Merge pull request #154 from yusufm/feat/cursor-telemetry-zoom-suggestions

feat: cursor telemetry-driven zoom suggestions

electron/electron-env.d.ts

@@ -32,6 +32,7 @@ interface Window {
     storeRecordedVideo: (videoData: ArrayBuffer, fileName: string) => Promise<{ success: boolean; path?: string; message?: string }>
     getRecordedVideoPath: () => Promise<{ success: boolean; path?: string; message?: string }>
     setRecordingState: (recording: boolean) => Promise<void>
+    getCursorTelemetry: (videoPath?: string) => Promise<{ success: boolean; samples: CursorTelemetryPoint[]; message?: string; error?: string }>
     onStopRecordingFromTray: (callback: () => void) => () => void
     openExternalUrl: (url: string) => Promise<{ success: boolean; error?: string }>
     saveExportedVideo: (videoData: ArrayBuffer, fileName: string) => Promise<{ success: boolean; path?: string; message?: string; cancelled?: boolean }>
@@ -52,3 +53,9 @@ interface ProcessedDesktopSource {
   thumbnail: string | null
   appIcon: string | null
 }
+
+interface CursorTelemetryPoint {
+  timeMs: number
+  cx: number
+  cy: number
+}

electron/ipc/handlers.ts

@@ -1,10 +1,62 @@
-import { ipcMain, desktopCapturer, BrowserWindow, shell, app, dialog } from 'electron'
+import { ipcMain, desktopCapturer, BrowserWindow, shell, app, dialog, screen } from 'electron'
 
 import fs from 'node:fs/promises'
 import path from 'node:path'
 import { RECORDINGS_DIR } from '../main'
 
 let selectedSource: any = null
+let currentVideoPath: string | null = null
+
+const CURSOR_TELEMETRY_VERSION = 1
+const CURSOR_SAMPLE_INTERVAL_MS = 100
+const MAX_CURSOR_SAMPLES = 60 * 60 * 10 // 1 hour @ 10Hz
+
+interface CursorTelemetryPoint {
+  timeMs: number
+  cx: number
+  cy: number
+}
+
+let cursorCaptureInterval: NodeJS.Timeout | null = null
+let cursorCaptureStartTimeMs = 0
+let activeCursorSamples: CursorTelemetryPoint[] = []
+let pendingCursorSamples: CursorTelemetryPoint[] = []
+
+function clamp(value: number, min: number, max: number) {
+  return Math.min(max, Math.max(min, value))
+}
+
+function stopCursorCapture() {
+  if (cursorCaptureInterval) {
+    clearInterval(cursorCaptureInterval)
+    cursorCaptureInterval = null
+  }
+}
+
+function sampleCursorPoint() {
+  const cursor = screen.getCursorScreenPoint()
+  const sourceDisplayId = Number(selectedSource?.display_id)
+  const sourceDisplay = Number.isFinite(sourceDisplayId)
+    ? screen.getAllDisplays().find((display) => display.id === sourceDisplayId) ?? null
+    : null
+  const display = sourceDisplay ?? screen.getDisplayNearestPoint(cursor)
+  const bounds = display.bounds
+  const width = Math.max(1, bounds.width)
+  const height = Math.max(1, bounds.height)
+
+  const cx = clamp((cursor.x - bounds.x) / width, 0, 1)
+  const cy = clamp((cursor.y - bounds.y) / height, 0, 1)
+
+  activeCursorSamples.push({
+    timeMs: Math.max(0, Date.now() - cursorCaptureStartTimeMs),
+    cx,
+    cy,
+  })
+
+  if (activeCursorSamples.length > MAX_CURSOR_SAMPLES) {
+    activeCursorSamples.shift()
+  }
+}
 
 export function registerIpcHandlers(
   createEditorWindow: () => void,
@@ -61,6 +113,17 @@ export function registerIpcHandlers(
       const videoPath = path.join(RECORDINGS_DIR, fileName)
       await fs.writeFile(videoPath, Buffer.from(videoData))
       currentVideoPath = videoPath;
+
+      const telemetryPath = `${videoPath}.cursor.json`
+      if (pendingCursorSamples.length > 0) {
+        await fs.writeFile(
+          telemetryPath,
+          JSON.stringify({ version: CURSOR_TELEMETRY_VERSION, samples: pendingCursorSamples }, null, 2),
+          'utf-8'
+        )
+      }
+      pendingCursorSamples = []
+
       return {
         success: true,
         path: videoPath,
@@ -98,12 +161,62 @@ export function registerIpcHandlers(
   })
 
   ipcMain.handle('set-recording-state', (_, recording: boolean) => {
+    if (recording) {
+      stopCursorCapture()
+      activeCursorSamples = []
+      pendingCursorSamples = []
+      cursorCaptureStartTimeMs = Date.now()
+      sampleCursorPoint()
+      cursorCaptureInterval = setInterval(sampleCursorPoint, CURSOR_SAMPLE_INTERVAL_MS)
+    } else {
+      stopCursorCapture()
+      pendingCursorSamples = [...activeCursorSamples]
+      activeCursorSamples = []
+    }
+
     const source = selectedSource || { name: 'Screen' }
     if (onRecordingStateChange) {
       onRecordingStateChange(recording, source.name)
     }
   })
 
+  ipcMain.handle('get-cursor-telemetry', async (_, videoPath?: string) => {
+    const targetVideoPath = videoPath ?? currentVideoPath
+    if (!targetVideoPath) {
+      return { success: true, samples: [] }
+    }
+
+    const telemetryPath = `${targetVideoPath}.cursor.json`
+    try {
+      const content = await fs.readFile(telemetryPath, 'utf-8')
+      const parsed = JSON.parse(content)
+      const rawSamples = Array.isArray(parsed)
+        ? parsed
+        : (Array.isArray(parsed?.samples) ? parsed.samples : [])
+
+      const samples: CursorTelemetryPoint[] = rawSamples
+        .filter((sample: unknown) => Boolean(sample && typeof sample === 'object'))
+        .map((sample: unknown) => {
+          const point = sample as Partial<CursorTelemetryPoint>
+          return {
+            timeMs: typeof point.timeMs === 'number' && Number.isFinite(point.timeMs) ? Math.max(0, point.timeMs) : 0,
+            cx: typeof point.cx === 'number' && Number.isFinite(point.cx) ? clamp(point.cx, 0, 1) : 0.5,
+            cy: typeof point.cy === 'number' && Number.isFinite(point.cy) ? clamp(point.cy, 0, 1) : 0.5,
+          }
+        })
+        .sort((a: CursorTelemetryPoint, b: CursorTelemetryPoint) => a.timeMs - b.timeMs)
+
+      return { success: true, samples }
+    } catch (error) {
+      const nodeError = error as NodeJS.ErrnoException
+      if (nodeError.code === 'ENOENT') {
+        return { success: true, samples: [] }
+      }
+      console.error('Failed to load cursor telemetry:', error)
+      return { success: false, message: 'Failed to load cursor telemetry', error: String(error), samples: [] }
+    }
+  })
+
 
   ipcMain.handle('open-external-url', async (_, url: string) => {
     try {
@@ -198,8 +311,6 @@ export function registerIpcHandlers(
     }
   });
 
-  let currentVideoPath: string | null = null;
-
   ipcMain.handle('set-current-video-path', (_, path: string) => {
     currentVideoPath = path;
     return { success: true };

electron/preload.ts

@@ -37,6 +37,9 @@ contextBridge.exposeInMainWorld('electronAPI', {
   setRecordingState: (recording: boolean) => {
     return ipcRenderer.invoke('set-recording-state', recording)
   },
+  getCursorTelemetry: (videoPath?: string) => {
+    return ipcRenderer.invoke('get-cursor-telemetry', videoPath)
+  },
   onStopRecordingFromTray: (callback: () => void) => {
     const listener = () => callback()
     ipcRenderer.on('stop-recording-from-tray', listener)

src/components/video-editor/VideoEditor.tsx

@@ -23,6 +23,7 @@ import {
   type ZoomDepth,
   type ZoomFocus,
   type ZoomRegion,
+  type CursorTelemetryPoint,
   type TrimRegion,
   type AnnotationRegion,
   type CropRegion,
@@ -50,6 +51,7 @@ export default function VideoEditor() {
   const [padding, setPadding] = useState(50);
   const [cropRegion, setCropRegion] = useState<CropRegion>(DEFAULT_CROP_REGION);
   const [zoomRegions, setZoomRegions] = useState<ZoomRegion[]>([]);
+  const [cursorTelemetry, setCursorTelemetry] = useState<CursorTelemetryPoint[]>([]);
   const [selectedZoomId, setSelectedZoomId] = useState<string | null>(null);
   const [trimRegions, setTrimRegions] = useState<TrimRegion[]>([]);
   const [selectedTrimId, setSelectedTrimId] = useState<string | null>(null);
@@ -89,6 +91,19 @@ export default function VideoEditor() {
     return fileUrl;
   };
 
+  const fromFileUrl = (fileUrl: string): string => {
+    if (!fileUrl.startsWith('file://')) {
+      return fileUrl;
+    }
+
+    try {
+      const url = new URL(fileUrl);
+      return decodeURIComponent(url.pathname);
+    } catch {
+      return fileUrl.replace(/^file:\/\//, '');
+    }
+  };
+
   useEffect(() => {
     async function loadVideo() {
       try {
@@ -109,6 +124,37 @@ export default function VideoEditor() {
     loadVideo();
   }, []);
 
+  useEffect(() => {
+    let mounted = true;
+
+    async function loadCursorTelemetry() {
+      if (!videoPath) {
+        if (mounted) {
+          setCursorTelemetry([]);
+        }
+        return;
+      }
+
+      try {
+        const result = await window.electronAPI.getCursorTelemetry(fromFileUrl(videoPath));
+        if (mounted) {
+          setCursorTelemetry(result.success ? result.samples : []);
+        }
+      } catch (telemetryError) {
+        console.warn('Unable to load cursor telemetry:', telemetryError);
+        if (mounted) {
+          setCursorTelemetry([]);
+        }
+      }
+    }
+
+    loadCursorTelemetry();
+
+    return () => {
+      mounted = false;
+    };
+  }, [videoPath]);
+
   // Initialize default wallpaper with resolved asset path
   useEffect(() => {
     let mounted = true;
@@ -180,6 +226,21 @@ export default function VideoEditor() {
     setSelectedAnnotationId(null);
   }, []);
 
+  const handleZoomSuggested = useCallback((span: Span, focus: ZoomFocus) => {
+    const id = `zoom-${nextZoomIdRef.current++}`;
+    const newRegion: ZoomRegion = {
+      id,
+      startMs: Math.round(span.start),
+      endMs: Math.round(span.end),
+      depth: DEFAULT_ZOOM_DEPTH,
+      focus: clampFocusToDepth(focus, DEFAULT_ZOOM_DEPTH),
+    };
+    setZoomRegions((prev) => [...prev, newRegion]);
+    setSelectedZoomId(id);
+    setSelectedTrimId(null);
+    setSelectedAnnotationId(null);
+  }, []);
+
   const handleTrimAdded = useCallback((span: Span) => {
     const id = `trim-${nextTrimIdRef.current++}`;
     const newRegion: TrimRegion = {
@@ -804,8 +865,10 @@ export default function VideoEditor() {
               videoDuration={duration}
               currentTime={currentTime}
               onSeek={handleSeek}
+              cursorTelemetry={cursorTelemetry}
               zoomRegions={zoomRegions}
               onZoomAdded={handleZoomAdded}
+              onZoomSuggested={handleZoomSuggested}
               onZoomSpanChange={handleZoomSpanChange}
               onZoomDelete={handleZoomDelete}
               selectedZoomId={selectedZoomId}
@@ -894,4 +957,4 @@ export default function VideoEditor() {
       />
     </div>
   );
-}
+}

src/components/video-editor/timeline/TimelineEditor.tsx

@@ -1,7 +1,7 @@
 import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { useTimelineContext } from "dnd-timeline";
 import { Button } from "@/components/ui/button";
-import { Plus, Scissors, ZoomIn, MessageSquare, ChevronDown, Check } from "lucide-react";
+import { Plus, Scissors, ZoomIn, MessageSquare, ChevronDown, Check, WandSparkles } from "lucide-react";
 import { toast } from "sonner";
 import { cn } from "@/lib/utils";
 import TimelineWrapper from "./TimelineWrapper";
@@ -9,7 +9,7 @@ import Row from "./Row";
 import Item from "./Item";
 import KeyframeMarkers from "./KeyframeMarkers";
 import type { Range, Span } from "dnd-timeline";
-import type { ZoomRegion, TrimRegion, AnnotationRegion } from "../types";
+import type { ZoomRegion, TrimRegion, AnnotationRegion, CursorTelemetryPoint, ZoomFocus } from "../types";
 import { v4 as uuidv4 } from 'uuid';
 import {
   DropdownMenu,
@@ -20,19 +20,23 @@ import {
 import { type AspectRatio, getAspectRatioLabel, ASPECT_RATIOS } from "@/utils/aspectRatioUtils";
 import { formatShortcut } from "@/utils/platformUtils";
 import { TutorialHelp } from "../TutorialHelp";
+import { detectZoomDwellCandidates, normalizeCursorTelemetry } from "./zoomSuggestionUtils";
 
 const ZOOM_ROW_ID = "row-zoom";
 const TRIM_ROW_ID = "row-trim";
 const ANNOTATION_ROW_ID = "row-annotation";
 const FALLBACK_RANGE_MS = 1000;
 const TARGET_MARKER_COUNT = 12;
+const SUGGESTION_SPACING_MS = 1800;
 
 interface TimelineEditorProps {
   videoDuration: number;
   currentTime: number;
   onSeek?: (time: number) => void;
+  cursorTelemetry?: CursorTelemetryPoint[];
   zoomRegions: ZoomRegion[];
   onZoomAdded: (span: Span) => void;
+  onZoomSuggested?: (span: Span, focus: ZoomFocus) => void;
   onZoomSpanChange: (id: string, span: Span) => void;
   onZoomDelete: (id: string) => void;
   selectedZoomId: string | null;
@@ -520,8 +524,10 @@ export default function TimelineEditor({
   videoDuration,
   currentTime,
   onSeek,
+  cursorTelemetry = [],
   zoomRegions,
   onZoomAdded,
+  onZoomSuggested,
   onZoomSpanChange,
   onZoomDelete,
   selectedZoomId,
@@ -716,6 +722,91 @@ export default function TimelineEditor({
     onZoomAdded({ start: startPos, end: startPos + actualDuration });
   }, [videoDuration, totalMs, currentTimeMs, zoomRegions, onZoomAdded, defaultRegionDurationMs]);
 
+  const handleSuggestZooms = useCallback(() => {
+    if (!videoDuration || videoDuration === 0 || totalMs === 0) {
+      return;
+    }
+
+    if (!onZoomSuggested) {
+      toast.error("Zoom suggestion handler unavailable");
+      return;
+    }
+
+    if (cursorTelemetry.length < 2) {
+      toast.info("No cursor telemetry available", {
+        description: "Record a screencast first to generate cursor-based suggestions.",
+      });
+      return;
+    }
+
+    const defaultDuration = Math.min(defaultRegionDurationMs, totalMs);
+    if (defaultDuration <= 0) {
+      return;
+    }
+
+    const reservedSpans = [...zoomRegions]
+      .map((region) => ({ start: region.startMs, end: region.endMs }))
+      .sort((a, b) => a.start - b.start);
+
+    const normalizedSamples = normalizeCursorTelemetry(cursorTelemetry, totalMs);
+
+    if (normalizedSamples.length < 2) {
+      toast.info("No usable cursor telemetry", {
+        description: "The recording does not include enough cursor movement data.",
+      });
+      return;
+    }
+
+    const dwellCandidates = detectZoomDwellCandidates(normalizedSamples);
+
+    if (dwellCandidates.length === 0) {
+      toast.info("No clear cursor dwell moments found", {
+        description: "Try a recording with slower cursor pauses on important actions.",
+      });
+      return;
+    }
+
+    const sortedCandidates = [...dwellCandidates].sort((a, b) => b.strength - a.strength);
+    const acceptedCenters: number[] = [];
+
+    let addedCount = 0;
+
+    sortedCandidates.forEach((candidate) => {
+      const tooCloseToAccepted = acceptedCenters.some(
+        (center) => Math.abs(center - candidate.centerTimeMs) < SUGGESTION_SPACING_MS,
+      );
+
+      if (tooCloseToAccepted) {
+        return;
+      }
+
+      const centeredStart = Math.round(candidate.centerTimeMs - defaultDuration / 2);
+      const candidateStart = Math.max(0, Math.min(centeredStart, totalMs - defaultDuration));
+      const candidateEnd = candidateStart + defaultDuration;
+      const hasOverlap = reservedSpans.some(
+        (span) => candidateEnd > span.start && candidateStart < span.end,
+      );
+
+      if (hasOverlap) {
+        return;
+      }
+
+      reservedSpans.push({ start: candidateStart, end: candidateEnd });
+      acceptedCenters.push(candidate.centerTimeMs);
+      onZoomSuggested({ start: candidateStart, end: candidateEnd }, candidate.focus);
+      addedCount += 1;
+    });
+
+    if (addedCount === 0) {
+      toast.info("No auto-zoom slots available", {
+        description: "Detected dwell points overlap existing zoom regions.",
+      });
+      return;
+    }
+
+    toast.success(`Added ${addedCount} cursor-based zoom suggestion${addedCount === 1 ? "" : "s"}`);
+  }, [videoDuration, totalMs, defaultRegionDurationMs, zoomRegions, onZoomSuggested, cursorTelemetry]);
+
   const handleAddTrim = useCallback(() => {
     if (!videoDuration || videoDuration === 0 || totalMs === 0 || !onTrimAdded) {
       return;
@@ -920,6 +1011,15 @@ export default function TimelineEditor({
           >
             <ZoomIn className="w-4 h-4" />
           </Button>
+          <Button
+            onClick={handleSuggestZooms}
+            variant="ghost"
+            size="icon"
+            className="h-7 w-7 text-slate-400 hover:text-[#34B27B] hover:bg-[#34B27B]/10 transition-all"
+            title="Suggest Zooms from Cursor"
+          >
+            <WandSparkles className="w-4 h-4" />
+          </Button>
           <Button
             onClick={handleAddTrim}
             variant="ghost"

src/components/video-editor/timeline/zoomSuggestionUtils.ts

@@ -0,0 +1,75 @@
+import type { CursorTelemetryPoint, ZoomFocus } from "../types";
+
+export const MIN_DWELL_DURATION_MS = 450;
+export const MAX_DWELL_DURATION_MS = 2600;
+export const DWELL_MOVE_THRESHOLD = 0.02;
+
+export interface ZoomDwellCandidate {
+  centerTimeMs: number;
+  focus: ZoomFocus;
+  strength: number;
+}
+
+function normalizeTelemetrySample(sample: CursorTelemetryPoint, totalMs: number): CursorTelemetryPoint {
+  return {
+    timeMs: Math.max(0, Math.min(sample.timeMs, totalMs)),
+    cx: Math.max(0, Math.min(sample.cx, 1)),
+    cy: Math.max(0, Math.min(sample.cy, 1)),
+  };
+}
+
+export function normalizeCursorTelemetry(
+  telemetry: CursorTelemetryPoint[],
+  totalMs: number,
+): CursorTelemetryPoint[] {
+  return [...telemetry]
+    .filter((sample) => Number.isFinite(sample.timeMs) && Number.isFinite(sample.cx) && Number.isFinite(sample.cy))
+    .sort((a, b) => a.timeMs - b.timeMs)
+    .map((sample) => normalizeTelemetrySample(sample, totalMs));
+}
+
+export function detectZoomDwellCandidates(samples: CursorTelemetryPoint[]): ZoomDwellCandidate[] {
+  if (samples.length < 2) {
+    return [];
+  }
+
+  const dwellCandidates: ZoomDwellCandidate[] = [];
+  let runStart = 0;
+
+  const pushRunIfDwell = (startIndex: number, endIndexExclusive: number) => {
+    if (endIndexExclusive - startIndex < 2) {
+      return;
+    }
+
+    const start = samples[startIndex];
+    const end = samples[endIndexExclusive - 1];
+    const runDuration = end.timeMs - start.timeMs;
+    if (runDuration < MIN_DWELL_DURATION_MS || runDuration > MAX_DWELL_DURATION_MS) {
+      return;
+    }
+
+    const runSamples = samples.slice(startIndex, endIndexExclusive);
+    const avgCx = runSamples.reduce((sum, sample) => sum + sample.cx, 0) / runSamples.length;
+    const avgCy = runSamples.reduce((sum, sample) => sum + sample.cy, 0) / runSamples.length;
+
+    dwellCandidates.push({
+      centerTimeMs: Math.round((start.timeMs + end.timeMs) / 2),
+      focus: { cx: avgCx, cy: avgCy },
+      strength: runDuration,
+    });
+  };
+
+  for (let index = 1; index < samples.length; index += 1) {
+    const prev = samples[index - 1];
+    const curr = samples[index];
+    const distance = Math.hypot(curr.cx - prev.cx, curr.cy - prev.cy);
+
+    if (distance > DWELL_MOVE_THRESHOLD) {
+      pushRunIfDwell(runStart, index);
+      runStart = index;
+    }
+  }
+  pushRunIfDwell(runStart, samples.length);
+
+  return dwellCandidates;
+}

src/components/video-editor/types.ts

@@ -13,6 +13,12 @@ export interface ZoomRegion {
   focus: ZoomFocus;
 }
 
+export interface CursorTelemetryPoint {
+  timeMs: number;
+  cx: number;
+  cy: number;
+}
+
 export interface TrimRegion {
   id: string;
   startMs: number;

src/vite-env.d.ts

@@ -9,6 +9,12 @@ interface ProcessedDesktopSource {
   appIcon: string | null;
 }
 
+interface CursorTelemetryPoint {
+  timeMs: number;
+  cx: number;
+  cy: number;
+}
+
 interface Window {
   electronAPI: {
     getSources: (opts: Electron.SourcesOptions) => Promise<ProcessedDesktopSource[]>
@@ -30,6 +36,12 @@ interface Window {
     }>
     getAssetBasePath: () => Promise<string | null>
     setRecordingState: (recording: boolean) => Promise<void>
+    getCursorTelemetry: (videoPath?: string) => Promise<{
+      success: boolean
+      samples: CursorTelemetryPoint[]
+      message?: string
+      error?: string
+    }>
     onStopRecordingFromTray: (callback: () => void) => () => void
     openExternalUrl: (url: string) => Promise<{ success: boolean; error?: string }>
     saveExportedVideo: (videoData: ArrayBuffer, fileName: string) => Promise<{