feat: add cursor telemetry-driven zoom suggestions

src/components/video-editor/VideoEditor.tsx

@@ -23,6 +23,7 @@ import {
   type ZoomDepth,
   type ZoomFocus,
   type ZoomRegion,
+  type CursorTelemetryPoint,
   type TrimRegion,
   type AnnotationRegion,
   type CropRegion,
@@ -50,6 +51,7 @@ export default function VideoEditor() {
   const [padding, setPadding] = useState(50);
   const [cropRegion, setCropRegion] = useState<CropRegion>(DEFAULT_CROP_REGION);
   const [zoomRegions, setZoomRegions] = useState<ZoomRegion[]>([]);
+  const [cursorTelemetry, setCursorTelemetry] = useState<CursorTelemetryPoint[]>([]);
   const [selectedZoomId, setSelectedZoomId] = useState<string | null>(null);
   const [trimRegions, setTrimRegions] = useState<TrimRegion[]>([]);
   const [selectedTrimId, setSelectedTrimId] = useState<string | null>(null);
@@ -89,6 +91,19 @@ export default function VideoEditor() {
     return fileUrl;
   };
 
+  const fromFileUrl = (fileUrl: string): string => {
+    if (!fileUrl.startsWith('file://')) {
+      return fileUrl;
+    }
+
+    try {
+      const url = new URL(fileUrl);
+      return decodeURIComponent(url.pathname);
+    } catch {
+      return fileUrl.replace(/^file:\/\//, '');
+    }
+  };
+
   useEffect(() => {
     async function loadVideo() {
       try {
@@ -109,6 +124,37 @@ export default function VideoEditor() {
     loadVideo();
   }, []);
 
+  useEffect(() => {
+    let mounted = true;
+
+    async function loadCursorTelemetry() {
+      if (!videoPath) {
+        if (mounted) {
+          setCursorTelemetry([]);
+        }
+        return;
+      }
+
+      try {
+        const result = await window.electronAPI.getCursorTelemetry(fromFileUrl(videoPath));
+        if (mounted) {
+          setCursorTelemetry(result.success ? result.samples : []);
+        }
+      } catch (telemetryError) {
+        console.warn('Unable to load cursor telemetry:', telemetryError);
+        if (mounted) {
+          setCursorTelemetry([]);
+        }
+      }
+    }
+
+    loadCursorTelemetry();
+
+    return () => {
+      mounted = false;
+    };
+  }, [videoPath]);
+
   // Initialize default wallpaper with resolved asset path
   useEffect(() => {
     let mounted = true;
@@ -180,6 +226,21 @@ export default function VideoEditor() {
     setSelectedAnnotationId(null);
   }, []);
 
+  const handleZoomSuggested = useCallback((span: Span, focus: ZoomFocus) => {
+    const id = `zoom-${nextZoomIdRef.current++}`;
+    const newRegion: ZoomRegion = {
+      id,
+      startMs: Math.round(span.start),
+      endMs: Math.round(span.end),
+      depth: DEFAULT_ZOOM_DEPTH,
+      focus: clampFocusToDepth(focus, DEFAULT_ZOOM_DEPTH),
+    };
+    setZoomRegions((prev) => [...prev, newRegion]);
+    setSelectedZoomId(id);
+    setSelectedTrimId(null);
+    setSelectedAnnotationId(null);
+  }, []);
+
   const handleTrimAdded = useCallback((span: Span) => {
     const id = `trim-${nextTrimIdRef.current++}`;
     const newRegion: TrimRegion = {
@@ -804,8 +865,10 @@ export default function VideoEditor() {
               videoDuration={duration}
               currentTime={currentTime}
               onSeek={handleSeek}
+              cursorTelemetry={cursorTelemetry}
               zoomRegions={zoomRegions}
               onZoomAdded={handleZoomAdded}
+              onZoomSuggested={handleZoomSuggested}
               onZoomSpanChange={handleZoomSpanChange}
               onZoomDelete={handleZoomDelete}
               selectedZoomId={selectedZoomId}
@@ -894,4 +957,4 @@ export default function VideoEditor() {
       />
     </div>
   );
-}
+}

src/components/video-editor/timeline/TimelineEditor.tsx

@@ -1,7 +1,7 @@
 import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { useTimelineContext } from "dnd-timeline";
 import { Button } from "@/components/ui/button";
-import { Plus, Scissors, ZoomIn, MessageSquare, ChevronDown, Check } from "lucide-react";
+import { Plus, Scissors, ZoomIn, MessageSquare, ChevronDown, Check, WandSparkles } from "lucide-react";
 import { toast } from "sonner";
 import { cn } from "@/lib/utils";
 import TimelineWrapper from "./TimelineWrapper";
@@ -9,7 +9,7 @@ import Row from "./Row";
 import Item from "./Item";
 import KeyframeMarkers from "./KeyframeMarkers";
 import type { Range, Span } from "dnd-timeline";
-import type { ZoomRegion, TrimRegion, AnnotationRegion } from "../types";
+import type { ZoomRegion, TrimRegion, AnnotationRegion, CursorTelemetryPoint, ZoomFocus } from "../types";
 import { v4 as uuidv4 } from 'uuid';
 import {
   DropdownMenu,
@@ -26,13 +26,19 @@ const TRIM_ROW_ID = "row-trim";
 const ANNOTATION_ROW_ID = "row-annotation";
 const FALLBACK_RANGE_MS = 1000;
 const TARGET_MARKER_COUNT = 12;
+const MIN_DWELL_DURATION_MS = 450;
+const MAX_DWELL_DURATION_MS = 2600;
+const DWELL_MOVE_THRESHOLD = 0.02;
+const SUGGESTION_SPACING_MS = 1800;
 
 interface TimelineEditorProps {
   videoDuration: number;
   currentTime: number;
   onSeek?: (time: number) => void;
+  cursorTelemetry?: CursorTelemetryPoint[];
   zoomRegions: ZoomRegion[];
   onZoomAdded: (span: Span) => void;
+  onZoomSuggested?: (span: Span, focus: ZoomFocus) => void;
   onZoomSpanChange: (id: string, span: Span) => void;
   onZoomDelete: (id: string) => void;
   selectedZoomId: string | null;
@@ -520,8 +526,10 @@ export default function TimelineEditor({
   videoDuration,
   currentTime,
   onSeek,
+  cursorTelemetry = [],
   zoomRegions,
   onZoomAdded,
+  onZoomSuggested,
   onZoomSpanChange,
   onZoomDelete,
   selectedZoomId,
@@ -716,6 +724,136 @@ export default function TimelineEditor({
     onZoomAdded({ start: startPos, end: startPos + actualDuration });
   }, [videoDuration, totalMs, currentTimeMs, zoomRegions, onZoomAdded, defaultRegionDurationMs]);
 
+  const handleSuggestZooms = useCallback(() => {
+    if (!videoDuration || videoDuration === 0 || totalMs === 0) {
+      return;
+    }
+
+    if (!onZoomSuggested) {
+      toast.error("Zoom suggestion handler unavailable");
+      return;
+    }
+
+    if (cursorTelemetry.length < 2) {
+      toast.info("No cursor telemetry available", {
+        description: "Record a screencast first to generate cursor-based suggestions.",
+      });
+      return;
+    }
+
+    const defaultDuration = Math.min(defaultRegionDurationMs, totalMs);
+    if (defaultDuration <= 0) {
+      return;
+    }
+
+    const reservedSpans = [...zoomRegions]
+      .map((region) => ({ start: region.startMs, end: region.endMs }))
+      .sort((a, b) => a.start - b.start);
+
+    const normalizedSamples = [...cursorTelemetry]
+      .filter((sample) => Number.isFinite(sample.timeMs) && Number.isFinite(sample.cx) && Number.isFinite(sample.cy))
+      .sort((a, b) => a.timeMs - b.timeMs)
+      .map((sample) => ({
+        timeMs: Math.max(0, Math.min(sample.timeMs, totalMs)),
+        cx: Math.max(0, Math.min(sample.cx, 1)),
+        cy: Math.max(0, Math.min(sample.cy, 1)),
+      }));
+
+    if (normalizedSamples.length < 2) {
+      toast.info("No usable cursor telemetry", {
+        description: "The recording does not include enough cursor movement data.",
+      });
+      return;
+    }
+
+    const dwellCandidates: Array<{ centerTimeMs: number; focus: ZoomFocus; strength: number }> = [];
+    let runStart = 0;
+
+    const pushRunIfDwell = (startIndex: number, endIndexExclusive: number) => {
+      if (endIndexExclusive - startIndex < 2) {
+        return;
+      }
+
+      const start = normalizedSamples[startIndex];
+      const end = normalizedSamples[endIndexExclusive - 1];
+      const runDuration = end.timeMs - start.timeMs;
+      if (runDuration < MIN_DWELL_DURATION_MS || runDuration > MAX_DWELL_DURATION_MS) {
+        return;
+      }
+
+      const runSamples = normalizedSamples.slice(startIndex, endIndexExclusive);
+      const avgCx = runSamples.reduce((sum, sample) => sum + sample.cx, 0) / runSamples.length;
+      const avgCy = runSamples.reduce((sum, sample) => sum + sample.cy, 0) / runSamples.length;
+
+      dwellCandidates.push({
+        centerTimeMs: Math.round((start.timeMs + end.timeMs) / 2),
+        focus: { cx: avgCx, cy: avgCy },
+        strength: runDuration,
+      });
+    };
+
+    for (let index = 1; index < normalizedSamples.length; index += 1) {
+      const prev = normalizedSamples[index - 1];
+      const curr = normalizedSamples[index];
+      const dx = curr.cx - prev.cx;
+      const dy = curr.cy - prev.cy;
+      const distance = Math.hypot(dx, dy);
+
+      if (distance > DWELL_MOVE_THRESHOLD) {
+        pushRunIfDwell(runStart, index);
+        runStart = index;
+      }
+    }
+    pushRunIfDwell(runStart, normalizedSamples.length);
+
+    if (dwellCandidates.length === 0) {
+      toast.info("No clear cursor dwell moments found", {
+        description: "Try a recording with slower cursor pauses on important actions.",
+      });
+      return;
+    }
+
+    const sortedCandidates = [...dwellCandidates].sort((a, b) => b.strength - a.strength);
+    const acceptedCenters: number[] = [];
+
+    let addedCount = 0;
+
+    sortedCandidates.forEach((candidate) => {
+      const tooCloseToAccepted = acceptedCenters.some(
+        (center) => Math.abs(center - candidate.centerTimeMs) < SUGGESTION_SPACING_MS,
+      );
+
+      if (tooCloseToAccepted) {
+        return;
+      }
+
+      const centeredStart = Math.round(candidate.centerTimeMs - defaultDuration / 2);
+      const candidateStart = Math.max(0, Math.min(centeredStart, totalMs - defaultDuration));
+      const candidateEnd = candidateStart + defaultDuration;
+      const hasOverlap = reservedSpans.some(
+        (span) => candidateEnd > span.start && candidateStart < span.end,
+      );
+
+      if (hasOverlap) {
+        return;
+      }
+
+      reservedSpans.push({ start: candidateStart, end: candidateEnd });
+      acceptedCenters.push(candidate.centerTimeMs);
+      onZoomSuggested({ start: candidateStart, end: candidateEnd }, candidate.focus);
+      addedCount += 1;
+    });
+
+    if (addedCount === 0) {
+      toast.info("No auto-zoom slots available", {
+        description: "Detected dwell points overlap existing zoom regions.",
+      });
+      return;
+    }
+
+    toast.success(`Added ${addedCount} cursor-based zoom suggestion${addedCount === 1 ? "" : "s"}`);
+  }, [videoDuration, totalMs, defaultRegionDurationMs, zoomRegions, onZoomSuggested, cursorTelemetry]);
+
   const handleAddTrim = useCallback(() => {
     if (!videoDuration || videoDuration === 0 || totalMs === 0 || !onTrimAdded) {
       return;
@@ -920,6 +1058,15 @@ export default function TimelineEditor({
           >
             <ZoomIn className="w-4 h-4" />
           </Button>
+          <Button
+            onClick={handleSuggestZooms}
+            variant="ghost"
+            size="icon"
+            className="h-7 w-7 text-slate-400 hover:text-[#34B27B] hover:bg-[#34B27B]/10 transition-all"
+            title="Suggest Zooms from Cursor"
+          >
+            <WandSparkles className="w-4 h-4" />
+          </Button>
           <Button
             onClick={handleAddTrim}
             variant="ghost"

src/components/video-editor/types.ts

@@ -13,6 +13,12 @@ export interface ZoomRegion {
   focus: ZoomFocus;
 }
 
+export interface CursorTelemetryPoint {
+  timeMs: number;
+  cx: number;
+  cy: number;
+}
+
 export interface TrimRegion {
   id: string;
   startMs: number;