From a2b9eea90aa96595bf808933c37410fda43b3624 Mon Sep 17 00:00:00 2001 From: Yusuf Mohsinally <463376+yusufm@users.noreply.github.com> Date: Wed, 18 Feb 2026 13:29:50 -0800 Subject: [PATCH 1/2] feat: add cursor telemetry-driven zoom suggestions --- electron/electron-env.d.ts | 7 + electron/ipc/handlers.ts | 117 +++++++++++++- electron/preload.ts | 3 + src/components/video-editor/VideoEditor.tsx | 65 +++++++- .../video-editor/timeline/TimelineEditor.tsx | 151 +++++++++++++++++- src/components/video-editor/types.ts | 6 + src/vite-env.d.ts | 12 ++ 7 files changed, 355 insertions(+), 6 deletions(-) diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index dba3f16..dda3d8d 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -32,6 +32,7 @@ interface Window { storeRecordedVideo: (videoData: ArrayBuffer, fileName: string) => Promise<{ success: boolean; path?: string; message?: string }> getRecordedVideoPath: () => Promise<{ success: boolean; path?: string; message?: string }> setRecordingState: (recording: boolean) => Promise + getCursorTelemetry: (videoPath?: string) => Promise<{ success: boolean; samples: CursorTelemetryPoint[]; message?: string; error?: string }> onStopRecordingFromTray: (callback: () => void) => () => void openExternalUrl: (url: string) => Promise<{ success: boolean; error?: string }> saveExportedVideo: (videoData: ArrayBuffer, fileName: string) => Promise<{ success: boolean; path?: string; message?: string; cancelled?: boolean }> @@ -52,3 +53,9 @@ interface ProcessedDesktopSource { thumbnail: string | null appIcon: string | null } + +interface CursorTelemetryPoint { + timeMs: number + cx: number + cy: number +} diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 867b72b..24a63a1 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -1,10 +1,62 @@ -import { ipcMain, desktopCapturer, BrowserWindow, shell, app, dialog } from 'electron' +import { ipcMain, desktopCapturer, BrowserWindow, shell, app, dialog, screen } from 'electron' import fs from 'node:fs/promises' import path from 'node:path' import { RECORDINGS_DIR } from '../main' let selectedSource: any = null +let currentVideoPath: string | null = null + +const CURSOR_TELEMETRY_VERSION = 1 +const CURSOR_SAMPLE_INTERVAL_MS = 100 +const MAX_CURSOR_SAMPLES = 60 * 60 * 10 // 1 hour @ 10Hz + +interface CursorTelemetryPoint { + timeMs: number + cx: number + cy: number +} + +let cursorCaptureInterval: NodeJS.Timeout | null = null +let cursorCaptureStartTimeMs = 0 +let activeCursorSamples: CursorTelemetryPoint[] = [] +let pendingCursorSamples: CursorTelemetryPoint[] = [] + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)) +} + +function stopCursorCapture() { + if (cursorCaptureInterval) { + clearInterval(cursorCaptureInterval) + cursorCaptureInterval = null + } +} + +function sampleCursorPoint() { + const cursor = screen.getCursorScreenPoint() + const sourceDisplayId = Number(selectedSource?.display_id) + const sourceDisplay = Number.isFinite(sourceDisplayId) + ? screen.getAllDisplays().find((display) => display.id === sourceDisplayId) ?? null + : null + const display = sourceDisplay ?? screen.getDisplayNearestPoint(cursor) + const bounds = display.bounds + const width = Math.max(1, bounds.width) + const height = Math.max(1, bounds.height) + + const cx = clamp((cursor.x - bounds.x) / width, 0, 1) + const cy = clamp((cursor.y - bounds.y) / height, 0, 1) + + activeCursorSamples.push({ + timeMs: Math.max(0, Date.now() - cursorCaptureStartTimeMs), + cx, + cy, + }) + + if (activeCursorSamples.length > MAX_CURSOR_SAMPLES) { + activeCursorSamples.shift() + } +} export function registerIpcHandlers( createEditorWindow: () => void, @@ -61,6 +113,17 @@ export function registerIpcHandlers( const videoPath = path.join(RECORDINGS_DIR, fileName) await fs.writeFile(videoPath, Buffer.from(videoData)) currentVideoPath = videoPath; + + const telemetryPath = `${videoPath}.cursor.json` + if (pendingCursorSamples.length > 0) { + await fs.writeFile( + telemetryPath, + JSON.stringify({ version: CURSOR_TELEMETRY_VERSION, samples: pendingCursorSamples }, null, 2), + 'utf-8' + ) + } + pendingCursorSamples = [] + return { success: true, path: videoPath, @@ -98,12 +161,62 @@ export function registerIpcHandlers( }) ipcMain.handle('set-recording-state', (_, recording: boolean) => { + if (recording) { + stopCursorCapture() + activeCursorSamples = [] + pendingCursorSamples = [] + cursorCaptureStartTimeMs = Date.now() + sampleCursorPoint() + cursorCaptureInterval = setInterval(sampleCursorPoint, CURSOR_SAMPLE_INTERVAL_MS) + } else { + stopCursorCapture() + pendingCursorSamples = [...activeCursorSamples] + activeCursorSamples = [] + } + const source = selectedSource || { name: 'Screen' } if (onRecordingStateChange) { onRecordingStateChange(recording, source.name) } }) + ipcMain.handle('get-cursor-telemetry', async (_, videoPath?: string) => { + const targetVideoPath = videoPath ?? currentVideoPath + if (!targetVideoPath) { + return { success: true, samples: [] } + } + + const telemetryPath = `${targetVideoPath}.cursor.json` + try { + const content = await fs.readFile(telemetryPath, 'utf-8') + const parsed = JSON.parse(content) + const rawSamples = Array.isArray(parsed) + ? parsed + : (Array.isArray(parsed?.samples) ? parsed.samples : []) + + const samples: CursorTelemetryPoint[] = rawSamples + .filter((sample: unknown) => Boolean(sample && typeof sample === 'object')) + .map((sample: unknown) => { + const point = sample as Partial + return { + timeMs: typeof point.timeMs === 'number' && Number.isFinite(point.timeMs) ? Math.max(0, point.timeMs) : 0, + cx: typeof point.cx === 'number' && Number.isFinite(point.cx) ? clamp(point.cx, 0, 1) : 0.5, + cy: typeof point.cy === 'number' && Number.isFinite(point.cy) ? clamp(point.cy, 0, 1) : 0.5, + } + }) + .sort((a: CursorTelemetryPoint, b: CursorTelemetryPoint) => a.timeMs - b.timeMs) + + return { success: true, samples } + } catch (error) { + const nodeError = error as NodeJS.ErrnoException + if (nodeError.code === 'ENOENT') { + return { success: true, samples: [] } + } + console.error('Failed to load cursor telemetry:', error) + return { success: false, message: 'Failed to load cursor telemetry', error: String(error), samples: [] } + } + }) + ipcMain.handle('open-external-url', async (_, url: string) => { try { @@ -198,8 +311,6 @@ export function registerIpcHandlers( } }); - let currentVideoPath: string | null = null; - ipcMain.handle('set-current-video-path', (_, path: string) => { currentVideoPath = path; return { success: true }; diff --git a/electron/preload.ts b/electron/preload.ts index 02fcc97..f58d8a8 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -37,6 +37,9 @@ contextBridge.exposeInMainWorld('electronAPI', { setRecordingState: (recording: boolean) => { return ipcRenderer.invoke('set-recording-state', recording) }, + getCursorTelemetry: (videoPath?: string) => { + return ipcRenderer.invoke('get-cursor-telemetry', videoPath) + }, onStopRecordingFromTray: (callback: () => void) => { const listener = () => callback() ipcRenderer.on('stop-recording-from-tray', listener) diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index c0a038e..d418950 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -23,6 +23,7 @@ import { type ZoomDepth, type ZoomFocus, type ZoomRegion, + type CursorTelemetryPoint, type TrimRegion, type AnnotationRegion, type CropRegion, @@ -50,6 +51,7 @@ export default function VideoEditor() { const [padding, setPadding] = useState(50); const [cropRegion, setCropRegion] = useState(DEFAULT_CROP_REGION); const [zoomRegions, setZoomRegions] = useState([]); + const [cursorTelemetry, setCursorTelemetry] = useState([]); const [selectedZoomId, setSelectedZoomId] = useState(null); const [trimRegions, setTrimRegions] = useState([]); const [selectedTrimId, setSelectedTrimId] = useState(null); @@ -89,6 +91,19 @@ export default function VideoEditor() { return fileUrl; }; + const fromFileUrl = (fileUrl: string): string => { + if (!fileUrl.startsWith('file://')) { + return fileUrl; + } + + try { + const url = new URL(fileUrl); + return decodeURIComponent(url.pathname); + } catch { + return fileUrl.replace(/^file:\/\//, ''); + } + }; + useEffect(() => { async function loadVideo() { try { @@ -109,6 +124,37 @@ export default function VideoEditor() { loadVideo(); }, []); + useEffect(() => { + let mounted = true; + + async function loadCursorTelemetry() { + if (!videoPath) { + if (mounted) { + setCursorTelemetry([]); + } + return; + } + + try { + const result = await window.electronAPI.getCursorTelemetry(fromFileUrl(videoPath)); + if (mounted) { + setCursorTelemetry(result.success ? result.samples : []); + } + } catch (telemetryError) { + console.warn('Unable to load cursor telemetry:', telemetryError); + if (mounted) { + setCursorTelemetry([]); + } + } + } + + loadCursorTelemetry(); + + return () => { + mounted = false; + }; + }, [videoPath]); + // Initialize default wallpaper with resolved asset path useEffect(() => { let mounted = true; @@ -180,6 +226,21 @@ export default function VideoEditor() { setSelectedAnnotationId(null); }, []); + const handleZoomSuggested = useCallback((span: Span, focus: ZoomFocus) => { + const id = `zoom-${nextZoomIdRef.current++}`; + const newRegion: ZoomRegion = { + id, + startMs: Math.round(span.start), + endMs: Math.round(span.end), + depth: DEFAULT_ZOOM_DEPTH, + focus: clampFocusToDepth(focus, DEFAULT_ZOOM_DEPTH), + }; + setZoomRegions((prev) => [...prev, newRegion]); + setSelectedZoomId(id); + setSelectedTrimId(null); + setSelectedAnnotationId(null); + }, []); + const handleTrimAdded = useCallback((span: Span) => { const id = `trim-${nextTrimIdRef.current++}`; const newRegion: TrimRegion = { @@ -804,8 +865,10 @@ export default function VideoEditor() { videoDuration={duration} currentTime={currentTime} onSeek={handleSeek} + cursorTelemetry={cursorTelemetry} zoomRegions={zoomRegions} onZoomAdded={handleZoomAdded} + onZoomSuggested={handleZoomSuggested} onZoomSpanChange={handleZoomSpanChange} onZoomDelete={handleZoomDelete} selectedZoomId={selectedZoomId} @@ -894,4 +957,4 @@ export default function VideoEditor() { /> ); -} \ No newline at end of file +} diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx index 9b091ef..5f965d0 100644 --- a/src/components/video-editor/timeline/TimelineEditor.tsx +++ b/src/components/video-editor/timeline/TimelineEditor.tsx @@ -1,7 +1,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useTimelineContext } from "dnd-timeline"; import { Button } from "@/components/ui/button"; -import { Plus, Scissors, ZoomIn, MessageSquare, ChevronDown, Check } from "lucide-react"; +import { Plus, Scissors, ZoomIn, MessageSquare, ChevronDown, Check, WandSparkles } from "lucide-react"; import { toast } from "sonner"; import { cn } from "@/lib/utils"; import TimelineWrapper from "./TimelineWrapper"; @@ -9,7 +9,7 @@ import Row from "./Row"; import Item from "./Item"; import KeyframeMarkers from "./KeyframeMarkers"; import type { Range, Span } from "dnd-timeline"; -import type { ZoomRegion, TrimRegion, AnnotationRegion } from "../types"; +import type { ZoomRegion, TrimRegion, AnnotationRegion, CursorTelemetryPoint, ZoomFocus } from "../types"; import { v4 as uuidv4 } from 'uuid'; import { DropdownMenu, @@ -26,13 +26,19 @@ const TRIM_ROW_ID = "row-trim"; const ANNOTATION_ROW_ID = "row-annotation"; const FALLBACK_RANGE_MS = 1000; const TARGET_MARKER_COUNT = 12; +const MIN_DWELL_DURATION_MS = 450; +const MAX_DWELL_DURATION_MS = 2600; +const DWELL_MOVE_THRESHOLD = 0.02; +const SUGGESTION_SPACING_MS = 1800; interface TimelineEditorProps { videoDuration: number; currentTime: number; onSeek?: (time: number) => void; + cursorTelemetry?: CursorTelemetryPoint[]; zoomRegions: ZoomRegion[]; onZoomAdded: (span: Span) => void; + onZoomSuggested?: (span: Span, focus: ZoomFocus) => void; onZoomSpanChange: (id: string, span: Span) => void; onZoomDelete: (id: string) => void; selectedZoomId: string | null; @@ -520,8 +526,10 @@ export default function TimelineEditor({ videoDuration, currentTime, onSeek, + cursorTelemetry = [], zoomRegions, onZoomAdded, + onZoomSuggested, onZoomSpanChange, onZoomDelete, selectedZoomId, @@ -716,6 +724,136 @@ export default function TimelineEditor({ onZoomAdded({ start: startPos, end: startPos + actualDuration }); }, [videoDuration, totalMs, currentTimeMs, zoomRegions, onZoomAdded, defaultRegionDurationMs]); + const handleSuggestZooms = useCallback(() => { + if (!videoDuration || videoDuration === 0 || totalMs === 0) { + return; + } + + if (!onZoomSuggested) { + toast.error("Zoom suggestion handler unavailable"); + return; + } + + if (cursorTelemetry.length < 2) { + toast.info("No cursor telemetry available", { + description: "Record a screencast first to generate cursor-based suggestions.", + }); + return; + } + + const defaultDuration = Math.min(defaultRegionDurationMs, totalMs); + if (defaultDuration <= 0) { + return; + } + + const reservedSpans = [...zoomRegions] + .map((region) => ({ start: region.startMs, end: region.endMs })) + .sort((a, b) => a.start - b.start); + + const normalizedSamples = [...cursorTelemetry] + .filter((sample) => Number.isFinite(sample.timeMs) && Number.isFinite(sample.cx) && Number.isFinite(sample.cy)) + .sort((a, b) => a.timeMs - b.timeMs) + .map((sample) => ({ + timeMs: Math.max(0, Math.min(sample.timeMs, totalMs)), + cx: Math.max(0, Math.min(sample.cx, 1)), + cy: Math.max(0, Math.min(sample.cy, 1)), + })); + + if (normalizedSamples.length < 2) { + toast.info("No usable cursor telemetry", { + description: "The recording does not include enough cursor movement data.", + }); + return; + } + + const dwellCandidates: Array<{ centerTimeMs: number; focus: ZoomFocus; strength: number }> = []; + let runStart = 0; + + const pushRunIfDwell = (startIndex: number, endIndexExclusive: number) => { + if (endIndexExclusive - startIndex < 2) { + return; + } + + const start = normalizedSamples[startIndex]; + const end = normalizedSamples[endIndexExclusive - 1]; + const runDuration = end.timeMs - start.timeMs; + if (runDuration < MIN_DWELL_DURATION_MS || runDuration > MAX_DWELL_DURATION_MS) { + return; + } + + const runSamples = normalizedSamples.slice(startIndex, endIndexExclusive); + const avgCx = runSamples.reduce((sum, sample) => sum + sample.cx, 0) / runSamples.length; + const avgCy = runSamples.reduce((sum, sample) => sum + sample.cy, 0) / runSamples.length; + + dwellCandidates.push({ + centerTimeMs: Math.round((start.timeMs + end.timeMs) / 2), + focus: { cx: avgCx, cy: avgCy }, + strength: runDuration, + }); + }; + + for (let index = 1; index < normalizedSamples.length; index += 1) { + const prev = normalizedSamples[index - 1]; + const curr = normalizedSamples[index]; + const dx = curr.cx - prev.cx; + const dy = curr.cy - prev.cy; + const distance = Math.hypot(dx, dy); + + if (distance > DWELL_MOVE_THRESHOLD) { + pushRunIfDwell(runStart, index); + runStart = index; + } + } + pushRunIfDwell(runStart, normalizedSamples.length); + + if (dwellCandidates.length === 0) { + toast.info("No clear cursor dwell moments found", { + description: "Try a recording with slower cursor pauses on important actions.", + }); + return; + } + + const sortedCandidates = [...dwellCandidates].sort((a, b) => b.strength - a.strength); + const acceptedCenters: number[] = []; + + let addedCount = 0; + + sortedCandidates.forEach((candidate) => { + const tooCloseToAccepted = acceptedCenters.some( + (center) => Math.abs(center - candidate.centerTimeMs) < SUGGESTION_SPACING_MS, + ); + + if (tooCloseToAccepted) { + return; + } + + const centeredStart = Math.round(candidate.centerTimeMs - defaultDuration / 2); + const candidateStart = Math.max(0, Math.min(centeredStart, totalMs - defaultDuration)); + const candidateEnd = candidateStart + defaultDuration; + const hasOverlap = reservedSpans.some( + (span) => candidateEnd > span.start && candidateStart < span.end, + ); + + if (hasOverlap) { + return; + } + + reservedSpans.push({ start: candidateStart, end: candidateEnd }); + acceptedCenters.push(candidate.centerTimeMs); + onZoomSuggested({ start: candidateStart, end: candidateEnd }, candidate.focus); + addedCount += 1; + }); + + if (addedCount === 0) { + toast.info("No auto-zoom slots available", { + description: "Detected dwell points overlap existing zoom regions.", + }); + return; + } + + toast.success(`Added ${addedCount} cursor-based zoom suggestion${addedCount === 1 ? "" : "s"}`); + }, [videoDuration, totalMs, defaultRegionDurationMs, zoomRegions, onZoomSuggested, cursorTelemetry]); + const handleAddTrim = useCallback(() => { if (!videoDuration || videoDuration === 0 || totalMs === 0 || !onTrimAdded) { return; @@ -920,6 +1058,15 @@ export default function TimelineEditor({ > +