diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 247b679..a489699 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -32,6 +32,7 @@ interface Window { storeRecordedVideo: (videoData: ArrayBuffer, fileName: string) => Promise<{ success: boolean; path?: string; message?: string }> getRecordedVideoPath: () => Promise<{ success: boolean; path?: string; message?: string }> setRecordingState: (recording: boolean) => Promise + getCursorTelemetry: (videoPath?: string) => Promise<{ success: boolean; samples: CursorTelemetryPoint[]; message?: string; error?: string }> onStopRecordingFromTray: (callback: () => void) => () => void openExternalUrl: (url: string) => Promise<{ success: boolean; error?: string }> saveExportedVideo: (videoData: ArrayBuffer, fileName: string) => Promise<{ success: boolean; path?: string; message?: string; cancelled?: boolean }> @@ -54,3 +55,9 @@ interface ProcessedDesktopSource { thumbnail: string | null appIcon: string | null } + +interface CursorTelemetryPoint { + timeMs: number + cx: number + cy: number +} diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index caa82ca..cec3ed9 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -1,4 +1,4 @@ -import { ipcMain, desktopCapturer, BrowserWindow, shell, app, dialog } from 'electron' +import { ipcMain, desktopCapturer, BrowserWindow, shell, app, dialog, screen } from 'electron' import fs from 'node:fs/promises' import path from 'node:path' @@ -7,6 +7,58 @@ import { RECORDINGS_DIR } from '../main' const SHORTCUTS_FILE = path.join(app.getPath('userData'), 'shortcuts.json') let selectedSource: any = null +let currentVideoPath: string | null = null + +const CURSOR_TELEMETRY_VERSION = 1 +const CURSOR_SAMPLE_INTERVAL_MS = 100 +const MAX_CURSOR_SAMPLES = 60 * 60 * 10 // 1 hour @ 10Hz + +interface CursorTelemetryPoint { + timeMs: number + cx: number + cy: number +} + +let cursorCaptureInterval: NodeJS.Timeout | null = null +let cursorCaptureStartTimeMs = 0 +let activeCursorSamples: CursorTelemetryPoint[] = [] +let pendingCursorSamples: CursorTelemetryPoint[] = [] + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)) +} + +function stopCursorCapture() { + if (cursorCaptureInterval) { + clearInterval(cursorCaptureInterval) + cursorCaptureInterval = null + } +} + +function sampleCursorPoint() { + const cursor = screen.getCursorScreenPoint() + const sourceDisplayId = Number(selectedSource?.display_id) + const sourceDisplay = Number.isFinite(sourceDisplayId) + ? screen.getAllDisplays().find((display) => display.id === sourceDisplayId) ?? null + : null + const display = sourceDisplay ?? screen.getDisplayNearestPoint(cursor) + const bounds = display.bounds + const width = Math.max(1, bounds.width) + const height = Math.max(1, bounds.height) + + const cx = clamp((cursor.x - bounds.x) / width, 0, 1) + const cy = clamp((cursor.y - bounds.y) / height, 0, 1) + + activeCursorSamples.push({ + timeMs: Math.max(0, Date.now() - cursorCaptureStartTimeMs), + cx, + cy, + }) + + if (activeCursorSamples.length > MAX_CURSOR_SAMPLES) { + activeCursorSamples.shift() + } +} export function registerIpcHandlers( createEditorWindow: () => void, @@ -63,6 +115,17 @@ export function registerIpcHandlers( const videoPath = path.join(RECORDINGS_DIR, fileName) await fs.writeFile(videoPath, Buffer.from(videoData)) currentVideoPath = videoPath; + + const telemetryPath = `${videoPath}.cursor.json` + if (pendingCursorSamples.length > 0) { + await fs.writeFile( + telemetryPath, + JSON.stringify({ version: CURSOR_TELEMETRY_VERSION, samples: pendingCursorSamples }, null, 2), + 'utf-8' + ) + } + pendingCursorSamples = [] + return { success: true, path: videoPath, @@ -100,12 +163,62 @@ export function registerIpcHandlers( }) ipcMain.handle('set-recording-state', (_, recording: boolean) => { + if (recording) { + stopCursorCapture() + activeCursorSamples = [] + pendingCursorSamples = [] + cursorCaptureStartTimeMs = Date.now() + sampleCursorPoint() + cursorCaptureInterval = setInterval(sampleCursorPoint, CURSOR_SAMPLE_INTERVAL_MS) + } else { + stopCursorCapture() + pendingCursorSamples = [...activeCursorSamples] + activeCursorSamples = [] + } + const source = selectedSource || { name: 'Screen' } if (onRecordingStateChange) { onRecordingStateChange(recording, source.name) } }) + ipcMain.handle('get-cursor-telemetry', async (_, videoPath?: string) => { + const targetVideoPath = videoPath ?? currentVideoPath + if (!targetVideoPath) { + return { success: true, samples: [] } + } + + const telemetryPath = `${targetVideoPath}.cursor.json` + try { + const content = await fs.readFile(telemetryPath, 'utf-8') + const parsed = JSON.parse(content) + const rawSamples = Array.isArray(parsed) + ? parsed + : (Array.isArray(parsed?.samples) ? parsed.samples : []) + + const samples: CursorTelemetryPoint[] = rawSamples + .filter((sample: unknown) => Boolean(sample && typeof sample === 'object')) + .map((sample: unknown) => { + const point = sample as Partial + return { + timeMs: typeof point.timeMs === 'number' && Number.isFinite(point.timeMs) ? Math.max(0, point.timeMs) : 0, + cx: typeof point.cx === 'number' && Number.isFinite(point.cx) ? clamp(point.cx, 0, 1) : 0.5, + cy: typeof point.cy === 'number' && Number.isFinite(point.cy) ? clamp(point.cy, 0, 1) : 0.5, + } + }) + .sort((a: CursorTelemetryPoint, b: CursorTelemetryPoint) => a.timeMs - b.timeMs) + + return { success: true, samples } + } catch (error) { + const nodeError = error as NodeJS.ErrnoException + if (nodeError.code === 'ENOENT') { + return { success: true, samples: [] } + } + console.error('Failed to load cursor telemetry:', error) + return { success: false, message: 'Failed to load cursor telemetry', error: String(error), samples: [] } + } + }) + ipcMain.handle('open-external-url', async (_, url: string) => { try { @@ -200,8 +313,6 @@ export function registerIpcHandlers( } }); - let currentVideoPath: string | null = null; - ipcMain.handle('set-current-video-path', (_, path: string) => { currentVideoPath = path; return { success: true }; diff --git a/electron/preload.ts b/electron/preload.ts index 34cf056..a609e7b 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -37,6 +37,9 @@ contextBridge.exposeInMainWorld('electronAPI', { setRecordingState: (recording: boolean) => { return ipcRenderer.invoke('set-recording-state', recording) }, + getCursorTelemetry: (videoPath?: string) => { + return ipcRenderer.invoke('get-cursor-telemetry', videoPath) + }, onStopRecordingFromTray: (callback: () => void) => { const listener = () => callback() ipcRenderer.on('stop-recording-from-tray', listener) diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index 66313f6..d1515ed 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -23,6 +23,7 @@ import { type ZoomDepth, type ZoomFocus, type ZoomRegion, + type CursorTelemetryPoint, type TrimRegion, type AnnotationRegion, type CropRegion, @@ -52,6 +53,7 @@ export default function VideoEditor() { const [padding, setPadding] = useState(50); const [cropRegion, setCropRegion] = useState(DEFAULT_CROP_REGION); const [zoomRegions, setZoomRegions] = useState([]); + const [cursorTelemetry, setCursorTelemetry] = useState([]); const [selectedZoomId, setSelectedZoomId] = useState(null); const [trimRegions, setTrimRegions] = useState([]); const [selectedTrimId, setSelectedTrimId] = useState(null); @@ -93,6 +95,19 @@ export default function VideoEditor() { return fileUrl; }; + const fromFileUrl = (fileUrl: string): string => { + if (!fileUrl.startsWith('file://')) { + return fileUrl; + } + + try { + const url = new URL(fileUrl); + return decodeURIComponent(url.pathname); + } catch { + return fileUrl.replace(/^file:\/\//, ''); + } + }; + useEffect(() => { async function loadVideo() { try { @@ -113,6 +128,37 @@ export default function VideoEditor() { loadVideo(); }, []); + useEffect(() => { + let mounted = true; + + async function loadCursorTelemetry() { + if (!videoPath) { + if (mounted) { + setCursorTelemetry([]); + } + return; + } + + try { + const result = await window.electronAPI.getCursorTelemetry(fromFileUrl(videoPath)); + if (mounted) { + setCursorTelemetry(result.success ? result.samples : []); + } + } catch (telemetryError) { + console.warn('Unable to load cursor telemetry:', telemetryError); + if (mounted) { + setCursorTelemetry([]); + } + } + } + + loadCursorTelemetry(); + + return () => { + mounted = false; + }; + }, [videoPath]); + // Initialize default wallpaper with resolved asset path useEffect(() => { let mounted = true; @@ -184,6 +230,21 @@ export default function VideoEditor() { setSelectedAnnotationId(null); }, []); + const handleZoomSuggested = useCallback((span: Span, focus: ZoomFocus) => { + const id = `zoom-${nextZoomIdRef.current++}`; + const newRegion: ZoomRegion = { + id, + startMs: Math.round(span.start), + endMs: Math.round(span.end), + depth: DEFAULT_ZOOM_DEPTH, + focus: clampFocusToDepth(focus, DEFAULT_ZOOM_DEPTH), + }; + setZoomRegions((prev) => [...prev, newRegion]); + setSelectedZoomId(id); + setSelectedTrimId(null); + setSelectedAnnotationId(null); + }, []); + const handleTrimAdded = useCallback((span: Span) => { const id = `trim-${nextTrimIdRef.current++}`; const newRegion: TrimRegion = { @@ -808,8 +869,10 @@ export default function VideoEditor() { videoDuration={duration} currentTime={currentTime} onSeek={handleSeek} + cursorTelemetry={cursorTelemetry} zoomRegions={zoomRegions} onZoomAdded={handleZoomAdded} + onZoomSuggested={handleZoomSuggested} onZoomSpanChange={handleZoomSpanChange} onZoomDelete={handleZoomDelete} selectedZoomId={selectedZoomId} @@ -898,4 +961,4 @@ export default function VideoEditor() { /> ); -} \ No newline at end of file +} diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx index dc2b2bd..2d0ced8 100644 --- a/src/components/video-editor/timeline/TimelineEditor.tsx +++ b/src/components/video-editor/timeline/TimelineEditor.tsx @@ -1,7 +1,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useTimelineContext } from "dnd-timeline"; import { Button } from "@/components/ui/button"; -import { Plus, Scissors, ZoomIn, MessageSquare, ChevronDown, Check } from "lucide-react"; +import { Plus, Scissors, ZoomIn, MessageSquare, ChevronDown, Check, WandSparkles } from "lucide-react"; import { toast } from "sonner"; import { cn } from "@/lib/utils"; import TimelineWrapper from "./TimelineWrapper"; @@ -9,7 +9,7 @@ import Row from "./Row"; import Item from "./Item"; import KeyframeMarkers from "./KeyframeMarkers"; import type { Range, Span } from "dnd-timeline"; -import type { ZoomRegion, TrimRegion, AnnotationRegion } from "../types"; +import type { ZoomRegion, TrimRegion, AnnotationRegion, CursorTelemetryPoint, ZoomFocus } from "../types"; import { v4 as uuidv4 } from 'uuid'; import { DropdownMenu, @@ -22,19 +22,23 @@ import { formatShortcut } from "@/utils/platformUtils"; import { TutorialHelp } from "../TutorialHelp"; import { useShortcuts } from "@/contexts/ShortcutsContext"; import { matchesShortcut } from "@/lib/shortcuts"; +import { detectZoomDwellCandidates, normalizeCursorTelemetry } from "./zoomSuggestionUtils"; const ZOOM_ROW_ID = "row-zoom"; const TRIM_ROW_ID = "row-trim"; const ANNOTATION_ROW_ID = "row-annotation"; const FALLBACK_RANGE_MS = 1000; const TARGET_MARKER_COUNT = 12; +const SUGGESTION_SPACING_MS = 1800; interface TimelineEditorProps { videoDuration: number; currentTime: number; onSeek?: (time: number) => void; + cursorTelemetry?: CursorTelemetryPoint[]; zoomRegions: ZoomRegion[]; onZoomAdded: (span: Span) => void; + onZoomSuggested?: (span: Span, focus: ZoomFocus) => void; onZoomSpanChange: (id: string, span: Span) => void; onZoomDelete: (id: string) => void; selectedZoomId: string | null; @@ -522,8 +526,10 @@ export default function TimelineEditor({ videoDuration, currentTime, onSeek, + cursorTelemetry = [], zoomRegions, onZoomAdded, + onZoomSuggested, onZoomSpanChange, onZoomDelete, selectedZoomId, @@ -719,6 +725,91 @@ export default function TimelineEditor({ onZoomAdded({ start: startPos, end: startPos + actualDuration }); }, [videoDuration, totalMs, currentTimeMs, zoomRegions, onZoomAdded, defaultRegionDurationMs]); + const handleSuggestZooms = useCallback(() => { + if (!videoDuration || videoDuration === 0 || totalMs === 0) { + return; + } + + if (!onZoomSuggested) { + toast.error("Zoom suggestion handler unavailable"); + return; + } + + if (cursorTelemetry.length < 2) { + toast.info("No cursor telemetry available", { + description: "Record a screencast first to generate cursor-based suggestions.", + }); + return; + } + + const defaultDuration = Math.min(defaultRegionDurationMs, totalMs); + if (defaultDuration <= 0) { + return; + } + + const reservedSpans = [...zoomRegions] + .map((region) => ({ start: region.startMs, end: region.endMs })) + .sort((a, b) => a.start - b.start); + + const normalizedSamples = normalizeCursorTelemetry(cursorTelemetry, totalMs); + + if (normalizedSamples.length < 2) { + toast.info("No usable cursor telemetry", { + description: "The recording does not include enough cursor movement data.", + }); + return; + } + + const dwellCandidates = detectZoomDwellCandidates(normalizedSamples); + + if (dwellCandidates.length === 0) { + toast.info("No clear cursor dwell moments found", { + description: "Try a recording with slower cursor pauses on important actions.", + }); + return; + } + + const sortedCandidates = [...dwellCandidates].sort((a, b) => b.strength - a.strength); + const acceptedCenters: number[] = []; + + let addedCount = 0; + + sortedCandidates.forEach((candidate) => { + const tooCloseToAccepted = acceptedCenters.some( + (center) => Math.abs(center - candidate.centerTimeMs) < SUGGESTION_SPACING_MS, + ); + + if (tooCloseToAccepted) { + return; + } + + const centeredStart = Math.round(candidate.centerTimeMs - defaultDuration / 2); + const candidateStart = Math.max(0, Math.min(centeredStart, totalMs - defaultDuration)); + const candidateEnd = candidateStart + defaultDuration; + const hasOverlap = reservedSpans.some( + (span) => candidateEnd > span.start && candidateStart < span.end, + ); + + if (hasOverlap) { + return; + } + + reservedSpans.push({ start: candidateStart, end: candidateEnd }); + acceptedCenters.push(candidate.centerTimeMs); + onZoomSuggested({ start: candidateStart, end: candidateEnd }, candidate.focus); + addedCount += 1; + }); + + if (addedCount === 0) { + toast.info("No auto-zoom slots available", { + description: "Detected dwell points overlap existing zoom regions.", + }); + return; + } + + toast.success(`Added ${addedCount} cursor-based zoom suggestion${addedCount === 1 ? "" : "s"}`); + }, [videoDuration, totalMs, defaultRegionDurationMs, zoomRegions, onZoomSuggested, cursorTelemetry]); + const handleAddTrim = useCallback(() => { if (!videoDuration || videoDuration === 0 || totalMs === 0 || !onTrimAdded) { return; @@ -923,6 +1014,15 @@ export default function TimelineEditor({ > +