diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 431cda1..1d73a9b 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -2,7 +2,7 @@ import fs from "node:fs/promises"; import { createRequire } from "node:module"; import os from "node:os"; import path from "node:path"; -import { fileURLToPath } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; const nodeRequire = createRequire(import.meta.url); @@ -27,8 +27,18 @@ import { type RecordingSession, type StoreRecordedSessionInput, } from "../../src/lib/recordingSession"; +import type { + CursorRecordingData, + CursorRecordingSample, + NativeCursorAsset, + ProjectFileResult, + ProjectPathResult, +} from "../../src/native/contracts"; import { mainT } from "../i18n"; import { RECORDINGS_DIR } from "../main"; +import { createCursorRecordingSession } from "../native-bridge/cursor/recording/factory"; +import type { CursorRecordingSession } from "../native-bridge/cursor/recording/session"; +import { registerNativeBridgeHandlers } from "./nativeBridge"; const PROJECT_FILE_EXTENSION = "openscreen"; const SHORTCUTS_FILE = path.join(app.getPath("userData"), "shortcuts.json"); @@ -193,7 +203,7 @@ type SelectedSource = { let selectedSource: SelectedSource | null = null; let currentProjectPath: string | null = null; -let currentRecordingSession: RecordingSession | null = null; +let currentVideoPath: string | null = null; function normalizePath(filePath: string) { return path.resolve(filePath); @@ -227,450 +237,180 @@ function isTrustedProjectPath(filePath?: string | null) { return normalizePath(filePath) === normalizePath(currentProjectPath); } -function setCurrentRecordingSessionState(session: RecordingSession | null) { - currentRecordingSession = session; -} - -function getSessionManifestPathForVideo(videoPath: string) { - const parsed = path.parse(videoPath); - const baseName = parsed.name.endsWith("-webcam") - ? parsed.name.slice(0, -"-webcam".length) - : parsed.name; - return path.join(parsed.dir, `${baseName}${RECORDING_SESSION_SUFFIX}`); -} - -async function loadRecordedSessionForVideoPath( - videoPath: string, -): Promise { - const normalizedVideoPath = normalizeVideoSourcePath(videoPath); - if (!normalizedVideoPath) { - return null; - } - - try { - const manifestPath = getSessionManifestPathForVideo(normalizedVideoPath); - const content = await fs.readFile(manifestPath, "utf-8"); - const session = normalizeRecordingSession(JSON.parse(content)); - if (!session) { - return null; - } - - const normalizedSession: RecordingSession = { - ...session, - screenVideoPath: normalizeVideoSourcePath(session.screenVideoPath) ?? session.screenVideoPath, - ...(session.webcamVideoPath - ? { - webcamVideoPath: - normalizeVideoSourcePath(session.webcamVideoPath) ?? session.webcamVideoPath, - } - : {}), - }; - - const targetPath = normalizePath(normalizedVideoPath); - const screenMatches = normalizePath(normalizedSession.screenVideoPath) === targetPath; - const webcamMatches = normalizedSession.webcamVideoPath - ? normalizePath(normalizedSession.webcamVideoPath) === targetPath - : false; - - return screenMatches || webcamMatches ? normalizedSession : null; - } catch { - return null; - } -} - -async function storeRecordedSessionFiles(payload: StoreRecordedSessionInput) { - const createdAt = - typeof payload.createdAt === "number" && Number.isFinite(payload.createdAt) - ? payload.createdAt - : Date.now(); - const screenVideoPath = resolveRecordingOutputPath(payload.screen.fileName); - await fs.writeFile(screenVideoPath, Buffer.from(payload.screen.videoData)); - - let webcamVideoPath: string | undefined; - if (payload.webcam) { - webcamVideoPath = resolveRecordingOutputPath(payload.webcam.fileName); - await fs.writeFile(webcamVideoPath, Buffer.from(payload.webcam.videoData)); - } - - const session: RecordingSession = webcamVideoPath - ? { screenVideoPath, webcamVideoPath, createdAt } - : { screenVideoPath, createdAt }; - setCurrentRecordingSessionState(session); - currentProjectPath = null; - - const telemetryPath = `${screenVideoPath}.cursor.json`; - const pendingBatch = cursorTelemetryBuffer.takeNextBatch(); - const pendingClicks = takeCursorClickTimestamps(); - if ((pendingBatch && pendingBatch.samples.length > 0) || pendingClicks.length > 0) { - try { - await fs.writeFile( - telemetryPath, - JSON.stringify( - { - version: CURSOR_TELEMETRY_VERSION, - samples: pendingBatch?.samples ?? [], - clicks: pendingClicks, - }, - null, - 2, - ), - "utf-8", - ); - } catch (err) { - if (pendingBatch) cursorTelemetryBuffer.prependBatch(pendingBatch); - throw err; - } - } - - const sessionManifestPath = path.join( - RECORDINGS_DIR, - `${path.parse(payload.screen.fileName).name}${RECORDING_SESSION_SUFFIX}`, - ); - await fs.writeFile(sessionManifestPath, JSON.stringify(session, null, 2), "utf-8"); - - return { - success: true, - path: screenVideoPath, - session, - message: "Recording session stored successfully", - }; -} - -const CURSOR_TELEMETRY_VERSION = 1; +const CURSOR_TELEMETRY_VERSION = 2; const CURSOR_SAMPLE_INTERVAL_MS = 100; const MAX_CURSOR_SAMPLES = 60 * 60 * 10; // 1 hour @ 10Hz -let cursorCaptureInterval: NodeJS.Timeout | null = null; -let cursorCaptureStartTimeMs = 0; -const cursorTelemetryBuffer = createCursorTelemetryBuffer({ - maxActiveSamples: MAX_CURSOR_SAMPLES, -}); - -// Mouse click timestamps (macOS only — uiohook-napi behind Accessibility). -const MAX_CURSOR_CLICKS = 60 * 60 * 60; // ~1 click/sec for an hour -let cursorClickTimestampsMs: number[] = []; -let uioHookInstance: { - start: () => void; - stop: () => void; - on: (...a: unknown[]) => void; - off?: (...a: unknown[]) => void; - removeListener?: (...a: unknown[]) => void; -} | null = null; -let uioHookMouseDownHandler: ((event: { time?: number }) => void) | null = null; -let uioHookFailureLogged = false; +let cursorRecordingSession: CursorRecordingSession | null = null; +let pendingCursorRecordingData: CursorRecordingData | null = null; function clamp(value: number, min: number, max: number) { return Math.min(max, Math.max(min, value)); } -function loadUioHookForClicks(): typeof uioHookInstance { - try { - // Dynamic require + try/catch so a broken native binary doesn't crash startup. - const mod = nodeRequire("uiohook-napi"); - const candidate = mod.uIOhook ?? mod.default?.uIOhook ?? mod.uiohook ?? mod.default; - if (candidate && typeof candidate.start === "function" && typeof candidate.on === "function") { - return candidate; - } - return null; - } catch (error) { - if (!uioHookFailureLogged) { - uioHookFailureLogged = true; - console.warn("[clickCapture] uiohook-napi unavailable:", error); - } +function normalizeCursorSample(sample: unknown): CursorRecordingSample | null { + if (!sample || typeof sample !== "object") { return null; } -} -function startClickCapture() { - if (process.platform !== "darwin") return; - if (uioHookInstance) return; - - // Passive check — the prompt fires from the renderer when the user toggles - // "Only on clicks" so it doesn't stack with the screen-recording prompt. - try { - if (!systemPreferences.isTrustedAccessibilityClient(false)) { - if (!uioHookFailureLogged) { - uioHookFailureLogged = true; - console.warn( - "[clickCapture] Accessibility permission not granted — click capture disabled.", - ); - } - return; - } - } catch { - // fall through; uiohook will fail defensively below - } - - const hook = loadUioHookForClicks(); - if (!hook) return; - - uioHookMouseDownHandler = (event) => { - const elapsed = Math.max(0, Date.now() - cursorCaptureStartTimeMs); - void event; - if (cursorClickTimestampsMs.length >= MAX_CURSOR_CLICKS) return; - cursorClickTimestampsMs.push(elapsed); + const point = sample as Partial; + return { + timeMs: + typeof point.timeMs === "number" && Number.isFinite(point.timeMs) + ? Math.max(0, point.timeMs) + : 0, + cx: typeof point.cx === "number" && Number.isFinite(point.cx) ? clamp(point.cx, 0, 1) : 0.5, + cy: typeof point.cy === "number" && Number.isFinite(point.cy) ? clamp(point.cy, 0, 1) : 0.5, + assetId: typeof point.assetId === "string" ? point.assetId : null, + visible: typeof point.visible === "boolean" ? point.visible : true, }; +} +function normalizeCursorAsset(asset: unknown): NativeCursorAsset | null { + if (!asset || typeof asset !== "object") { + return null; + } + + const candidate = asset as Partial; + if (typeof candidate.id !== "string" || typeof candidate.imageDataUrl !== "string") { + return null; + } + + return { + id: candidate.id, + platform: + candidate.platform === "win32" ? "win32" : process.platform === "darwin" ? "darwin" : "linux", + imageDataUrl: candidate.imageDataUrl, + width: + typeof candidate.width === "number" && Number.isFinite(candidate.width) + ? Math.max(1, Math.round(candidate.width)) + : 1, + height: + typeof candidate.height === "number" && Number.isFinite(candidate.height) + ? Math.max(1, Math.round(candidate.height)) + : 1, + hotspotX: + typeof candidate.hotspotX === "number" && Number.isFinite(candidate.hotspotX) + ? Math.max(0, Math.round(candidate.hotspotX)) + : 0, + hotspotY: + typeof candidate.hotspotY === "number" && Number.isFinite(candidate.hotspotY) + ? Math.max(0, Math.round(candidate.hotspotY)) + : 0, + scaleFactor: + typeof candidate.scaleFactor === "number" && Number.isFinite(candidate.scaleFactor) + ? Math.max(0.1, candidate.scaleFactor) + : undefined, + }; +} + +async function readCursorRecordingFile(targetVideoPath: string): Promise { + const telemetryPath = `${targetVideoPath}.cursor.json`; try { - hook.on("mousedown", uioHookMouseDownHandler); - hook.start(); - uioHookInstance = hook; + const content = await fs.readFile(telemetryPath, "utf-8"); + const parsed = JSON.parse(content); + const rawSamples = Array.isArray(parsed) + ? parsed + : Array.isArray(parsed?.samples) + ? parsed.samples + : []; + const rawAssets = Array.isArray(parsed?.assets) ? parsed.assets : []; + + const samples = rawSamples + .map((sample: unknown) => normalizeCursorSample(sample)) + .filter((sample: CursorRecordingSample | null): sample is CursorRecordingSample => + Boolean(sample), + ) + .sort((a: CursorRecordingSample, b: CursorRecordingSample) => a.timeMs - b.timeMs); + + const assets = rawAssets + .map((asset: unknown) => normalizeCursorAsset(asset)) + .filter((asset: NativeCursorAsset | null): asset is NativeCursorAsset => Boolean(asset)); + + return { + version: + typeof parsed?.version === "number" && Number.isFinite(parsed.version) ? parsed.version : 1, + provider: parsed?.provider === "native" ? "native" : "none", + samples, + assets, + }; } catch (error) { - if (!uioHookFailureLogged) { - uioHookFailureLogged = true; - console.warn("[clickCapture] failed to start uiohook:", error); + const nodeError = error as NodeJS.ErrnoException; + if (nodeError.code === "ENOENT") { + return { + version: CURSOR_TELEMETRY_VERSION, + provider: "none", + samples: [], + assets: [], + }; } - uioHookMouseDownHandler = null; + + console.error("Failed to load cursor telemetry:", error); + throw error; } } -function stopClickCapture() { - if (!uioHookInstance) return; +async function readCursorTelemetryFile(targetVideoPath: string) { try { - if (uioHookMouseDownHandler) { - if (typeof uioHookInstance.off === "function") { - uioHookInstance.off("mousedown", uioHookMouseDownHandler); - } else if (typeof uioHookInstance.removeListener === "function") { - uioHookInstance.removeListener("mousedown", uioHookMouseDownHandler); - } - } - uioHookInstance.stop(); + const recordingData = await readCursorRecordingFile(targetVideoPath); + return { + success: true, + samples: recordingData.samples.map((sample) => ({ + timeMs: sample.timeMs, + cx: sample.cx, + cy: sample.cy, + })), + }; } catch (error) { - console.warn("[clickCapture] failed to stop uiohook:", error); + console.error("Failed to load cursor telemetry:", error); + return { + success: false, + message: "Failed to load cursor telemetry", + error: String(error), + samples: [], + }; } - uioHookInstance = null; - uioHookMouseDownHandler = null; } -function takeCursorClickTimestamps(): number[] { - const out = cursorClickTimestampsMs; - cursorClickTimestampsMs = []; - return out; -} - -function stopCursorCapture() { - if (cursorCaptureInterval) { - clearInterval(cursorCaptureInterval); - cursorCaptureInterval = null; +function resolveAssetBasePath() { + try { + if (app.isPackaged) { + const assetPath = path.join(process.resourcesPath, "assets"); + return pathToFileURL(`${assetPath}${path.sep}`).toString(); + } + const assetPath = path.join(app.getAppPath(), "public", "assets"); + return pathToFileURL(`${assetPath}${path.sep}`).toString(); + } catch (err) { + console.error("Failed to resolve asset base path:", err); + return null; } - stopClickCapture(); } -function sampleCursorPoint() { +function getSelectedSourceBounds() { const cursor = screen.getCursorScreenPoint(); const sourceDisplayId = Number(selectedSource?.display_id); const sourceDisplay = Number.isFinite(sourceDisplayId) ? (screen.getAllDisplays().find((display) => display.id === sourceDisplayId) ?? null) : null; - const display = sourceDisplay ?? screen.getDisplayNearestPoint(cursor); - const bounds = display.bounds; - const width = Math.max(1, bounds.width); - const height = Math.max(1, bounds.height); - - const cx = clamp((cursor.x - bounds.x) / width, 0, 1); - const cy = clamp((cursor.y - bounds.y) / height, 0, 1); - - cursorTelemetryBuffer.push({ - timeMs: Math.max(0, Date.now() - cursorCaptureStartTimeMs), - cx, - cy, - }); + return (sourceDisplay ?? screen.getDisplayNearestPoint(cursor)).bounds; } export function registerIpcHandlers( createEditorWindow: () => void, createSourceSelectorWindow: () => BrowserWindow, - createCountdownOverlayWindow: () => BrowserWindow, getMainWindow: () => BrowserWindow | null, getSourceSelectorWindow: () => BrowserWindow | null, - getCountdownOverlayWindow: () => BrowserWindow | null, onRecordingStateChange?: (recording: boolean, sourceName: string) => void, - switchToHud?: () => void, ) { - const supportsWindowOpacity = process.platform !== "linux"; - const countdownOverlayState = { - visible: false, - value: null as number | null, - activeRunId: null as number | null, - hideCommitId: 0, - hideCommitTimer: null as ReturnType | null, - }; - const COUNTDOWN_OVERLAY_HIDE_DEBOUNCE_MS = 1200; - - const clearCountdownOverlayHideCommit = () => { - if (countdownOverlayState.hideCommitTimer) { - clearTimeout(countdownOverlayState.hideCommitTimer); - countdownOverlayState.hideCommitTimer = null; - } - }; - - const commitCountdownOverlayHide = (win: BrowserWindow, hideCommitId: number) => { - if (win.isDestroyed()) { - return; - } - - if (countdownOverlayState.visible || countdownOverlayState.hideCommitId !== hideCommitId) { - return; - } - - win.hide(); - if (supportsWindowOpacity) { - // Reset baseline opacity for the next show cycle. - win.setOpacity(1); - } - }; - - const flushCountdownOverlayState = (win: BrowserWindow) => { - if (win.isDestroyed()) { - return; - } - - clearCountdownOverlayHideCommit(); - win.webContents.send("countdown-overlay-value", countdownOverlayState.value); - if (!countdownOverlayState.visible) { - return; - } - - if (win.isVisible()) { - if (supportsWindowOpacity) { - win.setOpacity(1); - } - return; - } - - setTimeout(() => { - if (!win.isDestroyed() && countdownOverlayState.visible && !win.isVisible()) { - if (supportsWindowOpacity) { - win.setOpacity(0); - } - win.showInactive(); - - if (supportsWindowOpacity) { - setTimeout(() => { - if (!win.isDestroyed() && countdownOverlayState.visible && win.isVisible()) { - win.setOpacity(1); - } - }, 0); - } - } - }, 16); - }; - - ipcMain.handle("countdown-overlay-show", (_, value: number, runId: number) => { - countdownOverlayState.activeRunId = runId; - countdownOverlayState.visible = true; - countdownOverlayState.value = value; - - const win = getCountdownOverlayWindow() ?? createCountdownOverlayWindow(); - if (win.isDestroyed()) { - return; - } - - if (win.webContents.isLoading()) { - win.webContents.once("did-finish-load", () => { - if (!win.isDestroyed()) { - flushCountdownOverlayState(win); - } - }); - } else { - flushCountdownOverlayState(win); - } - }); - - ipcMain.handle("countdown-overlay-set-value", (_, value: number, runId: number) => { - if (countdownOverlayState.activeRunId !== runId || !countdownOverlayState.visible) { - return; - } - - countdownOverlayState.value = value; - - const win = getCountdownOverlayWindow(); - if (!win || win.isDestroyed()) { - return; - } - - if (win.webContents.isLoading()) { - return; - } - - win.webContents.send("countdown-overlay-value", value); - }); - - ipcMain.handle("countdown-overlay-hide", (_, runId: number) => { - if (countdownOverlayState.activeRunId !== runId) { - return; - } - - countdownOverlayState.visible = false; - countdownOverlayState.hideCommitId += 1; - const hideCommitId = countdownOverlayState.hideCommitId; - clearCountdownOverlayHideCommit(); - - const win = getCountdownOverlayWindow(); - if (!win || win.isDestroyed()) { - countdownOverlayState.value = null; - return; - } - - if (supportsWindowOpacity) { - // Hide visually immediately to avoid hide/show compositor flashes on rapid restart. - win.setOpacity(0); - } - - countdownOverlayState.value = null; - if (!win.webContents.isLoading()) { - win.webContents.send("countdown-overlay-value", countdownOverlayState.value); - } - - if (!supportsWindowOpacity) { - win.hide(); - return; - } - - countdownOverlayState.hideCommitTimer = setTimeout(() => { - countdownOverlayState.hideCommitTimer = null; - commitCountdownOverlayHide(win, hideCommitId); - }, COUNTDOWN_OVERLAY_HIDE_DEBOUNCE_MS); - }); - - ipcMain.handle("switch-to-hud", () => { - if (switchToHud) switchToHud(); - }); - ipcMain.handle("start-new-recording", () => { - try { - setCurrentRecordingSessionState(null); - if (switchToHud) { - switchToHud(); - } - return { success: true }; - } catch (error) { - console.error("Failed to start new recording:", error); - return { success: false, error: String(error) }; - } - }); - ipcMain.handle("get-sources", async (_, opts) => { - const ownWindowSourceIds = new Set( - BrowserWindow.getAllWindows() - .map((win) => { - try { - return win.getMediaSourceId(); - } catch { - return null; - } - }) - .filter((id): id is string => Boolean(id)), - ); const sources = await desktopCapturer.getSources(opts); - return sources - .filter((source) => !ownWindowSourceIds.has(source.id)) - .map((source) => ({ - id: source.id, - name: source.name, - display_id: source.display_id, - thumbnail: source.thumbnail ? source.thumbnail.toDataURL() : null, - appIcon: source.appIcon ? source.appIcon.toDataURL() : null, - })); + return sources.map((source) => ({ + id: source.id, + name: source.name, + display_id: source.display_id, + thumbnail: source.thumbnail ? source.thumbnail.toDataURL() : null, + appIcon: source.appIcon ? source.appIcon.toDataURL() : null, + })); }); ipcMain.handle("select-source", (_, source: SelectedSource) => { @@ -779,7 +519,25 @@ export function registerIpcHandlers( ipcMain.handle("store-recorded-session", async (_, payload: StoreRecordedSessionInput) => { try { - return await storeRecordedSessionFiles(payload); + const videoPath = path.join(RECORDINGS_DIR, fileName); + await fs.writeFile(videoPath, Buffer.from(videoData)); + currentProjectPath = null; + + const telemetryPath = `${videoPath}.cursor.json`; + if (pendingCursorRecordingData && pendingCursorRecordingData.samples.length > 0) { + await fs.writeFile( + telemetryPath, + JSON.stringify(pendingCursorRecordingData, null, 2), + "utf-8", + ); + } + pendingCursorRecordingData = null; + + return { + success: true, + path: videoPath, + message: "Video stored successfully", + }; } catch (error) { console.error("Failed to store recording session:", error); return { @@ -821,24 +579,7 @@ export function registerIpcHandlers( return { success: false, message: "No recorded video found" }; } - // Sort by most recently modified to reliably get the latest recording. - // Lexicographic sort is unreliable (e.g. recording-9.webm > recording-10.webm). - let latestVideo: string | null = null; - let latestMtimeMs = 0; - for (const file of videoFiles) { - try { - const stat = await fs.stat(path.join(RECORDINGS_DIR, file)); - if (stat.mtimeMs > latestMtimeMs) { - latestMtimeMs = stat.mtimeMs; - latestVideo = file; - } - } catch { - // Skip inaccessible files. - } - } - if (!latestVideo) { - return { success: false, message: "No recorded video found" }; - } + const latestVideo = videoFiles.sort().reverse()[0]; const videoPath = path.join(RECORDINGS_DIR, latestVideo); return { success: true, path: videoPath }; @@ -848,54 +589,38 @@ export function registerIpcHandlers( } }); - ipcMain.handle("read-binary-file", async (_, inputPath: string) => { - try { - const normalizedPath = normalizeVideoSourcePath(inputPath); - if (!normalizedPath) { - return { success: false, message: "Invalid file path" }; - } - - if (!isPathAllowed(normalizedPath)) { - console.warn( - "[read-binary-file] Rejected path outside allowed directories:", - normalizedPath, - ); - return { success: false, message: "Access denied: path outside allowed directories" }; - } - - const data = await fs.readFile(normalizedPath); - return { - success: true, - data: data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength), - path: normalizedPath, - }; - } catch (error) { - console.error("Failed to read binary file:", error); - return { - success: false, - message: "Failed to read binary file", - error: String(error), - }; - } - }); - - ipcMain.handle("set-recording-state", (_, recording: boolean, recordingId?: number) => { + ipcMain.handle("set-recording-state", async (_, recording: boolean) => { if (recording) { - stopCursorCapture(); - // The renderer is the source of truth for the recording id (it - // uses the same id as the saved fileName). Fall back to a - // timestamp only if the renderer didn't supply one, so the - // buffer always has a stable key per session. - const id = typeof recordingId === "number" ? recordingId : Date.now(); - cursorTelemetryBuffer.startSession(id); - cursorCaptureStartTimeMs = Date.now(); - cursorClickTimestampsMs = []; - startClickCapture(); - sampleCursorPoint(); - cursorCaptureInterval = setInterval(sampleCursorPoint, CURSOR_SAMPLE_INTERVAL_MS); + if (cursorRecordingSession) { + pendingCursorRecordingData = await cursorRecordingSession.stop(); + cursorRecordingSession = null; + } + + pendingCursorRecordingData = null; + cursorRecordingSession = createCursorRecordingSession({ + getDisplayBounds: getSelectedSourceBounds, + maxSamples: MAX_CURSOR_SAMPLES, + platform: process.platform, + sampleIntervalMs: CURSOR_SAMPLE_INTERVAL_MS, + }); + + try { + await cursorRecordingSession.start(); + } catch (error) { + console.error("Failed to start cursor recording session:", error); + cursorRecordingSession = null; + } } else { - stopCursorCapture(); - cursorTelemetryBuffer.endSession(); + if (cursorRecordingSession) { + try { + pendingCursorRecordingData = await cursorRecordingSession.stop(); + } catch (error) { + console.error("Failed to stop cursor recording session:", error); + pendingCursorRecordingData = null; + } finally { + cursorRecordingSession = null; + } + } } const source = selectedSource || { name: "Screen" }; @@ -904,10 +629,6 @@ export function registerIpcHandlers( } }); - ipcMain.handle("discard-cursor-telemetry", (_, recordingId: number) => { - cursorTelemetryBuffer.discardBatch(recordingId); - }); - ipcMain.handle("get-cursor-telemetry", async (_, videoPath?: string) => { const targetVideoPath = normalizeVideoSourcePath( videoPath ?? currentRecordingSession?.screenVideoPath, @@ -916,85 +637,12 @@ export function registerIpcHandlers( return { success: true, samples: [] }; } - if (!isPathAllowed(targetVideoPath)) { - console.warn( - "[get-cursor-telemetry] Rejected path outside allowed directories:", - targetVideoPath, - ); - return { success: true, samples: [] }; - } - - const telemetryPath = `${targetVideoPath}.cursor.json`; - try { - const content = await fs.readFile(telemetryPath, "utf-8"); - const parsed = JSON.parse(content); - const rawSamples = Array.isArray(parsed) - ? parsed - : Array.isArray(parsed?.samples) - ? parsed.samples - : []; - - const samples: CursorTelemetryPoint[] = rawSamples - .filter((sample: unknown) => Boolean(sample && typeof sample === "object")) - .map((sample: unknown) => { - const point = sample as Partial; - return { - timeMs: - typeof point.timeMs === "number" && Number.isFinite(point.timeMs) - ? Math.max(0, point.timeMs) - : 0, - cx: - typeof point.cx === "number" && Number.isFinite(point.cx) - ? clamp(point.cx, 0, 1) - : 0.5, - cy: - typeof point.cy === "number" && Number.isFinite(point.cy) - ? clamp(point.cy, 0, 1) - : 0.5, - }; - }) - .sort((a: CursorTelemetryPoint, b: CursorTelemetryPoint) => a.timeMs - b.timeMs); - - const rawClicks = Array.isArray(parsed?.clicks) ? parsed.clicks : []; - const clicks: number[] = rawClicks - .map((value: unknown) => - typeof value === "number" && Number.isFinite(value) ? Math.max(0, value) : null, - ) - .filter((v: number | null): v is number => v !== null) - .sort((a: number, b: number) => a - b); - - return { success: true, samples, clicks }; - } catch (error) { - const nodeError = error as NodeJS.ErrnoException; - if (nodeError.code === "ENOENT") { - return { success: true, samples: [], clicks: [] }; - } - console.error("Failed to load cursor telemetry:", error); - return { - success: false, - message: "Failed to load cursor telemetry", - error: String(error), - samples: [], - clicks: [], - }; - } + return readCursorTelemetryFile(targetVideoPath); }); ipcMain.handle("open-external-url", async (_, url: string) => { try { - const ALLOWED_SCHEMES = ["http:", "https:", "mailto:"]; - let parsed: URL; - try { - parsed = new URL(url); - } catch { - return { success: false, error: "Invalid URL" }; - } - - if (!ALLOWED_SCHEMES.includes(parsed.protocol)) { - return { success: false, error: `Unsupported URL scheme: ${parsed.protocol}` }; - } - - await shell.openExternal(parsed.toString()); + await shell.openExternal(url); return { success: true }; } catch (error) { console.error("Failed to open URL:", error); @@ -1002,15 +650,10 @@ export function registerIpcHandlers( } }); - /** - * Handles saving an exported video file. - * Shows a save dialog, normalizes the file path for the current OS, - * ensures the directory exists, and writes the video data. - * @param _ - Unused event parameter. - * @param videoData - The exported video as an ArrayBuffer. - * @param fileName - Suggested filename for the save dialog. - * @returns Object with success status, optional file path, and error details. - */ + // Return base path for assets so renderer can resolve file:// paths in production + ipcMain.handle("get-asset-base-path", () => { + return resolveAssetBasePath(); + }); ipcMain.handle("pick-export-save-path", async (_, fileName: string, exportFolder?: string) => { try { @@ -1082,7 +725,7 @@ export function registerIpcHandlers( return { success: true, - path: normalizedPath, + path: result.filePath, message: "Video exported successfully", }; } catch (error) { @@ -1094,6 +737,7 @@ export function registerIpcHandlers( }; } }); + ipcMain.handle("open-video-file-picker", async () => { try { const dialogOptions = buildDialogOptions( @@ -1117,17 +761,10 @@ export function registerIpcHandlers( return { success: false, canceled: true }; } - const approvedPath = await approveReadableVideoPath(result.filePaths[0]); - if (!approvedPath) { - return { - success: false, - message: "Selected file is not a supported video", - }; - } currentProjectPath = null; return { success: true, - path: approvedPath, + path: result.filePaths[0], }; } catch (error) { console.error("Failed to open file picker:", error); @@ -1166,75 +803,87 @@ export function registerIpcHandlers( ipcMain.handle( "save-project-file", async (_, projectData: unknown, suggestedName?: string, existingProjectPath?: string) => { - try { - const trustedExistingProjectPath = isTrustedProjectPath(existingProjectPath) - ? existingProjectPath - : null; - - if (trustedExistingProjectPath) { - await fs.writeFile( - trustedExistingProjectPath, - JSON.stringify(projectData, null, 2), - "utf-8", - ); - currentProjectPath = trustedExistingProjectPath; - return { - success: true, - path: trustedExistingProjectPath, - message: "Project saved successfully", - }; - } - - const safeName = (suggestedName || `project-${Date.now()}`).replace(/[^a-zA-Z0-9-_]/g, "_"); - const defaultName = safeName.endsWith(`.${PROJECT_FILE_EXTENSION}`) - ? safeName - : `${safeName}.${PROJECT_FILE_EXTENSION}`; - - const dialogOptions = buildDialogOptions( - { - title: mainT("dialogs", "fileDialogs.saveProject"), - defaultPath: path.join(RECORDINGS_DIR, defaultName), - filters: [ - { - name: mainT("dialogs", "fileDialogs.openscreenProject"), - extensions: [PROJECT_FILE_EXTENSION], - }, - { name: "JSON", extensions: ["json"] }, - ], - properties: ["createDirectory", "showOverwriteConfirmation"], - }, - getMainWindow(), - ); - const result = await dialog.showSaveDialog(dialogOptions); - - if (result.canceled || !result.filePath) { - return { - success: false, - canceled: true, - message: "Save project canceled", - }; - } - - await fs.writeFile(result.filePath, JSON.stringify(projectData, null, 2), "utf-8"); - currentProjectPath = result.filePath; - - return { - success: true, - path: result.filePath, - message: "Project saved successfully", - }; - } catch (error) { - console.error("Failed to save project file:", error); - return { - success: false, - message: "Failed to save project file", - error: String(error), - }; - } + return saveProjectFile(projectData, suggestedName, existingProjectPath); }, ); + async function saveProjectFile( + projectData: unknown, + suggestedName?: string, + existingProjectPath?: string, + ): Promise { + try { + const trustedExistingProjectPath = isTrustedProjectPath(existingProjectPath) + ? existingProjectPath + : null; + + if (trustedExistingProjectPath) { + await fs.writeFile( + trustedExistingProjectPath, + JSON.stringify(projectData, null, 2), + "utf-8", + ); + currentProjectPath = trustedExistingProjectPath; + return { + success: true, + path: trustedExistingProjectPath, + message: "Project saved successfully", + }; + } + + const safeName = (suggestedName || `project-${Date.now()}`).replace(/[^a-zA-Z0-9-_]/g, "_"); + const defaultName = safeName.endsWith(`.${PROJECT_FILE_EXTENSION}`) + ? safeName + : `${safeName}.${PROJECT_FILE_EXTENSION}`; + + const dialogOptions = buildDialogOptions( + { + title: mainT("dialogs", "fileDialogs.saveProject"), + defaultPath: path.join(RECORDINGS_DIR, defaultName), + filters: [ + { + name: mainT("dialogs", "fileDialogs.openscreenProject"), + extensions: [PROJECT_FILE_EXTENSION], + }, + { name: "JSON", extensions: ["json"] }, + ], + properties: ["createDirectory", "showOverwriteConfirmation"], + }, + getMainWindow(), + ); + const result = await dialog.showSaveDialog(dialogOptions); + + if (result.canceled || !result.filePath) { + return { + success: false, + canceled: true, + message: "Save project canceled", + }; + } + + await fs.writeFile(result.filePath, JSON.stringify(projectData, null, 2), "utf-8"); + currentProjectPath = result.filePath; + + return { + success: true, + path: result.filePath, + message: "Project saved successfully", + }; + } catch (error) { + console.error("Failed to save project file:", error); + return { + success: false, + message: "Failed to save project file", + error: String(error), + }; + } + } + ipcMain.handle("load-project-file", async () => { + return loadProjectFile(); + }); + + async function loadProjectFile(): Promise { try { const dialogOptions = buildDialogOptions( { @@ -1261,9 +910,19 @@ export function registerIpcHandlers( const filePath = result.filePaths[0]; const content = await fs.readFile(filePath, "utf-8"); const project = JSON.parse(content); - const session = await getApprovedProjectSession(project, filePath); currentProjectPath = filePath; - setCurrentRecordingSessionState(session); + if (project && typeof project === "object") { + const rawProject = project as { media?: unknown; videoPath?: unknown }; + const media = + normalizeProjectMedia(rawProject.media) ?? + (typeof rawProject.videoPath === "string" + ? { + screenVideoPath: + normalizeVideoSourcePath(rawProject.videoPath) ?? rawProject.videoPath, + } + : null); + setCurrentRecordingSessionState(media ? { ...media, createdAt: Date.now() } : null); + } return { success: true, @@ -1278,9 +937,13 @@ export function registerIpcHandlers( error: String(error), }; } - }); + } ipcMain.handle("load-current-project-file", async () => { + return loadCurrentProjectFile(); + }); + + async function loadCurrentProjectFile(): Promise { try { if (!currentProjectPath) { return { success: false, message: "No active project" }; @@ -1288,8 +951,18 @@ export function registerIpcHandlers( const content = await fs.readFile(currentProjectPath, "utf-8"); const project = JSON.parse(content); - const session = await getApprovedProjectSession(project, currentProjectPath); - setCurrentRecordingSessionState(session); + if (project && typeof project === "object") { + const rawProject = project as { media?: unknown; videoPath?: unknown }; + const media = + normalizeProjectMedia(rawProject.media) ?? + (typeof rawProject.videoPath === "string" + ? { + screenVideoPath: + normalizeVideoSourcePath(rawProject.videoPath) ?? rawProject.videoPath, + } + : null); + setCurrentRecordingSessionState(media ? { ...media, createdAt: Date.now() } : null); + } return { success: true, path: currentProjectPath, @@ -1303,55 +976,35 @@ export function registerIpcHandlers( error: String(error), }; } - }); - ipcMain.handle("set-current-recording-session", (_, session: RecordingSession | null) => { - const normalized = normalizeRecordingSession(session); - setCurrentRecordingSessionState(normalized); - currentProjectPath = null; - return { success: true, session: normalized ?? undefined }; + } + + ipcMain.handle("set-current-video-path", (_, path: string) => { + return setCurrentVideoPath(path); }); - ipcMain.handle("get-current-recording-session", () => { - return currentRecordingSession - ? { success: true, session: currentRecordingSession } - : { success: false }; - }); - - ipcMain.handle("set-current-video-path", async (_, path: string) => { - const normalizedPath = normalizeVideoSourcePath(path); - if (!normalizedPath || !isPathAllowed(normalizedPath)) { - return { success: false, message: "Video path has not been approved" }; - } - - const restoredSession = await loadRecordedSessionForVideoPath(normalizedPath); - if (restoredSession) { - // Approve all media paths from the restored session so they can be read later - approveFilePath(restoredSession.screenVideoPath); - if (restoredSession.webcamVideoPath) { - approveFilePath(restoredSession.webcamVideoPath); - } - setCurrentRecordingSessionState(restoredSession); - } else { - setCurrentRecordingSessionState({ - screenVideoPath: normalizedPath, - createdAt: Date.now(), - }); - } + function setCurrentVideoPath(path: string): ProjectPathResult { + currentVideoPath = normalizeVideoSourcePath(path) ?? path; currentProjectPath = null; return { success: true }; - }); + } ipcMain.handle("get-current-video-path", () => { - return currentRecordingSession?.screenVideoPath - ? { success: true, path: currentRecordingSession.screenVideoPath } - : { success: false }; + return getCurrentVideoPathResult(); }); + function getCurrentVideoPathResult(): ProjectPathResult { + return currentVideoPath ? { success: true, path: currentVideoPath } : { success: false }; + } + ipcMain.handle("clear-current-video-path", () => { - setCurrentRecordingSessionState(null); - return { success: true }; + return clearCurrentVideoPath(); }); + function clearCurrentVideoPath(): ProjectPathResult { + currentVideoPath = null; + return { success: true }; + } + ipcMain.handle("get-platform", () => { return process.platform; }); diff --git a/electron/native-bridge/cursor/recording/factory.ts b/electron/native-bridge/cursor/recording/factory.ts new file mode 100644 index 0000000..fe92991 --- /dev/null +++ b/electron/native-bridge/cursor/recording/factory.ts @@ -0,0 +1,29 @@ +import type { Rectangle } from "electron"; +import type { CursorRecordingSession } from "./session"; +import { TelemetryRecordingSession } from "./telemetryRecordingSession"; +import { WindowsNativeRecordingSession } from "./windowsNativeRecordingSession"; + +interface CreateCursorRecordingSessionOptions { + getDisplayBounds: () => Rectangle | null; + maxSamples: number; + platform: NodeJS.Platform; + sampleIntervalMs: number; +} + +export function createCursorRecordingSession( + options: CreateCursorRecordingSessionOptions, +): CursorRecordingSession { + if (options.platform === "win32") { + return new WindowsNativeRecordingSession({ + getDisplayBounds: options.getDisplayBounds, + maxSamples: options.maxSamples, + sampleIntervalMs: options.sampleIntervalMs, + }); + } + + return new TelemetryRecordingSession({ + getDisplayBounds: options.getDisplayBounds, + maxSamples: options.maxSamples, + sampleIntervalMs: options.sampleIntervalMs, + }); +} diff --git a/electron/native-bridge/cursor/recording/session.ts b/electron/native-bridge/cursor/recording/session.ts new file mode 100644 index 0000000..9cebe9f --- /dev/null +++ b/electron/native-bridge/cursor/recording/session.ts @@ -0,0 +1,6 @@ +import type { CursorRecordingData } from "../../../../src/native/contracts"; + +export interface CursorRecordingSession { + start(): Promise; + stop(): Promise; +} diff --git a/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts b/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts new file mode 100644 index 0000000..dd42871 --- /dev/null +++ b/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts @@ -0,0 +1,62 @@ +import { type Rectangle, screen } from "electron"; +import type { CursorRecordingData, CursorRecordingSample } from "../../../../src/native/contracts"; +import type { CursorRecordingSession } from "./session"; + +interface TelemetryRecordingSessionOptions { + getDisplayBounds: () => Rectangle | null; + maxSamples: number; + sampleIntervalMs: number; +} + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)); +} + +export class TelemetryRecordingSession implements CursorRecordingSession { + private samples: CursorRecordingSample[] = []; + private interval: NodeJS.Timeout | null = null; + private startTimeMs = 0; + + constructor(private readonly options: TelemetryRecordingSessionOptions) {} + + async start(): Promise { + this.samples = []; + this.startTimeMs = Date.now(); + this.captureSample(); + this.interval = setInterval(() => { + this.captureSample(); + }, this.options.sampleIntervalMs); + } + + async stop(): Promise { + if (this.interval) { + clearInterval(this.interval); + this.interval = null; + } + + return { + version: 2, + provider: "none", + samples: this.samples, + assets: [], + }; + } + + private captureSample() { + const cursor = screen.getCursorScreenPoint(); + const display = this.options.getDisplayBounds() ?? screen.getDisplayNearestPoint(cursor).bounds; + const width = Math.max(1, display.width); + const height = Math.max(1, display.height); + + this.samples.push({ + timeMs: Math.max(0, Date.now() - this.startTimeMs), + cx: clamp((cursor.x - display.x) / width, 0, 1), + cy: clamp((cursor.y - display.y) / height, 0, 1), + visible: true, + }); + + if (this.samples.length > this.options.maxSamples) { + this.samples.shift(); + } + } +} diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts new file mode 100644 index 0000000..a0540ed --- /dev/null +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts @@ -0,0 +1,326 @@ +import { type ChildProcessByStdio, spawn } from "node:child_process"; +import type { Readable } from "node:stream"; +import { type Rectangle, screen } from "electron"; +import type { + CursorRecordingData, + CursorRecordingSample, + NativeCursorAsset, +} from "../../../../src/native/contracts"; +import type { CursorRecordingSession } from "./session"; + +interface WindowsCursorSampleEvent { + type: "sample"; + timestampMs: number; + x: number; + y: number; + visible: boolean; + handle: string | null; + asset?: WindowsCursorAssetPayload; +} + +interface WindowsCursorReadyEvent { + type: "ready"; + timestampMs: number; +} + +interface WindowsCursorErrorEvent { + type: "error"; + timestampMs: number; + message: string; +} + +interface WindowsCursorAssetPayload { + id: string; + imageDataUrl: string; + width: number; + height: number; + hotspotX: number; + hotspotY: number; +} + +type WindowsCursorEvent = + | WindowsCursorSampleEvent + | WindowsCursorReadyEvent + | WindowsCursorErrorEvent; + +interface WindowsNativeRecordingSessionOptions { + getDisplayBounds: () => Rectangle | null; + maxSamples: number; + sampleIntervalMs: number; +} + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)); +} + +function buildPowerShellCommand(sampleIntervalMs: number) { + const script = String.raw` +$ErrorActionPreference = 'Stop' +Add-Type -AssemblyName System.Drawing + +$source = @" +using System; +using System.Runtime.InteropServices; + +public static class OpenScreenCursorInterop { + [StructLayout(LayoutKind.Sequential)] + public struct POINT { + public int X; + public int Y; + } + + [StructLayout(LayoutKind.Sequential)] + public struct CURSORINFO { + public int cbSize; + public int flags; + public IntPtr hCursor; + public POINT ptScreenPos; + } + + [StructLayout(LayoutKind.Sequential)] + public struct ICONINFO { + [MarshalAs(UnmanagedType.Bool)] + public bool fIcon; + public int xHotspot; + public int yHotspot; + public IntPtr hbmMask; + public IntPtr hbmColor; + } + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool GetCursorInfo(ref CURSORINFO pci); + + [DllImport("user32.dll", SetLastError = true)] + public static extern IntPtr CopyIcon(IntPtr hIcon); + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool DestroyIcon(IntPtr hIcon); + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool GetIconInfo(IntPtr hIcon, out ICONINFO piconinfo); + + [DllImport("gdi32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool DeleteObject(IntPtr hObject); +} +"@ + +Add-Type -TypeDefinition $source + +function Write-JsonLine($payload) { + [Console]::Out.WriteLine(($payload | ConvertTo-Json -Compress -Depth 6)) +} + +function Get-CursorAsset($cursorHandle, $cursorId) { + $copiedHandle = [OpenScreenCursorInterop]::CopyIcon($cursorHandle) + if ($copiedHandle -eq [IntPtr]::Zero) { + return $null + } + + $iconInfo = New-Object OpenScreenCursorInterop+ICONINFO + $hasIconInfo = [OpenScreenCursorInterop]::GetIconInfo($copiedHandle, [ref]$iconInfo) + + try { + $icon = [System.Drawing.Icon]::FromHandle($copiedHandle) + $bitmap = New-Object System.Drawing.Bitmap $icon.Width, $icon.Height, ([System.Drawing.Imaging.PixelFormat]::Format32bppArgb) + $graphics = [System.Drawing.Graphics]::FromImage($bitmap) + $memoryStream = New-Object System.IO.MemoryStream + + try { + $graphics.Clear([System.Drawing.Color]::Transparent) + $graphics.DrawIcon($icon, 0, 0) + $bitmap.Save($memoryStream, [System.Drawing.Imaging.ImageFormat]::Png) + $base64 = [System.Convert]::ToBase64String($memoryStream.ToArray()) + + return @{ + id = $cursorId + imageDataUrl = "data:image/png;base64,$base64" + width = $bitmap.Width + height = $bitmap.Height + hotspotX = if ($hasIconInfo) { $iconInfo.xHotspot } else { 0 } + hotspotY = if ($hasIconInfo) { $iconInfo.yHotspot } else { 0 } + } + } + finally { + $memoryStream.Dispose() + $graphics.Dispose() + $bitmap.Dispose() + $icon.Dispose() + } + } + finally { + if ($hasIconInfo) { + if ($iconInfo.hbmMask -ne [IntPtr]::Zero) { + [OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmMask) | Out-Null + } + if ($iconInfo.hbmColor -ne [IntPtr]::Zero) { + [OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmColor) | Out-Null + } + } + [OpenScreenCursorInterop]::DestroyIcon($copiedHandle) | Out-Null + } +} + +Write-JsonLine @{ type = 'ready'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() } + +$lastCursorId = $null +while ($true) { + $cursorInfo = New-Object OpenScreenCursorInterop+CURSORINFO + $cursorInfo.cbSize = [Runtime.InteropServices.Marshal]::SizeOf([type][OpenScreenCursorInterop+CURSORINFO]) + + if (-not [OpenScreenCursorInterop]::GetCursorInfo([ref]$cursorInfo)) { + Write-JsonLine @{ type = 'error'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds(); message = 'GetCursorInfo failed' } + Start-Sleep -Milliseconds ${sampleIntervalMs} + continue + } + + $visible = ($cursorInfo.flags -band 1) -ne 0 + $cursorId = if ($cursorInfo.hCursor -eq [IntPtr]::Zero) { $null } else { ('0x{0:X}' -f $cursorInfo.hCursor.ToInt64()) } + $asset = $null + + if ($visible -and $cursorId -and $cursorId -ne $lastCursorId) { + $asset = Get-CursorAsset -cursorHandle $cursorInfo.hCursor -cursorId $cursorId + $lastCursorId = $cursorId + } + + Write-JsonLine @{ + type = 'sample' + timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + x = $cursorInfo.ptScreenPos.X + y = $cursorInfo.ptScreenPos.Y + visible = $visible + handle = $cursorId + asset = $asset + } + + Start-Sleep -Milliseconds ${sampleIntervalMs} +} +`; + + return Buffer.from(script, "utf16le").toString("base64"); +} + +export class WindowsNativeRecordingSession implements CursorRecordingSession { + private assets = new Map(); + private samples: CursorRecordingSample[] = []; + private process: ChildProcessByStdio | null = null; + private lineBuffer = ""; + private startTimeMs = 0; + + constructor(private readonly options: WindowsNativeRecordingSessionOptions) {} + + async start(): Promise { + this.assets.clear(); + this.samples = []; + this.lineBuffer = ""; + this.startTimeMs = Date.now(); + + const encodedCommand = buildPowerShellCommand(this.options.sampleIntervalMs); + const child = spawn( + "powershell.exe", + [ + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-EncodedCommand", + encodedCommand, + ], + { + stdio: ["ignore", "pipe", "pipe"], + windowsHide: true, + }, + ); + + this.process = child; + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + this.handleStdoutChunk(chunk); + }); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + console.error("[cursor-native]", chunk.trim()); + }); + } + + async stop(): Promise { + const child = this.process; + this.process = null; + + if (child && !child.killed) { + child.kill(); + } + + return { + version: 2, + provider: this.assets.size > 0 ? "native" : "none", + samples: this.samples, + assets: [...this.assets.values()], + }; + } + + private handleStdoutChunk(chunk: string) { + this.lineBuffer += chunk; + const lines = this.lineBuffer.split(/\r?\n/); + this.lineBuffer = lines.pop() ?? ""; + + for (const line of lines) { + const trimmedLine = line.trim(); + if (!trimmedLine) { + continue; + } + + try { + const payload = JSON.parse(trimmedLine) as WindowsCursorEvent; + this.handleEvent(payload); + } catch (error) { + console.error("Failed to parse Windows cursor helper output:", error, trimmedLine); + } + } + } + + private handleEvent(payload: WindowsCursorEvent) { + if (payload.type === "error") { + console.error("Windows cursor helper error:", payload.message); + return; + } + + if (payload.type === "ready") { + return; + } + + if (payload.asset?.id && !this.assets.has(payload.asset.id)) { + const assetDisplay = screen.getDisplayNearestPoint({ x: payload.x, y: payload.y }); + this.assets.set(payload.asset.id, { + id: payload.asset.id, + platform: "win32", + imageDataUrl: payload.asset.imageDataUrl, + width: payload.asset.width, + height: payload.asset.height, + hotspotX: payload.asset.hotspotX, + hotspotY: payload.asset.hotspotY, + scaleFactor: assetDisplay.scaleFactor, + }); + } + + const bounds = this.options.getDisplayBounds() ?? screen.getPrimaryDisplay().bounds; + const width = Math.max(1, bounds.width); + const height = Math.max(1, bounds.height); + + this.samples.push({ + timeMs: Math.max(0, payload.timestampMs - this.startTimeMs), + cx: clamp((payload.x - bounds.x) / width, 0, 1), + cy: clamp((payload.y - bounds.y) / height, 0, 1), + assetId: payload.handle, + visible: payload.visible, + }); + + if (this.samples.length > this.options.maxSamples) { + this.samples.shift(); + } + } +} diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index 2612b68..1ed8938 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -43,6 +43,7 @@ import { getNativeAspectRatioValue, isPortraitAspectRatio, } from "@/utils/aspectRatioUtils"; +import { nativeBridgeClient, useCursorRecordingData, useCursorTelemetry } from "@/native"; import { ExportDialog } from "./ExportDialog"; import PlaybackControls from "./PlaybackControls"; import { @@ -61,7 +62,6 @@ import TimelineEditor from "./timeline/TimelineEditor"; import { type AnnotationRegion, type BlurData, - type CursorTelemetryPoint, clampFocusToDepth, DEFAULT_ANNOTATION_POSITION, DEFAULT_ANNOTATION_SIZE, @@ -133,8 +133,6 @@ export default function VideoEditor() { currentTimeRef.current = currentTime; const durationRef = useRef(duration); durationRef.current = duration; - const [cursorTelemetry, setCursorTelemetry] = useState([]); - const [cursorClickTimestamps, setCursorClickTimestamps] = useState([]); const [selectedZoomId, setSelectedZoomId] = useState(null); const [selectedTrimId, setSelectedTrimId] = useState(null); const [selectedSpeedId, setSelectedSpeedId] = useState(null); @@ -220,6 +218,13 @@ export default function VideoEditor() { const project = candidate; const sourcePath = project.videoPath; const normalizedEditor = normalizeProjectEditor(project.editor); + const inferredDurationMs = Math.max( + 0, + ...normalizedEditor.zoomRegions.map((region) => region.endMs), + ...normalizedEditor.trimRegions.map((region) => region.endMs), + ...normalizedEditor.speedRegions.map((region) => region.endMs), + ...normalizedEditor.annotationRegions.map((region) => region.endMs), + ); try { videoPlaybackRef.current?.pause(); @@ -228,7 +233,7 @@ export default function VideoEditor() { } setIsPlaying(false); setCurrentTime(0); - setDuration(0); + setDuration(inferredDurationMs > 0 ? inferredDurationMs / 1000 : 0); setError(null); setVideoSourcePath(sourcePath); @@ -357,7 +362,7 @@ export default function VideoEditor() { useEffect(() => { async function loadInitialData() { try { - const currentProjectResult = await window.electronAPI.loadCurrentProjectFile(); + const currentProjectResult = await nativeBridgeClient.project.loadCurrentProjectFile(); if (currentProjectResult.success && currentProjectResult.project) { const restored = await applyLoadedProject( currentProjectResult.project, @@ -394,7 +399,7 @@ export default function VideoEditor() { return; } - const result = await window.electronAPI.getCurrentVideoPath(); + const result = await nativeBridgeClient.project.getCurrentVideoPath(); if (result.success && result.path) { setVideoSourcePath(result.path); setVideoPath(toFileUrl(result.path)); @@ -483,7 +488,7 @@ export default function VideoEditor() { // Match the normalization path used by `currentProjectSnapshot` so the // post-save baseline compares equal and `hasUnsavedChanges` clears. const projectSnapshot = createProjectSnapshot(currentProjectMedia, editorState); - const result = await window.electronAPI.saveProjectFile( + const result = await nativeBridgeClient.project.saveProjectFile( projectData, fileNameBase, forceSaveAs ? undefined : (currentProjectPath ?? undefined), @@ -589,7 +594,7 @@ export default function VideoEditor() { }, []); const handleLoadProject = useCallback(async () => { - const result = await window.electronAPI.loadProjectFile(); + const result = await nativeBridgeClient.project.loadProjectFile(); if (result.canceled) { return; @@ -622,40 +627,16 @@ export default function VideoEditor() { }, [handleLoadProject, handleSaveProject, handleSaveProjectAs]); useEffect(() => { - let mounted = true; - - async function loadCursorTelemetry() { - const sourcePath = currentProjectMedia?.screenVideoPath ?? null; - - if (!sourcePath) { - if (mounted) { - setCursorTelemetry([]); - setCursorClickTimestamps([]); - } - return; - } - - try { - const result = await window.electronAPI.getCursorTelemetry(sourcePath); - if (mounted) { - setCursorTelemetry(result.success ? result.samples : []); - setCursorClickTimestamps(result.success ? (result.clicks ?? []) : []); - } - } catch (telemetryError) { - console.warn("Unable to load cursor telemetry:", telemetryError); - if (mounted) { - setCursorTelemetry([]); - setCursorClickTimestamps([]); - } - } + if (cursorTelemetryError) { + console.warn("Unable to load cursor telemetry:", cursorTelemetryError); } + }, [cursorTelemetryError]); - loadCursorTelemetry(); - - return () => { - mounted = false; - }; - }, [currentProjectMedia]); + useEffect(() => { + if (cursorRecordingDataError) { + console.warn("Unable to load cursor recording data:", cursorRecordingDataError); + } + }, [cursorRecordingDataError]); function togglePlayPause() { const playback = videoPlaybackRef.current; @@ -1495,6 +1476,7 @@ export default function VideoEditor() { padding, videoPadding: padding, cropRegion, + cursorRecordingData, annotationRegions, webcamLayoutPreset, webcamMaskShape, @@ -1636,6 +1618,7 @@ export default function VideoEditor() { borderRadius, padding, cropRegion, + cursorRecordingData, annotationRegions, webcamLayoutPreset, webcamMaskShape, @@ -1715,6 +1698,7 @@ export default function VideoEditor() { borderRadius, padding, cropRegion, + cursorRecordingData, annotationRegions, isPlaying, aspectRatio, diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx index a053c3a..12b1c25 100644 --- a/src/components/video-editor/VideoPlayback.tsx +++ b/src/components/video-editor/VideoPlayback.tsx @@ -27,6 +27,12 @@ import { } from "@/lib/compositeLayout"; import { classifyWallpaper, DEFAULT_WALLPAPER, resolveImageWallpaperUrl } from "@/lib/wallpaper"; import { getCssClipPath } from "@/lib/webcamMaskShapes"; +import { + getNativeCursorDisplayMetrics, + projectNativeCursorToStage, + resolveActiveNativeCursorFrame, +} from "@/lib/cursor/nativeCursor"; +import type { CursorRecordingData } from "@/native/contracts"; import { type AspectRatio, formatAspectRatioForCSS, @@ -123,6 +129,7 @@ interface VideoPlaybackProps { trimRegions?: TrimRegion[]; speedRegions?: SpeedRegion[]; aspectRatio: AspectRatio; + cursorRecordingData?: CursorRecordingData | null; annotationRegions?: AnnotationRegion[]; selectedAnnotationId?: string | null; onSelectAnnotation?: (id: string | null) => void; @@ -155,6 +162,22 @@ export interface VideoPlaybackRef { pause: () => void; } +function getResolvedVideoDuration(video: HTMLVideoElement): number | null { + if (Number.isFinite(video.duration) && video.duration > 0) { + return video.duration; + } + + if (video.seekable.length > 0) { + const lastRangeIndex = video.seekable.length - 1; + const seekableEnd = video.seekable.end(lastRangeIndex); + if (Number.isFinite(seekableEnd) && seekableEnd > 0) { + return seekableEnd; + } + } + + return null; +} + const VideoPlayback = forwardRef( ( { @@ -188,6 +211,7 @@ const VideoPlayback = forwardRef( trimRegions = [], speedRegions = [], aspectRatio, + cursorRecordingData, annotationRegions = [], selectedAnnotationId, onSelectAnnotation, @@ -843,6 +867,8 @@ const VideoPlayback = forwardRef( useEffect(() => { if (!videoPath) { + lastResolvedDurationRef.current = null; + isResolvingDurationRef.current = false; setVideoReady(false); return; } @@ -853,11 +879,18 @@ const VideoPlayback = forwardRef( video.currentTime = 0; allowPlaybackRef.current = false; lockedVideoDimensionsRef.current = null; + lastResolvedDurationRef.current = null; + isResolvingDurationRef.current = false; + if (durationResolutionTimeoutRef.current) { + clearTimeout(durationResolutionTimeoutRef.current); + durationResolutionTimeoutRef.current = null; + } setVideoReady(false); if (videoReadyRafRef.current) { cancelAnimationFrame(videoReadyRafRef.current); videoReadyRafRef.current = null; } + video.load(); }, [videoPath]); useEffect(() => { @@ -1299,8 +1332,12 @@ const VideoPlayback = forwardRef( const handleLoadedMetadata = (e: React.SyntheticEvent) => { const video = e.currentTarget; - onDurationChange(video.duration); - video.currentTime = 0; + const hasResolvedDuration = syncResolvedDuration(video); + if (!hasResolvedDuration) { + forceResolveDuration(video); + } else { + video.currentTime = 0; + } video.pause(); allowPlaybackRef.current = false; currentTimeRef.current = 0; @@ -1313,6 +1350,9 @@ const VideoPlayback = forwardRef( const waitForRenderableFrame = () => { const hasDimensions = video.videoWidth > 0 && video.videoHeight > 0; const hasData = video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA; + if (!syncResolvedDuration(video)) { + forceResolveDuration(video); + } if (hasDimensions && hasData) { videoReadyRafRef.current = null; setVideoReady(true); @@ -1412,6 +1452,10 @@ const VideoPlayback = forwardRef( window.clearTimeout(scrubEndTimerRef.current); scrubEndTimerRef.current = null; } + if (durationResolutionTimeoutRef.current) { + clearTimeout(durationResolutionTimeoutRef.current); + durationResolutionTimeoutRef.current = null; + } }; }, []); @@ -1527,6 +1571,22 @@ const VideoPlayback = forwardRef( className="absolute rounded-md border border-[#34B27B]/80 bg-[#34B27B]/20 shadow-[0_0_0_1px_rgba(52,178,123,0.35)]" style={{ display: "none", pointerEvents: "none" }} /> + {activeNativeCursor && nativeCursorStyle ? ( + + ) : null} {(() => { const filteredAnnotations = (annotationRegions || []).filter((annotation) => { if ( @@ -1672,11 +1732,24 @@ const VideoPlayback = forwardRef( ref={videoRef} src={videoPath} className="hidden" - preload="metadata" + preload="auto" + muted playsInline onLoadedMetadata={handleLoadedMetadata} onDurationChange={(e) => { - onDurationChange(e.currentTarget.duration); + if (!syncResolvedDuration(e.currentTarget)) { + forceResolveDuration(e.currentTarget); + } + }} + onLoadedData={(e) => { + if (!syncResolvedDuration(e.currentTarget)) { + forceResolveDuration(e.currentTarget); + } + }} + onCanPlay={(e) => { + if (!syncResolvedDuration(e.currentTarget)) { + forceResolveDuration(e.currentTarget); + } }} onError={() => onError("Failed to load video")} /> diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx index feac886..20faf65 100644 --- a/src/components/video-editor/timeline/TimelineEditor.tsx +++ b/src/components/video-editor/timeline/TimelineEditor.tsx @@ -51,6 +51,7 @@ const SUGGESTION_SPACING_MS = 1800; interface TimelineEditorProps { videoDuration: number; + hasVideoSource?: boolean; currentTime: number; onSeek?: (time: number) => void; cursorTelemetry?: CursorTelemetryPoint[]; @@ -766,6 +767,7 @@ function Timeline({ export default function TimelineEditor({ videoDuration, + hasVideoSource = false, currentTime, onSeek, cursorTelemetry = [], @@ -1439,8 +1441,14 @@ export default function TimelineEditor({
-

{t("emptyState.noVideo")}

-

{t("emptyState.dragAndDrop")}

+

+ {hasVideoSource ? "Loading Timeline" : "No Video Loaded"} +

+

+ {hasVideoSource + ? "Video opened, waiting for duration metadata" + : "Drag and drop a video to start editing"} +

); diff --git a/src/lib/cursor/nativeCursor.ts b/src/lib/cursor/nativeCursor.ts new file mode 100644 index 0000000..23e187b --- /dev/null +++ b/src/lib/cursor/nativeCursor.ts @@ -0,0 +1,101 @@ +import { type Container, Point } from "pixi.js"; +import type { CropRegion } from "@/components/video-editor/types"; +import type { + CursorRecordingData, + CursorRecordingSample, + NativeCursorAsset, +} from "@/native/contracts"; + +export interface ActiveNativeCursorFrame { + asset: NativeCursorAsset; + sample: CursorRecordingSample; +} + +interface ProjectNativeCursorOptions { + cameraContainer: Container; + cropRegion: CropRegion; + maskRect: { width: number; height: number }; + videoContainerPosition: { x: number; y: number }; + sample: CursorRecordingSample; +} + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)); +} + +function getCroppedCursorPosition(sample: CursorRecordingSample, cropRegion: CropRegion) { + if (cropRegion.width <= 0 || cropRegion.height <= 0) { + return null; + } + + const croppedCx = (sample.cx - cropRegion.x) / cropRegion.width; + const croppedCy = (sample.cy - cropRegion.y) / cropRegion.height; + + if (croppedCx < 0 || croppedCx > 1 || croppedCy < 0 || croppedCy > 1) { + return null; + } + + return { + cx: clamp(croppedCx, 0, 1), + cy: clamp(croppedCy, 0, 1), + }; +} + +export function resolveActiveNativeCursorFrame( + recordingData: CursorRecordingData | null | undefined, + timeMs: number, +): ActiveNativeCursorFrame | null { + if (!recordingData || recordingData.provider !== "native" || recordingData.assets.length === 0) { + return null; + } + + for (let index = recordingData.samples.length - 1; index >= 0; index -= 1) { + const sample = recordingData.samples[index]; + if (sample.timeMs > timeMs) { + continue; + } + + if (sample.visible === false || !sample.assetId) { + return null; + } + + const asset = recordingData.assets.find((candidate) => candidate.id === sample.assetId); + if (!asset) { + return null; + } + + return { sample, asset }; + } + + return null; +} + +export function projectNativeCursorToStage({ + cameraContainer, + cropRegion, + maskRect, + videoContainerPosition, + sample, +}: ProjectNativeCursorOptions) { + const croppedPosition = getCroppedCursorPosition(sample, cropRegion); + if (!croppedPosition) { + return null; + } + + const localPoint = new Point( + videoContainerPosition.x + croppedPosition.cx * maskRect.width, + videoContainerPosition.y + croppedPosition.cy * maskRect.height, + ); + + return cameraContainer.toGlobal(localPoint); +} + +export function getNativeCursorDisplayMetrics(asset: NativeCursorAsset, deviceScaleFactor: number) { + const scaleFactor = asset.scaleFactor ?? deviceScaleFactor ?? 1; + return { + width: asset.width / scaleFactor, + height: asset.height / scaleFactor, + hotspotX: asset.hotspotX / scaleFactor, + hotspotY: asset.hotspotY / scaleFactor, + }; +} diff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.ts index 017af83..f13735d 100644 --- a/src/lib/exporter/frameRenderer.ts +++ b/src/lib/exporter/frameRenderer.ts @@ -56,8 +56,14 @@ import { type Size, type StyledRenderRect, } from "@/lib/compositeLayout"; +import { + getNativeCursorDisplayMetrics, + projectNativeCursorToStage, + resolveActiveNativeCursorFrame, +} from "@/lib/cursor/nativeCursor"; import { BackgroundLoadError, classifyWallpaper, resolveImageWallpaperUrl } from "@/lib/wallpaper"; import { drawCanvasClipPath } from "@/lib/webcamMaskShapes"; +import type { CursorRecordingData, NativeCursorAsset } from "@/native/contracts"; import { renderAnnotations } from "./annotationRenderer"; import { getLinearGradientPoints, @@ -79,6 +85,7 @@ interface FrameRenderConfig { borderRadius?: number; padding?: number; cropRegion: CropRegion; + cursorRecordingData?: CursorRecordingData | null; videoWidth: number; videoHeight: number; webcamSize?: Size | null; @@ -136,6 +143,7 @@ export class FrameRenderer { private rasterCtx: CanvasRenderingContext2D | null = null; private threeDPass: ThreeDPass | null = null; private currentRotation3D: Rotation3D = { ...DEFAULT_ROTATION_3D }; + private cursorImageCache = new Map(); private config: FrameRenderConfig; private animationState: AnimationState; private layoutCache: LayoutCache | null = null; @@ -468,6 +476,8 @@ export class FrameRenderer { } } + await this.drawNativeCursor(timeMs); + // Render annotations on top of foreground (so they rotate with recording). if ( this.config.annotationRegions && @@ -543,7 +553,63 @@ export class FrameRenderer { } } - private updateLayout(webcamFrame?: VideoFrame | null): void { + private async drawNativeCursor(timeMs: number) { + if (!this.compositeCtx || !this.cameraContainer || !this.videoContainer || !this.layoutCache) { + return; + } + + const activeNativeCursor = resolveActiveNativeCursorFrame( + this.config.cursorRecordingData, + timeMs, + ); + if (!activeNativeCursor) { + return; + } + + const projectedPoint = projectNativeCursorToStage({ + cameraContainer: this.cameraContainer, + cropRegion: this.config.cropRegion, + maskRect: this.layoutCache.maskRect, + videoContainerPosition: { + x: this.videoContainer.x, + y: this.videoContainer.y, + }, + sample: activeNativeCursor.sample, + }); + if (!projectedPoint) { + return; + } + + const image = await this.getCursorImage(activeNativeCursor.asset); + const metrics = getNativeCursorDisplayMetrics(activeNativeCursor.asset, 1); + + this.compositeCtx.drawImage( + image, + projectedPoint.x - metrics.hotspotX, + projectedPoint.y - metrics.hotspotY, + metrics.width, + metrics.height, + ); + } + + private async getCursorImage(asset: NativeCursorAsset) { + const cachedImage = this.cursorImageCache.get(asset.id); + if (cachedImage) { + return cachedImage; + } + + const image = new Image(); + await new Promise((resolve, reject) => { + image.onload = () => resolve(); + image.onerror = () => reject(new Error(`Failed to load cursor asset ${asset.id}`)); + image.src = asset.imageDataUrl; + }); + + this.cursorImageCache.set(asset.id, image); + return image; + } + + private updateLayout(): void { if (!this.app || !this.videoSprite || !this.maskGraphics || !this.videoContainer) return; const { width, height } = this.config; @@ -999,5 +1065,6 @@ export class FrameRenderer { this.threeDPass.destroy(); this.threeDPass = null; } + this.cursorImageCache.clear(); } } diff --git a/src/lib/exporter/gifExporter.ts b/src/lib/exporter/gifExporter.ts index 0d7a432..02564db 100644 --- a/src/lib/exporter/gifExporter.ts +++ b/src/lib/exporter/gifExporter.ts @@ -11,6 +11,7 @@ import type { import { BackgroundLoadError } from "@/lib/wallpaper"; import { getPlatform } from "@/utils/platformUtils"; import { AsyncVideoFrameQueue } from "./asyncVideoFrameQueue"; +import type { CursorRecordingData } from "@/native/contracts"; import { FrameRenderer } from "./frameRenderer"; import { StreamingVideoDecoder } from "./streamingDecoder"; import type { @@ -47,6 +48,7 @@ interface GifExporterConfig { webcamMaskShape?: import("@/components/video-editor/types").WebcamMaskShape; webcamSizePreset?: WebcamSizePreset; webcamPosition?: { cx: number; cy: number } | null; + cursorRecordingData?: CursorRecordingData | null; annotationRegions?: AnnotationRegion[]; previewWidth?: number; previewHeight?: number; @@ -151,6 +153,7 @@ export class GifExporter { borderRadius: this.config.borderRadius, padding: this.config.padding, cropRegion: this.config.cropRegion, + cursorRecordingData: this.config.cursorRecordingData, videoWidth: videoInfo.width, videoHeight: videoInfo.height, webcamSize: webcamInfo ? { width: webcamInfo.width, height: webcamInfo.height } : null, diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index e064ba7..edddd05 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -10,6 +10,7 @@ import type { import { BackgroundLoadError } from "@/lib/wallpaper"; import { getPlatform } from "@/utils/platformUtils"; import { AsyncVideoFrameQueue } from "./asyncVideoFrameQueue"; +import type { CursorRecordingData } from "@/native/contracts"; import { AudioProcessor } from "./audioEncoder"; import { FrameRenderer } from "./frameRenderer"; import { VideoMuxer } from "./muxer"; @@ -38,6 +39,7 @@ interface VideoExporterConfig extends ExportConfig { webcamMaskShape?: import("@/components/video-editor/types").WebcamMaskShape; webcamSizePreset?: WebcamSizePreset; webcamPosition?: { cx: number; cy: number } | null; + cursorRecordingData?: CursorRecordingData | null; annotationRegions?: AnnotationRegion[]; previewWidth?: number; previewHeight?: number; @@ -146,6 +148,7 @@ export class VideoExporter { borderRadius: this.config.borderRadius, padding: this.config.padding, cropRegion: this.config.cropRegion, + cursorRecordingData: this.config.cursorRecordingData, videoWidth: videoInfo.width, videoHeight: videoInfo.height, webcamSize: webcamInfo ? { width: webcamInfo.width, height: webcamInfo.height } : null,