From e9650225bade7f4f2eca86f29c5f5f03145492df Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Thu, 26 Mar 2026 11:16:41 +0100 Subject: [PATCH] feat: add cursor overlay pipeline for high-fidelity cursor recording and playback - Implement native bridge for Windows cursor capture via PowerShell/C# - Add cursor-free capture using getDisplayMedia with setDisplayMediaRequestHandler - Update video player and exporters to support native cursor telemetry - Enable system audio capture on Windows via WASAPI loopback - Add interpolation for smoother cursor movement in playback and export - Improve cursor scaling and visibility handling in editor and playback --- electron/ipc/handlers.ts | 123 ++++-- electron/main.ts | 2 +- .../native-bridge/cursor/recording/factory.ts | 2 + .../windowsNativeRecordingSession.script.ts | 216 +++++++++++ .../windowsNativeRecordingSession.ts | 352 ++++++++---------- .../windowsNativeRecordingSession.types.ts | 49 +++ src/components/launch/LaunchWindow.tsx | 5 + src/components/video-editor/VideoEditor.tsx | 2 + src/components/video-editor/VideoPlayback.tsx | 81 +++- src/hooks/useScreenRecorder.ts | 50 +-- src/lib/cursor/nativeCursor.ts | 78 +++- src/lib/exporter/frameRenderer.ts | 19 +- src/lib/exporter/gifExporter.ts | 2 + src/lib/exporter/videoExporter.ts | 2 + 14 files changed, 686 insertions(+), 297 deletions(-) create mode 100644 electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts create mode 100644 electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 1d73a9b..d50cab3 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -4,8 +4,6 @@ import os from "node:os"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; -const nodeRequire = createRequire(import.meta.url); - import { app, BrowserWindow, @@ -16,10 +14,7 @@ import { shell, systemPreferences, } from "electron"; -import { - type CursorTelemetryPoint, - createCursorTelemetryBuffer, -} from "../../src/lib/cursorTelemetryBuffer"; +import type { DesktopCapturerSource } from "electron"; import { normalizeProjectMedia, normalizeRecordingSession, @@ -198,11 +193,24 @@ async function getApprovedProjectSession( type SelectedSource = { name: string; + id?: string; + display_id?: string; [key: string]: unknown; }; let selectedSource: SelectedSource | null = null; +let selectedDesktopSource: DesktopCapturerSource | null = null; +let lastEnumeratedSources = new Map(); let currentProjectPath: string | null = null; +let currentRecordingSession: RecordingSession | null = null; + +/** + * Returns the cached DesktopCapturerSource set when the user picked a source. + * Used by setDisplayMediaRequestHandler in main.ts for cursor-free capture. + */ +export function getSelectedDesktopSource(): DesktopCapturerSource | null { + return selectedDesktopSource; +} let currentVideoPath: string | null = null; function normalizePath(filePath: string) { @@ -238,16 +246,12 @@ function isTrustedProjectPath(filePath?: string | null) { } const CURSOR_TELEMETRY_VERSION = 2; -const CURSOR_SAMPLE_INTERVAL_MS = 100; -const MAX_CURSOR_SAMPLES = 60 * 60 * 10; // 1 hour @ 10Hz +const CURSOR_SAMPLE_INTERVAL_MS = 33; +const MAX_CURSOR_SAMPLES = 60 * 60 * 30; // 1 hour @ 30Hz let cursorRecordingSession: CursorRecordingSession | null = null; let pendingCursorRecordingData: CursorRecordingData | null = null; -function clamp(value: number, min: number, max: number) { - return Math.min(max, Math.max(min, value)); -} - function normalizeCursorSample(sample: unknown): CursorRecordingSample | null { if (!sample || typeof sample !== "object") { return null; @@ -259,8 +263,8 @@ function normalizeCursorSample(sample: unknown): CursorRecordingSample | null { typeof point.timeMs === "number" && Number.isFinite(point.timeMs) ? Math.max(0, point.timeMs) : 0, - cx: typeof point.cx === "number" && Number.isFinite(point.cx) ? clamp(point.cx, 0, 1) : 0.5, - cy: typeof point.cy === "number" && Number.isFinite(point.cy) ? clamp(point.cy, 0, 1) : 0.5, + cx: typeof point.cx === "number" && Number.isFinite(point.cx) ? point.cx : 0.5, + cy: typeof point.cy === "number" && Number.isFinite(point.cy) ? point.cy : 0.5, assetId: typeof point.assetId === "string" ? point.assetId : null, visible: typeof point.visible === "boolean" ? point.visible : true, }; @@ -395,6 +399,55 @@ function getSelectedSourceBounds() { return (sourceDisplay ?? screen.getDisplayNearestPoint(cursor)).bounds; } +function getSelectedSourceId() { + return typeof selectedSource?.id === "string" ? selectedSource.id : null; +} + +function setCurrentRecordingSessionState(session: RecordingSession | null) { + currentRecordingSession = session; + currentVideoPath = session?.screenVideoPath ?? null; +} + +async function storeRecordedSessionFiles(payload: StoreRecordedSessionInput) { + const createdAt = + typeof payload.createdAt === "number" && Number.isFinite(payload.createdAt) + ? payload.createdAt + : Date.now(); + const screenVideoPath = resolveRecordingOutputPath(payload.screen.fileName); + await fs.writeFile(screenVideoPath, Buffer.from(payload.screen.videoData)); + + let webcamVideoPath: string | undefined; + if (payload.webcam) { + webcamVideoPath = resolveRecordingOutputPath(payload.webcam.fileName); + await fs.writeFile(webcamVideoPath, Buffer.from(payload.webcam.videoData)); + } + + const session: RecordingSession = webcamVideoPath + ? { screenVideoPath, webcamVideoPath, createdAt } + : { screenVideoPath, createdAt }; + setCurrentRecordingSessionState(session); + currentProjectPath = null; + + const telemetryPath = `${screenVideoPath}.cursor.json`; + if (pendingCursorRecordingData && pendingCursorRecordingData.samples.length > 0) { + await fs.writeFile(telemetryPath, JSON.stringify(pendingCursorRecordingData, null, 2), "utf-8"); + } + pendingCursorRecordingData = null; + + const sessionManifestPath = path.join( + RECORDINGS_DIR, + `${path.parse(payload.screen.fileName).name}${RECORDING_SESSION_SUFFIX}`, + ); + await fs.writeFile(sessionManifestPath, JSON.stringify(session, null, 2), "utf-8"); + + return { + success: true, + path: screenVideoPath, + session, + message: "Recording session stored successfully", + }; +} + export function registerIpcHandlers( createEditorWindow: () => void, createSourceSelectorWindow: () => BrowserWindow, @@ -404,6 +457,7 @@ export function registerIpcHandlers( ) { ipcMain.handle("get-sources", async (_, opts) => { const sources = await desktopCapturer.getSources(opts); + lastEnumeratedSources = new Map(sources.map((source) => [source.id, source])); return sources.map((source) => ({ id: source.id, name: source.name, @@ -413,8 +467,26 @@ export function registerIpcHandlers( })); }); - ipcMain.handle("select-source", (_, source: SelectedSource) => { + ipcMain.handle("select-source", async (_, source: SelectedSource) => { selectedSource = source; + // Reuse the exact source object returned during enumeration to avoid + // Windows window-source id mismatches across separate getSources() calls. + selectedDesktopSource = + typeof source.id === "string" ? lastEnumeratedSources.get(source.id) ?? null : null; + + if (!selectedDesktopSource && typeof source.id === "string") { + try { + const sources = await desktopCapturer.getSources({ + types: ["screen", "window"], + thumbnailSize: { width: 0, height: 0 }, + fetchWindowIcons: true, + }); + lastEnumeratedSources = new Map(sources.map((candidate) => [candidate.id, candidate])); + selectedDesktopSource = lastEnumeratedSources.get(source.id) ?? null; + } catch { + selectedDesktopSource = null; + } + } const sourceSelectorWin = getSourceSelectorWindow(); if (sourceSelectorWin) { sourceSelectorWin.close(); @@ -519,25 +591,7 @@ export function registerIpcHandlers( ipcMain.handle("store-recorded-session", async (_, payload: StoreRecordedSessionInput) => { try { - const videoPath = path.join(RECORDINGS_DIR, fileName); - await fs.writeFile(videoPath, Buffer.from(videoData)); - currentProjectPath = null; - - const telemetryPath = `${videoPath}.cursor.json`; - if (pendingCursorRecordingData && pendingCursorRecordingData.samples.length > 0) { - await fs.writeFile( - telemetryPath, - JSON.stringify(pendingCursorRecordingData, null, 2), - "utf-8", - ); - } - pendingCursorRecordingData = null; - - return { - success: true, - path: videoPath, - message: "Video stored successfully", - }; + return await storeRecordedSessionFiles(payload); } catch (error) { console.error("Failed to store recording session:", error); return { @@ -602,6 +656,7 @@ export function registerIpcHandlers( maxSamples: MAX_CURSOR_SAMPLES, platform: process.platform, sampleIntervalMs: CURSOR_SAMPLE_INTERVAL_MS, + sourceId: getSelectedSourceId(), }); try { diff --git a/electron/main.ts b/electron/main.ts index 4e443a9..1db4740 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -13,7 +13,7 @@ import { Tray, } from "electron"; import { mainT, setMainLocale } from "./i18n"; -import { registerIpcHandlers } from "./ipc/handlers"; +import { getSelectedDesktopSource, registerIpcHandlers } from "./ipc/handlers"; import { createCountdownOverlayWindow, createEditorWindow, diff --git a/electron/native-bridge/cursor/recording/factory.ts b/electron/native-bridge/cursor/recording/factory.ts index fe92991..4e0f75c 100644 --- a/electron/native-bridge/cursor/recording/factory.ts +++ b/electron/native-bridge/cursor/recording/factory.ts @@ -8,6 +8,7 @@ interface CreateCursorRecordingSessionOptions { maxSamples: number; platform: NodeJS.Platform; sampleIntervalMs: number; + sourceId?: string | null; } export function createCursorRecordingSession( @@ -18,6 +19,7 @@ export function createCursorRecordingSession( getDisplayBounds: options.getDisplayBounds, maxSamples: options.maxSamples, sampleIntervalMs: options.sampleIntervalMs, + sourceId: options.sourceId, }); } diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts new file mode 100644 index 0000000..b7a11cb --- /dev/null +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts @@ -0,0 +1,216 @@ +export function parseWindowHandleFromSourceId(sourceId?: string | null) { + if (!sourceId?.startsWith("window:")) { + return null; + } + + const handlePart = sourceId.split(":")[1]; + if (!handlePart || !/^\d+$/.test(handlePart)) { + return null; + } + + return handlePart; +} + +export function buildPowerShellCommand(sampleIntervalMs: number, windowHandle?: string | null) { + const script = String.raw` +$ErrorActionPreference = 'Stop' +Add-Type -AssemblyName System.Drawing + +$targetWindowHandle = ${windowHandle ? `'${windowHandle}'` : '$null'} + +$source = @" +using System; +using System.Runtime.InteropServices; + +public static class OpenScreenCursorInterop { + [StructLayout(LayoutKind.Sequential)] + public struct POINT { + public int X; + public int Y; + } + + [StructLayout(LayoutKind.Sequential)] + public struct CURSORINFO { + public int cbSize; + public int flags; + public IntPtr hCursor; + public POINT ptScreenPos; + } + + [StructLayout(LayoutKind.Sequential)] + public struct ICONINFO { + [MarshalAs(UnmanagedType.Bool)] + public bool fIcon; + public int xHotspot; + public int yHotspot; + public IntPtr hbmMask; + public IntPtr hbmColor; + } + + [StructLayout(LayoutKind.Sequential)] + public struct RECT { + public int Left; + public int Top; + public int Right; + public int Bottom; + } + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool GetCursorInfo(ref CURSORINFO pci); + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool GetWindowRect(IntPtr hWnd, out RECT lpRect); + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool IsWindow(IntPtr hWnd); + + [DllImport("user32.dll", SetLastError = true)] + public static extern IntPtr CopyIcon(IntPtr hIcon); + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool DestroyIcon(IntPtr hIcon); + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool GetIconInfo(IntPtr hIcon, out ICONINFO piconinfo); + + [DllImport("gdi32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool DeleteObject(IntPtr hObject); +} +"@ + +Add-Type -TypeDefinition $source + +function Write-JsonLine($payload) { + [Console]::Out.WriteLine(($payload | ConvertTo-Json -Compress -Depth 6)) +} + +function Get-TargetBounds() { + if ([string]::IsNullOrWhiteSpace($targetWindowHandle)) { + return $null + } + + try { + $handleValue = [int64]::Parse($targetWindowHandle) + $windowHandle = [IntPtr]::new($handleValue) + if (-not [OpenScreenCursorInterop]::IsWindow($windowHandle)) { + return $null + } + + $rect = New-Object OpenScreenCursorInterop+RECT + if (-not [OpenScreenCursorInterop]::GetWindowRect($windowHandle, [ref]$rect)) { + return $null + } + + $width = $rect.Right - $rect.Left + $height = $rect.Bottom - $rect.Top + if ($width -le 0 -or $height -le 0) { + return $null + } + + return @{ + x = $rect.Left + y = $rect.Top + width = $width + height = $height + } + } + catch { + return $null + } +} + +function Get-CursorAsset($cursorHandle, $cursorId) { + $copiedHandle = [OpenScreenCursorInterop]::CopyIcon($cursorHandle) + if ($copiedHandle -eq [IntPtr]::Zero) { + return $null + } + + $iconInfo = New-Object OpenScreenCursorInterop+ICONINFO + $hasIconInfo = [OpenScreenCursorInterop]::GetIconInfo($copiedHandle, [ref]$iconInfo) + + try { + $icon = [System.Drawing.Icon]::FromHandle($copiedHandle) + $bitmap = New-Object System.Drawing.Bitmap $icon.Width, $icon.Height, ([System.Drawing.Imaging.PixelFormat]::Format32bppArgb) + $graphics = [System.Drawing.Graphics]::FromImage($bitmap) + $memoryStream = New-Object System.IO.MemoryStream + + try { + $graphics.Clear([System.Drawing.Color]::Transparent) + $graphics.DrawIcon($icon, 0, 0) + $bitmap.Save($memoryStream, [System.Drawing.Imaging.ImageFormat]::Png) + $base64 = [System.Convert]::ToBase64String($memoryStream.ToArray()) + + return @{ + id = $cursorId + imageDataUrl = "data:image/png;base64,$base64" + width = $bitmap.Width + height = $bitmap.Height + hotspotX = if ($hasIconInfo) { $iconInfo.xHotspot } else { 0 } + hotspotY = if ($hasIconInfo) { $iconInfo.yHotspot } else { 0 } + } + } + finally { + $memoryStream.Dispose() + $graphics.Dispose() + $bitmap.Dispose() + $icon.Dispose() + } + } + finally { + if ($hasIconInfo) { + if ($iconInfo.hbmMask -ne [IntPtr]::Zero) { + [OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmMask) | Out-Null + } + if ($iconInfo.hbmColor -ne [IntPtr]::Zero) { + [OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmColor) | Out-Null + } + } + [OpenScreenCursorInterop]::DestroyIcon($copiedHandle) | Out-Null + } +} + +Write-JsonLine @{ type = 'ready'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() } + +$lastCursorId = $null +while ($true) { + $cursorInfo = New-Object OpenScreenCursorInterop+CURSORINFO + $cursorInfo.cbSize = [Runtime.InteropServices.Marshal]::SizeOf([type][OpenScreenCursorInterop+CURSORINFO]) + + if (-not [OpenScreenCursorInterop]::GetCursorInfo([ref]$cursorInfo)) { + Write-JsonLine @{ type = 'error'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds(); message = 'GetCursorInfo failed' } + Start-Sleep -Milliseconds ${sampleIntervalMs} + continue + } + + $visible = ($cursorInfo.flags -band 1) -ne 0 + $cursorId = if ($cursorInfo.hCursor -eq [IntPtr]::Zero) { $null } else { ('0x{0:X}' -f $cursorInfo.hCursor.ToInt64()) } + $asset = $null + + if ($visible -and $cursorId -and $cursorId -ne $lastCursorId) { + $asset = Get-CursorAsset -cursorHandle $cursorInfo.hCursor -cursorId $cursorId + $lastCursorId = $cursorId + } + + Write-JsonLine @{ + type = 'sample' + timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + x = $cursorInfo.ptScreenPos.X + y = $cursorInfo.ptScreenPos.Y + visible = $visible + handle = $cursorId + bounds = Get-TargetBounds + asset = $asset + } + + Start-Sleep -Milliseconds ${sampleIntervalMs} +} +`; + + return Buffer.from(script, "utf16le").toString("base64"); +} diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts index a0540ed..d5e43d7 100644 --- a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts @@ -1,206 +1,23 @@ import { type ChildProcessByStdio, spawn } from "node:child_process"; import type { Readable } from "node:stream"; -import { type Rectangle, screen } from "electron"; +import { screen } from "electron"; import type { CursorRecordingData, CursorRecordingSample, NativeCursorAsset, } from "../../../../src/native/contracts"; import type { CursorRecordingSession } from "./session"; +import { buildPowerShellCommand, parseWindowHandleFromSourceId } from "./windowsNativeRecordingSession.script"; +import type { + WindowsCursorEvent, + WindowsNativeRecordingSessionOptions, +} from "./windowsNativeRecordingSession.types"; -interface WindowsCursorSampleEvent { - type: "sample"; - timestampMs: number; - x: number; - y: number; - visible: boolean; - handle: string | null; - asset?: WindowsCursorAssetPayload; -} +const READY_TIMEOUT_MS = 5_000; -interface WindowsCursorReadyEvent { - type: "ready"; - timestampMs: number; -} - -interface WindowsCursorErrorEvent { - type: "error"; - timestampMs: number; - message: string; -} - -interface WindowsCursorAssetPayload { - id: string; - imageDataUrl: string; - width: number; - height: number; - hotspotX: number; - hotspotY: number; -} - -type WindowsCursorEvent = - | WindowsCursorSampleEvent - | WindowsCursorReadyEvent - | WindowsCursorErrorEvent; - -interface WindowsNativeRecordingSessionOptions { - getDisplayBounds: () => Rectangle | null; - maxSamples: number; - sampleIntervalMs: number; -} - -function clamp(value: number, min: number, max: number) { - return Math.min(max, Math.max(min, value)); -} - -function buildPowerShellCommand(sampleIntervalMs: number) { - const script = String.raw` -$ErrorActionPreference = 'Stop' -Add-Type -AssemblyName System.Drawing - -$source = @" -using System; -using System.Runtime.InteropServices; - -public static class OpenScreenCursorInterop { - [StructLayout(LayoutKind.Sequential)] - public struct POINT { - public int X; - public int Y; - } - - [StructLayout(LayoutKind.Sequential)] - public struct CURSORINFO { - public int cbSize; - public int flags; - public IntPtr hCursor; - public POINT ptScreenPos; - } - - [StructLayout(LayoutKind.Sequential)] - public struct ICONINFO { - [MarshalAs(UnmanagedType.Bool)] - public bool fIcon; - public int xHotspot; - public int yHotspot; - public IntPtr hbmMask; - public IntPtr hbmColor; - } - - [DllImport("user32.dll", SetLastError = true)] - [return: MarshalAs(UnmanagedType.Bool)] - public static extern bool GetCursorInfo(ref CURSORINFO pci); - - [DllImport("user32.dll", SetLastError = true)] - public static extern IntPtr CopyIcon(IntPtr hIcon); - - [DllImport("user32.dll", SetLastError = true)] - [return: MarshalAs(UnmanagedType.Bool)] - public static extern bool DestroyIcon(IntPtr hIcon); - - [DllImport("user32.dll", SetLastError = true)] - [return: MarshalAs(UnmanagedType.Bool)] - public static extern bool GetIconInfo(IntPtr hIcon, out ICONINFO piconinfo); - - [DllImport("gdi32.dll", SetLastError = true)] - [return: MarshalAs(UnmanagedType.Bool)] - public static extern bool DeleteObject(IntPtr hObject); -} -"@ - -Add-Type -TypeDefinition $source - -function Write-JsonLine($payload) { - [Console]::Out.WriteLine(($payload | ConvertTo-Json -Compress -Depth 6)) -} - -function Get-CursorAsset($cursorHandle, $cursorId) { - $copiedHandle = [OpenScreenCursorInterop]::CopyIcon($cursorHandle) - if ($copiedHandle -eq [IntPtr]::Zero) { - return $null - } - - $iconInfo = New-Object OpenScreenCursorInterop+ICONINFO - $hasIconInfo = [OpenScreenCursorInterop]::GetIconInfo($copiedHandle, [ref]$iconInfo) - - try { - $icon = [System.Drawing.Icon]::FromHandle($copiedHandle) - $bitmap = New-Object System.Drawing.Bitmap $icon.Width, $icon.Height, ([System.Drawing.Imaging.PixelFormat]::Format32bppArgb) - $graphics = [System.Drawing.Graphics]::FromImage($bitmap) - $memoryStream = New-Object System.IO.MemoryStream - - try { - $graphics.Clear([System.Drawing.Color]::Transparent) - $graphics.DrawIcon($icon, 0, 0) - $bitmap.Save($memoryStream, [System.Drawing.Imaging.ImageFormat]::Png) - $base64 = [System.Convert]::ToBase64String($memoryStream.ToArray()) - - return @{ - id = $cursorId - imageDataUrl = "data:image/png;base64,$base64" - width = $bitmap.Width - height = $bitmap.Height - hotspotX = if ($hasIconInfo) { $iconInfo.xHotspot } else { 0 } - hotspotY = if ($hasIconInfo) { $iconInfo.yHotspot } else { 0 } - } - } - finally { - $memoryStream.Dispose() - $graphics.Dispose() - $bitmap.Dispose() - $icon.Dispose() - } - } - finally { - if ($hasIconInfo) { - if ($iconInfo.hbmMask -ne [IntPtr]::Zero) { - [OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmMask) | Out-Null - } - if ($iconInfo.hbmColor -ne [IntPtr]::Zero) { - [OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmColor) | Out-Null - } - } - [OpenScreenCursorInterop]::DestroyIcon($copiedHandle) | Out-Null - } -} - -Write-JsonLine @{ type = 'ready'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() } - -$lastCursorId = $null -while ($true) { - $cursorInfo = New-Object OpenScreenCursorInterop+CURSORINFO - $cursorInfo.cbSize = [Runtime.InteropServices.Marshal]::SizeOf([type][OpenScreenCursorInterop+CURSORINFO]) - - if (-not [OpenScreenCursorInterop]::GetCursorInfo([ref]$cursorInfo)) { - Write-JsonLine @{ type = 'error'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds(); message = 'GetCursorInfo failed' } - Start-Sleep -Milliseconds ${sampleIntervalMs} - continue - } - - $visible = ($cursorInfo.flags -band 1) -ne 0 - $cursorId = if ($cursorInfo.hCursor -eq [IntPtr]::Zero) { $null } else { ('0x{0:X}' -f $cursorInfo.hCursor.ToInt64()) } - $asset = $null - - if ($visible -and $cursorId -and $cursorId -ne $lastCursorId) { - $asset = Get-CursorAsset -cursorHandle $cursorInfo.hCursor -cursorId $cursorId - $lastCursorId = $cursorId - } - - Write-JsonLine @{ - type = 'sample' - timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() - x = $cursorInfo.ptScreenPos.X - y = $cursorInfo.ptScreenPos.Y - visible = $visible - handle = $cursorId - asset = $asset - } - - Start-Sleep -Milliseconds ${sampleIntervalMs} -} -`; - - return Buffer.from(script, "utf16le").toString("base64"); +interface NormalizedSample { + sample: CursorRecordingSample; + withinBounds: boolean; } export class WindowsNativeRecordingSession implements CursorRecordingSession { @@ -209,6 +26,11 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession { private process: ChildProcessByStdio | null = null; private lineBuffer = ""; private startTimeMs = 0; + private readyResolve: (() => void) | null = null; + private readyReject: ((error: Error) => void) | null = null; + private readyTimer: NodeJS.Timeout | null = null; + private sampleCount = 0; + private outOfBoundsSampleCount = 0; constructor(private readonly options: WindowsNativeRecordingSessionOptions) {} @@ -217,8 +39,13 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession { this.samples = []; this.lineBuffer = ""; this.startTimeMs = Date.now(); + this.sampleCount = 0; + this.outOfBoundsSampleCount = 0; - const encodedCommand = buildPowerShellCommand(this.options.sampleIntervalMs); + const encodedCommand = buildPowerShellCommand( + this.options.sampleIntervalMs, + parseWindowHandleFromSourceId(this.options.sourceId), + ); const child = spawn( "powershell.exe", [ @@ -237,24 +64,58 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession { ); this.process = child; + this.logDiagnostic("spawn", { + pid: child.pid ?? null, + sampleIntervalMs: this.options.sampleIntervalMs, + sourceId: this.options.sourceId ?? null, + windowHandle: parseWindowHandleFromSourceId(this.options.sourceId), + }); + child.stdout.setEncoding("utf8"); child.stdout.on("data", (chunk: string) => { this.handleStdoutChunk(chunk); }); child.stderr.setEncoding("utf8"); child.stderr.on("data", (chunk: string) => { - console.error("[cursor-native]", chunk.trim()); + const message = chunk.trim(); + if (message) { + this.logDiagnostic("stderr", { message }); + } + console.error("[cursor-native]", message); }); + child.once("exit", (code, signal) => { + this.logDiagnostic("exit", { + code, + signal, + sampleCount: this.sampleCount, + assetCount: this.assets.size, + outOfBoundsSampleCount: this.outOfBoundsSampleCount, + }); + this.rejectReady(new Error(`Windows cursor helper exited before ready (code=${code}, signal=${signal})`)); + }); + child.once("error", (error) => { + this.logDiagnostic("process-error", { message: error.message }); + this.rejectReady(error); + }); + + await this.waitUntilReady(); } async stop(): Promise { const child = this.process; this.process = null; + this.clearReadyState(); if (child && !child.killed) { child.kill(); } + this.logDiagnostic("stop", { + sampleCount: this.sampleCount, + assetCount: this.assets.size, + outOfBoundsSampleCount: this.outOfBoundsSampleCount, + }); + return { version: 2, provider: this.assets.size > 0 ? "native" : "none", @@ -285,11 +146,14 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession { private handleEvent(payload: WindowsCursorEvent) { if (payload.type === "error") { + this.logDiagnostic("helper-error", { message: payload.message }); console.error("Windows cursor helper error:", payload.message); return; } if (payload.type === "ready") { + this.logDiagnostic("ready", { timestampMs: payload.timestampMs }); + this.resolveReady(); return; } @@ -305,22 +169,100 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession { hotspotY: payload.asset.hotspotY, scaleFactor: assetDisplay.scaleFactor, }); + this.logDiagnostic("asset", { + id: payload.asset.id, + width: payload.asset.width, + height: payload.asset.height, + hotspotX: payload.asset.hotspotX, + hotspotY: payload.asset.hotspotY, + scaleFactor: assetDisplay.scaleFactor, + }); } - const bounds = this.options.getDisplayBounds() ?? screen.getPrimaryDisplay().bounds; - const width = Math.max(1, bounds.width); - const height = Math.max(1, bounds.height); + const normalized = this.normalizeSample(payload); + this.sampleCount += 1; + if (!normalized.withinBounds) { + this.outOfBoundsSampleCount += 1; + } - this.samples.push({ - timeMs: Math.max(0, payload.timestampMs - this.startTimeMs), - cx: clamp((payload.x - bounds.x) / width, 0, 1), - cy: clamp((payload.y - bounds.y) / height, 0, 1), - assetId: payload.handle, - visible: payload.visible, - }); + this.samples.push(normalized.sample); if (this.samples.length > this.options.maxSamples) { this.samples.shift(); } } + + private normalizeSample(payload: Extract): NormalizedSample { + const bounds = payload.bounds ?? this.options.getDisplayBounds() ?? screen.getPrimaryDisplay().bounds; + const width = Math.max(1, bounds.width); + const height = Math.max(1, bounds.height); + const normalizedX = (payload.x - bounds.x) / width; + const normalizedY = (payload.y - bounds.y) / height; + const withinBounds = normalizedX >= 0 && normalizedX <= 1 && normalizedY >= 0 && normalizedY <= 1; + + if (this.sampleCount === 0 || (!withinBounds && this.outOfBoundsSampleCount === 0)) { + this.logDiagnostic("sample", { + rawX: payload.x, + rawY: payload.y, + normalizedX, + normalizedY, + visible: payload.visible, + withinBounds, + bounds, + handle: payload.handle, + }); + } + + return { + withinBounds, + sample: { + timeMs: Math.max(0, payload.timestampMs - this.startTimeMs), + cx: normalizedX, + cy: normalizedY, + assetId: payload.handle, + visible: payload.visible && withinBounds, + }, + }; + } + + private waitUntilReady() { + return new Promise((resolve, reject) => { + this.readyResolve = resolve; + this.readyReject = reject; + this.readyTimer = setTimeout(() => { + this.rejectReady(new Error("Timed out waiting for Windows cursor helper readiness")); + }, READY_TIMEOUT_MS); + }); + } + + private resolveReady() { + const resolve = this.readyResolve; + this.clearReadyState(); + resolve?.(); + } + + private rejectReady(error: Error) { + const reject = this.readyReject; + this.clearReadyState(); + reject?.(error); + } + + private clearReadyState() { + if (this.readyTimer) { + clearTimeout(this.readyTimer); + this.readyTimer = null; + } + this.readyResolve = null; + this.readyReject = null; + } + + private logDiagnostic(event: string, data: Record) { + console.info( + "[cursor-native][win32]", + JSON.stringify({ + event, + ...data, + }), + ); + } } diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts new file mode 100644 index 0000000..6efd59d --- /dev/null +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts @@ -0,0 +1,49 @@ +import type { Rectangle } from "electron"; + +export interface WindowsCursorSampleEvent { + type: "sample"; + timestampMs: number; + x: number; + y: number; + visible: boolean; + handle: string | null; + bounds?: { + x: number; + y: number; + width: number; + height: number; + } | null; + asset?: WindowsCursorAssetPayload; +} + +export interface WindowsCursorReadyEvent { + type: "ready"; + timestampMs: number; +} + +export interface WindowsCursorErrorEvent { + type: "error"; + timestampMs: number; + message: string; +} + +export interface WindowsCursorAssetPayload { + id: string; + imageDataUrl: string; + width: number; + height: number; + hotspotX: number; + hotspotY: number; +} + +export type WindowsCursorEvent = + | WindowsCursorSampleEvent + | WindowsCursorReadyEvent + | WindowsCursorErrorEvent; + +export interface WindowsNativeRecordingSessionOptions { + getDisplayBounds: () => Rectangle | null; + maxSamples: number; + sampleIntervalMs: number; + sourceId?: string | null; +} diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index 260f4cb..57f79b3 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -259,6 +259,8 @@ export function LaunchWindow() { const [selectedSource, setSelectedSource] = useState("Screen"); const [hasSelectedSource, setHasSelectedSource] = useState(false); + const [, setHudPointerDownCount] = useState(0); + const [, setRecordPointerDownCount] = useState(0); useEffect(() => { const checkSelectedSource = async () => { @@ -541,6 +543,9 @@ export function LaunchWindow() { onClick={toggleMicrophone} disabled={recording} title={microphoneEnabled ? t("audio.disableMicrophone") : t("audio.enableMicrophone")} + onPointerDown={() => { + setRecordPointerDownCount((count) => count + 1); + }} > {microphoneEnabled ? getIcon("micOn", "text-green-400") diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index 1ed8938..842767a 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -1477,6 +1477,7 @@ export default function VideoEditor() { videoPadding: padding, cropRegion, cursorRecordingData, + cursorScale: showCursor ? cursorSize : 0, annotationRegions, webcamLayoutPreset, webcamMaskShape, @@ -1619,6 +1620,7 @@ export default function VideoEditor() { padding, cropRegion, cursorRecordingData, + cursorScale: showCursor ? cursorSize : 0, annotationRegions, webcamLayoutPreset, webcamMaskShape, diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx index 12b1c25..32db9d2 100644 --- a/src/components/video-editor/VideoPlayback.tsx +++ b/src/components/video-editor/VideoPlayback.tsx @@ -29,8 +29,9 @@ import { classifyWallpaper, DEFAULT_WALLPAPER, resolveImageWallpaperUrl } from " import { getCssClipPath } from "@/lib/webcamMaskShapes"; import { getNativeCursorDisplayMetrics, + hasNativeCursorRecordingData, projectNativeCursorToStage, - resolveActiveNativeCursorFrame, + resolveInterpolatedNativeCursorFrame, } from "@/lib/cursor/nativeCursor"; import type { CursorRecordingData } from "@/native/contracts"; import { @@ -635,6 +636,18 @@ const VideoPlayback = forwardRef( showCursorRef.current = showCursor; }, [showCursor]); + useEffect(() => { + hasNativeCursorRecordingRef.current = hasNativeCursorRecording; + }, [hasNativeCursorRecording]); + + useEffect(() => { + cursorRecordingDataRef.current = cursorRecordingData; + }, [cursorRecordingData]); + + useEffect(() => { + cropRegionRef.current = cropRegion; + }, [cropRegion]); + useEffect(() => { cursorSizeRef.current = cursorSize; }, [cursorSize]); @@ -1273,16 +1286,69 @@ const VideoPlayback = forwardRef( // Update cursor overlay const cursorOverlay = cursorOverlayRef.current; if (cursorOverlay) { - const timeMs = currentTimeRef.current; + const timeMs = currentTimeRef.current; // already in ms cursorOverlay.update( cursorTelemetryRef.current, timeMs, baseMaskRef.current, - showCursorRef.current, + showCursorRef.current && !hasNativeCursorRecordingRef.current, !isPlayingRef.current || isSeekingRef.current, ); } + // Update native cursor image position at ticker rate (60fps) + const nativeCursorImg = nativeCursorImgRef.current; + if (nativeCursorImg) { + const cameraContainerRc = cameraContainerRef.current; + const videoContainerRc = videoContainerRef.current; + if ( + hasNativeCursorRecordingRef.current && + showCursorRef.current && + cameraContainerRc && + videoContainerRc + ) { + const timeMs = currentTimeRef.current; // already in ms + const frame = resolveInterpolatedNativeCursorFrame( + cursorRecordingDataRef.current, + timeMs, + ); + if (frame) { + const projectedPoint = projectNativeCursorToStage({ + cameraContainer: cameraContainerRc, + cropRegion: cropRegionRef.current ?? { x: 0, y: 0, width: 1, height: 1 }, + maskRect: baseMaskRef.current, + videoContainerPosition: { + x: videoContainerRc.x, + y: videoContainerRc.y, + }, + sample: frame.sample, + }); + if (projectedPoint) { + const metrics = getNativeCursorDisplayMetrics( + frame.asset, + window.devicePixelRatio || 1, + ); + const scale = Math.max(0, cursorSizeRef.current); + if (nativeCursorImg.dataset.cursorId !== frame.asset.id) { + nativeCursorImg.src = frame.asset.imageDataUrl; + nativeCursorImg.dataset.cursorId = frame.asset.id; + } + nativeCursorImg.style.left = `${projectedPoint.x - metrics.hotspotX * scale}px`; + nativeCursorImg.style.top = `${projectedPoint.y - metrics.hotspotY * scale}px`; + nativeCursorImg.style.width = `${metrics.width * scale}px`; + nativeCursorImg.style.height = `${metrics.height * scale}px`; + nativeCursorImg.style.display = "block"; + } else { + nativeCursorImg.style.display = "none"; + } + } else { + nativeCursorImg.style.display = "none"; + } + } else { + nativeCursorImg.style.display = "none"; + } + } + const composite3D = composite3DRef.current; const outerWrapper = outerWrapperRef.current; if (composite3D && outerWrapper) { @@ -1571,17 +1637,14 @@ const VideoPlayback = forwardRef( className="absolute rounded-md border border-[#34B27B]/80 bg-[#34B27B]/20 shadow-[0_0_0_1px_rgba(52,178,123,0.35)]" style={{ display: "none", pointerEvents: "none" }} /> - {activeNativeCursor && nativeCursorStyle ? ( + {hasNativeCursorRecording ? ( 0 && + recordingData.assets.length > 0, + ); +} + function getCroppedCursorPosition(sample: CursorRecordingSample, cropRegion: CropRegion) { if (cropRegion.width <= 0 || cropRegion.height <= 0) { return null; @@ -45,7 +56,7 @@ export function resolveActiveNativeCursorFrame( recordingData: CursorRecordingData | null | undefined, timeMs: number, ): ActiveNativeCursorFrame | null { - if (!recordingData || recordingData.provider !== "native" || recordingData.assets.length === 0) { + if (!hasNativeCursorRecordingData(recordingData)) { return null; } @@ -70,6 +81,65 @@ export function resolveActiveNativeCursorFrame( return null; } +export function resolveInterpolatedNativeCursorFrame( + recordingData: CursorRecordingData | null | undefined, + timeMs: number, +): ActiveNativeCursorFrame | null { + if (!hasNativeCursorRecordingData(recordingData)) { + return null; + } + + const samples = recordingData.samples; + let activeIndex = -1; + + for (let index = samples.length - 1; index >= 0; index -= 1) { + if (samples[index].timeMs <= timeMs) { + activeIndex = index; + break; + } + } + + if (activeIndex < 0) { + return null; + } + + const activeSample = samples[activeIndex]; + if (activeSample.visible === false || !activeSample.assetId) { + return null; + } + + const asset = recordingData.assets.find((candidate) => candidate.id === activeSample.assetId); + if (!asset) { + return null; + } + + const nextSample = samples[activeIndex + 1]; + if ( + !nextSample || + nextSample.timeMs <= activeSample.timeMs || + nextSample.visible === false || + nextSample.assetId !== activeSample.assetId || + timeMs <= activeSample.timeMs + ) { + return { asset, sample: activeSample }; + } + + const interpolation = clamp( + (timeMs - activeSample.timeMs) / (nextSample.timeMs - activeSample.timeMs), + 0, + 1, + ); + + return { + asset, + sample: { + ...activeSample, + cx: activeSample.cx + (nextSample.cx - activeSample.cx) * interpolation, + cy: activeSample.cy + (nextSample.cy - activeSample.cy) * interpolation, + }, + }; +} + export function projectNativeCursorToStage({ cameraContainer, cropRegion, @@ -83,8 +153,8 @@ export function projectNativeCursorToStage({ } const localPoint = new Point( - videoContainerPosition.x + croppedPosition.cx * maskRect.width, - videoContainerPosition.y + croppedPosition.cy * maskRect.height, + videoContainerPosition.x + maskRect.x + croppedPosition.cx * maskRect.width, + videoContainerPosition.y + maskRect.y + croppedPosition.cy * maskRect.height, ); return cameraContainer.toGlobal(localPoint); diff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.ts index f13735d..6f605b0 100644 --- a/src/lib/exporter/frameRenderer.ts +++ b/src/lib/exporter/frameRenderer.ts @@ -59,7 +59,7 @@ import { import { getNativeCursorDisplayMetrics, projectNativeCursorToStage, - resolveActiveNativeCursorFrame, + resolveInterpolatedNativeCursorFrame, } from "@/lib/cursor/nativeCursor"; import { BackgroundLoadError, classifyWallpaper, resolveImageWallpaperUrl } from "@/lib/wallpaper"; import { drawCanvasClipPath } from "@/lib/webcamMaskShapes"; @@ -86,6 +86,7 @@ interface FrameRenderConfig { padding?: number; cropRegion: CropRegion; cursorRecordingData?: CursorRecordingData | null; + cursorScale?: number; videoWidth: number; videoHeight: number; webcamSize?: Size | null; @@ -558,7 +559,11 @@ export class FrameRenderer { return; } - const activeNativeCursor = resolveActiveNativeCursorFrame( + if ((this.config.cursorScale ?? 1) <= 0) { + return; + } + + const activeNativeCursor = resolveInterpolatedNativeCursorFrame( this.config.cursorRecordingData, timeMs, ); @@ -582,13 +587,13 @@ export class FrameRenderer { const image = await this.getCursorImage(activeNativeCursor.asset); const metrics = getNativeCursorDisplayMetrics(activeNativeCursor.asset, 1); - + const scale = Math.max(0, this.config.cursorScale ?? 1); this.compositeCtx.drawImage( image, - projectedPoint.x - metrics.hotspotX, - projectedPoint.y - metrics.hotspotY, - metrics.width, - metrics.height, + projectedPoint.x - metrics.hotspotX * scale, + projectedPoint.y - metrics.hotspotY * scale, + metrics.width * scale, + metrics.height * scale, ); } diff --git a/src/lib/exporter/gifExporter.ts b/src/lib/exporter/gifExporter.ts index 02564db..c1120af 100644 --- a/src/lib/exporter/gifExporter.ts +++ b/src/lib/exporter/gifExporter.ts @@ -49,6 +49,7 @@ interface GifExporterConfig { webcamSizePreset?: WebcamSizePreset; webcamPosition?: { cx: number; cy: number } | null; cursorRecordingData?: CursorRecordingData | null; + cursorScale?: number; annotationRegions?: AnnotationRegion[]; previewWidth?: number; previewHeight?: number; @@ -154,6 +155,7 @@ export class GifExporter { padding: this.config.padding, cropRegion: this.config.cropRegion, cursorRecordingData: this.config.cursorRecordingData, + cursorScale: this.config.cursorScale, videoWidth: videoInfo.width, videoHeight: videoInfo.height, webcamSize: webcamInfo ? { width: webcamInfo.width, height: webcamInfo.height } : null, diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index edddd05..fb38611 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -40,6 +40,7 @@ interface VideoExporterConfig extends ExportConfig { webcamSizePreset?: WebcamSizePreset; webcamPosition?: { cx: number; cy: number } | null; cursorRecordingData?: CursorRecordingData | null; + cursorScale?: number; annotationRegions?: AnnotationRegion[]; previewWidth?: number; previewHeight?: number; @@ -149,6 +150,7 @@ export class VideoExporter { padding: this.config.padding, cropRegion: this.config.cropRegion, cursorRecordingData: this.config.cursorRecordingData, + cursorScale: this.config.cursorScale, videoWidth: videoInfo.width, videoHeight: videoInfo.height, webcamSize: webcamInfo ? { width: webcamInfo.width, height: webcamInfo.height } : null,