diff --git a/README.md b/README.md index 7009a22..0403afe 100644 --- a/README.md +++ b/README.md @@ -171,6 +171,9 @@ See the documentation here: [OpenScreen Docs](https://deepwiki.com/siddharthvaddem/openscreen) Refresh if outdated. +Developer notes: +- [Windows native cursor test pipeline](docs/testing/windows-native-cursor.md) + ## Contributing Contributions are welcome - please **include screenshots or a short video** for any UI change or new user-facing feature. If it touches what users see or do, show it. Skip only when it genuinely doesn't apply. PRs that don't follow this will be closed. diff --git a/docs/testing/windows-native-cursor.md b/docs/testing/windows-native-cursor.md new file mode 100644 index 0000000..1abe04e --- /dev/null +++ b/docs/testing/windows-native-cursor.md @@ -0,0 +1,85 @@ +# Windows native cursor test pipeline + +This branch includes two Windows-focused diagnostics for fast iteration on native cursor capture and rendering. They are intentionally local developer tools: they create short videos and JSON reports so cursor changes can be inspected without doing a full manual record/edit/export cycle. + +## Native sampler diagnostic + +```powershell +npm run test:cursor-native:win +``` + +This script does not launch OpenScreen. It: + +- starts a Windows `GetCursorInfo` sampler +- moves the real OS pointer with `SetCursorPos` +- captures native cursor handles, hotspots, assets, and standard `IDC_*` cursor types +- writes normalized `CursorRecordingData` +- generates an abstract preview video +- generates a real-screen preview video using screenshots of the current desktop + +The output directory is printed in the command result, for example: + +```text +C:\Users\\AppData\Local\Temp\openscreen-cursor-native-... +``` + +Useful files: + +- `report.json`: sample counts, asset counts, cursor handles, and generated artifact paths +- `cursor-recording-data.json`: sidecar-compatible cursor data +- `preview.webm`: abstract path/asset/hotspot preview +- `real-capture-preview.webm`: real desktop screenshot background with reconstructed cursor overlay +- `assets/*.png`: raw cursor bitmaps captured from Windows + +Environment overrides: + +```powershell +$env:CURSOR_TEST_DURATION_MS = "3000" +$env:CURSOR_TEST_SAMPLE_INTERVAL_MS = "16" +$env:CURSOR_TEST_SCREEN_FRAME_INTERVAL_MS = "80" +$env:CURSOR_TEST_OUTPUT_DIR = "C:\temp\openscreen-cursor-test" +npm run test:cursor-native:win +``` + +## OpenScreen preview capture + +```powershell +npm run capture:openscreen-preview +``` + +This script launches the real Electron app, injects a fixture video plus cursor sidecar data, opens the editor, captures frames from the actual OpenScreen preview UI, and encodes them into a WebM. + +By default it uses the latest `cursor-recording-data.json` generated by `npm run test:cursor-native:win`. To force a specific sidecar: + +```powershell +$env:CURSOR_RECORDING_DATA_PATH = "C:\path\to\cursor-recording-data.json" +npm run capture:openscreen-preview +``` + +Useful environment overrides: + +```powershell +$env:OPENSCREEN_PREVIEW_SKIP_BUILD = "true" +$env:OPENSCREEN_PREVIEW_FRAME_COUNT = "120" +$env:OPENSCREEN_PREVIEW_FPS = "30" +$env:OPENSCREEN_PREVIEW_OUTPUT_DIR = "C:\temp\openscreen-preview" +npm run capture:openscreen-preview +``` + +Useful files: + +- `openscreen-preview.webm`: video of the real OpenScreen editor preview +- `frames/*.png`: captured preview frames +- `report.json`: fixture paths, source sidecar, frame count, and output path + +## What these tests validate + +Together, the scripts make it quick to inspect: + +- whether Windows cursor samples are visible and continuous +- whether native hotspots stay anchored when scaling to `3x` +- whether standard Windows cursors are recognized via `IDC_*` +- whether high-quality SVG cursor replacements follow the native hotspot +- whether the real OpenScreen preview renders the same cursor behavior as the diagnostic pipeline + +They are not a full substitute for an end-to-end manual recording pass. Before shipping cursor changes, also test a real capture session and export from the packaged app. diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 3a4dd3b..c5a1269 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -267,6 +267,7 @@ function normalizeCursorSample(sample: unknown): CursorRecordingSample | null { cy: typeof point.cy === "number" && Number.isFinite(point.cy) ? point.cy : 0.5, assetId: typeof point.assetId === "string" ? point.assetId : null, visible: typeof point.visible === "boolean" ? point.visible : true, + cursorType: typeof point.cursorType === "string" ? point.cursorType : null, }; } @@ -305,6 +306,7 @@ function normalizeCursorAsset(asset: unknown): NativeCursorAsset | null { typeof candidate.scaleFactor === "number" && Number.isFinite(candidate.scaleFactor) ? Math.max(0.1, candidate.scaleFactor) : undefined, + cursorType: typeof candidate.cursorType === "string" ? candidate.cursorType : null, }; } @@ -1079,6 +1081,20 @@ export function registerIpcHandlers( return setCurrentVideoPath(path); }); + ipcMain.handle("set-current-recording-session", (_, session: RecordingSession | null) => { + const normalizedSession = normalizeRecordingSession(session); + setCurrentRecordingSessionState(normalizedSession); + currentVideoPath = normalizedSession?.screenVideoPath ?? null; + currentProjectPath = null; + return { success: true, session: currentRecordingSession }; + }); + + ipcMain.handle("get-current-recording-session", () => { + return currentRecordingSession + ? { success: true, session: currentRecordingSession } + : { success: false }; + }); + function setCurrentVideoPath(path: string): ProjectPathResult { currentVideoPath = normalizeVideoSourcePath(path) ?? path; currentProjectPath = null; diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts index b7a11cb..5607134 100644 --- a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts @@ -16,7 +16,7 @@ export function buildPowerShellCommand(sampleIntervalMs: number, windowHandle?: $ErrorActionPreference = 'Stop' Add-Type -AssemblyName System.Drawing -$targetWindowHandle = ${windowHandle ? `'${windowHandle}'` : '$null'} +$targetWindowHandle = ${windowHandle ? `'${windowHandle}'` : "$null"} $source = @" using System; @@ -59,6 +59,9 @@ public static class OpenScreenCursorInterop { [return: MarshalAs(UnmanagedType.Bool)] public static extern bool GetCursorInfo(ref CURSORINFO pci); + [DllImport("user32.dll", SetLastError = true)] + public static extern IntPtr LoadCursor(IntPtr hInstance, IntPtr lpCursorName); + [DllImport("user32.dll", SetLastError = true)] [return: MarshalAs(UnmanagedType.Bool)] public static extern bool GetWindowRect(IntPtr hWnd, out RECT lpRect); @@ -86,6 +89,37 @@ public static class OpenScreenCursorInterop { Add-Type -TypeDefinition $source +$standardCursors = @{ + arrow = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32512)) + text = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32513)) + wait = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32514)) + crosshair = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32515)) + 'up-arrow' = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32516)) + 'resize-nwse' = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32642)) + 'resize-nesw' = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32643)) + 'resize-ew' = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32644)) + 'resize-ns' = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32645)) + move = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32646)) + 'not-allowed' = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32648)) + pointer = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32649)) + 'app-starting' = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32650)) + help = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32651)) +} + +function Get-StandardCursorType($cursorHandle) { + if ($cursorHandle -eq [IntPtr]::Zero) { + return $null + } + + foreach ($entry in $standardCursors.GetEnumerator()) { + if ($entry.Value -eq $cursorHandle) { + return $entry.Key + } + } + + return $null +} + function Write-JsonLine($payload) { [Console]::Out.WriteLine(($payload | ConvertTo-Json -Compress -Depth 6)) } @@ -190,10 +224,14 @@ while ($true) { $visible = ($cursorInfo.flags -band 1) -ne 0 $cursorId = if ($cursorInfo.hCursor -eq [IntPtr]::Zero) { $null } else { ('0x{0:X}' -f $cursorInfo.hCursor.ToInt64()) } + $cursorType = Get-StandardCursorType $cursorInfo.hCursor $asset = $null if ($visible -and $cursorId -and $cursorId -ne $lastCursorId) { $asset = Get-CursorAsset -cursorHandle $cursorInfo.hCursor -cursorId $cursorId + if ($asset -and $cursorType) { + $asset.cursorType = $cursorType + } $lastCursorId = $cursorId } @@ -204,6 +242,7 @@ while ($true) { y = $cursorInfo.ptScreenPos.Y visible = $visible handle = $cursorId + cursorType = $cursorType bounds = Get-TargetBounds asset = $asset } diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts index d5e43d7..632a74d 100644 --- a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts @@ -7,7 +7,10 @@ import type { NativeCursorAsset, } from "../../../../src/native/contracts"; import type { CursorRecordingSession } from "./session"; -import { buildPowerShellCommand, parseWindowHandleFromSourceId } from "./windowsNativeRecordingSession.script"; +import { + buildPowerShellCommand, + parseWindowHandleFromSourceId, +} from "./windowsNativeRecordingSession.script"; import type { WindowsCursorEvent, WindowsNativeRecordingSessionOptions, @@ -91,7 +94,9 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession { assetCount: this.assets.size, outOfBoundsSampleCount: this.outOfBoundsSampleCount, }); - this.rejectReady(new Error(`Windows cursor helper exited before ready (code=${code}, signal=${signal})`)); + this.rejectReady( + new Error(`Windows cursor helper exited before ready (code=${code}, signal=${signal})`), + ); }); child.once("error", (error) => { this.logDiagnostic("process-error", { message: error.message }); @@ -168,6 +173,7 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession { hotspotX: payload.asset.hotspotX, hotspotY: payload.asset.hotspotY, scaleFactor: assetDisplay.scaleFactor, + cursorType: payload.asset.cursorType ?? payload.cursorType ?? null, }); this.logDiagnostic("asset", { id: payload.asset.id, @@ -192,13 +198,17 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession { } } - private normalizeSample(payload: Extract): NormalizedSample { - const bounds = payload.bounds ?? this.options.getDisplayBounds() ?? screen.getPrimaryDisplay().bounds; + private normalizeSample( + payload: Extract, + ): NormalizedSample { + const bounds = + payload.bounds ?? this.options.getDisplayBounds() ?? screen.getPrimaryDisplay().bounds; const width = Math.max(1, bounds.width); const height = Math.max(1, bounds.height); const normalizedX = (payload.x - bounds.x) / width; const normalizedY = (payload.y - bounds.y) / height; - const withinBounds = normalizedX >= 0 && normalizedX <= 1 && normalizedY >= 0 && normalizedY <= 1; + const withinBounds = + normalizedX >= 0 && normalizedX <= 1 && normalizedY >= 0 && normalizedY <= 1; if (this.sampleCount === 0 || (!withinBounds && this.outOfBoundsSampleCount === 0)) { this.logDiagnostic("sample", { @@ -221,6 +231,7 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession { cy: normalizedY, assetId: payload.handle, visible: payload.visible && withinBounds, + cursorType: payload.cursorType ?? payload.asset?.cursorType ?? null, }, }; } diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts index 6efd59d..fdc4ab9 100644 --- a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts @@ -1,4 +1,5 @@ import type { Rectangle } from "electron"; +import type { NativeCursorType } from "../../../../src/native/contracts"; export interface WindowsCursorSampleEvent { type: "sample"; @@ -7,6 +8,7 @@ export interface WindowsCursorSampleEvent { y: number; visible: boolean; handle: string | null; + cursorType?: NativeCursorType | null; bounds?: { x: number; y: number; @@ -34,6 +36,7 @@ export interface WindowsCursorAssetPayload { height: number; hotspotX: number; hotspotY: number; + cursorType?: NativeCursorType | null; } export type WindowsCursorEvent = diff --git a/package.json b/package.json index 2ccb0b3..f81d99b 100644 --- a/package.json +++ b/package.json @@ -19,12 +19,14 @@ "lint:fix": "biome check --write .", "format": "biome format --write .", "i18n:check": "node scripts/i18n-check.mjs", - "preview": "vite preview", - "build:mac": "tsc && vite build && electron-builder --mac", - "build:win": "tsc && vite build && electron-builder --win --config.npmRebuild=false", - "build:linux": "tsc && vite build && electron-builder --linux AppImage deb pacman --config.npmRebuild=false", - "test": "vitest --run", + "preview": "vite preview", + "build:mac": "tsc && vite build && electron-builder --mac", + "build:win": "tsc && vite build && electron-builder --win --config.npmRebuild=false", + "build:linux": "tsc && vite build && electron-builder --linux AppImage deb pacman --config.npmRebuild=false", + "test": "vitest --run", "test:watch": "vitest", + "test:cursor-native:win": "node scripts/test-windows-native-cursor.mjs", + "capture:openscreen-preview": "node scripts/capture-openscreen-preview.mjs", "build-vite": "tsc && vite build", "test:browser": "vitest --config vitest.browser.config.ts --run", "test:browser:install": "playwright install --with-deps chromium-headless-shell", diff --git a/scripts/capture-openscreen-preview.mjs b/scripts/capture-openscreen-preview.mjs new file mode 100644 index 0000000..6c9b6eb --- /dev/null +++ b/scripts/capture-openscreen-preview.mjs @@ -0,0 +1,262 @@ +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { chromium, _electron as electron } from "@playwright/test"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.join(__dirname, ".."); +const MAIN_JS = path.join(ROOT, "dist-electron", "main.js"); +const TEST_VIDEO = path.join(ROOT, "tests", "fixtures", "sample.webm"); +const OUTPUT_DIR = + process.env.OPENSCREEN_PREVIEW_OUTPUT_DIR ?? + path.join(os.tmpdir(), `openscreen-real-preview-${Date.now()}`); +const FRAME_COUNT = Number(process.env.OPENSCREEN_PREVIEW_FRAME_COUNT ?? 90); +const FPS = Number(process.env.OPENSCREEN_PREVIEW_FPS ?? 30); + +function findLatestCursorRecordingData() { + const explicit = process.env.CURSOR_RECORDING_DATA_PATH; + if (explicit) { + if (!fs.existsSync(explicit)) { + throw new Error(`CURSOR_RECORDING_DATA_PATH does not exist: ${explicit}`); + } + return explicit; + } + + const tempDir = os.tmpdir(); + const candidates = fs + .readdirSync(tempDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && entry.name.startsWith("openscreen-cursor-native-")) + .map((entry) => path.join(tempDir, entry.name, "cursor-recording-data.json")) + .filter((candidate) => fs.existsSync(candidate)) + .map((candidate) => ({ path: candidate, mtimeMs: fs.statSync(candidate).mtimeMs })) + .sort((a, b) => b.mtimeMs - a.mtimeMs); + + if (!candidates[0]) { + throw new Error( + "No cursor-recording-data.json found. Run npm run test:cursor-native:win first.", + ); + } + + return candidates[0].path; +} + +function findPlaywrightChromiumExecutable(defaultPath) { + if (fs.existsSync(defaultPath)) { + return defaultPath; + } + + const baseDir = path.join(process.env.LOCALAPPDATA ?? "", "ms-playwright"); + if (!baseDir || !fs.existsSync(baseDir)) { + return defaultPath; + } + + const candidates = fs + .readdirSync(baseDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && entry.name.startsWith("chromium-")) + .map((entry) => path.join(baseDir, entry.name, "chrome-win64", "chrome.exe")) + .filter((candidate) => fs.existsSync(candidate)) + .sort() + .reverse(); + + return candidates[0] ?? defaultPath; +} + +function ensureBuildExists() { + if (!fs.existsSync(MAIN_JS)) { + throw new Error(`Missing ${MAIN_JS}. Run npm run build-vite first.`); + } + if (!fs.existsSync(path.join(ROOT, "dist", "index.html"))) { + throw new Error(`Missing renderer build. Run npm run build-vite first.`); + } +} + +function runNpmBuildViteIfRequested() { + if (process.env.OPENSCREEN_PREVIEW_SKIP_BUILD === "true") { + ensureBuildExists(); + return Promise.resolve(); + } + + return new Promise((resolve, reject) => { + const child = spawn("cmd.exe", ["/d", "/s", "/c", "npm run build-vite"], { + cwd: ROOT, + stdio: "inherit", + }); + child.once("error", reject); + child.once("exit", (code) => { + if (code === 0) resolve(); + else reject(new Error(`npm run build-vite failed with code ${code}`)); + }); + }); +} + +async function encodeFramesToWebm(framePaths, outputPath) { + const frameData = framePaths.map((framePath) => ({ + src: `data:image/png;base64,${fs.readFileSync(framePath).toString("base64")}`, + })); + const html = ` + + + + + +`; + + const browser = await chromium.launch({ + executablePath: findPlaywrightChromiumExecutable(chromium.executablePath()), + headless: true, + }); + try { + const page = await browser.newPage(); + await page.setContent(html); + const base64 = await page.evaluate(() => window.__encode()); + fs.writeFileSync(outputPath, Buffer.from(base64, "base64")); + } finally { + await browser.close(); + } +} + +fs.mkdirSync(OUTPUT_DIR, { recursive: true }); +const cursorRecordingDataPath = findLatestCursorRecordingData(); +const fixtureVideoPath = path.join(OUTPUT_DIR, "openscreen-preview-fixture.webm"); +const outputVideoPath = path.join(OUTPUT_DIR, "openscreen-preview.webm"); +fs.copyFileSync(TEST_VIDEO, fixtureVideoPath); +fs.copyFileSync(cursorRecordingDataPath, `${fixtureVideoPath}.cursor.json`); + +await runNpmBuildViteIfRequested(); + +const app = await electron.launch({ + args: [MAIN_JS, "--no-sandbox", "--enable-unsafe-swiftshader"], + env: { + ...process.env, + HEADLESS: "false", + }, +}); + +app.process().stdout?.on("data", (data) => process.stdout.write(`[electron] ${data}`)); +app.process().stderr?.on("data", (data) => process.stderr.write(`[electron] ${data}`)); + +const framesDir = path.join(OUTPUT_DIR, "frames"); +fs.mkdirSync(framesDir, { recursive: true }); + +try { + const hudWindow = await app.firstWindow({ timeout: 60_000 }); + await hudWindow.waitForLoadState("domcontentloaded"); + await hudWindow.evaluate(async () => { + for (let attempt = 0; attempt < 100; attempt += 1) { + try { + await window.electronAPI.getCurrentRecordingSession(); + await window.electronAPI.getCurrentVideoPath(); + return; + } catch { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + } + throw new Error("Timed out waiting for OpenScreen IPC handlers."); + }); + + try { + await hudWindow.evaluate(async (videoPath) => { + await window.electronAPI.setCurrentVideoPath(videoPath); + await window.electronAPI.switchToEditor(); + }, fixtureVideoPath); + } catch { + // switchToEditor closes the HUD page before the evaluate promise can always resolve. + } + + const editorWindow = await app.waitForEvent("window", { + predicate: (window) => window.url().includes("windowType=editor"), + timeout: 30_000, + }); + await editorWindow.waitForLoadState("domcontentloaded"); + await editorWindow.waitForSelector("video", { state: "attached", timeout: 30_000 }); + await editorWindow.waitForSelector("canvas", { state: "attached", timeout: 30_000 }); + await editorWindow.waitForSelector('img[aria-hidden="true"]', { + state: "attached", + timeout: 30_000, + }); + + await editorWindow.setViewportSize({ width: 1280, height: 800 }); + await editorWindow.evaluate(async () => { + await document.fonts.ready; + for (const video of [...document.querySelectorAll("video")]) { + video.muted = true; + video.currentTime = 0; + video.dispatchEvent(new Event("timeupdate")); + } + }); + await editorWindow.waitForTimeout(1000); + + const framePaths = []; + for (let index = 0; index < FRAME_COUNT; index += 1) { + const timeSec = index / FPS; + await editorWindow.evaluate((time) => { + for (const video of [...document.querySelectorAll("video")]) { + video.currentTime = Math.min(time, Math.max(0, video.duration || time)); + video.dispatchEvent(new Event("timeupdate")); + } + }, timeSec); + await editorWindow.waitForTimeout(40); + const framePath = path.join(framesDir, `frame-${String(index).padStart(4, "0")}.png`); + await editorWindow.screenshot({ path: framePath }); + framePaths.push(framePath); + } + + await encodeFramesToWebm(framePaths, outputVideoPath); + + const report = { + outputDir: OUTPUT_DIR, + sourceCursorRecordingDataPath: cursorRecordingDataPath, + fixtureVideoPath, + outputVideoPath, + frameCount: framePaths.length, + fps: FPS, + }; + fs.writeFileSync(path.join(OUTPUT_DIR, "report.json"), JSON.stringify(report, null, 2)); + console.log(JSON.stringify(report, null, 2)); +} finally { + await app.close(); +} diff --git a/scripts/test-windows-native-cursor.mjs b/scripts/test-windows-native-cursor.mjs new file mode 100644 index 0000000..2a8b34c --- /dev/null +++ b/scripts/test-windows-native-cursor.mjs @@ -0,0 +1,1113 @@ +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const SAMPLE_INTERVAL_MS = Number(process.env.CURSOR_TEST_SAMPLE_INTERVAL_MS ?? 25); +const DURATION_MS = Number(process.env.CURSOR_TEST_DURATION_MS ?? 1800); +const SCREEN_FRAME_INTERVAL_MS = Number(process.env.CURSOR_TEST_SCREEN_FRAME_INTERVAL_MS ?? 100); +const READY_TIMEOUT_MS = 5000; +const OUTPUT_DIR = + process.env.CURSOR_TEST_OUTPUT_DIR ?? + path.join(os.tmpdir(), `openscreen-cursor-native-${Date.now()}`); + +if (process.platform !== "win32") { + console.error("This diagnostic is Windows-only."); + process.exit(1); +} + +function encodePowerShell(script) { + return Buffer.from(script, "utf16le").toString("base64"); +} + +function quotePowerShellString(value) { + return `'${String(value).replaceAll("'", "''")}'`; +} + +function runPowerShell(script) { + return new Promise((resolve, reject) => { + const child = spawn( + "powershell.exe", + [ + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-EncodedCommand", + encodePowerShell(script), + ], + { stdio: ["ignore", "pipe", "pipe"], windowsHide: true }, + ); + + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.once("error", reject); + child.once("exit", (code, signal) => { + if (code === 0) { + resolve(stdout); + return; + } + + reject( + new Error(`PowerShell command failed (code=${code}, signal=${signal}): ${stderr.trim()}`), + ); + }); + }); +} + +function spawnPowerShell(script, { onStdout, onStderr } = {}) { + const child = spawn( + "powershell.exe", + [ + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-EncodedCommand", + encodePowerShell(script), + ], + { stdio: ["ignore", "pipe", "pipe"], windowsHide: true }, + ); + + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => onStdout?.(chunk)); + child.stderr.on("data", (chunk) => onStderr?.(chunk)); + + const done = new Promise((resolve, reject) => { + child.once("error", reject); + child.once("exit", (code, signal) => { + if (code === 0 || child.killed) { + resolve({ code, signal }); + return; + } + + reject(new Error(`PowerShell process failed (code=${code}, signal=${signal})`)); + }); + }); + + return { child, done }; +} + +function buildSamplerScript() { + return String.raw` +$ErrorActionPreference = 'Stop' +Add-Type -AssemblyName System.Drawing +Add-Type -AssemblyName System.Windows.Forms + +$source = @" +using System; +using System.Runtime.InteropServices; + +public static class OpenScreenCursorDiagnosticInterop { + [StructLayout(LayoutKind.Sequential)] + public struct POINT { + public int X; + public int Y; + } + + [StructLayout(LayoutKind.Sequential)] + public struct CURSORINFO { + public int cbSize; + public int flags; + public IntPtr hCursor; + public POINT ptScreenPos; + } + + [StructLayout(LayoutKind.Sequential)] + public struct ICONINFO { + [MarshalAs(UnmanagedType.Bool)] + public bool fIcon; + public int xHotspot; + public int yHotspot; + public IntPtr hbmMask; + public IntPtr hbmColor; + } + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool GetCursorInfo(ref CURSORINFO pci); + + [DllImport("user32.dll", SetLastError = true)] + public static extern IntPtr LoadCursor(IntPtr hInstance, IntPtr lpCursorName); + + [DllImport("user32.dll", SetLastError = true)] + public static extern IntPtr CopyIcon(IntPtr hIcon); + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool DestroyIcon(IntPtr hIcon); + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool GetIconInfo(IntPtr hIcon, out ICONINFO piconinfo); + + [DllImport("gdi32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool DeleteObject(IntPtr hObject); +} +"@ + +Add-Type -TypeDefinition $source + +$standardCursors = @{ + arrow = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32512)) + text = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32513)) + wait = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32514)) + crosshair = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32515)) + 'up-arrow' = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32516)) + 'resize-nwse' = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32642)) + 'resize-nesw' = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32643)) + 'resize-ew' = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32644)) + 'resize-ns' = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32645)) + move = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32646)) + 'not-allowed' = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32648)) + pointer = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32649)) + 'app-starting' = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32650)) + help = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32651)) +} + +function Get-StandardCursorType($cursorHandle) { + if ($cursorHandle -eq [IntPtr]::Zero) { + return $null + } + + foreach ($entry in $standardCursors.GetEnumerator()) { + if ($entry.Value -eq $cursorHandle) { + return $entry.Key + } + } + + return $null +} + +function Write-JsonLine($payload) { + [Console]::Out.WriteLine(($payload | ConvertTo-Json -Compress -Depth 6)) +} + +function Get-CursorAsset($cursorHandle, $cursorId) { + $copiedHandle = [OpenScreenCursorDiagnosticInterop]::CopyIcon($cursorHandle) + if ($copiedHandle -eq [IntPtr]::Zero) { + return $null + } + + $iconInfo = New-Object OpenScreenCursorDiagnosticInterop+ICONINFO + $hasIconInfo = [OpenScreenCursorDiagnosticInterop]::GetIconInfo($copiedHandle, [ref]$iconInfo) + + try { + $icon = [System.Drawing.Icon]::FromHandle($copiedHandle) + $bitmap = New-Object System.Drawing.Bitmap $icon.Width, $icon.Height, ([System.Drawing.Imaging.PixelFormat]::Format32bppArgb) + $graphics = [System.Drawing.Graphics]::FromImage($bitmap) + $memoryStream = New-Object System.IO.MemoryStream + + try { + $graphics.Clear([System.Drawing.Color]::Transparent) + $graphics.DrawIcon($icon, 0, 0) + $bitmap.Save($memoryStream, [System.Drawing.Imaging.ImageFormat]::Png) + $base64 = [System.Convert]::ToBase64String($memoryStream.ToArray()) + + return @{ + id = $cursorId + imageDataUrl = "data:image/png;base64,$base64" + width = $bitmap.Width + height = $bitmap.Height + hotspotX = if ($hasIconInfo) { $iconInfo.xHotspot } else { 0 } + hotspotY = if ($hasIconInfo) { $iconInfo.yHotspot } else { 0 } + } + } + finally { + $memoryStream.Dispose() + $graphics.Dispose() + $bitmap.Dispose() + $icon.Dispose() + } + } + finally { + if ($hasIconInfo) { + if ($iconInfo.hbmMask -ne [IntPtr]::Zero) { + [OpenScreenCursorDiagnosticInterop]::DeleteObject($iconInfo.hbmMask) | Out-Null + } + if ($iconInfo.hbmColor -ne [IntPtr]::Zero) { + [OpenScreenCursorDiagnosticInterop]::DeleteObject($iconInfo.hbmColor) | Out-Null + } + } + [OpenScreenCursorDiagnosticInterop]::DestroyIcon($copiedHandle) | Out-Null + } +} + +Write-JsonLine @{ type = 'ready'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() } + +$lastCursorId = $null +$screenBounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds +while ($true) { + $cursorInfo = New-Object OpenScreenCursorDiagnosticInterop+CURSORINFO + $cursorInfo.cbSize = [Runtime.InteropServices.Marshal]::SizeOf([type][OpenScreenCursorDiagnosticInterop+CURSORINFO]) + + if (-not [OpenScreenCursorDiagnosticInterop]::GetCursorInfo([ref]$cursorInfo)) { + Write-JsonLine @{ type = 'error'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds(); message = 'GetCursorInfo failed' } + Start-Sleep -Milliseconds ${SAMPLE_INTERVAL_MS} + continue + } + + $visible = ($cursorInfo.flags -band 1) -ne 0 + $cursorId = if ($cursorInfo.hCursor -eq [IntPtr]::Zero) { $null } else { ('0x{0:X}' -f $cursorInfo.hCursor.ToInt64()) } + $cursorType = Get-StandardCursorType $cursorInfo.hCursor + $asset = $null + + if ($visible -and $cursorId -and $cursorId -ne $lastCursorId) { + $asset = Get-CursorAsset -cursorHandle $cursorInfo.hCursor -cursorId $cursorId + if ($asset -and $cursorType) { + $asset.cursorType = $cursorType + } + $lastCursorId = $cursorId + } + + Write-JsonLine @{ + type = 'sample' + timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + x = $cursorInfo.ptScreenPos.X + y = $cursorInfo.ptScreenPos.Y + visible = $visible + handle = $cursorId + cursorType = $cursorType + bounds = @{ + x = $screenBounds.Left + y = $screenBounds.Top + width = $screenBounds.Width + height = $screenBounds.Height + } + asset = $asset + } + + Start-Sleep -Milliseconds ${SAMPLE_INTERVAL_MS} +} +`; +} + +function buildMousePathScript(durationMs) { + const stepMs = 120; + const steps = Math.max(8, Math.floor(durationMs / stepMs)); + + return String.raw` +$ErrorActionPreference = 'Stop' +Add-Type -AssemblyName System.Windows.Forms + +$source = @" +using System.Runtime.InteropServices; + +public static class OpenScreenMouseDiagnosticInterop { + [DllImport("user32.dll")] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool SetCursorPos(int X, int Y); +} +"@ + +Add-Type -TypeDefinition $source + +$bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds +$points = @() +for ($i = 0; $i -lt ${steps}; $i++) { + $t = if (${steps} -le 1) { 0 } else { $i / (${steps} - 1) } + $x = [int]($bounds.Left + 80 + (($bounds.Width - 160) * $t)) + $wave = [Math]::Sin($t * [Math]::PI * 2) + $y = [int]($bounds.Top + ($bounds.Height / 2) + ($wave * [Math]::Min(180, $bounds.Height / 4))) + $points += @{ x = $x; y = $y } +} + +foreach ($point in $points) { + [OpenScreenMouseDiagnosticInterop]::SetCursorPos($point.x, $point.y) | Out-Null + Start-Sleep -Milliseconds ${stepMs} +} +`; +} + +function buildScreenRecorderScript(outputDir, durationMs) { + const framesDir = path.join(outputDir, "screen-frames"); + + return String.raw` +$ErrorActionPreference = 'Stop' +Add-Type -AssemblyName System.Drawing +Add-Type -AssemblyName System.Windows.Forms + +$framesDir = ${quotePowerShellString(framesDir)} +New-Item -ItemType Directory -Force -Path $framesDir | Out-Null + +$bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds +$targetWidth = 960 +$targetHeight = [int]([Math]::Round($targetWidth * ($bounds.Height / $bounds.Width))) +$frames = New-Object System.Collections.Generic.List[object] +$stopwatch = [System.Diagnostics.Stopwatch]::StartNew() +$index = 0 + +while ($stopwatch.ElapsedMilliseconds -le ${durationMs + 700}) { + $sourceBitmap = New-Object System.Drawing.Bitmap $bounds.Width, $bounds.Height, ([System.Drawing.Imaging.PixelFormat]::Format32bppArgb) + $graphics = [System.Drawing.Graphics]::FromImage($sourceBitmap) + $scaledBitmap = New-Object System.Drawing.Bitmap $targetWidth, $targetHeight, ([System.Drawing.Imaging.PixelFormat]::Format32bppArgb) + $scaledGraphics = [System.Drawing.Graphics]::FromImage($scaledBitmap) + $timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + $fileName = ('frame_{0:D4}.png' -f $index) + $path = Join-Path $framesDir $fileName + + try { + $graphics.CopyFromScreen($bounds.Left, $bounds.Top, 0, 0, $bounds.Size) + $scaledGraphics.InterpolationMode = [System.Drawing.Drawing2D.InterpolationMode]::HighQualityBicubic + $scaledGraphics.DrawImage($sourceBitmap, 0, 0, $targetWidth, $targetHeight) + $scaledBitmap.Save($path, [System.Drawing.Imaging.ImageFormat]::Png) + $frames.Add(@{ + index = $index + timestampMs = $timestampMs + path = $path + width = $targetWidth + height = $targetHeight + bounds = @{ + x = $bounds.Left + y = $bounds.Top + width = $bounds.Width + height = $bounds.Height + } + }) | Out-Null + } + finally { + $scaledGraphics.Dispose() + $scaledBitmap.Dispose() + $graphics.Dispose() + $sourceBitmap.Dispose() + } + + $index += 1 + Start-Sleep -Milliseconds ${SCREEN_FRAME_INTERVAL_MS} +} + +($frames | ConvertTo-Json -Depth 6) | Set-Content -Path (Join-Path $framesDir 'frames.json') -Encoding UTF8 +`; +} + +function waitForReady(events) { + return new Promise((resolve, reject) => { + const startedAt = Date.now(); + const timer = setInterval(() => { + if (events.some((event) => event.type === "ready")) { + clearInterval(timer); + resolve(); + return; + } + + if (Date.now() - startedAt > READY_TIMEOUT_MS) { + clearInterval(timer); + reject(new Error("Timed out waiting for cursor sampler readiness.")); + } + }, 25); + }); +} + +function writeAssets(assets, outputDir) { + const assetDir = path.join(outputDir, "assets"); + fs.mkdirSync(assetDir, { recursive: true }); + + for (const asset of assets.values()) { + const base64 = asset.imageDataUrl?.replace(/^data:image\/png;base64,/, ""); + if (!base64) { + continue; + } + + const safeId = String(asset.id).replace(/[^a-zA-Z0-9_-]/g, "_"); + fs.writeFileSync(path.join(assetDir, `${safeId}.png`), Buffer.from(base64, "base64")); + } +} + +function toRecordingData(samples, assets) { + const firstTimestampMs = samples[0]?.timestampMs ?? Date.now(); + const normalizedSamples = samples.flatMap((sample) => { + const bounds = sample.bounds; + if (!bounds || bounds.width <= 0 || bounds.height <= 0) { + return []; + } + + return [ + { + timeMs: Math.max(0, sample.timestampMs - firstTimestampMs), + cx: (sample.x - bounds.x) / bounds.width, + cy: (sample.y - bounds.y) / bounds.height, + assetId: sample.handle, + visible: Boolean(sample.visible), + cursorType: sample.cursorType ?? null, + }, + ]; + }); + + return { + version: 2, + provider: assets.size > 0 ? "native" : "none", + samples: normalizedSamples, + assets: [...assets.values()].map((asset) => ({ + id: asset.id, + platform: "win32", + imageDataUrl: asset.imageDataUrl, + width: asset.width, + height: asset.height, + hotspotX: asset.hotspotX, + hotspotY: asset.hotspotY, + scaleFactor: 1, + cursorType: asset.cursorType ?? null, + })), + }; +} + +function escapeScriptJson(value) { + return JSON.stringify(value).replace(/ + + + + +OpenScreen native cursor diagnostic + + + +
+

OpenScreen native cursor diagnostic

+
+
${report.sampleCount}samples
+
${report.assetCount}assets
+
${report.uniquePositionCount}positions
+
${report.errorCount}errors
+
+

The red cross is the captured native hotspot. Native bitmaps are drawn at 1x, 2x, and 3x. The last cursor is a crisp vector 3x replacement anchored on the same hotspot.

+ +
+
+ + + +`; +} + +function readScreenFrames(outputDir, recordingStartTimestampMs) { + const framesJsonPath = path.join(outputDir, "screen-frames", "frames.json"); + if (!fs.existsSync(framesJsonPath)) { + return []; + } + + const rawFrames = JSON.parse(fs.readFileSync(framesJsonPath, "utf8").replace(/^\uFEFF/, "")); + const frames = Array.isArray(rawFrames) ? rawFrames : [rawFrames]; + + return frames + .filter((frame) => frame?.path && fs.existsSync(frame.path)) + .map((frame) => ({ + ...frame, + timeMs: Math.max(0, frame.timestampMs - recordingStartTimestampMs), + imageDataUrl: `data:image/png;base64,${fs.readFileSync(frame.path).toString("base64")}`, + })); +} + +function buildRealCaptureHtml(report, recordingData, screenFrames) { + return ` + + + + +OpenScreen native cursor real capture diagnostic + + + +
+

Real screen capture + reconstructed native cursor

+

Background frames are real Windows screenshots. Native bitmaps are reconstructed at 1x, 2x, and 3x; the last cursor is a crisp vector 3x replacement. The red cross marks the recorded hotspot.

+ +
+ + + + +`; +} + +function findPlaywrightChromiumExecutable(defaultPath) { + if (fs.existsSync(defaultPath)) { + return defaultPath; + } + + const baseDir = path.join(process.env.LOCALAPPDATA ?? "", "ms-playwright"); + if (!baseDir || !fs.existsSync(baseDir)) { + return defaultPath; + } + + const candidates = fs + .readdirSync(baseDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && entry.name.startsWith("chromium-")) + .map((entry) => path.join(baseDir, entry.name, "chrome-win64", "chrome.exe")) + .filter((candidate) => fs.existsSync(candidate)) + .sort() + .reverse(); + + return candidates[0] ?? defaultPath; +} + +async function writePreviewVideo(reportPath, outputPath) { + const { chromium } = await import("playwright"); + const browser = await chromium.launch({ + executablePath: findPlaywrightChromiumExecutable(chromium.executablePath()), + headless: true, + }); + try { + const page = await browser.newPage({ viewport: { width: 1180, height: 760 } }); + await page.goto(`file:///${reportPath.replaceAll("\\", "/")}`); + const base64 = await page.evaluate(() => window.__exportWebm()); + fs.writeFileSync(outputPath, Buffer.from(base64, "base64")); + } finally { + await browser.close(); + } +} + +function assertReport(report) { + const failures = []; + if (report.sampleCount < Math.floor(DURATION_MS / SAMPLE_INTERVAL_MS / 3)) { + failures.push(`Too few samples: ${report.sampleCount}.`); + } + if (report.visibleSampleCount === 0) { + failures.push("No visible cursor samples were captured."); + } + if (report.assetCount === 0) { + failures.push("No cursor asset PNG was captured."); + } + if (report.uniquePositionCount < 4) { + failures.push(`Cursor movement was not observed enough times: ${report.uniquePositionCount}.`); + } + if (report.errorCount > 0) { + failures.push(`Sampler reported ${report.errorCount} error event(s).`); + } + + if (failures.length > 0) { + throw new Error(failures.join(" ")); + } +} + +fs.mkdirSync(OUTPUT_DIR, { recursive: true }); + +const events = []; +const assets = new Map(); +let lineBuffer = ""; +let stoppingSampler = false; +const sampler = spawnPowerShell(buildSamplerScript(), { + onStdout: (chunk) => { + lineBuffer += chunk; + const lines = lineBuffer.split(/\r?\n/); + lineBuffer = lines.pop() ?? ""; + + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed) { + continue; + } + + const event = JSON.parse(trimmed); + events.push(event); + if (event.asset?.id && !assets.has(event.asset.id)) { + assets.set(event.asset.id, event.asset); + } + } + }, + onStderr: (chunk) => { + if (!stoppingSampler && !chunk.startsWith("#< CLIXML")) { + process.stderr.write(`[cursor-native-test] ${chunk}`); + } + }, +}); +let screenRecorder = null; + +try { + await waitForReady(events); + screenRecorder = spawnPowerShell(buildScreenRecorderScript(OUTPUT_DIR, DURATION_MS), { + onStderr: (chunk) => { + if (!chunk.startsWith("#< CLIXML") && !chunk.startsWith(" setTimeout(resolve, 150)); + await runPowerShell(buildMousePathScript(DURATION_MS)); + await new Promise((resolve) => setTimeout(resolve, Math.max(250, SAMPLE_INTERVAL_MS * 3))); + await screenRecorder.done; +} finally { + if (!sampler.child.killed) { + stoppingSampler = true; + sampler.child.kill(); + } + if (screenRecorder && !screenRecorder.child.killed) { + screenRecorder.child.kill(); + } +} + +const samples = events.filter((event) => event.type === "sample"); +const errors = events.filter((event) => event.type === "error"); +const recordingStartTimestampMs = samples[0]?.timestampMs ?? Date.now(); +const uniquePositions = new Set(samples.map((sample) => `${sample.x},${sample.y}`)); +const report = { + outputDir: OUTPUT_DIR, + sampleIntervalMs: SAMPLE_INTERVAL_MS, + durationMs: DURATION_MS, + eventCount: events.length, + sampleCount: samples.length, + visibleSampleCount: samples.filter((sample) => sample.visible).length, + assetCount: assets.size, + uniqueCursorHandleCount: new Set(samples.map((sample) => sample.handle).filter(Boolean)).size, + uniquePositionCount: uniquePositions.size, + errorCount: errors.length, + firstSample: samples[0] ?? null, + lastSample: samples.at(-1) ?? null, + assets: [...assets.values()].map((asset) => ({ + id: asset.id, + width: asset.width, + height: asset.height, + hotspotX: asset.hotspotX, + hotspotY: asset.hotspotY, + })), +}; +const recordingData = toRecordingData(samples, assets); +const screenFrames = readScreenFrames(OUTPUT_DIR, recordingStartTimestampMs); +const reportHtmlPath = path.join(OUTPUT_DIR, "report.html"); +const previewVideoPath = path.join(OUTPUT_DIR, "preview.webm"); +const realCaptureHtmlPath = path.join(OUTPUT_DIR, "real-capture-report.html"); +const realCaptureVideoPath = path.join(OUTPUT_DIR, "real-capture-preview.webm"); + +writeAssets(assets, OUTPUT_DIR); +fs.writeFileSync(path.join(OUTPUT_DIR, "events.json"), JSON.stringify(events, null, 2)); +fs.writeFileSync( + path.join(OUTPUT_DIR, "cursor-recording-data.json"), + JSON.stringify(recordingData, null, 2), +); +fs.writeFileSync(path.join(OUTPUT_DIR, "report.json"), JSON.stringify(report, null, 2)); +fs.writeFileSync(reportHtmlPath, buildVisualReportHtml(report, recordingData)); +if (screenFrames.length > 0) { + fs.writeFileSync(realCaptureHtmlPath, buildRealCaptureHtml(report, recordingData, screenFrames)); + report.screenFrameCount = screenFrames.length; +} + +try { + await writePreviewVideo(reportHtmlPath, previewVideoPath); + report.previewVideoPath = previewVideoPath; +} catch (error) { + report.previewVideoError = error instanceof Error ? error.message : String(error); +} + +if (screenFrames.length > 0) { + try { + await writePreviewVideo(realCaptureHtmlPath, realCaptureVideoPath); + report.realCaptureVideoPath = realCaptureVideoPath; + } catch (error) { + report.realCaptureVideoError = error instanceof Error ? error.message : String(error); + } +} + +fs.writeFileSync(path.join(OUTPUT_DIR, "report.json"), JSON.stringify(report, null, 2)); + +assertReport(report); + +console.log(JSON.stringify(report, null, 2)); diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx index d7b3836..0586e54 100644 --- a/src/components/video-editor/VideoPlayback.tsx +++ b/src/components/video-editor/VideoPlayback.tsx @@ -26,10 +26,10 @@ import { type WebcamSizePreset, } from "@/lib/compositeLayout"; import { - getNativeCursorDisplayMetrics, hasNativeCursorRecordingData, projectNativeCursorToStage, resolveInterpolatedNativeCursorFrame, + resolveNativeCursorRenderAsset, } from "@/lib/cursor/nativeCursor"; import { classifyWallpaper, DEFAULT_WALLPAPER, resolveImageWallpaperUrl } from "@/lib/wallpaper"; import { getCssClipPath } from "@/lib/webcamMaskShapes"; @@ -1324,19 +1324,20 @@ const VideoPlayback = forwardRef( sample: frame.sample, }); if (projectedPoint) { - const metrics = getNativeCursorDisplayMetrics( + const renderAsset = resolveNativeCursorRenderAsset( frame.asset, window.devicePixelRatio || 1, + frame.sample, ); const scale = Math.max(0, cursorSizeRef.current); - if (nativeCursorImg.dataset.cursorId !== frame.asset.id) { - nativeCursorImg.src = frame.asset.imageDataUrl; - nativeCursorImg.dataset.cursorId = frame.asset.id; + if (nativeCursorImg.dataset.cursorId !== renderAsset.id) { + nativeCursorImg.src = renderAsset.imageDataUrl; + nativeCursorImg.dataset.cursorId = renderAsset.id; } - nativeCursorImg.style.left = `${projectedPoint.x - metrics.hotspotX * scale}px`; - nativeCursorImg.style.top = `${projectedPoint.y - metrics.hotspotY * scale}px`; - nativeCursorImg.style.width = `${metrics.width * scale}px`; - nativeCursorImg.style.height = `${metrics.height * scale}px`; + nativeCursorImg.style.left = `${projectedPoint.x - renderAsset.hotspotX * scale}px`; + nativeCursorImg.style.top = `${projectedPoint.y - renderAsset.hotspotY * scale}px`; + nativeCursorImg.style.width = `${renderAsset.width * scale}px`; + nativeCursorImg.style.height = `${renderAsset.height * scale}px`; nativeCursorImg.style.display = "block"; } else { nativeCursorImg.style.display = "none"; diff --git a/src/lib/cursor/nativeCursor.ts b/src/lib/cursor/nativeCursor.ts index b32bd9e..d6ae220 100644 --- a/src/lib/cursor/nativeCursor.ts +++ b/src/lib/cursor/nativeCursor.ts @@ -1,9 +1,20 @@ import { type Container, Point } from "pixi.js"; +import crosshairUrl from "@/assets/cursors/Cursor=Cross.svg"; +import arrowUrl from "@/assets/cursors/Cursor=Default.svg"; +import pointerUrl from "@/assets/cursors/Cursor=Hand-(Pointing).svg"; +import notAllowedUrl from "@/assets/cursors/Cursor=Menu.svg"; +import moveUrl from "@/assets/cursors/Cursor=Move.svg"; +import resizeNeswUrl from "@/assets/cursors/Cursor=Resize-North-East-South-West.svg"; +import resizeNsUrl from "@/assets/cursors/Cursor=Resize-North-South.svg"; +import resizeNwseUrl from "@/assets/cursors/Cursor=Resize-North-West-South-East.svg"; +import resizeEwUrl from "@/assets/cursors/Cursor=Resize-West-East.svg"; +import textUrl from "@/assets/cursors/Cursor=Text-Cursor.svg"; import type { CropRegion } from "@/components/video-editor/types"; import type { CursorRecordingData, CursorRecordingSample, NativeCursorAsset, + NativeCursorType, } from "@/native/contracts"; export interface ActiveNativeCursorFrame { @@ -23,6 +34,87 @@ function clamp(value: number, min: number, max: number) { return Math.min(max, Math.max(min, value)); } +interface PrettyNativeCursorAsset { + imageDataUrl: string; + width: number; + height: number; + hotspotX: number; + hotspotY: number; +} + +const PRETTY_NATIVE_CURSOR_ASSETS: Partial> = { + arrow: { + imageDataUrl: arrowUrl, + width: 32, + height: 32, + hotspotX: 5.8, + hotspotY: 3.2, + }, + text: { + imageDataUrl: textUrl, + width: 32, + height: 32, + hotspotX: 16, + hotspotY: 16, + }, + pointer: { + imageDataUrl: pointerUrl, + width: 32, + height: 32, + hotspotX: 11.8, + hotspotY: 2.6, + }, + crosshair: { + imageDataUrl: crosshairUrl, + width: 32, + height: 32, + hotspotX: 16, + hotspotY: 16, + }, + "resize-ew": { + imageDataUrl: resizeEwUrl, + width: 32, + height: 32, + hotspotX: 16, + hotspotY: 16, + }, + "resize-ns": { + imageDataUrl: resizeNsUrl, + width: 32, + height: 32, + hotspotX: 16, + hotspotY: 16, + }, + "resize-nesw": { + imageDataUrl: resizeNeswUrl, + width: 32, + height: 32, + hotspotX: 16, + hotspotY: 16, + }, + "resize-nwse": { + imageDataUrl: resizeNwseUrl, + width: 32, + height: 32, + hotspotX: 16, + hotspotY: 16, + }, + move: { + imageDataUrl: moveUrl, + width: 32, + height: 32, + hotspotX: 16, + hotspotY: 16, + }, + "not-allowed": { + imageDataUrl: notAllowedUrl, + width: 32, + height: 32, + hotspotX: 16, + hotspotY: 16, + }, +}; + export function hasNativeCursorRecordingData( recordingData: CursorRecordingData | null | undefined, ): recordingData is CursorRecordingData { @@ -169,3 +261,39 @@ export function getNativeCursorDisplayMetrics(asset: NativeCursorAsset, deviceSc hotspotY: asset.hotspotY / scaleFactor, }; } + +export function resolvePrettyNativeCursorAsset( + asset: NativeCursorAsset, + sample?: CursorRecordingSample, +) { + const cursorType = sample?.cursorType ?? asset.cursorType ?? null; + return cursorType ? (PRETTY_NATIVE_CURSOR_ASSETS[cursorType] ?? null) : null; +} + +export function resolveNativeCursorRenderAsset( + asset: NativeCursorAsset, + deviceScaleFactor: number, + sample?: CursorRecordingSample, +) { + const prettyAsset = resolvePrettyNativeCursorAsset(asset, sample); + if (prettyAsset) { + return { + id: `pretty:${sample?.cursorType ?? asset.cursorType}`, + imageDataUrl: prettyAsset.imageDataUrl, + width: prettyAsset.width, + height: prettyAsset.height, + hotspotX: prettyAsset.hotspotX, + hotspotY: prettyAsset.hotspotY, + }; + } + + const metrics = getNativeCursorDisplayMetrics(asset, deviceScaleFactor); + return { + id: asset.id, + imageDataUrl: asset.imageDataUrl, + width: metrics.width, + height: metrics.height, + hotspotX: metrics.hotspotX, + hotspotY: metrics.hotspotY, + }; +} diff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.ts index c43908f..a1e20cc 100644 --- a/src/lib/exporter/frameRenderer.ts +++ b/src/lib/exporter/frameRenderer.ts @@ -57,13 +57,13 @@ import { type StyledRenderRect, } from "@/lib/compositeLayout"; import { - getNativeCursorDisplayMetrics, projectNativeCursorToStage, resolveInterpolatedNativeCursorFrame, + resolveNativeCursorRenderAsset, } from "@/lib/cursor/nativeCursor"; import { BackgroundLoadError, classifyWallpaper, resolveImageWallpaperUrl } from "@/lib/wallpaper"; import { drawCanvasClipPath } from "@/lib/webcamMaskShapes"; -import type { CursorRecordingData, NativeCursorAsset } from "@/native/contracts"; +import type { CursorRecordingData } from "@/native/contracts"; import { renderAnnotations } from "./annotationRenderer"; import { getLinearGradientPoints, @@ -585,19 +585,23 @@ export class FrameRenderer { return; } - const image = await this.getCursorImage(activeNativeCursor.asset); - const metrics = getNativeCursorDisplayMetrics(activeNativeCursor.asset, 1); + const renderAsset = resolveNativeCursorRenderAsset( + activeNativeCursor.asset, + 1, + activeNativeCursor.sample, + ); + const image = await this.getCursorImage(renderAsset); const scale = Math.max(0, this.config.cursorScale ?? 1); this.compositeCtx.drawImage( image, - projectedPoint.x - metrics.hotspotX * scale, - projectedPoint.y - metrics.hotspotY * scale, - metrics.width * scale, - metrics.height * scale, + projectedPoint.x - renderAsset.hotspotX * scale, + projectedPoint.y - renderAsset.hotspotY * scale, + renderAsset.width * scale, + renderAsset.height * scale, ); } - private async getCursorImage(asset: NativeCursorAsset) { + private async getCursorImage(asset: { id: string; imageDataUrl: string }) { const cachedImage = this.cursorImageCache.get(asset.id); if (cachedImage) { return cachedImage; diff --git a/src/native/contracts.ts b/src/native/contracts.ts index 73d53db..a3c9087 100644 --- a/src/native/contracts.ts +++ b/src/native/contracts.ts @@ -3,6 +3,21 @@ export const NATIVE_BRIDGE_VERSION = 1; export type NativePlatform = "darwin" | "win32" | "linux"; export type CursorProviderKind = "native" | "none"; +export type NativeCursorType = + | "arrow" + | "text" + | "pointer" + | "crosshair" + | "resize-ew" + | "resize-ns" + | "resize-nesw" + | "resize-nwse" + | "move" + | "not-allowed" + | "wait" + | "app-starting" + | "help" + | "up-arrow"; export interface CursorTelemetryPoint { timeMs: number; @@ -13,6 +28,7 @@ export interface CursorTelemetryPoint { export interface CursorRecordingSample extends CursorTelemetryPoint { assetId?: string | null; visible?: boolean; + cursorType?: NativeCursorType | null; } export interface NativeCursorAsset { @@ -24,6 +40,7 @@ export interface NativeCursorAsset { hotspotX: number; hotspotY: number; scaleFactor?: number; + cursorType?: NativeCursorType | null; } export interface CursorRecordingData {