feat: add cursor overlay pipeline for high-fidelity cursor recording and playback
- Implement native bridge for Windows cursor capture via PowerShell/C# - Add cursor-free capture using getDisplayMedia with setDisplayMediaRequestHandler - Update video player and exporters to support native cursor telemetry - Enable system audio capture on Windows via WASAPI loopback - Add interpolation for smoother cursor movement in playback and export - Improve cursor scaling and visibility handling in editor and playback
This commit is contained in:
Etienne Lescot
committed by
EtienneLescot
EtienneLescot
parent
248ebabcf1
commit
e9650225ba
+89
-34
@@ -4,8 +4,6 @@ import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
const nodeRequire = createRequire(import.meta.url);
|
||||
|
||||
import {
|
||||
app,
|
||||
BrowserWindow,
|
||||
@@ -16,10 +14,7 @@ import {
|
||||
shell,
|
||||
systemPreferences,
|
||||
} from "electron";
|
||||
import {
|
||||
type CursorTelemetryPoint,
|
||||
createCursorTelemetryBuffer,
|
||||
} from "../../src/lib/cursorTelemetryBuffer";
|
||||
import type { DesktopCapturerSource } from "electron";
|
||||
import {
|
||||
normalizeProjectMedia,
|
||||
normalizeRecordingSession,
|
||||
@@ -198,11 +193,24 @@ async function getApprovedProjectSession(
|
||||
|
||||
type SelectedSource = {
|
||||
name: string;
|
||||
id?: string;
|
||||
display_id?: string;
|
||||
[key: string]: unknown;
|
||||
};
|
||||
|
||||
let selectedSource: SelectedSource | null = null;
|
||||
let selectedDesktopSource: DesktopCapturerSource | null = null;
|
||||
let lastEnumeratedSources = new Map<string, DesktopCapturerSource>();
|
||||
let currentProjectPath: string | null = null;
|
||||
let currentRecordingSession: RecordingSession | null = null;
|
||||
|
||||
/**
|
||||
* Returns the cached DesktopCapturerSource set when the user picked a source.
|
||||
* Used by setDisplayMediaRequestHandler in main.ts for cursor-free capture.
|
||||
*/
|
||||
export function getSelectedDesktopSource(): DesktopCapturerSource | null {
|
||||
return selectedDesktopSource;
|
||||
}
|
||||
let currentVideoPath: string | null = null;
|
||||
|
||||
function normalizePath(filePath: string) {
|
||||
@@ -238,16 +246,12 @@ function isTrustedProjectPath(filePath?: string | null) {
|
||||
}
|
||||
|
||||
const CURSOR_TELEMETRY_VERSION = 2;
|
||||
const CURSOR_SAMPLE_INTERVAL_MS = 100;
|
||||
const MAX_CURSOR_SAMPLES = 60 * 60 * 10; // 1 hour @ 10Hz
|
||||
const CURSOR_SAMPLE_INTERVAL_MS = 33;
|
||||
const MAX_CURSOR_SAMPLES = 60 * 60 * 30; // 1 hour @ 30Hz
|
||||
|
||||
let cursorRecordingSession: CursorRecordingSession | null = null;
|
||||
let pendingCursorRecordingData: CursorRecordingData | null = null;
|
||||
|
||||
function clamp(value: number, min: number, max: number) {
|
||||
return Math.min(max, Math.max(min, value));
|
||||
}
|
||||
|
||||
function normalizeCursorSample(sample: unknown): CursorRecordingSample | null {
|
||||
if (!sample || typeof sample !== "object") {
|
||||
return null;
|
||||
@@ -259,8 +263,8 @@ function normalizeCursorSample(sample: unknown): CursorRecordingSample | null {
|
||||
typeof point.timeMs === "number" && Number.isFinite(point.timeMs)
|
||||
? Math.max(0, point.timeMs)
|
||||
: 0,
|
||||
cx: typeof point.cx === "number" && Number.isFinite(point.cx) ? clamp(point.cx, 0, 1) : 0.5,
|
||||
cy: typeof point.cy === "number" && Number.isFinite(point.cy) ? clamp(point.cy, 0, 1) : 0.5,
|
||||
cx: typeof point.cx === "number" && Number.isFinite(point.cx) ? point.cx : 0.5,
|
||||
cy: typeof point.cy === "number" && Number.isFinite(point.cy) ? point.cy : 0.5,
|
||||
assetId: typeof point.assetId === "string" ? point.assetId : null,
|
||||
visible: typeof point.visible === "boolean" ? point.visible : true,
|
||||
};
|
||||
@@ -395,6 +399,55 @@ function getSelectedSourceBounds() {
|
||||
return (sourceDisplay ?? screen.getDisplayNearestPoint(cursor)).bounds;
|
||||
}
|
||||
|
||||
function getSelectedSourceId() {
|
||||
return typeof selectedSource?.id === "string" ? selectedSource.id : null;
|
||||
}
|
||||
|
||||
function setCurrentRecordingSessionState(session: RecordingSession | null) {
|
||||
currentRecordingSession = session;
|
||||
currentVideoPath = session?.screenVideoPath ?? null;
|
||||
}
|
||||
|
||||
async function storeRecordedSessionFiles(payload: StoreRecordedSessionInput) {
|
||||
const createdAt =
|
||||
typeof payload.createdAt === "number" && Number.isFinite(payload.createdAt)
|
||||
? payload.createdAt
|
||||
: Date.now();
|
||||
const screenVideoPath = resolveRecordingOutputPath(payload.screen.fileName);
|
||||
await fs.writeFile(screenVideoPath, Buffer.from(payload.screen.videoData));
|
||||
|
||||
let webcamVideoPath: string | undefined;
|
||||
if (payload.webcam) {
|
||||
webcamVideoPath = resolveRecordingOutputPath(payload.webcam.fileName);
|
||||
await fs.writeFile(webcamVideoPath, Buffer.from(payload.webcam.videoData));
|
||||
}
|
||||
|
||||
const session: RecordingSession = webcamVideoPath
|
||||
? { screenVideoPath, webcamVideoPath, createdAt }
|
||||
: { screenVideoPath, createdAt };
|
||||
setCurrentRecordingSessionState(session);
|
||||
currentProjectPath = null;
|
||||
|
||||
const telemetryPath = `${screenVideoPath}.cursor.json`;
|
||||
if (pendingCursorRecordingData && pendingCursorRecordingData.samples.length > 0) {
|
||||
await fs.writeFile(telemetryPath, JSON.stringify(pendingCursorRecordingData, null, 2), "utf-8");
|
||||
}
|
||||
pendingCursorRecordingData = null;
|
||||
|
||||
const sessionManifestPath = path.join(
|
||||
RECORDINGS_DIR,
|
||||
`${path.parse(payload.screen.fileName).name}${RECORDING_SESSION_SUFFIX}`,
|
||||
);
|
||||
await fs.writeFile(sessionManifestPath, JSON.stringify(session, null, 2), "utf-8");
|
||||
|
||||
return {
|
||||
success: true,
|
||||
path: screenVideoPath,
|
||||
session,
|
||||
message: "Recording session stored successfully",
|
||||
};
|
||||
}
|
||||
|
||||
export function registerIpcHandlers(
|
||||
createEditorWindow: () => void,
|
||||
createSourceSelectorWindow: () => BrowserWindow,
|
||||
@@ -404,6 +457,7 @@ export function registerIpcHandlers(
|
||||
) {
|
||||
ipcMain.handle("get-sources", async (_, opts) => {
|
||||
const sources = await desktopCapturer.getSources(opts);
|
||||
lastEnumeratedSources = new Map(sources.map((source) => [source.id, source]));
|
||||
return sources.map((source) => ({
|
||||
id: source.id,
|
||||
name: source.name,
|
||||
@@ -413,8 +467,26 @@ export function registerIpcHandlers(
|
||||
}));
|
||||
});
|
||||
|
||||
ipcMain.handle("select-source", (_, source: SelectedSource) => {
|
||||
ipcMain.handle("select-source", async (_, source: SelectedSource) => {
|
||||
selectedSource = source;
|
||||
// Reuse the exact source object returned during enumeration to avoid
|
||||
// Windows window-source id mismatches across separate getSources() calls.
|
||||
selectedDesktopSource =
|
||||
typeof source.id === "string" ? lastEnumeratedSources.get(source.id) ?? null : null;
|
||||
|
||||
if (!selectedDesktopSource && typeof source.id === "string") {
|
||||
try {
|
||||
const sources = await desktopCapturer.getSources({
|
||||
types: ["screen", "window"],
|
||||
thumbnailSize: { width: 0, height: 0 },
|
||||
fetchWindowIcons: true,
|
||||
});
|
||||
lastEnumeratedSources = new Map(sources.map((candidate) => [candidate.id, candidate]));
|
||||
selectedDesktopSource = lastEnumeratedSources.get(source.id) ?? null;
|
||||
} catch {
|
||||
selectedDesktopSource = null;
|
||||
}
|
||||
}
|
||||
const sourceSelectorWin = getSourceSelectorWindow();
|
||||
if (sourceSelectorWin) {
|
||||
sourceSelectorWin.close();
|
||||
@@ -519,25 +591,7 @@ export function registerIpcHandlers(
|
||||
|
||||
ipcMain.handle("store-recorded-session", async (_, payload: StoreRecordedSessionInput) => {
|
||||
try {
|
||||
const videoPath = path.join(RECORDINGS_DIR, fileName);
|
||||
await fs.writeFile(videoPath, Buffer.from(videoData));
|
||||
currentProjectPath = null;
|
||||
|
||||
const telemetryPath = `${videoPath}.cursor.json`;
|
||||
if (pendingCursorRecordingData && pendingCursorRecordingData.samples.length > 0) {
|
||||
await fs.writeFile(
|
||||
telemetryPath,
|
||||
JSON.stringify(pendingCursorRecordingData, null, 2),
|
||||
"utf-8",
|
||||
);
|
||||
}
|
||||
pendingCursorRecordingData = null;
|
||||
|
||||
return {
|
||||
success: true,
|
||||
path: videoPath,
|
||||
message: "Video stored successfully",
|
||||
};
|
||||
return await storeRecordedSessionFiles(payload);
|
||||
} catch (error) {
|
||||
console.error("Failed to store recording session:", error);
|
||||
return {
|
||||
@@ -602,6 +656,7 @@ export function registerIpcHandlers(
|
||||
maxSamples: MAX_CURSOR_SAMPLES,
|
||||
platform: process.platform,
|
||||
sampleIntervalMs: CURSOR_SAMPLE_INTERVAL_MS,
|
||||
sourceId: getSelectedSourceId(),
|
||||
});
|
||||
|
||||
try {
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@ import {
|
||||
Tray,
|
||||
} from "electron";
|
||||
import { mainT, setMainLocale } from "./i18n";
|
||||
import { registerIpcHandlers } from "./ipc/handlers";
|
||||
import { getSelectedDesktopSource, registerIpcHandlers } from "./ipc/handlers";
|
||||
import {
|
||||
createCountdownOverlayWindow,
|
||||
createEditorWindow,
|
||||
|
||||
@@ -8,6 +8,7 @@ interface CreateCursorRecordingSessionOptions {
|
||||
maxSamples: number;
|
||||
platform: NodeJS.Platform;
|
||||
sampleIntervalMs: number;
|
||||
sourceId?: string | null;
|
||||
}
|
||||
|
||||
export function createCursorRecordingSession(
|
||||
@@ -18,6 +19,7 @@ export function createCursorRecordingSession(
|
||||
getDisplayBounds: options.getDisplayBounds,
|
||||
maxSamples: options.maxSamples,
|
||||
sampleIntervalMs: options.sampleIntervalMs,
|
||||
sourceId: options.sourceId,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
export function parseWindowHandleFromSourceId(sourceId?: string | null) {
|
||||
if (!sourceId?.startsWith("window:")) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const handlePart = sourceId.split(":")[1];
|
||||
if (!handlePart || !/^\d+$/.test(handlePart)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return handlePart;
|
||||
}
|
||||
|
||||
export function buildPowerShellCommand(sampleIntervalMs: number, windowHandle?: string | null) {
|
||||
const script = String.raw`
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Add-Type -AssemblyName System.Drawing
|
||||
|
||||
$targetWindowHandle = ${windowHandle ? `'${windowHandle}'` : '$null'}
|
||||
|
||||
$source = @"
|
||||
using System;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
public static class OpenScreenCursorInterop {
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
public struct POINT {
|
||||
public int X;
|
||||
public int Y;
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
public struct CURSORINFO {
|
||||
public int cbSize;
|
||||
public int flags;
|
||||
public IntPtr hCursor;
|
||||
public POINT ptScreenPos;
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
public struct ICONINFO {
|
||||
[MarshalAs(UnmanagedType.Bool)]
|
||||
public bool fIcon;
|
||||
public int xHotspot;
|
||||
public int yHotspot;
|
||||
public IntPtr hbmMask;
|
||||
public IntPtr hbmColor;
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
public struct RECT {
|
||||
public int Left;
|
||||
public int Top;
|
||||
public int Right;
|
||||
public int Bottom;
|
||||
}
|
||||
|
||||
[DllImport("user32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
public static extern bool GetCursorInfo(ref CURSORINFO pci);
|
||||
|
||||
[DllImport("user32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
public static extern bool GetWindowRect(IntPtr hWnd, out RECT lpRect);
|
||||
|
||||
[DllImport("user32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
public static extern bool IsWindow(IntPtr hWnd);
|
||||
|
||||
[DllImport("user32.dll", SetLastError = true)]
|
||||
public static extern IntPtr CopyIcon(IntPtr hIcon);
|
||||
|
||||
[DllImport("user32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
public static extern bool DestroyIcon(IntPtr hIcon);
|
||||
|
||||
[DllImport("user32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
public static extern bool GetIconInfo(IntPtr hIcon, out ICONINFO piconinfo);
|
||||
|
||||
[DllImport("gdi32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
public static extern bool DeleteObject(IntPtr hObject);
|
||||
}
|
||||
"@
|
||||
|
||||
Add-Type -TypeDefinition $source
|
||||
|
||||
function Write-JsonLine($payload) {
|
||||
[Console]::Out.WriteLine(($payload | ConvertTo-Json -Compress -Depth 6))
|
||||
}
|
||||
|
||||
function Get-TargetBounds() {
|
||||
if ([string]::IsNullOrWhiteSpace($targetWindowHandle)) {
|
||||
return $null
|
||||
}
|
||||
|
||||
try {
|
||||
$handleValue = [int64]::Parse($targetWindowHandle)
|
||||
$windowHandle = [IntPtr]::new($handleValue)
|
||||
if (-not [OpenScreenCursorInterop]::IsWindow($windowHandle)) {
|
||||
return $null
|
||||
}
|
||||
|
||||
$rect = New-Object OpenScreenCursorInterop+RECT
|
||||
if (-not [OpenScreenCursorInterop]::GetWindowRect($windowHandle, [ref]$rect)) {
|
||||
return $null
|
||||
}
|
||||
|
||||
$width = $rect.Right - $rect.Left
|
||||
$height = $rect.Bottom - $rect.Top
|
||||
if ($width -le 0 -or $height -le 0) {
|
||||
return $null
|
||||
}
|
||||
|
||||
return @{
|
||||
x = $rect.Left
|
||||
y = $rect.Top
|
||||
width = $width
|
||||
height = $height
|
||||
}
|
||||
}
|
||||
catch {
|
||||
return $null
|
||||
}
|
||||
}
|
||||
|
||||
function Get-CursorAsset($cursorHandle, $cursorId) {
|
||||
$copiedHandle = [OpenScreenCursorInterop]::CopyIcon($cursorHandle)
|
||||
if ($copiedHandle -eq [IntPtr]::Zero) {
|
||||
return $null
|
||||
}
|
||||
|
||||
$iconInfo = New-Object OpenScreenCursorInterop+ICONINFO
|
||||
$hasIconInfo = [OpenScreenCursorInterop]::GetIconInfo($copiedHandle, [ref]$iconInfo)
|
||||
|
||||
try {
|
||||
$icon = [System.Drawing.Icon]::FromHandle($copiedHandle)
|
||||
$bitmap = New-Object System.Drawing.Bitmap $icon.Width, $icon.Height, ([System.Drawing.Imaging.PixelFormat]::Format32bppArgb)
|
||||
$graphics = [System.Drawing.Graphics]::FromImage($bitmap)
|
||||
$memoryStream = New-Object System.IO.MemoryStream
|
||||
|
||||
try {
|
||||
$graphics.Clear([System.Drawing.Color]::Transparent)
|
||||
$graphics.DrawIcon($icon, 0, 0)
|
||||
$bitmap.Save($memoryStream, [System.Drawing.Imaging.ImageFormat]::Png)
|
||||
$base64 = [System.Convert]::ToBase64String($memoryStream.ToArray())
|
||||
|
||||
return @{
|
||||
id = $cursorId
|
||||
imageDataUrl = "data:image/png;base64,$base64"
|
||||
width = $bitmap.Width
|
||||
height = $bitmap.Height
|
||||
hotspotX = if ($hasIconInfo) { $iconInfo.xHotspot } else { 0 }
|
||||
hotspotY = if ($hasIconInfo) { $iconInfo.yHotspot } else { 0 }
|
||||
}
|
||||
}
|
||||
finally {
|
||||
$memoryStream.Dispose()
|
||||
$graphics.Dispose()
|
||||
$bitmap.Dispose()
|
||||
$icon.Dispose()
|
||||
}
|
||||
}
|
||||
finally {
|
||||
if ($hasIconInfo) {
|
||||
if ($iconInfo.hbmMask -ne [IntPtr]::Zero) {
|
||||
[OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmMask) | Out-Null
|
||||
}
|
||||
if ($iconInfo.hbmColor -ne [IntPtr]::Zero) {
|
||||
[OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmColor) | Out-Null
|
||||
}
|
||||
}
|
||||
[OpenScreenCursorInterop]::DestroyIcon($copiedHandle) | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
Write-JsonLine @{ type = 'ready'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() }
|
||||
|
||||
$lastCursorId = $null
|
||||
while ($true) {
|
||||
$cursorInfo = New-Object OpenScreenCursorInterop+CURSORINFO
|
||||
$cursorInfo.cbSize = [Runtime.InteropServices.Marshal]::SizeOf([type][OpenScreenCursorInterop+CURSORINFO])
|
||||
|
||||
if (-not [OpenScreenCursorInterop]::GetCursorInfo([ref]$cursorInfo)) {
|
||||
Write-JsonLine @{ type = 'error'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds(); message = 'GetCursorInfo failed' }
|
||||
Start-Sleep -Milliseconds ${sampleIntervalMs}
|
||||
continue
|
||||
}
|
||||
|
||||
$visible = ($cursorInfo.flags -band 1) -ne 0
|
||||
$cursorId = if ($cursorInfo.hCursor -eq [IntPtr]::Zero) { $null } else { ('0x{0:X}' -f $cursorInfo.hCursor.ToInt64()) }
|
||||
$asset = $null
|
||||
|
||||
if ($visible -and $cursorId -and $cursorId -ne $lastCursorId) {
|
||||
$asset = Get-CursorAsset -cursorHandle $cursorInfo.hCursor -cursorId $cursorId
|
||||
$lastCursorId = $cursorId
|
||||
}
|
||||
|
||||
Write-JsonLine @{
|
||||
type = 'sample'
|
||||
timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds()
|
||||
x = $cursorInfo.ptScreenPos.X
|
||||
y = $cursorInfo.ptScreenPos.Y
|
||||
visible = $visible
|
||||
handle = $cursorId
|
||||
bounds = Get-TargetBounds
|
||||
asset = $asset
|
||||
}
|
||||
|
||||
Start-Sleep -Milliseconds ${sampleIntervalMs}
|
||||
}
|
||||
`;
|
||||
|
||||
return Buffer.from(script, "utf16le").toString("base64");
|
||||
}
|
||||
@@ -1,206 +1,23 @@
|
||||
import { type ChildProcessByStdio, spawn } from "node:child_process";
|
||||
import type { Readable } from "node:stream";
|
||||
import { type Rectangle, screen } from "electron";
|
||||
import { screen } from "electron";
|
||||
import type {
|
||||
CursorRecordingData,
|
||||
CursorRecordingSample,
|
||||
NativeCursorAsset,
|
||||
} from "../../../../src/native/contracts";
|
||||
import type { CursorRecordingSession } from "./session";
|
||||
import { buildPowerShellCommand, parseWindowHandleFromSourceId } from "./windowsNativeRecordingSession.script";
|
||||
import type {
|
||||
WindowsCursorEvent,
|
||||
WindowsNativeRecordingSessionOptions,
|
||||
} from "./windowsNativeRecordingSession.types";
|
||||
|
||||
interface WindowsCursorSampleEvent {
|
||||
type: "sample";
|
||||
timestampMs: number;
|
||||
x: number;
|
||||
y: number;
|
||||
visible: boolean;
|
||||
handle: string | null;
|
||||
asset?: WindowsCursorAssetPayload;
|
||||
}
|
||||
const READY_TIMEOUT_MS = 5_000;
|
||||
|
||||
interface WindowsCursorReadyEvent {
|
||||
type: "ready";
|
||||
timestampMs: number;
|
||||
}
|
||||
|
||||
interface WindowsCursorErrorEvent {
|
||||
type: "error";
|
||||
timestampMs: number;
|
||||
message: string;
|
||||
}
|
||||
|
||||
interface WindowsCursorAssetPayload {
|
||||
id: string;
|
||||
imageDataUrl: string;
|
||||
width: number;
|
||||
height: number;
|
||||
hotspotX: number;
|
||||
hotspotY: number;
|
||||
}
|
||||
|
||||
type WindowsCursorEvent =
|
||||
| WindowsCursorSampleEvent
|
||||
| WindowsCursorReadyEvent
|
||||
| WindowsCursorErrorEvent;
|
||||
|
||||
interface WindowsNativeRecordingSessionOptions {
|
||||
getDisplayBounds: () => Rectangle | null;
|
||||
maxSamples: number;
|
||||
sampleIntervalMs: number;
|
||||
}
|
||||
|
||||
function clamp(value: number, min: number, max: number) {
|
||||
return Math.min(max, Math.max(min, value));
|
||||
}
|
||||
|
||||
function buildPowerShellCommand(sampleIntervalMs: number) {
|
||||
const script = String.raw`
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Add-Type -AssemblyName System.Drawing
|
||||
|
||||
$source = @"
|
||||
using System;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
public static class OpenScreenCursorInterop {
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
public struct POINT {
|
||||
public int X;
|
||||
public int Y;
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
public struct CURSORINFO {
|
||||
public int cbSize;
|
||||
public int flags;
|
||||
public IntPtr hCursor;
|
||||
public POINT ptScreenPos;
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
public struct ICONINFO {
|
||||
[MarshalAs(UnmanagedType.Bool)]
|
||||
public bool fIcon;
|
||||
public int xHotspot;
|
||||
public int yHotspot;
|
||||
public IntPtr hbmMask;
|
||||
public IntPtr hbmColor;
|
||||
}
|
||||
|
||||
[DllImport("user32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
public static extern bool GetCursorInfo(ref CURSORINFO pci);
|
||||
|
||||
[DllImport("user32.dll", SetLastError = true)]
|
||||
public static extern IntPtr CopyIcon(IntPtr hIcon);
|
||||
|
||||
[DllImport("user32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
public static extern bool DestroyIcon(IntPtr hIcon);
|
||||
|
||||
[DllImport("user32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
public static extern bool GetIconInfo(IntPtr hIcon, out ICONINFO piconinfo);
|
||||
|
||||
[DllImport("gdi32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
public static extern bool DeleteObject(IntPtr hObject);
|
||||
}
|
||||
"@
|
||||
|
||||
Add-Type -TypeDefinition $source
|
||||
|
||||
function Write-JsonLine($payload) {
|
||||
[Console]::Out.WriteLine(($payload | ConvertTo-Json -Compress -Depth 6))
|
||||
}
|
||||
|
||||
function Get-CursorAsset($cursorHandle, $cursorId) {
|
||||
$copiedHandle = [OpenScreenCursorInterop]::CopyIcon($cursorHandle)
|
||||
if ($copiedHandle -eq [IntPtr]::Zero) {
|
||||
return $null
|
||||
}
|
||||
|
||||
$iconInfo = New-Object OpenScreenCursorInterop+ICONINFO
|
||||
$hasIconInfo = [OpenScreenCursorInterop]::GetIconInfo($copiedHandle, [ref]$iconInfo)
|
||||
|
||||
try {
|
||||
$icon = [System.Drawing.Icon]::FromHandle($copiedHandle)
|
||||
$bitmap = New-Object System.Drawing.Bitmap $icon.Width, $icon.Height, ([System.Drawing.Imaging.PixelFormat]::Format32bppArgb)
|
||||
$graphics = [System.Drawing.Graphics]::FromImage($bitmap)
|
||||
$memoryStream = New-Object System.IO.MemoryStream
|
||||
|
||||
try {
|
||||
$graphics.Clear([System.Drawing.Color]::Transparent)
|
||||
$graphics.DrawIcon($icon, 0, 0)
|
||||
$bitmap.Save($memoryStream, [System.Drawing.Imaging.ImageFormat]::Png)
|
||||
$base64 = [System.Convert]::ToBase64String($memoryStream.ToArray())
|
||||
|
||||
return @{
|
||||
id = $cursorId
|
||||
imageDataUrl = "data:image/png;base64,$base64"
|
||||
width = $bitmap.Width
|
||||
height = $bitmap.Height
|
||||
hotspotX = if ($hasIconInfo) { $iconInfo.xHotspot } else { 0 }
|
||||
hotspotY = if ($hasIconInfo) { $iconInfo.yHotspot } else { 0 }
|
||||
}
|
||||
}
|
||||
finally {
|
||||
$memoryStream.Dispose()
|
||||
$graphics.Dispose()
|
||||
$bitmap.Dispose()
|
||||
$icon.Dispose()
|
||||
}
|
||||
}
|
||||
finally {
|
||||
if ($hasIconInfo) {
|
||||
if ($iconInfo.hbmMask -ne [IntPtr]::Zero) {
|
||||
[OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmMask) | Out-Null
|
||||
}
|
||||
if ($iconInfo.hbmColor -ne [IntPtr]::Zero) {
|
||||
[OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmColor) | Out-Null
|
||||
}
|
||||
}
|
||||
[OpenScreenCursorInterop]::DestroyIcon($copiedHandle) | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
Write-JsonLine @{ type = 'ready'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() }
|
||||
|
||||
$lastCursorId = $null
|
||||
while ($true) {
|
||||
$cursorInfo = New-Object OpenScreenCursorInterop+CURSORINFO
|
||||
$cursorInfo.cbSize = [Runtime.InteropServices.Marshal]::SizeOf([type][OpenScreenCursorInterop+CURSORINFO])
|
||||
|
||||
if (-not [OpenScreenCursorInterop]::GetCursorInfo([ref]$cursorInfo)) {
|
||||
Write-JsonLine @{ type = 'error'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds(); message = 'GetCursorInfo failed' }
|
||||
Start-Sleep -Milliseconds ${sampleIntervalMs}
|
||||
continue
|
||||
}
|
||||
|
||||
$visible = ($cursorInfo.flags -band 1) -ne 0
|
||||
$cursorId = if ($cursorInfo.hCursor -eq [IntPtr]::Zero) { $null } else { ('0x{0:X}' -f $cursorInfo.hCursor.ToInt64()) }
|
||||
$asset = $null
|
||||
|
||||
if ($visible -and $cursorId -and $cursorId -ne $lastCursorId) {
|
||||
$asset = Get-CursorAsset -cursorHandle $cursorInfo.hCursor -cursorId $cursorId
|
||||
$lastCursorId = $cursorId
|
||||
}
|
||||
|
||||
Write-JsonLine @{
|
||||
type = 'sample'
|
||||
timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds()
|
||||
x = $cursorInfo.ptScreenPos.X
|
||||
y = $cursorInfo.ptScreenPos.Y
|
||||
visible = $visible
|
||||
handle = $cursorId
|
||||
asset = $asset
|
||||
}
|
||||
|
||||
Start-Sleep -Milliseconds ${sampleIntervalMs}
|
||||
}
|
||||
`;
|
||||
|
||||
return Buffer.from(script, "utf16le").toString("base64");
|
||||
interface NormalizedSample {
|
||||
sample: CursorRecordingSample;
|
||||
withinBounds: boolean;
|
||||
}
|
||||
|
||||
export class WindowsNativeRecordingSession implements CursorRecordingSession {
|
||||
@@ -209,6 +26,11 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession {
|
||||
private process: ChildProcessByStdio<null, Readable, Readable> | null = null;
|
||||
private lineBuffer = "";
|
||||
private startTimeMs = 0;
|
||||
private readyResolve: (() => void) | null = null;
|
||||
private readyReject: ((error: Error) => void) | null = null;
|
||||
private readyTimer: NodeJS.Timeout | null = null;
|
||||
private sampleCount = 0;
|
||||
private outOfBoundsSampleCount = 0;
|
||||
|
||||
constructor(private readonly options: WindowsNativeRecordingSessionOptions) {}
|
||||
|
||||
@@ -217,8 +39,13 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession {
|
||||
this.samples = [];
|
||||
this.lineBuffer = "";
|
||||
this.startTimeMs = Date.now();
|
||||
this.sampleCount = 0;
|
||||
this.outOfBoundsSampleCount = 0;
|
||||
|
||||
const encodedCommand = buildPowerShellCommand(this.options.sampleIntervalMs);
|
||||
const encodedCommand = buildPowerShellCommand(
|
||||
this.options.sampleIntervalMs,
|
||||
parseWindowHandleFromSourceId(this.options.sourceId),
|
||||
);
|
||||
const child = spawn(
|
||||
"powershell.exe",
|
||||
[
|
||||
@@ -237,24 +64,58 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession {
|
||||
);
|
||||
|
||||
this.process = child;
|
||||
this.logDiagnostic("spawn", {
|
||||
pid: child.pid ?? null,
|
||||
sampleIntervalMs: this.options.sampleIntervalMs,
|
||||
sourceId: this.options.sourceId ?? null,
|
||||
windowHandle: parseWindowHandleFromSourceId(this.options.sourceId),
|
||||
});
|
||||
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk: string) => {
|
||||
this.handleStdoutChunk(chunk);
|
||||
});
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stderr.on("data", (chunk: string) => {
|
||||
console.error("[cursor-native]", chunk.trim());
|
||||
const message = chunk.trim();
|
||||
if (message) {
|
||||
this.logDiagnostic("stderr", { message });
|
||||
}
|
||||
console.error("[cursor-native]", message);
|
||||
});
|
||||
child.once("exit", (code, signal) => {
|
||||
this.logDiagnostic("exit", {
|
||||
code,
|
||||
signal,
|
||||
sampleCount: this.sampleCount,
|
||||
assetCount: this.assets.size,
|
||||
outOfBoundsSampleCount: this.outOfBoundsSampleCount,
|
||||
});
|
||||
this.rejectReady(new Error(`Windows cursor helper exited before ready (code=${code}, signal=${signal})`));
|
||||
});
|
||||
child.once("error", (error) => {
|
||||
this.logDiagnostic("process-error", { message: error.message });
|
||||
this.rejectReady(error);
|
||||
});
|
||||
|
||||
await this.waitUntilReady();
|
||||
}
|
||||
|
||||
async stop(): Promise<CursorRecordingData> {
|
||||
const child = this.process;
|
||||
this.process = null;
|
||||
this.clearReadyState();
|
||||
|
||||
if (child && !child.killed) {
|
||||
child.kill();
|
||||
}
|
||||
|
||||
this.logDiagnostic("stop", {
|
||||
sampleCount: this.sampleCount,
|
||||
assetCount: this.assets.size,
|
||||
outOfBoundsSampleCount: this.outOfBoundsSampleCount,
|
||||
});
|
||||
|
||||
return {
|
||||
version: 2,
|
||||
provider: this.assets.size > 0 ? "native" : "none",
|
||||
@@ -285,11 +146,14 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession {
|
||||
|
||||
private handleEvent(payload: WindowsCursorEvent) {
|
||||
if (payload.type === "error") {
|
||||
this.logDiagnostic("helper-error", { message: payload.message });
|
||||
console.error("Windows cursor helper error:", payload.message);
|
||||
return;
|
||||
}
|
||||
|
||||
if (payload.type === "ready") {
|
||||
this.logDiagnostic("ready", { timestampMs: payload.timestampMs });
|
||||
this.resolveReady();
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -305,22 +169,100 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession {
|
||||
hotspotY: payload.asset.hotspotY,
|
||||
scaleFactor: assetDisplay.scaleFactor,
|
||||
});
|
||||
this.logDiagnostic("asset", {
|
||||
id: payload.asset.id,
|
||||
width: payload.asset.width,
|
||||
height: payload.asset.height,
|
||||
hotspotX: payload.asset.hotspotX,
|
||||
hotspotY: payload.asset.hotspotY,
|
||||
scaleFactor: assetDisplay.scaleFactor,
|
||||
});
|
||||
}
|
||||
|
||||
const bounds = this.options.getDisplayBounds() ?? screen.getPrimaryDisplay().bounds;
|
||||
const width = Math.max(1, bounds.width);
|
||||
const height = Math.max(1, bounds.height);
|
||||
const normalized = this.normalizeSample(payload);
|
||||
this.sampleCount += 1;
|
||||
if (!normalized.withinBounds) {
|
||||
this.outOfBoundsSampleCount += 1;
|
||||
}
|
||||
|
||||
this.samples.push({
|
||||
timeMs: Math.max(0, payload.timestampMs - this.startTimeMs),
|
||||
cx: clamp((payload.x - bounds.x) / width, 0, 1),
|
||||
cy: clamp((payload.y - bounds.y) / height, 0, 1),
|
||||
assetId: payload.handle,
|
||||
visible: payload.visible,
|
||||
});
|
||||
this.samples.push(normalized.sample);
|
||||
|
||||
if (this.samples.length > this.options.maxSamples) {
|
||||
this.samples.shift();
|
||||
}
|
||||
}
|
||||
|
||||
private normalizeSample(payload: Extract<WindowsCursorEvent, { type: "sample" }>): NormalizedSample {
|
||||
const bounds = payload.bounds ?? this.options.getDisplayBounds() ?? screen.getPrimaryDisplay().bounds;
|
||||
const width = Math.max(1, bounds.width);
|
||||
const height = Math.max(1, bounds.height);
|
||||
const normalizedX = (payload.x - bounds.x) / width;
|
||||
const normalizedY = (payload.y - bounds.y) / height;
|
||||
const withinBounds = normalizedX >= 0 && normalizedX <= 1 && normalizedY >= 0 && normalizedY <= 1;
|
||||
|
||||
if (this.sampleCount === 0 || (!withinBounds && this.outOfBoundsSampleCount === 0)) {
|
||||
this.logDiagnostic("sample", {
|
||||
rawX: payload.x,
|
||||
rawY: payload.y,
|
||||
normalizedX,
|
||||
normalizedY,
|
||||
visible: payload.visible,
|
||||
withinBounds,
|
||||
bounds,
|
||||
handle: payload.handle,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
withinBounds,
|
||||
sample: {
|
||||
timeMs: Math.max(0, payload.timestampMs - this.startTimeMs),
|
||||
cx: normalizedX,
|
||||
cy: normalizedY,
|
||||
assetId: payload.handle,
|
||||
visible: payload.visible && withinBounds,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private waitUntilReady() {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
this.readyResolve = resolve;
|
||||
this.readyReject = reject;
|
||||
this.readyTimer = setTimeout(() => {
|
||||
this.rejectReady(new Error("Timed out waiting for Windows cursor helper readiness"));
|
||||
}, READY_TIMEOUT_MS);
|
||||
});
|
||||
}
|
||||
|
||||
private resolveReady() {
|
||||
const resolve = this.readyResolve;
|
||||
this.clearReadyState();
|
||||
resolve?.();
|
||||
}
|
||||
|
||||
private rejectReady(error: Error) {
|
||||
const reject = this.readyReject;
|
||||
this.clearReadyState();
|
||||
reject?.(error);
|
||||
}
|
||||
|
||||
private clearReadyState() {
|
||||
if (this.readyTimer) {
|
||||
clearTimeout(this.readyTimer);
|
||||
this.readyTimer = null;
|
||||
}
|
||||
this.readyResolve = null;
|
||||
this.readyReject = null;
|
||||
}
|
||||
|
||||
private logDiagnostic(event: string, data: Record<string, unknown>) {
|
||||
console.info(
|
||||
"[cursor-native][win32]",
|
||||
JSON.stringify({
|
||||
event,
|
||||
...data,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
import type { Rectangle } from "electron";
|
||||
|
||||
export interface WindowsCursorSampleEvent {
|
||||
type: "sample";
|
||||
timestampMs: number;
|
||||
x: number;
|
||||
y: number;
|
||||
visible: boolean;
|
||||
handle: string | null;
|
||||
bounds?: {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
} | null;
|
||||
asset?: WindowsCursorAssetPayload;
|
||||
}
|
||||
|
||||
export interface WindowsCursorReadyEvent {
|
||||
type: "ready";
|
||||
timestampMs: number;
|
||||
}
|
||||
|
||||
export interface WindowsCursorErrorEvent {
|
||||
type: "error";
|
||||
timestampMs: number;
|
||||
message: string;
|
||||
}
|
||||
|
||||
export interface WindowsCursorAssetPayload {
|
||||
id: string;
|
||||
imageDataUrl: string;
|
||||
width: number;
|
||||
height: number;
|
||||
hotspotX: number;
|
||||
hotspotY: number;
|
||||
}
|
||||
|
||||
export type WindowsCursorEvent =
|
||||
| WindowsCursorSampleEvent
|
||||
| WindowsCursorReadyEvent
|
||||
| WindowsCursorErrorEvent;
|
||||
|
||||
export interface WindowsNativeRecordingSessionOptions {
|
||||
getDisplayBounds: () => Rectangle | null;
|
||||
maxSamples: number;
|
||||
sampleIntervalMs: number;
|
||||
sourceId?: string | null;
|
||||
}
|
||||
@@ -259,6 +259,8 @@ export function LaunchWindow() {
|
||||
|
||||
const [selectedSource, setSelectedSource] = useState("Screen");
|
||||
const [hasSelectedSource, setHasSelectedSource] = useState(false);
|
||||
const [, setHudPointerDownCount] = useState(0);
|
||||
const [, setRecordPointerDownCount] = useState(0);
|
||||
|
||||
useEffect(() => {
|
||||
const checkSelectedSource = async () => {
|
||||
@@ -541,6 +543,9 @@ export function LaunchWindow() {
|
||||
onClick={toggleMicrophone}
|
||||
disabled={recording}
|
||||
title={microphoneEnabled ? t("audio.disableMicrophone") : t("audio.enableMicrophone")}
|
||||
onPointerDown={() => {
|
||||
setRecordPointerDownCount((count) => count + 1);
|
||||
}}
|
||||
>
|
||||
{microphoneEnabled
|
||||
? getIcon("micOn", "text-green-400")
|
||||
|
||||
@@ -1477,6 +1477,7 @@ export default function VideoEditor() {
|
||||
videoPadding: padding,
|
||||
cropRegion,
|
||||
cursorRecordingData,
|
||||
cursorScale: showCursor ? cursorSize : 0,
|
||||
annotationRegions,
|
||||
webcamLayoutPreset,
|
||||
webcamMaskShape,
|
||||
@@ -1619,6 +1620,7 @@ export default function VideoEditor() {
|
||||
padding,
|
||||
cropRegion,
|
||||
cursorRecordingData,
|
||||
cursorScale: showCursor ? cursorSize : 0,
|
||||
annotationRegions,
|
||||
webcamLayoutPreset,
|
||||
webcamMaskShape,
|
||||
|
||||
@@ -29,8 +29,9 @@ import { classifyWallpaper, DEFAULT_WALLPAPER, resolveImageWallpaperUrl } from "
|
||||
import { getCssClipPath } from "@/lib/webcamMaskShapes";
|
||||
import {
|
||||
getNativeCursorDisplayMetrics,
|
||||
hasNativeCursorRecordingData,
|
||||
projectNativeCursorToStage,
|
||||
resolveActiveNativeCursorFrame,
|
||||
resolveInterpolatedNativeCursorFrame,
|
||||
} from "@/lib/cursor/nativeCursor";
|
||||
import type { CursorRecordingData } from "@/native/contracts";
|
||||
import {
|
||||
@@ -635,6 +636,18 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
showCursorRef.current = showCursor;
|
||||
}, [showCursor]);
|
||||
|
||||
useEffect(() => {
|
||||
hasNativeCursorRecordingRef.current = hasNativeCursorRecording;
|
||||
}, [hasNativeCursorRecording]);
|
||||
|
||||
useEffect(() => {
|
||||
cursorRecordingDataRef.current = cursorRecordingData;
|
||||
}, [cursorRecordingData]);
|
||||
|
||||
useEffect(() => {
|
||||
cropRegionRef.current = cropRegion;
|
||||
}, [cropRegion]);
|
||||
|
||||
useEffect(() => {
|
||||
cursorSizeRef.current = cursorSize;
|
||||
}, [cursorSize]);
|
||||
@@ -1273,16 +1286,69 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
// Update cursor overlay
|
||||
const cursorOverlay = cursorOverlayRef.current;
|
||||
if (cursorOverlay) {
|
||||
const timeMs = currentTimeRef.current;
|
||||
const timeMs = currentTimeRef.current; // already in ms
|
||||
cursorOverlay.update(
|
||||
cursorTelemetryRef.current,
|
||||
timeMs,
|
||||
baseMaskRef.current,
|
||||
showCursorRef.current,
|
||||
showCursorRef.current && !hasNativeCursorRecordingRef.current,
|
||||
!isPlayingRef.current || isSeekingRef.current,
|
||||
);
|
||||
}
|
||||
|
||||
// Update native cursor image position at ticker rate (60fps)
|
||||
const nativeCursorImg = nativeCursorImgRef.current;
|
||||
if (nativeCursorImg) {
|
||||
const cameraContainerRc = cameraContainerRef.current;
|
||||
const videoContainerRc = videoContainerRef.current;
|
||||
if (
|
||||
hasNativeCursorRecordingRef.current &&
|
||||
showCursorRef.current &&
|
||||
cameraContainerRc &&
|
||||
videoContainerRc
|
||||
) {
|
||||
const timeMs = currentTimeRef.current; // already in ms
|
||||
const frame = resolveInterpolatedNativeCursorFrame(
|
||||
cursorRecordingDataRef.current,
|
||||
timeMs,
|
||||
);
|
||||
if (frame) {
|
||||
const projectedPoint = projectNativeCursorToStage({
|
||||
cameraContainer: cameraContainerRc,
|
||||
cropRegion: cropRegionRef.current ?? { x: 0, y: 0, width: 1, height: 1 },
|
||||
maskRect: baseMaskRef.current,
|
||||
videoContainerPosition: {
|
||||
x: videoContainerRc.x,
|
||||
y: videoContainerRc.y,
|
||||
},
|
||||
sample: frame.sample,
|
||||
});
|
||||
if (projectedPoint) {
|
||||
const metrics = getNativeCursorDisplayMetrics(
|
||||
frame.asset,
|
||||
window.devicePixelRatio || 1,
|
||||
);
|
||||
const scale = Math.max(0, cursorSizeRef.current);
|
||||
if (nativeCursorImg.dataset.cursorId !== frame.asset.id) {
|
||||
nativeCursorImg.src = frame.asset.imageDataUrl;
|
||||
nativeCursorImg.dataset.cursorId = frame.asset.id;
|
||||
}
|
||||
nativeCursorImg.style.left = `${projectedPoint.x - metrics.hotspotX * scale}px`;
|
||||
nativeCursorImg.style.top = `${projectedPoint.y - metrics.hotspotY * scale}px`;
|
||||
nativeCursorImg.style.width = `${metrics.width * scale}px`;
|
||||
nativeCursorImg.style.height = `${metrics.height * scale}px`;
|
||||
nativeCursorImg.style.display = "block";
|
||||
} else {
|
||||
nativeCursorImg.style.display = "none";
|
||||
}
|
||||
} else {
|
||||
nativeCursorImg.style.display = "none";
|
||||
}
|
||||
} else {
|
||||
nativeCursorImg.style.display = "none";
|
||||
}
|
||||
}
|
||||
|
||||
const composite3D = composite3DRef.current;
|
||||
const outerWrapper = outerWrapperRef.current;
|
||||
if (composite3D && outerWrapper) {
|
||||
@@ -1571,17 +1637,14 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
className="absolute rounded-md border border-[#34B27B]/80 bg-[#34B27B]/20 shadow-[0_0_0_1px_rgba(52,178,123,0.35)]"
|
||||
style={{ display: "none", pointerEvents: "none" }}
|
||||
/>
|
||||
{activeNativeCursor && nativeCursorStyle ? (
|
||||
{hasNativeCursorRecording ? (
|
||||
<img
|
||||
ref={nativeCursorImgRef}
|
||||
alt=""
|
||||
aria-hidden="true"
|
||||
src={activeNativeCursor.asset.imageDataUrl}
|
||||
className="absolute select-none"
|
||||
style={{
|
||||
left: nativeCursorStyle.left,
|
||||
top: nativeCursorStyle.top,
|
||||
width: nativeCursorStyle.width,
|
||||
height: nativeCursorStyle.height,
|
||||
display: "none",
|
||||
pointerEvents: "none",
|
||||
userSelect: "none",
|
||||
}}
|
||||
|
||||
@@ -25,7 +25,6 @@ const CODEC_ALIGNMENT = 2;
|
||||
|
||||
const RECORDER_TIMESLICE_MS = 1000;
|
||||
const BITS_PER_MEGABIT = 1_000_000;
|
||||
const CHROME_MEDIA_SOURCE = "desktop";
|
||||
const RECORDING_FILE_PREFIX = "recording-";
|
||||
const VIDEO_FILE_EXTENSION = ".webm";
|
||||
const WEBCAM_FILE_SUFFIX = "-webcam";
|
||||
@@ -576,42 +575,19 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
|
||||
|
||||
let screenMediaStream: MediaStream;
|
||||
|
||||
const videoConstraints = {
|
||||
mandatory: {
|
||||
chromeMediaSource: CHROME_MEDIA_SOURCE,
|
||||
chromeMediaSourceId: selectedSource.id,
|
||||
maxWidth: TARGET_WIDTH,
|
||||
maxHeight: TARGET_HEIGHT,
|
||||
maxFrameRate: TARGET_FRAME_RATE,
|
||||
minFrameRate: MIN_FRAME_RATE,
|
||||
},
|
||||
};
|
||||
|
||||
if (systemAudioEnabled) {
|
||||
try {
|
||||
screenMediaStream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
mandatory: {
|
||||
chromeMediaSource: CHROME_MEDIA_SOURCE,
|
||||
chromeMediaSourceId: selectedSource.id,
|
||||
},
|
||||
},
|
||||
video: videoConstraints,
|
||||
} as unknown as MediaStreamConstraints);
|
||||
} catch (audioErr) {
|
||||
console.warn("System audio capture failed, falling back to video-only:", audioErr);
|
||||
toast.error(t("recording.systemAudioUnavailable"));
|
||||
screenMediaStream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: false,
|
||||
video: videoConstraints,
|
||||
} as unknown as MediaStreamConstraints);
|
||||
}
|
||||
} else {
|
||||
screenMediaStream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: false,
|
||||
video: videoConstraints,
|
||||
} as unknown as MediaStreamConstraints);
|
||||
}
|
||||
// getDisplayMedia + setDisplayMediaRequestHandler (main.ts) supplies the
|
||||
// pre-selected source and honors cursor:"never" to exclude the system cursor
|
||||
// from every captured frame. System audio is provided via WASAPI loopback
|
||||
// on Windows when the user has enabled it.
|
||||
screenMediaStream = await navigator.mediaDevices.getDisplayMedia({
|
||||
video: {
|
||||
cursor: "never",
|
||||
width: { max: TARGET_WIDTH },
|
||||
height: { max: TARGET_HEIGHT },
|
||||
frameRate: { ideal: TARGET_FRAME_RATE, min: MIN_FRAME_RATE },
|
||||
} as MediaTrackConstraints,
|
||||
audio: systemAudioEnabled,
|
||||
} as DisplayMediaStreamOptions);
|
||||
screenStream.current = screenMediaStream;
|
||||
|
||||
if (!isCountdownRunActive(countdownRunToken)) {
|
||||
|
||||
@@ -14,7 +14,7 @@ export interface ActiveNativeCursorFrame {
|
||||
interface ProjectNativeCursorOptions {
|
||||
cameraContainer: Container;
|
||||
cropRegion: CropRegion;
|
||||
maskRect: { width: number; height: number };
|
||||
maskRect: { x: number; y: number; width: number; height: number };
|
||||
videoContainerPosition: { x: number; y: number };
|
||||
sample: CursorRecordingSample;
|
||||
}
|
||||
@@ -23,6 +23,17 @@ function clamp(value: number, min: number, max: number) {
|
||||
return Math.min(max, Math.max(min, value));
|
||||
}
|
||||
|
||||
export function hasNativeCursorRecordingData(
|
||||
recordingData: CursorRecordingData | null | undefined,
|
||||
): recordingData is CursorRecordingData {
|
||||
return Boolean(
|
||||
recordingData &&
|
||||
recordingData.provider === "native" &&
|
||||
recordingData.samples.length > 0 &&
|
||||
recordingData.assets.length > 0,
|
||||
);
|
||||
}
|
||||
|
||||
function getCroppedCursorPosition(sample: CursorRecordingSample, cropRegion: CropRegion) {
|
||||
if (cropRegion.width <= 0 || cropRegion.height <= 0) {
|
||||
return null;
|
||||
@@ -45,7 +56,7 @@ export function resolveActiveNativeCursorFrame(
|
||||
recordingData: CursorRecordingData | null | undefined,
|
||||
timeMs: number,
|
||||
): ActiveNativeCursorFrame | null {
|
||||
if (!recordingData || recordingData.provider !== "native" || recordingData.assets.length === 0) {
|
||||
if (!hasNativeCursorRecordingData(recordingData)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -70,6 +81,65 @@ export function resolveActiveNativeCursorFrame(
|
||||
return null;
|
||||
}
|
||||
|
||||
export function resolveInterpolatedNativeCursorFrame(
|
||||
recordingData: CursorRecordingData | null | undefined,
|
||||
timeMs: number,
|
||||
): ActiveNativeCursorFrame | null {
|
||||
if (!hasNativeCursorRecordingData(recordingData)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const samples = recordingData.samples;
|
||||
let activeIndex = -1;
|
||||
|
||||
for (let index = samples.length - 1; index >= 0; index -= 1) {
|
||||
if (samples[index].timeMs <= timeMs) {
|
||||
activeIndex = index;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (activeIndex < 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const activeSample = samples[activeIndex];
|
||||
if (activeSample.visible === false || !activeSample.assetId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const asset = recordingData.assets.find((candidate) => candidate.id === activeSample.assetId);
|
||||
if (!asset) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const nextSample = samples[activeIndex + 1];
|
||||
if (
|
||||
!nextSample ||
|
||||
nextSample.timeMs <= activeSample.timeMs ||
|
||||
nextSample.visible === false ||
|
||||
nextSample.assetId !== activeSample.assetId ||
|
||||
timeMs <= activeSample.timeMs
|
||||
) {
|
||||
return { asset, sample: activeSample };
|
||||
}
|
||||
|
||||
const interpolation = clamp(
|
||||
(timeMs - activeSample.timeMs) / (nextSample.timeMs - activeSample.timeMs),
|
||||
0,
|
||||
1,
|
||||
);
|
||||
|
||||
return {
|
||||
asset,
|
||||
sample: {
|
||||
...activeSample,
|
||||
cx: activeSample.cx + (nextSample.cx - activeSample.cx) * interpolation,
|
||||
cy: activeSample.cy + (nextSample.cy - activeSample.cy) * interpolation,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function projectNativeCursorToStage({
|
||||
cameraContainer,
|
||||
cropRegion,
|
||||
@@ -83,8 +153,8 @@ export function projectNativeCursorToStage({
|
||||
}
|
||||
|
||||
const localPoint = new Point(
|
||||
videoContainerPosition.x + croppedPosition.cx * maskRect.width,
|
||||
videoContainerPosition.y + croppedPosition.cy * maskRect.height,
|
||||
videoContainerPosition.x + maskRect.x + croppedPosition.cx * maskRect.width,
|
||||
videoContainerPosition.y + maskRect.y + croppedPosition.cy * maskRect.height,
|
||||
);
|
||||
|
||||
return cameraContainer.toGlobal(localPoint);
|
||||
|
||||
@@ -59,7 +59,7 @@ import {
|
||||
import {
|
||||
getNativeCursorDisplayMetrics,
|
||||
projectNativeCursorToStage,
|
||||
resolveActiveNativeCursorFrame,
|
||||
resolveInterpolatedNativeCursorFrame,
|
||||
} from "@/lib/cursor/nativeCursor";
|
||||
import { BackgroundLoadError, classifyWallpaper, resolveImageWallpaperUrl } from "@/lib/wallpaper";
|
||||
import { drawCanvasClipPath } from "@/lib/webcamMaskShapes";
|
||||
@@ -86,6 +86,7 @@ interface FrameRenderConfig {
|
||||
padding?: number;
|
||||
cropRegion: CropRegion;
|
||||
cursorRecordingData?: CursorRecordingData | null;
|
||||
cursorScale?: number;
|
||||
videoWidth: number;
|
||||
videoHeight: number;
|
||||
webcamSize?: Size | null;
|
||||
@@ -558,7 +559,11 @@ export class FrameRenderer {
|
||||
return;
|
||||
}
|
||||
|
||||
const activeNativeCursor = resolveActiveNativeCursorFrame(
|
||||
if ((this.config.cursorScale ?? 1) <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const activeNativeCursor = resolveInterpolatedNativeCursorFrame(
|
||||
this.config.cursorRecordingData,
|
||||
timeMs,
|
||||
);
|
||||
@@ -582,13 +587,13 @@ export class FrameRenderer {
|
||||
|
||||
const image = await this.getCursorImage(activeNativeCursor.asset);
|
||||
const metrics = getNativeCursorDisplayMetrics(activeNativeCursor.asset, 1);
|
||||
|
||||
const scale = Math.max(0, this.config.cursorScale ?? 1);
|
||||
this.compositeCtx.drawImage(
|
||||
image,
|
||||
projectedPoint.x - metrics.hotspotX,
|
||||
projectedPoint.y - metrics.hotspotY,
|
||||
metrics.width,
|
||||
metrics.height,
|
||||
projectedPoint.x - metrics.hotspotX * scale,
|
||||
projectedPoint.y - metrics.hotspotY * scale,
|
||||
metrics.width * scale,
|
||||
metrics.height * scale,
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -49,6 +49,7 @@ interface GifExporterConfig {
|
||||
webcamSizePreset?: WebcamSizePreset;
|
||||
webcamPosition?: { cx: number; cy: number } | null;
|
||||
cursorRecordingData?: CursorRecordingData | null;
|
||||
cursorScale?: number;
|
||||
annotationRegions?: AnnotationRegion[];
|
||||
previewWidth?: number;
|
||||
previewHeight?: number;
|
||||
@@ -154,6 +155,7 @@ export class GifExporter {
|
||||
padding: this.config.padding,
|
||||
cropRegion: this.config.cropRegion,
|
||||
cursorRecordingData: this.config.cursorRecordingData,
|
||||
cursorScale: this.config.cursorScale,
|
||||
videoWidth: videoInfo.width,
|
||||
videoHeight: videoInfo.height,
|
||||
webcamSize: webcamInfo ? { width: webcamInfo.width, height: webcamInfo.height } : null,
|
||||
|
||||
@@ -40,6 +40,7 @@ interface VideoExporterConfig extends ExportConfig {
|
||||
webcamSizePreset?: WebcamSizePreset;
|
||||
webcamPosition?: { cx: number; cy: number } | null;
|
||||
cursorRecordingData?: CursorRecordingData | null;
|
||||
cursorScale?: number;
|
||||
annotationRegions?: AnnotationRegion[];
|
||||
previewWidth?: number;
|
||||
previewHeight?: number;
|
||||
@@ -149,6 +150,7 @@ export class VideoExporter {
|
||||
padding: this.config.padding,
|
||||
cropRegion: this.config.cropRegion,
|
||||
cursorRecordingData: this.config.cursorRecordingData,
|
||||
cursorScale: this.config.cursorScale,
|
||||
videoWidth: videoInfo.width,
|
||||
videoHeight: videoInfo.height,
|
||||
webcamSize: webcamInfo ? { width: webcamInfo.width, height: webcamInfo.height } : null,
|
||||
|
||||
Reference in New Issue
Block a user