feat: add windows native cursor capture and rendering
This commit is contained in:
Etienne Lescot
committed by
EtienneLescot
EtienneLescot
parent
44f59bfa89
commit
248ebabcf1
+326
-673
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,29 @@
|
||||
import type { Rectangle } from "electron";
|
||||
import type { CursorRecordingSession } from "./session";
|
||||
import { TelemetryRecordingSession } from "./telemetryRecordingSession";
|
||||
import { WindowsNativeRecordingSession } from "./windowsNativeRecordingSession";
|
||||
|
||||
interface CreateCursorRecordingSessionOptions {
|
||||
getDisplayBounds: () => Rectangle | null;
|
||||
maxSamples: number;
|
||||
platform: NodeJS.Platform;
|
||||
sampleIntervalMs: number;
|
||||
}
|
||||
|
||||
export function createCursorRecordingSession(
|
||||
options: CreateCursorRecordingSessionOptions,
|
||||
): CursorRecordingSession {
|
||||
if (options.platform === "win32") {
|
||||
return new WindowsNativeRecordingSession({
|
||||
getDisplayBounds: options.getDisplayBounds,
|
||||
maxSamples: options.maxSamples,
|
||||
sampleIntervalMs: options.sampleIntervalMs,
|
||||
});
|
||||
}
|
||||
|
||||
return new TelemetryRecordingSession({
|
||||
getDisplayBounds: options.getDisplayBounds,
|
||||
maxSamples: options.maxSamples,
|
||||
sampleIntervalMs: options.sampleIntervalMs,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
import type { CursorRecordingData } from "../../../../src/native/contracts";
|
||||
|
||||
export interface CursorRecordingSession {
|
||||
start(): Promise<void>;
|
||||
stop(): Promise<CursorRecordingData>;
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
import { type Rectangle, screen } from "electron";
|
||||
import type { CursorRecordingData, CursorRecordingSample } from "../../../../src/native/contracts";
|
||||
import type { CursorRecordingSession } from "./session";
|
||||
|
||||
interface TelemetryRecordingSessionOptions {
|
||||
getDisplayBounds: () => Rectangle | null;
|
||||
maxSamples: number;
|
||||
sampleIntervalMs: number;
|
||||
}
|
||||
|
||||
function clamp(value: number, min: number, max: number) {
|
||||
return Math.min(max, Math.max(min, value));
|
||||
}
|
||||
|
||||
export class TelemetryRecordingSession implements CursorRecordingSession {
|
||||
private samples: CursorRecordingSample[] = [];
|
||||
private interval: NodeJS.Timeout | null = null;
|
||||
private startTimeMs = 0;
|
||||
|
||||
constructor(private readonly options: TelemetryRecordingSessionOptions) {}
|
||||
|
||||
async start(): Promise<void> {
|
||||
this.samples = [];
|
||||
this.startTimeMs = Date.now();
|
||||
this.captureSample();
|
||||
this.interval = setInterval(() => {
|
||||
this.captureSample();
|
||||
}, this.options.sampleIntervalMs);
|
||||
}
|
||||
|
||||
async stop(): Promise<CursorRecordingData> {
|
||||
if (this.interval) {
|
||||
clearInterval(this.interval);
|
||||
this.interval = null;
|
||||
}
|
||||
|
||||
return {
|
||||
version: 2,
|
||||
provider: "none",
|
||||
samples: this.samples,
|
||||
assets: [],
|
||||
};
|
||||
}
|
||||
|
||||
private captureSample() {
|
||||
const cursor = screen.getCursorScreenPoint();
|
||||
const display = this.options.getDisplayBounds() ?? screen.getDisplayNearestPoint(cursor).bounds;
|
||||
const width = Math.max(1, display.width);
|
||||
const height = Math.max(1, display.height);
|
||||
|
||||
this.samples.push({
|
||||
timeMs: Math.max(0, Date.now() - this.startTimeMs),
|
||||
cx: clamp((cursor.x - display.x) / width, 0, 1),
|
||||
cy: clamp((cursor.y - display.y) / height, 0, 1),
|
||||
visible: true,
|
||||
});
|
||||
|
||||
if (this.samples.length > this.options.maxSamples) {
|
||||
this.samples.shift();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,326 @@
|
||||
import { type ChildProcessByStdio, spawn } from "node:child_process";
|
||||
import type { Readable } from "node:stream";
|
||||
import { type Rectangle, screen } from "electron";
|
||||
import type {
|
||||
CursorRecordingData,
|
||||
CursorRecordingSample,
|
||||
NativeCursorAsset,
|
||||
} from "../../../../src/native/contracts";
|
||||
import type { CursorRecordingSession } from "./session";
|
||||
|
||||
interface WindowsCursorSampleEvent {
|
||||
type: "sample";
|
||||
timestampMs: number;
|
||||
x: number;
|
||||
y: number;
|
||||
visible: boolean;
|
||||
handle: string | null;
|
||||
asset?: WindowsCursorAssetPayload;
|
||||
}
|
||||
|
||||
interface WindowsCursorReadyEvent {
|
||||
type: "ready";
|
||||
timestampMs: number;
|
||||
}
|
||||
|
||||
interface WindowsCursorErrorEvent {
|
||||
type: "error";
|
||||
timestampMs: number;
|
||||
message: string;
|
||||
}
|
||||
|
||||
interface WindowsCursorAssetPayload {
|
||||
id: string;
|
||||
imageDataUrl: string;
|
||||
width: number;
|
||||
height: number;
|
||||
hotspotX: number;
|
||||
hotspotY: number;
|
||||
}
|
||||
|
||||
type WindowsCursorEvent =
|
||||
| WindowsCursorSampleEvent
|
||||
| WindowsCursorReadyEvent
|
||||
| WindowsCursorErrorEvent;
|
||||
|
||||
interface WindowsNativeRecordingSessionOptions {
|
||||
getDisplayBounds: () => Rectangle | null;
|
||||
maxSamples: number;
|
||||
sampleIntervalMs: number;
|
||||
}
|
||||
|
||||
function clamp(value: number, min: number, max: number) {
|
||||
return Math.min(max, Math.max(min, value));
|
||||
}
|
||||
|
||||
function buildPowerShellCommand(sampleIntervalMs: number) {
|
||||
const script = String.raw`
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Add-Type -AssemblyName System.Drawing
|
||||
|
||||
$source = @"
|
||||
using System;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
public static class OpenScreenCursorInterop {
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
public struct POINT {
|
||||
public int X;
|
||||
public int Y;
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
public struct CURSORINFO {
|
||||
public int cbSize;
|
||||
public int flags;
|
||||
public IntPtr hCursor;
|
||||
public POINT ptScreenPos;
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
public struct ICONINFO {
|
||||
[MarshalAs(UnmanagedType.Bool)]
|
||||
public bool fIcon;
|
||||
public int xHotspot;
|
||||
public int yHotspot;
|
||||
public IntPtr hbmMask;
|
||||
public IntPtr hbmColor;
|
||||
}
|
||||
|
||||
[DllImport("user32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
public static extern bool GetCursorInfo(ref CURSORINFO pci);
|
||||
|
||||
[DllImport("user32.dll", SetLastError = true)]
|
||||
public static extern IntPtr CopyIcon(IntPtr hIcon);
|
||||
|
||||
[DllImport("user32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
public static extern bool DestroyIcon(IntPtr hIcon);
|
||||
|
||||
[DllImport("user32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
public static extern bool GetIconInfo(IntPtr hIcon, out ICONINFO piconinfo);
|
||||
|
||||
[DllImport("gdi32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
public static extern bool DeleteObject(IntPtr hObject);
|
||||
}
|
||||
"@
|
||||
|
||||
Add-Type -TypeDefinition $source
|
||||
|
||||
function Write-JsonLine($payload) {
|
||||
[Console]::Out.WriteLine(($payload | ConvertTo-Json -Compress -Depth 6))
|
||||
}
|
||||
|
||||
function Get-CursorAsset($cursorHandle, $cursorId) {
|
||||
$copiedHandle = [OpenScreenCursorInterop]::CopyIcon($cursorHandle)
|
||||
if ($copiedHandle -eq [IntPtr]::Zero) {
|
||||
return $null
|
||||
}
|
||||
|
||||
$iconInfo = New-Object OpenScreenCursorInterop+ICONINFO
|
||||
$hasIconInfo = [OpenScreenCursorInterop]::GetIconInfo($copiedHandle, [ref]$iconInfo)
|
||||
|
||||
try {
|
||||
$icon = [System.Drawing.Icon]::FromHandle($copiedHandle)
|
||||
$bitmap = New-Object System.Drawing.Bitmap $icon.Width, $icon.Height, ([System.Drawing.Imaging.PixelFormat]::Format32bppArgb)
|
||||
$graphics = [System.Drawing.Graphics]::FromImage($bitmap)
|
||||
$memoryStream = New-Object System.IO.MemoryStream
|
||||
|
||||
try {
|
||||
$graphics.Clear([System.Drawing.Color]::Transparent)
|
||||
$graphics.DrawIcon($icon, 0, 0)
|
||||
$bitmap.Save($memoryStream, [System.Drawing.Imaging.ImageFormat]::Png)
|
||||
$base64 = [System.Convert]::ToBase64String($memoryStream.ToArray())
|
||||
|
||||
return @{
|
||||
id = $cursorId
|
||||
imageDataUrl = "data:image/png;base64,$base64"
|
||||
width = $bitmap.Width
|
||||
height = $bitmap.Height
|
||||
hotspotX = if ($hasIconInfo) { $iconInfo.xHotspot } else { 0 }
|
||||
hotspotY = if ($hasIconInfo) { $iconInfo.yHotspot } else { 0 }
|
||||
}
|
||||
}
|
||||
finally {
|
||||
$memoryStream.Dispose()
|
||||
$graphics.Dispose()
|
||||
$bitmap.Dispose()
|
||||
$icon.Dispose()
|
||||
}
|
||||
}
|
||||
finally {
|
||||
if ($hasIconInfo) {
|
||||
if ($iconInfo.hbmMask -ne [IntPtr]::Zero) {
|
||||
[OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmMask) | Out-Null
|
||||
}
|
||||
if ($iconInfo.hbmColor -ne [IntPtr]::Zero) {
|
||||
[OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmColor) | Out-Null
|
||||
}
|
||||
}
|
||||
[OpenScreenCursorInterop]::DestroyIcon($copiedHandle) | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
Write-JsonLine @{ type = 'ready'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() }
|
||||
|
||||
$lastCursorId = $null
|
||||
while ($true) {
|
||||
$cursorInfo = New-Object OpenScreenCursorInterop+CURSORINFO
|
||||
$cursorInfo.cbSize = [Runtime.InteropServices.Marshal]::SizeOf([type][OpenScreenCursorInterop+CURSORINFO])
|
||||
|
||||
if (-not [OpenScreenCursorInterop]::GetCursorInfo([ref]$cursorInfo)) {
|
||||
Write-JsonLine @{ type = 'error'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds(); message = 'GetCursorInfo failed' }
|
||||
Start-Sleep -Milliseconds ${sampleIntervalMs}
|
||||
continue
|
||||
}
|
||||
|
||||
$visible = ($cursorInfo.flags -band 1) -ne 0
|
||||
$cursorId = if ($cursorInfo.hCursor -eq [IntPtr]::Zero) { $null } else { ('0x{0:X}' -f $cursorInfo.hCursor.ToInt64()) }
|
||||
$asset = $null
|
||||
|
||||
if ($visible -and $cursorId -and $cursorId -ne $lastCursorId) {
|
||||
$asset = Get-CursorAsset -cursorHandle $cursorInfo.hCursor -cursorId $cursorId
|
||||
$lastCursorId = $cursorId
|
||||
}
|
||||
|
||||
Write-JsonLine @{
|
||||
type = 'sample'
|
||||
timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds()
|
||||
x = $cursorInfo.ptScreenPos.X
|
||||
y = $cursorInfo.ptScreenPos.Y
|
||||
visible = $visible
|
||||
handle = $cursorId
|
||||
asset = $asset
|
||||
}
|
||||
|
||||
Start-Sleep -Milliseconds ${sampleIntervalMs}
|
||||
}
|
||||
`;
|
||||
|
||||
return Buffer.from(script, "utf16le").toString("base64");
|
||||
}
|
||||
|
||||
export class WindowsNativeRecordingSession implements CursorRecordingSession {
|
||||
private assets = new Map<string, NativeCursorAsset>();
|
||||
private samples: CursorRecordingSample[] = [];
|
||||
private process: ChildProcessByStdio<null, Readable, Readable> | null = null;
|
||||
private lineBuffer = "";
|
||||
private startTimeMs = 0;
|
||||
|
||||
constructor(private readonly options: WindowsNativeRecordingSessionOptions) {}
|
||||
|
||||
async start(): Promise<void> {
|
||||
this.assets.clear();
|
||||
this.samples = [];
|
||||
this.lineBuffer = "";
|
||||
this.startTimeMs = Date.now();
|
||||
|
||||
const encodedCommand = buildPowerShellCommand(this.options.sampleIntervalMs);
|
||||
const child = spawn(
|
||||
"powershell.exe",
|
||||
[
|
||||
"-NoLogo",
|
||||
"-NoProfile",
|
||||
"-NonInteractive",
|
||||
"-ExecutionPolicy",
|
||||
"Bypass",
|
||||
"-EncodedCommand",
|
||||
encodedCommand,
|
||||
],
|
||||
{
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
windowsHide: true,
|
||||
},
|
||||
);
|
||||
|
||||
this.process = child;
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk: string) => {
|
||||
this.handleStdoutChunk(chunk);
|
||||
});
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stderr.on("data", (chunk: string) => {
|
||||
console.error("[cursor-native]", chunk.trim());
|
||||
});
|
||||
}
|
||||
|
||||
async stop(): Promise<CursorRecordingData> {
|
||||
const child = this.process;
|
||||
this.process = null;
|
||||
|
||||
if (child && !child.killed) {
|
||||
child.kill();
|
||||
}
|
||||
|
||||
return {
|
||||
version: 2,
|
||||
provider: this.assets.size > 0 ? "native" : "none",
|
||||
samples: this.samples,
|
||||
assets: [...this.assets.values()],
|
||||
};
|
||||
}
|
||||
|
||||
private handleStdoutChunk(chunk: string) {
|
||||
this.lineBuffer += chunk;
|
||||
const lines = this.lineBuffer.split(/\r?\n/);
|
||||
this.lineBuffer = lines.pop() ?? "";
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmedLine = line.trim();
|
||||
if (!trimmedLine) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const payload = JSON.parse(trimmedLine) as WindowsCursorEvent;
|
||||
this.handleEvent(payload);
|
||||
} catch (error) {
|
||||
console.error("Failed to parse Windows cursor helper output:", error, trimmedLine);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private handleEvent(payload: WindowsCursorEvent) {
|
||||
if (payload.type === "error") {
|
||||
console.error("Windows cursor helper error:", payload.message);
|
||||
return;
|
||||
}
|
||||
|
||||
if (payload.type === "ready") {
|
||||
return;
|
||||
}
|
||||
|
||||
if (payload.asset?.id && !this.assets.has(payload.asset.id)) {
|
||||
const assetDisplay = screen.getDisplayNearestPoint({ x: payload.x, y: payload.y });
|
||||
this.assets.set(payload.asset.id, {
|
||||
id: payload.asset.id,
|
||||
platform: "win32",
|
||||
imageDataUrl: payload.asset.imageDataUrl,
|
||||
width: payload.asset.width,
|
||||
height: payload.asset.height,
|
||||
hotspotX: payload.asset.hotspotX,
|
||||
hotspotY: payload.asset.hotspotY,
|
||||
scaleFactor: assetDisplay.scaleFactor,
|
||||
});
|
||||
}
|
||||
|
||||
const bounds = this.options.getDisplayBounds() ?? screen.getPrimaryDisplay().bounds;
|
||||
const width = Math.max(1, bounds.width);
|
||||
const height = Math.max(1, bounds.height);
|
||||
|
||||
this.samples.push({
|
||||
timeMs: Math.max(0, payload.timestampMs - this.startTimeMs),
|
||||
cx: clamp((payload.x - bounds.x) / width, 0, 1),
|
||||
cy: clamp((payload.y - bounds.y) / height, 0, 1),
|
||||
assetId: payload.handle,
|
||||
visible: payload.visible,
|
||||
});
|
||||
|
||||
if (this.samples.length > this.options.maxSamples) {
|
||||
this.samples.shift();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -43,6 +43,7 @@ import {
|
||||
getNativeAspectRatioValue,
|
||||
isPortraitAspectRatio,
|
||||
} from "@/utils/aspectRatioUtils";
|
||||
import { nativeBridgeClient, useCursorRecordingData, useCursorTelemetry } from "@/native";
|
||||
import { ExportDialog } from "./ExportDialog";
|
||||
import PlaybackControls from "./PlaybackControls";
|
||||
import {
|
||||
@@ -61,7 +62,6 @@ import TimelineEditor from "./timeline/TimelineEditor";
|
||||
import {
|
||||
type AnnotationRegion,
|
||||
type BlurData,
|
||||
type CursorTelemetryPoint,
|
||||
clampFocusToDepth,
|
||||
DEFAULT_ANNOTATION_POSITION,
|
||||
DEFAULT_ANNOTATION_SIZE,
|
||||
@@ -133,8 +133,6 @@ export default function VideoEditor() {
|
||||
currentTimeRef.current = currentTime;
|
||||
const durationRef = useRef(duration);
|
||||
durationRef.current = duration;
|
||||
const [cursorTelemetry, setCursorTelemetry] = useState<CursorTelemetryPoint[]>([]);
|
||||
const [cursorClickTimestamps, setCursorClickTimestamps] = useState<number[]>([]);
|
||||
const [selectedZoomId, setSelectedZoomId] = useState<string | null>(null);
|
||||
const [selectedTrimId, setSelectedTrimId] = useState<string | null>(null);
|
||||
const [selectedSpeedId, setSelectedSpeedId] = useState<string | null>(null);
|
||||
@@ -220,6 +218,13 @@ export default function VideoEditor() {
|
||||
const project = candidate;
|
||||
const sourcePath = project.videoPath;
|
||||
const normalizedEditor = normalizeProjectEditor(project.editor);
|
||||
const inferredDurationMs = Math.max(
|
||||
0,
|
||||
...normalizedEditor.zoomRegions.map((region) => region.endMs),
|
||||
...normalizedEditor.trimRegions.map((region) => region.endMs),
|
||||
...normalizedEditor.speedRegions.map((region) => region.endMs),
|
||||
...normalizedEditor.annotationRegions.map((region) => region.endMs),
|
||||
);
|
||||
|
||||
try {
|
||||
videoPlaybackRef.current?.pause();
|
||||
@@ -228,7 +233,7 @@ export default function VideoEditor() {
|
||||
}
|
||||
setIsPlaying(false);
|
||||
setCurrentTime(0);
|
||||
setDuration(0);
|
||||
setDuration(inferredDurationMs > 0 ? inferredDurationMs / 1000 : 0);
|
||||
|
||||
setError(null);
|
||||
setVideoSourcePath(sourcePath);
|
||||
@@ -357,7 +362,7 @@ export default function VideoEditor() {
|
||||
useEffect(() => {
|
||||
async function loadInitialData() {
|
||||
try {
|
||||
const currentProjectResult = await window.electronAPI.loadCurrentProjectFile();
|
||||
const currentProjectResult = await nativeBridgeClient.project.loadCurrentProjectFile();
|
||||
if (currentProjectResult.success && currentProjectResult.project) {
|
||||
const restored = await applyLoadedProject(
|
||||
currentProjectResult.project,
|
||||
@@ -394,7 +399,7 @@ export default function VideoEditor() {
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await window.electronAPI.getCurrentVideoPath();
|
||||
const result = await nativeBridgeClient.project.getCurrentVideoPath();
|
||||
if (result.success && result.path) {
|
||||
setVideoSourcePath(result.path);
|
||||
setVideoPath(toFileUrl(result.path));
|
||||
@@ -483,7 +488,7 @@ export default function VideoEditor() {
|
||||
// Match the normalization path used by `currentProjectSnapshot` so the
|
||||
// post-save baseline compares equal and `hasUnsavedChanges` clears.
|
||||
const projectSnapshot = createProjectSnapshot(currentProjectMedia, editorState);
|
||||
const result = await window.electronAPI.saveProjectFile(
|
||||
const result = await nativeBridgeClient.project.saveProjectFile(
|
||||
projectData,
|
||||
fileNameBase,
|
||||
forceSaveAs ? undefined : (currentProjectPath ?? undefined),
|
||||
@@ -589,7 +594,7 @@ export default function VideoEditor() {
|
||||
}, []);
|
||||
|
||||
const handleLoadProject = useCallback(async () => {
|
||||
const result = await window.electronAPI.loadProjectFile();
|
||||
const result = await nativeBridgeClient.project.loadProjectFile();
|
||||
|
||||
if (result.canceled) {
|
||||
return;
|
||||
@@ -622,40 +627,16 @@ export default function VideoEditor() {
|
||||
}, [handleLoadProject, handleSaveProject, handleSaveProjectAs]);
|
||||
|
||||
useEffect(() => {
|
||||
let mounted = true;
|
||||
|
||||
async function loadCursorTelemetry() {
|
||||
const sourcePath = currentProjectMedia?.screenVideoPath ?? null;
|
||||
|
||||
if (!sourcePath) {
|
||||
if (mounted) {
|
||||
setCursorTelemetry([]);
|
||||
setCursorClickTimestamps([]);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await window.electronAPI.getCursorTelemetry(sourcePath);
|
||||
if (mounted) {
|
||||
setCursorTelemetry(result.success ? result.samples : []);
|
||||
setCursorClickTimestamps(result.success ? (result.clicks ?? []) : []);
|
||||
}
|
||||
} catch (telemetryError) {
|
||||
console.warn("Unable to load cursor telemetry:", telemetryError);
|
||||
if (mounted) {
|
||||
setCursorTelemetry([]);
|
||||
setCursorClickTimestamps([]);
|
||||
}
|
||||
}
|
||||
if (cursorTelemetryError) {
|
||||
console.warn("Unable to load cursor telemetry:", cursorTelemetryError);
|
||||
}
|
||||
}, [cursorTelemetryError]);
|
||||
|
||||
loadCursorTelemetry();
|
||||
|
||||
return () => {
|
||||
mounted = false;
|
||||
};
|
||||
}, [currentProjectMedia]);
|
||||
useEffect(() => {
|
||||
if (cursorRecordingDataError) {
|
||||
console.warn("Unable to load cursor recording data:", cursorRecordingDataError);
|
||||
}
|
||||
}, [cursorRecordingDataError]);
|
||||
|
||||
function togglePlayPause() {
|
||||
const playback = videoPlaybackRef.current;
|
||||
@@ -1495,6 +1476,7 @@ export default function VideoEditor() {
|
||||
padding,
|
||||
videoPadding: padding,
|
||||
cropRegion,
|
||||
cursorRecordingData,
|
||||
annotationRegions,
|
||||
webcamLayoutPreset,
|
||||
webcamMaskShape,
|
||||
@@ -1636,6 +1618,7 @@ export default function VideoEditor() {
|
||||
borderRadius,
|
||||
padding,
|
||||
cropRegion,
|
||||
cursorRecordingData,
|
||||
annotationRegions,
|
||||
webcamLayoutPreset,
|
||||
webcamMaskShape,
|
||||
@@ -1715,6 +1698,7 @@ export default function VideoEditor() {
|
||||
borderRadius,
|
||||
padding,
|
||||
cropRegion,
|
||||
cursorRecordingData,
|
||||
annotationRegions,
|
||||
isPlaying,
|
||||
aspectRatio,
|
||||
|
||||
@@ -27,6 +27,12 @@ import {
|
||||
} from "@/lib/compositeLayout";
|
||||
import { classifyWallpaper, DEFAULT_WALLPAPER, resolveImageWallpaperUrl } from "@/lib/wallpaper";
|
||||
import { getCssClipPath } from "@/lib/webcamMaskShapes";
|
||||
import {
|
||||
getNativeCursorDisplayMetrics,
|
||||
projectNativeCursorToStage,
|
||||
resolveActiveNativeCursorFrame,
|
||||
} from "@/lib/cursor/nativeCursor";
|
||||
import type { CursorRecordingData } from "@/native/contracts";
|
||||
import {
|
||||
type AspectRatio,
|
||||
formatAspectRatioForCSS,
|
||||
@@ -123,6 +129,7 @@ interface VideoPlaybackProps {
|
||||
trimRegions?: TrimRegion[];
|
||||
speedRegions?: SpeedRegion[];
|
||||
aspectRatio: AspectRatio;
|
||||
cursorRecordingData?: CursorRecordingData | null;
|
||||
annotationRegions?: AnnotationRegion[];
|
||||
selectedAnnotationId?: string | null;
|
||||
onSelectAnnotation?: (id: string | null) => void;
|
||||
@@ -155,6 +162,22 @@ export interface VideoPlaybackRef {
|
||||
pause: () => void;
|
||||
}
|
||||
|
||||
function getResolvedVideoDuration(video: HTMLVideoElement): number | null {
|
||||
if (Number.isFinite(video.duration) && video.duration > 0) {
|
||||
return video.duration;
|
||||
}
|
||||
|
||||
if (video.seekable.length > 0) {
|
||||
const lastRangeIndex = video.seekable.length - 1;
|
||||
const seekableEnd = video.seekable.end(lastRangeIndex);
|
||||
if (Number.isFinite(seekableEnd) && seekableEnd > 0) {
|
||||
return seekableEnd;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
(
|
||||
{
|
||||
@@ -188,6 +211,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
trimRegions = [],
|
||||
speedRegions = [],
|
||||
aspectRatio,
|
||||
cursorRecordingData,
|
||||
annotationRegions = [],
|
||||
selectedAnnotationId,
|
||||
onSelectAnnotation,
|
||||
@@ -843,6 +867,8 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
|
||||
useEffect(() => {
|
||||
if (!videoPath) {
|
||||
lastResolvedDurationRef.current = null;
|
||||
isResolvingDurationRef.current = false;
|
||||
setVideoReady(false);
|
||||
return;
|
||||
}
|
||||
@@ -853,11 +879,18 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
video.currentTime = 0;
|
||||
allowPlaybackRef.current = false;
|
||||
lockedVideoDimensionsRef.current = null;
|
||||
lastResolvedDurationRef.current = null;
|
||||
isResolvingDurationRef.current = false;
|
||||
if (durationResolutionTimeoutRef.current) {
|
||||
clearTimeout(durationResolutionTimeoutRef.current);
|
||||
durationResolutionTimeoutRef.current = null;
|
||||
}
|
||||
setVideoReady(false);
|
||||
if (videoReadyRafRef.current) {
|
||||
cancelAnimationFrame(videoReadyRafRef.current);
|
||||
videoReadyRafRef.current = null;
|
||||
}
|
||||
video.load();
|
||||
}, [videoPath]);
|
||||
|
||||
useEffect(() => {
|
||||
@@ -1299,8 +1332,12 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
|
||||
const handleLoadedMetadata = (e: React.SyntheticEvent<HTMLVideoElement, Event>) => {
|
||||
const video = e.currentTarget;
|
||||
onDurationChange(video.duration);
|
||||
video.currentTime = 0;
|
||||
const hasResolvedDuration = syncResolvedDuration(video);
|
||||
if (!hasResolvedDuration) {
|
||||
forceResolveDuration(video);
|
||||
} else {
|
||||
video.currentTime = 0;
|
||||
}
|
||||
video.pause();
|
||||
allowPlaybackRef.current = false;
|
||||
currentTimeRef.current = 0;
|
||||
@@ -1313,6 +1350,9 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
const waitForRenderableFrame = () => {
|
||||
const hasDimensions = video.videoWidth > 0 && video.videoHeight > 0;
|
||||
const hasData = video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA;
|
||||
if (!syncResolvedDuration(video)) {
|
||||
forceResolveDuration(video);
|
||||
}
|
||||
if (hasDimensions && hasData) {
|
||||
videoReadyRafRef.current = null;
|
||||
setVideoReady(true);
|
||||
@@ -1412,6 +1452,10 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
window.clearTimeout(scrubEndTimerRef.current);
|
||||
scrubEndTimerRef.current = null;
|
||||
}
|
||||
if (durationResolutionTimeoutRef.current) {
|
||||
clearTimeout(durationResolutionTimeoutRef.current);
|
||||
durationResolutionTimeoutRef.current = null;
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
@@ -1527,6 +1571,22 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
className="absolute rounded-md border border-[#34B27B]/80 bg-[#34B27B]/20 shadow-[0_0_0_1px_rgba(52,178,123,0.35)]"
|
||||
style={{ display: "none", pointerEvents: "none" }}
|
||||
/>
|
||||
{activeNativeCursor && nativeCursorStyle ? (
|
||||
<img
|
||||
alt=""
|
||||
aria-hidden="true"
|
||||
src={activeNativeCursor.asset.imageDataUrl}
|
||||
className="absolute select-none"
|
||||
style={{
|
||||
left: nativeCursorStyle.left,
|
||||
top: nativeCursorStyle.top,
|
||||
width: nativeCursorStyle.width,
|
||||
height: nativeCursorStyle.height,
|
||||
pointerEvents: "none",
|
||||
userSelect: "none",
|
||||
}}
|
||||
/>
|
||||
) : null}
|
||||
{(() => {
|
||||
const filteredAnnotations = (annotationRegions || []).filter((annotation) => {
|
||||
if (
|
||||
@@ -1672,11 +1732,24 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
ref={videoRef}
|
||||
src={videoPath}
|
||||
className="hidden"
|
||||
preload="metadata"
|
||||
preload="auto"
|
||||
muted
|
||||
playsInline
|
||||
onLoadedMetadata={handleLoadedMetadata}
|
||||
onDurationChange={(e) => {
|
||||
onDurationChange(e.currentTarget.duration);
|
||||
if (!syncResolvedDuration(e.currentTarget)) {
|
||||
forceResolveDuration(e.currentTarget);
|
||||
}
|
||||
}}
|
||||
onLoadedData={(e) => {
|
||||
if (!syncResolvedDuration(e.currentTarget)) {
|
||||
forceResolveDuration(e.currentTarget);
|
||||
}
|
||||
}}
|
||||
onCanPlay={(e) => {
|
||||
if (!syncResolvedDuration(e.currentTarget)) {
|
||||
forceResolveDuration(e.currentTarget);
|
||||
}
|
||||
}}
|
||||
onError={() => onError("Failed to load video")}
|
||||
/>
|
||||
|
||||
@@ -51,6 +51,7 @@ const SUGGESTION_SPACING_MS = 1800;
|
||||
|
||||
interface TimelineEditorProps {
|
||||
videoDuration: number;
|
||||
hasVideoSource?: boolean;
|
||||
currentTime: number;
|
||||
onSeek?: (time: number) => void;
|
||||
cursorTelemetry?: CursorTelemetryPoint[];
|
||||
@@ -766,6 +767,7 @@ function Timeline({
|
||||
|
||||
export default function TimelineEditor({
|
||||
videoDuration,
|
||||
hasVideoSource = false,
|
||||
currentTime,
|
||||
onSeek,
|
||||
cursorTelemetry = [],
|
||||
@@ -1439,8 +1441,14 @@ export default function TimelineEditor({
|
||||
<Plus className="w-6 h-6 text-slate-600" />
|
||||
</div>
|
||||
<div className="text-center">
|
||||
<p className="text-sm font-medium text-slate-300">{t("emptyState.noVideo")}</p>
|
||||
<p className="text-xs text-slate-500 mt-1">{t("emptyState.dragAndDrop")}</p>
|
||||
<p className="text-sm font-medium text-slate-300">
|
||||
{hasVideoSource ? "Loading Timeline" : "No Video Loaded"}
|
||||
</p>
|
||||
<p className="text-xs text-slate-500 mt-1">
|
||||
{hasVideoSource
|
||||
? "Video opened, waiting for duration metadata"
|
||||
: "Drag and drop a video to start editing"}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
import { type Container, Point } from "pixi.js";
|
||||
import type { CropRegion } from "@/components/video-editor/types";
|
||||
import type {
|
||||
CursorRecordingData,
|
||||
CursorRecordingSample,
|
||||
NativeCursorAsset,
|
||||
} from "@/native/contracts";
|
||||
|
||||
export interface ActiveNativeCursorFrame {
|
||||
asset: NativeCursorAsset;
|
||||
sample: CursorRecordingSample;
|
||||
}
|
||||
|
||||
interface ProjectNativeCursorOptions {
|
||||
cameraContainer: Container;
|
||||
cropRegion: CropRegion;
|
||||
maskRect: { width: number; height: number };
|
||||
videoContainerPosition: { x: number; y: number };
|
||||
sample: CursorRecordingSample;
|
||||
}
|
||||
|
||||
function clamp(value: number, min: number, max: number) {
|
||||
return Math.min(max, Math.max(min, value));
|
||||
}
|
||||
|
||||
function getCroppedCursorPosition(sample: CursorRecordingSample, cropRegion: CropRegion) {
|
||||
if (cropRegion.width <= 0 || cropRegion.height <= 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const croppedCx = (sample.cx - cropRegion.x) / cropRegion.width;
|
||||
const croppedCy = (sample.cy - cropRegion.y) / cropRegion.height;
|
||||
|
||||
if (croppedCx < 0 || croppedCx > 1 || croppedCy < 0 || croppedCy > 1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
cx: clamp(croppedCx, 0, 1),
|
||||
cy: clamp(croppedCy, 0, 1),
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveActiveNativeCursorFrame(
|
||||
recordingData: CursorRecordingData | null | undefined,
|
||||
timeMs: number,
|
||||
): ActiveNativeCursorFrame | null {
|
||||
if (!recordingData || recordingData.provider !== "native" || recordingData.assets.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
for (let index = recordingData.samples.length - 1; index >= 0; index -= 1) {
|
||||
const sample = recordingData.samples[index];
|
||||
if (sample.timeMs > timeMs) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (sample.visible === false || !sample.assetId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const asset = recordingData.assets.find((candidate) => candidate.id === sample.assetId);
|
||||
if (!asset) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return { sample, asset };
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export function projectNativeCursorToStage({
|
||||
cameraContainer,
|
||||
cropRegion,
|
||||
maskRect,
|
||||
videoContainerPosition,
|
||||
sample,
|
||||
}: ProjectNativeCursorOptions) {
|
||||
const croppedPosition = getCroppedCursorPosition(sample, cropRegion);
|
||||
if (!croppedPosition) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const localPoint = new Point(
|
||||
videoContainerPosition.x + croppedPosition.cx * maskRect.width,
|
||||
videoContainerPosition.y + croppedPosition.cy * maskRect.height,
|
||||
);
|
||||
|
||||
return cameraContainer.toGlobal(localPoint);
|
||||
}
|
||||
|
||||
export function getNativeCursorDisplayMetrics(asset: NativeCursorAsset, deviceScaleFactor: number) {
|
||||
const scaleFactor = asset.scaleFactor ?? deviceScaleFactor ?? 1;
|
||||
return {
|
||||
width: asset.width / scaleFactor,
|
||||
height: asset.height / scaleFactor,
|
||||
hotspotX: asset.hotspotX / scaleFactor,
|
||||
hotspotY: asset.hotspotY / scaleFactor,
|
||||
};
|
||||
}
|
||||
@@ -56,8 +56,14 @@ import {
|
||||
type Size,
|
||||
type StyledRenderRect,
|
||||
} from "@/lib/compositeLayout";
|
||||
import {
|
||||
getNativeCursorDisplayMetrics,
|
||||
projectNativeCursorToStage,
|
||||
resolveActiveNativeCursorFrame,
|
||||
} from "@/lib/cursor/nativeCursor";
|
||||
import { BackgroundLoadError, classifyWallpaper, resolveImageWallpaperUrl } from "@/lib/wallpaper";
|
||||
import { drawCanvasClipPath } from "@/lib/webcamMaskShapes";
|
||||
import type { CursorRecordingData, NativeCursorAsset } from "@/native/contracts";
|
||||
import { renderAnnotations } from "./annotationRenderer";
|
||||
import {
|
||||
getLinearGradientPoints,
|
||||
@@ -79,6 +85,7 @@ interface FrameRenderConfig {
|
||||
borderRadius?: number;
|
||||
padding?: number;
|
||||
cropRegion: CropRegion;
|
||||
cursorRecordingData?: CursorRecordingData | null;
|
||||
videoWidth: number;
|
||||
videoHeight: number;
|
||||
webcamSize?: Size | null;
|
||||
@@ -136,6 +143,7 @@ export class FrameRenderer {
|
||||
private rasterCtx: CanvasRenderingContext2D | null = null;
|
||||
private threeDPass: ThreeDPass | null = null;
|
||||
private currentRotation3D: Rotation3D = { ...DEFAULT_ROTATION_3D };
|
||||
private cursorImageCache = new Map<string, HTMLImageElement>();
|
||||
private config: FrameRenderConfig;
|
||||
private animationState: AnimationState;
|
||||
private layoutCache: LayoutCache | null = null;
|
||||
@@ -468,6 +476,8 @@ export class FrameRenderer {
|
||||
}
|
||||
}
|
||||
|
||||
await this.drawNativeCursor(timeMs);
|
||||
|
||||
// Render annotations on top of foreground (so they rotate with recording).
|
||||
if (
|
||||
this.config.annotationRegions &&
|
||||
@@ -543,7 +553,63 @@ export class FrameRenderer {
|
||||
}
|
||||
}
|
||||
|
||||
private updateLayout(webcamFrame?: VideoFrame | null): void {
|
||||
private async drawNativeCursor(timeMs: number) {
|
||||
if (!this.compositeCtx || !this.cameraContainer || !this.videoContainer || !this.layoutCache) {
|
||||
return;
|
||||
}
|
||||
|
||||
const activeNativeCursor = resolveActiveNativeCursorFrame(
|
||||
this.config.cursorRecordingData,
|
||||
timeMs,
|
||||
);
|
||||
if (!activeNativeCursor) {
|
||||
return;
|
||||
}
|
||||
|
||||
const projectedPoint = projectNativeCursorToStage({
|
||||
cameraContainer: this.cameraContainer,
|
||||
cropRegion: this.config.cropRegion,
|
||||
maskRect: this.layoutCache.maskRect,
|
||||
videoContainerPosition: {
|
||||
x: this.videoContainer.x,
|
||||
y: this.videoContainer.y,
|
||||
},
|
||||
sample: activeNativeCursor.sample,
|
||||
});
|
||||
if (!projectedPoint) {
|
||||
return;
|
||||
}
|
||||
|
||||
const image = await this.getCursorImage(activeNativeCursor.asset);
|
||||
const metrics = getNativeCursorDisplayMetrics(activeNativeCursor.asset, 1);
|
||||
|
||||
this.compositeCtx.drawImage(
|
||||
image,
|
||||
projectedPoint.x - metrics.hotspotX,
|
||||
projectedPoint.y - metrics.hotspotY,
|
||||
metrics.width,
|
||||
metrics.height,
|
||||
);
|
||||
}
|
||||
|
||||
private async getCursorImage(asset: NativeCursorAsset) {
|
||||
const cachedImage = this.cursorImageCache.get(asset.id);
|
||||
if (cachedImage) {
|
||||
return cachedImage;
|
||||
}
|
||||
|
||||
const image = new Image();
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
image.onload = () => resolve();
|
||||
image.onerror = () => reject(new Error(`Failed to load cursor asset ${asset.id}`));
|
||||
image.src = asset.imageDataUrl;
|
||||
});
|
||||
|
||||
this.cursorImageCache.set(asset.id, image);
|
||||
return image;
|
||||
}
|
||||
|
||||
private updateLayout(): void {
|
||||
if (!this.app || !this.videoSprite || !this.maskGraphics || !this.videoContainer) return;
|
||||
|
||||
const { width, height } = this.config;
|
||||
@@ -999,5 +1065,6 @@ export class FrameRenderer {
|
||||
this.threeDPass.destroy();
|
||||
this.threeDPass = null;
|
||||
}
|
||||
this.cursorImageCache.clear();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import type {
|
||||
import { BackgroundLoadError } from "@/lib/wallpaper";
|
||||
import { getPlatform } from "@/utils/platformUtils";
|
||||
import { AsyncVideoFrameQueue } from "./asyncVideoFrameQueue";
|
||||
import type { CursorRecordingData } from "@/native/contracts";
|
||||
import { FrameRenderer } from "./frameRenderer";
|
||||
import { StreamingVideoDecoder } from "./streamingDecoder";
|
||||
import type {
|
||||
@@ -47,6 +48,7 @@ interface GifExporterConfig {
|
||||
webcamMaskShape?: import("@/components/video-editor/types").WebcamMaskShape;
|
||||
webcamSizePreset?: WebcamSizePreset;
|
||||
webcamPosition?: { cx: number; cy: number } | null;
|
||||
cursorRecordingData?: CursorRecordingData | null;
|
||||
annotationRegions?: AnnotationRegion[];
|
||||
previewWidth?: number;
|
||||
previewHeight?: number;
|
||||
@@ -151,6 +153,7 @@ export class GifExporter {
|
||||
borderRadius: this.config.borderRadius,
|
||||
padding: this.config.padding,
|
||||
cropRegion: this.config.cropRegion,
|
||||
cursorRecordingData: this.config.cursorRecordingData,
|
||||
videoWidth: videoInfo.width,
|
||||
videoHeight: videoInfo.height,
|
||||
webcamSize: webcamInfo ? { width: webcamInfo.width, height: webcamInfo.height } : null,
|
||||
|
||||
@@ -10,6 +10,7 @@ import type {
|
||||
import { BackgroundLoadError } from "@/lib/wallpaper";
|
||||
import { getPlatform } from "@/utils/platformUtils";
|
||||
import { AsyncVideoFrameQueue } from "./asyncVideoFrameQueue";
|
||||
import type { CursorRecordingData } from "@/native/contracts";
|
||||
import { AudioProcessor } from "./audioEncoder";
|
||||
import { FrameRenderer } from "./frameRenderer";
|
||||
import { VideoMuxer } from "./muxer";
|
||||
@@ -38,6 +39,7 @@ interface VideoExporterConfig extends ExportConfig {
|
||||
webcamMaskShape?: import("@/components/video-editor/types").WebcamMaskShape;
|
||||
webcamSizePreset?: WebcamSizePreset;
|
||||
webcamPosition?: { cx: number; cy: number } | null;
|
||||
cursorRecordingData?: CursorRecordingData | null;
|
||||
annotationRegions?: AnnotationRegion[];
|
||||
previewWidth?: number;
|
||||
previewHeight?: number;
|
||||
@@ -146,6 +148,7 @@ export class VideoExporter {
|
||||
borderRadius: this.config.borderRadius,
|
||||
padding: this.config.padding,
|
||||
cropRegion: this.config.cropRegion,
|
||||
cursorRecordingData: this.config.cursorRecordingData,
|
||||
videoWidth: videoInfo.width,
|
||||
videoHeight: videoInfo.height,
|
||||
webcamSize: webcamInfo ? { width: webcamInfo.width, height: webcamInfo.height } : null,
|
||||
|
||||
Reference in New Issue
Block a user