From 87240a919ee1aa25426e0795fde2dee4c9586ab0 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Tue, 5 May 2026 11:04:08 +0200 Subject: [PATCH] fix: align native cursor preview and export --- electron/ipc/handlers.ts | 31 ++++++++- .../native-bridge/cursor/recording/factory.ts | 3 + .../recording/telemetryRecordingSession.ts | 3 +- .../windowsNativeRecordingSession.ts | 2 +- .../windowsNativeRecordingSession.types.ts | 1 + scripts/capture-openscreen-preview.mjs | 4 -- src/components/video-editor/VideoPlayback.tsx | 64 ++++++++----------- src/hooks/useScreenRecorder.ts | 6 +- src/lib/cursor/nativeCursor.ts | 19 ++++-- 9 files changed, 82 insertions(+), 51 deletions(-) diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 7b16f2f..4c306ee 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -651,7 +651,7 @@ export function registerIpcHandlers( } }); - ipcMain.handle("set-recording-state", async (_, recording: boolean) => { + ipcMain.handle("set-recording-state", async (_, recording: boolean, recordingId?: number) => { if (recording) { if (cursorRecordingSession) { pendingCursorRecordingData = await cursorRecordingSession.stop(); @@ -665,6 +665,8 @@ export function registerIpcHandlers( platform: process.platform, sampleIntervalMs: CURSOR_SAMPLE_INTERVAL_MS, sourceId: getSelectedSourceId(), + startTimeMs: + typeof recordingId === "number" && Number.isFinite(recordingId) ? recordingId : undefined, }); try { @@ -824,6 +826,7 @@ export function registerIpcHandlers( return { success: false, canceled: true }; } + approveFilePath(result.filePaths[0]); currentProjectPath = null; return { success: true, @@ -863,6 +866,32 @@ export function registerIpcHandlers( } }); + ipcMain.handle("read-binary-file", async (_, filePath: string) => { + try { + const normalizedPath = await approveReadableVideoPath(filePath); + if (!normalizedPath) { + return { + success: false, + message: "File path is not approved or is not a supported video file", + }; + } + + const data = await fs.readFile(normalizedPath); + return { + success: true, + data: data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength), + path: normalizedPath, + }; + } catch (error) { + console.error("Failed to read binary file:", error); + return { + success: false, + message: "Failed to read binary file", + error: String(error), + }; + } + }); + ipcMain.handle( "save-project-file", async (_, projectData: unknown, suggestedName?: string, existingProjectPath?: string) => { diff --git a/electron/native-bridge/cursor/recording/factory.ts b/electron/native-bridge/cursor/recording/factory.ts index 4e0f75c..52d6079 100644 --- a/electron/native-bridge/cursor/recording/factory.ts +++ b/electron/native-bridge/cursor/recording/factory.ts @@ -9,6 +9,7 @@ interface CreateCursorRecordingSessionOptions { platform: NodeJS.Platform; sampleIntervalMs: number; sourceId?: string | null; + startTimeMs?: number; } export function createCursorRecordingSession( @@ -20,6 +21,7 @@ export function createCursorRecordingSession( maxSamples: options.maxSamples, sampleIntervalMs: options.sampleIntervalMs, sourceId: options.sourceId, + startTimeMs: options.startTimeMs, }); } @@ -27,5 +29,6 @@ export function createCursorRecordingSession( getDisplayBounds: options.getDisplayBounds, maxSamples: options.maxSamples, sampleIntervalMs: options.sampleIntervalMs, + startTimeMs: options.startTimeMs, }); } diff --git a/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts b/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts index dd42871..e719d8e 100644 --- a/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts +++ b/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts @@ -6,6 +6,7 @@ interface TelemetryRecordingSessionOptions { getDisplayBounds: () => Rectangle | null; maxSamples: number; sampleIntervalMs: number; + startTimeMs?: number; } function clamp(value: number, min: number, max: number) { @@ -21,7 +22,7 @@ export class TelemetryRecordingSession implements CursorRecordingSession { async start(): Promise { this.samples = []; - this.startTimeMs = Date.now(); + this.startTimeMs = this.options.startTimeMs ?? Date.now(); this.captureSample(); this.interval = setInterval(() => { this.captureSample(); diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts index 632a74d..8075fe3 100644 --- a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts @@ -41,7 +41,7 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession { this.assets.clear(); this.samples = []; this.lineBuffer = ""; - this.startTimeMs = Date.now(); + this.startTimeMs = this.options.startTimeMs ?? Date.now(); this.sampleCount = 0; this.outOfBoundsSampleCount = 0; diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts index fdc4ab9..5afc012 100644 --- a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts @@ -49,4 +49,5 @@ export interface WindowsNativeRecordingSessionOptions { maxSamples: number; sampleIntervalMs: number; sourceId?: string | null; + startTimeMs?: number; } diff --git a/scripts/capture-openscreen-preview.mjs b/scripts/capture-openscreen-preview.mjs index 6c9b6eb..25f86db 100644 --- a/scripts/capture-openscreen-preview.mjs +++ b/scripts/capture-openscreen-preview.mjs @@ -214,10 +214,6 @@ try { await editorWindow.waitForLoadState("domcontentloaded"); await editorWindow.waitForSelector("video", { state: "attached", timeout: 30_000 }); await editorWindow.waitForSelector("canvas", { state: "attached", timeout: 30_000 }); - await editorWindow.waitForSelector('img[aria-hidden="true"]', { - state: "attached", - timeout: 30_000, - }); await editorWindow.setViewportSize({ width: 1280, height: 800 }); await editorWindow.evaluate(async () => { diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx index 0586e54..840101d 100644 --- a/src/components/video-editor/VideoPlayback.tsx +++ b/src/components/video-editor/VideoPlayback.tsx @@ -27,7 +27,7 @@ import { } from "@/lib/compositeLayout"; import { hasNativeCursorRecordingData, - projectNativeCursorToStage, + projectNativeCursorToLocal, resolveInterpolatedNativeCursorFrame, resolveNativeCursorRenderAsset, } from "@/lib/cursor/nativeCursor"; @@ -841,6 +841,12 @@ const VideoPlayback = forwardRef( cursorContainerRef.current = cursorContainer; cameraContainer.addChild(cursorContainer); + const nativeCursorSprite = new Sprite(Texture.EMPTY); + nativeCursorSprite.visible = false; + nativeCursorSprite.eventMode = "none"; + nativeCursorSpriteRef.current = nativeCursorSprite; + cursorContainer.addChild(nativeCursorSprite); + // Cursor overlay - rendered above the masked video if (cursorOverlayEnabled) { const cursorOverlay = new PixiCursorOverlay({ @@ -863,6 +869,8 @@ const VideoPlayback = forwardRef( cursorOverlayRef.current.destroy(); cursorOverlayRef.current = null; } + nativeCursorSpriteRef.current = null; + nativeCursorTextureIdRef.current = null; if (app && app.renderer) { app.destroy(true, { children: true, @@ -1296,25 +1304,18 @@ const VideoPlayback = forwardRef( ); } - // Update native cursor image position at ticker rate (60fps) - const nativeCursorImg = nativeCursorImgRef.current; - if (nativeCursorImg) { - const cameraContainerRc = cameraContainerRef.current; + // Update native cursor sprite in the same PIXI coordinate space as the video. + const nativeCursorSprite = nativeCursorSpriteRef.current; + if (nativeCursorSprite) { const videoContainerRc = videoContainerRef.current; - if ( - hasNativeCursorRecordingRef.current && - showCursorRef.current && - cameraContainerRc && - videoContainerRc - ) { + if (hasNativeCursorRecordingRef.current && showCursorRef.current && videoContainerRc) { const timeMs = currentTimeRef.current; // already in ms const frame = resolveInterpolatedNativeCursorFrame( cursorRecordingDataRef.current, timeMs, ); if (frame) { - const projectedPoint = projectNativeCursorToStage({ - cameraContainer: cameraContainerRc, + const projectedPoint = projectNativeCursorToLocal({ cropRegion: cropRegionRef.current ?? { x: 0, y: 0, width: 1, height: 1 }, maskRect: baseMaskRef.current, videoContainerPosition: { @@ -1330,23 +1331,25 @@ const VideoPlayback = forwardRef( frame.sample, ); const scale = Math.max(0, cursorSizeRef.current); - if (nativeCursorImg.dataset.cursorId !== renderAsset.id) { - nativeCursorImg.src = renderAsset.imageDataUrl; - nativeCursorImg.dataset.cursorId = renderAsset.id; + if (nativeCursorTextureIdRef.current !== renderAsset.id) { + nativeCursorSprite.texture = Texture.from(renderAsset.imageDataUrl); + nativeCursorTextureIdRef.current = renderAsset.id; } - nativeCursorImg.style.left = `${projectedPoint.x - renderAsset.hotspotX * scale}px`; - nativeCursorImg.style.top = `${projectedPoint.y - renderAsset.hotspotY * scale}px`; - nativeCursorImg.style.width = `${renderAsset.width * scale}px`; - nativeCursorImg.style.height = `${renderAsset.height * scale}px`; - nativeCursorImg.style.display = "block"; + nativeCursorSprite.position.set( + projectedPoint.x - renderAsset.hotspotX * scale, + projectedPoint.y - renderAsset.hotspotY * scale, + ); + nativeCursorSprite.width = renderAsset.width * scale; + nativeCursorSprite.height = renderAsset.height * scale; + nativeCursorSprite.visible = true; } else { - nativeCursorImg.style.display = "none"; + nativeCursorSprite.visible = false; } } else { - nativeCursorImg.style.display = "none"; + nativeCursorSprite.visible = false; } } else { - nativeCursorImg.style.display = "none"; + nativeCursorSprite.visible = false; } } @@ -1638,19 +1641,6 @@ const VideoPlayback = forwardRef( className="absolute rounded-md border border-[#34B27B]/80 bg-[#34B27B]/20 shadow-[0_0_0_1px_rgba(52,178,123,0.35)]" style={{ display: "none", pointerEvents: "none" }} /> - {hasNativeCursorRecording ? ( - - ) : null} {(() => { const filteredAnnotations = (annotationRegions || []).filter((annotation) => { if ( diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 7cd86a7..717a6cd 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -719,6 +719,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { return; } + recordingId.current = Date.now(); + const activeRecordingId = recordingId.current; screenRecorder.current = createRecorderHandle(stream.current, { mimeType, videoBitsPerSecond, @@ -741,9 +743,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }); } - recordingId.current = Date.now(); accumulatedDurationMs.current = 0; - segmentStartedAt.current = Date.now(); + segmentStartedAt.current = activeRecordingId; allowAutoFinalize.current = true; setRecording(true); setPaused(false); @@ -752,7 +753,6 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const activeScreenRecorder = screenRecorder.current; const activeWebcamRecorder = webcamRecorder.current; - const activeRecordingId = recordingId.current; if (activeScreenRecorder) { activeScreenRecorder.recorder.addEventListener( "stop", diff --git a/src/lib/cursor/nativeCursor.ts b/src/lib/cursor/nativeCursor.ts index 30f7a46..5c38c4d 100644 --- a/src/lib/cursor/nativeCursor.ts +++ b/src/lib/cursor/nativeCursor.ts @@ -27,13 +27,16 @@ export interface ActiveNativeCursorFrame { } interface ProjectNativeCursorOptions { - cameraContainer: Container; cropRegion: CropRegion; maskRect: { x: number; y: number; width: number; height: number }; videoContainerPosition: { x: number; y: number }; sample: CursorRecordingSample; } +interface ProjectNativeCursorToStageOptions extends ProjectNativeCursorOptions { + cameraContainer: Container; +} + function clamp(value: number, min: number, max: number) { return Math.min(max, Math.max(min, value)); } @@ -264,8 +267,7 @@ export function resolveInterpolatedNativeCursorFrame( }; } -export function projectNativeCursorToStage({ - cameraContainer, +export function projectNativeCursorToLocal({ cropRegion, maskRect, videoContainerPosition, @@ -276,11 +278,20 @@ export function projectNativeCursorToStage({ return null; } - const localPoint = new Point( + return new Point( videoContainerPosition.x + maskRect.x + croppedPosition.cx * maskRect.width, videoContainerPosition.y + maskRect.y + croppedPosition.cy * maskRect.height, ); +} +export function projectNativeCursorToStage({ + cameraContainer, + ...options +}: ProjectNativeCursorToStageOptions) { + const localPoint = projectNativeCursorToLocal(options); + if (!localPoint) { + return null; + } return cameraContainer.toGlobal(localPoint); }