Merge branch 'main' into codex/editor-defaults-ssot

src/lib/exporter/gifExporter.ts

@@ -11,9 +11,9 @@ import type {
 import { BackgroundLoadError } from "@/lib/wallpaper";
 import type { CursorRecordingData } from "@/native/contracts";
 import { getPlatform } from "@/utils/platformUtils";
-import { AsyncVideoFrameQueue } from "./asyncVideoFrameQueue";
 import { FrameRenderer } from "./frameRenderer";
 import { StreamingVideoDecoder } from "./streamingDecoder";
+import { TimestampedVideoFrameQueue } from "./timestampedVideoFrameQueue";
 import type {
 	ExportProgress,
 	ExportResult,
@@ -124,7 +124,7 @@ export class GifExporter {
 	}
 
 	async export(): Promise<ExportResult> {
-		let webcamFrameQueue: AsyncVideoFrameQueue | null = null;
+		let webcamFrameQueue: TimestampedVideoFrameQueue | null = null;
 
 		const warnings: string[] = [];
 		const onWarning = (message: string) => warnings.push(message);
@@ -216,7 +216,7 @@ export class GifExporter {
 			console.log("[GifExporter] Using streaming decode (web-demuxer + VideoDecoder)");
 
 			let frameIndex = 0;
-			webcamFrameQueue = this.config.webcamVideoUrl ? new AsyncVideoFrameQueue() : null;
+			webcamFrameQueue = this.config.webcamVideoUrl ? new TimestampedVideoFrameQueue() : null;
 			let stopWebcamDecode = false;
 			let webcamDecodeError: Error | null = null;
 			const webcamDecodePromise =
@@ -228,7 +228,7 @@ export class GifExporter {
 									this.config.frameRate,
 									this.config.trimRegions,
 									this.config.speedRegions,
-									async (webcamFrame) => {
+									async (webcamFrame, _exportTimestampUs, webcamSourceTimestampMs) => {
 										while (queue.length >= 12 && !this.cancelled && !stopWebcamDecode) {
 											await new Promise((resolve) => setTimeout(resolve, 2));
 										}
@@ -236,7 +236,7 @@ export class GifExporter {
 											webcamFrame.close();
 											return;
 										}
-										queue.enqueue(webcamFrame);
+										queue.enqueue(webcamFrame, webcamSourceTimestampMs);
 									},
 									onWarning,
 								)
@@ -266,7 +266,9 @@ export class GifExporter {
 							return;
 						}
 
-						webcamFrame = webcamFrameQueue ? await webcamFrameQueue.dequeue() : null;
+						webcamFrame = webcamFrameQueue
+							? await webcamFrameQueue.frameAt(sourceTimestampMs)
+							: null;
 						const renderer = this.renderer;
 						if (this.cancelled || !renderer) {
 							return;

src/lib/exporter/timestampedVideoFrameQueue.test.ts

@@ -0,0 +1,93 @@
+import { describe, expect, it, vi } from "vitest";
+import { TimestampedVideoFrameQueue } from "./timestampedVideoFrameQueue";
+
+class MockVideoFrame {
+	timestamp: number;
+	closed = false;
+
+	constructor(source: MockVideoFrame | number) {
+		this.timestamp = typeof source === "number" ? source : source.timestamp;
+	}
+
+	close() {
+		this.closed = true;
+	}
+}
+
+function restoreVideoFrame(originalVideoFrame: typeof globalThis.VideoFrame | undefined) {
+	if (originalVideoFrame === undefined) {
+		delete (globalThis as { VideoFrame?: typeof globalThis.VideoFrame }).VideoFrame;
+		return;
+	}
+
+	vi.stubGlobal("VideoFrame", originalVideoFrame);
+}
+
+describe("TimestampedVideoFrameQueue", () => {
+	it("samples the latest webcam frame at or before the requested source timestamp", async () => {
+		const originalVideoFrame = globalThis.VideoFrame;
+		vi.stubGlobal("VideoFrame", MockVideoFrame);
+		try {
+			const queue = new TimestampedVideoFrameQueue();
+			const frame0 = new MockVideoFrame(0) as unknown as VideoFrame;
+			const frame33 = new MockVideoFrame(33_000) as unknown as VideoFrame;
+			const frame66 = new MockVideoFrame(66_000) as unknown as VideoFrame;
+
+			queue.enqueue(frame0, 0);
+			queue.enqueue(frame33, 33);
+			queue.enqueue(frame66, 66);
+			queue.close();
+
+			const sampled0 = await queue.frameAt(0);
+			const sampled20 = await queue.frameAt(20);
+			const sampled40 = await queue.frameAt(40);
+			const sampled80 = await queue.frameAt(80);
+
+			expect(sampled0?.timestamp).toBe(0);
+			expect(sampled20?.timestamp).toBe(0);
+			expect(sampled40?.timestamp).toBe(33_000);
+			expect(sampled80?.timestamp).toBe(66_000);
+
+			sampled0?.close();
+			sampled20?.close();
+			sampled40?.close();
+			sampled80?.close();
+			queue.destroy();
+		} finally {
+			restoreVideoFrame(originalVideoFrame);
+		}
+	});
+
+	it("waits for a newer frame before falling back to the held frame while open", async () => {
+		const originalVideoFrame = globalThis.VideoFrame;
+		vi.stubGlobal("VideoFrame", MockVideoFrame);
+		try {
+			const queue = new TimestampedVideoFrameQueue();
+			const frame0 = new MockVideoFrame(0) as unknown as VideoFrame;
+			const frame33 = new MockVideoFrame(33_000) as unknown as VideoFrame;
+
+			queue.enqueue(frame0, 0);
+			const sampled0 = await queue.frameAt(0);
+			let resolved = false;
+			const pending = queue.frameAt(33).then((frame) => {
+				resolved = true;
+				return frame;
+			});
+
+			await Promise.resolve();
+			expect(resolved).toBe(false);
+
+			queue.enqueue(frame33, 33);
+			const sampled33 = await pending;
+
+			expect(sampled0?.timestamp).toBe(0);
+			expect(sampled33?.timestamp).toBe(33_000);
+
+			sampled0?.close();
+			sampled33?.close();
+			queue.destroy();
+		} finally {
+			restoreVideoFrame(originalVideoFrame);
+		}
+	});
+});

src/lib/exporter/timestampedVideoFrameQueue.ts

@@ -0,0 +1,110 @@
+type TimestampedVideoFrame = {
+	frame: VideoFrame;
+	sourceTimestampMs: number;
+};
+
+type PendingConsumer = {
+	resolve: () => void;
+	reject: (error: Error) => void;
+};
+
+const TIMESTAMP_EPSILON_MS = 0.5;
+
+export class TimestampedVideoFrameQueue {
+	private frames: TimestampedVideoFrame[] = [];
+	private consumers: PendingConsumer[] = [];
+	private error: Error | null = null;
+	private closed = false;
+	private heldFrame: TimestampedVideoFrame | null = null;
+
+	get length() {
+		return this.frames.length;
+	}
+
+	enqueue(frame: VideoFrame, sourceTimestampMs: number) {
+		if (this.closed) {
+			frame.close();
+			return;
+		}
+
+		this.frames.push({ frame, sourceTimestampMs });
+		const consumers = this.consumers.splice(0);
+		for (const consumer of consumers) {
+			consumer.resolve();
+		}
+	}
+
+	fail(error: Error) {
+		this.error = error;
+		this.closed = true;
+		const consumers = this.consumers.splice(0);
+		for (const consumer of consumers) {
+			consumer.reject(error);
+		}
+		this.closeOwnedFrames();
+	}
+
+	close() {
+		this.closed = true;
+		const consumers = this.consumers.splice(0);
+		for (const consumer of consumers) {
+			consumer.resolve();
+		}
+	}
+
+	async frameAt(sourceTimestampMs: number): Promise<VideoFrame | null> {
+		for (;;) {
+			if (this.error) {
+				throw this.error;
+			}
+
+			const next = this.frames[0] ?? null;
+			if (next && next.sourceTimestampMs <= sourceTimestampMs + TIMESTAMP_EPSILON_MS) {
+				this.replaceHeldFrame(this.frames.shift() ?? null);
+				continue;
+			}
+
+			if (
+				this.heldFrame &&
+				(next ||
+					this.closed ||
+					this.heldFrame.sourceTimestampMs >= sourceTimestampMs - TIMESTAMP_EPSILON_MS)
+			) {
+				return new VideoFrame(this.heldFrame.frame, {
+					timestamp: this.heldFrame.frame.timestamp,
+				});
+			}
+
+			if (next || this.closed) {
+				return null;
+			}
+
+			await new Promise<void>((resolve, reject) => {
+				this.consumers.push({ resolve, reject });
+			});
+		}
+	}
+
+	destroy() {
+		this.close();
+		this.closeOwnedFrames();
+	}
+
+	private replaceHeldFrame(frame: TimestampedVideoFrame | null) {
+		if (this.heldFrame) {
+			this.heldFrame.frame.close();
+		}
+		this.heldFrame = frame;
+	}
+
+	private closeOwnedFrames() {
+		if (this.heldFrame) {
+			this.heldFrame.frame.close();
+			this.heldFrame = null;
+		}
+		for (const item of this.frames) {
+			item.frame.close();
+		}
+		this.frames = [];
+	}
+}

src/lib/exporter/videoExporter.ts

@@ -10,11 +10,11 @@ import type {
 import { BackgroundLoadError } from "@/lib/wallpaper";
 import type { CursorRecordingData } from "@/native/contracts";
 import { getPlatform } from "@/utils/platformUtils";
-import { AsyncVideoFrameQueue } from "./asyncVideoFrameQueue";
 import { AudioProcessor } from "./audioEncoder";
 import { FrameRenderer } from "./frameRenderer";
 import { VideoMuxer } from "./muxer";
 import { StreamingVideoDecoder } from "./streamingDecoder";
+import { TimestampedVideoFrameQueue } from "./timestampedVideoFrameQueue";
 import type { ExportConfig, ExportProgress, ExportResult } from "./types";
 
 const ENCODER_STALL_TIMEOUT_MS = 15_000;
@@ -195,7 +195,7 @@ export class VideoExporter {
 	private async exportWithEncoderPreference(
 		encoderPreference: HardwareAcceleration,
 	): Promise<ExportResult> {
-		let webcamFrameQueue: AsyncVideoFrameQueue | null = null;
+		let webcamFrameQueue: TimestampedVideoFrameQueue | null = null;
 		let stopWebcamDecode = false;
 		let webcamDecodeError: Error | null = null;
 		let webcamDecodePromise: Promise<void> | null = null;
@@ -290,7 +290,7 @@ export class VideoExporter {
 					? Math.min(this.MAX_ENCODE_QUEUE, 32)
 					: this.MAX_ENCODE_QUEUE;
 
-			webcamFrameQueue = this.config.webcamVideoUrl ? new AsyncVideoFrameQueue() : null;
+			webcamFrameQueue = this.config.webcamVideoUrl ? new TimestampedVideoFrameQueue() : null;
 			webcamDecodePromise =
 				webcamDecoder && webcamFrameQueue
 					? (() => {
@@ -300,7 +300,7 @@ export class VideoExporter {
 									this.config.frameRate,
 									this.config.trimRegions,
 									this.config.speedRegions,
-									async (webcamFrame) => {
+									async (webcamFrame, _exportTimestampUs, webcamSourceTimestampMs) => {
 										while (queue.length >= 12 && !this.cancelled && !stopWebcamDecode) {
 											await new Promise((resolve) => setTimeout(resolve, 2));
 										}
@@ -308,7 +308,7 @@ export class VideoExporter {
 											webcamFrame.close();
 											return;
 										}
-										queue.enqueue(webcamFrame);
+										queue.enqueue(webcamFrame, webcamSourceTimestampMs);
 									},
 									onWarning,
 								)
@@ -342,7 +342,9 @@ export class VideoExporter {
 						}
 
 						const timestamp = frameIndex * frameDuration;
-						webcamFrame = webcamFrameQueue ? await webcamFrameQueue.dequeue() : null;
+						webcamFrame = webcamFrameQueue
+							? await webcamFrameQueue.frameAt(sourceTimestampMs)
+							: null;
 						if (this.cancelled) {
 							return;
 						}