From 4f68df1db8f4176f1fac048b634a2bb276c67e92 Mon Sep 17 00:00:00 2001 From: Siddharth Date: Thu, 12 Mar 2026 21:16:20 -0700 Subject: [PATCH] fix exporter --- src/lib/exporter/streamingDecoder.ts | 232 ++++++++++++++++++--------- 1 file changed, 155 insertions(+), 77 deletions(-) diff --git a/src/lib/exporter/streamingDecoder.ts b/src/lib/exporter/streamingDecoder.ts index 8d4eb24..2d0a5a3 100644 --- a/src/lib/exporter/streamingDecoder.ts +++ b/src/lib/exporter/streamingDecoder.ts @@ -5,6 +5,7 @@ export interface DecodedVideoInfo { width: number; height: number; duration: number; // seconds + streamDuration?: number; // seconds frameRate: number; codec: string; hasAudio: boolean; @@ -23,7 +24,7 @@ type OnFrameCallback = ( * Way faster than seeking an HTMLVideoElement per frame. * * Frames in trimmed regions are decoded (needed for P/B-frame state) but discarded. - * Non-trimmed frames get buffered per segment and resampled to the target frame rate. + * Kept frames are resampled to the target frame rate in a streaming pass. */ export class StreamingVideoDecoder { private demuxer: WebDemuxer | null = null; @@ -61,6 +62,10 @@ export class StreamingVideoDecoder { width: videoStream?.width || 1920, height: videoStream?.height || 1080, duration: mediaInfo.duration, + streamDuration: + typeof videoStream?.duration === "number" && Number.isFinite(videoStream.duration) + ? videoStream.duration + : undefined, frameRate, codec: videoStream?.codec_string || "unknown", hasAudio: !!audioStream, @@ -81,11 +86,17 @@ export class StreamingVideoDecoder { } const decoderConfig = await this.demuxer.getDecoderConfig("video"); + const codec = this.metadata.codec.toLowerCase(); + const shouldPreferSoftwareDecode = codec.includes("av01") || codec.includes("av1"); const segments = this.splitBySpeed( this.computeSegments(this.metadata.duration, trimRegions), speedRegions, ); + const segmentOutputFrameCounts = segments.map((segment) => + Math.ceil(((segment.endSec - segment.startSec) / segment.speed) * targetFrameRate), + ); const frameDurationUs = 1_000_000 / targetFrameRate; + const epsilonSec = 0.001; // Async frame queue — decoder pushes, consumer pulls const pendingFrames: VideoFrame[] = []; @@ -112,7 +123,22 @@ export class StreamingVideoDecoder { } }, }); - this.decoder.configure(decoderConfig); + const preferredDecoderConfig = shouldPreferSoftwareDecode + ? { + ...decoderConfig, + hardwareAcceleration: "prefer-software" as const, + } + : decoderConfig; + + try { + this.decoder.configure(preferredDecoderConfig); + } catch (error) { + if (!shouldPreferSoftwareDecode) { + throw error; + } + // Fall back to default decoder config if software preference isn't supported. + this.decoder.configure(decoderConfig); + } const getNextFrame = (): Promise => { if (decodeError) throw decodeError; @@ -123,8 +149,10 @@ export class StreamingVideoDecoder { }); }; - // One forward stream through the whole file - const reader = this.demuxer.read("video").getReader(); + // One forward stream through the whole file. + // Pass explicit range because some containers are truncated when no end is provided. + const readEndSec = Math.max(this.metadata.duration, this.metadata.streamDuration ?? 0) + 0.5; + const reader = this.demuxer.read("video", 0, readEndSec).getReader(); // Feed chunks to decoder in background with backpressure const feedPromise = (async () => { @@ -133,7 +161,11 @@ export class StreamingVideoDecoder { const { done, value: chunk } = await reader.read(); if (done || !chunk) break; - while (this.decoder!.decodeQueueSize > 10 && !this.cancelled) { + // Backpressure on both decode queue and decoded frame backlog. + while ( + (this.decoder!.decodeQueueSize > 10 || pendingFrames.length > 24) && + !this.cancelled + ) { await new Promise((resolve) => setTimeout(resolve, 1)); } if (this.cancelled) break; @@ -158,65 +190,131 @@ export class StreamingVideoDecoder { // Route decoded frames into segments by timestamp, then deliver with VFR→CFR resampling let segmentIdx = 0; + let segmentFrameIndex = 0; let exportFrameIndex = 0; - let segmentBuffer: VideoFrame[] = []; + let lastDecodedFrameSec: number | null = null; + let heldFrame: VideoFrame | null = null; + let heldFrameSec = 0; + + const emitHeldFrameForTarget = async (segment: { + startSec: number; + endSec: number; + speed: number; + }) => { + if (!heldFrame) return false; + const segmentFrameCount = segmentOutputFrameCounts[segmentIdx]; + if (segmentFrameIndex >= segmentFrameCount) return false; + + const sourceTimeSec = + segment.startSec + (segmentFrameIndex / targetFrameRate) * segment.speed; + if (sourceTimeSec >= segment.endSec - epsilonSec) return false; + + const clone = new VideoFrame(heldFrame, { timestamp: heldFrame.timestamp }); + await onFrame(clone, exportFrameIndex * frameDurationUs, sourceTimeSec * 1000); + segmentFrameIndex++; + exportFrameIndex++; + return true; + }; while (!this.cancelled && segmentIdx < segments.length) { const frame = await getNextFrame(); if (!frame) break; const frameTimeSec = frame.timestamp / 1_000_000; - const currentSegment = segments[segmentIdx]; + lastDecodedFrameSec = frameTimeSec; - // Before current segment — trimmed or pre-video - if (frameTimeSec < currentSegment.startSec - 0.001) { + // Finalize completed segments before handling this frame. + while ( + segmentIdx < segments.length && + frameTimeSec >= segments[segmentIdx].endSec - epsilonSec + ) { + const segment = segments[segmentIdx]; + while (!this.cancelled && (await emitHeldFrameForTarget(segment))) { + // Keep emitting remaining output frames for this segment from the last known frame. + } + + segmentIdx++; + segmentFrameIndex = 0; + if ( + heldFrame && + segmentIdx < segments.length && + heldFrameSec < segments[segmentIdx].startSec - epsilonSec + ) { + heldFrame.close(); + heldFrame = null; + } + } + + if (segmentIdx >= segments.length) { frame.close(); continue; } - // Past current segment — flush buffer and advance - if (frameTimeSec >= currentSegment.endSec - 0.001) { - exportFrameIndex = await this.deliverSegment( - segmentBuffer, - currentSegment, - targetFrameRate, - frameDurationUs, - exportFrameIndex, - onFrame, - ); - for (const f of segmentBuffer) f.close(); - segmentBuffer = []; + const currentSegment = segments[segmentIdx]; - segmentIdx++; - while ( - segmentIdx < segments.length && - frameTimeSec >= segments[segmentIdx].endSec - 0.001 - ) { - segmentIdx++; - } - - if (segmentIdx < segments.length && frameTimeSec >= segments[segmentIdx].startSec - 0.001) { - segmentBuffer.push(frame); - } else { - frame.close(); - } + // Before current segment (trimmed region or pre-roll). + if (frameTimeSec < currentSegment.startSec - epsilonSec) { + frame.close(); continue; } - segmentBuffer.push(frame); + if (!heldFrame) { + heldFrame = frame; + heldFrameSec = frameTimeSec; + continue; + } + + // Any target timestamp before this midpoint is closer to heldFrame than current frame. + const handoffBoundarySec = (heldFrameSec + frameTimeSec) / 2; + while (!this.cancelled) { + const segmentFrameCount = segmentOutputFrameCounts[segmentIdx]; + if (segmentFrameIndex >= segmentFrameCount) { + break; + } + + const sourceTimeSec = + currentSegment.startSec + (segmentFrameIndex / targetFrameRate) * currentSegment.speed; + if (sourceTimeSec >= currentSegment.endSec - epsilonSec) { + break; + } + if (sourceTimeSec > handoffBoundarySec) { + break; + } + + const clone = new VideoFrame(heldFrame, { timestamp: heldFrame.timestamp }); + await onFrame(clone, exportFrameIndex * frameDurationUs, sourceTimeSec * 1000); + segmentFrameIndex++; + exportFrameIndex++; + } + + heldFrame.close(); + heldFrame = frame; + heldFrameSec = frameTimeSec; } - // Flush last segment - if (segmentBuffer.length > 0 && segmentIdx < segments.length) { - exportFrameIndex = await this.deliverSegment( - segmentBuffer, - segments[segmentIdx], - targetFrameRate, - frameDurationUs, - exportFrameIndex, - onFrame, - ); - for (const f of segmentBuffer) f.close(); + // Flush remaining output frames for the last decoded frame. + if (heldFrame && segmentIdx < segments.length) { + while (!this.cancelled && segmentIdx < segments.length) { + const segment = segments[segmentIdx]; + if (heldFrameSec < segment.startSec - epsilonSec) { + break; + } + + while (!this.cancelled && (await emitHeldFrameForTarget(segment))) { + // Keep emitting output frames for the active segment. + } + + segmentIdx++; + segmentFrameIndex = 0; + if ( + segmentIdx < segments.length && + heldFrameSec < segments[segmentIdx].startSec - epsilonSec + ) { + break; + } + } + heldFrame.close(); + heldFrame = null; } // Drain leftover decoded frames @@ -239,39 +337,17 @@ export class StreamingVideoDecoder { this.decoder.close(); } this.decoder = null; - } - /** - * Resample buffered frames to fill the target frame count for this segment. - * Handles VFR sources by duplicating/decimating as needed. - */ - private async deliverSegment( - frames: VideoFrame[], - segment: { startSec: number; endSec: number; speed: number }, - targetFrameRate: number, - frameDurationUs: number, - startExportFrameIndex: number, - onFrame: OnFrameCallback, - ): Promise { - if (frames.length === 0) return startExportFrameIndex; - - const segmentFrameCount = Math.ceil( - ((segment.endSec - segment.startSec) / segment.speed) * targetFrameRate, - ); - let exportFrameIndex = startExportFrameIndex; - - for (let i = 0; i < segmentFrameCount && !this.cancelled; i++) { - const sourceIdx = Math.min( - Math.floor((i * frames.length) / segmentFrameCount), - frames.length - 1, + const requiredEndSec = segments.length > 0 ? segments[segments.length - 1].endSec : 0; + if ( + !this.cancelled && + lastDecodedFrameSec !== null && + requiredEndSec - lastDecodedFrameSec > 1 + ) { + throw new Error( + `Video decode ended early at ${lastDecodedFrameSec.toFixed(3)}s (needed ${requiredEndSec.toFixed(3)}s).`, ); - const sourceFrame = frames[sourceIdx]; - const clone = new VideoFrame(sourceFrame, { timestamp: sourceFrame.timestamp }); - await onFrame(clone, exportFrameIndex * frameDurationUs, sourceFrame.timestamp / 1000); - exportFrameIndex++; } - - return exportFrameIndex; } private computeSegments( @@ -364,7 +440,9 @@ export class StreamingVideoDecoder { if (this.demuxer) { try { this.demuxer.destroy(); - } catch {} + } catch { + /* ignore */ + } this.demuxer = null; } }