import { WebDemuxer } from "web-demuxer"; import type { SpeedRegion, TrimRegion } from "@/components/video-editor/types"; export interface DecodedVideoInfo { width: number; height: number; duration: number; // seconds frameRate: number; codec: string; hasAudio: boolean; audioCodec?: string; } /** Caller must close the VideoFrame after use. */ type OnFrameCallback = ( frame: VideoFrame, exportTimestampUs: number, sourceTimestampMs: number, ) => Promise; /** * Decodes video frames via web-demuxer + VideoDecoder in a single forward pass. * Way faster than seeking an HTMLVideoElement per frame. * * Frames in trimmed regions are decoded (needed for P/B-frame state) but discarded. * Non-trimmed frames get buffered per segment and resampled to the target frame rate. */ export class StreamingVideoDecoder { private demuxer: WebDemuxer | null = null; private decoder: VideoDecoder | null = null; private cancelled = false; private metadata: DecodedVideoInfo | null = null; async loadMetadata(videoUrl: string): Promise { const response = await fetch(videoUrl); const blob = await response.blob(); const filename = videoUrl.split("/").pop() || "video"; const file = new File([blob], filename, { type: blob.type }); // Relative URL so it resolves correctly in both dev (http) and packaged (file://) builds const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href; this.demuxer = new WebDemuxer({ wasmFilePath: wasmUrl }); await this.demuxer.load(file); const mediaInfo = await this.demuxer.getMediaInfo(); const videoStream = mediaInfo.streams.find((s) => s.codec_type_string === "video"); let frameRate = 60; if (videoStream?.avg_frame_rate) { const parts = videoStream.avg_frame_rate.split("/"); if (parts.length === 2) { const num = parseInt(parts[0], 10); const den = parseInt(parts[1], 10); if (den > 0 && num > 0) frameRate = num / den; } } const audioStream = mediaInfo.streams.find((s) => s.codec_type_string === "audio"); this.metadata = { width: videoStream?.width || 1920, height: videoStream?.height || 1080, duration: mediaInfo.duration, frameRate, codec: videoStream?.codec_string || "unknown", hasAudio: !!audioStream, audioCodec: audioStream?.codec_string, }; return this.metadata; } async decodeAll( targetFrameRate: number, trimRegions: TrimRegion[] | undefined, speedRegions: SpeedRegion[] | undefined, onFrame: OnFrameCallback, ): Promise { if (!this.demuxer || !this.metadata) { throw new Error("Must call loadMetadata() before decodeAll()"); } const decoderConfig = await this.demuxer.getDecoderConfig("video"); const segments = this.splitBySpeed( this.computeSegments(this.metadata.duration, trimRegions), speedRegions, ); const frameDurationUs = 1_000_000 / targetFrameRate; // Async frame queue — decoder pushes, consumer pulls const pendingFrames: VideoFrame[] = []; let frameResolve: ((frame: VideoFrame | null) => void) | null = null; let decodeError: Error | null = null; let decodeDone = false; this.decoder = new VideoDecoder({ output: (frame: VideoFrame) => { if (frameResolve) { const resolve = frameResolve; frameResolve = null; resolve(frame); } else { pendingFrames.push(frame); } }, error: (e: DOMException) => { decodeError = new Error(`VideoDecoder error: ${e.message}`); if (frameResolve) { const resolve = frameResolve; frameResolve = null; resolve(null); } }, }); this.decoder.configure(decoderConfig); const getNextFrame = (): Promise => { if (decodeError) throw decodeError; if (pendingFrames.length > 0) return Promise.resolve(pendingFrames.shift()!); if (decodeDone) return Promise.resolve(null); return new Promise((resolve) => { frameResolve = resolve; }); }; // One forward stream through the whole file const reader = this.demuxer.read("video").getReader(); // Feed chunks to decoder in background with backpressure const feedPromise = (async () => { try { while (!this.cancelled) { const { done, value: chunk } = await reader.read(); if (done || !chunk) break; while (this.decoder!.decodeQueueSize > 10 && !this.cancelled) { await new Promise((resolve) => setTimeout(resolve, 1)); } if (this.cancelled) break; this.decoder!.decode(chunk); } if (!this.cancelled && this.decoder!.state === "configured") { await this.decoder!.flush(); } } catch (e) { decodeError = e instanceof Error ? e : new Error(String(e)); } finally { decodeDone = true; if (frameResolve) { const resolve = frameResolve; frameResolve = null; resolve(null); } } })(); // Route decoded frames into segments by timestamp, then deliver with VFR→CFR resampling let segmentIdx = 0; let exportFrameIndex = 0; let segmentBuffer: VideoFrame[] = []; while (!this.cancelled && segmentIdx < segments.length) { const frame = await getNextFrame(); if (!frame) break; const frameTimeSec = frame.timestamp / 1_000_000; const currentSegment = segments[segmentIdx]; // Before current segment — trimmed or pre-video if (frameTimeSec < currentSegment.startSec - 0.001) { frame.close(); continue; } // Past current segment — flush buffer and advance if (frameTimeSec >= currentSegment.endSec - 0.001) { exportFrameIndex = await this.deliverSegment( segmentBuffer, currentSegment, targetFrameRate, frameDurationUs, exportFrameIndex, onFrame, ); for (const f of segmentBuffer) f.close(); segmentBuffer = []; segmentIdx++; while ( segmentIdx < segments.length && frameTimeSec >= segments[segmentIdx].endSec - 0.001 ) { segmentIdx++; } if (segmentIdx < segments.length && frameTimeSec >= segments[segmentIdx].startSec - 0.001) { segmentBuffer.push(frame); } else { frame.close(); } continue; } segmentBuffer.push(frame); } // Flush last segment if (segmentBuffer.length > 0 && segmentIdx < segments.length) { exportFrameIndex = await this.deliverSegment( segmentBuffer, segments[segmentIdx], targetFrameRate, frameDurationUs, exportFrameIndex, onFrame, ); for (const f of segmentBuffer) f.close(); } // Drain leftover decoded frames while (!decodeDone) { const frame = await getNextFrame(); if (!frame) break; frame.close(); } try { reader.cancel(); } catch { /* already closed */ } await feedPromise; for (const f of pendingFrames) f.close(); pendingFrames.length = 0; if (this.decoder?.state === "configured") { this.decoder.close(); } this.decoder = null; } /** * Resample buffered frames to fill the target frame count for this segment. * Handles VFR sources by duplicating/decimating as needed. */ private async deliverSegment( frames: VideoFrame[], segment: { startSec: number; endSec: number; speed: number }, targetFrameRate: number, frameDurationUs: number, startExportFrameIndex: number, onFrame: OnFrameCallback, ): Promise { if (frames.length === 0) return startExportFrameIndex; const segmentFrameCount = Math.ceil( ((segment.endSec - segment.startSec) / segment.speed) * targetFrameRate, ); let exportFrameIndex = startExportFrameIndex; for (let i = 0; i < segmentFrameCount && !this.cancelled; i++) { const sourceIdx = Math.min( Math.floor((i * frames.length) / segmentFrameCount), frames.length - 1, ); const sourceFrame = frames[sourceIdx]; const clone = new VideoFrame(sourceFrame, { timestamp: sourceFrame.timestamp }); await onFrame(clone, exportFrameIndex * frameDurationUs, sourceFrame.timestamp / 1000); exportFrameIndex++; } return exportFrameIndex; } private computeSegments( totalDuration: number, trimRegions?: TrimRegion[], ): Array<{ startSec: number; endSec: number }> { if (!trimRegions || trimRegions.length === 0) { return [{ startSec: 0, endSec: totalDuration }]; } const sorted = [...trimRegions].sort((a, b) => a.startMs - b.startMs); const segments: Array<{ startSec: number; endSec: number }> = []; let cursor = 0; for (const trim of sorted) { const trimStart = trim.startMs / 1000; const trimEnd = trim.endMs / 1000; if (cursor < trimStart) { segments.push({ startSec: cursor, endSec: trimStart }); } cursor = trimEnd; } if (cursor < totalDuration) { segments.push({ startSec: cursor, endSec: totalDuration }); } return segments; } getEffectiveDuration(trimRegions?: TrimRegion[], speedRegions?: SpeedRegion[]): number { if (!this.metadata) throw new Error("Must call loadMetadata() first"); const trimSegments = this.computeSegments(this.metadata.duration, trimRegions); const speedSegments = this.splitBySpeed(trimSegments, speedRegions); return speedSegments.reduce((sum, seg) => sum + (seg.endSec - seg.startSec) / seg.speed, 0); } private splitBySpeed( segments: Array<{ startSec: number; endSec: number }>, speedRegions?: SpeedRegion[], ): Array<{ startSec: number; endSec: number; speed: number }> { if (!speedRegions || speedRegions.length === 0) return segments.map((s) => ({ ...s, speed: 1 })); const result: Array<{ startSec: number; endSec: number; speed: number }> = []; for (const segment of segments) { const overlapping = speedRegions .filter((sr) => sr.startMs / 1000 < segment.endSec && sr.endMs / 1000 > segment.startSec) .sort((a, b) => a.startMs - b.startMs); if (overlapping.length === 0) { result.push({ ...segment, speed: 1 }); continue; } let cursor = segment.startSec; for (const sr of overlapping) { const srStart = Math.max(sr.startMs / 1000, segment.startSec); const srEnd = Math.min(sr.endMs / 1000, segment.endSec); if (cursor < srStart) result.push({ startSec: cursor, endSec: srStart, speed: 1 }); result.push({ startSec: srStart, endSec: srEnd, speed: sr.speed }); cursor = srEnd; } if (cursor < segment.endSec) result.push({ startSec: cursor, endSec: segment.endSec, speed: 1 }); } return result.filter((s) => s.endSec - s.startSec > 0.0001); } getDemuxer(): WebDemuxer | null { return this.demuxer; } cancel(): void { this.cancelled = true; } destroy(): void { this.cancelled = true; if (this.decoder) { try { if (this.decoder.state === "configured") this.decoder.close(); } catch { /* ignore */ } this.decoder = null; } if (this.demuxer) { try { this.demuxer.destroy(); } catch {} this.demuxer = null; } } }