From 0d5c4529d1a51bf01a3bdc12f9afc420a21e1df3 Mon Sep 17 00:00:00 2001 From: Siddharth Date: Sun, 23 Nov 2025 12:24:56 -0700 Subject: [PATCH] migrate to mediabunny --- package-lock.json | 51 +++++---- package.json | 4 +- src/lib/exporter/muxer.ts | 114 ++++++++++---------- src/lib/exporter/videoExporter.ts | 174 ++++++++++++++++-------------- 4 files changed, 184 insertions(+), 159 deletions(-) diff --git a/package-lock.json b/package-lock.json index a903707..c20e433 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,7 +21,7 @@ "clsx": "^2.1.1", "dnd-timeline": "^2.2.0", "lucide-react": "^0.545.0", - "mp4-muxer": "^5.2.2", + "mediabunny": "^1.25.1", "mp4box": "^2.2.0", "pixi.js": "^8.14.0", "react": "^18.2.0", @@ -2869,6 +2869,15 @@ "@types/ms": "*" } }, + "node_modules/@types/dom-mediacapture-transform": { + "version": "0.1.11", + "resolved": "https://registry.npmjs.org/@types/dom-mediacapture-transform/-/dom-mediacapture-transform-0.1.11.tgz", + "integrity": "sha512-Y2p+nGf1bF2XMttBnsVPHUWzRRZzqUoJAKmiP10b5umnO6DDrWI0BrGDJy1pOHoOULVmGSfFNkQrAlC5dcj6nQ==", + "license": "MIT", + "dependencies": { + "@types/dom-webcodecs": "*" + } + }, "node_modules/@types/dom-webcodecs": { "version": "0.1.17", "resolved": "https://registry.npmjs.org/@types/dom-webcodecs/-/dom-webcodecs-0.1.17.tgz", @@ -2990,12 +2999,6 @@ "license": "MIT", "optional": true }, - "node_modules/@types/wicg-file-system-access": { - "version": "2020.9.8", - "resolved": "https://registry.npmjs.org/@types/wicg-file-system-access/-/wicg-file-system-access-2020.9.8.tgz", - "integrity": "sha512-ggMz8nOygG7d/stpH40WVaNvBwuyYLnrg5Mbyf6bmsj/8+gb6Ei4ZZ9/4PNpcPNTT8th9Q8sM8wYmWGjMWLX/A==", - "license": "MIT" - }, "node_modules/@types/yauzl": { "version": "2.10.3", "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", @@ -7543,6 +7546,29 @@ "node": ">= 0.4" } }, + "node_modules/mediabunny": { + "version": "1.25.1", + "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.25.1.tgz", + "integrity": "sha512-0Rrd47PMCVJbTPA7IJaXPCupV5/RZ/icgr+a0qExRJAr0n5vB4fsGSo+fdHIehG0CrddXtVRvNZwFtJz709yfA==", + "license": "MPL-2.0", + "workspaces": [ + "packages/*" + ], + "dependencies": { + "@types/dom-mediacapture-transform": "^0.1.11", + "@types/dom-webcodecs": "0.1.13" + }, + "funding": { + "type": "individual", + "url": "https://github.com/sponsors/Vanilagy" + } + }, + "node_modules/mediabunny/node_modules/@types/dom-webcodecs": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/@types/dom-webcodecs/-/dom-webcodecs-0.1.13.tgz", + "integrity": "sha512-O5hkiFIcjjszPIYyUSyvScyvrBoV3NOEEZx/pMlsu44TKzWNkLVBBxnxJz42in5n3QIolYOcBYFCPZZ0h8SkwQ==", + "license": "MIT" + }, "node_modules/merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", @@ -7872,17 +7898,6 @@ "node": ">=10" } }, - "node_modules/mp4-muxer": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/mp4-muxer/-/mp4-muxer-5.2.2.tgz", - "integrity": "sha512-dhozjTywI0h2qFzeShagt8YYw811fh1XlwiDCE2f6Aeqf6xG2CyuShoSa5E0AZDO8pPF0JOZ3wOmWBNWIGdSpQ==", - "deprecated": "This library is superseded by Mediabunny. Please migrate to it.", - "license": "MIT", - "dependencies": { - "@types/dom-webcodecs": "^0.1.6", - "@types/wicg-file-system-access": "^2020.9.5" - } - }, "node_modules/mp4box": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/mp4box/-/mp4box-2.2.0.tgz", diff --git a/package.json b/package.json index 24d80ab..598b4d6 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "clsx": "^2.1.1", "dnd-timeline": "^2.2.0", "lucide-react": "^0.545.0", - "mp4-muxer": "^5.2.2", + "mediabunny": "^1.25.1", "mp4box": "^2.2.0", "pixi.js": "^8.14.0", "react": "^18.2.0", @@ -56,4 +56,4 @@ "vite-plugin-electron-renderer": "^0.14.5" }, "main": "dist-electron/main.js" -} \ No newline at end of file +} diff --git a/src/lib/exporter/muxer.ts b/src/lib/exporter/muxer.ts index b787db3..dd07f78 100644 --- a/src/lib/exporter/muxer.ts +++ b/src/lib/exporter/muxer.ts @@ -1,31 +1,20 @@ import type { ExportConfig } from './types'; - -interface MP4MuxerOptions { - target: any; // ArrayBufferTarget instance - video: { - codec: string; - width: number; - height: number; - }; - audio?: { - codec: string; - numberOfChannels: number; - sampleRate: number; - }; - fastStart?: 'in-memory' | 'fragmented'; -} - -interface Muxer { - addVideoChunk(chunk: EncodedVideoChunk, meta?: EncodedVideoChunkMetadata): void; - addAudioChunk(chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata): void; - finalize(): void; - target?: any; -} +import { + Output, + Mp4OutputFormat, + BufferTarget, + EncodedVideoPacketSource, + EncodedAudioPacketSource, + EncodedPacket +} from 'mediabunny'; export class VideoMuxer { - private muxer: Muxer | null = null; - private config: ExportConfig; + private output: Output | null = null; + private videoSource: EncodedVideoPacketSource | null = null; + private audioSource: EncodedAudioPacketSource | null = null; private hasAudio: boolean; + private target: BufferTarget | null = null; + private config: ExportConfig; constructor(config: ExportConfig, hasAudio = false) { this.config = config; @@ -33,57 +22,68 @@ export class VideoMuxer { } async initialize(): Promise { - const MP4MuxerModule = await import('mp4-muxer'); - const MP4MuxerClass = (MP4MuxerModule as any).Muxer || MP4MuxerModule.default; - const ArrayBufferTarget = (MP4MuxerModule as any).ArrayBufferTarget; + // Create the buffer target + this.target = new BufferTarget(); - const target = new ArrayBufferTarget(); - - const options: MP4MuxerOptions = { - target, - video: { - codec: 'avc', - width: this.config.width, - height: this.config.height, - }, - fastStart: 'in-memory', - }; + this.output = new Output({ + format: new Mp4OutputFormat({ + fastStart: 'in-memory', + }), + target: this.target, + }); + // Create video source - codec will be deduced from metadata + this.videoSource = new EncodedVideoPacketSource('avc'); + this.output.addVideoTrack(this.videoSource, { + frameRate: this.config.frameRate, + }); + + // Create audio source if needed if (this.hasAudio) { - options.audio = { - codec: 'opus', - numberOfChannels: 2, - sampleRate: 48000, - }; + this.audioSource = new EncodedAudioPacketSource('opus'); + this.output.addAudioTrack(this.audioSource); } - this.muxer = new MP4MuxerClass(options) as Muxer; + // Start the output to begin accepting media data + await this.output.start(); } - addVideoChunk(chunk: EncodedVideoChunk, meta?: EncodedVideoChunkMetadata): void { - if (!this.muxer) { + async addVideoChunk(chunk: EncodedVideoChunk, meta?: EncodedVideoChunkMetadata): Promise { + if (!this.videoSource) { throw new Error('Muxer not initialized'); } - this.muxer.addVideoChunk(chunk, meta); + + // Convert WebCodecs chunk to Mediabunny packet + const packet = EncodedPacket.fromEncodedChunk(chunk); + + // Add metadata with the first chunk + await this.videoSource.add(packet, meta); } - addAudioChunk(chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata): void { - if (!this.muxer) { - throw new Error('Muxer not initialized'); - } - if (!this.hasAudio) { + async addAudioChunk(chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata): Promise { + if (!this.audioSource) { throw new Error('Audio not configured for this muxer'); } - this.muxer.addAudioChunk(chunk, meta); + + // Convert WebCodecs chunk to Mediabunny packet + const packet = EncodedPacket.fromEncodedChunk(chunk); + + // Add metadata with the first chunk + await this.audioSource.add(packet, meta); } - finalize(): Blob { - if (!this.muxer) { + async finalize(): Promise { + if (!this.output || !this.target) { throw new Error('Muxer not initialized'); } - this.muxer.finalize(); - const buffer = this.muxer.target.buffer; + await this.output.finalize(); + const buffer = this.target.buffer; + + if (!buffer) { + throw new Error('Failed to finalize output'); + } + return new Blob([buffer], { type: 'video/mp4' }); } } diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index b7c630a..e2dcf17 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -21,12 +21,14 @@ export class VideoExporter { private encoder: VideoEncoder | null = null; private muxer: VideoMuxer | null = null; private cancelled = false; - private encodedChunks: EncodedVideoChunk[] = []; private encodeQueue = 0; // Increased queue size for better throughput with hardware encoding private readonly MAX_ENCODE_QUEUE = 120; private videoDescription: Uint8Array | undefined; private videoColorSpace: VideoColorSpaceInit | undefined; + // Track muxing promises for parallel processing + private muxingPromises: Promise[] = []; + private chunkCount = 0; constructor(config: VideoExporterConfig) { this.config = config; @@ -69,72 +71,67 @@ export class VideoExporter { throw new Error('Video element not available'); } - // Process frames with optimized seeking + // Process frames continuously without batching delays const frameDuration = 1_000_000 / this.config.frameRate; // in microseconds let frameIndex = 0; const timeStep = 1 / this.config.frameRate; - const BATCH_SIZE = 5; // Process frames in batches for better throughput while (frameIndex < totalFrames && !this.cancelled) { - // Process a batch of frames - const batchEnd = Math.min(frameIndex + BATCH_SIZE, totalFrames); - - for (let i = frameIndex; i < batchEnd && !this.cancelled; i++) { - const timestamp = i * frameDuration; - const videoTime = i * timeStep; + const i = frameIndex; + const timestamp = i * frameDuration; + const videoTime = i * timeStep; - // Seek if needed or wait for first frame to be ready - const needsSeek = Math.abs(videoElement.currentTime - videoTime) > 0.001; - if (needsSeek || i === 0) { - if (needsSeek) { - videoElement.currentTime = videoTime; - } - // Wait for video frame to be ready - await new Promise(resolve => { - videoElement.requestVideoFrameCallback(() => resolve()); - }); + // Seek if needed or wait for first frame to be ready + const needsSeek = Math.abs(videoElement.currentTime - videoTime) > 0.001; + if (needsSeek || i === 0) { + if (needsSeek) { + videoElement.currentTime = videoTime; } - - // Create a VideoFrame from the video element (on GPU!) - const videoFrame = new VideoFrame(videoElement, { - timestamp, + // Wait for video frame to be ready + await new Promise(resolve => { + videoElement.requestVideoFrameCallback(() => resolve()); }); - - // Render the frame with all effects - await this.renderer!.renderFrame(videoFrame, timestamp); - - videoFrame.close(); - - const canvas = this.renderer!.getCanvas(); - - // Create VideoFrame from canvas on GPU without reading pixels - // @ts-ignore - colorSpace not in TypeScript definitions but works at runtime - const exportFrame = new VideoFrame(canvas, { - timestamp, - duration: frameDuration, - colorSpace: { - primaries: 'bt709', - transfer: 'iec61966-2-1', - matrix: 'rgb', - fullRange: true, - }, - }); - - if (this.encoder && this.encoder.state === 'configured') { - this.encodeQueue++; - this.encoder.encode(exportFrame, { keyFrame: i % 150 === 0 }); - } - exportFrame.close(); } + + // Create a VideoFrame from the video element (on GPU!) + const videoFrame = new VideoFrame(videoElement, { + timestamp, + }); + + // Render the frame with all effects + await this.renderer!.renderFrame(videoFrame, timestamp); - // Wait for encoder queue once per batch + videoFrame.close(); + + const canvas = this.renderer!.getCanvas(); + + // Create VideoFrame from canvas on GPU without reading pixels + // @ts-ignore - colorSpace not in TypeScript definitions but works at runtime + const exportFrame = new VideoFrame(canvas, { + timestamp, + duration: frameDuration, + colorSpace: { + primaries: 'bt709', + transfer: 'iec61966-2-1', + matrix: 'rgb', + fullRange: true, + }, + }); + + // Check encoder queue before encoding to keep it full while (this.encodeQueue >= this.MAX_ENCODE_QUEUE && !this.cancelled) { await new Promise(resolve => setTimeout(resolve, 0)); } - frameIndex = batchEnd; + if (this.encoder && this.encoder.state === 'configured') { + this.encodeQueue++; + this.encoder.encode(exportFrame, { keyFrame: i % 150 === 0 }); + } + exportFrame.close(); - // Batch progress updates to reduce callback overhead + frameIndex++; + + // Update progress if (this.config.onProgress) { this.config.onProgress({ currentFrame: frameIndex, @@ -154,35 +151,11 @@ export class VideoExporter { await this.encoder.flush(); } - // Add all chunks to muxer with metadata - for (let i = 0; i < this.encodedChunks.length; i++) { - const chunk = this.encodedChunks[i]; - const meta: EncodedVideoChunkMetadata = {}; - - // Add decoder config for the first chunk - if (i === 0 && this.videoDescription) { - // Use captured colorSpace from encoder or fallback to default sRGB colorspace - const colorSpace = this.videoColorSpace || { - primaries: 'bt709', - transfer: 'iec61966-2-1', - matrix: 'rgb', - fullRange: true, - }; - - meta.decoderConfig = { - codec: this.config.codec || 'avc1.640033', - codedWidth: this.config.width, - codedHeight: this.config.height, - description: this.videoDescription, - colorSpace, - }; - } - - this.muxer!.addVideoChunk(chunk, meta); - } + // Wait for all muxing operations to complete + await Promise.all(this.muxingPromises); // Finalize muxer and get output blob - const blob = this.muxer!.finalize(); + const blob = await this.muxer!.finalize(); return { success: true, blob }; } catch (error) { @@ -197,8 +170,9 @@ export class VideoExporter { } private async initializeEncoder(): Promise { - this.encodedChunks = []; this.encodeQueue = 0; + this.muxingPromises = []; + this.chunkCount = 0; let videoDescription: Uint8Array | undefined; this.encoder = new VideoEncoder({ @@ -213,7 +187,42 @@ export class VideoExporter { if (meta?.decoderConfig?.colorSpace && !this.videoColorSpace) { this.videoColorSpace = meta.decoderConfig.colorSpace; } - this.encodedChunks.push(chunk); + + // Stream chunk to muxer immediately (parallel processing) + const isFirstChunk = this.chunkCount === 0; + this.chunkCount++; + + const muxingPromise = (async () => { + try { + if (isFirstChunk && this.videoDescription) { + // Add decoder config for the first chunk + const colorSpace = this.videoColorSpace || { + primaries: 'bt709', + transfer: 'iec61966-2-1', + matrix: 'rgb', + fullRange: true, + }; + + const metadata: EncodedVideoChunkMetadata = { + decoderConfig: { + codec: this.config.codec || 'avc1.640033', + codedWidth: this.config.width, + codedHeight: this.config.height, + description: this.videoDescription, + colorSpace, + }, + }; + + await this.muxer!.addVideoChunk(chunk, metadata); + } else { + await this.muxer!.addVideoChunk(chunk, meta); + } + } catch (error) { + console.error('Muxing error:', error); + } + })(); + + this.muxingPromises.push(muxingPromise); this.encodeQueue--; }, error: (error) => { @@ -271,8 +280,9 @@ export class VideoExporter { } this.muxer = null; - this.encodedChunks = []; this.encodeQueue = 0; + this.muxingPromises = []; + this.chunkCount = 0; this.videoDescription = undefined; this.videoColorSpace = undefined; }