huanld/openscreen
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

migrate to mediabunny

Browse Source
This commit is contained in:
Siddharth
2025-11-23 12:24:56 -07:00
parent 4a78bb999b
commit 0d5c4529d1
4 changed files with 184 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package-lock.json
Generated
+33 -18
View File
@@ -21,7 +21,7 @@
"clsx": "^2.1.1",
"dnd-timeline": "^2.2.0",
"lucide-react": "^0.545.0",
"mp4-muxer": "^5.2.2",
"mediabunny": "^1.25.1",
"mp4box": "^2.2.0",
"pixi.js": "^8.14.0",
"react": "^18.2.0",
@@ -2869,6 +2869,15 @@
"@types/ms": "*"
}
},
"node_modules/@types/dom-mediacapture-transform": {
"version": "0.1.11",
"resolved": "https://registry.npmjs.org/@types/dom-mediacapture-transform/-/dom-mediacapture-transform-0.1.11.tgz",
"integrity": "sha512-Y2p+nGf1bF2XMttBnsVPHUWzRRZzqUoJAKmiP10b5umnO6DDrWI0BrGDJy1pOHoOULVmGSfFNkQrAlC5dcj6nQ==",
"license": "MIT",
"dependencies": {
"@types/dom-webcodecs": "*"
}
},
"node_modules/@types/dom-webcodecs": {
"version": "0.1.17",
"resolved": "https://registry.npmjs.org/@types/dom-webcodecs/-/dom-webcodecs-0.1.17.tgz",
@@ -2990,12 +2999,6 @@
"license": "MIT",
"optional": true
},
"node_modules/@types/wicg-file-system-access": {
"version": "2020.9.8",
"resolved": "https://registry.npmjs.org/@types/wicg-file-system-access/-/wicg-file-system-access-2020.9.8.tgz",
"integrity": "sha512-ggMz8nOygG7d/stpH40WVaNvBwuyYLnrg5Mbyf6bmsj/8+gb6Ei4ZZ9/4PNpcPNTT8th9Q8sM8wYmWGjMWLX/A==",
"license": "MIT"
},
"node_modules/@types/yauzl": {
"version": "2.10.3",
"resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz",
@@ -7543,6 +7546,29 @@
"node": ">= 0.4"
}
},
"node_modules/mediabunny": {
"version": "1.25.1",
"resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.25.1.tgz",
"integrity": "sha512-0Rrd47PMCVJbTPA7IJaXPCupV5/RZ/icgr+a0qExRJAr0n5vB4fsGSo+fdHIehG0CrddXtVRvNZwFtJz709yfA==",
"license": "MPL-2.0",
"workspaces": [
"packages/*"
],
"dependencies": {
"@types/dom-mediacapture-transform": "^0.1.11",
"@types/dom-webcodecs": "0.1.13"
},
"funding": {
"type": "individual",
"url": "https://github.com/sponsors/Vanilagy"
}
},
"node_modules/mediabunny/node_modules/@types/dom-webcodecs": {
"version": "0.1.13",
"resolved": "https://registry.npmjs.org/@types/dom-webcodecs/-/dom-webcodecs-0.1.13.tgz",
"integrity": "sha512-O5hkiFIcjjszPIYyUSyvScyvrBoV3NOEEZx/pMlsu44TKzWNkLVBBxnxJz42in5n3QIolYOcBYFCPZZ0h8SkwQ==",
"license": "MIT"
},
"node_modules/merge2": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz",
@@ -7872,17 +7898,6 @@
"node": ">=10"
}
},
"node_modules/mp4-muxer": {
"version": "5.2.2",
"resolved": "https://registry.npmjs.org/mp4-muxer/-/mp4-muxer-5.2.2.tgz",
"integrity": "sha512-dhozjTywI0h2qFzeShagt8YYw811fh1XlwiDCE2f6Aeqf6xG2CyuShoSa5E0AZDO8pPF0JOZ3wOmWBNWIGdSpQ==",
"deprecated": "This library is superseded by Mediabunny. Please migrate to it.",
"license": "MIT",
"dependencies": {
"@types/dom-webcodecs": "^0.1.6",
"@types/wicg-file-system-access": "^2020.9.5"
}
},
"node_modules/mp4box": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/mp4box/-/mp4box-2.2.0.tgz",
package.json
+2 -2
View File
@@ -24,7 +24,7 @@
"clsx": "^2.1.1",
"dnd-timeline": "^2.2.0",
"lucide-react": "^0.545.0",
"mp4-muxer": "^5.2.2",
"mediabunny": "^1.25.1",
"mp4box": "^2.2.0",
"pixi.js": "^8.14.0",
"react": "^18.2.0",
@@ -56,4 +56,4 @@
"vite-plugin-electron-renderer": "^0.14.5"
},
"main": "dist-electron/main.js"
}
}
src/lib/exporter/muxer.ts
+57 -57
View File
@@ -1,31 +1,20 @@
import type { ExportConfig } from './types';
interface MP4MuxerOptions {
target: any; // ArrayBufferTarget instance
video: {
codec: string;
width: number;
height: number;
};
audio?: {
codec: string;
numberOfChannels: number;
sampleRate: number;
};
fastStart?: 'in-memory' | 'fragmented';
}
interface Muxer {
addVideoChunk(chunk: EncodedVideoChunk, meta?: EncodedVideoChunkMetadata): void;
addAudioChunk(chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata): void;
finalize(): void;
target?: any;
}
import {
Output,
Mp4OutputFormat,
BufferTarget,
EncodedVideoPacketSource,
EncodedAudioPacketSource,
EncodedPacket
} from 'mediabunny';
export class VideoMuxer {
private muxer: Muxer | null = null;
private config: ExportConfig;
private output: Output | null = null;
private videoSource: EncodedVideoPacketSource | null = null;
private audioSource: EncodedAudioPacketSource | null = null;
private hasAudio: boolean;
private target: BufferTarget | null = null;
private config: ExportConfig;
constructor(config: ExportConfig, hasAudio = false) {
this.config = config;
@@ -33,57 +22,68 @@ export class VideoMuxer {
}
async initialize(): Promise<void> {
const MP4MuxerModule = await import('mp4-muxer');
const MP4MuxerClass = (MP4MuxerModule as any).Muxer || MP4MuxerModule.default;
const ArrayBufferTarget = (MP4MuxerModule as any).ArrayBufferTarget;
// Create the buffer target
this.target = new BufferTarget();
const target = new ArrayBufferTarget();
const options: MP4MuxerOptions = {
target,
video: {
codec: 'avc',
width: this.config.width,
height: this.config.height,
},
fastStart: 'in-memory',
};
this.output = new Output({
format: new Mp4OutputFormat({
fastStart: 'in-memory',
}),
target: this.target,
});
// Create video source - codec will be deduced from metadata
this.videoSource = new EncodedVideoPacketSource('avc');
this.output.addVideoTrack(this.videoSource, {
frameRate: this.config.frameRate,
});
// Create audio source if needed
if (this.hasAudio) {
options.audio = {
codec: 'opus',
numberOfChannels: 2,
sampleRate: 48000,
};
this.audioSource = new EncodedAudioPacketSource('opus');
this.output.addAudioTrack(this.audioSource);
}
this.muxer = new MP4MuxerClass(options) as Muxer;
// Start the output to begin accepting media data
await this.output.start();
}
addVideoChunk(chunk: EncodedVideoChunk, meta?: EncodedVideoChunkMetadata): void {
if (!this.muxer) {
async addVideoChunk(chunk: EncodedVideoChunk, meta?: EncodedVideoChunkMetadata): Promise<void> {
if (!this.videoSource) {
throw new Error('Muxer not initialized');
}
this.muxer.addVideoChunk(chunk, meta);
// Convert WebCodecs chunk to Mediabunny packet
const packet = EncodedPacket.fromEncodedChunk(chunk);
// Add metadata with the first chunk
await this.videoSource.add(packet, meta);
}
addAudioChunk(chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata): void {
if (!this.muxer) {
throw new Error('Muxer not initialized');
}
if (!this.hasAudio) {
async addAudioChunk(chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata): Promise<void> {
if (!this.audioSource) {
throw new Error('Audio not configured for this muxer');
}
this.muxer.addAudioChunk(chunk, meta);
// Convert WebCodecs chunk to Mediabunny packet
const packet = EncodedPacket.fromEncodedChunk(chunk);
// Add metadata with the first chunk
await this.audioSource.add(packet, meta);
}
finalize(): Blob {
if (!this.muxer) {
async finalize(): Promise<Blob> {
if (!this.output || !this.target) {
throw new Error('Muxer not initialized');
}
this.muxer.finalize();
const buffer = this.muxer.target.buffer;
await this.output.finalize();
const buffer = this.target.buffer;
if (!buffer) {
throw new Error('Failed to finalize output');
}
return new Blob([buffer], { type: 'video/mp4' });
}
}
src/lib/exporter/videoExporter.ts
+92 -82
View File
@@ -21,12 +21,14 @@ export class VideoExporter {
private encoder: VideoEncoder | null = null;
private muxer: VideoMuxer | null = null;
private cancelled = false;
private encodedChunks: EncodedVideoChunk[] = [];
private encodeQueue = 0;
// Increased queue size for better throughput with hardware encoding
private readonly MAX_ENCODE_QUEUE = 120;
private videoDescription: Uint8Array | undefined;
private videoColorSpace: VideoColorSpaceInit | undefined;
// Track muxing promises for parallel processing
private muxingPromises: Promise<void>[] = [];
private chunkCount = 0;
constructor(config: VideoExporterConfig) {
this.config = config;
@@ -69,72 +71,67 @@ export class VideoExporter {
throw new Error('Video element not available');
}
// Process frames with optimized seeking
// Process frames continuously without batching delays
const frameDuration = 1_000_000 / this.config.frameRate; // in microseconds
let frameIndex = 0;
const timeStep = 1 / this.config.frameRate;
const BATCH_SIZE = 5; // Process frames in batches for better throughput
while (frameIndex < totalFrames && !this.cancelled) {
// Process a batch of frames
const batchEnd = Math.min(frameIndex + BATCH_SIZE, totalFrames);
for (let i = frameIndex; i < batchEnd && !this.cancelled; i++) {
const timestamp = i * frameDuration;
const videoTime = i * timeStep;
const i = frameIndex;
const timestamp = i * frameDuration;
const videoTime = i * timeStep;
// Seek if needed or wait for first frame to be ready
const needsSeek = Math.abs(videoElement.currentTime - videoTime) > 0.001;
if (needsSeek || i === 0) {
if (needsSeek) {
videoElement.currentTime = videoTime;
}
// Wait for video frame to be ready
await new Promise<void>(resolve => {
videoElement.requestVideoFrameCallback(() => resolve());
});
// Seek if needed or wait for first frame to be ready
const needsSeek = Math.abs(videoElement.currentTime - videoTime) > 0.001;
if (needsSeek || i === 0) {
if (needsSeek) {
videoElement.currentTime = videoTime;
}
// Create a VideoFrame from the video element (on GPU!)
const videoFrame = new VideoFrame(videoElement, {
timestamp,
// Wait for video frame to be ready
await new Promise<void>(resolve => {
videoElement.requestVideoFrameCallback(() => resolve());
});
// Render the frame with all effects
await this.renderer!.renderFrame(videoFrame, timestamp);
videoFrame.close();
const canvas = this.renderer!.getCanvas();
// Create VideoFrame from canvas on GPU without reading pixels
// @ts-ignore - colorSpace not in TypeScript definitions but works at runtime
const exportFrame = new VideoFrame(canvas, {
timestamp,
duration: frameDuration,
colorSpace: {
primaries: 'bt709',
transfer: 'iec61966-2-1',
matrix: 'rgb',
fullRange: true,
},
});
if (this.encoder && this.encoder.state === 'configured') {
this.encodeQueue++;
this.encoder.encode(exportFrame, { keyFrame: i % 150 === 0 });
}
exportFrame.close();
}
// Create a VideoFrame from the video element (on GPU!)
const videoFrame = new VideoFrame(videoElement, {
timestamp,
});
// Render the frame with all effects
await this.renderer!.renderFrame(videoFrame, timestamp);
// Wait for encoder queue once per batch
videoFrame.close();
const canvas = this.renderer!.getCanvas();
// Create VideoFrame from canvas on GPU without reading pixels
// @ts-ignore - colorSpace not in TypeScript definitions but works at runtime
const exportFrame = new VideoFrame(canvas, {
timestamp,
duration: frameDuration,
colorSpace: {
primaries: 'bt709',
transfer: 'iec61966-2-1',
matrix: 'rgb',
fullRange: true,
},
});
// Check encoder queue before encoding to keep it full
while (this.encodeQueue >= this.MAX_ENCODE_QUEUE && !this.cancelled) {
await new Promise(resolve => setTimeout(resolve, 0));
}
frameIndex = batchEnd;
if (this.encoder && this.encoder.state === 'configured') {
this.encodeQueue++;
this.encoder.encode(exportFrame, { keyFrame: i % 150 === 0 });
}
exportFrame.close();
// Batch progress updates to reduce callback overhead
frameIndex++;
// Update progress
if (this.config.onProgress) {
this.config.onProgress({
currentFrame: frameIndex,
@@ -154,35 +151,11 @@ export class VideoExporter {
await this.encoder.flush();
}
// Add all chunks to muxer with metadata
for (let i = 0; i < this.encodedChunks.length; i++) {
const chunk = this.encodedChunks[i];
const meta: EncodedVideoChunkMetadata = {};
// Add decoder config for the first chunk
if (i === 0 && this.videoDescription) {
// Use captured colorSpace from encoder or fallback to default sRGB colorspace
const colorSpace = this.videoColorSpace || {
primaries: 'bt709',
transfer: 'iec61966-2-1',
matrix: 'rgb',
fullRange: true,
};
meta.decoderConfig = {
codec: this.config.codec || 'avc1.640033',
codedWidth: this.config.width,
codedHeight: this.config.height,
description: this.videoDescription,
colorSpace,
};
}
this.muxer!.addVideoChunk(chunk, meta);
}
// Wait for all muxing operations to complete
await Promise.all(this.muxingPromises);
// Finalize muxer and get output blob
const blob = this.muxer!.finalize();
const blob = await this.muxer!.finalize();
return { success: true, blob };
} catch (error) {
@@ -197,8 +170,9 @@ export class VideoExporter {
}
private async initializeEncoder(): Promise<void> {
this.encodedChunks = [];
this.encodeQueue = 0;
this.muxingPromises = [];
this.chunkCount = 0;
let videoDescription: Uint8Array | undefined;
this.encoder = new VideoEncoder({
@@ -213,7 +187,42 @@ export class VideoExporter {
if (meta?.decoderConfig?.colorSpace && !this.videoColorSpace) {
this.videoColorSpace = meta.decoderConfig.colorSpace;
}
this.encodedChunks.push(chunk);
// Stream chunk to muxer immediately (parallel processing)
const isFirstChunk = this.chunkCount === 0;
this.chunkCount++;
const muxingPromise = (async () => {
try {
if (isFirstChunk && this.videoDescription) {
// Add decoder config for the first chunk
const colorSpace = this.videoColorSpace || {
primaries: 'bt709',
transfer: 'iec61966-2-1',
matrix: 'rgb',
fullRange: true,
};
const metadata: EncodedVideoChunkMetadata = {
decoderConfig: {
codec: this.config.codec || 'avc1.640033',
codedWidth: this.config.width,
codedHeight: this.config.height,
description: this.videoDescription,
colorSpace,
},
};
await this.muxer!.addVideoChunk(chunk, metadata);
} else {
await this.muxer!.addVideoChunk(chunk, meta);
}
} catch (error) {
console.error('Muxing error:', error);
}
})();
this.muxingPromises.push(muxingPromise);
this.encodeQueue--;
},
error: (error) => {
@@ -271,8 +280,9 @@ export class VideoExporter {
}
this.muxer = null;
this.encodedChunks = [];
this.encodeQueue = 0;
this.muxingPromises = [];
this.chunkCount = 0;
this.videoDescription = undefined;
this.videoColorSpace = undefined;
}