huanld/openscreen
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

revert exporter

Browse Source
This commit is contained in:
Siddharth
2025-12-04 10:22:20 -07:00
parent 3a4ec9c470
commit 7a7db0b277
3 changed files with 273 additions and 457 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/exporter/videoExporter.ts
+91 -88
View File
@@ -31,9 +31,11 @@ export class VideoExporter {
private muxer: VideoMuxer | null = null;
private cancelled = false;
private encodeQueue = 0;
// Increased queue size for better throughput with hardware encoding
private readonly MAX_ENCODE_QUEUE = 120;
private videoDescription: Uint8Array | undefined;
private videoColorSpace: VideoColorSpaceInit | undefined;
// Track muxing promises for parallel processing
private muxingPromises: Promise<void>[] = [];
private chunkCount = 0;
@@ -54,20 +56,20 @@ export class VideoExporter {
const trimRegions = this.config.trimRegions || [];
// Sort trim regions by start time
const sortedTrims = [...trimRegions].sort((a, b) => a.startMs - b.startMs);
let sourceTimeMs = effectiveTimeMs;
for (const trim of sortedTrims) {
// If the source time hasn't reached this trim region yet, we're done
if (sourceTimeMs < trim.startMs) {
break;
}
// Add the duration of this trim region to the source time
const trimDuration = trim.endMs - trim.startMs;
sourceTimeMs += trimDuration;
}
return sourceTimeMs;
}
@@ -75,12 +77,12 @@ export class VideoExporter {
try {
this.cleanup();
this.cancelled = false;
const exportStartTime = performance.now();
// Initialize decoder and load video
this.decoder = new VideoFileDecoder();
const videoInfo = await this.decoder.loadVideo(this.config.videoUrl);
// Initialize frame renderer
this.renderer = new FrameRenderer({
width: this.config.width,
height: this.config.height,
@@ -101,82 +103,74 @@ export class VideoExporter {
});
await this.renderer.initialize();
// Initialize video encoder
await this.initializeEncoder();
// Initialize muxer
this.muxer = new VideoMuxer(this.config, false);
await this.muxer.initialize();
// Get the video element for frame extraction
const videoElement = this.decoder.getVideoElement();
if (!videoElement) {
throw new Error('Video element not available');
}
// Calculate frame count after trimming
// Calculate effective duration and frame count (excluding trim regions)
const effectiveDuration = this.getEffectiveDuration(videoInfo.duration);
const totalFrames = Math.ceil(effectiveDuration * this.config.frameRate);
const frameDuration = 1_000_000 / this.config.frameRate;
console.log('[VideoExporter] Original duration:', videoInfo.duration, 's');
console.log('[VideoExporter] Effective duration:', effectiveDuration, 's');
console.log('[VideoExporter] Total frames to export:', totalFrames);
// Process frames continuously without batching delays
const frameDuration = 1_000_000 / this.config.frameRate; // in microseconds
let frameIndex = 0;
const timeStep = 1 / this.config.frameRate;
videoElement.muted = true;
if (videoElement.readyState < 2) {
await new Promise<void>(r => {
videoElement.addEventListener('loadeddata', () => r(), { once: true });
});
}
while (frameIndex < totalFrames && !this.cancelled) {
const i = frameIndex;
const timestamp = i * frameDuration;
// Pipeline: Decode 10 frames ahead to overlap decode/render/encode operations
const DECODE_AHEAD = 10;
const frameQueue: { frame: VideoFrame; timestamp: number; sourceTimestamp: number }[] = [];
// Decode a single frame from source video
const decodeFrame = async (idx: number) => {
if (idx >= totalFrames) return;
const timestamp = idx * frameDuration;
const effectiveTimeMs = (idx * timeStep) * 1000;
// Map effective time to source time (accounting for trim regions)
const effectiveTimeMs = (i * timeStep) * 1000;
const sourceTimeMs = this.mapEffectiveToSourceTime(effectiveTimeMs);
const videoTime = sourceTimeMs / 1000;
const sourceTimestamp = sourceTimeMs * 1000;
// Seek to frame position
// Seek if needed or wait for first frame to be ready
const needsSeek = Math.abs(videoElement.currentTime - videoTime) > 0.001;
if (needsSeek || idx === 0) {
if (needsSeek) {
// Attach listener BEFORE setting currentTime to avoid race condition
const seekedPromise = new Promise<void>(resolve => {
videoElement.addEventListener('seeked', () => resolve(), { once: true });
});
videoElement.currentTime = videoTime;
await new Promise<void>(r => {
videoElement.addEventListener('seeked', () => r(), { once: true });
await seekedPromise;
} else if (i === 0) {
// Only for the very first frame, wait for it to be ready
await new Promise<void>(resolve => {
videoElement.requestVideoFrameCallback(() => resolve());
});
}
// Create VideoFrame from current video element position
const videoFrame = new VideoFrame(videoElement, { timestamp });
frameQueue.push({ frame: videoFrame, timestamp, sourceTimestamp });
};
// Pre-decode first batch of frames
for (let i = 0; i < Math.min(DECODE_AHEAD, totalFrames); i++) {
await decodeFrame(i);
}
let frameIndex = 0;
let decodeIndex = DECODE_AHEAD;
// Main processing loop
while (frameIndex < totalFrames && !this.cancelled) {
// Wait for decoded frame to be available
while (frameQueue.length === 0 && frameIndex < totalFrames) {
await new Promise(r => setTimeout(r, 1));
}
if (frameQueue.length === 0) break;
const { frame: videoFrame, timestamp, sourceTimestamp } = frameQueue.shift()!;
// Create a VideoFrame from the video element (on GPU!)
const videoFrame = new VideoFrame(videoElement, {
timestamp,
});
// Render frame with effects using PixiJS
// Render the frame with all effects using source timestamp
const sourceTimestamp = sourceTimeMs * 1000; // Convert to microseconds
await this.renderer!.renderFrame(videoFrame, sourceTimestamp);
videoFrame.close();
// Create VideoFrame directly from canvas (GPU-level)
const canvas = this.renderer!.getCanvas();
// @ts-ignore
// Create VideoFrame from canvas on GPU without reading pixels
// @ts-ignore - colorSpace not in TypeScript definitions but works at runtime
const exportFrame = new VideoFrame(canvas, {
timestamp,
duration: frameDuration,
@@ -188,26 +182,23 @@ export class VideoExporter {
},
});
// Wait if encoder queue is full
// Check encoder queue before encoding to keep it full
while (this.encodeQueue >= this.MAX_ENCODE_QUEUE && !this.cancelled) {
await new Promise(r => setTimeout(r, 0));
await new Promise(resolve => setTimeout(resolve, 0));
}
// Encode frame using hardware acceleration
if (this.encoder && this.encoder.state === 'configured') {
this.encodeQueue++;
this.encoder.encode(exportFrame, { keyFrame: frameIndex % 150 === 0 });
this.encoder.encode(exportFrame, { keyFrame: i % 150 === 0 });
} else {
console.warn(`[Frame ${i}] Encoder not ready! State: ${this.encoder?.state}`);
}
exportFrame.close();
frameIndex++;
// Decode next frame in parallel while we process current frame
if (decodeIndex < totalFrames) {
decodeFrame(decodeIndex++).catch(e => console.error('[VideoExporter] Decode error:', e));
}
// Update progress
if (this.config.onProgress) {
this.config.onProgress({
currentFrame: frameIndex,
@@ -222,18 +213,20 @@ export class VideoExporter {
return { success: false, error: 'Export cancelled' };
}
// Finalize encoding
if (this.encoder && this.encoder.state === 'configured') {
await this.encoder.flush();
}
// Wait for all muxing operations to complete
await Promise.all(this.muxingPromises);
// Finalize muxer and get output blob
const blob = await this.muxer!.finalize();
const totalTime = performance.now() - exportStartTime;
console.log(`[VideoExporter] Export complete in ${(totalTime/1000).toFixed(2)}s (${totalFrames} frames)`);
return { success: true, blob };
} catch (error) {
console.error('[VideoExporter] Export error:', error);
console.error('Export error:', error);
return {
success: false,
error: error instanceof Error ? error.message : String(error),
@@ -247,33 +240,36 @@ export class VideoExporter {
this.encodeQueue = 0;
this.muxingPromises = [];
this.chunkCount = 0;
let videoDescription: Uint8Array | undefined;
// Create VideoEncoder with hardware acceleration
this.encoder = new VideoEncoder({
output: (chunk, meta) => {
// Capture codec description and color space from first chunk
if (meta?.decoderConfig?.description && !this.videoDescription) {
// Capture decoder config metadata from encoder output
if (meta?.decoderConfig?.description && !videoDescription) {
const desc = meta.decoderConfig.description;
this.videoDescription = new Uint8Array(desc instanceof ArrayBuffer ? desc : (desc as any));
videoDescription = new Uint8Array(desc instanceof ArrayBuffer ? desc : (desc as any));
this.videoDescription = videoDescription;
}
// Capture colorSpace from encoder metadata if provided
if (meta?.decoderConfig?.colorSpace && !this.videoColorSpace) {
this.videoColorSpace = meta.decoderConfig.colorSpace;
}
// Stream chunk to muxer immediately (parallel processing)
const isFirstChunk = this.chunkCount === 0;
this.chunkCount++;
// Send encoded chunk to muxer
const muxingPromise = (async () => {
try {
if (isFirstChunk && this.videoDescription) {
// Add decoder config for the first chunk
const colorSpace = this.videoColorSpace || {
primaries: 'bt709',
transfer: 'iec61966-2-1',
matrix: 'rgb',
fullRange: true,
};
const metadata: EncodedVideoChunkMetadata = {
decoderConfig: {
codec: this.config.codec || 'avc1.640033',
@@ -283,27 +279,28 @@ export class VideoExporter {
colorSpace,
},
};
await this.muxer!.addVideoChunk(chunk, metadata);
} else {
await this.muxer!.addVideoChunk(chunk, meta);
}
} catch (error) {
console.error('[VideoExporter] Muxing error:', error);
console.error('Muxing error:', error);
}
})();
this.muxingPromises.push(muxingPromise);
this.encodeQueue--;
},
error: (error) => {
console.error('[VideoExporter] Encoder error:', error);
// Stop export encoding failed
this.cancelled = true;
},
});
// Configure encoder with hardware acceleration
const codec = this.config.codec || 'avc1.640033';
const encoderConfig: VideoEncoderConfig = {
codec,
width: this.config.width,
@@ -315,17 +312,23 @@ export class VideoExporter {
hardwareAcceleration: 'prefer-hardware',
};
const support = await VideoEncoder.isConfigSupported(encoderConfig);
if (support.supported) {
// Check hardware support first
const hardwareSupport = await VideoEncoder.isConfigSupported(encoderConfig);
if (hardwareSupport.supported) {
// Use hardware encoding
console.log('[VideoExporter] Using hardware acceleration');
this.encoder.configure(encoderConfig);
} else {
// Fallback to software encoding
// Fall back to software encoding
console.log('[VideoExporter] Hardware not supported, using software encoding');
encoderConfig.hardwareAcceleration = 'prefer-software';
const softwareSupport = await VideoEncoder.isConfigSupported(encoderConfig);
if (!softwareSupport.supported) {
throw new Error('Video encoding not supported');
throw new Error('Video encoding not supported on this system');
}
this.encoder.configure(encoderConfig);
}
}