cleanup+ readme updates

This commit is contained in:
Siddharth
2025-11-18 00:58:09 -07:00
parent fd8417b221
commit d9a9f48ab9
18 changed files with 40 additions and 147 deletions
+3 -2
View File
@@ -15,11 +15,11 @@ If you don't want to pay $29/month for Screen Studio but want a much simpler ver
Okay, let's be real: Screen Studio is an awesome product and this is definitely not a 1:1 clone. OpenScreen is a much simpler take, just the basics for folks who want control and don't want to pay. If you need all the fancy features, your best bet is to support Screen Studio (they really do a great job, haha). But if you just want something free (no gotchas) and open, this project does the job!
OpenScreen is 100% free for personal and commercial use. Use it, remix it, build on it. (Just be cool and give a shoutout if you feel like it!)
OpenScreen is 100% free for personal and commercial use. Use it, modify it, distribute it. (Just be cool 😁 and give a shoutout if you feel like it !)
**⚠️ DISCLAIMER: This is very much in beta and might be buggy here and there (but hope you have a good experience!).**
**⚠️ DISCLAIMER: This is very much in <span style="display: inline-block; background: transparent; color: #111; font-size: 0.85em; font-weight: 600; border-radius: 8px; border: 1.5px solid #111; padding: 2px 10px; margin-right: 2px; vertical-align: middle;">beta</span> and might be buggy here and there (but hope you have a good experience!).**
</p>
<p align="center">
@@ -53,6 +53,7 @@ Download the latest installer for your platform from the [GitHub Releases](https
- TypeScript
- Vite
- PixiJS
- dnd-timeline
---
+3
View File
@@ -1,3 +1,6 @@
// uiohook-napi [WIP: scoping out mouse tracking with cross platform support for potential auto zoom/ post processing cursor effects]
// not currently being used.
import { uIOhook } from 'uiohook-napi'
let isMouseTrackingActive = false
-1
View File
@@ -147,7 +147,6 @@ app.whenReady().then(async () => {
() => mainWindow,
() => sourceSelectorWindow,
(recording: boolean, sourceName: string) => {
// removed unused assignment to _isRecording
selectedSourceName = sourceName
if (recording) {
if (!tray) createTray();
-2
View File
@@ -104,7 +104,6 @@ export function SourceSelector() {
</div>
</Card>
))}
{/* Removed scroll hint gradient for clean look */}
</div>
</TabsContent>
<TabsContent value="windows" className="h-full">
@@ -144,7 +143,6 @@ export function SourceSelector() {
</div>
</Card>
))}
{/* Removed scroll hint gradient for clean look */}
</div>
</TabsContent>
</div>
@@ -23,7 +23,6 @@ export function CropControl({ videoElement, cropRegion, onCropChange }: CropCont
const [dragStart, setDragStart] = useState({ x: 0, y: 0 });
const [initialCrop, setInitialCrop] = useState<CropRegion>(cropRegion);
// Draw video preview at high quality
useEffect(() => {
if (!videoElement || !canvasRef.current) return;
@@ -31,7 +30,6 @@ export function CropControl({ videoElement, cropRegion, onCropChange }: CropCont
const ctx = canvas.getContext('2d', { alpha: false });
if (!ctx) return;
// Set canvas to actual video dimensions for high quality
canvas.width = videoElement.videoWidth || 1920;
canvas.height = videoElement.videoHeight || 1080;
@@ -62,7 +60,6 @@ export function CropControl({ videoElement, cropRegion, onCropChange }: CropCont
});
setInitialCrop(cropRegion);
// Capture pointer for smooth dragging
e.currentTarget.setPointerCapture(e.pointerId);
};
@@ -79,11 +76,8 @@ export function CropControl({ videoElement, cropRegion, onCropChange }: CropCont
switch (isDragging) {
case 'top': {
// Calculate new y position
const newY = Math.max(0, initialCrop.y + deltaY);
// Calculate the bottom edge (which should stay fixed)
const bottom = initialCrop.y + initialCrop.height;
// Ensure minimum height of 0.1
newCrop.y = Math.min(newY, bottom - 0.1);
newCrop.height = bottom - newCrop.y;
break;
@@ -92,11 +86,8 @@ export function CropControl({ videoElement, cropRegion, onCropChange }: CropCont
newCrop.height = Math.max(0.1, Math.min(initialCrop.height + deltaY, 1 - initialCrop.y));
break;
case 'left': {
// Calculate new x position
const newX = Math.max(0, initialCrop.x + deltaX);
// Calculate the right edge (which should stay fixed)
const right = initialCrop.x + initialCrop.width;
// Ensure minimum width of 0.1
newCrop.x = Math.min(newX, right - 0.1);
newCrop.width = right - newCrop.x;
break;
@@ -114,7 +105,6 @@ export function CropControl({ videoElement, cropRegion, onCropChange }: CropCont
try {
e.currentTarget.releasePointerCapture(e.pointerId);
} catch {
// ignore
}
}
setIsDragging(null);
@@ -140,7 +130,6 @@ export function CropControl({ videoElement, cropRegion, onCropChange }: CropCont
style={{ imageRendering: 'auto' }}
/>
{/* Dark overlay outside crop */}
<div className="absolute inset-0 pointer-events-none" style={{ transition: 'none' }}>
<svg width="100%" height="100%" className="absolute inset-0" style={{ transition: 'none' }}>
<defs>
@@ -167,8 +156,6 @@ export function CropControl({ videoElement, cropRegion, onCropChange }: CropCont
</svg>
</div>
{/* Crop region - 4 straight lines */}
{/* Top line */}
<div
className={cn(
"absolute h-[3px] cursor-ns-resize z-20 pointer-events-auto bg-green-500"
@@ -184,7 +171,6 @@ export function CropControl({ videoElement, cropRegion, onCropChange }: CropCont
onPointerDown={(e) => handlePointerDown(e, 'top')}
/>
{/* Bottom line */}
<div
className={cn(
"absolute h-[3px] cursor-ns-resize z-20 pointer-events-auto bg-green-500"
@@ -200,7 +186,6 @@ export function CropControl({ videoElement, cropRegion, onCropChange }: CropCont
onPointerDown={(e) => handlePointerDown(e, 'bottom')}
/>
{/* Left line */}
<div
className={cn(
"absolute w-[3px] cursor-ew-resize z-20 pointer-events-auto bg-green-500"
@@ -216,7 +201,6 @@ export function CropControl({ videoElement, cropRegion, onCropChange }: CropCont
onPointerDown={(e) => handlePointerDown(e, 'left')}
/>
{/* Right line */}
<div
className={cn(
"absolute w-[3px] cursor-ew-resize z-20 pointer-events-auto bg-green-500"
@@ -210,15 +210,13 @@ export function SettingsPanel({ selected, onWallpaperChange, selectedZoomDepth,
{(wallpaperPaths.length > 0 ? wallpaperPaths : WALLPAPER_RELATIVE.map(p => `/${p}`)).map((path, idx) => {
const isSelected = (() => {
if (!selected) return false;
// exact match
if (selected === path) return true;
// file:// vs absolute path mismatch: compare by filename suffix
try {
const clean = (s: string) => s.replace(/^file:\/\//, '').replace(/^\//, '')
if (clean(selected).endsWith(clean(path))) return true;
if (clean(path).endsWith(clean(selected))) return true;
} catch {
// ignore
}
return false;
})();
@@ -181,13 +181,11 @@ export default function VideoEditor() {
setExportError(null);
try {
// Pause video during export
const wasPlaying = isPlaying;
if (wasPlaying) {
videoPlaybackRef.current?.pause();
}
// Always export at 1920x1080 (16:9)
const width = 1920;
const height = 1080;
@@ -212,7 +210,6 @@ export default function VideoEditor() {
const result = await exporter.export();
if (result.success && result.blob) {
// Save the blob using Electron
const arrayBuffer = await result.blob.arrayBuffer();
const timestamp = Date.now();
const fileName = `export-${timestamp}.mp4`;
@@ -230,7 +227,6 @@ export default function VideoEditor() {
toast.error(result.error || 'Export failed');
}
// Resume playback if it was playing
if (wasPlaying) {
videoPlaybackRef.current?.play();
}
+1 -16
View File
@@ -165,7 +165,6 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(({
}
}, [updateOverlayForRegion, cropRegion]);
// Keep layoutVideoContent ref updated
useEffect(() => {
layoutVideoContentRef.current = layoutVideoContent;
}, [layoutVideoContent]);
@@ -262,7 +261,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(({
try {
event.currentTarget.releasePointerCapture(event.pointerId);
} catch {
// ignore release errors when pointer capture is already cleared
}
};
@@ -286,7 +285,6 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(({
isPlayingRef.current = isPlaying;
}, [isPlaying]);
// Reset animation state and transforms when crop changes
useEffect(() => {
if (!pixiReady || !videoReady) return;
@@ -306,7 +304,6 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(({
video.pause();
}
// Reset animation state so the ticker starts from identity once it resumes
animationStateRef.current = {
scale: 1,
focusX: DEFAULT_FOCUS.cx,
@@ -317,7 +314,6 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(({
blurFilterRef.current.blur = 0;
}
// Defer layout to the next frame so DOM measurements include the new crop UI state
requestAnimationFrame(() => {
const container = cameraContainerRef.current;
const videoStage = videoContainerRef.current;
@@ -327,7 +323,6 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(({
return;
}
// Reset all transform hierarchies to identity
container.scale.set(1);
container.position.set(0, 0);
videoStage.scale.set(1);
@@ -335,10 +330,8 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(({
sprite.scale.set(1);
sprite.position.set(0, 0);
// Now layoutVideoContent will apply the correct transforms for the new crop
layoutVideoContent();
// Apply an explicit identity transform to ensure no residual camera offset
applyZoomTransform({
cameraContainer: container,
blurFilter: blurFilterRef.current,
@@ -351,12 +344,10 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(({
isPlaying: false,
});
// Restart ticker on a second frame to avoid running mid-layout
requestAnimationFrame(() => {
const finalApp = appRef.current;
if (wasPlaying && video) {
video.play().catch(() => {
/* ignore */
});
}
if (tickerWasStarted && finalApp?.ticker) {
@@ -402,7 +393,6 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(({
overlayEl.style.pointerEvents = isPlaying ? 'none' : 'auto';
}, [selectedZoom, isPlaying]);
// Initialize PixiJS application
useEffect(() => {
const container = containerRef.current;
if (!container) return;
@@ -410,7 +400,6 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(({
let mounted = true;
let app: PIXI.Application | null = null;
// Initialize the app
(async () => {
app = new PIXI.Application();
@@ -423,7 +412,6 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(({
autoDensity: true,
});
// Lock ticker to 60fps for consistent animation speed across all displays
app.ticker.maxFPS = 60;
if (!mounted) {
@@ -690,7 +678,6 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(({
return
}
// If it's a solid color or CSS gradient, use it directly
if (wallpaper.startsWith('#') || wallpaper.startsWith('linear-gradient') || wallpaper.startsWith('radial-gradient')) {
if (mounted) setResolvedWallpaper(wallpaper)
return
@@ -708,8 +695,6 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(({
if (mounted) setResolvedWallpaper(wallpaper)
return
}
// Otherwise assume it's a relative path like 'wallpapers/wallpaper1.jpg'
const p = await getAssetPath(wallpaper.replace(/^\//, ''))
if (mounted) setResolvedWallpaper(p)
} catch (err) {
@@ -151,7 +151,7 @@ function PlaybackCursor({
<div
className="absolute top-0 bottom-0 pointer-events-none z-50"
style={{
[sideProperty === "right" ? "marginRight" : "marginLeft"]: `${sidebarWidth - 8}px`, // reduce margin
[sideProperty === "right" ? "marginRight" : "marginLeft"]: `${sidebarWidth - 8}px`,
}}
>
<div
@@ -185,7 +185,6 @@ function PlaybackCursor({
</div>
</div>
{/* Subtle glow at top */}
<div className="absolute -top-1 left-1/2 -translate-x-1/2 w-2 h-2 bg-red-500/30 rounded-full blur-sm" />
</div>
</div>
@@ -403,7 +402,6 @@ export default function TimelineEditor({
// Snap if gap is 2ms or less
return zoomRegions.some((region) => {
if (region.id === excludeId) return false;
// If the new span is within 2ms of another region, treat as overlap (snap)
const gapBefore = newSpan.start - region.endMs;
const gapAfter = region.startMs - newSpan.end;
if (gapBefore > 0 && gapBefore <= 2) return true;
@@ -149,8 +149,6 @@ export default function TimelineWrapper({
[clampRange, onRangeChange, totalMs],
);
// To maximize granularity, disable grid snapping by not passing rangeGridSizeDefinition
// and allow pixel-level movement for items.
return (
<TimelineContext
range={range}
@@ -158,7 +156,6 @@ export default function TimelineWrapper({
onResizeEnd={onResizeEnd}
onDragEnd={onDragEnd}
autoScroll={{ enabled: false }}
// Remove rangeGridSizeDefinition to avoid snap effect
>
{children}
</TimelineContext>
@@ -18,8 +18,7 @@ export function clampFocusToStage(
const windowWidth = stageSize.width / zoomScale;
const windowHeight = stageSize.height / zoomScale;
// Calculate margins - focus must stay far enough from edges so zoom window stays within stage bounds
const marginX = windowWidth / (2 * stageSize.width);
const marginY = windowHeight / (2 * stageSize.height);
@@ -24,7 +24,7 @@ interface LayoutResult {
export function layoutVideoContent(params: LayoutParams): LayoutResult | null {
const { container, app, videoSprite, maskGraphics, videoElement, cropRegion, lockedVideoDimensions } = params;
// Use locked dimensions if available, otherwise use current video dimensions
const videoWidth = lockedVideoDimensions?.width || videoElement.videoWidth;
const videoHeight = lockedVideoDimensions?.height || videoElement.videoHeight;
@@ -37,7 +37,6 @@ export function createVideoEventHandlers(params: VideoEventHandlersParams) {
}
const handlePlay = () => {
// Prevent autoplay during seek operations
if (isSeekingRef.current) {
video.pause();
return;
@@ -69,7 +68,6 @@ export function createVideoEventHandlers(params: VideoEventHandlersParams) {
const handleSeeked = () => {
isSeekingRef.current = false;
// Keep video paused after seek if it wasn't playing
if (!isPlayingRef.current && !video.paused) {
video.pause();
}
@@ -79,7 +77,6 @@ export function createVideoEventHandlers(params: VideoEventHandlersParams) {
const handleSeeking = () => {
isSeekingRef.current = true;
// Prevent autoplay during seek if video was paused
if (!isPlayingRef.current && !video.paused) {
video.pause();
}
+8 -45
View File
@@ -24,10 +24,8 @@ interface AnimationState {
focusY: number;
}
/**
* Renders video frames with all effects (background, zoom, crop, blur, shadow)
* to an offscreen canvas for export.
*/
// Renders video frames with all effects (background, zoom, crop, blur, shadow) to an offscreen canvas for export.
export class FrameRenderer {
private app: PIXI.Application | null = null;
private cameraContainer: PIXI.Container | null = null;
@@ -65,7 +63,6 @@ export class FrameRenderer {
}
// Initialize PixiJS app with transparent background (background rendered separately)
// Use 2x resolution to match Retina displays and ensure blur quality matches preview
this.app = new PIXI.Application();
await this.app.init({
canvas,
@@ -73,7 +70,7 @@ export class FrameRenderer {
height: this.config.height,
backgroundAlpha: 0,
antialias: true,
resolution: 2, // Match typical Retina/high-DPI displays for blur quality
resolution: 2,
autoDensity: true,
});
@@ -150,20 +147,18 @@ export class FrameRenderer {
img.src = imageUrl;
});
// Draw the image using cover and center positioning (like CSS bg-cover bg-center)
// Draw the image using cover and center positioning
const imgAspect = img.width / img.height;
const canvasAspect = this.config.width / this.config.height;
let drawWidth, drawHeight, drawX, drawY;
if (imgAspect > canvasAspect) {
// Image is wider - fit to height and crop width
drawHeight = this.config.height;
drawWidth = drawHeight * imgAspect;
drawX = (this.config.width - drawWidth) / 2;
drawY = 0;
} else {
// Image is taller - fit to width and crop height
drawWidth = this.config.width;
drawHeight = drawWidth / imgAspect;
drawX = 0;
@@ -172,13 +167,10 @@ export class FrameRenderer {
bgCtx.drawImage(img, drawX, drawY, drawWidth, drawHeight);
} else if (wallpaper.startsWith('#')) {
// Solid color
bgCtx.fillStyle = wallpaper;
bgCtx.fillRect(0, 0, this.config.width, this.config.height);
} else if (wallpaper.startsWith('linear-gradient') || wallpaper.startsWith('radial-gradient')) {
// Gradient - parse and create CanvasGradient"}
// Simple gradient parser for common cases
const gradientMatch = wallpaper.match(/(linear|radial)-gradient\((.+)\)/);
if (gradientMatch) {
const [, type, params] = gradientMatch;
@@ -187,15 +179,10 @@ export class FrameRenderer {
let gradient: CanvasGradient;
if (type === 'linear') {
// Default to top-to-bottom if no direction specified
gradient = bgCtx.createLinearGradient(0, 0, 0, this.config.height);
// Parse color stops
parts.forEach((part, index) => {
// Skip direction keywords
if (part.startsWith('to ') || part.includes('deg')) return;
// Extract color (everything before optional percentage/position)
const colorMatch = part.match(/^(#[0-9a-fA-F]{3,8}|rgba?\([^)]+\)|[a-z]+)/);
if (colorMatch) {
const color = colorMatch[1];
@@ -204,7 +191,6 @@ export class FrameRenderer {
}
});
} else {
// Radial gradient - center circle
const cx = this.config.width / 2;
const cy = this.config.height / 2;
const radius = Math.max(this.config.width, this.config.height) / 2;
@@ -228,19 +214,17 @@ export class FrameRenderer {
bgCtx.fillRect(0, 0, this.config.width, this.config.height);
}
} else {
// Unknown format, try to use as fillStyle (might be a named color like 'red', 'blue', etc.)
bgCtx.fillStyle = wallpaper;
bgCtx.fillRect(0, 0, this.config.width, this.config.height);
}
} catch (error) {
console.error('[FrameRenderer] Error setting up background, using fallback:', error);
// Fallback to black background
bgCtx.fillStyle = '#000000';
bgCtx.fillRect(0, 0, this.config.width, this.config.height);
}
// Store the background canvas for compositing
this.backgroundSprite = bgCanvas as any; // Reuse the field to store canvas"}
this.backgroundSprite = bgCanvas as any;
}
async renderFrame(videoFrame: VideoFrame, timestamp: number): Promise<void> {
@@ -248,7 +232,7 @@ export class FrameRenderer {
throw new Error('Renderer not initialized');
}
this.currentVideoTime = timestamp / 1000000; // convert microseconds to seconds
this.currentVideoTime = timestamp / 1000000;
// Create or update video sprite from VideoFrame
if (!this.videoSprite) {
@@ -264,10 +248,8 @@ export class FrameRenderer {
// Apply layout
this.updateLayout();
// Apply zoom effects normalized to 60fps (1 tick per video frame)
// This ensures consistent animation speed regardless of display refresh rate
const timeMs = this.currentVideoTime * 1000;
const TICKS_PER_FRAME = 1; // 60fps standard - 1 animation update per video frame
const TICKS_PER_FRAME = 1;
let maxMotionIntensity = 0;
for (let i = 0; i < TICKS_PER_FRAME; i++) {
@@ -285,7 +267,7 @@ export class FrameRenderer {
focusX: this.animationState.focusX,
focusY: this.animationState.focusY,
motionIntensity: maxMotionIntensity,
isPlaying: true, // Enable motion blur
isPlaying: true,
});
// Render the PixiJS stage to its canvas (video only, transparent background)
@@ -355,10 +337,6 @@ export class FrameRenderer {
return clampFocusToStageUtil(focus, depth as any, this.layoutCache);
}
/**
* Updates animation state for one tick and returns motion intensity.
* This simulates one PixiJS ticker update.
*/
private updateAnimationState(timeMs: number): number {
if (!this.cameraContainer || !this.layoutCache) return 0;
@@ -368,12 +346,10 @@ export class FrameRenderer {
let targetScaleFactor = 1;
let targetFocus = { ...defaultFocus };
// Match the preview logic exactly
if (region && strength > 0) {
const zoomScale = ZOOM_DEPTH_SCALES[region.depth];
const regionFocus = this.clampFocusToStage(region.focus, region.depth);
// Interpolate scale and focus based on region strength (exponential easing)
targetScaleFactor = 1 + (zoomScale - 1) * strength;
targetFocus = {
cx: defaultFocus.cx + (regionFocus.cx - defaultFocus.cx) * strength,
@@ -395,7 +371,6 @@ export class FrameRenderer {
let nextFocusX = prevFocusX;
let nextFocusY = prevFocusY;
// Apply smooth exponential easing
if (Math.abs(scaleDelta) > MIN_DELTA) {
nextScale = prevScale + scaleDelta * SMOOTHING_FACTOR;
} else {
@@ -418,7 +393,6 @@ export class FrameRenderer {
state.focusX = nextFocusX;
state.focusY = nextFocusY;
// Calculate and return motion intensity for blur
return Math.max(
Math.abs(nextScale - prevScale),
Math.abs(nextFocusX - prevFocusX),
@@ -442,7 +416,6 @@ export class FrameRenderer {
const bgCanvas = this.backgroundSprite as any as HTMLCanvasElement;
if (this.config.showBlur) {
// Apply CSS blur(2px) to background
ctx.save();
ctx.filter = 'blur(2px)';
ctx.drawImage(bgCanvas, 0, 0, w, h);
@@ -456,22 +429,14 @@ export class FrameRenderer {
// Step 2: Draw video layer with shadows on top of background
if (this.config.showShadow && this.shadowCanvas && this.shadowCtx) {
// CSS drop-shadow creates layered shadows. We need to composite them properly.
// The key is to draw all shadows UNDER the video content, not draw video multiple times
const shadowCtx = this.shadowCtx;
shadowCtx.clearRect(0, 0, w, h);
// Apply all three shadow layers in a single draw call using composite filter
// This matches CSS drop-shadow behavior exactly - note: no 'px' on X offset in CSS syntax
shadowCtx.save();
shadowCtx.filter = 'drop-shadow(0 12px 48px rgba(0,0,0,0.7)) drop-shadow(0 4px 16px rgba(0,0,0,0.5)) drop-shadow(0 2px 8px rgba(0,0,0,0.3))';
shadowCtx.drawImage(videoCanvas, 0, 0, w, h);
shadowCtx.restore();
// Draw shadow canvas (which has shadows + video) on top of background
ctx.drawImage(this.shadowCanvas, 0, 0, w, h);
} else {
// No shadows, just draw video directly on top of background
ctx.drawImage(videoCanvas, 0, 0, w, h);
}
}
@@ -480,7 +445,6 @@ export class FrameRenderer {
if (!this.compositeCanvas) {
throw new Error('Renderer not initialized');
}
// Return the composite canvas which includes shadows
return this.compositeCanvas;
}
@@ -496,7 +460,6 @@ export class FrameRenderer {
this.videoSprite.destroy();
this.videoSprite = null;
}
// backgroundSprite is now a canvas, just null it
this.backgroundSprite = null;
if (this.app) {
this.app.destroy(true, { children: true, texture: true, textureSource: true });
+3
View File
@@ -3,3 +3,6 @@ export { VideoFileDecoder } from './videoDecoder';
export { FrameRenderer } from './frameRenderer';
export { VideoMuxer } from './muxer';
export type { ExportConfig, ExportProgress, ExportResult, VideoFrameData } from './types';
// Ref: https://pietrasiak.com/fast-video-rendering-and-encoding-using-web-apis
+1 -6
View File
@@ -22,10 +22,6 @@ interface Muxer {
target?: any;
}
/**
* Video muxer that combines encoded video and audio tracks into a final MP4 file.
* Uses mp4-muxer library for efficient muxing without re-encoding.
*/
export class VideoMuxer {
private muxer: Muxer | null = null;
private config: ExportConfig;
@@ -37,7 +33,6 @@ export class VideoMuxer {
}
async initialize(): Promise<void> {
// Dynamically import mp4-muxer
const MP4MuxerModule = await import('mp4-muxer');
const MP4MuxerClass = (MP4MuxerModule as any).Muxer || MP4MuxerModule.default;
const ArrayBufferTarget = (MP4MuxerModule as any).ArrayBufferTarget;
@@ -47,7 +42,7 @@ export class VideoMuxer {
const options: MP4MuxerOptions = {
target,
video: {
codec: 'avc', // mp4-muxer only accepts 'avc', not full codec string
codec: 'avc',
width: this.config.width,
height: this.config.height,
},
+2 -7
View File
@@ -6,17 +6,12 @@ export interface DecodedVideoInfo {
codec: string;
}
/**
* Simple video decoder for WebM files using native VideoDecoder API.
* For export, we'll use a different approach - directly rendering from the HTML video element.
*/
export class VideoFileDecoder {
private decoder: VideoDecoder | null = null;
private info: DecodedVideoInfo | null = null;
private videoElement: HTMLVideoElement | null = null;
async loadVideo(videoUrl: string): Promise<DecodedVideoInfo> {
// Create a video element to get video info
this.videoElement = document.createElement('video');
this.videoElement.src = videoUrl;
this.videoElement.preload = 'metadata';
@@ -29,8 +24,8 @@ export class VideoFileDecoder {
width: video.videoWidth,
height: video.videoHeight,
duration: video.duration,
frameRate: 60, // 60fps for smooth playback
codec: 'avc1.640033', // H.264 High Profile Level 5.1
frameRate: 60,
codec: 'avc1.640033',
};
resolve(this.info);
+15 -33
View File
@@ -14,12 +14,6 @@ interface VideoExporterConfig extends ExportConfig {
onProgress?: (progress: ExportProgress) => void;
}
/**
* Fast video exporter using VideoFrame and VideoEncoder APIs.
* Avoids reading pixel data into JavaScript memory for maximum performance.
*
* Based on: https://pietrasiak.com/fast-video-rendering-and-encoding-using-web-apis
*/
export class VideoExporter {
private config: VideoExporterConfig;
private decoder: VideoFileDecoder | null = null;
@@ -29,7 +23,7 @@ export class VideoExporter {
private cancelled = false;
private encodedChunks: EncodedVideoChunk[] = [];
private encodeQueue = 0;
private readonly MAX_ENCODE_QUEUE = 60; // Increased for better throughput
private readonly MAX_ENCODE_QUEUE = 60;
private videoDescription: Uint8Array | undefined;
constructor(config: VideoExporterConfig) {
@@ -38,15 +32,14 @@ export class VideoExporter {
async export(): Promise<ExportResult> {
try {
// Clean up any previous export state
this.cleanup();
this.cancelled = false;
// Step 1: Initialize decoder and load video
// Initialize decoder and load video
this.decoder = new VideoFileDecoder();
const videoInfo = await this.decoder.loadVideo(this.config.videoUrl);
// Step 2: Initialize frame renderer
// Initialize frame renderer
this.renderer = new FrameRenderer({
width: this.config.width,
height: this.config.height,
@@ -60,26 +53,26 @@ export class VideoExporter {
});
await this.renderer.initialize();
// Step 3: Initialize video encoder
// Initialize video encoder
const totalFrames = Math.ceil(videoInfo.duration * this.config.frameRate);
await this.initializeEncoder();
// Step 4: Initialize muxer
// Initialize muxer
this.muxer = new VideoMuxer(this.config, false);
await this.muxer.initialize();
// Step 5: Get the video element for frame extraction
// Get the video element for frame extraction
const videoElement = this.decoder.getVideoElement();
if (!videoElement) {
throw new Error('Video element not available');
}
// Step 6: Process frames with optimized seeking
// Process frames with optimized seeking
const frameDuration = 1_000_000 / this.config.frameRate; // in microseconds
let frameIndex = 0;
const timeStep = 1 / this.config.frameRate;
// Optimize: Pre-load first frame
// Pre-load first frame
videoElement.currentTime = 0;
await new Promise(resolve => {
const onSeeked = () => {
@@ -92,7 +85,7 @@ export class VideoExporter {
while (frameIndex < totalFrames && !this.cancelled) {
const timestamp = frameIndex * frameDuration;
const videoTime = frameIndex * timeStep;
// Seek to frame (optimized: only seek if not already there)
// Seek to frame (only seek if not already there)
if (Math.abs(videoElement.currentTime - videoTime) > 0.001) {
videoElement.currentTime = videoTime;
await Promise.race([
@@ -104,7 +97,7 @@ export class VideoExporter {
};
videoElement.addEventListener('seeked', onSeeked, { once: true });
}),
new Promise(resolve => setTimeout(resolve, 200)) // higher is slower but better capture
new Promise(resolve => setTimeout(resolve, 200)) // higher this number, slower the export, but better capture/ no frame drops
]);
}
@@ -116,31 +109,26 @@ export class VideoExporter {
// Render the frame with all effects
await this.renderer!.renderFrame(videoFrame, timestamp);
// Close the video frame as we're done with it
videoFrame.close();
// Wait if encode queue is too large (backpressure)
while (this.encodeQueue >= this.MAX_ENCODE_QUEUE && !this.cancelled) {
await new Promise(resolve => setTimeout(resolve, 1));
}
if (this.cancelled) break;
// Create VideoFrame from rendered canvas (on GPU, no pixel read!)
const canvas = this.renderer!.getCanvas();
const exportFrame = new VideoFrame(canvas, {
timestamp,
duration: frameDuration,
});
// Encode the frame (check if encoder is still valid)
if (this.encoder && this.encoder.state === 'configured') {
this.encodeQueue++;
this.encoder.encode(exportFrame, { keyFrame: frameIndex % 150 === 0 });
}
exportFrame.close();
// Report progress
frameIndex++;
if (this.config.onProgress) {
@@ -157,12 +145,12 @@ export class VideoExporter {
return { success: false, error: 'Export cancelled' };
}
// Step 7: Finalize encoding
// Finalize encoding
if (this.encoder && this.encoder.state === 'configured') {
await this.encoder.flush();
}
// Step 8: Add all chunks to muxer with metadata
// Add all chunks to muxer with metadata
for (let i = 0; i < this.encodedChunks.length; i++) {
const chunk = this.encodedChunks[i];
const meta: EncodedVideoChunkMetadata = {};
@@ -180,7 +168,7 @@ export class VideoExporter {
this.muxer!.addVideoChunk(chunk, meta);
}
// Step 9: Finalize muxer and get output blob
// Finalize muxer and get output blob
const blob = this.muxer!.finalize();
return { success: true, blob };
@@ -202,7 +190,6 @@ export class VideoExporter {
this.encoder = new VideoEncoder({
output: (chunk, meta) => {
// Store the first chunk's metadata (contains codec description)
if (meta?.decoderConfig?.description && !videoDescription) {
const desc = meta.decoderConfig.description;
videoDescription = new Uint8Array(desc instanceof ArrayBuffer ? desc : (desc as any));
@@ -216,7 +203,6 @@ export class VideoExporter {
},
});
// Configure encoder for H.264 (AVC) with level 5.1 for high resolution support
const codec = this.config.codec || 'avc1.640033';
this.encoder.configure({
@@ -225,20 +211,18 @@ export class VideoExporter {
height: this.config.height,
bitrate: this.config.bitrate,
framerate: this.config.frameRate,
latencyMode: 'realtime', // Changed from 'quality' for faster encoding
latencyMode: 'realtime',
bitrateMode: 'variable',
hardwareAcceleration: 'prefer-hardware', // Use GPU encoding
hardwareAcceleration: 'prefer-hardware',
} as VideoEncoderConfig);
}
cancel(): void {
this.cancelled = true;
// Immediately cleanup to stop encoding
this.cleanup();
}
private cleanup(): void {
// Close encoder safely
if (this.encoder) {
try {
if (this.encoder.state === 'configured') {
@@ -250,7 +234,6 @@ export class VideoExporter {
this.encoder = null;
}
// Destroy decoder
if (this.decoder) {
try {
this.decoder.destroy();
@@ -260,7 +243,6 @@ export class VideoExporter {
this.decoder = null;
}
// Destroy renderer
if (this.renderer) {
try {
this.renderer.destroy();