Siddharth
534 lines
19 KiB
TypeScript
534 lines
19 KiB
TypeScript
import { Application, Container, Sprite, Graphics, BlurFilter, Texture } from 'pixi.js';
|
|
import type { ZoomRegion, CropRegion, AnnotationRegion } from '@/components/video-editor/types';
|
|
import { ZOOM_DEPTH_SCALES } from '@/components/video-editor/types';
|
|
import { findDominantRegion } from '@/components/video-editor/videoPlayback/zoomRegionUtils';
|
|
import { applyZoomTransform } from '@/components/video-editor/videoPlayback/zoomTransform';
|
|
import { DEFAULT_FOCUS, SMOOTHING_FACTOR, MIN_DELTA } from '@/components/video-editor/videoPlayback/constants';
|
|
import { clampFocusToStage as clampFocusToStageUtil } from '@/components/video-editor/videoPlayback/focusUtils';
|
|
import { renderAnnotations } from './annotationRenderer';
|
|
|
|
interface FrameRenderConfig {
|
|
width: number;
|
|
height: number;
|
|
wallpaper: string;
|
|
zoomRegions: ZoomRegion[];
|
|
showShadow: boolean;
|
|
shadowIntensity: number;
|
|
showBlur: boolean;
|
|
motionBlurEnabled?: boolean;
|
|
borderRadius?: number;
|
|
padding?: number;
|
|
cropRegion: CropRegion;
|
|
videoWidth: number;
|
|
videoHeight: number;
|
|
annotationRegions?: AnnotationRegion[];
|
|
previewWidth?: number;
|
|
previewHeight?: number;
|
|
}
|
|
|
|
interface AnimationState {
|
|
scale: number;
|
|
focusX: number;
|
|
focusY: number;
|
|
}
|
|
|
|
// Renders video frames with all effects (background, zoom, crop, blur, shadow) to an offscreen canvas for export.
|
|
|
|
export class FrameRenderer {
|
|
private app: Application | null = null;
|
|
private cameraContainer: Container | null = null;
|
|
private videoContainer: Container | null = null;
|
|
private videoSprite: Sprite | null = null;
|
|
private backgroundSprite: Sprite | null = null;
|
|
private maskGraphics: Graphics | null = null;
|
|
private blurFilter: BlurFilter | null = null;
|
|
private shadowCanvas: HTMLCanvasElement | null = null;
|
|
private shadowCtx: CanvasRenderingContext2D | null = null;
|
|
private compositeCanvas: HTMLCanvasElement | null = null;
|
|
private compositeCtx: CanvasRenderingContext2D | null = null;
|
|
private config: FrameRenderConfig;
|
|
private animationState: AnimationState;
|
|
private layoutCache: any = null;
|
|
private currentVideoTime = 0;
|
|
|
|
constructor(config: FrameRenderConfig) {
|
|
this.config = config;
|
|
this.animationState = {
|
|
scale: 1,
|
|
focusX: DEFAULT_FOCUS.cx,
|
|
focusY: DEFAULT_FOCUS.cy,
|
|
};
|
|
}
|
|
|
|
async initialize(): Promise<void> {
|
|
// Create canvas for rendering
|
|
const canvas = document.createElement('canvas');
|
|
canvas.width = this.config.width;
|
|
canvas.height = this.config.height;
|
|
|
|
// Try to set colorSpace if supported (may not be available on all platforms)
|
|
try {
|
|
if (canvas && 'colorSpace' in canvas) {
|
|
// @ts-ignore
|
|
canvas.colorSpace = 'srgb';
|
|
}
|
|
} catch (error) {
|
|
// Silently ignore colorSpace errors on platforms that don't support it
|
|
console.warn('[FrameRenderer] colorSpace not supported on this platform:', error);
|
|
}
|
|
|
|
// Initialize PixiJS with optimized settings for export performance
|
|
this.app = new Application();
|
|
await this.app.init({
|
|
canvas,
|
|
width: this.config.width,
|
|
height: this.config.height,
|
|
backgroundAlpha: 0,
|
|
antialias: true,
|
|
resolution: 1,
|
|
autoDensity: true,
|
|
});
|
|
|
|
// Setup containers
|
|
this.cameraContainer = new Container();
|
|
this.videoContainer = new Container();
|
|
this.app.stage.addChild(this.cameraContainer);
|
|
this.cameraContainer.addChild(this.videoContainer);
|
|
|
|
// Setup background (render separately, not in PixiJS)
|
|
await this.setupBackground();
|
|
|
|
// Setup blur filter for video container
|
|
this.blurFilter = new BlurFilter();
|
|
this.blurFilter.quality = 5;
|
|
this.blurFilter.resolution = this.app.renderer.resolution;
|
|
this.blurFilter.blur = 0;
|
|
this.videoContainer.filters = [this.blurFilter];
|
|
|
|
// Setup composite canvas for final output with shadows
|
|
this.compositeCanvas = document.createElement('canvas');
|
|
this.compositeCanvas.width = this.config.width;
|
|
this.compositeCanvas.height = this.config.height;
|
|
this.compositeCtx = this.compositeCanvas.getContext('2d', { willReadFrequently: false });
|
|
|
|
if (!this.compositeCtx) {
|
|
throw new Error('Failed to get 2D context for composite canvas');
|
|
}
|
|
|
|
// Setup shadow canvas if needed
|
|
if (this.config.showShadow) {
|
|
this.shadowCanvas = document.createElement('canvas');
|
|
this.shadowCanvas.width = this.config.width;
|
|
this.shadowCanvas.height = this.config.height;
|
|
this.shadowCtx = this.shadowCanvas.getContext('2d', { willReadFrequently: false });
|
|
|
|
if (!this.shadowCtx) {
|
|
throw new Error('Failed to get 2D context for shadow canvas');
|
|
}
|
|
}
|
|
|
|
// Setup mask
|
|
this.maskGraphics = new Graphics();
|
|
this.videoContainer.addChild(this.maskGraphics);
|
|
this.videoContainer.mask = this.maskGraphics;
|
|
}
|
|
|
|
private async setupBackground(): Promise<void> {
|
|
const wallpaper = this.config.wallpaper;
|
|
|
|
// Create background canvas for separate rendering (not affected by zoom)
|
|
const bgCanvas = document.createElement('canvas');
|
|
bgCanvas.width = this.config.width;
|
|
bgCanvas.height = this.config.height;
|
|
const bgCtx = bgCanvas.getContext('2d')!;
|
|
|
|
try {
|
|
// Render background based on type
|
|
if (wallpaper.startsWith('file://') || wallpaper.startsWith('data:') || wallpaper.startsWith('/') || wallpaper.startsWith('http')) {
|
|
// Image background
|
|
const img = new Image();
|
|
// Don't set crossOrigin for same-origin images to avoid CORS taint
|
|
// Only set it for cross-origin URLs
|
|
let imageUrl: string;
|
|
if (wallpaper.startsWith('http')) {
|
|
imageUrl = wallpaper;
|
|
if (!imageUrl.startsWith(window.location.origin)) {
|
|
img.crossOrigin = 'anonymous';
|
|
}
|
|
} else if (wallpaper.startsWith('file://') || wallpaper.startsWith('data:')) {
|
|
imageUrl = wallpaper;
|
|
} else {
|
|
imageUrl = window.location.origin + wallpaper;
|
|
}
|
|
|
|
await new Promise<void>((resolve, reject) => {
|
|
img.onload = () => resolve();
|
|
img.onerror = (err) => {
|
|
console.error('[FrameRenderer] Failed to load background image:', imageUrl, err);
|
|
reject(new Error(`Failed to load background image: ${imageUrl}`));
|
|
};
|
|
img.src = imageUrl;
|
|
});
|
|
|
|
// Draw the image using cover and center positioning
|
|
const imgAspect = img.width / img.height;
|
|
const canvasAspect = this.config.width / this.config.height;
|
|
|
|
let drawWidth, drawHeight, drawX, drawY;
|
|
|
|
if (imgAspect > canvasAspect) {
|
|
drawHeight = this.config.height;
|
|
drawWidth = drawHeight * imgAspect;
|
|
drawX = (this.config.width - drawWidth) / 2;
|
|
drawY = 0;
|
|
} else {
|
|
drawWidth = this.config.width;
|
|
drawHeight = drawWidth / imgAspect;
|
|
drawX = 0;
|
|
drawY = (this.config.height - drawHeight) / 2;
|
|
}
|
|
|
|
bgCtx.drawImage(img, drawX, drawY, drawWidth, drawHeight);
|
|
} else if (wallpaper.startsWith('#')) {
|
|
bgCtx.fillStyle = wallpaper;
|
|
bgCtx.fillRect(0, 0, this.config.width, this.config.height);
|
|
} else if (wallpaper.startsWith('linear-gradient') || wallpaper.startsWith('radial-gradient')) {
|
|
|
|
const gradientMatch = wallpaper.match(/(linear|radial)-gradient\((.+)\)/);
|
|
if (gradientMatch) {
|
|
const [, type, params] = gradientMatch;
|
|
const parts = params.split(',').map(s => s.trim());
|
|
|
|
let gradient: CanvasGradient;
|
|
|
|
if (type === 'linear') {
|
|
gradient = bgCtx.createLinearGradient(0, 0, 0, this.config.height);
|
|
parts.forEach((part, index) => {
|
|
if (part.startsWith('to ') || part.includes('deg')) return;
|
|
|
|
const colorMatch = part.match(/^(#[0-9a-fA-F]{3,8}|rgba?\([^)]+\)|[a-z]+)/);
|
|
if (colorMatch) {
|
|
const color = colorMatch[1];
|
|
const position = index / (parts.length - 1);
|
|
gradient.addColorStop(position, color);
|
|
}
|
|
});
|
|
} else {
|
|
const cx = this.config.width / 2;
|
|
const cy = this.config.height / 2;
|
|
const radius = Math.max(this.config.width, this.config.height) / 2;
|
|
gradient = bgCtx.createRadialGradient(cx, cy, 0, cx, cy, radius);
|
|
|
|
parts.forEach((part, index) => {
|
|
const colorMatch = part.match(/^(#[0-9a-fA-F]{3,8}|rgba?\([^)]+\)|[a-z]+)/);
|
|
if (colorMatch) {
|
|
const color = colorMatch[1];
|
|
const position = index / (parts.length - 1);
|
|
gradient.addColorStop(position, color);
|
|
}
|
|
});
|
|
}
|
|
|
|
bgCtx.fillStyle = gradient;
|
|
bgCtx.fillRect(0, 0, this.config.width, this.config.height);
|
|
} else {
|
|
console.warn('[FrameRenderer] Could not parse gradient, using black fallback');
|
|
bgCtx.fillStyle = '#000000';
|
|
bgCtx.fillRect(0, 0, this.config.width, this.config.height);
|
|
}
|
|
} else {
|
|
bgCtx.fillStyle = wallpaper;
|
|
bgCtx.fillRect(0, 0, this.config.width, this.config.height);
|
|
}
|
|
} catch (error) {
|
|
console.error('[FrameRenderer] Error setting up background, using fallback:', error);
|
|
bgCtx.fillStyle = '#000000';
|
|
bgCtx.fillRect(0, 0, this.config.width, this.config.height);
|
|
}
|
|
|
|
// Store the background canvas for compositing
|
|
this.backgroundSprite = bgCanvas as any;
|
|
}
|
|
|
|
async renderFrame(videoFrame: VideoFrame, timestamp: number): Promise<void> {
|
|
if (!this.app || !this.videoContainer || !this.cameraContainer) {
|
|
throw new Error('Renderer not initialized');
|
|
}
|
|
|
|
this.currentVideoTime = timestamp / 1000000;
|
|
|
|
// Create or update video sprite from VideoFrame
|
|
if (!this.videoSprite) {
|
|
const texture = Texture.from(videoFrame as any);
|
|
this.videoSprite = new Sprite(texture);
|
|
this.videoContainer.addChild(this.videoSprite);
|
|
} else {
|
|
// Destroy old texture to avoid memory leaks, then create new one
|
|
const oldTexture = this.videoSprite.texture;
|
|
const newTexture = Texture.from(videoFrame as any);
|
|
this.videoSprite.texture = newTexture;
|
|
oldTexture.destroy(true);
|
|
}
|
|
|
|
// Apply layout
|
|
this.updateLayout();
|
|
|
|
const timeMs = this.currentVideoTime * 1000;
|
|
const TICKS_PER_FRAME = 1;
|
|
|
|
let maxMotionIntensity = 0;
|
|
for (let i = 0; i < TICKS_PER_FRAME; i++) {
|
|
const motionIntensity = this.updateAnimationState(timeMs);
|
|
maxMotionIntensity = Math.max(maxMotionIntensity, motionIntensity);
|
|
}
|
|
|
|
// Apply transform once with maximum motion intensity from all ticks
|
|
applyZoomTransform({
|
|
cameraContainer: this.cameraContainer,
|
|
blurFilter: this.blurFilter,
|
|
stageSize: this.layoutCache.stageSize,
|
|
baseMask: this.layoutCache.maskRect,
|
|
zoomScale: this.animationState.scale,
|
|
focusX: this.animationState.focusX,
|
|
focusY: this.animationState.focusY,
|
|
motionIntensity: maxMotionIntensity,
|
|
isPlaying: true,
|
|
motionBlurEnabled: this.config.motionBlurEnabled ?? false,
|
|
});
|
|
|
|
// Render the PixiJS stage to its canvas (video only, transparent background)
|
|
this.app.renderer.render(this.app.stage);
|
|
|
|
// Composite with shadows to final output canvas
|
|
this.compositeWithShadows();
|
|
|
|
// Render annotations on top if present
|
|
if (this.config.annotationRegions && this.config.annotationRegions.length > 0 && this.compositeCtx) {
|
|
// Calculate scale factor based on export vs preview dimensions
|
|
const previewWidth = this.config.previewWidth || 1920;
|
|
const previewHeight = this.config.previewHeight || 1080;
|
|
const scaleX = this.config.width / previewWidth;
|
|
const scaleY = this.config.height / previewHeight;
|
|
const scaleFactor = (scaleX + scaleY) / 2;
|
|
|
|
await renderAnnotations(
|
|
this.compositeCtx,
|
|
this.config.annotationRegions,
|
|
this.config.width,
|
|
this.config.height,
|
|
timeMs,
|
|
scaleFactor
|
|
);
|
|
}
|
|
}
|
|
|
|
private updateLayout(): void {
|
|
if (!this.app || !this.videoSprite || !this.maskGraphics || !this.videoContainer) return;
|
|
|
|
const { width, height } = this.config;
|
|
const { cropRegion, borderRadius = 0, padding = 0 } = this.config;
|
|
const videoWidth = this.config.videoWidth;
|
|
const videoHeight = this.config.videoHeight;
|
|
|
|
// Calculate cropped video dimensions
|
|
const cropStartX = cropRegion.x;
|
|
const cropStartY = cropRegion.y;
|
|
const cropEndX = cropRegion.x + cropRegion.width;
|
|
const cropEndY = cropRegion.y + cropRegion.height;
|
|
|
|
const croppedVideoWidth = videoWidth * (cropEndX - cropStartX);
|
|
const croppedVideoHeight = videoHeight * (cropEndY - cropStartY);
|
|
|
|
// Calculate scale to fit in viewport
|
|
// Padding is a percentage (0-100), where 50% ~ 0.8 scale
|
|
const paddingScale = 1.0 - (padding / 100) * 0.4;
|
|
const viewportWidth = width * paddingScale;
|
|
const viewportHeight = height * paddingScale;
|
|
const scale = Math.min(viewportWidth / croppedVideoWidth, viewportHeight / croppedVideoHeight);
|
|
|
|
// Position video sprite
|
|
this.videoSprite.width = videoWidth * scale;
|
|
this.videoSprite.height = videoHeight * scale;
|
|
|
|
const cropPixelX = cropStartX * videoWidth * scale;
|
|
const cropPixelY = cropStartY * videoHeight * scale;
|
|
this.videoSprite.x = -cropPixelX;
|
|
this.videoSprite.y = -cropPixelY;
|
|
|
|
// Position video container
|
|
const croppedDisplayWidth = croppedVideoWidth * scale;
|
|
const croppedDisplayHeight = croppedVideoHeight * scale;
|
|
const centerOffsetX = (width - croppedDisplayWidth) / 2;
|
|
const centerOffsetY = (height - croppedDisplayHeight) / 2;
|
|
this.videoContainer.x = centerOffsetX;
|
|
this.videoContainer.y = centerOffsetY;
|
|
|
|
// scale border radius by export/preview canvas ratio
|
|
const previewWidth = this.config.previewWidth || 1920;
|
|
const previewHeight = this.config.previewHeight || 1080;
|
|
const canvasScaleFactor = Math.min(width / previewWidth, height / previewHeight);
|
|
const scaledBorderRadius = borderRadius * canvasScaleFactor;
|
|
|
|
this.maskGraphics.clear();
|
|
this.maskGraphics.roundRect(0, 0, croppedDisplayWidth, croppedDisplayHeight, scaledBorderRadius);
|
|
this.maskGraphics.fill({ color: 0xffffff });
|
|
|
|
// Cache layout info
|
|
this.layoutCache = {
|
|
stageSize: { width, height },
|
|
videoSize: { width: croppedVideoWidth, height: croppedVideoHeight },
|
|
baseScale: scale,
|
|
baseOffset: { x: centerOffsetX, y: centerOffsetY },
|
|
maskRect: { x: 0, y: 0, width: croppedDisplayWidth, height: croppedDisplayHeight },
|
|
};
|
|
}
|
|
|
|
private clampFocusToStage(focus: { cx: number; cy: number }, depth: number): { cx: number; cy: number } {
|
|
if (!this.layoutCache) return focus;
|
|
return clampFocusToStageUtil(focus, depth as any, this.layoutCache);
|
|
}
|
|
|
|
private updateAnimationState(timeMs: number): number {
|
|
if (!this.cameraContainer || !this.layoutCache) return 0;
|
|
|
|
const { region, strength } = findDominantRegion(this.config.zoomRegions, timeMs);
|
|
|
|
const defaultFocus = DEFAULT_FOCUS;
|
|
let targetScaleFactor = 1;
|
|
let targetFocus = { ...defaultFocus };
|
|
|
|
if (region && strength > 0) {
|
|
const zoomScale = ZOOM_DEPTH_SCALES[region.depth];
|
|
const regionFocus = this.clampFocusToStage(region.focus, region.depth);
|
|
|
|
targetScaleFactor = 1 + (zoomScale - 1) * strength;
|
|
targetFocus = {
|
|
cx: defaultFocus.cx + (regionFocus.cx - defaultFocus.cx) * strength,
|
|
cy: defaultFocus.cy + (regionFocus.cy - defaultFocus.cy) * strength,
|
|
};
|
|
}
|
|
|
|
const state = this.animationState;
|
|
|
|
const prevScale = state.scale;
|
|
const prevFocusX = state.focusX;
|
|
const prevFocusY = state.focusY;
|
|
|
|
const scaleDelta = targetScaleFactor - state.scale;
|
|
const focusXDelta = targetFocus.cx - state.focusX;
|
|
const focusYDelta = targetFocus.cy - state.focusY;
|
|
|
|
let nextScale = prevScale;
|
|
let nextFocusX = prevFocusX;
|
|
let nextFocusY = prevFocusY;
|
|
|
|
if (Math.abs(scaleDelta) > MIN_DELTA) {
|
|
nextScale = prevScale + scaleDelta * SMOOTHING_FACTOR;
|
|
} else {
|
|
nextScale = targetScaleFactor;
|
|
}
|
|
|
|
if (Math.abs(focusXDelta) > MIN_DELTA) {
|
|
nextFocusX = prevFocusX + focusXDelta * SMOOTHING_FACTOR;
|
|
} else {
|
|
nextFocusX = targetFocus.cx;
|
|
}
|
|
|
|
if (Math.abs(focusYDelta) > MIN_DELTA) {
|
|
nextFocusY = prevFocusY + focusYDelta * SMOOTHING_FACTOR;
|
|
} else {
|
|
nextFocusY = targetFocus.cy;
|
|
}
|
|
|
|
state.scale = nextScale;
|
|
state.focusX = nextFocusX;
|
|
state.focusY = nextFocusY;
|
|
|
|
return Math.max(
|
|
Math.abs(nextScale - prevScale),
|
|
Math.abs(nextFocusX - prevFocusX),
|
|
Math.abs(nextFocusY - prevFocusY)
|
|
);
|
|
}
|
|
|
|
private compositeWithShadows(): void {
|
|
if (!this.compositeCanvas || !this.compositeCtx || !this.app) return;
|
|
|
|
const videoCanvas = this.app.canvas as HTMLCanvasElement;
|
|
const ctx = this.compositeCtx;
|
|
const w = this.compositeCanvas.width;
|
|
const h = this.compositeCanvas.height;
|
|
|
|
// Clear composite canvas
|
|
ctx.clearRect(0, 0, w, h);
|
|
|
|
// Step 1: Draw background layer (with optional blur, not affected by zoom)
|
|
if (this.backgroundSprite) {
|
|
const bgCanvas = this.backgroundSprite as any as HTMLCanvasElement;
|
|
|
|
if (this.config.showBlur) {
|
|
ctx.save();
|
|
ctx.filter = 'blur(6px)'; // Canvas blur is weaker than CSS
|
|
ctx.drawImage(bgCanvas, 0, 0, w, h);
|
|
ctx.restore();
|
|
} else {
|
|
ctx.drawImage(bgCanvas, 0, 0, w, h);
|
|
}
|
|
} else {
|
|
console.warn('[FrameRenderer] No background sprite found during compositing!');
|
|
}
|
|
|
|
// Draw video layer with shadows on top of background
|
|
if (this.config.showShadow && this.config.shadowIntensity > 0 && this.shadowCanvas && this.shadowCtx) {
|
|
const shadowCtx = this.shadowCtx;
|
|
shadowCtx.clearRect(0, 0, w, h);
|
|
shadowCtx.save();
|
|
|
|
// Calculate shadow parameters based on intensity (0-1)
|
|
const intensity = this.config.shadowIntensity;
|
|
const baseBlur1 = 48 * intensity;
|
|
const baseBlur2 = 16 * intensity;
|
|
const baseBlur3 = 8 * intensity;
|
|
const baseAlpha1 = 0.7 * intensity;
|
|
const baseAlpha2 = 0.5 * intensity;
|
|
const baseAlpha3 = 0.3 * intensity;
|
|
const baseOffset = 12 * intensity;
|
|
|
|
shadowCtx.filter = `drop-shadow(0 ${baseOffset}px ${baseBlur1}px rgba(0,0,0,${baseAlpha1})) drop-shadow(0 ${baseOffset/3}px ${baseBlur2}px rgba(0,0,0,${baseAlpha2})) drop-shadow(0 ${baseOffset/6}px ${baseBlur3}px rgba(0,0,0,${baseAlpha3}))`;
|
|
shadowCtx.drawImage(videoCanvas, 0, 0, w, h);
|
|
shadowCtx.restore();
|
|
ctx.drawImage(this.shadowCanvas, 0, 0, w, h);
|
|
} else {
|
|
ctx.drawImage(videoCanvas, 0, 0, w, h);
|
|
}
|
|
}
|
|
|
|
getCanvas(): HTMLCanvasElement {
|
|
if (!this.compositeCanvas) {
|
|
throw new Error('Renderer not initialized');
|
|
}
|
|
return this.compositeCanvas;
|
|
}
|
|
|
|
|
|
destroy(): void {
|
|
if (this.videoSprite) {
|
|
this.videoSprite.destroy();
|
|
this.videoSprite = null;
|
|
}
|
|
this.backgroundSprite = null;
|
|
if (this.app) {
|
|
this.app.destroy(true, { children: true, texture: true, textureSource: true });
|
|
this.app = null;
|
|
}
|
|
this.cameraContainer = null;
|
|
this.videoContainer = null;
|
|
this.maskGraphics = null;
|
|
this.blurFilter = null;
|
|
this.shadowCanvas = null;
|
|
this.shadowCtx = null;
|
|
this.compositeCanvas = null;
|
|
this.compositeCtx = null;
|
|
}
|
|
}
|