merge: resolve conflicts and update video playback system

This commit is contained in:
AmitwalaH
2026-04-24 15:07:05 +05:30
129 changed files with 9956 additions and 6443 deletions
+80
View File
@@ -0,0 +1,80 @@
import { describe, expect, it } from "vitest";
import { applyMosaicToImageData, getBlurOverlayColor, normalizeBlurColor } from "./blurEffects";
function createTestImageData(width: number, height: number) {
const data = new Uint8ClampedArray(width * height * 4);
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
const offset = (y * width + x) * 4;
data[offset] = x * 20 + y;
data[offset + 1] = y * 20 + x;
data[offset + 2] = (x + y) * 10;
data[offset + 3] = 255;
}
}
return {
data,
width,
height,
} as ImageData;
}
describe("applyMosaicToImageData", () => {
it("collapses each block to a single representative color", () => {
const imageData = createTestImageData(4, 4);
const original = new Uint8ClampedArray(imageData.data);
applyMosaicToImageData(imageData, 2);
const topLeft = Array.from(imageData.data.slice(0, 4));
const topRightOffset = (1 * 4 + 1) * 4;
const topRight = Array.from(imageData.data.slice(topRightOffset, topRightOffset + 4));
expect(topLeft).toEqual(topRight);
expect(Array.from(original.slice(0, 4))).not.toEqual(topLeft);
});
it("reduces unique pixel colors, making the transform information-lossy", () => {
const imageData = createTestImageData(8, 8);
const before = new Set<string>();
const after = new Set<string>();
for (let i = 0; i < imageData.data.length; i += 4) {
before.add(
`${imageData.data[i]}-${imageData.data[i + 1]}-${imageData.data[i + 2]}-${imageData.data[i + 3]}`,
);
}
applyMosaicToImageData(imageData, 4);
for (let i = 0; i < imageData.data.length; i += 4) {
after.add(
`${imageData.data[i]}-${imageData.data[i + 1]}-${imageData.data[i + 2]}-${imageData.data[i + 3]}`,
);
}
expect(after.size).toBeLessThan(before.size);
expect(after.size).toBe(4);
});
});
describe("blur color helpers", () => {
it("normalizes invalid blur colors to white", () => {
expect(normalizeBlurColor("black")).toBe("black");
expect(normalizeBlurColor("invalid")).toBe("white");
});
it("returns a dark overlay when black blur color is selected", () => {
expect(
getBlurOverlayColor({
type: "blur",
shape: "rectangle",
color: "black",
intensity: 12,
blockSize: 12,
}),
).toBe("rgba(0, 0, 0, 0.18)");
});
});
+113
View File
@@ -0,0 +1,113 @@
import {
type BlurColor,
type BlurData,
type BlurType,
DEFAULT_BLUR_BLOCK_SIZE,
DEFAULT_BLUR_INTENSITY,
MAX_BLUR_BLOCK_SIZE,
MAX_BLUR_INTENSITY,
MIN_BLUR_BLOCK_SIZE,
MIN_BLUR_INTENSITY,
} from "@/components/video-editor/types";
function clamp(value: number, min: number, max: number) {
if (!Number.isFinite(value)) return min;
return Math.min(max, Math.max(min, value));
}
export function normalizeBlurType(value: unknown): BlurType {
return value === "mosaic" ? "mosaic" : "blur";
}
export function normalizeBlurColor(value: unknown): BlurColor {
return value === "black" ? "black" : "white";
}
export function getNormalizedBlurIntensity(blurData?: BlurData | null): number {
return clamp(
blurData?.intensity ?? DEFAULT_BLUR_INTENSITY,
MIN_BLUR_INTENSITY,
MAX_BLUR_INTENSITY,
);
}
export function getNormalizedMosaicBlockSize(blurData?: BlurData | null, scaleFactor = 1): number {
const rawBlockSize = clamp(
blurData?.blockSize ?? DEFAULT_BLUR_BLOCK_SIZE,
MIN_BLUR_BLOCK_SIZE,
MAX_BLUR_BLOCK_SIZE,
);
return Math.max(1, Math.round(rawBlockSize * Math.max(scaleFactor, 0.01)));
}
export function getBlurOverlayColor(blurData?: BlurData | null): string {
const blurColor = normalizeBlurColor(blurData?.color);
const blurType = normalizeBlurType(blurData?.type);
if (blurColor === "black") {
return blurType === "mosaic" ? "rgba(0, 0, 0, 0.72)" : "rgba(0, 0, 0, 0.56)";
}
return blurType === "mosaic" ? "rgba(255, 255, 255, 0.06)" : "rgba(255, 255, 255, 0.02)";
}
export function getMosaicGridOverlayColor(blurData?: BlurData | null): string {
return normalizeBlurColor(blurData?.color) === "black"
? "rgba(255,255,255,0.05)"
: "rgba(255,255,255,0.04)";
}
export function applyMosaicToImageData(imageData: ImageData, blockSize: number): ImageData {
const width = imageData.width;
const height = imageData.height;
const data = imageData.data;
const normalizedBlockSize = Math.max(1, Math.floor(blockSize));
if (width <= 0 || height <= 0 || normalizedBlockSize <= 1) {
return imageData;
}
for (let blockY = 0; blockY < height; blockY += normalizedBlockSize) {
for (let blockX = 0; blockX < width; blockX += normalizedBlockSize) {
const blockWidth = Math.min(normalizedBlockSize, width - blockX);
const blockHeight = Math.min(normalizedBlockSize, height - blockY);
const pixelCount = blockWidth * blockHeight;
if (pixelCount <= 0) {
continue;
}
let redTotal = 0;
let greenTotal = 0;
let blueTotal = 0;
let alphaTotal = 0;
for (let y = blockY; y < blockY + blockHeight; y++) {
for (let x = blockX; x < blockX + blockWidth; x++) {
const offset = (y * width + x) * 4;
redTotal += data[offset];
greenTotal += data[offset + 1];
blueTotal += data[offset + 2];
alphaTotal += data[offset + 3];
}
}
const averageRed = Math.round(redTotal / pixelCount);
const averageGreen = Math.round(greenTotal / pixelCount);
const averageBlue = Math.round(blueTotal / pixelCount);
const averageAlpha = Math.round(alphaTotal / pixelCount);
for (let y = blockY; y < blockY + blockHeight; y++) {
for (let x = blockX; x < blockX + blockWidth; x++) {
const offset = (y * width + x) * 4;
data[offset] = averageRed;
data[offset + 1] = averageGreen;
data[offset + 2] = averageBlue;
data[offset + 3] = averageAlpha;
}
}
}
}
return imageData;
}
+132 -18
View File
@@ -24,16 +24,111 @@ describe("computeCompositeLayout", () => {
webcamSize: { width: 1920, height: 1080 },
});
const refDim = Math.sqrt(1280 * 720);
const defaultFraction = 25 / 100; // DEFAULT_WEBCAM_SIZE_PRESET = 25
expect(layout).not.toBeNull();
expect(layout!.webcamRect).not.toBeNull();
expect(layout!.webcamRect!.width).toBeLessThanOrEqual(Math.round(1280 * 0.18) + 1);
expect(layout!.webcamRect!.height).toBeLessThanOrEqual(Math.round(720 * 0.18) + 1);
expect(layout!.webcamRect!.width).toBeLessThanOrEqual(Math.round(refDim * defaultFraction) + 1);
expect(layout!.webcamRect!.height).toBeLessThanOrEqual(
Math.round(refDim * defaultFraction) + 1,
);
expect(
Math.abs(layout!.webcamRect!.width * 1080 - layout!.webcamRect!.height * 1920),
).toBeLessThanOrEqual(1920);
});
it("uses cover-style full-width stacking in vertical stack mode", () => {
it("produces consistent webcam size across landscape and portrait aspect ratios", () => {
const webcamSize = { width: 1280, height: 720 };
const landscape = computeCompositeLayout({
canvasSize: { width: 1920, height: 1080 },
screenSize: { width: 1920, height: 1080 },
webcamSize,
webcamSizePreset: 50,
});
const portrait = computeCompositeLayout({
canvasSize: { width: 1080, height: 1920 },
screenSize: { width: 1080, height: 1920 },
webcamSize,
webcamSizePreset: 50,
});
expect(landscape).not.toBeNull();
expect(portrait).not.toBeNull();
// Same total pixel count — webcam area should be comparable
const landscapeArea = landscape!.webcamRect!.width * landscape!.webcamRect!.height;
const portraitArea = portrait!.webcamRect!.width * portrait!.webcamRect!.height;
expect(landscapeArea).toBe(portraitArea);
});
it("scales the webcam proportionally as webcamSizePreset increases", () => {
const canvasSize = { width: 1920, height: 1080 };
const screenSize = { width: 1920, height: 1080 };
const webcamSize = { width: 1280, height: 720 };
const small = computeCompositeLayout({
canvasSize,
screenSize,
webcamSize,
webcamSizePreset: 10,
});
const medium = computeCompositeLayout({
canvasSize,
screenSize,
webcamSize,
webcamSizePreset: 25,
});
const large = computeCompositeLayout({
canvasSize,
screenSize,
webcamSize,
webcamSizePreset: 50,
});
expect(small!.webcamRect!.width).toBeLessThan(medium!.webcamRect!.width);
expect(medium!.webcamRect!.width).toBeLessThan(large!.webcamRect!.width);
expect(small!.webcamRect!.height).toBeLessThan(medium!.webcamRect!.height);
expect(medium!.webcamRect!.height).toBeLessThan(large!.webcamRect!.height);
});
it("clamps webcamSizePreset to the valid range (1050)", () => {
const canvasSize = { width: 1920, height: 1080 };
const screenSize = { width: 1920, height: 1080 };
const webcamSize = { width: 1280, height: 720 };
const atMin = computeCompositeLayout({
canvasSize,
screenSize,
webcamSize,
webcamSizePreset: 10,
});
const belowMin = computeCompositeLayout({
canvasSize,
screenSize,
webcamSize,
webcamSizePreset: 1,
});
const atMax = computeCompositeLayout({
canvasSize,
screenSize,
webcamSize,
webcamSizePreset: 50,
});
const aboveMax = computeCompositeLayout({
canvasSize,
screenSize,
webcamSize,
webcamSizePreset: 100,
});
// Values below 10 should clamp to 10
expect(belowMin!.webcamRect!.width).toBe(atMin!.webcamRect!.width);
expect(belowMin!.webcamRect!.height).toBe(atMin!.webcamRect!.height);
// Values above 50 should clamp to 50
expect(aboveMax!.webcamRect!.width).toBe(atMax!.webcamRect!.width);
expect(aboveMax!.webcamRect!.height).toBe(atMax!.webcamRect!.height);
});
it("centers the combined screen and webcam stack in vertical stack mode", () => {
const layout = computeCompositeLayout({
canvasSize: { width: 1920, height: 1080 },
maxContentSize: { width: 1536, height: 864 },
@@ -43,23 +138,19 @@ describe("computeCompositeLayout", () => {
});
expect(layout).not.toBeNull();
expect(layout?.screenRect).toEqual({
x: 0,
y: 0,
width: 1920,
height: 0,
});
expect(layout?.webcamRect).toEqual({
x: 0,
y: 0,
width: 1920,
height: 1080,
borderRadius: 0,
});
expect(layout?.screenCover).toBe(true);
// Webcam is full-width at the bottom
expect(layout!.webcamRect).not.toBeNull();
expect(layout!.webcamRect!.x).toBe(0);
expect(layout!.webcamRect!.width).toBe(1920);
expect(layout!.webcamRect!.borderRadius).toBe(0);
// Screen fills remaining space at the top (cover mode)
expect(layout!.screenRect.x).toBe(0);
expect(layout!.screenRect.y).toBe(0);
expect(layout!.screenRect.width).toBe(1920);
expect(layout!.screenCover).toBe(true);
});
it("fills the canvas with the screen when vertical stack has no webcam", () => {
it("keeps the screen full-canvas and omits the webcam when dimensions are unavailable in stack mode", () => {
const layout = computeCompositeLayout({
canvasSize: { width: 1920, height: 1080 },
maxContentSize: { width: 1536, height: 864 },
@@ -78,6 +169,29 @@ describe("computeCompositeLayout", () => {
expect(layout?.screenCover).toBe(true);
});
it("uses a 2:1 split layout in dual frame mode", () => {
const layout = computeCompositeLayout({
canvasSize: { width: 1920, height: 1080 },
maxContentSize: { width: 1536, height: 864 },
screenSize: { width: 1920, height: 1080 },
webcamSize: { width: 1280, height: 720 },
layoutPreset: "dual-frame",
});
expect(layout).not.toBeNull();
expect(layout?.webcamRect).not.toBeNull();
expect(layout?.screenRect.y).toBe(108);
expect(layout?.screenRect.height).toBe(864);
expect(layout?.screenBorderRadius).toBe(layout?.webcamRect?.borderRadius);
expect(layout?.webcamRect?.y).toBe(108);
expect(layout?.webcamRect?.height).toBe(864);
expect(layout?.webcamRect?.x).toBeGreaterThan(layout?.screenRect.x ?? 0);
expect(
Math.abs((layout?.screenRect.width ?? 0) - 2 * (layout?.webcamRect?.width ?? 0)),
).toBeLessThanOrEqual(1);
expect(layout?.screenCover).toBe(true);
});
it("forces circular and square masks to use square dimensions", () => {
const circularLayout = computeCompositeLayout({
canvasSize: { width: 1920, height: 1080 },
+120 -10
View File
@@ -15,7 +15,9 @@ export interface Size {
height: number;
}
export type WebcamLayoutPreset = "picture-in-picture" | "vertical-stack";
export type WebcamLayoutPreset = "picture-in-picture" | "vertical-stack" | "dual-frame";
/** Webcam size as a percentage of the canvas reference dimension (1050). */
export type WebcamSizePreset = number;
export interface WebcamLayoutShadow {
color: string;
@@ -32,7 +34,6 @@ interface BorderRadiusRule {
interface OverlayTransform {
type: "overlay";
maxStageFraction: number;
marginFraction: number;
minMargin: number;
minSize: number;
@@ -43,9 +44,17 @@ interface StackTransform {
gap: number;
}
interface SplitTransform {
type: "split";
gapFraction: number;
minGap: number;
screenUnits: number;
webcamUnits: number;
}
export interface WebcamLayoutPresetDefinition {
label: string;
transform: OverlayTransform | StackTransform;
transform: OverlayTransform | StackTransform | SplitTransform;
borderRadius: BorderRadiusRule;
shadow: WebcamLayoutShadow | null;
}
@@ -53,11 +62,18 @@ export interface WebcamLayoutPresetDefinition {
export interface WebcamCompositeLayout {
screenRect: RenderRect;
webcamRect: StyledRenderRect | null;
screenBorderRadius?: number;
/** When true, the video should be scaled to cover screenRect (cropping overflow). */
screenCover?: boolean;
}
const MAX_STAGE_FRACTION = 0.18;
/** Convert a webcam size percentage (1050) to a fraction of the reference dimension. */
function webcamSizeToFraction(percent: number): number {
const safe = Number.isFinite(percent) ? percent : 25;
const clamped = Math.max(10, Math.min(50, safe));
return clamped / 100;
}
const MARGIN_FRACTION = 0.02;
const MAX_BORDER_RADIUS = 24;
const WEBCAM_LAYOUT_PRESET_MAP: Record<WebcamLayoutPreset, WebcamLayoutPresetDefinition> = {
@@ -65,7 +81,6 @@ const WEBCAM_LAYOUT_PRESET_MAP: Record<WebcamLayoutPreset, WebcamLayoutPresetDef
label: "Picture in Picture",
transform: {
type: "overlay",
maxStageFraction: MAX_STAGE_FRACTION,
marginFraction: MARGIN_FRACTION,
minMargin: 0,
minSize: 0,
@@ -95,6 +110,22 @@ const WEBCAM_LAYOUT_PRESET_MAP: Record<WebcamLayoutPreset, WebcamLayoutPresetDef
},
shadow: null,
},
"dual-frame": {
label: "Dual Frame",
transform: {
type: "split",
gapFraction: 0.02,
minGap: 12,
screenUnits: 2,
webcamUnits: 1,
},
borderRadius: {
max: MAX_BORDER_RADIUS,
min: 12,
fraction: 0.06,
},
shadow: null,
},
};
export const WEBCAM_LAYOUT_PRESETS = Object.entries(WEBCAM_LAYOUT_PRESET_MAP).map(
@@ -125,6 +156,7 @@ export function computeCompositeLayout(params: {
screenSize: Size;
webcamSize?: Size | null;
layoutPreset?: WebcamLayoutPreset;
webcamSizePreset?: WebcamSizePreset;
webcamPosition?: { cx: number; cy: number } | null;
webcamMaskShape?: import("@/components/video-editor/types").WebcamMaskShape;
}): WebcamCompositeLayout | null {
@@ -134,6 +166,7 @@ export function computeCompositeLayout(params: {
screenSize,
webcamSize,
layoutPreset = "picture-in-picture",
webcamSizePreset = 25,
webcamPosition,
webcamMaskShape = "rectangle",
} = params;
@@ -143,6 +176,8 @@ export function computeCompositeLayout(params: {
const webcamHeight = webcamSize?.height;
const preset = getWebcamLayoutPresetDefinition(layoutPreset);
const MAX_STAGE_FRACTION = webcamSizeToFraction(webcamSizePreset);
if (canvasWidth <= 0 || canvasHeight <= 0 || screenWidth <= 0 || screenHeight <= 0) {
return null;
}
@@ -183,6 +218,69 @@ export function computeCompositeLayout(params: {
};
}
if (preset.transform.type === "split") {
const screenRect = centerRect({
canvasSize,
size: screenSize,
maxSize: maxContentSize,
});
if (!webcamWidth || !webcamHeight || webcamWidth <= 0 || webcamHeight <= 0) {
return { screenRect, webcamRect: null };
}
const contentWidth = Math.min(canvasWidth, Math.max(1, Math.round(maxContentSize.width)));
const contentHeight = Math.min(canvasHeight, Math.max(1, Math.round(maxContentSize.height)));
const contentX = Math.max(0, Math.floor((canvasWidth - contentWidth) / 2));
const contentY = Math.max(0, Math.floor((canvasHeight - contentHeight) / 2));
const gap = Math.max(
preset.transform.minGap,
Math.round(contentWidth * preset.transform.gapFraction),
);
const totalUnits = preset.transform.screenUnits + preset.transform.webcamUnits;
const availableWidth = Math.max(1, contentWidth - gap);
const screenSlotWidth = Math.max(
1,
Math.round((availableWidth * preset.transform.screenUnits) / totalUnits),
);
const webcamSlotWidth = Math.max(1, availableWidth - screenSlotWidth);
const screenSlot = {
x: contentX,
y: contentY,
width: screenSlotWidth,
height: contentHeight,
};
const webcamSlot = {
x: contentX + screenSlotWidth + gap,
y: contentY,
width: webcamSlotWidth,
height: contentHeight,
};
const webcamBorderRadius = Math.min(
preset.borderRadius.max,
Math.max(
preset.borderRadius.min,
Math.round(Math.min(webcamSlot.width, webcamSlot.height) * preset.borderRadius.fraction),
),
);
return {
screenRect: screenSlot,
screenBorderRadius: webcamBorderRadius,
webcamRect: {
x: webcamSlot.x,
y: webcamSlot.y,
width: webcamSlot.width,
height: webcamSlot.height,
borderRadius: webcamBorderRadius,
maskShape: "rectangle",
},
screenCover: true,
};
}
const transform = preset.transform;
const screenRect = centerRect({
canvasSize,
@@ -198,8 +296,11 @@ export function computeCompositeLayout(params: {
transform.minMargin,
Math.round(Math.min(canvasWidth, canvasHeight) * transform.marginFraction),
);
const maxWidth = Math.max(transform.minSize, canvasWidth * transform.maxStageFraction);
const maxHeight = Math.max(transform.minSize, canvasHeight * transform.maxStageFraction);
// Use geometric mean so the webcam occupies a consistent visual proportion
// regardless of whether the canvas is portrait or landscape.
const referenceDim = Math.sqrt(canvasWidth * canvasHeight);
const maxWidth = Math.max(transform.minSize, referenceDim * MAX_STAGE_FRACTION);
const maxHeight = Math.max(transform.minSize, referenceDim * MAX_STAGE_FRACTION);
const scale = Math.min(maxWidth / webcamWidth, maxHeight / webcamHeight);
let width = Math.round(webcamWidth * scale);
let height = Math.round(webcamHeight * scale);
@@ -258,7 +359,16 @@ export function computeCompositeLayout(params: {
function centerRect(params: { canvasSize: Size; size: Size; maxSize: Size }): RenderRect {
const { canvasSize, size, maxSize } = params;
const { width: canvasWidth, height: canvasHeight } = canvasSize;
return centerRectInBounds({
bounds: { x: 0, y: 0, width: canvasSize.width, height: canvasSize.height },
size,
maxSize,
});
}
function centerRectInBounds(params: { bounds: RenderRect; size: Size; maxSize: Size }): RenderRect {
const { bounds, size, maxSize } = params;
const { x: boundsX, y: boundsY, width: boundsWidth, height: boundsHeight } = bounds;
const { width, height } = size;
const { width: maxWidth, height: maxHeight } = maxSize;
const scale = Math.min(maxWidth / width, maxHeight / height, 1);
@@ -266,8 +376,8 @@ function centerRect(params: { canvasSize: Size; size: Size; maxSize: Size }): Re
const resolvedHeight = Math.round(height * scale);
return {
x: Math.max(0, Math.floor((canvasWidth - resolvedWidth) / 2)),
y: Math.max(0, Math.floor((canvasHeight - resolvedHeight) / 2)),
x: boundsX + Math.max(0, Math.floor((boundsWidth - resolvedWidth) / 2)),
y: boundsY + Math.max(0, Math.floor((boundsHeight - resolvedHeight) / 2)),
width: resolvedWidth,
height: resolvedHeight,
};
+148 -6
View File
@@ -1,4 +1,47 @@
import type { AnnotationRegion, ArrowDirection } from "@/components/video-editor/types";
import { type AnnotationRegion, type ArrowDirection } from "@/components/video-editor/types";
import {
applyMosaicToImageData,
getBlurOverlayColor,
getNormalizedBlurIntensity,
getNormalizedMosaicBlockSize,
normalizeBlurType,
} from "@/lib/blurEffects";
let blurScratchCanvas: HTMLCanvasElement | null = null;
let blurScratchCtx: CanvasRenderingContext2D | null = null;
// Matches a single code point whose script is Han (including non-BMP
// Extension A-F), Hiragana, Katakana (including halfwidth forms), or
// Hangul. Used to split CJK text at character boundaries during wrap,
// since CJK scripts have no word-separating whitespace. Unicode script
// property escapes require ES2018+; tsconfig target is ES2020.
const CJK_CHAR = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
function tokenizeForWrap(line: string): string[] {
// Split Latin text on whitespace (preserving the whitespace as its own token,
// matching the original behavior), and split CJK runs into individual
// characters so each one becomes a breakable unit. This mirrors the editor's
// CSS `word-break: break-word` handling for CJK content.
const tokens: string[] = [];
let buffer = "";
const chars = Array.from(line);
const flushBuffer = () => {
if (buffer) {
tokens.push(...buffer.split(/(\s+)/).filter((s) => s.length > 0));
buffer = "";
}
};
for (const ch of chars) {
if (CJK_CHAR.test(ch)) {
flushBuffer();
tokens.push(ch);
} else {
buffer += ch;
}
}
flushBuffer();
return tokens;
}
// SVG path data for each arrow direction
const ARROW_PATHS: Record<ArrowDirection, string[]> = {
@@ -96,6 +139,101 @@ function renderArrow(
ctx.restore();
}
function drawBlurPath(
ctx: CanvasRenderingContext2D,
annotation: AnnotationRegion,
x: number,
y: number,
width: number,
height: number,
) {
const shape = annotation.blurData?.shape || "rectangle";
if (shape === "rectangle") {
ctx.beginPath();
ctx.rect(x, y, width, height);
return;
}
if (shape === "oval") {
ctx.beginPath();
ctx.ellipse(x + width / 2, y + height / 2, width / 2, height / 2, 0, 0, Math.PI * 2);
return;
}
const points = annotation.blurData?.freehandPoints;
if (shape === "freehand" && points && points.length >= 3) {
ctx.beginPath();
ctx.moveTo(x + (points[0].x / 100) * width, y + (points[0].y / 100) * height);
for (let i = 1; i < points.length; i++) {
ctx.lineTo(x + (points[i].x / 100) * width, y + (points[i].y / 100) * height);
}
ctx.closePath();
return;
}
ctx.beginPath();
ctx.rect(x, y, width, height);
}
function renderBlur(
ctx: CanvasRenderingContext2D,
annotation: AnnotationRegion,
x: number,
y: number,
width: number,
height: number,
scaleFactor: number,
) {
const canvas = ctx.canvas;
const blurType = normalizeBlurType(annotation.blurData?.type);
const blurRadius = Math.max(
1,
Math.round(getNormalizedBlurIntensity(annotation.blurData) * scaleFactor),
);
const samplePadding =
blurType === "mosaic"
? Math.max(0, Math.ceil(getNormalizedMosaicBlockSize(annotation.blurData, scaleFactor)))
: Math.max(2, Math.ceil(blurRadius * 2));
const sx = Math.max(0, Math.floor(x) - samplePadding);
const sy = Math.max(0, Math.floor(y) - samplePadding);
const ex = Math.min(canvas.width, Math.ceil(x + width) + samplePadding);
const ey = Math.min(canvas.height, Math.ceil(y + height) + samplePadding);
const sw = Math.max(0, ex - sx);
const sh = Math.max(0, ey - sy);
if (sw <= 0 || sh <= 0) return;
if (!blurScratchCanvas || !blurScratchCtx) {
blurScratchCanvas = document.createElement("canvas");
blurScratchCtx = blurScratchCanvas.getContext("2d");
}
if (!blurScratchCanvas || !blurScratchCtx) return;
blurScratchCanvas.width = sw;
blurScratchCanvas.height = sh;
blurScratchCtx.clearRect(0, 0, sw, sh);
blurScratchCtx.drawImage(canvas, sx, sy, sw, sh, 0, 0, sw, sh);
if (blurType === "mosaic") {
const imageData = blurScratchCtx.getImageData(0, 0, sw, sh);
applyMosaicToImageData(
imageData,
getNormalizedMosaicBlockSize(annotation.blurData, scaleFactor),
);
blurScratchCtx.putImageData(imageData, 0, 0);
}
ctx.save();
drawBlurPath(ctx, annotation, x, y, width, height);
ctx.clip();
ctx.filter = blurType === "mosaic" ? "none" : `blur(${blurRadius}px)`;
ctx.drawImage(blurScratchCanvas, sx, sy);
ctx.filter = "none";
ctx.fillStyle = getBlurOverlayColor(annotation.blurData);
ctx.fillRect(sx, sy, sw, sh);
ctx.restore();
}
function renderText(
ctx: CanvasRenderingContext2D,
annotation: AnnotationRegion,
@@ -144,13 +282,13 @@ function renderText(
lines.push("");
continue;
}
const words = rawLine.split(/(\s+)/);
const tokens = tokenizeForWrap(rawLine);
let current = "";
for (const word of words) {
const test = current + word;
for (const token of tokens) {
const test = current + token;
if (current && ctx.measureText(test).width > availableWidth) {
lines.push(current);
current = word.trimStart();
current = token.trimStart();
} else {
current = test;
}
@@ -268,7 +406,7 @@ export async function renderAnnotations(
): Promise<void> {
// Filter active annotations at current time
const activeAnnotations = annotations.filter(
(ann) => currentTimeMs >= ann.startMs && currentTimeMs <= ann.endMs,
(ann) => currentTimeMs >= ann.startMs && currentTimeMs < ann.endMs,
);
// Sort by z-index (lower first, so higher z-index draws on top)
@@ -304,6 +442,10 @@ export async function renderAnnotations(
);
}
break;
case "blur":
renderBlur(ctx, annotation, x, y, width, height, scaleFactor);
break;
}
}
}
+108 -22
View File
@@ -5,6 +5,7 @@ import type { VideoMuxer } from "./muxer";
const AUDIO_BITRATE = 128_000;
const DECODE_BACKPRESSURE_LIMIT = 20;
const MIN_SPEED_REGION_DELTA_MS = 0.0001;
const SEEK_TIMEOUT_MS = 5_000;
export class AudioProcessor {
private cancelled = false;
@@ -18,9 +19,9 @@ export class AudioProcessor {
demuxer: WebDemuxer,
muxer: VideoMuxer,
videoUrl: string,
trimRegions?: TrimRegion[],
speedRegions?: SpeedRegion[],
readEndSec?: number,
trimRegions: TrimRegion[] | undefined,
speedRegions: SpeedRegion[] | undefined,
validatedDurationSec: number,
): Promise<void> {
const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) : [];
const sortedSpeedRegions = speedRegions
@@ -35,14 +36,19 @@ export class AudioProcessor {
videoUrl,
sortedTrims,
sortedSpeedRegions,
validatedDurationSec,
);
if (!this.cancelled) {
if (!this.cancelled && renderedAudioBlob.size > 0) {
await this.muxRenderedAudioBlob(renderedAudioBlob, muxer);
return;
}
return;
}
// No speed edits: keep the original demux/decode/encode path with trim timestamp remap.
// The +0.5s buffer mirrors streamingDecoder.decodeAll's read window so the trim-only
// and speed-aware paths agree on how far to read past the validated duration boundary.
const readEndSec = validatedDurationSec + 0.5;
await this.processTrimOnlyAudio(demuxer, muxer, sortedTrims, readEndSec);
}
@@ -55,7 +61,7 @@ export class AudioProcessor {
): Promise<void> {
let audioConfig: AudioDecoderConfig;
try {
audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig;
audioConfig = await demuxer.getDecoderConfig("audio");
} catch {
console.warn("[AudioProcessor] No audio track found, skipping");
return;
@@ -80,11 +86,10 @@ export class AudioProcessor {
typeof readEndSec === "number" && Number.isFinite(readEndSec)
? Math.max(0, readEndSec)
: undefined;
const audioStream = (
const audioStream =
safeReadEndSec !== undefined
? demuxer.read("audio", 0, safeReadEndSec)
: demuxer.read("audio")
) as ReadableStream<EncodedAudioChunk>;
: demuxer.read("audio");
const reader = audioStream.getReader();
try {
@@ -187,6 +192,7 @@ export class AudioProcessor {
videoUrl: string,
trimRegions: TrimRegion[],
speedRegions: SpeedRegion[],
validatedDurationSec: number,
): Promise<Blob> {
const media = document.createElement("audio");
media.src = videoUrl;
@@ -211,15 +217,44 @@ export class AudioProcessor {
const destinationNode = audioContext.createMediaStreamDestination();
sourceNode.connect(destinationNode);
const { recorder, recordedBlobPromise } = this.startAudioRecording(destinationNode.stream);
let rafId: number | null = null;
let recorder: MediaRecorder | null = null;
let recordedBlobPromise: Promise<Blob> | null = null;
try {
if (audioContext.state === "suspended") {
await audioContext.resume();
}
await this.seekTo(media, 0);
// Skip past any initial trim region(s) before recording starts to avoid
// capturing trimmed audio during the first rAF frames of playback.
// Loops to handle back-to-back or overlapping trims at t=0.
const effectiveEnd = validatedDurationSec;
let startPosition = 0;
for (let i = 0; i <= trimRegions.length; i++) {
const activeTrim = this.findActiveTrimRegion(startPosition * 1000, trimRegions);
if (!activeTrim) break;
startPosition = activeTrim.endMs / 1000;
if (startPosition >= effectiveEnd) break;
}
if (startPosition >= effectiveEnd) {
// All content is trimmed — return silent blob
return new Blob([], { type: "audio/webm" });
}
await this.seekTo(media, startPosition);
// Set initial playback rate for the starting position
const initialSpeedRegion = this.findActiveSpeedRegion(startPosition * 1000, speedRegions);
if (initialSpeedRegion) {
media.playbackRate = initialSpeedRegion.speed;
}
// Start recording only AFTER seeking past trims
const recording = this.startAudioRecording(destinationNode.stream);
recorder = recording.recorder;
recordedBlobPromise = recording.recordedBlobPromise;
await media.play();
await new Promise<void>((resolve, reject) => {
@@ -249,24 +284,66 @@ export class AudioProcessor {
return;
}
// Stop playback at validated duration — browser's media.duration
// may be inflated from bad container metadata.
if (media.currentTime >= validatedDurationSec) {
media.pause();
cleanup();
resolve();
return;
}
const currentTimeMs = media.currentTime * 1000;
const activeTrimRegion = this.findActiveTrimRegion(currentTimeMs, trimRegions);
if (activeTrimRegion && !media.paused && !media.ended) {
const skipToTime = activeTrimRegion.endMs / 1000;
if (skipToTime >= media.duration) {
if (skipToTime >= media.duration || skipToTime >= validatedDurationSec) {
media.pause();
cleanup();
resolve();
return;
}
// Pause recording during trim seek to prevent capturing
// silence/noise as the audio element seeks.
media.pause();
if (recorder?.state === "recording") recorder.pause();
const onSeeked = () => {
clearTimeout(seekTimer);
if (this.cancelled) {
cleanup();
resolve();
return;
}
if (recorder?.state === "paused") recorder.resume();
media
.play()
.then(() => {
if (!this.cancelled) rafId = requestAnimationFrame(tick);
})
.catch((err) => {
cleanup();
reject(
new Error(
`Failed to resume playback after trim seek: ${err instanceof Error ? err.message : String(err)}`,
),
);
});
};
const seekTimer = window.setTimeout(() => {
media.removeEventListener("seeked", onSeeked);
cleanup();
reject(new Error("Audio seek timed out while skipping trim region"));
}, SEEK_TIMEOUT_MS);
media.addEventListener("seeked", onSeeked, { once: true });
media.currentTime = skipToTime;
} else {
const activeSpeedRegion = this.findActiveSpeedRegion(currentTimeMs, speedRegions);
const playbackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1;
if (Math.abs(media.playbackRate - playbackRate) > 0.0001) {
media.playbackRate = playbackRate;
}
return;
}
const activeSpeedRegion = this.findActiveSpeedRegion(currentTimeMs, speedRegions);
const playbackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1;
if (Math.abs(media.playbackRate - playbackRate) > 0.0001) {
media.playbackRate = playbackRate;
}
if (!media.paused && !media.ended) {
@@ -286,7 +363,7 @@ export class AudioProcessor {
cancelAnimationFrame(rafId);
}
media.pause();
if (recorder.state !== "inactive") {
if (recorder && recorder.state !== "inactive") {
recorder.stop();
}
destinationNode.stream.getTracks().forEach((track) => track.stop());
@@ -297,6 +374,12 @@ export class AudioProcessor {
media.load();
}
if (!recordedBlobPromise) {
// Invariant: either an early return above fires, or startAudioRecording ran and
// populated recordedBlobPromise before the playback Promise resolved. Reaching
// here means that contract was broken — fail loud instead of returning silence.
throw new Error("Audio recorder finished without assigning recordedBlobPromise");
}
const recordedBlob = await recordedBlobPromise;
if (this.cancelled) {
throw new Error("Export cancelled");
@@ -314,8 +397,8 @@ export class AudioProcessor {
try {
await demuxer.load(file);
const audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig;
const reader = (demuxer.read("audio") as ReadableStream<EncodedAudioChunk>).getReader();
const audioConfig = await demuxer.getDecoderConfig("audio");
const reader = demuxer.read("audio").getReader();
let isFirstChunk = true;
try {
@@ -459,7 +542,10 @@ export class AudioProcessor {
}
private cloneWithTimestamp(src: AudioData, newTimestamp: number): AudioData {
const isPlanar = src.format?.includes("planar") ?? false;
if (!src.format) {
throw new Error("AudioData format is required for cloning");
}
const isPlanar = src.format.includes("planar");
const numPlanes = isPlanar ? src.numberOfChannels : 1;
let totalSize = 0;
@@ -476,7 +562,7 @@ export class AudioProcessor {
}
return new AudioData({
format: src.format!,
format: src.format,
sampleRate: src.sampleRate,
numberOfFrames: src.numberOfFrames,
numberOfChannels: src.numberOfChannels,
+88 -10
View File
@@ -13,6 +13,7 @@ import type {
CropRegion,
SpeedRegion,
WebcamLayoutPreset,
WebcamSizePreset,
ZoomDepth,
ZoomRegion,
} from "@/components/video-editor/types";
@@ -70,12 +71,14 @@ interface FrameRenderConfig {
webcamSize?: Size | null;
webcamLayoutPreset?: WebcamLayoutPreset;
webcamMaskShape?: import("@/components/video-editor/types").WebcamMaskShape;
webcamSizePreset?: WebcamSizePreset;
webcamPosition?: { cx: number; cy: number } | null;
annotationRegions?: AnnotationRegion[];
speedRegions?: SpeedRegion[];
previewWidth?: number;
previewHeight?: number;
cursorTelemetry?: import("@/components/video-editor/types").CursorTelemetryPoint[];
platform: string;
}
interface AnimationState {
@@ -112,6 +115,8 @@ export class FrameRenderer {
private shadowCtx: CanvasRenderingContext2D | null = null;
private compositeCanvas: HTMLCanvasElement | null = null;
private compositeCtx: CanvasRenderingContext2D | null = null;
private rasterCanvas: HTMLCanvasElement | null = null;
private rasterCtx: CanvasRenderingContext2D | null = null;
private config: FrameRenderConfig;
private animationState: AnimationState;
private layoutCache: LayoutCache | null = null;
@@ -120,9 +125,11 @@ export class FrameRenderer {
private smoothedAutoFocus: { cx: number; cy: number } | null = null;
private prevAnimationTimeMs: number | null = null;
private prevTargetProgress = 0;
private isLinux = false;
constructor(config: FrameRenderConfig) {
this.config = config;
this.isLinux = config.platform === "linux";
this.animationState = {
scale: 1,
focusX: DEFAULT_FOCUS.cx,
@@ -183,14 +190,24 @@ export class FrameRenderer {
this.compositeCanvas = document.createElement("canvas");
this.compositeCanvas.width = this.config.width;
this.compositeCanvas.height = this.config.height;
// On Linux, getImageData() is called frequently causing frequent CPU readback
this.compositeCtx = this.compositeCanvas.getContext("2d", {
willReadFrequently: false,
willReadFrequently: this.isLinux,
});
if (!this.compositeCtx) {
throw new Error("Failed to get 2D context for composite canvas");
}
this.rasterCanvas = document.createElement("canvas");
this.rasterCanvas.width = this.config.width;
this.rasterCanvas.height = this.config.height;
this.rasterCtx = this.rasterCanvas.getContext("2d");
if (!this.rasterCtx) {
throw new Error("Failed to get 2D context for raster canvas");
}
// Setup shadow canvas if needed
if (this.config.showShadow) {
this.shadowCanvas = document.createElement("canvas");
@@ -453,6 +470,7 @@ export class FrameRenderer {
screenSize: { width: croppedVideoWidth, height: croppedVideoHeight },
webcamSize: webcamFrame ? this.config.webcamSize : null,
layoutPreset: this.config.webcamLayoutPreset,
webcamSizePreset: this.config.webcamSizePreset,
webcamPosition: this.config.webcamPosition,
webcamMaskShape: this.config.webcamMaskShape,
});
@@ -494,7 +512,12 @@ export class FrameRenderer {
const previewWidth = this.config.previewWidth ?? this.config.width;
const previewHeight = this.config.previewHeight ?? this.config.height;
const canvasScaleFactor = Math.min(width / previewWidth, height / previewHeight);
const scaledBorderRadius = compositeLayout.screenCover ? 0 : borderRadius * canvasScaleFactor;
const scaledBorderRadius =
compositeLayout.screenBorderRadius != null
? compositeLayout.screenBorderRadius
: compositeLayout.screenCover
? 0
: borderRadius * canvasScaleFactor;
this.maskGraphics.clear();
this.maskGraphics.roundRect(0, 0, screenRect.width, screenRect.height, scaledBorderRadius);
@@ -522,16 +545,10 @@ export class FrameRenderer {
private updateAnimationState(timeMs: number): number {
if (!this.cameraContainer || !this.layoutCache) return 0;
const bmEx = this.layoutCache.maskRect;
const ssEx = this.layoutCache.stageSize;
const viewportRatio =
bmEx.width > 0 && bmEx.height > 0
? { widthRatio: ssEx.width / bmEx.width, heightRatio: ssEx.height / bmEx.height }
: undefined;
const { region, strength, blendedScale, transition } = findDominantRegion(
this.config.zoomRegions,
timeMs,
{ connectZooms: true, cursorTelemetry: this.config.cursorTelemetry, viewportRatio },
{ connectZooms: true, cursorTelemetry: this.config.cursorTelemetry },
);
const defaultFocus = DEFAULT_FOCUS;
@@ -675,10 +692,49 @@ export class FrameRenderer {
);
}
// On Linux/Wayland the implicit GPU→2D texture-sharing path
// used by drawImage(webglCanvas) can fail silently (EGL/Ozone),
// producing green/empty frames. Explicit gl.readPixels always
// copies from GPU to CPU memory, bypassing that path.
private readbackVideoCanvas(): HTMLCanvasElement {
const glCanvas = this.app!.canvas as HTMLCanvasElement;
const gl =
(glCanvas.getContext("webgl2") as WebGL2RenderingContext | null) ??
(glCanvas.getContext("webgl") as WebGLRenderingContext | null);
if (!gl || !this.rasterCanvas || !this.rasterCtx) {
return glCanvas;
}
const w = glCanvas.width;
const h = glCanvas.height;
const buf = new Uint8Array(w * h * 4);
gl.readPixels(0, 0, w, h, gl.RGBA, gl.UNSIGNED_BYTE, buf);
// readPixels returns rows bottom-to-top; flip vertically
const rowSize = w * 4;
const temp = new Uint8Array(rowSize);
for (let top = 0, bot = h - 1; top < bot; top++, bot--) {
const tOff = top * rowSize;
const bOff = bot * rowSize;
temp.set(buf.subarray(tOff, tOff + rowSize));
buf.copyWithin(tOff, bOff, bOff + rowSize);
buf.set(temp, bOff);
}
const imageData = new ImageData(new Uint8ClampedArray(buf.buffer), w, h);
this.rasterCtx.putImageData(imageData, 0, 0);
return this.rasterCanvas;
}
private compositeWithShadows(webcamFrame?: VideoFrame | null): void {
if (!this.compositeCanvas || !this.compositeCtx || !this.app) return;
const videoCanvas = this.app.canvas as HTMLCanvasElement;
const videoCanvas = this.isLinux
? this.readbackVideoCanvas()
: (this.app.canvas as HTMLCanvasElement);
const ctx = this.compositeCtx;
const w = this.compositeCanvas.width;
const h = this.compositeCanvas.height;
@@ -735,6 +791,22 @@ export class FrameRenderer {
if (webcamFrame && webcamRect) {
const preset = getWebcamLayoutPresetDefinition(this.config.webcamLayoutPreset);
const shape = webcamRect.maskShape ?? this.config.webcamMaskShape ?? "rectangle";
const sourceWidth =
("displayWidth" in webcamFrame && webcamFrame.displayWidth > 0
? webcamFrame.displayWidth
: webcamFrame.codedWidth) || webcamRect.width;
const sourceHeight =
("displayHeight" in webcamFrame && webcamFrame.displayHeight > 0
? webcamFrame.displayHeight
: webcamFrame.codedHeight) || webcamRect.height;
const sourceAspect = sourceWidth / sourceHeight;
const targetAspect = webcamRect.width / webcamRect.height;
const sourceCropWidth =
sourceAspect > targetAspect ? Math.round(sourceHeight * targetAspect) : sourceWidth;
const sourceCropHeight =
sourceAspect > targetAspect ? sourceHeight : Math.round(sourceWidth / targetAspect);
const sourceCropX = Math.max(0, Math.round((sourceWidth - sourceCropWidth) / 2));
const sourceCropY = Math.max(0, Math.round((sourceHeight - sourceCropHeight) / 2));
ctx.save();
drawCanvasClipPath(
ctx,
@@ -756,6 +828,10 @@ export class FrameRenderer {
ctx.clip();
ctx.drawImage(
webcamFrame as unknown as CanvasImageSource,
sourceCropX,
sourceCropY,
sourceCropWidth,
sourceCropHeight,
webcamRect.x,
webcamRect.y,
webcamRect.width,
@@ -795,5 +871,7 @@ export class FrameRenderer {
this.shadowCtx = null;
this.compositeCanvas = null;
this.compositeCtx = null;
this.rasterCanvas = null;
this.rasterCtx = null;
}
}
@@ -0,0 +1,43 @@
import { describe, expect, it } from "vitest";
import sampleVideoUrl from "../../../tests/fixtures/sample.webm?url";
import { GifExporter } from "./gifExporter";
import type { ExportProgress } from "./types";
describe("GifExporter (real browser)", () => {
it("exports a valid GIF blob from a real video", async () => {
const progressEvents: ExportProgress[] = [];
const exporter = new GifExporter({
videoUrl: sampleVideoUrl,
width: 320,
height: 180,
frameRate: 15,
loop: true,
sizePreset: "medium",
wallpaper: "#1a1a2e",
zoomRegions: [],
showShadow: false,
shadowIntensity: 0,
showBlur: false,
cropRegion: { x: 0, y: 0, width: 1, height: 1 },
onProgress: (p) => progressEvents.push(p),
});
const result = await exporter.export();
expect(result.success, result.error).toBe(true);
expect(result.blob).toBeInstanceOf(Blob);
const buf = await result.blob!.arrayBuffer();
const header = new TextDecoder().decode(new Uint8Array(buf, 0, 6));
expect(header).toMatch(/^GIF8[79]a/);
expect(result.blob!.size).toBeGreaterThan(1024);
expect(progressEvents.length).toBeGreaterThan(0);
const finalizing = progressEvents.filter((p) => p.phase === "finalizing");
expect(finalizing.length).toBeGreaterThan(0);
expect(finalizing.at(-1)!.percentage).toBe(100);
});
});
+10 -2
View File
@@ -5,8 +5,10 @@ import type {
SpeedRegion,
TrimRegion,
WebcamLayoutPreset,
WebcamSizePreset,
ZoomRegion,
} from "@/components/video-editor/types";
import { getPlatform } from "@/utils/platformUtils";
import { AsyncVideoFrameQueue } from "./asyncVideoFrameQueue";
import { FrameRenderer } from "./frameRenderer";
import { StreamingVideoDecoder } from "./streamingDecoder";
@@ -42,6 +44,7 @@ interface GifExporterConfig {
cropRegion: CropRegion;
webcamLayoutPreset?: WebcamLayoutPreset;
webcamMaskShape?: import("@/components/video-editor/types").WebcamMaskShape;
webcamSizePreset?: WebcamSizePreset;
webcamPosition?: { cx: number; cy: number } | null;
annotationRegions?: AnnotationRegion[];
previewWidth?: number;
@@ -113,7 +116,10 @@ export class GifExporter {
async export(): Promise<ExportResult> {
let webcamFrameQueue: AsyncVideoFrameQueue | null = null;
try {
const platform = await getPlatform();
this.cleanup();
this.cancelled = false;
@@ -144,12 +150,14 @@ export class GifExporter {
webcamSize: webcamInfo ? { width: webcamInfo.width, height: webcamInfo.height } : null,
webcamLayoutPreset: this.config.webcamLayoutPreset,
webcamMaskShape: this.config.webcamMaskShape,
webcamSizePreset: this.config.webcamSizePreset,
webcamPosition: this.config.webcamPosition,
annotationRegions: this.config.annotationRegions,
speedRegions: this.config.speedRegions,
previewWidth: this.config.previewWidth,
previewHeight: this.config.previewHeight,
cursorTelemetry: this.config.cursorTelemetry,
platform,
});
await this.renderer.initialize();
@@ -171,11 +179,11 @@ export class GifExporter {
});
// Calculate effective duration and frame count (excluding trim regions)
const effectiveDuration = this.streamingDecoder.getEffectiveDuration(
const { effectiveDuration, totalFrames } = this.streamingDecoder.getExportMetrics(
this.config.frameRate,
this.config.trimRegions,
this.config.speedRegions,
);
const totalFrames = Math.ceil(effectiveDuration * this.config.frameRate);
// Calculate frame delay in milliseconds (gif.js uses ms)
const frameDelay = Math.round(1000 / this.config.frameRate);
+40 -1
View File
@@ -1,5 +1,44 @@
import { describe, expect, it } from "vitest";
import { shouldFailDecodeEndedEarly } from "./streamingDecoder";
import { shouldFailDecodeEndedEarly, validateDuration } from "./streamingDecoder";
describe("validateDuration", () => {
it("returns scanned duration when container reports Infinity", () => {
expect(validateDuration(Infinity, 15.3)).toBe(15.3);
});
it("returns scanned duration when container reports 0", () => {
expect(validateDuration(0, 15.3)).toBe(15.3);
});
it("returns scanned duration when container reports NaN", () => {
expect(validateDuration(NaN, 15.3)).toBe(15.3);
});
it("returns scanned duration when container is inflated beyond threshold", () => {
expect(validateDuration(42, 15.3)).toBe(15.3);
});
it("returns container duration when values are close", () => {
expect(validateDuration(15.5, 15.3)).toBe(15.5);
});
it("returns container duration when scanned is slightly higher", () => {
// container < scanned (scanned overshoot from last frame duration)
expect(validateDuration(15.0, 15.3)).toBe(15.0);
});
it("returns scanned duration when container under-reports beyond threshold", () => {
expect(validateDuration(10, 15.3)).toBe(15.3);
});
it("returns container duration when scanned is zero (corrupted/empty file)", () => {
expect(validateDuration(10, 0)).toBe(10);
});
it("returns 0 when both container is NaN and scanned is zero", () => {
expect(validateDuration(NaN, 0)).toBe(0);
});
});
describe("shouldFailDecodeEndedEarly", () => {
it("does not fail once every segment has been satisfied", () => {
+178 -20
View File
@@ -2,6 +2,52 @@ import { WebDemuxer } from "web-demuxer";
import type { SpeedRegion, TrimRegion } from "@/components/video-editor/types";
const SOURCE_LOAD_TIMEOUT_MS = 60_000;
const EPSILON_SEC = 0.001;
/**
* Build a full WebCodecs-compatible AV1 codec string from the AV1CodecConfigurationRecord.
* web-demuxer may return a bare "av01" when the WASM-side parser fails to read
* the extradata (e.g. raw OBU sequence header from WebM instead of ISOBMFF av1C box).
* This function parses the record if present, otherwise returns a safe default.
*
* @see https://aomediacodec.github.io/av1-isobmff/#av1codecconfigurationbox-section
*/
function buildAV1CodecString(description?: BufferSource): string {
const fallback = "av01.0.01M.08";
if (!description) return fallback;
const bytes =
description instanceof ArrayBuffer
? new Uint8Array(description)
: new Uint8Array(description.buffer, description.byteOffset, description.byteLength);
// AV1CodecConfigurationRecord layout (4+ bytes):
// Byte 0: marker (1) | version (7)
// Byte 1: seq_profile (3) | seq_level_idx_0 (5)
// Byte 2: seq_tier_0 (1) | high_bitdepth (1) | twelve_bit (1) | ...
// The spec says version should be 1, but Chrome/Electron's MediaRecorder
// may write version 127 (0xFF first byte). We accept any version as long
// as the marker bit is set and the record is long enough.
if (bytes.length < 4) return fallback;
if (!(bytes[0] & 0x80)) return fallback; // marker bit must be 1
// Byte 1: seq_profile (3) | seq_level_idx_0 (5)
const profile = (bytes[1] >> 5) & 0x07;
const level = bytes[1] & 0x1f;
// Byte 2: seq_tier_0 (1) | high_bitdepth (1) | twelve_bit (1) | monochrome (1) | ...
const tier = (bytes[2] >> 7) & 0x01;
const highBitdepth = (bytes[2] >> 6) & 0x01;
const twelveBit = (bytes[2] >> 5) & 0x01;
let bitdepth = 8;
if (highBitdepth) bitdepth = twelveBit ? 12 : 10;
const tierChar = tier ? "H" : "M";
const levelStr = level.toString().padStart(2, "0");
const bitdepthStr = bitdepth.toString().padStart(2, "0");
return `av01.${profile}.${levelStr}${tierChar}.${bitdepthStr}`;
}
export interface DecodedVideoInfo {
width: number;
@@ -24,6 +70,37 @@ type EarlyDecodeEndCheck = {
const EARLY_DECODE_END_THRESHOLD_SEC = 1;
const METADATA_TAIL_TOLERANCE_SEC = 1.5;
const STREAM_DURATION_MATCH_TOLERANCE_SEC = 0.25;
const DURATION_DIVERGENCE_THRESHOLD_SEC = 1.5;
// Fallback upper bound for the packet scan when no reliable duration hint is
// available. Explicit end is required (some containers are truncated without
// one), but the hint-derived bound would cap the scan prematurely when
// container/stream duration are missing or corrupt.
const SCAN_UNBOUNDED_FALLBACK_SEC = 24 * 60 * 60;
/**
* Validate container duration against actual packet timestamps.
*
* Chrome/Electron's MediaRecorder writes WebM containers with unreliable
* Duration fields (often Infinity, 0, or inflated) — especially on Linux.
* This function picks the most trustworthy duration value.
*
* @param containerDuration Duration from the container-level metadata
* @param scannedDuration Duration derived from actual packet timestamps (ground truth)
*/
export function validateDuration(containerDuration: number, scannedDuration: number): number {
if (scannedDuration <= 0) {
// Zero scanned duration means corrupted/empty file — fall back to container
// (downstream shouldFailDecodeEndedEarly will catch truly empty files)
return Number.isFinite(containerDuration) ? Math.max(containerDuration, 0) : 0;
}
if (!Number.isFinite(containerDuration) || containerDuration <= 0) {
return scannedDuration;
}
if (Math.abs(containerDuration - scannedDuration) > DURATION_DIVERGENCE_THRESHOLD_SEC) {
return scannedDuration;
}
return containerDuration;
}
export function shouldFailDecodeEndedEarly({
cancelled,
@@ -155,10 +232,43 @@ export class StreamingVideoDecoder {
const audioStream = mediaInfo.streams.find((s) => s.codec_type_string === "audio");
// Scan video packets to find the true content boundary.
// MediaRecorder (especially on Linux) writes unreliable container durations.
// Packet timestamps are ground truth — no decode needed, just timestamp reads.
// Pass explicit range because some containers are truncated without one.
// Sanitize because mediaInfo.duration can be NaN/Infinity (Chromium Linux bug),
// which would propagate into demuxer.read() as an invalid endpoint.
const containerDurationSec = Number.isFinite(mediaInfo.duration) ? mediaInfo.duration : 0;
const streamDurationSec =
typeof videoStream?.duration === "number" && Number.isFinite(videoStream.duration)
? videoStream.duration
: 0;
const hintedDurationSec = Math.max(containerDurationSec, streamDurationSec, 0);
const scanEndSec =
hintedDurationSec > 0 ? hintedDurationSec + 0.5 : SCAN_UNBOUNDED_FALLBACK_SEC;
let maxPacketEndUs = 0;
const scanReader = this.demuxer.read("video", 0, scanEndSec).getReader();
try {
while (true) {
const { done, value } = await scanReader.read();
if (done || !value) break;
const endUs = value.timestamp + (value.duration ?? 0);
if (endUs > maxPacketEndUs) maxPacketEndUs = endUs;
}
} finally {
try {
await scanReader.cancel();
} catch {
/* already closed */
}
}
const scannedDuration = maxPacketEndUs / 1_000_000;
const validatedDuration = validateDuration(mediaInfo.duration, scannedDuration);
this.metadata = {
width: videoStream?.width || 1920,
height: videoStream?.height || 1080,
duration: mediaInfo.duration,
duration: validatedDuration,
streamDuration:
typeof videoStream?.duration === "number" && Number.isFinite(videoStream.duration)
? videoStream.duration
@@ -171,7 +281,15 @@ export class StreamingVideoDecoder {
return this.metadata;
}
/**
* Decodes all video frames from the loaded source and invokes a callback for each.
* Handles trimming and speed adjustments, and resamples to the target frame rate.
* On Windows, early decode termination is tolerated to work around driver quirks.
* @param targetFrameRate - Desired output frame rate.
* @param trimRegions - Array of time regions to keep (others discarded).
* @param speedRegions - Array of speed adjustments for specific time ranges.
* @param onFrame - Async callback receiving each decoded VideoFrame.
*/
async decodeAll(
targetFrameRate: number,
trimRegions: TrimRegion[] | undefined,
@@ -183,17 +301,30 @@ export class StreamingVideoDecoder {
}
const decoderConfig = await this.demuxer.getDecoderConfig("video");
const codec = this.metadata.codec.toLowerCase();
// web-demuxer may return a bare "av01" for AV1 in WebM containers when the
// extradata isn't in the expected ISOBMFF format. WebCodecs requires the
// full parametrized form (e.g. "av01.0.05M.08").
if (/^av01$/i.test(decoderConfig.codec)) {
decoderConfig.codec = buildAV1CodecString(
decoderConfig.description as BufferSource | undefined,
);
}
const codec = decoderConfig.codec.toLowerCase();
const shouldPreferSoftwareDecode = codec.includes("av01") || codec.includes("av1");
const segments = this.splitBySpeed(
this.computeSegments(this.metadata.duration, trimRegions),
speedRegions,
);
const requiredEndSec = segments[segments.length - 1]?.endSec ?? 0;
const segmentOutputFrameCounts = segments.map((segment) =>
Math.ceil(((segment.endSec - segment.startSec) / segment.speed) * targetFrameRate),
Math.ceil(
((segment.endSec - segment.startSec - EPSILON_SEC) / segment.speed) * targetFrameRate,
),
);
const frameDurationUs = 1_000_000 / targetFrameRate;
const epsilonSec = 0.001;
// Async frame queue — decoder pushes, consumer pulls
const pendingFrames: VideoFrame[] = [];
@@ -248,7 +379,7 @@ export class StreamingVideoDecoder {
// One forward stream through the whole file.
// Pass explicit range because some containers are truncated when no end is provided.
const readEndSec = Math.max(this.metadata.duration, this.metadata.streamDuration ?? 0) + 0.5;
const readEndSec = this.metadata.duration + 0.5;
const reader = this.demuxer.read("video", 0, readEndSec).getReader();
// Feed chunks to decoder in background with backpressure
@@ -304,7 +435,7 @@ export class StreamingVideoDecoder {
const sourceTimeSec =
segment.startSec + (segmentFrameIndex / targetFrameRate) * segment.speed;
if (sourceTimeSec >= segment.endSec - epsilonSec) return false;
if (sourceTimeSec >= segment.endSec - EPSILON_SEC) return false;
const clone = new VideoFrame(heldFrame, { timestamp: heldFrame.timestamp });
await onFrame(clone, exportFrameIndex * frameDurationUs, sourceTimeSec * 1000);
@@ -323,7 +454,7 @@ export class StreamingVideoDecoder {
// Finalize completed segments before handling this frame.
while (
segmentIdx < segments.length &&
frameTimeSec >= segments[segmentIdx].endSec - epsilonSec
frameTimeSec >= segments[segmentIdx].endSec - EPSILON_SEC
) {
const segment = segments[segmentIdx];
while (!this.cancelled && (await emitHeldFrameForTarget(segment))) {
@@ -335,7 +466,7 @@ export class StreamingVideoDecoder {
if (
heldFrame &&
segmentIdx < segments.length &&
heldFrameSec < segments[segmentIdx].startSec - epsilonSec
heldFrameSec < segments[segmentIdx].startSec - EPSILON_SEC
) {
heldFrame.close();
heldFrame = null;
@@ -350,7 +481,7 @@ export class StreamingVideoDecoder {
const currentSegment = segments[segmentIdx];
// Before current segment (trimmed region or pre-roll).
if (frameTimeSec < currentSegment.startSec - epsilonSec) {
if (frameTimeSec < currentSegment.startSec - EPSILON_SEC) {
frame.close();
continue;
}
@@ -371,7 +502,7 @@ export class StreamingVideoDecoder {
const sourceTimeSec =
currentSegment.startSec + (segmentFrameIndex / targetFrameRate) * currentSegment.speed;
if (sourceTimeSec >= currentSegment.endSec - epsilonSec) {
if (sourceTimeSec >= currentSegment.endSec - EPSILON_SEC) {
break;
}
if (sourceTimeSec > handoffBoundarySec) {
@@ -393,7 +524,7 @@ export class StreamingVideoDecoder {
if (heldFrame && segmentIdx < segments.length) {
while (!this.cancelled && segmentIdx < segments.length) {
const segment = segments[segmentIdx];
if (heldFrameSec < segment.startSec - epsilonSec) {
if (heldFrameSec < segment.startSec - EPSILON_SEC) {
break;
}
@@ -405,7 +536,7 @@ export class StreamingVideoDecoder {
segmentFrameIndex = 0;
if (
segmentIdx < segments.length &&
heldFrameSec < segments[segmentIdx].startSec - epsilonSec
heldFrameSec < segments[segmentIdx].startSec - EPSILON_SEC
) {
break;
}
@@ -435,7 +566,8 @@ export class StreamingVideoDecoder {
}
this.decoder = null;
const requiredEndSec = segments.length > 0 ? segments[segments.length - 1].endSec : 0;
const isWindows = typeof navigator !== "undefined" && /Windows/.test(navigator.userAgent);
if (
shouldFailDecodeEndedEarly({
cancelled: this.cancelled,
@@ -446,9 +578,22 @@ export class StreamingVideoDecoder {
) {
const decodedAtLabel =
lastDecodedFrameSec === null ? "no decoded frame" : `${lastDecodedFrameSec.toFixed(3)}s`;
throw new Error(
`Video decode ended early at ${decodedAtLabel} (needed ${requiredEndSec.toFixed(3)}s).`,
);
const decodeGapSec =
lastDecodedFrameSec === null ? Infinity : requiredEndSec - lastDecodedFrameSec;
// On Windows, tolerate a small decode gap: up to 10% of required duration, capped at 3 seconds.
const maxToleratedGap = Math.min(3.0, requiredEndSec * 0.1);
if (isWindows && lastDecodedFrameSec !== null && decodeGapSec <= maxToleratedGap) {
console.warn(
`[StreamingVideoDecoder] Decode ended early on Windows with a gap of ${decodeGapSec.toFixed(2)}s ` +
`(max tolerated: ${maxToleratedGap.toFixed(2)}s) proceeding anyway.`,
);
} else {
throw new Error(
`Video decode ended early at ${decodedAtLabel} (needed ${requiredEndSec.toFixed(3)}s).`,
);
}
}
}
@@ -480,11 +625,24 @@ export class StreamingVideoDecoder {
return segments;
}
getEffectiveDuration(trimRegions?: TrimRegion[], speedRegions?: SpeedRegion[]): number {
getExportMetrics(
targetFrameRate: number,
trimRegions?: TrimRegion[],
speedRegions?: SpeedRegion[],
): { effectiveDuration: number; totalFrames: number } {
if (!this.metadata) throw new Error("Must call loadMetadata() first");
const trimSegments = this.computeSegments(this.metadata.duration, trimRegions);
const speedSegments = this.splitBySpeed(trimSegments, speedRegions);
return speedSegments.reduce((sum, seg) => sum + (seg.endSec - seg.startSec) / seg.speed, 0);
const segments = this.splitBySpeed(trimSegments, speedRegions);
return {
effectiveDuration: segments.reduce(
(sum, seg) => sum + (seg.endSec - seg.startSec) / seg.speed,
0,
),
totalFrames: segments.reduce((sum, seg) => {
const segDur = seg.endSec - seg.startSec - EPSILON_SEC;
return sum + Math.max(0, Math.ceil((segDur / seg.speed) * targetFrameRate));
}, 0),
};
}
private splitBySpeed(
@@ -0,0 +1,43 @@
import { describe, expect, it } from "vitest";
import sampleVideoUrl from "../../../tests/fixtures/sample.webm?url";
import type { ExportProgress } from "./types";
import { VideoExporter } from "./videoExporter";
describe("VideoExporter (real browser)", () => {
it("exports a valid MP4 blob from a real video", async () => {
const progressEvents: ExportProgress[] = [];
const exporter = new VideoExporter({
videoUrl: sampleVideoUrl,
width: 320,
height: 180,
frameRate: 15,
bitrate: 1_000_000,
wallpaper: "#1a1a2e",
zoomRegions: [],
showShadow: false,
shadowIntensity: 0,
showBlur: false,
cropRegion: { x: 0, y: 0, width: 1, height: 1 },
onProgress: (p) => progressEvents.push(p),
});
const result = await exporter.export();
expect(result.success, result.error).toBe(true);
expect(result.blob).toBeInstanceOf(Blob);
const buf = await result.blob!.arrayBuffer();
const bytes = new Uint8Array(buf);
const ftyp = new TextDecoder().decode(bytes.slice(4, 8));
expect(ftyp).toBe("ftyp");
expect(result.blob!.size).toBeGreaterThan(1024);
expect(progressEvents.length).toBeGreaterThan(0);
const finalizing = progressEvents.filter((p) => p.phase === "finalizing");
expect(finalizing.length).toBeGreaterThan(0);
expect(finalizing.at(-1)!.percentage).toBe(100);
});
});
+34 -29
View File
@@ -4,8 +4,10 @@ import type {
SpeedRegion,
TrimRegion,
WebcamLayoutPreset,
WebcamSizePreset,
ZoomRegion,
} from "@/components/video-editor/types";
import { getPlatform } from "@/utils/platformUtils";
import { AsyncVideoFrameQueue } from "./asyncVideoFrameQueue";
import { AudioProcessor } from "./audioEncoder";
import { FrameRenderer } from "./frameRenderer";
@@ -33,6 +35,7 @@ interface VideoExporterConfig extends ExportConfig {
cropRegion: CropRegion;
webcamLayoutPreset?: WebcamLayoutPreset;
webcamMaskShape?: import("@/components/video-editor/types").WebcamMaskShape;
webcamSizePreset?: WebcamSizePreset;
webcamPosition?: { cx: number; cy: number } | null;
annotationRegions?: AnnotationRegion[];
previewWidth?: number;
@@ -110,6 +113,8 @@ export class VideoExporter {
this.fatalEncoderError = null;
try {
const platform = await getPlatform();
const streamingDecoder = new StreamingVideoDecoder();
this.streamingDecoder = streamingDecoder;
const videoInfo = await streamingDecoder.loadMetadata(this.config.videoUrl);
@@ -137,12 +142,14 @@ export class VideoExporter {
webcamSize: webcamInfo ? { width: webcamInfo.width, height: webcamInfo.height } : null,
webcamLayoutPreset: this.config.webcamLayoutPreset,
webcamMaskShape: this.config.webcamMaskShape,
webcamSizePreset: this.config.webcamSizePreset,
webcamPosition: this.config.webcamPosition,
annotationRegions: this.config.annotationRegions,
speedRegions: this.config.speedRegions,
previewWidth: this.config.previewWidth,
previewHeight: this.config.previewHeight,
cursorTelemetry: this.config.cursorTelemetry,
platform,
});
this.renderer = renderer;
await renderer.initialize();
@@ -154,17 +161,11 @@ export class VideoExporter {
this.muxer = muxer;
await muxer.initialize();
const effectiveDuration = streamingDecoder.getEffectiveDuration(
const { totalFrames } = streamingDecoder.getExportMetrics(
this.config.frameRate,
this.config.trimRegions,
this.config.speedRegions,
);
const totalFrames = Math.ceil(effectiveDuration * this.config.frameRate);
const readEndSec = Math.max(videoInfo.duration, videoInfo.streamDuration ?? 0) + 0.5;
console.log("[VideoExporter] Original duration:", videoInfo.duration, "s");
console.log("[VideoExporter] Effective duration:", effectiveDuration, "s");
console.log("[VideoExporter] Total frames to export:", totalFrames);
console.log("[VideoExporter] Using streaming decode (web-demuxer + VideoDecoder)");
const frameDuration = 1_000_000 / this.config.frameRate;
let frameIndex = 0;
@@ -234,25 +235,29 @@ export class VideoExporter {
const canvas = renderer.getCanvas();
// Read raw pixels from the canvas instead of passing
// the canvas directly to VideoFrame. On some Linux
// systems the GPU shared-image path (EGL/Ozone) fails
// silently, producing empty frames.
const canvasCtx = canvas.getContext("2d")!;
const imageData = canvasCtx.getImageData(0, 0, canvas.width, canvas.height);
const exportFrame = new VideoFrame(imageData.data.buffer, {
format: "RGBA",
codedWidth: canvas.width,
codedHeight: canvas.height,
timestamp,
duration: frameDuration,
colorSpace: {
primaries: "bt709",
transfer: "iec61966-2-1",
matrix: "rgb",
fullRange: true,
},
});
let exportFrame: VideoFrame;
// On some Linux systems the GPU shared-image path (EGL/Ozone) fails
// silently, producing empty frames, so we force a CPU readback instead.
if (platform === "linux") {
const canvasCtx = canvas.getContext("2d")!;
const imageData = canvasCtx.getImageData(0, 0, canvas.width, canvas.height);
exportFrame = new VideoFrame(imageData.data.buffer, {
format: "RGBA",
codedWidth: canvas.width,
codedHeight: canvas.height,
timestamp,
duration: frameDuration,
colorSpace: {
primaries: "bt709",
transfer: "iec61966-2-1",
matrix: "rgb",
fullRange: true,
},
});
} else {
exportFrame = new VideoFrame(canvas, { timestamp, duration: frameDuration });
}
while (
this.encoder &&
@@ -343,7 +348,7 @@ export class VideoExporter {
this.config.videoUrl,
this.config.trimRegions,
this.config.speedRegions,
readEndSec,
videoInfo.duration,
);
}
}
@@ -419,7 +424,7 @@ export class VideoExporter {
})();
this.muxingPromises.push(muxingPromise);
this.encodeQueue--;
this.encodeQueue = Math.max(0, this.encodeQueue - 1);
},
error: (error) => {
console.error("[VideoExporter] Encoder error:", error);
+1 -3
View File
@@ -17,9 +17,7 @@ export async function requestCameraAccess(): Promise<CameraAccessResult> {
if (window.electronAPI?.requestCameraAccess) {
try {
const electronResult = await window.electronAPI.requestCameraAccess();
if (!electronResult.success || !electronResult.granted) {
return electronResult;
}
return electronResult;
} catch (error) {
return {
success: false,
+5 -1
View File
@@ -3,6 +3,7 @@ export const SHORTCUT_ACTIONS = [
"addTrim",
"addSpeed",
"addAnnotation",
"addBlur",
"addKeyframe",
"deleteSelected",
"playPause",
@@ -108,6 +109,7 @@ export const DEFAULT_SHORTCUTS: ShortcutsConfig = {
addTrim: { key: "t" },
addSpeed: { key: "s" },
addAnnotation: { key: "a" },
addBlur: { key: "b" },
addKeyframe: { key: "f" },
deleteSelected: { key: "d", ctrl: true },
playPause: { key: " " },
@@ -118,6 +120,7 @@ export const SHORTCUT_LABELS: Record<ShortcutAction, string> = {
addTrim: "Add Trim",
addSpeed: "Add Speed",
addAnnotation: "Add Annotation",
addBlur: "Add Blur",
addKeyframe: "Add Keyframe",
deleteSelected: "Delete Selected",
playPause: "Play / Pause",
@@ -125,9 +128,10 @@ export const SHORTCUT_LABELS: Record<ShortcutAction, string> = {
export function matchesShortcut(
e: KeyboardEvent,
binding: ShortcutBinding,
binding: ShortcutBinding | undefined,
isMacPlatform: boolean,
): boolean {
if (!binding) return false;
if (e.key.toLowerCase() !== binding.key.toLowerCase()) return false;
const primaryMod = isMacPlatform ? e.metaKey : e.ctrlKey;