merge: resolve conflicts and update video playback system
This commit is contained in:
@@ -0,0 +1,80 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { applyMosaicToImageData, getBlurOverlayColor, normalizeBlurColor } from "./blurEffects";
|
||||
|
||||
function createTestImageData(width: number, height: number) {
|
||||
const data = new Uint8ClampedArray(width * height * 4);
|
||||
|
||||
for (let y = 0; y < height; y++) {
|
||||
for (let x = 0; x < width; x++) {
|
||||
const offset = (y * width + x) * 4;
|
||||
data[offset] = x * 20 + y;
|
||||
data[offset + 1] = y * 20 + x;
|
||||
data[offset + 2] = (x + y) * 10;
|
||||
data[offset + 3] = 255;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
data,
|
||||
width,
|
||||
height,
|
||||
} as ImageData;
|
||||
}
|
||||
|
||||
describe("applyMosaicToImageData", () => {
|
||||
it("collapses each block to a single representative color", () => {
|
||||
const imageData = createTestImageData(4, 4);
|
||||
const original = new Uint8ClampedArray(imageData.data);
|
||||
|
||||
applyMosaicToImageData(imageData, 2);
|
||||
|
||||
const topLeft = Array.from(imageData.data.slice(0, 4));
|
||||
const topRightOffset = (1 * 4 + 1) * 4;
|
||||
const topRight = Array.from(imageData.data.slice(topRightOffset, topRightOffset + 4));
|
||||
expect(topLeft).toEqual(topRight);
|
||||
|
||||
expect(Array.from(original.slice(0, 4))).not.toEqual(topLeft);
|
||||
});
|
||||
|
||||
it("reduces unique pixel colors, making the transform information-lossy", () => {
|
||||
const imageData = createTestImageData(8, 8);
|
||||
const before = new Set<string>();
|
||||
const after = new Set<string>();
|
||||
|
||||
for (let i = 0; i < imageData.data.length; i += 4) {
|
||||
before.add(
|
||||
`${imageData.data[i]}-${imageData.data[i + 1]}-${imageData.data[i + 2]}-${imageData.data[i + 3]}`,
|
||||
);
|
||||
}
|
||||
|
||||
applyMosaicToImageData(imageData, 4);
|
||||
|
||||
for (let i = 0; i < imageData.data.length; i += 4) {
|
||||
after.add(
|
||||
`${imageData.data[i]}-${imageData.data[i + 1]}-${imageData.data[i + 2]}-${imageData.data[i + 3]}`,
|
||||
);
|
||||
}
|
||||
|
||||
expect(after.size).toBeLessThan(before.size);
|
||||
expect(after.size).toBe(4);
|
||||
});
|
||||
});
|
||||
|
||||
describe("blur color helpers", () => {
|
||||
it("normalizes invalid blur colors to white", () => {
|
||||
expect(normalizeBlurColor("black")).toBe("black");
|
||||
expect(normalizeBlurColor("invalid")).toBe("white");
|
||||
});
|
||||
|
||||
it("returns a dark overlay when black blur color is selected", () => {
|
||||
expect(
|
||||
getBlurOverlayColor({
|
||||
type: "blur",
|
||||
shape: "rectangle",
|
||||
color: "black",
|
||||
intensity: 12,
|
||||
blockSize: 12,
|
||||
}),
|
||||
).toBe("rgba(0, 0, 0, 0.18)");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,113 @@
|
||||
import {
|
||||
type BlurColor,
|
||||
type BlurData,
|
||||
type BlurType,
|
||||
DEFAULT_BLUR_BLOCK_SIZE,
|
||||
DEFAULT_BLUR_INTENSITY,
|
||||
MAX_BLUR_BLOCK_SIZE,
|
||||
MAX_BLUR_INTENSITY,
|
||||
MIN_BLUR_BLOCK_SIZE,
|
||||
MIN_BLUR_INTENSITY,
|
||||
} from "@/components/video-editor/types";
|
||||
|
||||
function clamp(value: number, min: number, max: number) {
|
||||
if (!Number.isFinite(value)) return min;
|
||||
return Math.min(max, Math.max(min, value));
|
||||
}
|
||||
|
||||
export function normalizeBlurType(value: unknown): BlurType {
|
||||
return value === "mosaic" ? "mosaic" : "blur";
|
||||
}
|
||||
|
||||
export function normalizeBlurColor(value: unknown): BlurColor {
|
||||
return value === "black" ? "black" : "white";
|
||||
}
|
||||
|
||||
export function getNormalizedBlurIntensity(blurData?: BlurData | null): number {
|
||||
return clamp(
|
||||
blurData?.intensity ?? DEFAULT_BLUR_INTENSITY,
|
||||
MIN_BLUR_INTENSITY,
|
||||
MAX_BLUR_INTENSITY,
|
||||
);
|
||||
}
|
||||
|
||||
export function getNormalizedMosaicBlockSize(blurData?: BlurData | null, scaleFactor = 1): number {
|
||||
const rawBlockSize = clamp(
|
||||
blurData?.blockSize ?? DEFAULT_BLUR_BLOCK_SIZE,
|
||||
MIN_BLUR_BLOCK_SIZE,
|
||||
MAX_BLUR_BLOCK_SIZE,
|
||||
);
|
||||
return Math.max(1, Math.round(rawBlockSize * Math.max(scaleFactor, 0.01)));
|
||||
}
|
||||
|
||||
export function getBlurOverlayColor(blurData?: BlurData | null): string {
|
||||
const blurColor = normalizeBlurColor(blurData?.color);
|
||||
const blurType = normalizeBlurType(blurData?.type);
|
||||
|
||||
if (blurColor === "black") {
|
||||
return blurType === "mosaic" ? "rgba(0, 0, 0, 0.72)" : "rgba(0, 0, 0, 0.56)";
|
||||
}
|
||||
|
||||
return blurType === "mosaic" ? "rgba(255, 255, 255, 0.06)" : "rgba(255, 255, 255, 0.02)";
|
||||
}
|
||||
|
||||
export function getMosaicGridOverlayColor(blurData?: BlurData | null): string {
|
||||
return normalizeBlurColor(blurData?.color) === "black"
|
||||
? "rgba(255,255,255,0.05)"
|
||||
: "rgba(255,255,255,0.04)";
|
||||
}
|
||||
|
||||
export function applyMosaicToImageData(imageData: ImageData, blockSize: number): ImageData {
|
||||
const width = imageData.width;
|
||||
const height = imageData.height;
|
||||
const data = imageData.data;
|
||||
const normalizedBlockSize = Math.max(1, Math.floor(blockSize));
|
||||
|
||||
if (width <= 0 || height <= 0 || normalizedBlockSize <= 1) {
|
||||
return imageData;
|
||||
}
|
||||
|
||||
for (let blockY = 0; blockY < height; blockY += normalizedBlockSize) {
|
||||
for (let blockX = 0; blockX < width; blockX += normalizedBlockSize) {
|
||||
const blockWidth = Math.min(normalizedBlockSize, width - blockX);
|
||||
const blockHeight = Math.min(normalizedBlockSize, height - blockY);
|
||||
const pixelCount = blockWidth * blockHeight;
|
||||
|
||||
if (pixelCount <= 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let redTotal = 0;
|
||||
let greenTotal = 0;
|
||||
let blueTotal = 0;
|
||||
let alphaTotal = 0;
|
||||
|
||||
for (let y = blockY; y < blockY + blockHeight; y++) {
|
||||
for (let x = blockX; x < blockX + blockWidth; x++) {
|
||||
const offset = (y * width + x) * 4;
|
||||
redTotal += data[offset];
|
||||
greenTotal += data[offset + 1];
|
||||
blueTotal += data[offset + 2];
|
||||
alphaTotal += data[offset + 3];
|
||||
}
|
||||
}
|
||||
|
||||
const averageRed = Math.round(redTotal / pixelCount);
|
||||
const averageGreen = Math.round(greenTotal / pixelCount);
|
||||
const averageBlue = Math.round(blueTotal / pixelCount);
|
||||
const averageAlpha = Math.round(alphaTotal / pixelCount);
|
||||
|
||||
for (let y = blockY; y < blockY + blockHeight; y++) {
|
||||
for (let x = blockX; x < blockX + blockWidth; x++) {
|
||||
const offset = (y * width + x) * 4;
|
||||
data[offset] = averageRed;
|
||||
data[offset + 1] = averageGreen;
|
||||
data[offset + 2] = averageBlue;
|
||||
data[offset + 3] = averageAlpha;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return imageData;
|
||||
}
|
||||
+132
-18
@@ -24,16 +24,111 @@ describe("computeCompositeLayout", () => {
|
||||
webcamSize: { width: 1920, height: 1080 },
|
||||
});
|
||||
|
||||
const refDim = Math.sqrt(1280 * 720);
|
||||
const defaultFraction = 25 / 100; // DEFAULT_WEBCAM_SIZE_PRESET = 25
|
||||
expect(layout).not.toBeNull();
|
||||
expect(layout!.webcamRect).not.toBeNull();
|
||||
expect(layout!.webcamRect!.width).toBeLessThanOrEqual(Math.round(1280 * 0.18) + 1);
|
||||
expect(layout!.webcamRect!.height).toBeLessThanOrEqual(Math.round(720 * 0.18) + 1);
|
||||
expect(layout!.webcamRect!.width).toBeLessThanOrEqual(Math.round(refDim * defaultFraction) + 1);
|
||||
expect(layout!.webcamRect!.height).toBeLessThanOrEqual(
|
||||
Math.round(refDim * defaultFraction) + 1,
|
||||
);
|
||||
expect(
|
||||
Math.abs(layout!.webcamRect!.width * 1080 - layout!.webcamRect!.height * 1920),
|
||||
).toBeLessThanOrEqual(1920);
|
||||
});
|
||||
|
||||
it("uses cover-style full-width stacking in vertical stack mode", () => {
|
||||
it("produces consistent webcam size across landscape and portrait aspect ratios", () => {
|
||||
const webcamSize = { width: 1280, height: 720 };
|
||||
const landscape = computeCompositeLayout({
|
||||
canvasSize: { width: 1920, height: 1080 },
|
||||
screenSize: { width: 1920, height: 1080 },
|
||||
webcamSize,
|
||||
webcamSizePreset: 50,
|
||||
});
|
||||
const portrait = computeCompositeLayout({
|
||||
canvasSize: { width: 1080, height: 1920 },
|
||||
screenSize: { width: 1080, height: 1920 },
|
||||
webcamSize,
|
||||
webcamSizePreset: 50,
|
||||
});
|
||||
|
||||
expect(landscape).not.toBeNull();
|
||||
expect(portrait).not.toBeNull();
|
||||
// Same total pixel count — webcam area should be comparable
|
||||
const landscapeArea = landscape!.webcamRect!.width * landscape!.webcamRect!.height;
|
||||
const portraitArea = portrait!.webcamRect!.width * portrait!.webcamRect!.height;
|
||||
expect(landscapeArea).toBe(portraitArea);
|
||||
});
|
||||
|
||||
it("scales the webcam proportionally as webcamSizePreset increases", () => {
|
||||
const canvasSize = { width: 1920, height: 1080 };
|
||||
const screenSize = { width: 1920, height: 1080 };
|
||||
const webcamSize = { width: 1280, height: 720 };
|
||||
|
||||
const small = computeCompositeLayout({
|
||||
canvasSize,
|
||||
screenSize,
|
||||
webcamSize,
|
||||
webcamSizePreset: 10,
|
||||
});
|
||||
const medium = computeCompositeLayout({
|
||||
canvasSize,
|
||||
screenSize,
|
||||
webcamSize,
|
||||
webcamSizePreset: 25,
|
||||
});
|
||||
const large = computeCompositeLayout({
|
||||
canvasSize,
|
||||
screenSize,
|
||||
webcamSize,
|
||||
webcamSizePreset: 50,
|
||||
});
|
||||
|
||||
expect(small!.webcamRect!.width).toBeLessThan(medium!.webcamRect!.width);
|
||||
expect(medium!.webcamRect!.width).toBeLessThan(large!.webcamRect!.width);
|
||||
expect(small!.webcamRect!.height).toBeLessThan(medium!.webcamRect!.height);
|
||||
expect(medium!.webcamRect!.height).toBeLessThan(large!.webcamRect!.height);
|
||||
});
|
||||
|
||||
it("clamps webcamSizePreset to the valid range (10–50)", () => {
|
||||
const canvasSize = { width: 1920, height: 1080 };
|
||||
const screenSize = { width: 1920, height: 1080 };
|
||||
const webcamSize = { width: 1280, height: 720 };
|
||||
|
||||
const atMin = computeCompositeLayout({
|
||||
canvasSize,
|
||||
screenSize,
|
||||
webcamSize,
|
||||
webcamSizePreset: 10,
|
||||
});
|
||||
const belowMin = computeCompositeLayout({
|
||||
canvasSize,
|
||||
screenSize,
|
||||
webcamSize,
|
||||
webcamSizePreset: 1,
|
||||
});
|
||||
const atMax = computeCompositeLayout({
|
||||
canvasSize,
|
||||
screenSize,
|
||||
webcamSize,
|
||||
webcamSizePreset: 50,
|
||||
});
|
||||
const aboveMax = computeCompositeLayout({
|
||||
canvasSize,
|
||||
screenSize,
|
||||
webcamSize,
|
||||
webcamSizePreset: 100,
|
||||
});
|
||||
|
||||
// Values below 10 should clamp to 10
|
||||
expect(belowMin!.webcamRect!.width).toBe(atMin!.webcamRect!.width);
|
||||
expect(belowMin!.webcamRect!.height).toBe(atMin!.webcamRect!.height);
|
||||
// Values above 50 should clamp to 50
|
||||
expect(aboveMax!.webcamRect!.width).toBe(atMax!.webcamRect!.width);
|
||||
expect(aboveMax!.webcamRect!.height).toBe(atMax!.webcamRect!.height);
|
||||
});
|
||||
|
||||
it("centers the combined screen and webcam stack in vertical stack mode", () => {
|
||||
const layout = computeCompositeLayout({
|
||||
canvasSize: { width: 1920, height: 1080 },
|
||||
maxContentSize: { width: 1536, height: 864 },
|
||||
@@ -43,23 +138,19 @@ describe("computeCompositeLayout", () => {
|
||||
});
|
||||
|
||||
expect(layout).not.toBeNull();
|
||||
expect(layout?.screenRect).toEqual({
|
||||
x: 0,
|
||||
y: 0,
|
||||
width: 1920,
|
||||
height: 0,
|
||||
});
|
||||
expect(layout?.webcamRect).toEqual({
|
||||
x: 0,
|
||||
y: 0,
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
borderRadius: 0,
|
||||
});
|
||||
expect(layout?.screenCover).toBe(true);
|
||||
// Webcam is full-width at the bottom
|
||||
expect(layout!.webcamRect).not.toBeNull();
|
||||
expect(layout!.webcamRect!.x).toBe(0);
|
||||
expect(layout!.webcamRect!.width).toBe(1920);
|
||||
expect(layout!.webcamRect!.borderRadius).toBe(0);
|
||||
// Screen fills remaining space at the top (cover mode)
|
||||
expect(layout!.screenRect.x).toBe(0);
|
||||
expect(layout!.screenRect.y).toBe(0);
|
||||
expect(layout!.screenRect.width).toBe(1920);
|
||||
expect(layout!.screenCover).toBe(true);
|
||||
});
|
||||
|
||||
it("fills the canvas with the screen when vertical stack has no webcam", () => {
|
||||
it("keeps the screen full-canvas and omits the webcam when dimensions are unavailable in stack mode", () => {
|
||||
const layout = computeCompositeLayout({
|
||||
canvasSize: { width: 1920, height: 1080 },
|
||||
maxContentSize: { width: 1536, height: 864 },
|
||||
@@ -78,6 +169,29 @@ describe("computeCompositeLayout", () => {
|
||||
expect(layout?.screenCover).toBe(true);
|
||||
});
|
||||
|
||||
it("uses a 2:1 split layout in dual frame mode", () => {
|
||||
const layout = computeCompositeLayout({
|
||||
canvasSize: { width: 1920, height: 1080 },
|
||||
maxContentSize: { width: 1536, height: 864 },
|
||||
screenSize: { width: 1920, height: 1080 },
|
||||
webcamSize: { width: 1280, height: 720 },
|
||||
layoutPreset: "dual-frame",
|
||||
});
|
||||
|
||||
expect(layout).not.toBeNull();
|
||||
expect(layout?.webcamRect).not.toBeNull();
|
||||
expect(layout?.screenRect.y).toBe(108);
|
||||
expect(layout?.screenRect.height).toBe(864);
|
||||
expect(layout?.screenBorderRadius).toBe(layout?.webcamRect?.borderRadius);
|
||||
expect(layout?.webcamRect?.y).toBe(108);
|
||||
expect(layout?.webcamRect?.height).toBe(864);
|
||||
expect(layout?.webcamRect?.x).toBeGreaterThan(layout?.screenRect.x ?? 0);
|
||||
expect(
|
||||
Math.abs((layout?.screenRect.width ?? 0) - 2 * (layout?.webcamRect?.width ?? 0)),
|
||||
).toBeLessThanOrEqual(1);
|
||||
expect(layout?.screenCover).toBe(true);
|
||||
});
|
||||
|
||||
it("forces circular and square masks to use square dimensions", () => {
|
||||
const circularLayout = computeCompositeLayout({
|
||||
canvasSize: { width: 1920, height: 1080 },
|
||||
|
||||
+120
-10
@@ -15,7 +15,9 @@ export interface Size {
|
||||
height: number;
|
||||
}
|
||||
|
||||
export type WebcamLayoutPreset = "picture-in-picture" | "vertical-stack";
|
||||
export type WebcamLayoutPreset = "picture-in-picture" | "vertical-stack" | "dual-frame";
|
||||
/** Webcam size as a percentage of the canvas reference dimension (10–50). */
|
||||
export type WebcamSizePreset = number;
|
||||
|
||||
export interface WebcamLayoutShadow {
|
||||
color: string;
|
||||
@@ -32,7 +34,6 @@ interface BorderRadiusRule {
|
||||
|
||||
interface OverlayTransform {
|
||||
type: "overlay";
|
||||
maxStageFraction: number;
|
||||
marginFraction: number;
|
||||
minMargin: number;
|
||||
minSize: number;
|
||||
@@ -43,9 +44,17 @@ interface StackTransform {
|
||||
gap: number;
|
||||
}
|
||||
|
||||
interface SplitTransform {
|
||||
type: "split";
|
||||
gapFraction: number;
|
||||
minGap: number;
|
||||
screenUnits: number;
|
||||
webcamUnits: number;
|
||||
}
|
||||
|
||||
export interface WebcamLayoutPresetDefinition {
|
||||
label: string;
|
||||
transform: OverlayTransform | StackTransform;
|
||||
transform: OverlayTransform | StackTransform | SplitTransform;
|
||||
borderRadius: BorderRadiusRule;
|
||||
shadow: WebcamLayoutShadow | null;
|
||||
}
|
||||
@@ -53,11 +62,18 @@ export interface WebcamLayoutPresetDefinition {
|
||||
export interface WebcamCompositeLayout {
|
||||
screenRect: RenderRect;
|
||||
webcamRect: StyledRenderRect | null;
|
||||
screenBorderRadius?: number;
|
||||
/** When true, the video should be scaled to cover screenRect (cropping overflow). */
|
||||
screenCover?: boolean;
|
||||
}
|
||||
|
||||
const MAX_STAGE_FRACTION = 0.18;
|
||||
/** Convert a webcam size percentage (10–50) to a fraction of the reference dimension. */
|
||||
function webcamSizeToFraction(percent: number): number {
|
||||
const safe = Number.isFinite(percent) ? percent : 25;
|
||||
const clamped = Math.max(10, Math.min(50, safe));
|
||||
return clamped / 100;
|
||||
}
|
||||
|
||||
const MARGIN_FRACTION = 0.02;
|
||||
const MAX_BORDER_RADIUS = 24;
|
||||
const WEBCAM_LAYOUT_PRESET_MAP: Record<WebcamLayoutPreset, WebcamLayoutPresetDefinition> = {
|
||||
@@ -65,7 +81,6 @@ const WEBCAM_LAYOUT_PRESET_MAP: Record<WebcamLayoutPreset, WebcamLayoutPresetDef
|
||||
label: "Picture in Picture",
|
||||
transform: {
|
||||
type: "overlay",
|
||||
maxStageFraction: MAX_STAGE_FRACTION,
|
||||
marginFraction: MARGIN_FRACTION,
|
||||
minMargin: 0,
|
||||
minSize: 0,
|
||||
@@ -95,6 +110,22 @@ const WEBCAM_LAYOUT_PRESET_MAP: Record<WebcamLayoutPreset, WebcamLayoutPresetDef
|
||||
},
|
||||
shadow: null,
|
||||
},
|
||||
"dual-frame": {
|
||||
label: "Dual Frame",
|
||||
transform: {
|
||||
type: "split",
|
||||
gapFraction: 0.02,
|
||||
minGap: 12,
|
||||
screenUnits: 2,
|
||||
webcamUnits: 1,
|
||||
},
|
||||
borderRadius: {
|
||||
max: MAX_BORDER_RADIUS,
|
||||
min: 12,
|
||||
fraction: 0.06,
|
||||
},
|
||||
shadow: null,
|
||||
},
|
||||
};
|
||||
|
||||
export const WEBCAM_LAYOUT_PRESETS = Object.entries(WEBCAM_LAYOUT_PRESET_MAP).map(
|
||||
@@ -125,6 +156,7 @@ export function computeCompositeLayout(params: {
|
||||
screenSize: Size;
|
||||
webcamSize?: Size | null;
|
||||
layoutPreset?: WebcamLayoutPreset;
|
||||
webcamSizePreset?: WebcamSizePreset;
|
||||
webcamPosition?: { cx: number; cy: number } | null;
|
||||
webcamMaskShape?: import("@/components/video-editor/types").WebcamMaskShape;
|
||||
}): WebcamCompositeLayout | null {
|
||||
@@ -134,6 +166,7 @@ export function computeCompositeLayout(params: {
|
||||
screenSize,
|
||||
webcamSize,
|
||||
layoutPreset = "picture-in-picture",
|
||||
webcamSizePreset = 25,
|
||||
webcamPosition,
|
||||
webcamMaskShape = "rectangle",
|
||||
} = params;
|
||||
@@ -143,6 +176,8 @@ export function computeCompositeLayout(params: {
|
||||
const webcamHeight = webcamSize?.height;
|
||||
const preset = getWebcamLayoutPresetDefinition(layoutPreset);
|
||||
|
||||
const MAX_STAGE_FRACTION = webcamSizeToFraction(webcamSizePreset);
|
||||
|
||||
if (canvasWidth <= 0 || canvasHeight <= 0 || screenWidth <= 0 || screenHeight <= 0) {
|
||||
return null;
|
||||
}
|
||||
@@ -183,6 +218,69 @@ export function computeCompositeLayout(params: {
|
||||
};
|
||||
}
|
||||
|
||||
if (preset.transform.type === "split") {
|
||||
const screenRect = centerRect({
|
||||
canvasSize,
|
||||
size: screenSize,
|
||||
maxSize: maxContentSize,
|
||||
});
|
||||
|
||||
if (!webcamWidth || !webcamHeight || webcamWidth <= 0 || webcamHeight <= 0) {
|
||||
return { screenRect, webcamRect: null };
|
||||
}
|
||||
|
||||
const contentWidth = Math.min(canvasWidth, Math.max(1, Math.round(maxContentSize.width)));
|
||||
const contentHeight = Math.min(canvasHeight, Math.max(1, Math.round(maxContentSize.height)));
|
||||
const contentX = Math.max(0, Math.floor((canvasWidth - contentWidth) / 2));
|
||||
const contentY = Math.max(0, Math.floor((canvasHeight - contentHeight) / 2));
|
||||
const gap = Math.max(
|
||||
preset.transform.minGap,
|
||||
Math.round(contentWidth * preset.transform.gapFraction),
|
||||
);
|
||||
const totalUnits = preset.transform.screenUnits + preset.transform.webcamUnits;
|
||||
const availableWidth = Math.max(1, contentWidth - gap);
|
||||
const screenSlotWidth = Math.max(
|
||||
1,
|
||||
Math.round((availableWidth * preset.transform.screenUnits) / totalUnits),
|
||||
);
|
||||
const webcamSlotWidth = Math.max(1, availableWidth - screenSlotWidth);
|
||||
|
||||
const screenSlot = {
|
||||
x: contentX,
|
||||
y: contentY,
|
||||
width: screenSlotWidth,
|
||||
height: contentHeight,
|
||||
};
|
||||
const webcamSlot = {
|
||||
x: contentX + screenSlotWidth + gap,
|
||||
y: contentY,
|
||||
width: webcamSlotWidth,
|
||||
height: contentHeight,
|
||||
};
|
||||
|
||||
const webcamBorderRadius = Math.min(
|
||||
preset.borderRadius.max,
|
||||
Math.max(
|
||||
preset.borderRadius.min,
|
||||
Math.round(Math.min(webcamSlot.width, webcamSlot.height) * preset.borderRadius.fraction),
|
||||
),
|
||||
);
|
||||
|
||||
return {
|
||||
screenRect: screenSlot,
|
||||
screenBorderRadius: webcamBorderRadius,
|
||||
webcamRect: {
|
||||
x: webcamSlot.x,
|
||||
y: webcamSlot.y,
|
||||
width: webcamSlot.width,
|
||||
height: webcamSlot.height,
|
||||
borderRadius: webcamBorderRadius,
|
||||
maskShape: "rectangle",
|
||||
},
|
||||
screenCover: true,
|
||||
};
|
||||
}
|
||||
|
||||
const transform = preset.transform;
|
||||
const screenRect = centerRect({
|
||||
canvasSize,
|
||||
@@ -198,8 +296,11 @@ export function computeCompositeLayout(params: {
|
||||
transform.minMargin,
|
||||
Math.round(Math.min(canvasWidth, canvasHeight) * transform.marginFraction),
|
||||
);
|
||||
const maxWidth = Math.max(transform.minSize, canvasWidth * transform.maxStageFraction);
|
||||
const maxHeight = Math.max(transform.minSize, canvasHeight * transform.maxStageFraction);
|
||||
// Use geometric mean so the webcam occupies a consistent visual proportion
|
||||
// regardless of whether the canvas is portrait or landscape.
|
||||
const referenceDim = Math.sqrt(canvasWidth * canvasHeight);
|
||||
const maxWidth = Math.max(transform.minSize, referenceDim * MAX_STAGE_FRACTION);
|
||||
const maxHeight = Math.max(transform.minSize, referenceDim * MAX_STAGE_FRACTION);
|
||||
const scale = Math.min(maxWidth / webcamWidth, maxHeight / webcamHeight);
|
||||
let width = Math.round(webcamWidth * scale);
|
||||
let height = Math.round(webcamHeight * scale);
|
||||
@@ -258,7 +359,16 @@ export function computeCompositeLayout(params: {
|
||||
|
||||
function centerRect(params: { canvasSize: Size; size: Size; maxSize: Size }): RenderRect {
|
||||
const { canvasSize, size, maxSize } = params;
|
||||
const { width: canvasWidth, height: canvasHeight } = canvasSize;
|
||||
return centerRectInBounds({
|
||||
bounds: { x: 0, y: 0, width: canvasSize.width, height: canvasSize.height },
|
||||
size,
|
||||
maxSize,
|
||||
});
|
||||
}
|
||||
|
||||
function centerRectInBounds(params: { bounds: RenderRect; size: Size; maxSize: Size }): RenderRect {
|
||||
const { bounds, size, maxSize } = params;
|
||||
const { x: boundsX, y: boundsY, width: boundsWidth, height: boundsHeight } = bounds;
|
||||
const { width, height } = size;
|
||||
const { width: maxWidth, height: maxHeight } = maxSize;
|
||||
const scale = Math.min(maxWidth / width, maxHeight / height, 1);
|
||||
@@ -266,8 +376,8 @@ function centerRect(params: { canvasSize: Size; size: Size; maxSize: Size }): Re
|
||||
const resolvedHeight = Math.round(height * scale);
|
||||
|
||||
return {
|
||||
x: Math.max(0, Math.floor((canvasWidth - resolvedWidth) / 2)),
|
||||
y: Math.max(0, Math.floor((canvasHeight - resolvedHeight) / 2)),
|
||||
x: boundsX + Math.max(0, Math.floor((boundsWidth - resolvedWidth) / 2)),
|
||||
y: boundsY + Math.max(0, Math.floor((boundsHeight - resolvedHeight) / 2)),
|
||||
width: resolvedWidth,
|
||||
height: resolvedHeight,
|
||||
};
|
||||
|
||||
@@ -1,4 +1,47 @@
|
||||
import type { AnnotationRegion, ArrowDirection } from "@/components/video-editor/types";
|
||||
import { type AnnotationRegion, type ArrowDirection } from "@/components/video-editor/types";
|
||||
import {
|
||||
applyMosaicToImageData,
|
||||
getBlurOverlayColor,
|
||||
getNormalizedBlurIntensity,
|
||||
getNormalizedMosaicBlockSize,
|
||||
normalizeBlurType,
|
||||
} from "@/lib/blurEffects";
|
||||
|
||||
let blurScratchCanvas: HTMLCanvasElement | null = null;
|
||||
let blurScratchCtx: CanvasRenderingContext2D | null = null;
|
||||
|
||||
// Matches a single code point whose script is Han (including non-BMP
|
||||
// Extension A-F), Hiragana, Katakana (including halfwidth forms), or
|
||||
// Hangul. Used to split CJK text at character boundaries during wrap,
|
||||
// since CJK scripts have no word-separating whitespace. Unicode script
|
||||
// property escapes require ES2018+; tsconfig target is ES2020.
|
||||
const CJK_CHAR = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
|
||||
|
||||
function tokenizeForWrap(line: string): string[] {
|
||||
// Split Latin text on whitespace (preserving the whitespace as its own token,
|
||||
// matching the original behavior), and split CJK runs into individual
|
||||
// characters so each one becomes a breakable unit. This mirrors the editor's
|
||||
// CSS `word-break: break-word` handling for CJK content.
|
||||
const tokens: string[] = [];
|
||||
let buffer = "";
|
||||
const chars = Array.from(line);
|
||||
const flushBuffer = () => {
|
||||
if (buffer) {
|
||||
tokens.push(...buffer.split(/(\s+)/).filter((s) => s.length > 0));
|
||||
buffer = "";
|
||||
}
|
||||
};
|
||||
for (const ch of chars) {
|
||||
if (CJK_CHAR.test(ch)) {
|
||||
flushBuffer();
|
||||
tokens.push(ch);
|
||||
} else {
|
||||
buffer += ch;
|
||||
}
|
||||
}
|
||||
flushBuffer();
|
||||
return tokens;
|
||||
}
|
||||
|
||||
// SVG path data for each arrow direction
|
||||
const ARROW_PATHS: Record<ArrowDirection, string[]> = {
|
||||
@@ -96,6 +139,101 @@ function renderArrow(
|
||||
ctx.restore();
|
||||
}
|
||||
|
||||
function drawBlurPath(
|
||||
ctx: CanvasRenderingContext2D,
|
||||
annotation: AnnotationRegion,
|
||||
x: number,
|
||||
y: number,
|
||||
width: number,
|
||||
height: number,
|
||||
) {
|
||||
const shape = annotation.blurData?.shape || "rectangle";
|
||||
if (shape === "rectangle") {
|
||||
ctx.beginPath();
|
||||
ctx.rect(x, y, width, height);
|
||||
return;
|
||||
}
|
||||
|
||||
if (shape === "oval") {
|
||||
ctx.beginPath();
|
||||
ctx.ellipse(x + width / 2, y + height / 2, width / 2, height / 2, 0, 0, Math.PI * 2);
|
||||
return;
|
||||
}
|
||||
|
||||
const points = annotation.blurData?.freehandPoints;
|
||||
if (shape === "freehand" && points && points.length >= 3) {
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(x + (points[0].x / 100) * width, y + (points[0].y / 100) * height);
|
||||
for (let i = 1; i < points.length; i++) {
|
||||
ctx.lineTo(x + (points[i].x / 100) * width, y + (points[i].y / 100) * height);
|
||||
}
|
||||
ctx.closePath();
|
||||
return;
|
||||
}
|
||||
|
||||
ctx.beginPath();
|
||||
ctx.rect(x, y, width, height);
|
||||
}
|
||||
|
||||
function renderBlur(
|
||||
ctx: CanvasRenderingContext2D,
|
||||
annotation: AnnotationRegion,
|
||||
x: number,
|
||||
y: number,
|
||||
width: number,
|
||||
height: number,
|
||||
scaleFactor: number,
|
||||
) {
|
||||
const canvas = ctx.canvas;
|
||||
const blurType = normalizeBlurType(annotation.blurData?.type);
|
||||
|
||||
const blurRadius = Math.max(
|
||||
1,
|
||||
Math.round(getNormalizedBlurIntensity(annotation.blurData) * scaleFactor),
|
||||
);
|
||||
const samplePadding =
|
||||
blurType === "mosaic"
|
||||
? Math.max(0, Math.ceil(getNormalizedMosaicBlockSize(annotation.blurData, scaleFactor)))
|
||||
: Math.max(2, Math.ceil(blurRadius * 2));
|
||||
const sx = Math.max(0, Math.floor(x) - samplePadding);
|
||||
const sy = Math.max(0, Math.floor(y) - samplePadding);
|
||||
const ex = Math.min(canvas.width, Math.ceil(x + width) + samplePadding);
|
||||
const ey = Math.min(canvas.height, Math.ceil(y + height) + samplePadding);
|
||||
const sw = Math.max(0, ex - sx);
|
||||
const sh = Math.max(0, ey - sy);
|
||||
if (sw <= 0 || sh <= 0) return;
|
||||
|
||||
if (!blurScratchCanvas || !blurScratchCtx) {
|
||||
blurScratchCanvas = document.createElement("canvas");
|
||||
blurScratchCtx = blurScratchCanvas.getContext("2d");
|
||||
}
|
||||
if (!blurScratchCanvas || !blurScratchCtx) return;
|
||||
|
||||
blurScratchCanvas.width = sw;
|
||||
blurScratchCanvas.height = sh;
|
||||
blurScratchCtx.clearRect(0, 0, sw, sh);
|
||||
blurScratchCtx.drawImage(canvas, sx, sy, sw, sh, 0, 0, sw, sh);
|
||||
|
||||
if (blurType === "mosaic") {
|
||||
const imageData = blurScratchCtx.getImageData(0, 0, sw, sh);
|
||||
applyMosaicToImageData(
|
||||
imageData,
|
||||
getNormalizedMosaicBlockSize(annotation.blurData, scaleFactor),
|
||||
);
|
||||
blurScratchCtx.putImageData(imageData, 0, 0);
|
||||
}
|
||||
|
||||
ctx.save();
|
||||
drawBlurPath(ctx, annotation, x, y, width, height);
|
||||
ctx.clip();
|
||||
ctx.filter = blurType === "mosaic" ? "none" : `blur(${blurRadius}px)`;
|
||||
ctx.drawImage(blurScratchCanvas, sx, sy);
|
||||
ctx.filter = "none";
|
||||
ctx.fillStyle = getBlurOverlayColor(annotation.blurData);
|
||||
ctx.fillRect(sx, sy, sw, sh);
|
||||
ctx.restore();
|
||||
}
|
||||
|
||||
function renderText(
|
||||
ctx: CanvasRenderingContext2D,
|
||||
annotation: AnnotationRegion,
|
||||
@@ -144,13 +282,13 @@ function renderText(
|
||||
lines.push("");
|
||||
continue;
|
||||
}
|
||||
const words = rawLine.split(/(\s+)/);
|
||||
const tokens = tokenizeForWrap(rawLine);
|
||||
let current = "";
|
||||
for (const word of words) {
|
||||
const test = current + word;
|
||||
for (const token of tokens) {
|
||||
const test = current + token;
|
||||
if (current && ctx.measureText(test).width > availableWidth) {
|
||||
lines.push(current);
|
||||
current = word.trimStart();
|
||||
current = token.trimStart();
|
||||
} else {
|
||||
current = test;
|
||||
}
|
||||
@@ -268,7 +406,7 @@ export async function renderAnnotations(
|
||||
): Promise<void> {
|
||||
// Filter active annotations at current time
|
||||
const activeAnnotations = annotations.filter(
|
||||
(ann) => currentTimeMs >= ann.startMs && currentTimeMs <= ann.endMs,
|
||||
(ann) => currentTimeMs >= ann.startMs && currentTimeMs < ann.endMs,
|
||||
);
|
||||
|
||||
// Sort by z-index (lower first, so higher z-index draws on top)
|
||||
@@ -304,6 +442,10 @@ export async function renderAnnotations(
|
||||
);
|
||||
}
|
||||
break;
|
||||
|
||||
case "blur":
|
||||
renderBlur(ctx, annotation, x, y, width, height, scaleFactor);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import type { VideoMuxer } from "./muxer";
|
||||
const AUDIO_BITRATE = 128_000;
|
||||
const DECODE_BACKPRESSURE_LIMIT = 20;
|
||||
const MIN_SPEED_REGION_DELTA_MS = 0.0001;
|
||||
const SEEK_TIMEOUT_MS = 5_000;
|
||||
|
||||
export class AudioProcessor {
|
||||
private cancelled = false;
|
||||
@@ -18,9 +19,9 @@ export class AudioProcessor {
|
||||
demuxer: WebDemuxer,
|
||||
muxer: VideoMuxer,
|
||||
videoUrl: string,
|
||||
trimRegions?: TrimRegion[],
|
||||
speedRegions?: SpeedRegion[],
|
||||
readEndSec?: number,
|
||||
trimRegions: TrimRegion[] | undefined,
|
||||
speedRegions: SpeedRegion[] | undefined,
|
||||
validatedDurationSec: number,
|
||||
): Promise<void> {
|
||||
const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) : [];
|
||||
const sortedSpeedRegions = speedRegions
|
||||
@@ -35,14 +36,19 @@ export class AudioProcessor {
|
||||
videoUrl,
|
||||
sortedTrims,
|
||||
sortedSpeedRegions,
|
||||
validatedDurationSec,
|
||||
);
|
||||
if (!this.cancelled) {
|
||||
if (!this.cancelled && renderedAudioBlob.size > 0) {
|
||||
await this.muxRenderedAudioBlob(renderedAudioBlob, muxer);
|
||||
return;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// No speed edits: keep the original demux/decode/encode path with trim timestamp remap.
|
||||
// The +0.5s buffer mirrors streamingDecoder.decodeAll's read window so the trim-only
|
||||
// and speed-aware paths agree on how far to read past the validated duration boundary.
|
||||
const readEndSec = validatedDurationSec + 0.5;
|
||||
await this.processTrimOnlyAudio(demuxer, muxer, sortedTrims, readEndSec);
|
||||
}
|
||||
|
||||
@@ -55,7 +61,7 @@ export class AudioProcessor {
|
||||
): Promise<void> {
|
||||
let audioConfig: AudioDecoderConfig;
|
||||
try {
|
||||
audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig;
|
||||
audioConfig = await demuxer.getDecoderConfig("audio");
|
||||
} catch {
|
||||
console.warn("[AudioProcessor] No audio track found, skipping");
|
||||
return;
|
||||
@@ -80,11 +86,10 @@ export class AudioProcessor {
|
||||
typeof readEndSec === "number" && Number.isFinite(readEndSec)
|
||||
? Math.max(0, readEndSec)
|
||||
: undefined;
|
||||
const audioStream = (
|
||||
const audioStream =
|
||||
safeReadEndSec !== undefined
|
||||
? demuxer.read("audio", 0, safeReadEndSec)
|
||||
: demuxer.read("audio")
|
||||
) as ReadableStream<EncodedAudioChunk>;
|
||||
: demuxer.read("audio");
|
||||
const reader = audioStream.getReader();
|
||||
|
||||
try {
|
||||
@@ -187,6 +192,7 @@ export class AudioProcessor {
|
||||
videoUrl: string,
|
||||
trimRegions: TrimRegion[],
|
||||
speedRegions: SpeedRegion[],
|
||||
validatedDurationSec: number,
|
||||
): Promise<Blob> {
|
||||
const media = document.createElement("audio");
|
||||
media.src = videoUrl;
|
||||
@@ -211,15 +217,44 @@ export class AudioProcessor {
|
||||
const destinationNode = audioContext.createMediaStreamDestination();
|
||||
sourceNode.connect(destinationNode);
|
||||
|
||||
const { recorder, recordedBlobPromise } = this.startAudioRecording(destinationNode.stream);
|
||||
let rafId: number | null = null;
|
||||
let recorder: MediaRecorder | null = null;
|
||||
let recordedBlobPromise: Promise<Blob> | null = null;
|
||||
|
||||
try {
|
||||
if (audioContext.state === "suspended") {
|
||||
await audioContext.resume();
|
||||
}
|
||||
|
||||
await this.seekTo(media, 0);
|
||||
// Skip past any initial trim region(s) before recording starts to avoid
|
||||
// capturing trimmed audio during the first rAF frames of playback.
|
||||
// Loops to handle back-to-back or overlapping trims at t=0.
|
||||
const effectiveEnd = validatedDurationSec;
|
||||
let startPosition = 0;
|
||||
for (let i = 0; i <= trimRegions.length; i++) {
|
||||
const activeTrim = this.findActiveTrimRegion(startPosition * 1000, trimRegions);
|
||||
if (!activeTrim) break;
|
||||
startPosition = activeTrim.endMs / 1000;
|
||||
if (startPosition >= effectiveEnd) break;
|
||||
}
|
||||
|
||||
if (startPosition >= effectiveEnd) {
|
||||
// All content is trimmed — return silent blob
|
||||
return new Blob([], { type: "audio/webm" });
|
||||
}
|
||||
|
||||
await this.seekTo(media, startPosition);
|
||||
|
||||
// Set initial playback rate for the starting position
|
||||
const initialSpeedRegion = this.findActiveSpeedRegion(startPosition * 1000, speedRegions);
|
||||
if (initialSpeedRegion) {
|
||||
media.playbackRate = initialSpeedRegion.speed;
|
||||
}
|
||||
|
||||
// Start recording only AFTER seeking past trims
|
||||
const recording = this.startAudioRecording(destinationNode.stream);
|
||||
recorder = recording.recorder;
|
||||
recordedBlobPromise = recording.recordedBlobPromise;
|
||||
await media.play();
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
@@ -249,24 +284,66 @@ export class AudioProcessor {
|
||||
return;
|
||||
}
|
||||
|
||||
// Stop playback at validated duration — browser's media.duration
|
||||
// may be inflated from bad container metadata.
|
||||
if (media.currentTime >= validatedDurationSec) {
|
||||
media.pause();
|
||||
cleanup();
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
|
||||
const currentTimeMs = media.currentTime * 1000;
|
||||
const activeTrimRegion = this.findActiveTrimRegion(currentTimeMs, trimRegions);
|
||||
|
||||
if (activeTrimRegion && !media.paused && !media.ended) {
|
||||
const skipToTime = activeTrimRegion.endMs / 1000;
|
||||
if (skipToTime >= media.duration) {
|
||||
if (skipToTime >= media.duration || skipToTime >= validatedDurationSec) {
|
||||
media.pause();
|
||||
cleanup();
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
// Pause recording during trim seek to prevent capturing
|
||||
// silence/noise as the audio element seeks.
|
||||
media.pause();
|
||||
if (recorder?.state === "recording") recorder.pause();
|
||||
const onSeeked = () => {
|
||||
clearTimeout(seekTimer);
|
||||
if (this.cancelled) {
|
||||
cleanup();
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
if (recorder?.state === "paused") recorder.resume();
|
||||
media
|
||||
.play()
|
||||
.then(() => {
|
||||
if (!this.cancelled) rafId = requestAnimationFrame(tick);
|
||||
})
|
||||
.catch((err) => {
|
||||
cleanup();
|
||||
reject(
|
||||
new Error(
|
||||
`Failed to resume playback after trim seek: ${err instanceof Error ? err.message : String(err)}`,
|
||||
),
|
||||
);
|
||||
});
|
||||
};
|
||||
const seekTimer = window.setTimeout(() => {
|
||||
media.removeEventListener("seeked", onSeeked);
|
||||
cleanup();
|
||||
reject(new Error("Audio seek timed out while skipping trim region"));
|
||||
}, SEEK_TIMEOUT_MS);
|
||||
media.addEventListener("seeked", onSeeked, { once: true });
|
||||
media.currentTime = skipToTime;
|
||||
} else {
|
||||
const activeSpeedRegion = this.findActiveSpeedRegion(currentTimeMs, speedRegions);
|
||||
const playbackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1;
|
||||
if (Math.abs(media.playbackRate - playbackRate) > 0.0001) {
|
||||
media.playbackRate = playbackRate;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const activeSpeedRegion = this.findActiveSpeedRegion(currentTimeMs, speedRegions);
|
||||
const playbackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1;
|
||||
if (Math.abs(media.playbackRate - playbackRate) > 0.0001) {
|
||||
media.playbackRate = playbackRate;
|
||||
}
|
||||
|
||||
if (!media.paused && !media.ended) {
|
||||
@@ -286,7 +363,7 @@ export class AudioProcessor {
|
||||
cancelAnimationFrame(rafId);
|
||||
}
|
||||
media.pause();
|
||||
if (recorder.state !== "inactive") {
|
||||
if (recorder && recorder.state !== "inactive") {
|
||||
recorder.stop();
|
||||
}
|
||||
destinationNode.stream.getTracks().forEach((track) => track.stop());
|
||||
@@ -297,6 +374,12 @@ export class AudioProcessor {
|
||||
media.load();
|
||||
}
|
||||
|
||||
if (!recordedBlobPromise) {
|
||||
// Invariant: either an early return above fires, or startAudioRecording ran and
|
||||
// populated recordedBlobPromise before the playback Promise resolved. Reaching
|
||||
// here means that contract was broken — fail loud instead of returning silence.
|
||||
throw new Error("Audio recorder finished without assigning recordedBlobPromise");
|
||||
}
|
||||
const recordedBlob = await recordedBlobPromise;
|
||||
if (this.cancelled) {
|
||||
throw new Error("Export cancelled");
|
||||
@@ -314,8 +397,8 @@ export class AudioProcessor {
|
||||
|
||||
try {
|
||||
await demuxer.load(file);
|
||||
const audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig;
|
||||
const reader = (demuxer.read("audio") as ReadableStream<EncodedAudioChunk>).getReader();
|
||||
const audioConfig = await demuxer.getDecoderConfig("audio");
|
||||
const reader = demuxer.read("audio").getReader();
|
||||
let isFirstChunk = true;
|
||||
|
||||
try {
|
||||
@@ -459,7 +542,10 @@ export class AudioProcessor {
|
||||
}
|
||||
|
||||
private cloneWithTimestamp(src: AudioData, newTimestamp: number): AudioData {
|
||||
const isPlanar = src.format?.includes("planar") ?? false;
|
||||
if (!src.format) {
|
||||
throw new Error("AudioData format is required for cloning");
|
||||
}
|
||||
const isPlanar = src.format.includes("planar");
|
||||
const numPlanes = isPlanar ? src.numberOfChannels : 1;
|
||||
|
||||
let totalSize = 0;
|
||||
@@ -476,7 +562,7 @@ export class AudioProcessor {
|
||||
}
|
||||
|
||||
return new AudioData({
|
||||
format: src.format!,
|
||||
format: src.format,
|
||||
sampleRate: src.sampleRate,
|
||||
numberOfFrames: src.numberOfFrames,
|
||||
numberOfChannels: src.numberOfChannels,
|
||||
|
||||
@@ -13,6 +13,7 @@ import type {
|
||||
CropRegion,
|
||||
SpeedRegion,
|
||||
WebcamLayoutPreset,
|
||||
WebcamSizePreset,
|
||||
ZoomDepth,
|
||||
ZoomRegion,
|
||||
} from "@/components/video-editor/types";
|
||||
@@ -70,12 +71,14 @@ interface FrameRenderConfig {
|
||||
webcamSize?: Size | null;
|
||||
webcamLayoutPreset?: WebcamLayoutPreset;
|
||||
webcamMaskShape?: import("@/components/video-editor/types").WebcamMaskShape;
|
||||
webcamSizePreset?: WebcamSizePreset;
|
||||
webcamPosition?: { cx: number; cy: number } | null;
|
||||
annotationRegions?: AnnotationRegion[];
|
||||
speedRegions?: SpeedRegion[];
|
||||
previewWidth?: number;
|
||||
previewHeight?: number;
|
||||
cursorTelemetry?: import("@/components/video-editor/types").CursorTelemetryPoint[];
|
||||
platform: string;
|
||||
}
|
||||
|
||||
interface AnimationState {
|
||||
@@ -112,6 +115,8 @@ export class FrameRenderer {
|
||||
private shadowCtx: CanvasRenderingContext2D | null = null;
|
||||
private compositeCanvas: HTMLCanvasElement | null = null;
|
||||
private compositeCtx: CanvasRenderingContext2D | null = null;
|
||||
private rasterCanvas: HTMLCanvasElement | null = null;
|
||||
private rasterCtx: CanvasRenderingContext2D | null = null;
|
||||
private config: FrameRenderConfig;
|
||||
private animationState: AnimationState;
|
||||
private layoutCache: LayoutCache | null = null;
|
||||
@@ -120,9 +125,11 @@ export class FrameRenderer {
|
||||
private smoothedAutoFocus: { cx: number; cy: number } | null = null;
|
||||
private prevAnimationTimeMs: number | null = null;
|
||||
private prevTargetProgress = 0;
|
||||
private isLinux = false;
|
||||
|
||||
constructor(config: FrameRenderConfig) {
|
||||
this.config = config;
|
||||
this.isLinux = config.platform === "linux";
|
||||
this.animationState = {
|
||||
scale: 1,
|
||||
focusX: DEFAULT_FOCUS.cx,
|
||||
@@ -183,14 +190,24 @@ export class FrameRenderer {
|
||||
this.compositeCanvas = document.createElement("canvas");
|
||||
this.compositeCanvas.width = this.config.width;
|
||||
this.compositeCanvas.height = this.config.height;
|
||||
|
||||
// On Linux, getImageData() is called frequently causing frequent CPU readback
|
||||
this.compositeCtx = this.compositeCanvas.getContext("2d", {
|
||||
willReadFrequently: false,
|
||||
willReadFrequently: this.isLinux,
|
||||
});
|
||||
|
||||
if (!this.compositeCtx) {
|
||||
throw new Error("Failed to get 2D context for composite canvas");
|
||||
}
|
||||
|
||||
this.rasterCanvas = document.createElement("canvas");
|
||||
this.rasterCanvas.width = this.config.width;
|
||||
this.rasterCanvas.height = this.config.height;
|
||||
this.rasterCtx = this.rasterCanvas.getContext("2d");
|
||||
if (!this.rasterCtx) {
|
||||
throw new Error("Failed to get 2D context for raster canvas");
|
||||
}
|
||||
|
||||
// Setup shadow canvas if needed
|
||||
if (this.config.showShadow) {
|
||||
this.shadowCanvas = document.createElement("canvas");
|
||||
@@ -453,6 +470,7 @@ export class FrameRenderer {
|
||||
screenSize: { width: croppedVideoWidth, height: croppedVideoHeight },
|
||||
webcamSize: webcamFrame ? this.config.webcamSize : null,
|
||||
layoutPreset: this.config.webcamLayoutPreset,
|
||||
webcamSizePreset: this.config.webcamSizePreset,
|
||||
webcamPosition: this.config.webcamPosition,
|
||||
webcamMaskShape: this.config.webcamMaskShape,
|
||||
});
|
||||
@@ -494,7 +512,12 @@ export class FrameRenderer {
|
||||
const previewWidth = this.config.previewWidth ?? this.config.width;
|
||||
const previewHeight = this.config.previewHeight ?? this.config.height;
|
||||
const canvasScaleFactor = Math.min(width / previewWidth, height / previewHeight);
|
||||
const scaledBorderRadius = compositeLayout.screenCover ? 0 : borderRadius * canvasScaleFactor;
|
||||
const scaledBorderRadius =
|
||||
compositeLayout.screenBorderRadius != null
|
||||
? compositeLayout.screenBorderRadius
|
||||
: compositeLayout.screenCover
|
||||
? 0
|
||||
: borderRadius * canvasScaleFactor;
|
||||
|
||||
this.maskGraphics.clear();
|
||||
this.maskGraphics.roundRect(0, 0, screenRect.width, screenRect.height, scaledBorderRadius);
|
||||
@@ -522,16 +545,10 @@ export class FrameRenderer {
|
||||
private updateAnimationState(timeMs: number): number {
|
||||
if (!this.cameraContainer || !this.layoutCache) return 0;
|
||||
|
||||
const bmEx = this.layoutCache.maskRect;
|
||||
const ssEx = this.layoutCache.stageSize;
|
||||
const viewportRatio =
|
||||
bmEx.width > 0 && bmEx.height > 0
|
||||
? { widthRatio: ssEx.width / bmEx.width, heightRatio: ssEx.height / bmEx.height }
|
||||
: undefined;
|
||||
const { region, strength, blendedScale, transition } = findDominantRegion(
|
||||
this.config.zoomRegions,
|
||||
timeMs,
|
||||
{ connectZooms: true, cursorTelemetry: this.config.cursorTelemetry, viewportRatio },
|
||||
{ connectZooms: true, cursorTelemetry: this.config.cursorTelemetry },
|
||||
);
|
||||
|
||||
const defaultFocus = DEFAULT_FOCUS;
|
||||
@@ -675,10 +692,49 @@ export class FrameRenderer {
|
||||
);
|
||||
}
|
||||
|
||||
// On Linux/Wayland the implicit GPU→2D texture-sharing path
|
||||
// used by drawImage(webglCanvas) can fail silently (EGL/Ozone),
|
||||
// producing green/empty frames. Explicit gl.readPixels always
|
||||
// copies from GPU to CPU memory, bypassing that path.
|
||||
private readbackVideoCanvas(): HTMLCanvasElement {
|
||||
const glCanvas = this.app!.canvas as HTMLCanvasElement;
|
||||
const gl =
|
||||
(glCanvas.getContext("webgl2") as WebGL2RenderingContext | null) ??
|
||||
(glCanvas.getContext("webgl") as WebGLRenderingContext | null);
|
||||
|
||||
if (!gl || !this.rasterCanvas || !this.rasterCtx) {
|
||||
return glCanvas;
|
||||
}
|
||||
|
||||
const w = glCanvas.width;
|
||||
const h = glCanvas.height;
|
||||
const buf = new Uint8Array(w * h * 4);
|
||||
gl.readPixels(0, 0, w, h, gl.RGBA, gl.UNSIGNED_BYTE, buf);
|
||||
|
||||
// readPixels returns rows bottom-to-top; flip vertically
|
||||
const rowSize = w * 4;
|
||||
const temp = new Uint8Array(rowSize);
|
||||
for (let top = 0, bot = h - 1; top < bot; top++, bot--) {
|
||||
const tOff = top * rowSize;
|
||||
const bOff = bot * rowSize;
|
||||
temp.set(buf.subarray(tOff, tOff + rowSize));
|
||||
buf.copyWithin(tOff, bOff, bOff + rowSize);
|
||||
buf.set(temp, bOff);
|
||||
}
|
||||
|
||||
const imageData = new ImageData(new Uint8ClampedArray(buf.buffer), w, h);
|
||||
this.rasterCtx.putImageData(imageData, 0, 0);
|
||||
|
||||
return this.rasterCanvas;
|
||||
}
|
||||
|
||||
private compositeWithShadows(webcamFrame?: VideoFrame | null): void {
|
||||
if (!this.compositeCanvas || !this.compositeCtx || !this.app) return;
|
||||
|
||||
const videoCanvas = this.app.canvas as HTMLCanvasElement;
|
||||
const videoCanvas = this.isLinux
|
||||
? this.readbackVideoCanvas()
|
||||
: (this.app.canvas as HTMLCanvasElement);
|
||||
|
||||
const ctx = this.compositeCtx;
|
||||
const w = this.compositeCanvas.width;
|
||||
const h = this.compositeCanvas.height;
|
||||
@@ -735,6 +791,22 @@ export class FrameRenderer {
|
||||
if (webcamFrame && webcamRect) {
|
||||
const preset = getWebcamLayoutPresetDefinition(this.config.webcamLayoutPreset);
|
||||
const shape = webcamRect.maskShape ?? this.config.webcamMaskShape ?? "rectangle";
|
||||
const sourceWidth =
|
||||
("displayWidth" in webcamFrame && webcamFrame.displayWidth > 0
|
||||
? webcamFrame.displayWidth
|
||||
: webcamFrame.codedWidth) || webcamRect.width;
|
||||
const sourceHeight =
|
||||
("displayHeight" in webcamFrame && webcamFrame.displayHeight > 0
|
||||
? webcamFrame.displayHeight
|
||||
: webcamFrame.codedHeight) || webcamRect.height;
|
||||
const sourceAspect = sourceWidth / sourceHeight;
|
||||
const targetAspect = webcamRect.width / webcamRect.height;
|
||||
const sourceCropWidth =
|
||||
sourceAspect > targetAspect ? Math.round(sourceHeight * targetAspect) : sourceWidth;
|
||||
const sourceCropHeight =
|
||||
sourceAspect > targetAspect ? sourceHeight : Math.round(sourceWidth / targetAspect);
|
||||
const sourceCropX = Math.max(0, Math.round((sourceWidth - sourceCropWidth) / 2));
|
||||
const sourceCropY = Math.max(0, Math.round((sourceHeight - sourceCropHeight) / 2));
|
||||
ctx.save();
|
||||
drawCanvasClipPath(
|
||||
ctx,
|
||||
@@ -756,6 +828,10 @@ export class FrameRenderer {
|
||||
ctx.clip();
|
||||
ctx.drawImage(
|
||||
webcamFrame as unknown as CanvasImageSource,
|
||||
sourceCropX,
|
||||
sourceCropY,
|
||||
sourceCropWidth,
|
||||
sourceCropHeight,
|
||||
webcamRect.x,
|
||||
webcamRect.y,
|
||||
webcamRect.width,
|
||||
@@ -795,5 +871,7 @@ export class FrameRenderer {
|
||||
this.shadowCtx = null;
|
||||
this.compositeCanvas = null;
|
||||
this.compositeCtx = null;
|
||||
this.rasterCanvas = null;
|
||||
this.rasterCtx = null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import sampleVideoUrl from "../../../tests/fixtures/sample.webm?url";
|
||||
import { GifExporter } from "./gifExporter";
|
||||
import type { ExportProgress } from "./types";
|
||||
|
||||
describe("GifExporter (real browser)", () => {
|
||||
it("exports a valid GIF blob from a real video", async () => {
|
||||
const progressEvents: ExportProgress[] = [];
|
||||
|
||||
const exporter = new GifExporter({
|
||||
videoUrl: sampleVideoUrl,
|
||||
width: 320,
|
||||
height: 180,
|
||||
frameRate: 15,
|
||||
loop: true,
|
||||
sizePreset: "medium",
|
||||
wallpaper: "#1a1a2e",
|
||||
zoomRegions: [],
|
||||
showShadow: false,
|
||||
shadowIntensity: 0,
|
||||
showBlur: false,
|
||||
cropRegion: { x: 0, y: 0, width: 1, height: 1 },
|
||||
onProgress: (p) => progressEvents.push(p),
|
||||
});
|
||||
|
||||
const result = await exporter.export();
|
||||
|
||||
expect(result.success, result.error).toBe(true);
|
||||
expect(result.blob).toBeInstanceOf(Blob);
|
||||
|
||||
const buf = await result.blob!.arrayBuffer();
|
||||
const header = new TextDecoder().decode(new Uint8Array(buf, 0, 6));
|
||||
expect(header).toMatch(/^GIF8[79]a/);
|
||||
|
||||
expect(result.blob!.size).toBeGreaterThan(1024);
|
||||
|
||||
expect(progressEvents.length).toBeGreaterThan(0);
|
||||
|
||||
const finalizing = progressEvents.filter((p) => p.phase === "finalizing");
|
||||
expect(finalizing.length).toBeGreaterThan(0);
|
||||
expect(finalizing.at(-1)!.percentage).toBe(100);
|
||||
});
|
||||
});
|
||||
@@ -5,8 +5,10 @@ import type {
|
||||
SpeedRegion,
|
||||
TrimRegion,
|
||||
WebcamLayoutPreset,
|
||||
WebcamSizePreset,
|
||||
ZoomRegion,
|
||||
} from "@/components/video-editor/types";
|
||||
import { getPlatform } from "@/utils/platformUtils";
|
||||
import { AsyncVideoFrameQueue } from "./asyncVideoFrameQueue";
|
||||
import { FrameRenderer } from "./frameRenderer";
|
||||
import { StreamingVideoDecoder } from "./streamingDecoder";
|
||||
@@ -42,6 +44,7 @@ interface GifExporterConfig {
|
||||
cropRegion: CropRegion;
|
||||
webcamLayoutPreset?: WebcamLayoutPreset;
|
||||
webcamMaskShape?: import("@/components/video-editor/types").WebcamMaskShape;
|
||||
webcamSizePreset?: WebcamSizePreset;
|
||||
webcamPosition?: { cx: number; cy: number } | null;
|
||||
annotationRegions?: AnnotationRegion[];
|
||||
previewWidth?: number;
|
||||
@@ -113,7 +116,10 @@ export class GifExporter {
|
||||
|
||||
async export(): Promise<ExportResult> {
|
||||
let webcamFrameQueue: AsyncVideoFrameQueue | null = null;
|
||||
|
||||
try {
|
||||
const platform = await getPlatform();
|
||||
|
||||
this.cleanup();
|
||||
this.cancelled = false;
|
||||
|
||||
@@ -144,12 +150,14 @@ export class GifExporter {
|
||||
webcamSize: webcamInfo ? { width: webcamInfo.width, height: webcamInfo.height } : null,
|
||||
webcamLayoutPreset: this.config.webcamLayoutPreset,
|
||||
webcamMaskShape: this.config.webcamMaskShape,
|
||||
webcamSizePreset: this.config.webcamSizePreset,
|
||||
webcamPosition: this.config.webcamPosition,
|
||||
annotationRegions: this.config.annotationRegions,
|
||||
speedRegions: this.config.speedRegions,
|
||||
previewWidth: this.config.previewWidth,
|
||||
previewHeight: this.config.previewHeight,
|
||||
cursorTelemetry: this.config.cursorTelemetry,
|
||||
platform,
|
||||
});
|
||||
await this.renderer.initialize();
|
||||
|
||||
@@ -171,11 +179,11 @@ export class GifExporter {
|
||||
});
|
||||
|
||||
// Calculate effective duration and frame count (excluding trim regions)
|
||||
const effectiveDuration = this.streamingDecoder.getEffectiveDuration(
|
||||
const { effectiveDuration, totalFrames } = this.streamingDecoder.getExportMetrics(
|
||||
this.config.frameRate,
|
||||
this.config.trimRegions,
|
||||
this.config.speedRegions,
|
||||
);
|
||||
const totalFrames = Math.ceil(effectiveDuration * this.config.frameRate);
|
||||
|
||||
// Calculate frame delay in milliseconds (gif.js uses ms)
|
||||
const frameDelay = Math.round(1000 / this.config.frameRate);
|
||||
|
||||
@@ -1,5 +1,44 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { shouldFailDecodeEndedEarly } from "./streamingDecoder";
|
||||
import { shouldFailDecodeEndedEarly, validateDuration } from "./streamingDecoder";
|
||||
|
||||
describe("validateDuration", () => {
|
||||
it("returns scanned duration when container reports Infinity", () => {
|
||||
expect(validateDuration(Infinity, 15.3)).toBe(15.3);
|
||||
});
|
||||
|
||||
it("returns scanned duration when container reports 0", () => {
|
||||
expect(validateDuration(0, 15.3)).toBe(15.3);
|
||||
});
|
||||
|
||||
it("returns scanned duration when container reports NaN", () => {
|
||||
expect(validateDuration(NaN, 15.3)).toBe(15.3);
|
||||
});
|
||||
|
||||
it("returns scanned duration when container is inflated beyond threshold", () => {
|
||||
expect(validateDuration(42, 15.3)).toBe(15.3);
|
||||
});
|
||||
|
||||
it("returns container duration when values are close", () => {
|
||||
expect(validateDuration(15.5, 15.3)).toBe(15.5);
|
||||
});
|
||||
|
||||
it("returns container duration when scanned is slightly higher", () => {
|
||||
// container < scanned (scanned overshoot from last frame duration)
|
||||
expect(validateDuration(15.0, 15.3)).toBe(15.0);
|
||||
});
|
||||
|
||||
it("returns scanned duration when container under-reports beyond threshold", () => {
|
||||
expect(validateDuration(10, 15.3)).toBe(15.3);
|
||||
});
|
||||
|
||||
it("returns container duration when scanned is zero (corrupted/empty file)", () => {
|
||||
expect(validateDuration(10, 0)).toBe(10);
|
||||
});
|
||||
|
||||
it("returns 0 when both container is NaN and scanned is zero", () => {
|
||||
expect(validateDuration(NaN, 0)).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("shouldFailDecodeEndedEarly", () => {
|
||||
it("does not fail once every segment has been satisfied", () => {
|
||||
|
||||
@@ -2,6 +2,52 @@ import { WebDemuxer } from "web-demuxer";
|
||||
import type { SpeedRegion, TrimRegion } from "@/components/video-editor/types";
|
||||
|
||||
const SOURCE_LOAD_TIMEOUT_MS = 60_000;
|
||||
const EPSILON_SEC = 0.001;
|
||||
/**
|
||||
* Build a full WebCodecs-compatible AV1 codec string from the AV1CodecConfigurationRecord.
|
||||
* web-demuxer may return a bare "av01" when the WASM-side parser fails to read
|
||||
* the extradata (e.g. raw OBU sequence header from WebM instead of ISOBMFF av1C box).
|
||||
* This function parses the record if present, otherwise returns a safe default.
|
||||
*
|
||||
* @see https://aomediacodec.github.io/av1-isobmff/#av1codecconfigurationbox-section
|
||||
*/
|
||||
function buildAV1CodecString(description?: BufferSource): string {
|
||||
const fallback = "av01.0.01M.08";
|
||||
|
||||
if (!description) return fallback;
|
||||
|
||||
const bytes =
|
||||
description instanceof ArrayBuffer
|
||||
? new Uint8Array(description)
|
||||
: new Uint8Array(description.buffer, description.byteOffset, description.byteLength);
|
||||
|
||||
// AV1CodecConfigurationRecord layout (4+ bytes):
|
||||
// Byte 0: marker (1) | version (7)
|
||||
// Byte 1: seq_profile (3) | seq_level_idx_0 (5)
|
||||
// Byte 2: seq_tier_0 (1) | high_bitdepth (1) | twelve_bit (1) | ...
|
||||
// The spec says version should be 1, but Chrome/Electron's MediaRecorder
|
||||
// may write version 127 (0xFF first byte). We accept any version as long
|
||||
// as the marker bit is set and the record is long enough.
|
||||
if (bytes.length < 4) return fallback;
|
||||
if (!(bytes[0] & 0x80)) return fallback; // marker bit must be 1
|
||||
|
||||
// Byte 1: seq_profile (3) | seq_level_idx_0 (5)
|
||||
const profile = (bytes[1] >> 5) & 0x07;
|
||||
const level = bytes[1] & 0x1f;
|
||||
|
||||
// Byte 2: seq_tier_0 (1) | high_bitdepth (1) | twelve_bit (1) | monochrome (1) | ...
|
||||
const tier = (bytes[2] >> 7) & 0x01;
|
||||
const highBitdepth = (bytes[2] >> 6) & 0x01;
|
||||
const twelveBit = (bytes[2] >> 5) & 0x01;
|
||||
let bitdepth = 8;
|
||||
if (highBitdepth) bitdepth = twelveBit ? 12 : 10;
|
||||
|
||||
const tierChar = tier ? "H" : "M";
|
||||
const levelStr = level.toString().padStart(2, "0");
|
||||
const bitdepthStr = bitdepth.toString().padStart(2, "0");
|
||||
|
||||
return `av01.${profile}.${levelStr}${tierChar}.${bitdepthStr}`;
|
||||
}
|
||||
|
||||
export interface DecodedVideoInfo {
|
||||
width: number;
|
||||
@@ -24,6 +70,37 @@ type EarlyDecodeEndCheck = {
|
||||
const EARLY_DECODE_END_THRESHOLD_SEC = 1;
|
||||
const METADATA_TAIL_TOLERANCE_SEC = 1.5;
|
||||
const STREAM_DURATION_MATCH_TOLERANCE_SEC = 0.25;
|
||||
const DURATION_DIVERGENCE_THRESHOLD_SEC = 1.5;
|
||||
// Fallback upper bound for the packet scan when no reliable duration hint is
|
||||
// available. Explicit end is required (some containers are truncated without
|
||||
// one), but the hint-derived bound would cap the scan prematurely when
|
||||
// container/stream duration are missing or corrupt.
|
||||
const SCAN_UNBOUNDED_FALLBACK_SEC = 24 * 60 * 60;
|
||||
|
||||
/**
|
||||
* Validate container duration against actual packet timestamps.
|
||||
*
|
||||
* Chrome/Electron's MediaRecorder writes WebM containers with unreliable
|
||||
* Duration fields (often Infinity, 0, or inflated) — especially on Linux.
|
||||
* This function picks the most trustworthy duration value.
|
||||
*
|
||||
* @param containerDuration Duration from the container-level metadata
|
||||
* @param scannedDuration Duration derived from actual packet timestamps (ground truth)
|
||||
*/
|
||||
export function validateDuration(containerDuration: number, scannedDuration: number): number {
|
||||
if (scannedDuration <= 0) {
|
||||
// Zero scanned duration means corrupted/empty file — fall back to container
|
||||
// (downstream shouldFailDecodeEndedEarly will catch truly empty files)
|
||||
return Number.isFinite(containerDuration) ? Math.max(containerDuration, 0) : 0;
|
||||
}
|
||||
if (!Number.isFinite(containerDuration) || containerDuration <= 0) {
|
||||
return scannedDuration;
|
||||
}
|
||||
if (Math.abs(containerDuration - scannedDuration) > DURATION_DIVERGENCE_THRESHOLD_SEC) {
|
||||
return scannedDuration;
|
||||
}
|
||||
return containerDuration;
|
||||
}
|
||||
|
||||
export function shouldFailDecodeEndedEarly({
|
||||
cancelled,
|
||||
@@ -155,10 +232,43 @@ export class StreamingVideoDecoder {
|
||||
|
||||
const audioStream = mediaInfo.streams.find((s) => s.codec_type_string === "audio");
|
||||
|
||||
// Scan video packets to find the true content boundary.
|
||||
// MediaRecorder (especially on Linux) writes unreliable container durations.
|
||||
// Packet timestamps are ground truth — no decode needed, just timestamp reads.
|
||||
// Pass explicit range because some containers are truncated without one.
|
||||
// Sanitize because mediaInfo.duration can be NaN/Infinity (Chromium Linux bug),
|
||||
// which would propagate into demuxer.read() as an invalid endpoint.
|
||||
const containerDurationSec = Number.isFinite(mediaInfo.duration) ? mediaInfo.duration : 0;
|
||||
const streamDurationSec =
|
||||
typeof videoStream?.duration === "number" && Number.isFinite(videoStream.duration)
|
||||
? videoStream.duration
|
||||
: 0;
|
||||
const hintedDurationSec = Math.max(containerDurationSec, streamDurationSec, 0);
|
||||
const scanEndSec =
|
||||
hintedDurationSec > 0 ? hintedDurationSec + 0.5 : SCAN_UNBOUNDED_FALLBACK_SEC;
|
||||
let maxPacketEndUs = 0;
|
||||
const scanReader = this.demuxer.read("video", 0, scanEndSec).getReader();
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await scanReader.read();
|
||||
if (done || !value) break;
|
||||
const endUs = value.timestamp + (value.duration ?? 0);
|
||||
if (endUs > maxPacketEndUs) maxPacketEndUs = endUs;
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
await scanReader.cancel();
|
||||
} catch {
|
||||
/* already closed */
|
||||
}
|
||||
}
|
||||
const scannedDuration = maxPacketEndUs / 1_000_000;
|
||||
const validatedDuration = validateDuration(mediaInfo.duration, scannedDuration);
|
||||
|
||||
this.metadata = {
|
||||
width: videoStream?.width || 1920,
|
||||
height: videoStream?.height || 1080,
|
||||
duration: mediaInfo.duration,
|
||||
duration: validatedDuration,
|
||||
streamDuration:
|
||||
typeof videoStream?.duration === "number" && Number.isFinite(videoStream.duration)
|
||||
? videoStream.duration
|
||||
@@ -171,7 +281,15 @@ export class StreamingVideoDecoder {
|
||||
|
||||
return this.metadata;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decodes all video frames from the loaded source and invokes a callback for each.
|
||||
* Handles trimming and speed adjustments, and resamples to the target frame rate.
|
||||
* On Windows, early decode termination is tolerated to work around driver quirks.
|
||||
* @param targetFrameRate - Desired output frame rate.
|
||||
* @param trimRegions - Array of time regions to keep (others discarded).
|
||||
* @param speedRegions - Array of speed adjustments for specific time ranges.
|
||||
* @param onFrame - Async callback receiving each decoded VideoFrame.
|
||||
*/
|
||||
async decodeAll(
|
||||
targetFrameRate: number,
|
||||
trimRegions: TrimRegion[] | undefined,
|
||||
@@ -183,17 +301,30 @@ export class StreamingVideoDecoder {
|
||||
}
|
||||
|
||||
const decoderConfig = await this.demuxer.getDecoderConfig("video");
|
||||
const codec = this.metadata.codec.toLowerCase();
|
||||
|
||||
// web-demuxer may return a bare "av01" for AV1 in WebM containers when the
|
||||
// extradata isn't in the expected ISOBMFF format. WebCodecs requires the
|
||||
// full parametrized form (e.g. "av01.0.05M.08").
|
||||
if (/^av01$/i.test(decoderConfig.codec)) {
|
||||
decoderConfig.codec = buildAV1CodecString(
|
||||
decoderConfig.description as BufferSource | undefined,
|
||||
);
|
||||
}
|
||||
|
||||
const codec = decoderConfig.codec.toLowerCase();
|
||||
const shouldPreferSoftwareDecode = codec.includes("av01") || codec.includes("av1");
|
||||
const segments = this.splitBySpeed(
|
||||
this.computeSegments(this.metadata.duration, trimRegions),
|
||||
speedRegions,
|
||||
);
|
||||
const requiredEndSec = segments[segments.length - 1]?.endSec ?? 0;
|
||||
|
||||
const segmentOutputFrameCounts = segments.map((segment) =>
|
||||
Math.ceil(((segment.endSec - segment.startSec) / segment.speed) * targetFrameRate),
|
||||
Math.ceil(
|
||||
((segment.endSec - segment.startSec - EPSILON_SEC) / segment.speed) * targetFrameRate,
|
||||
),
|
||||
);
|
||||
const frameDurationUs = 1_000_000 / targetFrameRate;
|
||||
const epsilonSec = 0.001;
|
||||
|
||||
// Async frame queue — decoder pushes, consumer pulls
|
||||
const pendingFrames: VideoFrame[] = [];
|
||||
@@ -248,7 +379,7 @@ export class StreamingVideoDecoder {
|
||||
|
||||
// One forward stream through the whole file.
|
||||
// Pass explicit range because some containers are truncated when no end is provided.
|
||||
const readEndSec = Math.max(this.metadata.duration, this.metadata.streamDuration ?? 0) + 0.5;
|
||||
const readEndSec = this.metadata.duration + 0.5;
|
||||
const reader = this.demuxer.read("video", 0, readEndSec).getReader();
|
||||
|
||||
// Feed chunks to decoder in background with backpressure
|
||||
@@ -304,7 +435,7 @@ export class StreamingVideoDecoder {
|
||||
|
||||
const sourceTimeSec =
|
||||
segment.startSec + (segmentFrameIndex / targetFrameRate) * segment.speed;
|
||||
if (sourceTimeSec >= segment.endSec - epsilonSec) return false;
|
||||
if (sourceTimeSec >= segment.endSec - EPSILON_SEC) return false;
|
||||
|
||||
const clone = new VideoFrame(heldFrame, { timestamp: heldFrame.timestamp });
|
||||
await onFrame(clone, exportFrameIndex * frameDurationUs, sourceTimeSec * 1000);
|
||||
@@ -323,7 +454,7 @@ export class StreamingVideoDecoder {
|
||||
// Finalize completed segments before handling this frame.
|
||||
while (
|
||||
segmentIdx < segments.length &&
|
||||
frameTimeSec >= segments[segmentIdx].endSec - epsilonSec
|
||||
frameTimeSec >= segments[segmentIdx].endSec - EPSILON_SEC
|
||||
) {
|
||||
const segment = segments[segmentIdx];
|
||||
while (!this.cancelled && (await emitHeldFrameForTarget(segment))) {
|
||||
@@ -335,7 +466,7 @@ export class StreamingVideoDecoder {
|
||||
if (
|
||||
heldFrame &&
|
||||
segmentIdx < segments.length &&
|
||||
heldFrameSec < segments[segmentIdx].startSec - epsilonSec
|
||||
heldFrameSec < segments[segmentIdx].startSec - EPSILON_SEC
|
||||
) {
|
||||
heldFrame.close();
|
||||
heldFrame = null;
|
||||
@@ -350,7 +481,7 @@ export class StreamingVideoDecoder {
|
||||
const currentSegment = segments[segmentIdx];
|
||||
|
||||
// Before current segment (trimmed region or pre-roll).
|
||||
if (frameTimeSec < currentSegment.startSec - epsilonSec) {
|
||||
if (frameTimeSec < currentSegment.startSec - EPSILON_SEC) {
|
||||
frame.close();
|
||||
continue;
|
||||
}
|
||||
@@ -371,7 +502,7 @@ export class StreamingVideoDecoder {
|
||||
|
||||
const sourceTimeSec =
|
||||
currentSegment.startSec + (segmentFrameIndex / targetFrameRate) * currentSegment.speed;
|
||||
if (sourceTimeSec >= currentSegment.endSec - epsilonSec) {
|
||||
if (sourceTimeSec >= currentSegment.endSec - EPSILON_SEC) {
|
||||
break;
|
||||
}
|
||||
if (sourceTimeSec > handoffBoundarySec) {
|
||||
@@ -393,7 +524,7 @@ export class StreamingVideoDecoder {
|
||||
if (heldFrame && segmentIdx < segments.length) {
|
||||
while (!this.cancelled && segmentIdx < segments.length) {
|
||||
const segment = segments[segmentIdx];
|
||||
if (heldFrameSec < segment.startSec - epsilonSec) {
|
||||
if (heldFrameSec < segment.startSec - EPSILON_SEC) {
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -405,7 +536,7 @@ export class StreamingVideoDecoder {
|
||||
segmentFrameIndex = 0;
|
||||
if (
|
||||
segmentIdx < segments.length &&
|
||||
heldFrameSec < segments[segmentIdx].startSec - epsilonSec
|
||||
heldFrameSec < segments[segmentIdx].startSec - EPSILON_SEC
|
||||
) {
|
||||
break;
|
||||
}
|
||||
@@ -435,7 +566,8 @@ export class StreamingVideoDecoder {
|
||||
}
|
||||
this.decoder = null;
|
||||
|
||||
const requiredEndSec = segments.length > 0 ? segments[segments.length - 1].endSec : 0;
|
||||
const isWindows = typeof navigator !== "undefined" && /Windows/.test(navigator.userAgent);
|
||||
|
||||
if (
|
||||
shouldFailDecodeEndedEarly({
|
||||
cancelled: this.cancelled,
|
||||
@@ -446,9 +578,22 @@ export class StreamingVideoDecoder {
|
||||
) {
|
||||
const decodedAtLabel =
|
||||
lastDecodedFrameSec === null ? "no decoded frame" : `${lastDecodedFrameSec.toFixed(3)}s`;
|
||||
throw new Error(
|
||||
`Video decode ended early at ${decodedAtLabel} (needed ${requiredEndSec.toFixed(3)}s).`,
|
||||
);
|
||||
const decodeGapSec =
|
||||
lastDecodedFrameSec === null ? Infinity : requiredEndSec - lastDecodedFrameSec;
|
||||
|
||||
// On Windows, tolerate a small decode gap: up to 10% of required duration, capped at 3 seconds.
|
||||
const maxToleratedGap = Math.min(3.0, requiredEndSec * 0.1);
|
||||
|
||||
if (isWindows && lastDecodedFrameSec !== null && decodeGapSec <= maxToleratedGap) {
|
||||
console.warn(
|
||||
`[StreamingVideoDecoder] Decode ended early on Windows with a gap of ${decodeGapSec.toFixed(2)}s ` +
|
||||
`(max tolerated: ${maxToleratedGap.toFixed(2)}s) – proceeding anyway.`,
|
||||
);
|
||||
} else {
|
||||
throw new Error(
|
||||
`Video decode ended early at ${decodedAtLabel} (needed ${requiredEndSec.toFixed(3)}s).`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -480,11 +625,24 @@ export class StreamingVideoDecoder {
|
||||
return segments;
|
||||
}
|
||||
|
||||
getEffectiveDuration(trimRegions?: TrimRegion[], speedRegions?: SpeedRegion[]): number {
|
||||
getExportMetrics(
|
||||
targetFrameRate: number,
|
||||
trimRegions?: TrimRegion[],
|
||||
speedRegions?: SpeedRegion[],
|
||||
): { effectiveDuration: number; totalFrames: number } {
|
||||
if (!this.metadata) throw new Error("Must call loadMetadata() first");
|
||||
const trimSegments = this.computeSegments(this.metadata.duration, trimRegions);
|
||||
const speedSegments = this.splitBySpeed(trimSegments, speedRegions);
|
||||
return speedSegments.reduce((sum, seg) => sum + (seg.endSec - seg.startSec) / seg.speed, 0);
|
||||
const segments = this.splitBySpeed(trimSegments, speedRegions);
|
||||
return {
|
||||
effectiveDuration: segments.reduce(
|
||||
(sum, seg) => sum + (seg.endSec - seg.startSec) / seg.speed,
|
||||
0,
|
||||
),
|
||||
totalFrames: segments.reduce((sum, seg) => {
|
||||
const segDur = seg.endSec - seg.startSec - EPSILON_SEC;
|
||||
return sum + Math.max(0, Math.ceil((segDur / seg.speed) * targetFrameRate));
|
||||
}, 0),
|
||||
};
|
||||
}
|
||||
|
||||
private splitBySpeed(
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import sampleVideoUrl from "../../../tests/fixtures/sample.webm?url";
|
||||
import type { ExportProgress } from "./types";
|
||||
import { VideoExporter } from "./videoExporter";
|
||||
|
||||
describe("VideoExporter (real browser)", () => {
|
||||
it("exports a valid MP4 blob from a real video", async () => {
|
||||
const progressEvents: ExportProgress[] = [];
|
||||
|
||||
const exporter = new VideoExporter({
|
||||
videoUrl: sampleVideoUrl,
|
||||
width: 320,
|
||||
height: 180,
|
||||
frameRate: 15,
|
||||
bitrate: 1_000_000,
|
||||
wallpaper: "#1a1a2e",
|
||||
zoomRegions: [],
|
||||
showShadow: false,
|
||||
shadowIntensity: 0,
|
||||
showBlur: false,
|
||||
cropRegion: { x: 0, y: 0, width: 1, height: 1 },
|
||||
onProgress: (p) => progressEvents.push(p),
|
||||
});
|
||||
|
||||
const result = await exporter.export();
|
||||
|
||||
expect(result.success, result.error).toBe(true);
|
||||
expect(result.blob).toBeInstanceOf(Blob);
|
||||
|
||||
const buf = await result.blob!.arrayBuffer();
|
||||
const bytes = new Uint8Array(buf);
|
||||
const ftyp = new TextDecoder().decode(bytes.slice(4, 8));
|
||||
expect(ftyp).toBe("ftyp");
|
||||
|
||||
expect(result.blob!.size).toBeGreaterThan(1024);
|
||||
|
||||
expect(progressEvents.length).toBeGreaterThan(0);
|
||||
|
||||
const finalizing = progressEvents.filter((p) => p.phase === "finalizing");
|
||||
expect(finalizing.length).toBeGreaterThan(0);
|
||||
expect(finalizing.at(-1)!.percentage).toBe(100);
|
||||
});
|
||||
});
|
||||
@@ -4,8 +4,10 @@ import type {
|
||||
SpeedRegion,
|
||||
TrimRegion,
|
||||
WebcamLayoutPreset,
|
||||
WebcamSizePreset,
|
||||
ZoomRegion,
|
||||
} from "@/components/video-editor/types";
|
||||
import { getPlatform } from "@/utils/platformUtils";
|
||||
import { AsyncVideoFrameQueue } from "./asyncVideoFrameQueue";
|
||||
import { AudioProcessor } from "./audioEncoder";
|
||||
import { FrameRenderer } from "./frameRenderer";
|
||||
@@ -33,6 +35,7 @@ interface VideoExporterConfig extends ExportConfig {
|
||||
cropRegion: CropRegion;
|
||||
webcamLayoutPreset?: WebcamLayoutPreset;
|
||||
webcamMaskShape?: import("@/components/video-editor/types").WebcamMaskShape;
|
||||
webcamSizePreset?: WebcamSizePreset;
|
||||
webcamPosition?: { cx: number; cy: number } | null;
|
||||
annotationRegions?: AnnotationRegion[];
|
||||
previewWidth?: number;
|
||||
@@ -110,6 +113,8 @@ export class VideoExporter {
|
||||
this.fatalEncoderError = null;
|
||||
|
||||
try {
|
||||
const platform = await getPlatform();
|
||||
|
||||
const streamingDecoder = new StreamingVideoDecoder();
|
||||
this.streamingDecoder = streamingDecoder;
|
||||
const videoInfo = await streamingDecoder.loadMetadata(this.config.videoUrl);
|
||||
@@ -137,12 +142,14 @@ export class VideoExporter {
|
||||
webcamSize: webcamInfo ? { width: webcamInfo.width, height: webcamInfo.height } : null,
|
||||
webcamLayoutPreset: this.config.webcamLayoutPreset,
|
||||
webcamMaskShape: this.config.webcamMaskShape,
|
||||
webcamSizePreset: this.config.webcamSizePreset,
|
||||
webcamPosition: this.config.webcamPosition,
|
||||
annotationRegions: this.config.annotationRegions,
|
||||
speedRegions: this.config.speedRegions,
|
||||
previewWidth: this.config.previewWidth,
|
||||
previewHeight: this.config.previewHeight,
|
||||
cursorTelemetry: this.config.cursorTelemetry,
|
||||
platform,
|
||||
});
|
||||
this.renderer = renderer;
|
||||
await renderer.initialize();
|
||||
@@ -154,17 +161,11 @@ export class VideoExporter {
|
||||
this.muxer = muxer;
|
||||
await muxer.initialize();
|
||||
|
||||
const effectiveDuration = streamingDecoder.getEffectiveDuration(
|
||||
const { totalFrames } = streamingDecoder.getExportMetrics(
|
||||
this.config.frameRate,
|
||||
this.config.trimRegions,
|
||||
this.config.speedRegions,
|
||||
);
|
||||
const totalFrames = Math.ceil(effectiveDuration * this.config.frameRate);
|
||||
const readEndSec = Math.max(videoInfo.duration, videoInfo.streamDuration ?? 0) + 0.5;
|
||||
|
||||
console.log("[VideoExporter] Original duration:", videoInfo.duration, "s");
|
||||
console.log("[VideoExporter] Effective duration:", effectiveDuration, "s");
|
||||
console.log("[VideoExporter] Total frames to export:", totalFrames);
|
||||
console.log("[VideoExporter] Using streaming decode (web-demuxer + VideoDecoder)");
|
||||
|
||||
const frameDuration = 1_000_000 / this.config.frameRate;
|
||||
let frameIndex = 0;
|
||||
@@ -234,25 +235,29 @@ export class VideoExporter {
|
||||
|
||||
const canvas = renderer.getCanvas();
|
||||
|
||||
// Read raw pixels from the canvas instead of passing
|
||||
// the canvas directly to VideoFrame. On some Linux
|
||||
// systems the GPU shared-image path (EGL/Ozone) fails
|
||||
// silently, producing empty frames.
|
||||
const canvasCtx = canvas.getContext("2d")!;
|
||||
const imageData = canvasCtx.getImageData(0, 0, canvas.width, canvas.height);
|
||||
const exportFrame = new VideoFrame(imageData.data.buffer, {
|
||||
format: "RGBA",
|
||||
codedWidth: canvas.width,
|
||||
codedHeight: canvas.height,
|
||||
timestamp,
|
||||
duration: frameDuration,
|
||||
colorSpace: {
|
||||
primaries: "bt709",
|
||||
transfer: "iec61966-2-1",
|
||||
matrix: "rgb",
|
||||
fullRange: true,
|
||||
},
|
||||
});
|
||||
let exportFrame: VideoFrame;
|
||||
|
||||
// On some Linux systems the GPU shared-image path (EGL/Ozone) fails
|
||||
// silently, producing empty frames, so we force a CPU readback instead.
|
||||
if (platform === "linux") {
|
||||
const canvasCtx = canvas.getContext("2d")!;
|
||||
const imageData = canvasCtx.getImageData(0, 0, canvas.width, canvas.height);
|
||||
exportFrame = new VideoFrame(imageData.data.buffer, {
|
||||
format: "RGBA",
|
||||
codedWidth: canvas.width,
|
||||
codedHeight: canvas.height,
|
||||
timestamp,
|
||||
duration: frameDuration,
|
||||
colorSpace: {
|
||||
primaries: "bt709",
|
||||
transfer: "iec61966-2-1",
|
||||
matrix: "rgb",
|
||||
fullRange: true,
|
||||
},
|
||||
});
|
||||
} else {
|
||||
exportFrame = new VideoFrame(canvas, { timestamp, duration: frameDuration });
|
||||
}
|
||||
|
||||
while (
|
||||
this.encoder &&
|
||||
@@ -343,7 +348,7 @@ export class VideoExporter {
|
||||
this.config.videoUrl,
|
||||
this.config.trimRegions,
|
||||
this.config.speedRegions,
|
||||
readEndSec,
|
||||
videoInfo.duration,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -419,7 +424,7 @@ export class VideoExporter {
|
||||
})();
|
||||
|
||||
this.muxingPromises.push(muxingPromise);
|
||||
this.encodeQueue--;
|
||||
this.encodeQueue = Math.max(0, this.encodeQueue - 1);
|
||||
},
|
||||
error: (error) => {
|
||||
console.error("[VideoExporter] Encoder error:", error);
|
||||
|
||||
@@ -17,9 +17,7 @@ export async function requestCameraAccess(): Promise<CameraAccessResult> {
|
||||
if (window.electronAPI?.requestCameraAccess) {
|
||||
try {
|
||||
const electronResult = await window.electronAPI.requestCameraAccess();
|
||||
if (!electronResult.success || !electronResult.granted) {
|
||||
return electronResult;
|
||||
}
|
||||
return electronResult;
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
|
||||
@@ -3,6 +3,7 @@ export const SHORTCUT_ACTIONS = [
|
||||
"addTrim",
|
||||
"addSpeed",
|
||||
"addAnnotation",
|
||||
"addBlur",
|
||||
"addKeyframe",
|
||||
"deleteSelected",
|
||||
"playPause",
|
||||
@@ -108,6 +109,7 @@ export const DEFAULT_SHORTCUTS: ShortcutsConfig = {
|
||||
addTrim: { key: "t" },
|
||||
addSpeed: { key: "s" },
|
||||
addAnnotation: { key: "a" },
|
||||
addBlur: { key: "b" },
|
||||
addKeyframe: { key: "f" },
|
||||
deleteSelected: { key: "d", ctrl: true },
|
||||
playPause: { key: " " },
|
||||
@@ -118,6 +120,7 @@ export const SHORTCUT_LABELS: Record<ShortcutAction, string> = {
|
||||
addTrim: "Add Trim",
|
||||
addSpeed: "Add Speed",
|
||||
addAnnotation: "Add Annotation",
|
||||
addBlur: "Add Blur",
|
||||
addKeyframe: "Add Keyframe",
|
||||
deleteSelected: "Delete Selected",
|
||||
playPause: "Play / Pause",
|
||||
@@ -125,9 +128,10 @@ export const SHORTCUT_LABELS: Record<ShortcutAction, string> = {
|
||||
|
||||
export function matchesShortcut(
|
||||
e: KeyboardEvent,
|
||||
binding: ShortcutBinding,
|
||||
binding: ShortcutBinding | undefined,
|
||||
isMacPlatform: boolean,
|
||||
): boolean {
|
||||
if (!binding) return false;
|
||||
if (e.key.toLowerCase() !== binding.key.toLowerCase()) return false;
|
||||
|
||||
const primaryMod = isMacPlatform ? e.metaKey : e.ctrlKey;
|
||||
|
||||
Reference in New Issue
Block a user