3d iso,tilt

This commit is contained in:
Siddharth
2026-05-03 17:54:21 -07:00
parent 6fc19314dd
commit 190d5d8ecb
9 changed files with 979 additions and 222 deletions
+41 -1
View File
@@ -54,13 +54,19 @@ import type {
CropRegion,
FigureData,
PlaybackSpeed,
Rotation3DPreset,
WebcamLayoutPreset,
WebcamMaskShape,
WebcamSizePreset,
ZoomDepth,
ZoomFocusMode,
} from "./types";
import { DEFAULT_WEBCAM_SIZE_PRESET, MAX_PLAYBACK_SPEED, SPEED_OPTIONS } from "./types";
import {
DEFAULT_WEBCAM_SIZE_PRESET,
MAX_PLAYBACK_SPEED,
ROTATION_3D_PRESET_ORDER,
SPEED_OPTIONS,
} from "./types";
function CustomSpeedInput({
value,
@@ -168,6 +174,8 @@ interface SettingsPanelProps {
hasCursorTelemetry?: boolean;
selectedZoomId?: string | null;
onZoomDelete?: (id: string) => void;
selectedZoomRotationPreset?: Rotation3DPreset | null;
onZoomRotationPresetChange?: (preset: Rotation3DPreset | null) => void;
selectedTrimId?: string | null;
onTrimDelete?: (id: string) => void;
shadowIntensity?: number;
@@ -258,6 +266,8 @@ export function SettingsPanel({
hasCursorTelemetry = false,
selectedZoomId,
onZoomDelete,
selectedZoomRotationPreset,
onZoomRotationPresetChange,
selectedTrimId,
onTrimDelete,
shadowIntensity = 0,
@@ -647,6 +657,36 @@ export function SettingsPanel({
)}
</div>
)}
{zoomEnabled && (
<div className="mt-4">
<span className="text-sm font-medium text-slate-200 mb-2 block">
{t("zoom.threeD.title")}
</span>
<div className="grid grid-cols-3 gap-1.5">
{ROTATION_3D_PRESET_ORDER.map((preset) => {
const isActive = selectedZoomRotationPreset === preset;
return (
<Button
key={preset}
type="button"
onClick={() => onZoomRotationPresetChange?.(isActive ? null : preset)}
className={cn(
"h-auto w-full rounded-lg border px-1 py-2 text-center shadow-sm transition-all duration-200 ease-out cursor-pointer",
isActive
? "border-[#34B27B] bg-[#34B27B] text-white shadow-[#34B27B]/20"
: "border-white/5 bg-white/5 text-slate-400 hover:bg-white/10 hover:border-white/10 hover:text-slate-200",
)}
>
<span className="text-xs font-semibold capitalize">
{t(`zoom.threeD.preset.${preset}`)}
</span>
</Button>
);
})}
</div>
</div>
)}
{zoomEnabled && (
<Button
onClick={handleDeleteClick}
@@ -67,6 +67,7 @@ import {
DEFAULT_ZOOM_DEPTH,
type FigureData,
type PlaybackSpeed,
type Rotation3DPreset,
type SpeedRegion,
type TrimRegion,
type ZoomDepth,
@@ -837,6 +838,23 @@ export default function VideoEditor() {
[selectedZoomId, pushState],
);
const handleZoomRotationPresetChange = useCallback(
(preset: Rotation3DPreset | null) => {
if (!selectedZoomId) return;
pushState((prev) => ({
zoomRegions: prev.zoomRegions.map((region) => {
if (region.id !== selectedZoomId) return region;
if (preset === null) {
const { rotationPreset: _p, ...rest } = region;
return rest;
}
return { ...region, rotationPreset: preset };
}),
}));
},
[selectedZoomId, pushState],
);
const handleTrimDelete = useCallback(
(id: string) => {
pushState((prev) => ({
@@ -1996,6 +2014,12 @@ export default function VideoEditor() {
hasCursorTelemetry={cursorTelemetry.length > 0}
selectedZoomId={selectedZoomId}
onZoomDelete={handleZoomDelete}
selectedZoomRotationPreset={
selectedZoomId
? (zoomRegions.find((z) => z.id === selectedZoomId)?.rotationPreset ?? null)
: null
}
onZoomRotationPresetChange={handleZoomRotationPresetChange}
selectedTrimId={selectedTrimId}
onTrimDelete={handleTrimDelete}
shadowIntensity={shadowIntensity}
+242 -179
View File
@@ -36,6 +36,11 @@ import { AnnotationOverlay } from "./AnnotationOverlay";
import {
type AnnotationRegion,
type BlurData,
computeRotation3DContainScale,
DEFAULT_ROTATION_3D,
isRotation3DIdentity,
lerpRotation3D,
rotation3DPerspective,
type SpeedRegion,
type TrimRegion,
ZOOM_DEPTH_SCALES,
@@ -200,6 +205,8 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
const overlayRef = useRef<HTMLDivElement | null>(null);
const focusIndicatorRef = useRef<HTMLDivElement | null>(null);
const composite3DRef = useRef<HTMLDivElement | null>(null);
const outerWrapperRef = useRef<HTMLDivElement | null>(null);
const [webcamLayout, setWebcamLayout] = useState<StyledRenderRect | null>(null);
const [webcamDimensions, setWebcamDimensions] = useState<Size | null>(null);
const currentTimeRef = useRef(0);
@@ -921,8 +928,10 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
};
let lastMotionBlurActive: boolean | null = null;
let lastTransformIsIdentity = true;
let lastPerspectiveValue = 0;
const ticker = () => {
const { region, strength, blendedScale, transition } = findDominantRegion(
const { region, strength, blendedScale, rotation3D, transition } = findDominantRegion(
zoomRegionsRef.current,
currentTimeRef.current,
{
@@ -1129,6 +1138,44 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
lastMotionBlurActive = false;
}
}
const composite3D = composite3DRef.current;
const outerWrapper = outerWrapperRef.current;
if (composite3D && outerWrapper) {
const effectiveRotation =
region && targetProgress > 0 && !shouldShowUnzoomedView
? lerpRotation3D(DEFAULT_ROTATION_3D, rotation3D, targetProgress)
: DEFAULT_ROTATION_3D;
const isIdentity = isRotation3DIdentity(effectiveRotation);
if (isIdentity) {
if (!lastTransformIsIdentity) {
composite3D.style.transform = "";
composite3D.style.willChange = "auto";
lastTransformIsIdentity = true;
}
if (lastPerspectiveValue !== 0) {
outerWrapper.style.perspective = "";
lastPerspectiveValue = 0;
}
} else {
const wrapperW = outerWrapper.clientWidth || 1;
const wrapperH = outerWrapper.clientHeight || 1;
const persp = rotation3DPerspective(wrapperW, wrapperH);
const containScale = computeRotation3DContainScale(
effectiveRotation,
wrapperW,
wrapperH,
persp,
);
composite3D.style.transform = `scale(${containScale}) rotateX(${effectiveRotation.rotationX}deg) rotateY(${effectiveRotation.rotationY}deg) rotateZ(${effectiveRotation.rotationZ}deg)`;
composite3D.style.willChange = "transform";
lastTransformIsIdentity = false;
if (persp !== lastPerspectiveValue) {
outerWrapper.style.perspective = `${persp}px`;
lastPerspectiveValue = persp;
}
}
}
};
app.ticker.add(ticker);
@@ -1270,6 +1317,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
return (
<div
ref={outerWrapperRef}
className="relative rounded-sm overflow-hidden"
style={{
width: "100%",
@@ -1294,189 +1342,204 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
}}
/>
<div
ref={containerRef}
ref={composite3DRef}
className="absolute inset-0"
style={{
filter:
showShadow && shadowIntensity > 0
? `drop-shadow(0 ${shadowIntensity * 12}px ${shadowIntensity * 48}px rgba(0,0,0,${shadowIntensity * 0.7})) drop-shadow(0 ${shadowIntensity * 4}px ${shadowIntensity * 16}px rgba(0,0,0,${shadowIntensity * 0.5})) drop-shadow(0 ${shadowIntensity * 2}px ${shadowIntensity * 8}px rgba(0,0,0,${shadowIntensity * 0.3}))`
: "none",
transformStyle: "preserve-3d",
transformOrigin: "center center",
}}
/>
{webcamVideoPath &&
(() => {
const clipPath = getCssClipPath(webcamLayout?.maskShape ?? "rectangle");
const useClipPath = !!clipPath;
return (
<div
className="absolute"
style={{
left: webcamLayout?.x ?? 0,
top: webcamLayout?.y ?? 0,
width: webcamLayout?.width ?? 0,
height: webcamLayout?.height ?? 0,
zIndex: 20,
opacity: webcamLayout ? 1 : 0,
filter:
useClipPath && webcamCssBoxShadow !== "none"
? `drop-shadow(${webcamCssBoxShadow})`
: undefined,
}}
>
<video
ref={webcamVideoRef}
src={webcamVideoPath}
className={`w-full h-full object-cover ${webcamLayoutPreset === "picture-in-picture" ? "cursor-grab active:cursor-grabbing" : "pointer-events-none"}`}
style={{
borderRadius: useClipPath ? 0 : (webcamLayout?.borderRadius ?? 0),
clipPath: clipPath ?? undefined,
boxShadow: useClipPath ? "none" : webcamCssBoxShadow,
backgroundColor: "#000",
}}
onPointerDown={handleWebcamPointerDown}
onPointerMove={handleWebcamPointerMove}
onPointerUp={handleWebcamPointerUp}
onPointerLeave={handleWebcamPointerUp}
muted
preload="metadata"
playsInline
/>
</div>
);
})()}
{/* Only render overlay after PIXI and video are fully initialized */}
{pixiReady && videoReady && (
>
<div
ref={setOverlayRefs}
className="absolute inset-0 select-none"
style={{ pointerEvents: "auto", zIndex: 30 }}
onPointerDown={handleOverlayPointerDown}
onPointerMove={handleOverlayPointerMove}
onPointerUp={handleOverlayPointerUp}
onPointerLeave={handleOverlayPointerLeave}
>
<div
ref={focusIndicatorRef}
className="absolute rounded-md border border-[#34B27B]/80 bg-[#34B27B]/20 shadow-[0_0_0_1px_rgba(52,178,123,0.35)]"
style={{ display: "none", pointerEvents: "none" }}
/>
{(() => {
const filteredAnnotations = (annotationRegions || []).filter((annotation) => {
if (typeof annotation.startMs !== "number" || typeof annotation.endMs !== "number")
return false;
if (annotation.id === selectedAnnotationId) return true;
const timeMs = Math.round(currentTime * 1000);
return timeMs >= annotation.startMs && timeMs < annotation.endMs;
});
const filteredBlurRegions = (blurRegions || []).filter((blurRegion) => {
if (typeof blurRegion.startMs !== "number" || typeof blurRegion.endMs !== "number")
return false;
if (blurRegion.id === selectedBlurId) return true;
const timeMs = Math.round(currentTime * 1000);
return timeMs >= blurRegion.startMs && timeMs < blurRegion.endMs;
});
const sorted = [
...filteredAnnotations.map((annotation) => ({
kind: "annotation" as const,
region: annotation,
})),
...filteredBlurRegions.map((blurRegion) => ({
kind: "blur" as const,
region: blurRegion,
})),
].sort((a, b) => a.region.zIndex - b.region.zIndex);
const previewSnapshotCanvas =
filteredBlurRegions.length > 0
? (() => {
const app = appRef.current;
if (!app?.renderer?.extract) return null;
try {
return app.renderer.extract.canvas(app.stage);
} catch {
return null;
}
})()
: null;
// Handle click-through cycling: when clicking same annotation, cycle to next
const handleAnnotationClick = (clickedId: string) => {
if (!onSelectAnnotation) return;
// If clicking on already selected annotation and there are multiple overlapping
if (clickedId === selectedAnnotationId && filteredAnnotations.length > 1) {
// Find current index and cycle to next
const currentIndex = filteredAnnotations.findIndex((a) => a.id === clickedId);
const nextIndex = (currentIndex + 1) % filteredAnnotations.length;
onSelectAnnotation(filteredAnnotations[nextIndex].id);
} else {
// First click or clicking different annotation
onSelectAnnotation(clickedId);
}
};
const handleBlurClick = (clickedId: string) => {
if (!onSelectBlur) return;
if (clickedId === selectedBlurId && filteredBlurRegions.length > 1) {
const currentIndex = filteredBlurRegions.findIndex((a) => a.id === clickedId);
const nextIndex = (currentIndex + 1) % filteredBlurRegions.length;
onSelectBlur(filteredBlurRegions[nextIndex].id);
} else {
onSelectBlur(clickedId);
}
};
return sorted.map((item) => (
<AnnotationOverlay
key={
item.kind === "blur"
? `${item.region.id}-${overlaySize.width}-${overlaySize.height}-${item.region.blurData?.type ?? "blur"}-${item.region.blurData?.shape ?? "rectangle"}-${item.region.blurData?.color ?? "white"}-${Math.round(item.region.blurData?.blockSize ?? 0)}-${Math.round(item.region.blurData?.intensity ?? 0)}-${(item.region.blurData?.freehandPoints ?? []).map((p) => `${Math.round(p.x)}_${Math.round(p.y)}`).join("-")}`
: `${item.region.id}-${overlaySize.width}-${overlaySize.height}`
}
annotation={item.region}
isSelected={
item.kind === "blur"
? item.region.id === selectedBlurId
: item.region.id === selectedAnnotationId
}
containerWidth={overlaySize.width}
containerHeight={overlaySize.height}
onPositionChange={(id, position) =>
item.kind === "blur"
? onBlurPositionChange?.(id, position)
: onAnnotationPositionChange?.(id, position)
}
onSizeChange={(id, size) =>
item.kind === "blur"
? onBlurSizeChange?.(id, size)
: onAnnotationSizeChange?.(id, size)
}
onBlurDataChange={
item.kind === "blur"
? (id, blurData) => onBlurDataChange?.(id, blurData)
: undefined
}
onBlurDataCommit={item.kind === "blur" ? onBlurDataCommit : undefined}
onClick={item.kind === "blur" ? handleBlurClick : handleAnnotationClick}
zIndex={item.region.zIndex}
isSelectedBoost={
item.kind === "blur"
? item.region.id === selectedBlurId
: item.region.id === selectedAnnotationId
}
previewSourceCanvas={previewSnapshotCanvas}
previewFrameVersion={Math.round(currentTime * 1000)}
/>
));
ref={containerRef}
className="absolute inset-0"
style={{
filter:
showShadow && shadowIntensity > 0
? `drop-shadow(0 ${shadowIntensity * 12}px ${shadowIntensity * 48}px rgba(0,0,0,${shadowIntensity * 0.7})) drop-shadow(0 ${shadowIntensity * 4}px ${shadowIntensity * 16}px rgba(0,0,0,${shadowIntensity * 0.5})) drop-shadow(0 ${shadowIntensity * 2}px ${shadowIntensity * 8}px rgba(0,0,0,${shadowIntensity * 0.3}))`
: "none",
}}
/>
{webcamVideoPath &&
(() => {
const clipPath = getCssClipPath(webcamLayout?.maskShape ?? "rectangle");
const useClipPath = !!clipPath;
return (
<div
className="absolute"
style={{
left: webcamLayout?.x ?? 0,
top: webcamLayout?.y ?? 0,
width: webcamLayout?.width ?? 0,
height: webcamLayout?.height ?? 0,
zIndex: 20,
opacity: webcamLayout ? 1 : 0,
filter:
useClipPath && webcamCssBoxShadow !== "none"
? `drop-shadow(${webcamCssBoxShadow})`
: undefined,
}}
>
<video
ref={webcamVideoRef}
src={webcamVideoPath}
className={`w-full h-full object-cover ${webcamLayoutPreset === "picture-in-picture" ? "cursor-grab active:cursor-grabbing" : "pointer-events-none"}`}
style={{
borderRadius: useClipPath ? 0 : (webcamLayout?.borderRadius ?? 0),
clipPath: clipPath ?? undefined,
boxShadow: useClipPath ? "none" : webcamCssBoxShadow,
backgroundColor: "#000",
}}
onPointerDown={handleWebcamPointerDown}
onPointerMove={handleWebcamPointerMove}
onPointerUp={handleWebcamPointerUp}
onPointerLeave={handleWebcamPointerUp}
muted
preload="metadata"
playsInline
/>
</div>
);
})()}
</div>
)}
{/* Only render overlay after PIXI and video are fully initialized */}
{pixiReady && videoReady && (
<div
ref={setOverlayRefs}
className="absolute inset-0 select-none"
style={{ pointerEvents: "auto", zIndex: 30 }}
onPointerDown={handleOverlayPointerDown}
onPointerMove={handleOverlayPointerMove}
onPointerUp={handleOverlayPointerUp}
onPointerLeave={handleOverlayPointerLeave}
>
<div
ref={focusIndicatorRef}
className="absolute rounded-md border border-[#34B27B]/80 bg-[#34B27B]/20 shadow-[0_0_0_1px_rgba(52,178,123,0.35)]"
style={{ display: "none", pointerEvents: "none" }}
/>
{(() => {
const filteredAnnotations = (annotationRegions || []).filter((annotation) => {
if (
typeof annotation.startMs !== "number" ||
typeof annotation.endMs !== "number"
)
return false;
if (annotation.id === selectedAnnotationId) return true;
const timeMs = Math.round(currentTime * 1000);
return timeMs >= annotation.startMs && timeMs < annotation.endMs;
});
const filteredBlurRegions = (blurRegions || []).filter((blurRegion) => {
if (
typeof blurRegion.startMs !== "number" ||
typeof blurRegion.endMs !== "number"
)
return false;
if (blurRegion.id === selectedBlurId) return true;
const timeMs = Math.round(currentTime * 1000);
return timeMs >= blurRegion.startMs && timeMs < blurRegion.endMs;
});
const sorted = [
...filteredAnnotations.map((annotation) => ({
kind: "annotation" as const,
region: annotation,
})),
...filteredBlurRegions.map((blurRegion) => ({
kind: "blur" as const,
region: blurRegion,
})),
].sort((a, b) => a.region.zIndex - b.region.zIndex);
const previewSnapshotCanvas =
filteredBlurRegions.length > 0
? (() => {
const app = appRef.current;
if (!app?.renderer?.extract) return null;
try {
return app.renderer.extract.canvas(app.stage);
} catch {
return null;
}
})()
: null;
// Handle click-through cycling: when clicking same annotation, cycle to next
const handleAnnotationClick = (clickedId: string) => {
if (!onSelectAnnotation) return;
// If clicking on already selected annotation and there are multiple overlapping
if (clickedId === selectedAnnotationId && filteredAnnotations.length > 1) {
// Find current index and cycle to next
const currentIndex = filteredAnnotations.findIndex((a) => a.id === clickedId);
const nextIndex = (currentIndex + 1) % filteredAnnotations.length;
onSelectAnnotation(filteredAnnotations[nextIndex].id);
} else {
// First click or clicking different annotation
onSelectAnnotation(clickedId);
}
};
const handleBlurClick = (clickedId: string) => {
if (!onSelectBlur) return;
if (clickedId === selectedBlurId && filteredBlurRegions.length > 1) {
const currentIndex = filteredBlurRegions.findIndex((a) => a.id === clickedId);
const nextIndex = (currentIndex + 1) % filteredBlurRegions.length;
onSelectBlur(filteredBlurRegions[nextIndex].id);
} else {
onSelectBlur(clickedId);
}
};
return sorted.map((item) => (
<AnnotationOverlay
key={
item.kind === "blur"
? `${item.region.id}-${overlaySize.width}-${overlaySize.height}-${item.region.blurData?.type ?? "blur"}-${item.region.blurData?.shape ?? "rectangle"}-${item.region.blurData?.color ?? "white"}-${Math.round(item.region.blurData?.blockSize ?? 0)}-${Math.round(item.region.blurData?.intensity ?? 0)}-${(item.region.blurData?.freehandPoints ?? []).map((p) => `${Math.round(p.x)}_${Math.round(p.y)}`).join("-")}`
: `${item.region.id}-${overlaySize.width}-${overlaySize.height}`
}
annotation={item.region}
isSelected={
item.kind === "blur"
? item.region.id === selectedBlurId
: item.region.id === selectedAnnotationId
}
containerWidth={overlaySize.width}
containerHeight={overlaySize.height}
onPositionChange={(id, position) =>
item.kind === "blur"
? onBlurPositionChange?.(id, position)
: onAnnotationPositionChange?.(id, position)
}
onSizeChange={(id, size) =>
item.kind === "blur"
? onBlurSizeChange?.(id, size)
: onAnnotationSizeChange?.(id, size)
}
onBlurDataChange={
item.kind === "blur"
? (id, blurData) => onBlurDataChange?.(id, blurData)
: undefined
}
onBlurDataCommit={item.kind === "blur" ? onBlurDataCommit : undefined}
onClick={item.kind === "blur" ? handleBlurClick : handleAnnotationClick}
zIndex={item.region.zIndex}
isSelectedBoost={
item.kind === "blur"
? item.region.id === selectedBlurId
: item.region.id === selectedAnnotationId
}
previewSourceCanvas={previewSnapshotCanvas}
previewFrameVersion={Math.round(currentTime * 1000)}
/>
));
})()}
</div>
)}
</div>
<video
ref={videoRef}
src={videoPath}
@@ -251,6 +251,12 @@ export function normalizeProjectEditor(editor: Partial<ProjectEditorState>): Pro
const startMs = Math.max(0, Math.min(rawStart, rawEnd));
const endMs = Math.max(startMs + 1, rawEnd);
const validPreset =
region.rotationPreset === "iso" ||
region.rotationPreset === "left" ||
region.rotationPreset === "right"
? region.rotationPreset
: undefined;
return {
id: region.id,
startMs,
@@ -261,6 +267,7 @@ export function normalizeProjectEditor(editor: Partial<ProjectEditorState>): Pro
cy: clamp(isFiniteNumber(region.focus?.cy) ? region.focus.cy : 0.5, 0, 1),
},
focusMode: region.focusMode === "auto" ? "auto" : "manual",
...(validPreset ? { rotationPreset: validPreset } : {}),
};
})
: [];
+129
View File
@@ -26,6 +26,37 @@ export interface ZoomFocus {
cy: number; // normalized vertical center (0-1)
}
export interface Rotation3D {
rotationX: number;
rotationY: number;
rotationZ: number;
}
export const DEFAULT_ROTATION_3D: Rotation3D = {
rotationX: 0,
rotationY: 0,
rotationZ: 0,
};
export type Rotation3DPreset = "iso" | "left" | "right";
export const ROTATION_3D_PRESETS: Record<Rotation3DPreset, Rotation3D> = {
iso: { rotationX: -10, rotationY: -16, rotationZ: 0 },
left: { rotationX: 0, rotationY: -22, rotationZ: 0 },
right: { rotationX: 0, rotationY: 22, rotationZ: 0 },
};
export const ROTATION_3D_PRESET_ORDER: Rotation3DPreset[] = ["iso", "left", "right"];
/** Perspective distance in CSS px is computed at render-time as this factor times
* min(viewport width, viewport height). Same factor used in preview and export so
* the visual look is identical regardless of canvas resolution. */
export const ROTATION_3D_PERSPECTIVE_FACTOR = 2.6;
export function rotation3DPerspective(width: number, height: number): number {
return Math.min(width, height) * ROTATION_3D_PERSPECTIVE_FACTOR;
}
export interface ZoomRegion {
id: string;
startMs: number;
@@ -33,6 +64,104 @@ export interface ZoomRegion {
depth: ZoomDepth;
focus: ZoomFocus;
focusMode?: ZoomFocusMode;
rotationPreset?: Rotation3DPreset;
}
export function getRotation3D(region: Pick<ZoomRegion, "rotationPreset">): Rotation3D {
if (!region.rotationPreset) return DEFAULT_ROTATION_3D;
return ROTATION_3D_PRESETS[region.rotationPreset];
}
export function isRotation3DIdentity(r: Rotation3D, eps = 0.01): boolean {
return Math.abs(r.rotationX) < eps && Math.abs(r.rotationY) < eps && Math.abs(r.rotationZ) < eps;
}
export function lerpRotation3D(a: Rotation3D, b: Rotation3D, t: number): Rotation3D {
return {
rotationX: a.rotationX + (b.rotationX - a.rotationX) * t,
rotationY: a.rotationY + (b.rotationY - a.rotationY) * t,
rotationZ: a.rotationZ + (b.rotationZ - a.rotationZ) * t,
};
}
/**
* Compute the maximum uniform scale that, when applied alongside `rot` and a perspective
* of `perspective` CSS px, keeps the projected bounding box of a `width × height` element
* inside its original `width × height` rectangle. Returns 1 when no scaling is needed.
*
* Math: project each rotated corner onto the screen via x' = x·P/(Pz); take the worst-case
* |x'|/|y'| against the half-extents and return the limiting ratio. This makes the rotated
* recording sit *inside* the zoom window instead of bleeding past it.
*/
export function computeRotation3DContainScale(
rot: Rotation3D,
width: number,
height: number,
perspective: number,
): number {
const a = (rot.rotationX * Math.PI) / 180;
const b = (rot.rotationY * Math.PI) / 180;
const g = (rot.rotationZ * Math.PI) / 180;
const ca = Math.cos(a);
const sa = Math.sin(a);
const cb = Math.cos(b);
const sb = Math.sin(b);
const cg = Math.cos(g);
const sg = Math.sin(g);
const halfW = width / 2;
const halfH = height / 2;
const corners: Array<[number, number]> = [
[-halfW, -halfH],
[halfW, -halfH],
[halfW, halfH],
[-halfW, halfH],
];
let maxAbsX = 0;
let maxAbsY = 0;
for (const [x0, y0] of corners) {
// CSS "rotateX(α) rotateY(β) rotateZ(γ)" reads right-to-left: Z first, then Y, then X.
let px = x0;
let py = y0;
let pz = 0;
// rotateZ
const zx = px * cg - py * sg;
const zy = px * sg + py * cg;
px = zx;
py = zy;
// rotateY
const yx = px * cb + pz * sb;
const yz = -px * sb + pz * cb;
px = yx;
pz = yz;
// rotateX
const xy = py * ca - pz * sa;
const xz = py * sa + pz * ca;
py = xy;
pz = xz;
// Perspective projection: viewer at (0, 0, P), looking toward z. A point at z=pz
// is scaled by P / (P pz). When perspective ≤ 0 we treat as orthographic.
if (perspective > 0) {
const denom = perspective - pz;
if (denom <= 0) return 1; // pathological — skip scaling rather than crash
const f = perspective / denom;
px *= f;
py *= f;
}
if (Math.abs(px) > maxAbsX) maxAbsX = Math.abs(px);
if (Math.abs(py) > maxAbsY) maxAbsY = Math.abs(py);
}
if (maxAbsX === 0 || maxAbsY === 0) return 1;
const sx = halfW / maxAbsX;
const sy = halfH / maxAbsY;
return Math.min(sx, sy, 1);
}
export interface CursorTelemetryPoint {
@@ -1,5 +1,5 @@
import type { CursorTelemetryPoint, ZoomFocus, ZoomRegion } from "../types";
import { ZOOM_DEPTH_SCALES } from "../types";
import type { CursorTelemetryPoint, Rotation3D, ZoomFocus, ZoomRegion } from "../types";
import { DEFAULT_ROTATION_3D, getRotation3D, lerpRotation3D, ZOOM_DEPTH_SCALES } from "../types";
import { TRANSITION_WINDOW_MS, ZOOM_IN_TRANSITION_WINDOW_MS } from "./constants";
import { interpolateCursorAt } from "./cursorFollowUtils";
import { clampFocusToScale } from "./focusUtils";
@@ -164,6 +164,7 @@ function getActiveRegion(
},
strength: activeRegions[0].strength,
blendedScale: null,
rotation3D: getRotation3D(activeRegion),
};
}
@@ -189,6 +190,7 @@ function getConnectedRegionHold(
},
strength: 1,
blendedScale: null,
rotation3D: getRotation3D(pair.nextRegion),
};
}
}
@@ -233,6 +235,11 @@ function getConnectedRegionTransition(
viewportRatio,
);
const transitionFocus = getLinearFocus(currentFocus, nextFocus, transitionProgress);
const transitionRotation = lerpRotation3D(
getRotation3D(currentRegion),
getRotation3D(nextRegion),
transitionProgress,
);
return {
region: {
@@ -241,6 +248,7 @@ function getConnectedRegionTransition(
},
strength: 1,
blendedScale: transitionScale,
rotation3D: transitionRotation,
transition: {
progress: transitionProgress,
startFocus: currentFocus,
@@ -258,6 +266,7 @@ type DominantRegionResult = {
region: ZoomRegion | null;
strength: number;
blendedScale: number | null;
rotation3D: Rotation3D;
transition: ConnectedPanTransition | null;
};
@@ -309,14 +318,26 @@ export function findDominantRegion(
const activeRegion = getActiveRegion(regions, timeMs, connectedPairs, telemetry, vr);
result = activeRegion
? { ...activeRegion, transition: null }
: { region: null, strength: 0, blendedScale: null, transition: null };
: {
region: null,
strength: 0,
blendedScale: null,
rotation3D: DEFAULT_ROTATION_3D,
transition: null,
};
}
}
} else {
const activeRegion = getActiveRegion(regions, timeMs, connectedPairs, telemetry, vr);
result = activeRegion
? { ...activeRegion, transition: null }
: { region: null, strength: 0, blendedScale: null, transition: null };
: {
region: null,
strength: 0,
blendedScale: null,
rotation3D: DEFAULT_ROTATION_3D,
transition: null,
};
}
dominantRegionCache = {
+8
View File
@@ -8,6 +8,14 @@
"manual": "Manual",
"auto": "Auto",
"autoDescription": "Camera follows the recorded cursor position"
},
"threeD": {
"title": "3D Rotation",
"preset": {
"iso": "Iso",
"left": "Left",
"right": "Right"
}
}
},
"speed": {
+147 -38
View File
@@ -11,13 +11,19 @@ import { MotionBlurFilter } from "pixi-filters/motion-blur";
import type {
AnnotationRegion,
CropRegion,
Rotation3D,
SpeedRegion,
WebcamLayoutPreset,
WebcamSizePreset,
ZoomDepth,
ZoomRegion,
} from "@/components/video-editor/types";
import { ZOOM_DEPTH_SCALES } from "@/components/video-editor/types";
import {
DEFAULT_ROTATION_3D,
isRotation3DIdentity,
lerpRotation3D,
ZOOM_DEPTH_SCALES,
} from "@/components/video-editor/types";
import {
AUTO_FOLLOW_RAMP_DISTANCE,
AUTO_FOLLOW_SMOOTHING_FACTOR,
@@ -60,6 +66,7 @@ import {
parseCssGradient,
resolveLinearGradientAngle,
} from "./gradientParser";
import { createThreeDPass, type ThreeDPass } from "./threeDPass";
interface FrameRenderConfig {
width: number;
@@ -124,8 +131,12 @@ export class FrameRenderer {
private shadowCtx: CanvasRenderingContext2D | null = null;
private compositeCanvas: HTMLCanvasElement | null = null;
private compositeCtx: CanvasRenderingContext2D | null = null;
private foregroundCanvas: HTMLCanvasElement | null = null;
private foregroundCtx: CanvasRenderingContext2D | null = null;
private rasterCanvas: HTMLCanvasElement | null = null;
private rasterCtx: CanvasRenderingContext2D | null = null;
private threeDPass: ThreeDPass | null = null;
private currentRotation3D: Rotation3D = { ...DEFAULT_ROTATION_3D };
private config: FrameRenderConfig;
private animationState: AnimationState;
private layoutCache: LayoutCache | null = null;
@@ -217,6 +228,19 @@ export class FrameRenderer {
throw new Error("Failed to get 2D context for raster canvas");
}
// Foreground canvas: holds recording + shadow + webcam + cursor + annotations,
// transparent background. The 3D rotation pass operates only on this layer so
// the wallpaper stays flat behind the rotated content (matching preview).
this.foregroundCanvas = document.createElement("canvas");
this.foregroundCanvas.width = this.config.width;
this.foregroundCanvas.height = this.config.height;
this.foregroundCtx = this.foregroundCanvas.getContext("2d", {
willReadFrequently: this.isLinux,
});
if (!this.foregroundCtx) {
throw new Error("Failed to get 2D context for foreground canvas");
}
// Setup shadow canvas if needed
if (this.config.showShadow) {
this.shadowCanvas = document.createElement("canvas");
@@ -235,6 +259,13 @@ export class FrameRenderer {
this.maskGraphics = new Graphics();
this.videoContainer.addChild(this.maskGraphics);
this.videoContainer.mask = this.maskGraphics;
try {
this.threeDPass = createThreeDPass(this.config.width, this.config.height);
} catch (error) {
console.warn("[FrameRenderer] 3D pass unavailable, rotation fields will be ignored:", error);
this.threeDPass = null;
}
}
private async setupBackground(): Promise<void> {
@@ -392,15 +423,18 @@ export class FrameRenderer {
// Render the PixiJS stage to its canvas (video only, transparent background)
this.app.renderer.render(this.app.stage);
// Composite with shadows to final output canvas
this.compositeWithShadows(webcamFrame);
// Skip baking the shadow when the WebGL rotation pass will run — it'd alias to
// a hard edge through bilinear sampling. We re-apply shadow fresh after rotation.
const willRotate = !isRotation3DIdentity(this.currentRotation3D);
this.compositeWithShadows(webcamFrame, !willRotate);
// Cursor highlight overlay (rendered above video, below annotations)
// Drawn onto foreground so it rotates with the recording.
if (
this.config.cursorHighlight?.enabled &&
this.config.cursorTelemetry &&
this.config.cursorTelemetry.length > 0 &&
this.compositeCtx
this.foregroundCtx
) {
const emphasisAlpha = clickEmphasisAlpha(
timeMs,
@@ -423,7 +457,7 @@ export class FrameRenderer {
const previewH = this.config.previewHeight ?? this.config.height;
const cursorScale = (this.config.width / previewW + this.config.height / previewH) / 2;
drawCursorHighlightCanvas(
this.compositeCtx,
this.foregroundCtx,
canvasX,
canvasY,
{
@@ -435,13 +469,12 @@ export class FrameRenderer {
}
}
// Render annotations on top if present
// Render annotations on top of foreground (so they rotate with recording).
if (
this.config.annotationRegions &&
this.config.annotationRegions.length > 0 &&
this.compositeCtx
this.foregroundCtx
) {
// Calculate scale factor based on export vs preview dimensions
const previewWidth = this.config.previewWidth ?? this.config.width;
const previewHeight = this.config.previewHeight ?? this.config.height;
const scaleX = this.config.width / previewWidth;
@@ -449,7 +482,7 @@ export class FrameRenderer {
const scaleFactor = (scaleX + scaleY) / 2;
await renderAnnotations(
this.compositeCtx,
this.foregroundCtx,
this.config.annotationRegions,
this.config.width,
this.config.height,
@@ -457,6 +490,58 @@ export class FrameRenderer {
scaleFactor,
);
}
// Apply 3D rotation to foreground only. Wallpaper (on compositeCanvas) is untouched.
if (willRotate && this.threeDPass && this.foregroundCanvas && this.foregroundCtx) {
const passCanvas = this.threeDPass.apply(this.foregroundCanvas, this.currentRotation3D);
const w = this.foregroundCanvas.width;
const h = this.foregroundCanvas.height;
this.foregroundCtx.clearRect(0, 0, w, h);
if (this.isLinux) {
// drawImage(webglCanvas) is unreliable on Linux/Wayland — use readPixels.
const pixels = this.threeDPass.readPixels();
const imageData = this.foregroundCtx.createImageData(w, h);
imageData.data.set(pixels);
this.foregroundCtx.putImageData(imageData, 0, 0);
} else {
this.foregroundCtx.drawImage(passCanvas, 0, 0);
}
}
// Apply shadow fresh on the rotated silhouette (flat path already baked it
// in compositeWithShadows, so guard on willRotate to avoid doubling).
// Same 3-layer filter chain as `main` — keeps the soft Gaussian intact.
if (
willRotate &&
this.config.showShadow &&
this.config.shadowIntensity > 0 &&
this.shadowCanvas &&
this.shadowCtx &&
this.foregroundCanvas
) {
const shadowCtx = this.shadowCtx;
const w = this.foregroundCanvas.width;
const h = this.foregroundCanvas.height;
shadowCtx.clearRect(0, 0, w, h);
shadowCtx.save();
const intensity = this.config.shadowIntensity;
const baseBlur1 = 48 * intensity;
const baseBlur2 = 16 * intensity;
const baseBlur3 = 8 * intensity;
const baseAlpha1 = 0.7 * intensity;
const baseAlpha2 = 0.5 * intensity;
const baseAlpha3 = 0.3 * intensity;
const baseOffset = 12 * intensity;
shadowCtx.filter = `drop-shadow(0 ${baseOffset}px ${baseBlur1}px rgba(0,0,0,${baseAlpha1})) drop-shadow(0 ${baseOffset / 3}px ${baseBlur2}px rgba(0,0,0,${baseAlpha2})) drop-shadow(0 ${baseOffset / 6}px ${baseBlur3}px rgba(0,0,0,${baseAlpha3}))`;
shadowCtx.drawImage(this.foregroundCanvas, 0, 0, w, h);
shadowCtx.restore();
if (this.compositeCtx) {
this.compositeCtx.drawImage(this.shadowCanvas, 0, 0);
}
} else if (this.compositeCtx && this.foregroundCanvas) {
// Flat path or 3D-without-shadow: stamp foreground directly.
this.compositeCtx.drawImage(this.foregroundCanvas, 0, 0);
}
}
private updateLayout(webcamFrame?: VideoFrame | null): void {
@@ -564,7 +649,7 @@ export class FrameRenderer {
private updateAnimationState(timeMs: number): number {
if (!this.cameraContainer || !this.layoutCache) return 0;
const { region, strength, blendedScale, transition } = findDominantRegion(
const { region, strength, blendedScale, rotation3D, transition } = findDominantRegion(
this.config.zoomRegions,
timeMs,
{ connectZooms: true, cursorTelemetry: this.config.cursorTelemetry },
@@ -575,6 +660,11 @@ export class FrameRenderer {
let targetFocus = { ...defaultFocus };
let targetProgress = 0;
this.currentRotation3D =
region && strength > 0
? lerpRotation3D(DEFAULT_ROTATION_3D, rotation3D, strength)
: { ...DEFAULT_ROTATION_3D };
if (region && strength > 0) {
const zoomScale = blendedScale ?? ZOOM_DEPTH_SCALES[region.depth];
const regionFocus = this.clampFocusToStage(region.focus, region.depth);
@@ -747,38 +837,52 @@ export class FrameRenderer {
return this.rasterCanvas;
}
private compositeWithShadows(webcamFrame?: VideoFrame | null): void {
if (!this.compositeCanvas || !this.compositeCtx || !this.app) return;
// `applyShadowToRecording` is false when the 3D pass will rotate this canvas
// next — the shadow gets re-applied after rotation to avoid aliasing.
private compositeWithShadows(
webcamFrame: VideoFrame | null | undefined,
applyShadowToRecording: boolean,
): void {
if (
!this.compositeCanvas ||
!this.compositeCtx ||
!this.foregroundCanvas ||
!this.foregroundCtx ||
!this.app
)
return;
const videoCanvas = this.isLinux
? this.readbackVideoCanvas()
: (this.app.canvas as HTMLCanvasElement);
const ctx = this.compositeCtx;
const bgCtx = this.compositeCtx;
const fgCtx = this.foregroundCtx;
const w = this.compositeCanvas.width;
const h = this.compositeCanvas.height;
// Clear composite canvas
ctx.clearRect(0, 0, w, h);
// Step 1: Draw background layer (with optional blur, not affected by zoom)
// Background layer (compositeCanvas): wallpaper only. Stays flat — never
// touched by the 3D rotation pass, matching preview behavior.
bgCtx.clearRect(0, 0, w, h);
if (this.backgroundSprite) {
const bgCanvas = this.backgroundSprite;
if (this.config.showBlur) {
ctx.save();
ctx.filter = "blur(6px)"; // Canvas blur is weaker than CSS
ctx.drawImage(bgCanvas, 0, 0, w, h);
ctx.restore();
bgCtx.save();
bgCtx.filter = "blur(6px)"; // Canvas blur is weaker than CSS
bgCtx.drawImage(bgCanvas, 0, 0, w, h);
bgCtx.restore();
} else {
ctx.drawImage(bgCanvas, 0, 0, w, h);
bgCtx.drawImage(bgCanvas, 0, 0, w, h);
}
} else {
console.warn("[FrameRenderer] No background sprite found during compositing!");
}
// Draw video layer with shadows on top of background
// Foreground (transparent): recording + webcam. Shadow only baked here on
// the flat path; the 3D path applies it after rotation (see renderFrame).
fgCtx.clearRect(0, 0, w, h);
if (
applyShadowToRecording &&
this.config.showShadow &&
this.config.shadowIntensity > 0 &&
this.shadowCanvas &&
@@ -788,7 +892,6 @@ export class FrameRenderer {
shadowCtx.clearRect(0, 0, w, h);
shadowCtx.save();
// Calculate shadow parameters based on intensity (0-1)
const intensity = this.config.shadowIntensity;
const baseBlur1 = 48 * intensity;
const baseBlur2 = 16 * intensity;
@@ -801,9 +904,9 @@ export class FrameRenderer {
shadowCtx.filter = `drop-shadow(0 ${baseOffset}px ${baseBlur1}px rgba(0,0,0,${baseAlpha1})) drop-shadow(0 ${baseOffset / 3}px ${baseBlur2}px rgba(0,0,0,${baseAlpha2})) drop-shadow(0 ${baseOffset / 6}px ${baseBlur3}px rgba(0,0,0,${baseAlpha3}))`;
shadowCtx.drawImage(videoCanvas, 0, 0, w, h);
shadowCtx.restore();
ctx.drawImage(this.shadowCanvas, 0, 0, w, h);
fgCtx.drawImage(this.shadowCanvas, 0, 0, w, h);
} else {
ctx.drawImage(videoCanvas, 0, 0, w, h);
fgCtx.drawImage(videoCanvas, 0, 0, w, h);
}
const webcamRect = this.layoutCache?.webcamRect ?? null;
@@ -826,9 +929,9 @@ export class FrameRenderer {
sourceAspect > targetAspect ? sourceHeight : Math.round(sourceWidth / targetAspect);
const sourceCropX = Math.max(0, Math.round((sourceWidth - sourceCropWidth) / 2));
const sourceCropY = Math.max(0, Math.round((sourceHeight - sourceCropHeight) / 2));
ctx.save();
fgCtx.save();
drawCanvasClipPath(
ctx,
fgCtx,
webcamRect.x,
webcamRect.y,
webcamRect.width,
@@ -837,15 +940,15 @@ export class FrameRenderer {
webcamRect.borderRadius,
);
if (preset.shadow) {
ctx.shadowColor = preset.shadow.color;
ctx.shadowBlur = preset.shadow.blur;
ctx.shadowOffsetX = preset.shadow.offsetX;
ctx.shadowOffsetY = preset.shadow.offsetY;
fgCtx.shadowColor = preset.shadow.color;
fgCtx.shadowBlur = preset.shadow.blur;
fgCtx.shadowOffsetX = preset.shadow.offsetX;
fgCtx.shadowOffsetY = preset.shadow.offsetY;
}
ctx.fillStyle = "#000000";
ctx.fill();
ctx.clip();
ctx.drawImage(
fgCtx.fillStyle = "#000000";
fgCtx.fill();
fgCtx.clip();
fgCtx.drawImage(
webcamFrame as unknown as CanvasImageSource,
sourceCropX,
sourceCropY,
@@ -856,7 +959,7 @@ export class FrameRenderer {
webcamRect.width,
webcamRect.height,
);
ctx.restore();
fgCtx.restore();
}
}
@@ -890,7 +993,13 @@ export class FrameRenderer {
this.shadowCtx = null;
this.compositeCanvas = null;
this.compositeCtx = null;
this.foregroundCanvas = null;
this.foregroundCtx = null;
this.rasterCanvas = null;
this.rasterCtx = null;
if (this.threeDPass) {
this.threeDPass.destroy();
this.threeDPass = null;
}
}
}
+356
View File
@@ -0,0 +1,356 @@
import type { Rotation3D } from "@/components/video-editor/types";
import {
computeRotation3DContainScale,
isRotation3DIdentity,
rotation3DPerspective,
} from "@/components/video-editor/types";
// CSS uses +y down, WebGL clip space uses +y up. We do all rotation math in CSS
// convention (top-left origin, +y down) to match the preview, then flip
// gl_Position.y at the end so WebGL's clip space lands the input's top edge at
// the top of the output viewport.
const VERTEX_SHADER = `#version 300 es
in vec2 aPos;
in vec2 aUV;
out vec2 vUV;
uniform mat4 uMvp;
uniform vec2 uSize;
void main() {
vUV = aUV;
vec2 px = (aPos - 0.5) * uSize;
vec4 clip = uMvp * vec4(px, 0.0, 1.0);
clip.y = -clip.y;
gl_Position = clip;
}
`;
const FRAGMENT_SHADER = `#version 300 es
precision highp float;
in vec2 vUV;
out vec4 fragColor;
uniform sampler2D uTex;
void main() {
fragColor = texture(uTex, vUV);
}
`;
function deg2rad(deg: number): number {
return (deg * Math.PI) / 180;
}
function multiplyMat4(a: Float32Array, b: Float32Array): Float32Array {
const out = new Float32Array(16);
for (let i = 0; i < 4; i += 1) {
for (let j = 0; j < 4; j += 1) {
let s = 0;
for (let k = 0; k < 4; k += 1) {
s += a[k * 4 + j] * b[i * 4 + k];
}
out[i * 4 + j] = s;
}
}
return out;
}
function rotationXMat(rad: number): Float32Array {
const c = Math.cos(rad);
const s = Math.sin(rad);
return new Float32Array([1, 0, 0, 0, 0, c, s, 0, 0, -s, c, 0, 0, 0, 0, 1]);
}
function rotationYMat(rad: number): Float32Array {
const c = Math.cos(rad);
const s = Math.sin(rad);
return new Float32Array([c, 0, -s, 0, 0, 1, 0, 0, s, 0, c, 0, 0, 0, 0, 1]);
}
function rotationZMat(rad: number): Float32Array {
const c = Math.cos(rad);
const s = Math.sin(rad);
return new Float32Array([c, s, 0, 0, -s, c, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]);
}
function translationMat(x: number, y: number, z: number): Float32Array {
return new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, x, y, z, 1]);
}
function perspectiveMat(fovY: number, aspect: number, near: number, far: number): Float32Array {
const f = 1 / Math.tan(fovY / 2);
const nf = 1 / (near - far);
return new Float32Array([
f / aspect,
0,
0,
0,
0,
f,
0,
0,
0,
0,
(far + near) * nf,
-1,
0,
0,
2 * far * near * nf,
0,
]);
}
function scaleMat(s: number): Float32Array {
return new Float32Array([s, 0, 0, 0, 0, s, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]);
}
export function buildMvpMatrix(rot: Rotation3D, w: number, h: number): Float32Array {
const rx = rotationXMat(deg2rad(rot.rotationX));
const ry = rotationYMat(deg2rad(rot.rotationY));
const rz = rotationZMat(deg2rad(rot.rotationZ));
const rotMat = multiplyMat4(rz, multiplyMat4(ry, rx));
const perspective = rotation3DPerspective(w, h);
const containScale = computeRotation3DContainScale(rot, w, h, perspective);
const rotScaled = multiplyMat4(rotMat, scaleMat(containScale));
const d = perspective;
const fovY = 2 * Math.atan2(h / 2, d);
const proj = perspectiveMat(fovY, w / h, 0.1, d * 4 + Math.max(w, h));
const view = translationMat(0, 0, -d);
return multiplyMat4(proj, multiplyMat4(view, rotScaled));
}
function compileShader(gl: WebGL2RenderingContext, type: number, source: string): WebGLShader {
const shader = gl.createShader(type);
if (!shader) throw new Error("Failed to create shader");
gl.shaderSource(shader, source);
gl.compileShader(shader);
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
const info = gl.getShaderInfoLog(shader);
gl.deleteShader(shader);
throw new Error(`Shader compile failed: ${info}`);
}
return shader;
}
function createProgram(gl: WebGL2RenderingContext): WebGLProgram {
const vs = compileShader(gl, gl.VERTEX_SHADER, VERTEX_SHADER);
const fs = compileShader(gl, gl.FRAGMENT_SHADER, FRAGMENT_SHADER);
const program = gl.createProgram();
if (!program) throw new Error("Failed to create program");
gl.attachShader(program, vs);
gl.attachShader(program, fs);
gl.linkProgram(program);
if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
const info = gl.getProgramInfoLog(program);
gl.deleteProgram(program);
throw new Error(`Program link failed: ${info}`);
}
gl.deleteShader(vs);
gl.deleteShader(fs);
return program;
}
export interface ThreeDPass {
apply(srcCanvas: HTMLCanvasElement | OffscreenCanvas, rot: Rotation3D): HTMLCanvasElement;
/**
* Reads back the most recent apply() result into a Uint8ClampedArray suitable
* for ImageData. Use this on platforms where drawImage(webglCanvas) is unreliable.
*/
readPixels(): Uint8ClampedArray;
resize(width: number, height: number): void;
destroy(): void;
}
export function createThreeDPass(width: number, height: number): ThreeDPass {
const canvas = document.createElement("canvas");
canvas.width = width;
canvas.height = height;
const gl = canvas.getContext("webgl2", { premultipliedAlpha: true, alpha: true });
if (!gl) throw new Error("WebGL2 not available for 3D pass");
const program = createProgram(gl);
// biome-ignore lint/correctness/useHookAtTopLevel: WebGL API, not a React hook
gl.useProgram(program);
const aPos = gl.getAttribLocation(program, "aPos");
const aUV = gl.getAttribLocation(program, "aUV");
const uMvp = gl.getUniformLocation(program, "uMvp");
const uSize = gl.getUniformLocation(program, "uSize");
const uTex = gl.getUniformLocation(program, "uTex");
const vao = gl.createVertexArray();
gl.bindVertexArray(vao);
// Quad: two triangles sharing UVs consistently per corner.
// pos.y ranges 0 (top of input) → 1 (bottom of input) following CSS convention.
// UV.y is inverted (1 - pos.y) so that with UNPACK_FLIP_Y_WEBGL the texture
// sample at the top of the input lands at the top of the rendered quad.
// TL: pos(0,0) uv(0,1) TR: pos(1,0) uv(1,1)
// BL: pos(0,1) uv(0,0) BR: pos(1,1) uv(1,0)
const verts = new Float32Array([
// aPos.x, aPos.y, aUV.x, aUV.y
0,
0,
0,
1, // TL
1,
0,
1,
1, // TR
0,
1,
0,
0, // BL
0,
1,
0,
0, // BL
1,
0,
1,
1, // TR (was 1,0,1,0 — broken)
1,
1,
1,
0, // BR
]);
const vbo = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, vbo);
gl.bufferData(gl.ARRAY_BUFFER, verts, gl.STATIC_DRAW);
gl.enableVertexAttribArray(aPos);
gl.vertexAttribPointer(aPos, 2, gl.FLOAT, false, 16, 0);
gl.enableVertexAttribArray(aUV);
gl.vertexAttribPointer(aUV, 2, gl.FLOAT, false, 16, 8);
const texture = gl.createTexture();
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, texture);
// Plain bilinear, NO mipmaps. Mipmaps pre-blur the texture for downsampling, but
// at our moderate rotation angles (≤22°) the receding edge would still pick a
// smaller mipmap level, which softens fine details — specifically the few-pixel
// rounded-corner anti-alias ramp and the shadow's Gaussian falloff. The result
// is "rounding looks like a hard corner / shadow looks grimy". Sampling level 0
// directly preserves the source crispness.
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
// Anisotropic filtering still helps without mipmaps: at oblique viewing angles
// it samples multiple texels along the gradient direction at level 0, recovering
// detail that plain bilinear would lose. Cap to the device max (16× typical).
const anisoExt =
gl.getExtension("EXT_texture_filter_anisotropic") ||
gl.getExtension("MOZ_EXT_texture_filter_anisotropic") ||
gl.getExtension("WEBKIT_EXT_texture_filter_anisotropic");
if (anisoExt) {
const maxAniso = gl.getParameter(anisoExt.MAX_TEXTURE_MAX_ANISOTROPY_EXT) as number;
gl.texParameterf(gl.TEXTURE_2D, anisoExt.TEXTURE_MAX_ANISOTROPY_EXT, Math.min(16, maxAniso));
}
gl.uniform1i(uTex, 0);
let currentSize = { width, height };
const apply = (
srcCanvas: HTMLCanvasElement | OffscreenCanvas,
rot: Rotation3D,
): HTMLCanvasElement => {
gl.viewport(0, 0, currentSize.width, currentSize.height);
gl.clearColor(0, 0, 0, 0);
gl.clear(gl.COLOR_BUFFER_BIT);
gl.useProgram(program);
gl.bindVertexArray(vao);
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
// CRITICAL: premultiply on upload. The source 2D canvas stores non-premultiplied
// RGBA (alpha=0 areas have RGB=0). Bilinear filtering between an inside-the-shape
// texel (alpha=1, RGB=color) and an outside texel (alpha=0, RGB=0) in
// non-premultiplied space yields (color/2, alpha=0.5), which the
// premultipliedAlpha:true canvas then interprets as half-strength color — visible
// as a dark halo around rounded corners and softened/grimy shadows. Premultiplying
// at upload time makes the bilinear math operate in linear-light premultiplied
// space, which is exactly the math used for compositing. Edges and shadows then
// reproduce the source crisply.
gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, true);
gl.texImage2D(
gl.TEXTURE_2D,
0,
gl.RGBA,
gl.RGBA,
gl.UNSIGNED_BYTE,
srcCanvas as TexImageSource,
);
const mvp = isRotation3DIdentity(rot)
? buildMvpMatrix(
{ rotationX: 0, rotationY: 0, rotationZ: 0 },
currentSize.width,
currentSize.height,
)
: buildMvpMatrix(rot, currentSize.width, currentSize.height);
gl.uniformMatrix4fv(uMvp, false, mvp);
gl.uniform2f(uSize, currentSize.width, currentSize.height);
gl.drawArrays(gl.TRIANGLES, 0, 6);
return canvas;
};
const resize = (w: number, h: number) => {
if (w === currentSize.width && h === currentSize.height) return;
canvas.width = w;
canvas.height = h;
currentSize = { width: w, height: h };
};
const readPixels = (): Uint8ClampedArray => {
const w = currentSize.width;
const h = currentSize.height;
const buf = new Uint8Array(w * h * 4);
gl.readPixels(0, 0, w, h, gl.RGBA, gl.UNSIGNED_BYTE, buf);
// gl.readPixels is bottom-up; flip to top-down for ImageData. We also need
// to un-premultiply the alpha here: the framebuffer holds premultiplied RGBA
// (we set UNPACK_PREMULTIPLY_ALPHA_WEBGL=true on upload), but ImageData /
// putImageData expect non-premultiplied. Without this divide, semi-transparent
// pixels get interpreted as darker than they should be.
const rowSize = w * 4;
const out = new Uint8ClampedArray(buf.length);
for (let row = 0; row < h; row += 1) {
const src = (h - 1 - row) * rowSize;
const dst = row * rowSize;
for (let col = 0; col < rowSize; col += 4) {
const r = buf[src + col];
const g = buf[src + col + 1];
const b = buf[src + col + 2];
const a = buf[src + col + 3];
if (a === 0) {
out[dst + col] = 0;
out[dst + col + 1] = 0;
out[dst + col + 2] = 0;
out[dst + col + 3] = 0;
} else if (a === 255) {
out[dst + col] = r;
out[dst + col + 1] = g;
out[dst + col + 2] = b;
out[dst + col + 3] = 255;
} else {
const inv = 255 / a;
out[dst + col] = Math.min(255, Math.round(r * inv));
out[dst + col + 1] = Math.min(255, Math.round(g * inv));
out[dst + col + 2] = Math.min(255, Math.round(b * inv));
out[dst + col + 3] = a;
}
}
}
return out;
};
const destroy = () => {
gl.deleteProgram(program);
gl.deleteBuffer(vbo);
gl.deleteVertexArray(vao);
gl.deleteTexture(texture);
};
return { apply, readPixels, resize, destroy };
}