Add webcam recording overlay support

This commit is contained in:
Marcus Schiesser
2026-03-17 19:09:34 +08:00
parent 881acdb26f
commit 2fb5b3b574
18 changed files with 1048 additions and 186 deletions
+7 -6
View File
@@ -37,12 +37,13 @@
],
"icon": "icons/icons/mac/icon.icns",
"artifactName": "${productName}-Mac-${arch}-${version}-Installer.${ext}",
"extendInfo": {
"NSAudioCaptureUsageDescription": "OpenScreen needs audio capture permission to record system audio.",
"NSMicrophoneUsageDescription": "OpenScreen needs microphone access to record voice audio.",
"NSCameraUseContinuityCameraDeviceType": true,
"com.apple.security.device.audio-input": true
}
"extendInfo": {
"NSAudioCaptureUsageDescription": "OpenScreen needs audio capture permission to record system audio.",
"NSMicrophoneUsageDescription": "OpenScreen needs microphone access to record voice audio.",
"NSCameraUsageDescription": "OpenScreen needs camera access to record webcam video.",
"NSCameraUseContinuityCameraDeviceType": true,
"com.apple.security.device.audio-input": true
}
},
"linux": {
"target": [
+32 -2
View File
@@ -33,8 +33,28 @@ interface Window {
storeRecordedVideo: (
videoData: ArrayBuffer,
fileName: string,
) => Promise<{ success: boolean; path?: string; message?: string }>;
getRecordedVideoPath: () => Promise<{ success: boolean; path?: string; message?: string }>;
) => Promise<{
success: boolean;
path?: string;
session?: import("../src/lib/recordingSession").RecordingSession;
message?: string;
error?: string;
}>;
storeRecordedSession: (
payload: import("../src/lib/recordingSession").StoreRecordedSessionInput,
) => Promise<{
success: boolean;
path?: string;
session?: import("../src/lib/recordingSession").RecordingSession;
message?: string;
error?: string;
}>;
getRecordedVideoPath: () => Promise<{
success: boolean;
path?: string;
message?: string;
error?: string;
}>;
setRecordingState: (recording: boolean) => Promise<void>;
getCursorTelemetry: (videoPath?: string) => Promise<{
success: boolean;
@@ -50,7 +70,17 @@ interface Window {
) => Promise<{ success: boolean; path?: string; message?: string; canceled?: boolean }>;
openVideoFilePicker: () => Promise<{ success: boolean; path?: string; canceled?: boolean }>;
setCurrentVideoPath: (path: string) => Promise<{ success: boolean }>;
setCurrentRecordingSession: (
session: import("../src/lib/recordingSession").RecordingSession | null,
) => Promise<{
success: boolean;
session?: import("../src/lib/recordingSession").RecordingSession;
}>;
getCurrentVideoPath: () => Promise<{ success: boolean; path?: string }>;
getCurrentRecordingSession: () => Promise<{
success: boolean;
session?: import("../src/lib/recordingSession").RecordingSession;
}>;
readBinaryFile: (filePath: string) => Promise<{
success: boolean;
data?: ArrayBuffer;
+129 -36
View File
@@ -2,10 +2,17 @@ import fs from "node:fs/promises";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { app, BrowserWindow, desktopCapturer, dialog, ipcMain, screen, shell } from "electron";
import {
normalizeProjectMedia,
normalizeRecordingSession,
type RecordingSession,
type StoreRecordedSessionInput,
} from "../../src/lib/recordingSession";
import { RECORDINGS_DIR } from "../main";
const PROJECT_FILE_EXTENSION = "openscreen";
const SHORTCUTS_FILE = path.join(app.getPath("userData"), "shortcuts.json");
const RECORDING_SESSION_SUFFIX = ".session.json";
type SelectedSource = {
name: string;
@@ -14,6 +21,7 @@ type SelectedSource = {
let selectedSource: SelectedSource | null = null;
let currentProjectPath: string | null = null;
let currentRecordingSession: RecordingSession | null = null;
function normalizePath(filePath: string) {
return path.resolve(filePath);
@@ -47,6 +55,54 @@ function isTrustedProjectPath(filePath?: string | null) {
return normalizePath(filePath) === normalizePath(currentProjectPath);
}
function setCurrentRecordingSessionState(session: RecordingSession | null) {
currentRecordingSession = session;
}
async function storeRecordedSessionFiles(payload: StoreRecordedSessionInput) {
const createdAt =
typeof payload.createdAt === "number" && Number.isFinite(payload.createdAt)
? payload.createdAt
: Date.now();
const screenVideoPath = path.join(RECORDINGS_DIR, payload.screen.fileName);
await fs.writeFile(screenVideoPath, Buffer.from(payload.screen.videoData));
let webcamVideoPath: string | undefined;
if (payload.webcam) {
webcamVideoPath = path.join(RECORDINGS_DIR, payload.webcam.fileName);
await fs.writeFile(webcamVideoPath, Buffer.from(payload.webcam.videoData));
}
const session: RecordingSession = webcamVideoPath
? { screenVideoPath, webcamVideoPath, createdAt }
: { screenVideoPath, createdAt };
setCurrentRecordingSessionState(session);
currentProjectPath = null;
const telemetryPath = `${screenVideoPath}.cursor.json`;
if (pendingCursorSamples.length > 0) {
await fs.writeFile(
telemetryPath,
JSON.stringify({ version: CURSOR_TELEMETRY_VERSION, samples: pendingCursorSamples }, null, 2),
"utf-8",
);
}
pendingCursorSamples = [];
const sessionManifestPath = path.join(
RECORDINGS_DIR,
`${path.parse(payload.screen.fileName).name}${RECORDING_SESSION_SUFFIX}`,
);
await fs.writeFile(sessionManifestPath, JSON.stringify(session, null, 2), "utf-8");
return {
success: true,
path: screenVideoPath,
session,
message: "Recording session stored successfully",
};
}
const CURSOR_TELEMETRY_VERSION = 1;
const CURSOR_SAMPLE_INTERVAL_MS = 100;
const MAX_CURSOR_SAMPLES = 60 * 60 * 10; // 1 hour @ 10Hz
@@ -146,36 +202,30 @@ export function registerIpcHandlers(
createEditorWindow();
});
ipcMain.handle("store-recorded-video", async (_, videoData: ArrayBuffer, fileName: string) => {
ipcMain.handle("store-recorded-session", async (_, payload: StoreRecordedSessionInput) => {
try {
const videoPath = path.join(RECORDINGS_DIR, fileName);
await fs.writeFile(videoPath, Buffer.from(videoData));
currentProjectPath = null;
const telemetryPath = `${videoPath}.cursor.json`;
if (pendingCursorSamples.length > 0) {
await fs.writeFile(
telemetryPath,
JSON.stringify(
{ version: CURSOR_TELEMETRY_VERSION, samples: pendingCursorSamples },
null,
2,
),
"utf-8",
);
}
pendingCursorSamples = [];
return {
success: true,
path: videoPath,
message: "Video stored successfully",
};
return await storeRecordedSessionFiles(payload);
} catch (error) {
console.error("Failed to store video:", error);
console.error("Failed to store recording session:", error);
return {
success: false,
message: "Failed to store video",
message: "Failed to store recording session",
error: String(error),
};
}
});
ipcMain.handle("store-recorded-video", async (_, videoData: ArrayBuffer, fileName: string) => {
try {
return await storeRecordedSessionFiles({
screen: { videoData, fileName },
createdAt: Date.now(),
});
} catch (error) {
console.error("Failed to store recorded video:", error);
return {
success: false,
message: "Failed to store recorded video",
error: String(error),
};
}
@@ -183,8 +233,14 @@ export function registerIpcHandlers(
ipcMain.handle("get-recorded-video-path", async () => {
try {
if (currentRecordingSession?.screenVideoPath) {
return { success: true, path: currentRecordingSession.screenVideoPath };
}
const files = await fs.readdir(RECORDINGS_DIR);
const videoFiles = files.filter((file) => file.endsWith(".webm"));
const videoFiles = files.filter(
(file) => file.endsWith(".webm") && !file.endsWith("-webcam.webm"),
);
if (videoFiles.length === 0) {
return { success: false, message: "No recorded video found" };
@@ -244,7 +300,9 @@ export function registerIpcHandlers(
});
ipcMain.handle("get-cursor-telemetry", async (_, videoPath?: string) => {
const targetVideoPath = normalizeVideoSourcePath(videoPath ?? currentVideoPath);
const targetVideoPath = normalizeVideoSourcePath(
videoPath ?? currentRecordingSession?.screenVideoPath,
);
if (!targetVideoPath) {
return { success: true, samples: [] };
}
@@ -416,7 +474,6 @@ export function registerIpcHandlers(
}
});
let currentVideoPath: string | null = null;
ipcMain.handle(
"save-project-file",
async (_, projectData: unknown, suggestedName?: string, existingProjectPath?: string) => {
@@ -502,8 +559,17 @@ export function registerIpcHandlers(
const content = await fs.readFile(filePath, "utf-8");
const project = JSON.parse(content);
currentProjectPath = filePath;
if (project && typeof project === "object" && typeof project.videoPath === "string") {
currentVideoPath = normalizeVideoSourcePath(project.videoPath) ?? project.videoPath;
if (project && typeof project === "object") {
const rawProject = project as { media?: unknown; videoPath?: unknown };
const media =
normalizeProjectMedia(rawProject.media) ??
(typeof rawProject.videoPath === "string"
? {
screenVideoPath:
normalizeVideoSourcePath(rawProject.videoPath) ?? rawProject.videoPath,
}
: null);
setCurrentRecordingSessionState(media ? { ...media, createdAt: Date.now() } : null);
}
return {
@@ -529,8 +595,17 @@ export function registerIpcHandlers(
const content = await fs.readFile(currentProjectPath, "utf-8");
const project = JSON.parse(content);
if (project && typeof project === "object" && typeof project.videoPath === "string") {
currentVideoPath = normalizeVideoSourcePath(project.videoPath) ?? project.videoPath;
if (project && typeof project === "object") {
const rawProject = project as { media?: unknown; videoPath?: unknown };
const media =
normalizeProjectMedia(rawProject.media) ??
(typeof rawProject.videoPath === "string"
? {
screenVideoPath:
normalizeVideoSourcePath(rawProject.videoPath) ?? rawProject.videoPath,
}
: null);
setCurrentRecordingSessionState(media ? { ...media, createdAt: Date.now() } : null);
}
return {
success: true,
@@ -546,18 +621,36 @@ export function registerIpcHandlers(
};
}
});
ipcMain.handle("set-current-recording-session", (_, session: RecordingSession | null) => {
const normalized = normalizeRecordingSession(session);
setCurrentRecordingSessionState(normalized);
currentProjectPath = null;
return { success: true, session: normalized ?? undefined };
});
ipcMain.handle("get-current-recording-session", () => {
return currentRecordingSession
? { success: true, session: currentRecordingSession }
: { success: false };
});
ipcMain.handle("set-current-video-path", (_, path: string) => {
currentVideoPath = normalizeVideoSourcePath(path) ?? path;
setCurrentRecordingSessionState({
screenVideoPath: normalizeVideoSourcePath(path) ?? path,
createdAt: Date.now(),
});
currentProjectPath = null;
return { success: true };
});
ipcMain.handle("get-current-video-path", () => {
return currentVideoPath ? { success: true, path: currentVideoPath } : { success: false };
return currentRecordingSession?.screenVideoPath
? { success: true, path: currentRecordingSession.screenVideoPath }
: { success: false };
});
ipcMain.handle("clear-current-video-path", () => {
currentVideoPath = null;
setCurrentRecordingSessionState(null);
return { success: true };
});
+10
View File
@@ -1,4 +1,5 @@
import { contextBridge, ipcRenderer } from "electron";
import type { RecordingSession, StoreRecordedSessionInput } from "../src/lib/recordingSession";
contextBridge.exposeInMainWorld("electronAPI", {
hudOverlayHide: () => {
@@ -30,6 +31,9 @@ contextBridge.exposeInMainWorld("electronAPI", {
storeRecordedVideo: (videoData: ArrayBuffer, fileName: string) => {
return ipcRenderer.invoke("store-recorded-video", videoData, fileName);
},
storeRecordedSession: (payload: StoreRecordedSessionInput) => {
return ipcRenderer.invoke("store-recorded-session", payload);
},
getRecordedVideoPath: () => {
return ipcRenderer.invoke("get-recorded-video-path");
@@ -57,9 +61,15 @@ contextBridge.exposeInMainWorld("electronAPI", {
setCurrentVideoPath: (path: string) => {
return ipcRenderer.invoke("set-current-video-path", path);
},
setCurrentRecordingSession: (session: RecordingSession | null) => {
return ipcRenderer.invoke("set-current-recording-session", session);
},
getCurrentVideoPath: () => {
return ipcRenderer.invoke("get-current-video-path");
},
getCurrentRecordingSession: () => {
return ipcRenderer.invoke("get-current-recording-session");
},
readBinaryFile: (filePath: string) => {
return ipcRenderer.invoke("read-binary-file", filePath);
},
+24 -1
View File
@@ -4,7 +4,16 @@ import { BsRecordCircle } from "react-icons/bs";
import { FaRegStopCircle } from "react-icons/fa";
import { FaFolderOpen } from "react-icons/fa6";
import { FiMinus, FiX } from "react-icons/fi";
import { MdMic, MdMicOff, MdMonitor, MdVideoFile, MdVolumeOff, MdVolumeUp } from "react-icons/md";
import {
MdMic,
MdMicOff,
MdMonitor,
MdVideocam,
MdVideocamOff,
MdVideoFile,
MdVolumeOff,
MdVolumeUp,
} from "react-icons/md";
import { RxDragHandleDots2 } from "react-icons/rx";
import { useAudioLevelMeter } from "../../hooks/useAudioLevelMeter";
import { useMicrophoneDevices } from "../../hooks/useMicrophoneDevices";
@@ -23,6 +32,8 @@ const ICON_CONFIG = {
volumeOff: { icon: MdVolumeOff, size: ICON_SIZE },
micOn: { icon: MdMic, size: ICON_SIZE },
micOff: { icon: MdMicOff, size: ICON_SIZE },
webcamOn: { icon: MdVideocam, size: ICON_SIZE },
webcamOff: { icon: MdVideocamOff, size: ICON_SIZE },
stop: { icon: FaRegStopCircle, size: ICON_SIZE },
record: { icon: BsRecordCircle, size: ICON_SIZE },
videoFile: { icon: MdVideoFile, size: ICON_SIZE },
@@ -57,6 +68,8 @@ export function LaunchWindow() {
setMicrophoneDeviceId,
systemAudioEnabled,
setSystemAudioEnabled,
webcamEnabled,
setWebcamEnabled,
} = useScreenRecorder();
const [recordingStart, setRecordingStart] = useState<number | null>(null);
const [elapsed, setElapsed] = useState(0);
@@ -233,6 +246,16 @@ export function LaunchWindow() {
? getIcon("micOn", "text-green-400")
: getIcon("micOff", "text-white/40")}
</button>
<button
className={`${hudIconBtnClasses} ${webcamEnabled ? "drop-shadow-[0_0_4px_rgba(74,222,128,0.4)]" : ""}`}
onClick={() => !recording && setWebcamEnabled(!webcamEnabled)}
disabled={recording}
title={webcamEnabled ? "Disable webcam" : "Enable webcam"}
>
{webcamEnabled
? getIcon("webcamOn", "text-green-400")
: getIcon("webcamOff", "text-white/40")}
</button>
</div>
{/* Record/Stop group */}
+68 -16
View File
@@ -17,6 +17,7 @@ import {
type GifSizePreset,
VideoExporter,
} from "@/lib/exporter";
import type { ProjectMedia } from "@/lib/recordingSession";
import { matchesShortcut } from "@/lib/shortcuts";
import { getAspectRatioValue, getNativeAspectRatioValue } from "@/utils/aspectRatioUtils";
import { ExportDialog } from "./ExportDialog";
@@ -26,6 +27,7 @@ import {
deriveNextId,
fromFileUrl,
normalizeProjectEditor,
resolveProjectMedia,
toFileUrl,
validateProjectData,
} from "./projectPersistence";
@@ -79,6 +81,8 @@ export default function VideoEditor() {
// ── Non-undoable state
const [videoPath, setVideoPath] = useState<string | null>(null);
const [videoSourcePath, setVideoSourcePath] = useState<string | null>(null);
const [webcamVideoPath, setWebcamVideoPath] = useState<string | null>(null);
const [webcamVideoSourcePath, setWebcamVideoSourcePath] = useState<string | null>(null);
const [currentProjectPath, setCurrentProjectPath] = useState<string | null>(null);
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
@@ -111,6 +115,19 @@ export default function VideoEditor() {
const nextAnnotationZIndexRef = useRef(1);
const exporterRef = useRef<VideoExporter | null>(null);
const currentProjectMedia = useMemo<ProjectMedia | null>(() => {
const screenVideoPath = videoSourcePath ?? (videoPath ? fromFileUrl(videoPath) : null);
if (!screenVideoPath) {
return null;
}
const webcamSourcePath =
webcamVideoSourcePath ?? (webcamVideoPath ? fromFileUrl(webcamVideoPath) : null);
return webcamSourcePath
? { screenVideoPath, webcamVideoPath: webcamSourcePath }
: { screenVideoPath };
}, [videoPath, videoSourcePath, webcamVideoPath, webcamVideoSourcePath]);
const applyLoadedProject = useCallback(
async (candidate: unknown, path?: string | null) => {
if (!validateProjectData(candidate)) {
@@ -118,7 +135,12 @@ export default function VideoEditor() {
}
const project = candidate;
const sourcePath = fromFileUrl(project.videoPath);
const media = resolveProjectMedia(project);
if (!media) {
return false;
}
const sourcePath = fromFileUrl(media.screenVideoPath);
const webcamSourcePath = media.webcamVideoPath ? fromFileUrl(media.webcamVideoPath) : null;
const normalizedEditor = normalizeProjectEditor(project.editor);
try {
@@ -133,6 +155,8 @@ export default function VideoEditor() {
setError(null);
setVideoSourcePath(sourcePath);
setVideoPath(toFileUrl(sourcePath));
setWebcamVideoSourcePath(webcamSourcePath);
setWebcamVideoPath(webcamSourcePath ? toFileUrl(webcamSourcePath) : null);
setCurrentProjectPath(path ?? null);
pushState({
@@ -182,19 +206,27 @@ export default function VideoEditor() {
0,
) + 1;
setLastSavedSnapshot(JSON.stringify(createProjectData(sourcePath, normalizedEditor)));
setLastSavedSnapshot(
JSON.stringify(
createProjectData(
webcamSourcePath
? { screenVideoPath: sourcePath, webcamVideoPath: webcamSourcePath }
: { screenVideoPath: sourcePath },
normalizedEditor,
),
),
);
return true;
},
[pushState],
);
const currentProjectSnapshot = useMemo(() => {
const sourcePath = videoSourcePath ?? (videoPath ? fromFileUrl(videoPath) : null);
if (!sourcePath) {
if (!currentProjectMedia) {
return null;
}
return JSON.stringify(
createProjectData(sourcePath, {
createProjectData(currentProjectMedia, {
wallpaper,
shadowIntensity,
showBlur,
@@ -215,8 +247,7 @@ export default function VideoEditor() {
}),
);
}, [
videoPath,
videoSourcePath,
currentProjectMedia,
wallpaper,
shadowIntensity,
showBlur,
@@ -257,11 +288,29 @@ export default function VideoEditor() {
}
}
const currentSessionResult = await window.electronAPI.getCurrentRecordingSession();
if (currentSessionResult.success && currentSessionResult.session) {
const session = currentSessionResult.session;
const sourcePath = fromFileUrl(session.screenVideoPath);
const webcamSourcePath = session.webcamVideoPath
? fromFileUrl(session.webcamVideoPath)
: null;
setVideoSourcePath(sourcePath);
setVideoPath(toFileUrl(sourcePath));
setWebcamVideoSourcePath(webcamSourcePath);
setWebcamVideoPath(webcamSourcePath ? toFileUrl(webcamSourcePath) : null);
setCurrentProjectPath(null);
setLastSavedSnapshot(null);
return;
}
const result = await window.electronAPI.getCurrentVideoPath();
if (result.success && result.path) {
const sourcePath = fromFileUrl(result.path);
setVideoSourcePath(sourcePath);
setVideoPath(toFileUrl(sourcePath));
setWebcamVideoSourcePath(null);
setWebcamVideoPath(null);
setCurrentProjectPath(null);
setLastSavedSnapshot(null);
} else {
@@ -284,13 +333,12 @@ export default function VideoEditor() {
return false;
}
const sourcePath = videoSourcePath ?? fromFileUrl(videoPath);
if (!sourcePath) {
if (!currentProjectMedia) {
toast.error("Unable to determine source video path");
return false;
}
const projectData = createProjectData(sourcePath, {
const projectData = createProjectData(currentProjectMedia, {
wallpaper,
shadowIntensity,
showBlur,
@@ -311,7 +359,7 @@ export default function VideoEditor() {
});
const fileNameBase =
sourcePath
currentProjectMedia.screenVideoPath
.split(/[\\/]/)
.pop()
?.replace(/\.[^.]+$/, "") || `project-${Date.now()}`;
@@ -341,8 +389,7 @@ export default function VideoEditor() {
return true;
},
[
videoPath,
videoSourcePath,
currentProjectMedia,
currentProjectPath,
wallpaper,
shadowIntensity,
@@ -361,6 +408,7 @@ export default function VideoEditor() {
gifFrameRate,
gifLoop,
gifSizePreset,
videoPath,
],
);
@@ -420,7 +468,7 @@ export default function VideoEditor() {
let mounted = true;
async function loadCursorTelemetry() {
const sourcePath = videoSourcePath ?? (videoPath ? fromFileUrl(videoPath) : null);
const sourcePath = currentProjectMedia?.screenVideoPath ?? null;
if (!sourcePath) {
if (mounted) {
@@ -447,7 +495,7 @@ export default function VideoEditor() {
return () => {
mounted = false;
};
}, [videoPath, videoSourcePath]);
}, [currentProjectMedia]);
function togglePlayPause() {
const playback = videoPlaybackRef.current;
@@ -921,6 +969,7 @@ export default function VideoEditor() {
// GIF Export
const gifExporter = new GifExporter({
videoUrl: videoPath,
webcamVideoUrl: webcamVideoPath || undefined,
width: settings.gifConfig.width,
height: settings.gifConfig.height,
frameRate: settings.gifConfig.frameRate,
@@ -1048,6 +1097,7 @@ export default function VideoEditor() {
const exporter = new VideoExporter({
videoUrl: videoPath,
webcamVideoUrl: webcamVideoPath || undefined,
width: exportWidth,
height: exportHeight,
frameRate: 60,
@@ -1115,6 +1165,7 @@ export default function VideoEditor() {
},
[
videoPath,
webcamVideoPath,
wallpaper,
zoomRegions,
trimRegions,
@@ -1251,10 +1302,11 @@ export default function VideoEditor() {
}}
>
<VideoPlayback
key={videoPath || "no-video"}
key={`${videoPath || "no-video"}:${webcamVideoPath || "no-webcam"}`}
aspectRatio={aspectRatio}
ref={videoPlaybackRef}
videoPath={videoPath || ""}
webcamVideoPath={webcamVideoPath || undefined}
onDurationChange={setDuration}
onTimeUpdate={setCurrentTime}
currentTime={currentTime}
+117 -1
View File
@@ -19,6 +19,7 @@ import {
useState,
} from "react";
import { getAssetPath } from "@/lib/assetPath";
import { computeWebcamOverlayLayout, type WebcamOverlayLayout } from "@/lib/webcamOverlay";
import {
type AspectRatio,
formatAspectRatioForCSS,
@@ -55,6 +56,7 @@ import {
interface VideoPlaybackProps {
videoPath: string;
webcamVideoPath?: string;
onDurationChange: (duration: number) => void;
onTimeUpdate: (time: number) => void;
currentTime: number;
@@ -98,6 +100,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
(
{
videoPath,
webcamVideoPath,
onDurationChange,
onTimeUpdate,
currentTime,
@@ -129,7 +132,9 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
ref,
) => {
const videoRef = useRef<HTMLVideoElement | null>(null);
const webcamVideoRef = useRef<HTMLVideoElement | null>(null);
const containerRef = useRef<HTMLDivElement | null>(null);
const stageRef = useRef<HTMLDivElement | null>(null);
const appRef = useRef<Application | null>(null);
const videoSpriteRef = useRef<Sprite | null>(null);
const videoContainerRef = useRef<Container | null>(null);
@@ -139,6 +144,11 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
const [videoReady, setVideoReady] = useState(false);
const overlayRef = useRef<HTMLDivElement | null>(null);
const focusIndicatorRef = useRef<HTMLDivElement | null>(null);
const [webcamLayout, setWebcamLayout] = useState<WebcamOverlayLayout | null>(null);
const [webcamDimensions, setWebcamDimensions] = useState<{
width: number;
height: number;
} | null>(null);
const currentTimeRef = useRef(0);
const zoomRegionsRef = useRef<ZoomRegion[]>([]);
const selectedZoomIdRef = useRef<string | null>(null);
@@ -901,6 +911,90 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
const [resolvedWallpaper, setResolvedWallpaper] = useState<string | null>(null);
useEffect(() => {
const webcamVideo = webcamVideoRef.current;
if (!webcamVideo || !webcamVideoPath) {
setWebcamDimensions(null);
return;
}
const handleLoadedMetadata = () => {
if (webcamVideo.videoWidth > 0 && webcamVideo.videoHeight > 0) {
setWebcamDimensions({
width: webcamVideo.videoWidth,
height: webcamVideo.videoHeight,
});
}
};
webcamVideo.addEventListener("loadedmetadata", handleLoadedMetadata);
handleLoadedMetadata();
return () => {
webcamVideo.removeEventListener("loadedmetadata", handleLoadedMetadata);
};
}, [webcamVideoPath]);
useEffect(() => {
const stage = stageRef.current;
if (!stage || !webcamDimensions) {
setWebcamLayout(null);
return;
}
const updateLayout = () => {
const layout = computeWebcamOverlayLayout({
stageWidth: stage.clientWidth,
stageHeight: stage.clientHeight,
videoWidth: webcamDimensions.width,
videoHeight: webcamDimensions.height,
});
setWebcamLayout(layout);
};
updateLayout();
if (typeof ResizeObserver === "undefined") {
return;
}
const observer = new ResizeObserver(updateLayout);
observer.observe(stage);
return () => observer.disconnect();
}, [webcamDimensions]);
useEffect(() => {
const webcamVideo = webcamVideoRef.current;
if (!webcamVideo || !webcamVideoPath) {
return;
}
if (!isPlaying) {
webcamVideo.pause();
if (Math.abs(webcamVideo.currentTime - currentTime) > 0.05) {
webcamVideo.currentTime = currentTime;
}
return;
}
if (Math.abs(webcamVideo.currentTime - currentTime) > 0.15) {
webcamVideo.currentTime = currentTime;
}
webcamVideo.play().catch(() => {
// Ignore webcam autoplay restoration failures.
});
}, [currentTime, isPlaying, webcamVideoPath]);
useEffect(() => {
const webcamVideo = webcamVideoRef.current;
if (!webcamVideo || !webcamVideoPath) {
return;
}
webcamVideo.pause();
webcamVideo.currentTime = 0;
}, [webcamVideoPath]);
useEffect(() => {
let mounted = true;
(async () => {
@@ -975,6 +1069,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
return (
<div
ref={stageRef}
className="relative rounded-sm overflow-hidden"
style={{
width: "100%",
@@ -1008,12 +1103,33 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
: "none",
}}
/>
{webcamVideoPath && (
<video
ref={webcamVideoRef}
src={webcamVideoPath}
className="absolute object-cover pointer-events-none"
style={{
left: webcamLayout?.x ?? 0,
top: webcamLayout?.y ?? 0,
width: webcamLayout?.width ?? 0,
height: webcamLayout?.height ?? 0,
borderRadius: webcamLayout?.borderRadius ?? 0,
boxShadow: "0 12px 36px rgba(0,0,0,0.35), 0 4px 12px rgba(0,0,0,0.22)",
zIndex: 20,
opacity: webcamLayout ? 1 : 0,
backgroundColor: "#000",
}}
muted
preload="metadata"
playsInline
/>
)}
{/* Only render overlay after PIXI and video are fully initialized */}
{pixiReady && videoReady && (
<div
ref={overlayRef}
className="absolute inset-0 select-none"
style={{ pointerEvents: "none" }}
style={{ pointerEvents: "none", zIndex: 30 }}
onPointerDown={handleOverlayPointerDown}
onPointerMove={handleOverlayPointerMove}
onPointerUp={handleOverlayPointerUp}
@@ -0,0 +1,57 @@
import { describe, expect, it } from "vitest";
import {
createProjectData,
PROJECT_VERSION,
resolveProjectMedia,
validateProjectData,
} from "./projectPersistence";
describe("projectPersistence media compatibility", () => {
it("accepts legacy projects with a single videoPath", () => {
const project = {
version: 1,
videoPath: "/tmp/screen.webm",
editor: {},
};
expect(validateProjectData(project)).toBe(true);
expect(resolveProjectMedia(project)).toEqual({
screenVideoPath: "/tmp/screen.webm",
});
});
it("creates version 2 projects with explicit media", () => {
const project = createProjectData(
{
screenVideoPath: "/tmp/screen.webm",
webcamVideoPath: "/tmp/webcam.webm",
},
{
wallpaper: "/wallpapers/wallpaper1.jpg",
shadowIntensity: 0,
showBlur: false,
motionBlurAmount: 0,
borderRadius: 0,
padding: 50,
cropRegion: { x: 0, y: 0, width: 1, height: 1 },
zoomRegions: [],
trimRegions: [],
speedRegions: [],
annotationRegions: [],
aspectRatio: "16:9",
exportQuality: "good",
exportFormat: "mp4",
gifFrameRate: 15,
gifLoop: true,
gifSizePreset: "medium",
},
);
expect(project.version).toBe(PROJECT_VERSION);
expect(project.media).toEqual({
screenVideoPath: "/tmp/screen.webm",
webcamVideoPath: "/tmp/webcam.webm",
});
expect(validateProjectData(project)).toBe(true);
});
});
@@ -1,4 +1,6 @@
import type { ExportFormat, ExportQuality, GifFrameRate, GifSizePreset } from "@/lib/exporter";
import type { ProjectMedia } from "@/lib/recordingSession";
import { normalizeProjectMedia } from "@/lib/recordingSession";
import { ASPECT_RATIOS, type AspectRatio } from "@/utils/aspectRatioUtils";
import {
type AnnotationRegion,
@@ -22,7 +24,7 @@ export const WALLPAPER_PATHS = Array.from(
(_, i) => `/wallpapers/wallpaper${i + 1}.jpg`,
);
export const PROJECT_VERSION = 1;
export const PROJECT_VERSION = 2;
export interface ProjectEditorState {
wallpaper: string;
@@ -46,8 +48,9 @@ export interface ProjectEditorState {
export interface EditorProjectData {
version: number;
videoPath: string;
media?: ProjectMedia;
editor: ProjectEditorState;
videoPath?: string;
}
function isFiniteNumber(value: unknown): value is number {
@@ -139,11 +142,26 @@ export function validateProjectData(candidate: unknown): candidate is EditorProj
if (!candidate || typeof candidate !== "object") return false;
const project = candidate as Partial<EditorProjectData>;
if (typeof project.version !== "number") return false;
if (typeof project.videoPath !== "string" || !project.videoPath) return false;
if (!resolveProjectMedia(project)) return false;
if (!project.editor || typeof project.editor !== "object") return false;
return true;
}
export function resolveProjectMedia(
candidate: Partial<EditorProjectData> | { media?: unknown; videoPath?: unknown },
): ProjectMedia | null {
const media = normalizeProjectMedia(candidate.media);
if (media) {
return media;
}
if (typeof candidate.videoPath === "string" && candidate.videoPath.trim()) {
return { screenVideoPath: candidate.videoPath };
}
return null;
}
export function normalizeProjectEditor(editor: Partial<ProjectEditorState>): ProjectEditorState {
const validAspectRatios = new Set<AspectRatio>(ASPECT_RATIOS);
@@ -346,12 +364,12 @@ export function normalizeProjectEditor(editor: Partial<ProjectEditorState>): Pro
}
export function createProjectData(
videoPath: string,
media: ProjectMedia,
editor: ProjectEditorState,
): EditorProjectData {
return {
version: PROJECT_VERSION,
videoPath,
media,
editor,
};
}
+195 -113
View File
@@ -1,8 +1,7 @@
import { fixWebmDuration } from "@fix-webm-duration/fix";
import { useEffect, useRef, useState } from "react";
import { useCallback, useEffect, useRef, useState } from "react";
import { toast } from "sonner";
// Target visually lossless 4K @ 60fps; fall back gracefully when hardware cannot keep up
const TARGET_FRAME_RATE = 60;
const MIN_FRAME_RATE = 30;
const TARGET_WIDTH = 3840;
@@ -12,18 +11,15 @@ const QHD_WIDTH = 2560;
const QHD_HEIGHT = 1440;
const QHD_PIXELS = QHD_WIDTH * QHD_HEIGHT;
// Bitrates (bits per second) per resolution tier
const BITRATE_4K = 45_000_000;
const BITRATE_QHD = 28_000_000;
const BITRATE_BASE = 18_000_000;
const HIGH_FRAME_RATE_THRESHOLD = 60;
const HIGH_FRAME_RATE_BOOST = 1.7;
// Fallback track settings when the driver reports nothing
const DEFAULT_WIDTH = 1920;
const DEFAULT_HEIGHT = 1080;
// Codec alignment: VP9/AV1 require dimensions divisible by 2
const CODEC_ALIGNMENT = 2;
const RECORDER_TIMESLICE_MS = 1000;
@@ -31,12 +27,15 @@ const BITS_PER_MEGABIT = 1_000_000;
const CHROME_MEDIA_SOURCE = "desktop";
const RECORDING_FILE_PREFIX = "recording-";
const VIDEO_FILE_EXTENSION = ".webm";
const WEBCAM_FILE_SUFFIX = "-webcam";
const AUDIO_BITRATE_VOICE = 128_000;
const AUDIO_BITRATE_SYSTEM = 192_000;
// Boost mic slightly when mixing with system audio so voice isn't drowned out
const MIC_GAIN_BOOST = 1.4;
const WEBCAM_TARGET_WIDTH = 1280;
const WEBCAM_TARGET_HEIGHT = 720;
const WEBCAM_TARGET_FRAME_RATE = 30;
type UseScreenRecorderReturn = {
recording: boolean;
@@ -47,20 +46,52 @@ type UseScreenRecorderReturn = {
setMicrophoneDeviceId: (deviceId: string | undefined) => void;
systemAudioEnabled: boolean;
setSystemAudioEnabled: (enabled: boolean) => void;
webcamEnabled: boolean;
setWebcamEnabled: (enabled: boolean) => void;
};
type RecorderHandle = {
recorder: MediaRecorder;
recordedBlobPromise: Promise<Blob>;
};
function createRecorderHandle(stream: MediaStream, options: MediaRecorderOptions): RecorderHandle {
const recorder = new MediaRecorder(stream, options);
const chunks: Blob[] = [];
const mimeType = options.mimeType || "video/webm";
const recordedBlobPromise = new Promise<Blob>((resolve, reject) => {
recorder.ondataavailable = (event: BlobEvent) => {
if (event.data && event.data.size > 0) {
chunks.push(event.data);
}
};
recorder.onerror = () => {
reject(new Error("Recording failed"));
};
recorder.onstop = () => {
resolve(new Blob(chunks, { type: mimeType }));
};
});
recorder.start(RECORDER_TIMESLICE_MS);
return { recorder, recordedBlobPromise };
}
export function useScreenRecorder(): UseScreenRecorderReturn {
const [recording, setRecording] = useState(false);
const [microphoneEnabled, setMicrophoneEnabled] = useState(false);
const [microphoneDeviceId, setMicrophoneDeviceId] = useState<string | undefined>(undefined);
const [systemAudioEnabled, setSystemAudioEnabled] = useState(false);
const mediaRecorder = useRef<MediaRecorder | null>(null);
const [webcamEnabled, setWebcamEnabled] = useState(false);
const screenRecorder = useRef<RecorderHandle | null>(null);
const webcamRecorder = useRef<RecorderHandle | null>(null);
const stream = useRef<MediaStream | null>(null);
const screenStream = useRef<MediaStream | null>(null);
const microphoneStream = useRef<MediaStream | null>(null);
const webcamStream = useRef<MediaStream | null>(null);
const mixingContext = useRef<AudioContext | null>(null);
const chunks = useRef<Blob[]>([]);
const startTime = useRef<number>(0);
const recordingId = useRef<number>(0);
const selectMimeType = () => {
const preferred = [
@@ -90,30 +121,109 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
return Math.round(BITRATE_BASE * highFrameRateBoost);
};
const stopRecording = useRef(() => {
if (mediaRecorder.current?.state === "recording") {
if (stream.current) {
stream.current.getTracks().forEach((track) => track.stop());
}
if (screenStream.current) {
screenStream.current.getTracks().forEach((track) => track.stop());
screenStream.current = null;
}
if (microphoneStream.current) {
microphoneStream.current.getTracks().forEach((track) => track.stop());
microphoneStream.current = null;
}
if (mixingContext.current) {
mixingContext.current.close().catch(() => {
// Ignore close errors during recorder teardown.
});
mixingContext.current = null;
}
mediaRecorder.current.stop();
setRecording(false);
window.electronAPI?.setRecordingState(false);
const teardownMedia = useCallback(() => {
if (stream.current) {
stream.current.getTracks().forEach((track) => track.stop());
stream.current = null;
}
if (screenStream.current) {
screenStream.current.getTracks().forEach((track) => track.stop());
screenStream.current = null;
}
if (microphoneStream.current) {
microphoneStream.current.getTracks().forEach((track) => track.stop());
microphoneStream.current = null;
}
if (webcamStream.current) {
webcamStream.current.getTracks().forEach((track) => track.stop());
webcamStream.current = null;
}
if (mixingContext.current) {
mixingContext.current.close().catch(() => {
// Ignore close errors during recorder teardown.
});
mixingContext.current = null;
}
}, []);
const stopRecording = useRef(() => {
const activeScreenRecorder = screenRecorder.current;
if (activeScreenRecorder?.recorder.state !== "recording") {
return;
}
const activeWebcamRecorder = webcamRecorder.current;
const duration = Date.now() - startTime.current;
const activeRecordingId = recordingId.current;
screenRecorder.current = null;
webcamRecorder.current = null;
try {
activeScreenRecorder.recorder.stop();
} catch {
// Recorder may already be stopping.
}
if (activeWebcamRecorder) {
try {
activeWebcamRecorder.recorder.stop();
} catch {
// Recorder may already be stopping.
}
}
teardownMedia();
setRecording(false);
window.electronAPI?.setRecordingState(false);
void (async () => {
try {
const screenBlob = await activeScreenRecorder.recordedBlobPromise;
if (screenBlob.size === 0) {
return;
}
const fixedScreenBlob = await fixWebmDuration(screenBlob, duration);
let fixedWebcamBlob: Blob | null = null;
if (activeWebcamRecorder) {
const webcamBlob = await activeWebcamRecorder.recordedBlobPromise.catch(() => null);
if (webcamBlob && webcamBlob.size > 0) {
fixedWebcamBlob = await fixWebmDuration(webcamBlob, duration);
}
}
const screenFileName = `${RECORDING_FILE_PREFIX}${activeRecordingId}${VIDEO_FILE_EXTENSION}`;
const webcamFileName = `${RECORDING_FILE_PREFIX}${activeRecordingId}${WEBCAM_FILE_SUFFIX}${VIDEO_FILE_EXTENSION}`;
const result = await window.electronAPI.storeRecordedSession({
screen: {
videoData: await fixedScreenBlob.arrayBuffer(),
fileName: screenFileName,
},
webcam: fixedWebcamBlob
? {
videoData: await fixedWebcamBlob.arrayBuffer(),
fileName: webcamFileName,
}
: undefined,
createdAt: activeRecordingId,
});
if (!result.success) {
console.error("Failed to store recording session:", result.message);
return;
}
if (result.session) {
await window.electronAPI.setCurrentRecordingSession(result.session);
} else if (result.path) {
await window.electronAPI.setCurrentVideoPath(result.path);
}
await window.electronAPI.switchToEditor();
} catch (error) {
console.error("Error saving recording:", error);
}
})();
});
useEffect(() => {
@@ -128,29 +238,25 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
return () => {
if (cleanup) cleanup();
if (mediaRecorder.current?.state === "recording") {
mediaRecorder.current.stop();
if (screenRecorder.current?.recorder.state === "recording") {
try {
screenRecorder.current.recorder.stop();
} catch {
// Ignore recorder teardown errors during cleanup.
}
}
if (stream.current) {
stream.current.getTracks().forEach((track) => track.stop());
stream.current = null;
}
if (screenStream.current) {
screenStream.current.getTracks().forEach((track) => track.stop());
screenStream.current = null;
}
if (microphoneStream.current) {
microphoneStream.current.getTracks().forEach((track) => track.stop());
microphoneStream.current = null;
}
if (mixingContext.current) {
mixingContext.current.close().catch(() => {
// Ignore close errors during cleanup.
});
mixingContext.current = null;
if (webcamRecorder.current?.recorder.state === "recording") {
try {
webcamRecorder.current.recorder.stop();
} catch {
// Ignore recorder teardown errors during cleanup.
}
}
screenRecorder.current = null;
webcamRecorder.current = null;
teardownMedia();
};
}, []);
}, [teardownMedia]);
const startRecording = async () => {
try {
@@ -200,7 +306,6 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
}
screenStream.current = screenMediaStream;
// If microphone is enabled, request mic stream
if (microphoneEnabled) {
try {
microphoneStream.current = await navigator.mediaDevices.getUserMedia({
@@ -225,7 +330,22 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
}
}
// Combine streams
if (webcamEnabled) {
try {
webcamStream.current = await navigator.mediaDevices.getUserMedia({
audio: false,
video: {
width: { ideal: WEBCAM_TARGET_WIDTH },
height: { ideal: WEBCAM_TARGET_HEIGHT },
frameRate: { ideal: WEBCAM_TARGET_FRAME_RATE, max: WEBCAM_TARGET_FRAME_RATE },
},
});
} catch (cameraError) {
console.warn("Failed to get webcam access:", cameraError);
toast.error("Camera access denied. Recording will continue without webcam.");
}
}
stream.current = new MediaStream();
const videoTrack = screenMediaStream.getVideoTracks()[0];
if (!videoTrack) {
@@ -237,7 +357,6 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
const micAudioTrack = microphoneStream.current?.getAudioTracks()[0];
if (systemAudioTrack && micAudioTrack) {
// Mix system audio + mic using Web Audio API
const ctx = new AudioContext();
mixingContext.current = ctx;
const systemSource = ctx.createMediaStreamSource(new MediaStream([systemAudioTrack]));
@@ -253,6 +372,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
} else if (micAudioTrack) {
stream.current.addTrack(micAudioTrack);
}
try {
await videoTrack.applyConstraints({
frameRate: { ideal: TARGET_FRAME_RATE, max: TARGET_FRAME_RATE },
@@ -272,7 +392,6 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
frameRate = TARGET_FRAME_RATE,
} = videoTrack.getSettings();
// Ensure dimensions are divisible by 2 for VP9/AV1 codec compatibility
width = Math.floor(width / CODEC_ALIGNMENT) * CODEC_ALIGNMENT;
height = Math.floor(height / CODEC_ALIGNMENT) * CODEC_ALIGNMENT;
@@ -286,54 +405,30 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
);
const hasAudio = stream.current.getAudioTracks().length > 0;
chunks.current = [];
const recorder = new MediaRecorder(stream.current, {
screenRecorder.current = createRecorderHandle(stream.current, {
mimeType,
videoBitsPerSecond,
...(hasAudio
? { audioBitsPerSecond: systemAudioTrack ? AUDIO_BITRATE_SYSTEM : AUDIO_BITRATE_VOICE }
: {}),
});
mediaRecorder.current = recorder;
recorder.ondataavailable = (e) => {
if (e.data && e.data.size > 0) chunks.current.push(e.data);
};
recorder.onstop = async () => {
stream.current = null;
if (chunks.current.length === 0) return;
const duration = Date.now() - startTime.current;
const recordedChunks = chunks.current;
const buggyBlob = new Blob(recordedChunks, { type: mimeType });
// Clear chunks early to free memory immediately after blob creation
chunks.current = [];
const timestamp = Date.now();
const videoFileName = `${RECORDING_FILE_PREFIX}${timestamp}${VIDEO_FILE_EXTENSION}`;
screenRecorder.current.recorder.addEventListener(
"error",
() => {
setRecording(false);
},
{ once: true },
);
try {
const videoBlob = await fixWebmDuration(buggyBlob, duration);
const arrayBuffer = await videoBlob.arrayBuffer();
const videoResult = await window.electronAPI.storeRecordedVideo(
arrayBuffer,
videoFileName,
);
if (!videoResult.success) {
console.error("Failed to store video:", videoResult.message);
return;
}
if (webcamStream.current) {
webcamRecorder.current = createRecorderHandle(webcamStream.current, {
mimeType,
videoBitsPerSecond: Math.min(videoBitsPerSecond, BITRATE_BASE),
});
}
if (videoResult.path) {
await window.electronAPI.setCurrentVideoPath(videoResult.path);
}
await window.electronAPI.switchToEditor();
} catch (error) {
console.error("Error saving recording:", error);
}
};
recorder.onerror = () => setRecording(false);
recorder.start(RECORDER_TIMESLICE_MS);
startTime.current = Date.now();
recordingId.current = Date.now();
startTime.current = recordingId.current;
setRecording(true);
window.electronAPI?.setRecordingState(true);
} catch (error) {
@@ -345,24 +440,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
toast.error(errorMsg);
}
setRecording(false);
if (stream.current) {
stream.current.getTracks().forEach((track) => track.stop());
stream.current = null;
}
if (screenStream.current) {
screenStream.current.getTracks().forEach((track) => track.stop());
screenStream.current = null;
}
if (microphoneStream.current) {
microphoneStream.current.getTracks().forEach((track) => track.stop());
microphoneStream.current = null;
}
if (mixingContext.current) {
mixingContext.current.close().catch(() => {
// Ignore close errors during error recovery.
});
mixingContext.current = null;
}
screenRecorder.current = null;
webcamRecorder.current = null;
teardownMedia();
}
};
@@ -379,5 +459,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
setMicrophoneDeviceId,
systemAudioEnabled,
setSystemAudioEnabled,
webcamEnabled,
setWebcamEnabled,
};
}
+77
View File
@@ -0,0 +1,77 @@
type PendingConsumer = {
resolve: (frame: VideoFrame | null) => void;
reject: (error: Error) => void;
};
export class AsyncVideoFrameQueue {
private frames: VideoFrame[] = [];
private consumers: PendingConsumer[] = [];
private error: Error | null = null;
private closed = false;
get length() {
return this.frames.length;
}
enqueue(frame: VideoFrame) {
if (this.closed) {
frame.close();
return;
}
const consumer = this.consumers.shift();
if (consumer) {
consumer.resolve(frame);
return;
}
this.frames.push(frame);
}
fail(error: Error) {
this.error = error;
this.closed = true;
const consumers = this.consumers.splice(0);
for (const consumer of consumers) {
consumer.reject(error);
}
for (const frame of this.frames) {
frame.close();
}
this.frames = [];
}
close() {
this.closed = true;
const consumers = this.consumers.splice(0);
for (const consumer of consumers) {
consumer.resolve(null);
}
}
async dequeue(): Promise<VideoFrame | null> {
if (this.error) {
throw this.error;
}
if (this.frames.length > 0) {
return this.frames.shift() ?? null;
}
if (this.closed) {
return null;
}
return await new Promise<VideoFrame | null>((resolve, reject) => {
this.consumers.push({ resolve, reject });
});
}
destroy() {
this.close();
for (const frame of this.frames) {
frame.close();
}
this.frames = [];
}
}
+40 -3
View File
@@ -30,6 +30,7 @@ import {
createMotionBlurState,
type MotionBlurState,
} from "@/components/video-editor/videoPlayback/zoomTransform";
import { computeWebcamOverlayLayout } from "@/lib/webcamOverlay";
import { renderAnnotations } from "./annotationRenderer";
interface FrameRenderConfig {
@@ -46,6 +47,8 @@ interface FrameRenderConfig {
cropRegion: CropRegion;
videoWidth: number;
videoHeight: number;
webcamWidth?: number;
webcamHeight?: number;
annotationRegions?: AnnotationRegion[];
speedRegions?: SpeedRegion[];
previewWidth?: number;
@@ -301,7 +304,11 @@ export class FrameRenderer {
this.backgroundSprite = bgCanvas;
}
async renderFrame(videoFrame: VideoFrame, timestamp: number): Promise<void> {
async renderFrame(
videoFrame: VideoFrame,
timestamp: number,
webcamFrame?: VideoFrame | null,
): Promise<void> {
if (!this.app || !this.videoContainer || !this.cameraContainer) {
throw new Error("Renderer not initialized");
}
@@ -360,7 +367,7 @@ export class FrameRenderer {
this.app.renderer.render(this.app.stage);
// Composite with shadows to final output canvas
this.compositeWithShadows();
this.compositeWithShadows(webcamFrame);
// Render annotations on top if present
if (
@@ -565,7 +572,7 @@ export class FrameRenderer {
);
}
private compositeWithShadows(): void {
private compositeWithShadows(webcamFrame?: VideoFrame | null): void {
if (!this.compositeCanvas || !this.compositeCtx || !this.app) return;
const videoCanvas = this.app.canvas as HTMLCanvasElement;
@@ -620,6 +627,36 @@ export class FrameRenderer {
} else {
ctx.drawImage(videoCanvas, 0, 0, w, h);
}
if (webcamFrame && this.config.webcamWidth && this.config.webcamHeight) {
const layout = computeWebcamOverlayLayout({
stageWidth: w,
stageHeight: h,
videoWidth: this.config.webcamWidth,
videoHeight: this.config.webcamHeight,
});
if (layout) {
ctx.save();
ctx.beginPath();
ctx.roundRect(layout.x, layout.y, layout.width, layout.height, layout.borderRadius);
ctx.closePath();
ctx.shadowColor = "rgba(0,0,0,0.35)";
ctx.shadowBlur = 24;
ctx.shadowOffsetY = 10;
ctx.fillStyle = "#000000";
ctx.fill();
ctx.clip();
ctx.drawImage(
webcamFrame as unknown as CanvasImageSource,
layout.x,
layout.y,
layout.width,
layout.height,
);
ctx.restore();
}
}
}
getCanvas(): HTMLCanvasElement {
+51 -1
View File
@@ -6,6 +6,7 @@ import type {
TrimRegion,
ZoomRegion,
} from "@/components/video-editor/types";
import { AsyncVideoFrameQueue } from "./asyncVideoFrameQueue";
import { FrameRenderer } from "./frameRenderer";
import { StreamingVideoDecoder } from "./streamingDecoder";
import type {
@@ -20,6 +21,7 @@ const GIF_WORKER_URL = new URL("gif.js/dist/gif.worker.js", import.meta.url).toS
interface GifExporterConfig {
videoUrl: string;
webcamVideoUrl?: string;
width: number;
height: number;
frameRate: GifFrameRate;
@@ -80,6 +82,7 @@ export function calculateOutputDimensions(
export class GifExporter {
private config: GifExporterConfig;
private streamingDecoder: StreamingVideoDecoder | null = null;
private webcamDecoder: StreamingVideoDecoder | null = null;
private renderer: FrameRenderer | null = null;
private gif: GIF | null = null;
private cancelled = false;
@@ -96,6 +99,11 @@ export class GifExporter {
// Initialize streaming decoder and load video metadata
this.streamingDecoder = new StreamingVideoDecoder();
const videoInfo = await this.streamingDecoder.loadMetadata(this.config.videoUrl);
let webcamInfo: Awaited<ReturnType<StreamingVideoDecoder["loadMetadata"]>> | null = null;
if (this.config.webcamVideoUrl) {
this.webcamDecoder = new StreamingVideoDecoder();
webcamInfo = await this.webcamDecoder.loadMetadata(this.config.webcamVideoUrl);
}
// Initialize frame renderer
this.renderer = new FrameRenderer({
@@ -112,6 +120,8 @@ export class GifExporter {
cropRegion: this.config.cropRegion,
videoWidth: videoInfo.width,
videoHeight: videoInfo.height,
webcamWidth: webcamInfo?.width,
webcamHeight: webcamInfo?.height,
annotationRegions: this.config.annotationRegions,
speedRegions: this.config.speedRegions,
previewWidth: this.config.previewWidth,
@@ -155,6 +165,29 @@ export class GifExporter {
console.log("[GifExporter] Using streaming decode (web-demuxer + VideoDecoder)");
let frameIndex = 0;
const webcamFrameQueue = this.config.webcamVideoUrl ? new AsyncVideoFrameQueue() : null;
const webcamDecodePromise =
this.webcamDecoder && webcamFrameQueue
? this.webcamDecoder
.decodeAll(
this.config.frameRate,
this.config.trimRegions,
this.config.speedRegions,
async (webcamFrame) => {
while (webcamFrameQueue.length >= 12 && !this.cancelled) {
await new Promise((resolve) => setTimeout(resolve, 2));
}
webcamFrameQueue.enqueue(webcamFrame);
},
)
.then(() => {
webcamFrameQueue.close();
})
.catch((error) => {
webcamFrameQueue.fail(error instanceof Error ? error : new Error(String(error)));
throw error;
})
: null;
// Stream decode and process frames — no seeking!
await this.streamingDecoder.decodeAll(
@@ -167,10 +200,13 @@ export class GifExporter {
return;
}
const webcamFrame = webcamFrameQueue ? await webcamFrameQueue.dequeue() : null;
// Render the frame with all effects using source timestamp
const sourceTimestampUs = sourceTimestampMs * 1000; // Convert to microseconds
await this.renderer!.renderFrame(videoFrame, sourceTimestampUs);
await this.renderer!.renderFrame(videoFrame, sourceTimestampUs, webcamFrame);
videoFrame.close();
webcamFrame?.close();
// Get the rendered canvas and add to GIF
const canvas = this.renderer!.getCanvas();
@@ -196,6 +232,8 @@ export class GifExporter {
return { success: false, error: "Export cancelled" };
}
await webcamDecodePromise;
// Update progress to show we're now in the finalizing phase
if (this.config.onProgress) {
this.config.onProgress({
@@ -248,6 +286,9 @@ export class GifExporter {
if (this.streamingDecoder) {
this.streamingDecoder.cancel();
}
if (this.webcamDecoder) {
this.webcamDecoder.cancel();
}
if (this.gif) {
this.gif.abort();
}
@@ -264,6 +305,15 @@ export class GifExporter {
this.streamingDecoder = null;
}
if (this.webcamDecoder) {
try {
this.webcamDecoder.destroy();
} catch (e) {
console.warn("Error destroying webcam decoder:", e);
}
this.webcamDecoder = null;
}
if (this.renderer) {
try {
this.renderer.destroy();
+50 -1
View File
@@ -5,6 +5,7 @@ import type {
TrimRegion,
ZoomRegion,
} from "@/components/video-editor/types";
import { AsyncVideoFrameQueue } from "./asyncVideoFrameQueue";
import { AudioProcessor } from "./audioEncoder";
import { FrameRenderer } from "./frameRenderer";
import { VideoMuxer } from "./muxer";
@@ -13,6 +14,7 @@ import type { ExportConfig, ExportProgress, ExportResult } from "./types";
interface VideoExporterConfig extends ExportConfig {
videoUrl: string;
webcamVideoUrl?: string;
wallpaper: string;
zoomRegions: ZoomRegion[];
trimRegions?: TrimRegion[];
@@ -38,6 +40,7 @@ export class VideoExporter {
private encoder: VideoEncoder | null = null;
private muxer: VideoMuxer | null = null;
private audioProcessor: AudioProcessor | null = null;
private webcamDecoder: StreamingVideoDecoder | null = null;
private cancelled = false;
private encodeQueue = 0;
// Increased queue size for better throughput with hardware encoding
@@ -60,6 +63,11 @@ export class VideoExporter {
// Initialize streaming decoder and load video metadata
this.streamingDecoder = new StreamingVideoDecoder();
const videoInfo = await this.streamingDecoder.loadMetadata(this.config.videoUrl);
let webcamInfo: Awaited<ReturnType<StreamingVideoDecoder["loadMetadata"]>> | null = null;
if (this.config.webcamVideoUrl) {
this.webcamDecoder = new StreamingVideoDecoder();
webcamInfo = await this.webcamDecoder.loadMetadata(this.config.webcamVideoUrl);
}
// Initialize frame renderer
this.renderer = new FrameRenderer({
@@ -76,6 +84,8 @@ export class VideoExporter {
cropRegion: this.config.cropRegion,
videoWidth: videoInfo.width,
videoHeight: videoInfo.height,
webcamWidth: webcamInfo?.width,
webcamHeight: webcamInfo?.height,
annotationRegions: this.config.annotationRegions,
speedRegions: this.config.speedRegions,
previewWidth: this.config.previewWidth,
@@ -106,6 +116,29 @@ export class VideoExporter {
const frameDuration = 1_000_000 / this.config.frameRate; // in microseconds
let frameIndex = 0;
const webcamFrameQueue = this.config.webcamVideoUrl ? new AsyncVideoFrameQueue() : null;
const webcamDecodePromise =
this.webcamDecoder && webcamFrameQueue
? this.webcamDecoder
.decodeAll(
this.config.frameRate,
this.config.trimRegions,
this.config.speedRegions,
async (webcamFrame) => {
while (webcamFrameQueue.length >= 12 && !this.cancelled) {
await new Promise((resolve) => setTimeout(resolve, 2));
}
webcamFrameQueue.enqueue(webcamFrame);
},
)
.then(() => {
webcamFrameQueue.close();
})
.catch((error) => {
webcamFrameQueue.fail(error instanceof Error ? error : new Error(String(error)));
throw error;
})
: null;
// Stream decode and process frames — no seeking!
await this.streamingDecoder.decodeAll(
@@ -119,11 +152,13 @@ export class VideoExporter {
}
const timestamp = frameIndex * frameDuration;
const webcamFrame = webcamFrameQueue ? await webcamFrameQueue.dequeue() : null;
// Render the frame with all effects using source timestamp
const sourceTimestampUs = sourceTimestampMs * 1000; // Convert to microseconds
await this.renderer!.renderFrame(videoFrame, sourceTimestampUs);
await this.renderer!.renderFrame(videoFrame, sourceTimestampUs, webcamFrame);
videoFrame.close();
webcamFrame?.close();
const canvas = this.renderer!.getCanvas();
@@ -176,6 +211,8 @@ export class VideoExporter {
return { success: false, error: "Export cancelled" };
}
await webcamDecodePromise;
// Finalize encoding
if (this.encoder && this.encoder.state === "configured") {
await this.encoder.flush();
@@ -332,6 +369,9 @@ export class VideoExporter {
if (this.streamingDecoder) {
this.streamingDecoder.cancel();
}
if (this.webcamDecoder) {
this.webcamDecoder.cancel();
}
if (this.audioProcessor) {
this.audioProcessor.cancel();
}
@@ -359,6 +399,15 @@ export class VideoExporter {
this.streamingDecoder = null;
}
if (this.webcamDecoder) {
try {
this.webcamDecoder.destroy();
} catch (e) {
console.warn("Error destroying webcam decoder:", e);
}
this.webcamDecoder = null;
}
if (this.renderer) {
try {
this.renderer.destroy();
+69
View File
@@ -0,0 +1,69 @@
export interface ProjectMedia {
screenVideoPath: string;
webcamVideoPath?: string;
}
export interface RecordingSession extends ProjectMedia {
createdAt: number;
}
export interface RecordedVideoAssetInput {
fileName: string;
videoData: ArrayBuffer;
}
export interface StoreRecordedSessionInput {
screen: RecordedVideoAssetInput;
webcam?: RecordedVideoAssetInput;
createdAt?: number;
}
function normalizePath(value: unknown): string | undefined {
if (typeof value !== "string") {
return undefined;
}
const trimmed = value.trim();
return trimmed ? trimmed : undefined;
}
export function normalizeProjectMedia(candidate: unknown): ProjectMedia | null {
if (!candidate || typeof candidate !== "object") {
return null;
}
const raw = candidate as Partial<ProjectMedia>;
const screenVideoPath = normalizePath(raw.screenVideoPath);
if (!screenVideoPath) {
return null;
}
const webcamVideoPath = normalizePath(raw.webcamVideoPath);
return webcamVideoPath
? { screenVideoPath, webcamVideoPath }
: {
screenVideoPath,
};
}
export function normalizeRecordingSession(candidate: unknown): RecordingSession | null {
if (!candidate || typeof candidate !== "object") {
return null;
}
const raw = candidate as Partial<RecordingSession>;
const media = normalizeProjectMedia(raw);
if (!media) {
return null;
}
return {
...media,
createdAt:
typeof raw.createdAt === "number" && Number.isFinite(raw.createdAt)
? raw.createdAt
: Date.now(),
};
}
+33
View File
@@ -0,0 +1,33 @@
import { describe, expect, it } from "vitest";
import { computeWebcamOverlayLayout } from "./webcamOverlay";
describe("computeWebcamOverlayLayout", () => {
it("anchors the overlay in the lower-right corner", () => {
const layout = computeWebcamOverlayLayout({
stageWidth: 1920,
stageHeight: 1080,
videoWidth: 1280,
videoHeight: 720,
});
expect(layout).not.toBeNull();
expect(layout!.x + layout!.width).toBeLessThanOrEqual(1920);
expect(layout!.y + layout!.height).toBeLessThanOrEqual(1080);
expect(layout!.x).toBeGreaterThan(1920 / 2);
expect(layout!.y).toBeGreaterThan(1080 / 2);
});
it("keeps the overlay within the configured stage fraction while preserving aspect ratio", () => {
const layout = computeWebcamOverlayLayout({
stageWidth: 1280,
stageHeight: 720,
videoWidth: 1920,
videoHeight: 1080,
});
expect(layout).not.toBeNull();
expect(layout!.width).toBeLessThanOrEqual(Math.round(1280 * 0.18) + 1);
expect(layout!.height).toBeLessThanOrEqual(Math.round(720 * 0.18) + 1);
expect(layout!.width / layout!.height).toBeCloseTo(1920 / 1080, 2);
});
});
+45
View File
@@ -0,0 +1,45 @@
export interface WebcamOverlayLayout {
x: number;
y: number;
width: number;
height: number;
margin: number;
borderRadius: number;
}
const MAX_STAGE_FRACTION = 0.18;
const MARGIN_FRACTION = 0.02;
const MIN_SIZE = 96;
const MAX_BORDER_RADIUS = 24;
export function computeWebcamOverlayLayout(params: {
stageWidth: number;
stageHeight: number;
videoWidth: number;
videoHeight: number;
}): WebcamOverlayLayout | null {
const { stageWidth, stageHeight, videoWidth, videoHeight } = params;
if (stageWidth <= 0 || stageHeight <= 0 || videoWidth <= 0 || videoHeight <= 0) {
return null;
}
const margin = Math.max(12, Math.round(Math.min(stageWidth, stageHeight) * MARGIN_FRACTION));
const maxWidth = Math.max(MIN_SIZE, stageWidth * MAX_STAGE_FRACTION);
const maxHeight = Math.max(MIN_SIZE, stageHeight * MAX_STAGE_FRACTION);
const scale = Math.min(maxWidth / videoWidth, maxHeight / videoHeight);
const width = Math.round(videoWidth * scale);
const height = Math.round(videoHeight * scale);
return {
x: Math.max(0, Math.round(stageWidth - margin - width)),
y: Math.max(0, Math.round(stageHeight - margin - height)),
width,
height,
margin,
borderRadius: Math.min(
MAX_BORDER_RADIUS,
Math.max(12, Math.round(Math.min(width, height) * 0.12)),
),
};
}
+21 -1
View File
@@ -28,7 +28,17 @@ interface Window {
) => Promise<{
success: boolean;
path?: string;
message: string;
session?: import("./lib/recordingSession").RecordingSession;
message?: string;
error?: string;
}>;
storeRecordedSession: (
payload: import("./lib/recordingSession").StoreRecordedSessionInput,
) => Promise<{
success: boolean;
path?: string;
session?: import("./lib/recordingSession").RecordingSession;
message?: string;
error?: string;
}>;
getRecordedVideoPath: () => Promise<{
@@ -58,7 +68,17 @@ interface Window {
}>;
openVideoFilePicker: () => Promise<{ success: boolean; path?: string; canceled?: boolean }>;
setCurrentVideoPath: (path: string) => Promise<{ success: boolean }>;
setCurrentRecordingSession: (
session: import("./lib/recordingSession").RecordingSession | null,
) => Promise<{
success: boolean;
session?: import("./lib/recordingSession").RecordingSession;
}>;
getCurrentVideoPath: () => Promise<{ success: boolean; path?: string }>;
getCurrentRecordingSession: () => Promise<{
success: boolean;
session?: import("./lib/recordingSession").RecordingSession;
}>;
clearCurrentVideoPath: () => Promise<{ success: boolean }>;
saveProjectFile: (
projectData: unknown,