@@ -1760,7 +1747,7 @@ export function SettingsPanel({
setShowCropDropdown(false)}
className="hover:bg-white/10 text-slate-400 hover:text-white"
>
setShowCropModal(false)}
+ onClick={() => setShowCropDropdown(false)}
size="lg"
className="bg-[#34B27B] hover:bg-[#34B27B]/90 text-white"
>
diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx
index e1f6a60..2612b68 100644
--- a/src/components/video-editor/VideoEditor.tsx
+++ b/src/components/video-editor/VideoEditor.tsx
@@ -67,6 +67,10 @@ import {
DEFAULT_ANNOTATION_SIZE,
DEFAULT_ANNOTATION_STYLE,
DEFAULT_BLUR_DATA,
+ DEFAULT_CURSOR_CLICK_BOUNCE,
+ DEFAULT_CURSOR_MOTION_BLUR,
+ DEFAULT_CURSOR_SIZE,
+ DEFAULT_CURSOR_SMOOTHING,
DEFAULT_FIGURE_DATA,
DEFAULT_PLAYBACK_SPEED,
DEFAULT_ZOOM_DEPTH,
@@ -156,7 +160,13 @@ export default function VideoEditor() {
const [isFullscreen, setIsFullscreen] = useState(false);
const [showCloseConfirmDialog, setShowCloseConfirmDialog] = useState(false);
- const playerContainerRef = useRef(null);
+ // Cursor & motion blur visual settings (non-undoable preferences)
+ const [showCursor, setShowCursor] = useState(true);
+ const [cursorSize, setCursorSize] = useState(DEFAULT_CURSOR_SIZE);
+ const [cursorSmoothing, setCursorSmoothing] = useState(DEFAULT_CURSOR_SMOOTHING);
+ const [cursorMotionBlur, setCursorMotionBlur] = useState(DEFAULT_CURSOR_MOTION_BLUR);
+ const [cursorClickBounce, setCursorClickBounce] = useState(DEFAULT_CURSOR_CLICK_BOUNCE);
+
const videoPlaybackRef = useRef(null);
const nextZoomIdRef = useRef(1);
@@ -208,12 +218,7 @@ export default function VideoEditor() {
}
const project = candidate;
- const media = resolveProjectMedia(project);
- if (!media) {
- return false;
- }
- const sourcePath = fromFileUrl(media.screenVideoPath);
- const webcamSourcePath = media.webcamVideoPath ? fromFileUrl(media.webcamVideoPath) : null;
+ const sourcePath = project.videoPath;
const normalizedEditor = normalizeProjectEditor(project.editor);
try {
@@ -391,11 +396,8 @@ export default function VideoEditor() {
const result = await window.electronAPI.getCurrentVideoPath();
if (result.success && result.path) {
- const sourcePath = fromFileUrl(result.path);
- setVideoSourcePath(sourcePath);
- setVideoPath(toFileUrl(sourcePath));
- setWebcamVideoSourcePath(null);
- setWebcamVideoPath(null);
+ setVideoSourcePath(result.path);
+ setVideoPath(toFileUrl(result.path));
setCurrentProjectPath(null);
setLastSavedSnapshot(
createProjectSnapshot({ screenVideoPath: sourcePath }, INITIAL_EDITOR_STATE),
diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx
index 90a3dbd..a053c3a 100644
--- a/src/components/video-editor/VideoPlayback.tsx
+++ b/src/components/video-editor/VideoPlayback.tsx
@@ -36,7 +36,12 @@ import { AnnotationOverlay } from "./AnnotationOverlay";
import {
type AnnotationRegion,
type BlurData,
+ type CursorTelemetryPoint,
computeRotation3DContainScale,
+ DEFAULT_CURSOR_CLICK_BOUNCE,
+ DEFAULT_CURSOR_MOTION_BLUR,
+ DEFAULT_CURSOR_SIZE,
+ DEFAULT_CURSOR_SMOOTHING,
DEFAULT_ROTATION_3D,
getZoomScale,
isRotation3DIdentity,
@@ -130,9 +135,14 @@ interface VideoPlaybackProps {
onBlurSizeChange?: (id: string, size: { width: number; height: number }) => void;
onBlurDataChange?: (id: string, blurData: BlurData) => void;
onBlurDataCommit?: () => void;
- cursorTelemetry?: import("./types").CursorTelemetryPoint[];
+ cursorTelemetry?: CursorTelemetryPoint[];
cursorHighlight?: CursorHighlightConfig;
cursorClickTimestamps?: number[];
+ showCursor?: boolean;
+ cursorSize?: number;
+ cursorSmoothing?: number;
+ cursorMotionBlur?: number;
+ cursorClickBounce?: number;
}
export interface VideoPlaybackRef {
@@ -193,6 +203,11 @@ const VideoPlayback = forwardRef(
cursorTelemetry = [],
cursorHighlight = DEFAULT_CURSOR_HIGHLIGHT,
cursorClickTimestamps = [],
+ showCursor = false,
+ cursorSize = DEFAULT_CURSOR_SIZE,
+ cursorSmoothing = DEFAULT_CURSOR_SMOOTHING,
+ cursorMotionBlur = DEFAULT_CURSOR_MOTION_BLUR,
+ cursorClickBounce = DEFAULT_CURSOR_CLICK_BOUNCE,
},
ref,
) => {
@@ -203,6 +218,7 @@ const VideoPlayback = forwardRef(
const videoSpriteRef = useRef(null);
const videoContainerRef = useRef(null);
const cameraContainerRef = useRef(null);
+ const cursorContainerRef = useRef(null);
const timeUpdateAnimationRef = useRef(null);
const [pixiReady, setPixiReady] = useState(false);
const [videoReady, setVideoReady] = useState(false);
@@ -217,7 +233,7 @@ const VideoPlayback = forwardRef(
const [webcamDimensions, setWebcamDimensions] = useState(null);
const currentTimeRef = useRef(0);
const zoomRegionsRef = useRef([]);
- const cursorTelemetryRef = useRef([]);
+ const cursorTelemetryRef = useRef([]);
const cursorHighlightRef = useRef(DEFAULT_CURSOR_HIGHLIGHT);
const cursorClickTimestampsRef = useRef([]);
const cursorHighlightGraphicsRef = useRef(null);
@@ -257,6 +273,12 @@ const VideoPlayback = forwardRef(
const trimRegionsRef = useRef([]);
const speedRegionsRef = useRef([]);
const motionBlurAmountRef = useRef(motionBlurAmount);
+ const cursorOverlayRef = useRef(null);
+ const showCursorRef = useRef(showCursor);
+ const cursorSizeRef = useRef(cursorSize);
+ const cursorSmoothingRef = useRef(cursorSmoothing);
+ const cursorMotionBlurRef = useRef(cursorMotionBlur);
+ const cursorClickBounceRef = useRef(cursorClickBounce);
const motionBlurStateRef = useRef(createMotionBlurState());
const onTimeUpdateRef = useRef(onTimeUpdate);
const onPlayStateChangeRef = useRef(onPlayStateChange);
@@ -581,6 +603,41 @@ const VideoPlayback = forwardRef(
motionBlurAmountRef.current = motionBlurAmount;
}, [motionBlurAmount]);
+ useEffect(() => {
+ cursorTelemetryRef.current = cursorTelemetry;
+ }, [cursorTelemetry]);
+
+ useEffect(() => {
+ showCursorRef.current = showCursor;
+ }, [showCursor]);
+
+ useEffect(() => {
+ cursorSizeRef.current = cursorSize;
+ }, [cursorSize]);
+
+ useEffect(() => {
+ cursorSmoothingRef.current = cursorSmoothing;
+ }, [cursorSmoothing]);
+
+ useEffect(() => {
+ cursorMotionBlurRef.current = cursorMotionBlur;
+ }, [cursorMotionBlur]);
+
+ useEffect(() => {
+ cursorClickBounceRef.current = cursorClickBounce;
+ }, [cursorClickBounce]);
+
+ // Sync cursor overlay config when settings change
+ useEffect(() => {
+ const overlay = cursorOverlayRef.current;
+ if (!overlay) return;
+ overlay.setDotRadius(DEFAULT_CURSOR_CONFIG.dotRadius * cursorSize);
+ overlay.setSmoothingFactor(cursorSmoothing);
+ overlay.setMotionBlur(cursorMotionBlur);
+ overlay.setClickBounce(cursorClickBounce);
+ overlay.reset();
+ }, [cursorSize, cursorSmoothing, cursorMotionBlur, cursorClickBounce]);
+
useEffect(() => {
onTimeUpdateRef.current = onTimeUpdate;
}, [onTimeUpdate]);
@@ -700,6 +757,13 @@ const VideoPlayback = forwardRef(
let app: Application | null = null;
(async () => {
+ let cursorOverlayEnabled = true;
+ try {
+ await preloadCursorAssets();
+ } catch {
+ cursorOverlayEnabled = false;
+ }
+
app = new Application();
await app.init({
@@ -735,12 +799,33 @@ const VideoPlayback = forwardRef(
videoContainerRef.current = videoContainer;
cameraContainer.addChild(videoContainer);
+ // Cursor container - rendered above video
+ const cursorContainer = new Container();
+ cursorContainerRef.current = cursorContainer;
+ cameraContainer.addChild(cursorContainer);
+
+ // Cursor overlay - rendered above the masked video
+ if (cursorOverlayEnabled) {
+ const cursorOverlay = new PixiCursorOverlay({
+ dotRadius: DEFAULT_CURSOR_CONFIG.dotRadius * cursorSizeRef.current,
+ smoothingFactor: cursorSmoothingRef.current,
+ motionBlur: cursorMotionBlurRef.current,
+ clickBounce: cursorClickBounceRef.current,
+ });
+ cursorOverlayRef.current = cursorOverlay;
+ cursorContainer.addChild(cursorOverlay.container);
+ }
+
setPixiReady(true);
})();
return () => {
mounted = false;
setPixiReady(false);
+ if (cursorOverlayRef.current) {
+ cursorOverlayRef.current.destroy();
+ cursorOverlayRef.current = null;
+ }
if (app && app.renderer) {
app.destroy(true, {
children: true,
@@ -751,6 +836,7 @@ const VideoPlayback = forwardRef(
appRef.current = null;
cameraContainerRef.current = null;
videoContainerRef.current = null;
+ cursorContainerRef.current = null;
videoSpriteRef.current = null;
};
}, []);
@@ -780,8 +866,9 @@ const VideoPlayback = forwardRef(
const video = videoRef.current;
const app = appRef.current;
const videoContainer = videoContainerRef.current;
+ const cursorContainer = cursorContainerRef.current;
- if (!video || !app || !videoContainer) return;
+ if (!video || !app || !videoContainer || !cursorContainer) return;
if (video.videoWidth === 0 || video.videoHeight === 0) return;
const source = VideoSource.from(video);
@@ -801,6 +888,9 @@ const VideoPlayback = forwardRef(
videoContainer.addChild(maskGraphics);
videoContainer.mask = maskGraphics;
maskGraphicsRef.current = maskGraphics;
+ if (cursorOverlayRef.current) {
+ cursorContainer.addChild(cursorOverlayRef.current.container);
+ }
const cursorHighlightGraphics = new Graphics();
cursorHighlightGraphics.visible = false;
@@ -1147,6 +1237,19 @@ const VideoPlayback = forwardRef(
}
}
+ // Update cursor overlay
+ const cursorOverlay = cursorOverlayRef.current;
+ if (cursorOverlay) {
+ const timeMs = currentTimeRef.current;
+ cursorOverlay.update(
+ cursorTelemetryRef.current,
+ timeMs,
+ baseMaskRef.current,
+ showCursorRef.current,
+ !isPlayingRef.current || isSeekingRef.current,
+ );
+ }
+
const composite3D = composite3DRef.current;
const outerWrapper = outerWrapperRef.current;
if (composite3D && outerWrapper) {
diff --git a/src/components/video-editor/projectPersistence.ts b/src/components/video-editor/projectPersistence.ts
index aec570f..a614ad0 100644
--- a/src/components/video-editor/projectPersistence.ts
+++ b/src/components/video-editor/projectPersistence.ts
@@ -119,50 +119,21 @@ function clamp(value: number, min: number, max: number) {
return Math.min(max, Math.max(min, value));
}
-function isFileUrl(value: string): boolean {
- return /^file:\/\//i.test(value);
-}
-
-function encodePathSegments(pathname: string, keepWindowsDrive = false): string {
- return pathname
- .split("/")
- .map((segment, index) => {
- if (!segment) return "";
- if (keepWindowsDrive && index === 1 && /^[a-zA-Z]:$/.test(segment)) {
- return segment;
- }
- return encodeURIComponent(segment);
- })
- .join("/");
-}
-
export function toFileUrl(filePath: string): string {
const normalized = filePath.replace(/\\/g, "/");
-
- // Windows drive path: C:/Users/...
- if (/^[a-zA-Z]:\//.test(normalized)) {
- return `file://${encodePathSegments(`/${normalized}`, true)}`;
+ if (normalized.match(/^[a-zA-Z]:/)) {
+ return `file:///${encodeURI(normalized)}`;
}
-
- // UNC path: //server/share/...
- if (normalized.startsWith("//")) {
- const [host, ...pathParts] = normalized.replace(/^\/+/, "").split("/");
- const encodedPath = pathParts.map((part) => encodeURIComponent(part)).join("/");
- return encodedPath ? `file://${host}/${encodedPath}` : `file://${host}/`;
- }
-
- const absolutePath = normalized.startsWith("/") ? normalized : `/${normalized}`;
- return `file://${encodePathSegments(absolutePath)}`;
+ return `file://${encodeURI(normalized)}`;
}
export function fromFileUrl(fileUrl: string): string {
- const value = fileUrl.trim();
- if (!isFileUrl(value)) {
+ if (!fileUrl.startsWith("file://")) {
return fileUrl;
}
try {
- const url = new URL(value);
+ const url = new URL(fileUrl);
const pathname = decodeURIComponent(url.pathname);
if (url.host && url.host !== "localhost") {
@@ -175,13 +146,7 @@ export function fromFileUrl(fileUrl: string): string {
return pathname;
} catch {
- const rawFallbackPath = value.replace(/^file:\/\//i, "");
- let fallbackPath = rawFallbackPath;
- try {
- fallbackPath = decodeURIComponent(rawFallbackPath);
- } catch {
- // Keep raw best-effort path if percent decoding fails.
- }
+ const fallbackPath = decodeURIComponent(fileUrl.replace(/^file:\/\//, ""));
return fallbackPath.replace(/^\/([a-zA-Z]:)/, "$1");
}
}
diff --git a/src/components/video-editor/types.ts b/src/components/video-editor/types.ts
index 093f168..d6571be 100644
--- a/src/components/video-editor/types.ts
+++ b/src/components/video-editor/types.ts
@@ -170,8 +170,32 @@ export interface CursorTelemetryPoint {
timeMs: number;
cx: number;
cy: number;
+ interactionType?: "move" | "click" | "double-click" | "right-click" | "middle-click" | "mouseup";
+ cursorType?:
+ | "arrow"
+ | "text"
+ | "pointer"
+ | "crosshair"
+ | "open-hand"
+ | "closed-hand"
+ | "resize-ew"
+ | "resize-ns"
+ | "not-allowed";
}
+export interface CursorVisualSettings {
+ size: number;
+ smoothing: number;
+ motionBlur: number;
+ clickBounce: number;
+}
+
+export const DEFAULT_CURSOR_SIZE = 3.0;
+export const DEFAULT_CURSOR_SMOOTHING = 0.67;
+export const DEFAULT_CURSOR_MOTION_BLUR = 0.35;
+export const DEFAULT_CURSOR_CLICK_BOUNCE = 2.5;
+export const DEFAULT_ZOOM_MOTION_BLUR = 0.35;
+
export interface TrimRegion {
id: string;
startMs: number;
diff --git a/src/components/video-editor/videoPlayback/cursorRenderer.ts b/src/components/video-editor/videoPlayback/cursorRenderer.ts
new file mode 100644
index 0000000..7b912c5
--- /dev/null
+++ b/src/components/video-editor/videoPlayback/cursorRenderer.ts
@@ -0,0 +1,766 @@
+import { Assets, BlurFilter, Container, Graphics, Sprite, Texture } from "pixi.js";
+import { MotionBlurFilter } from "pixi-filters/motion-blur";
+import type { CursorTelemetryPoint } from "../types";
+import {
+ createSpringState,
+ getCursorSpringConfig,
+ resetSpringState,
+ stepSpringValue,
+} from "./motionSmoothing";
+import { UPLOADED_CURSOR_SAMPLE_SIZE, uploadedCursorAssets } from "./uploadedCursorAssets";
+
+type CursorAssetKey = NonNullable;
+
+/** System cursor asset from native helper (macOS only). */
+type SystemCursorAsset = {
+ dataUrl: string;
+ width: number;
+ height: number;
+ hotspotX: number;
+ hotspotY: number;
+};
+
+type LoadedCursorAsset = {
+ texture: Texture;
+ image: HTMLImageElement;
+ aspectRatio: number;
+ anchorX: number;
+ anchorY: number;
+};
+
+export interface CursorViewportRect {
+ x: number;
+ y: number;
+ width: number;
+ height: number;
+}
+
+/**
+ * Configuration for cursor rendering.
+ */
+export interface CursorRenderConfig {
+ /** Base cursor height in pixels (at reference width of 1920px) */
+ dotRadius: number;
+ /** Cursor fill color (hex number for PixiJS) */
+ dotColor: number;
+ /** Cursor opacity (0–1) */
+ dotAlpha: number;
+ /** Unused, kept for interface compatibility */
+ trailLength: number;
+ /** Smoothing factor for cursor interpolation (0–1, lower = smoother/slower) */
+ smoothingFactor: number;
+ /** Directional cursor motion blur amount. */
+ motionBlur: number;
+ /** Click bounce multiplier. */
+ clickBounce: number;
+}
+
+export const DEFAULT_CURSOR_CONFIG: CursorRenderConfig = {
+ dotRadius: 28,
+ dotColor: 0xffffff,
+ dotAlpha: 0.95,
+ trailLength: 0,
+ smoothingFactor: 0.18,
+ motionBlur: 0,
+ clickBounce: 1,
+};
+
+const REFERENCE_WIDTH = 1920;
+const MIN_CURSOR_VIEWPORT_SCALE = 0.55;
+const CLICK_ANIMATION_MS = 140;
+const CLICK_RING_FADE_MS = 240;
+const CURSOR_MOTION_BLUR_BASE_MULTIPLIER = 0.08;
+const CURSOR_TIME_DISCONTINUITY_MS = 100;
+const CURSOR_SVG_DROP_SHADOW_FILTER = "drop-shadow(0px 2px 3px rgba(0, 0, 0, 0.35))";
+const CURSOR_SHADOW_COLOR = 0x000000;
+const CURSOR_SHADOW_ALPHA = 0.35;
+const CURSOR_SHADOW_OFFSET_X = 0;
+const CURSOR_SHADOW_OFFSET_Y = 2;
+const CURSOR_SHADOW_BLUR = 3;
+const CURSOR_SHADOW_PADDING = 12;
+
+let cursorAssetsPromise: Promise | null = null;
+let loadedCursorAssets: Partial> = {};
+const SUPPORTED_CURSOR_KEYS: CursorAssetKey[] = [
+ "arrow",
+ "text",
+ "pointer",
+ "crosshair",
+ "open-hand",
+ "closed-hand",
+ "resize-ew",
+ "resize-ns",
+ "not-allowed",
+];
+
+function loadImage(dataUrl: string) {
+ return new Promise((resolve, reject) => {
+ const image = new Image();
+ image.onload = () => resolve(image);
+ image.onerror = () =>
+ reject(new Error(`Failed to load cursor image: ${dataUrl.slice(0, 128)}`));
+ image.src = dataUrl;
+ });
+}
+
+function clamp(value: number, min: number, max: number) {
+ return Math.min(max, Math.max(min, value));
+}
+
+function getNormalizedAnchor(
+ systemAsset: SystemCursorAsset | undefined,
+ fallbackAnchor: { x: number; y: number },
+) {
+ if (!systemAsset || systemAsset.width <= 0 || systemAsset.height <= 0) {
+ return fallbackAnchor;
+ }
+
+ return {
+ x: clamp(systemAsset.hotspotX / systemAsset.width, 0, 1),
+ y: clamp(systemAsset.hotspotY / systemAsset.height, 0, 1),
+ };
+}
+
+/**
+ * Loads an SVG at `sampleSize × sampleSize`, crops the trim region out of it,
+ * and returns a PNG data-URL of the cropped result. This is required because
+ * SVG files have their own natural pixel size (e.g. 32×32) which does not
+ * match the 1024-sample coordinate space used by the trim measurements.
+ */
+async function rasterizeAndCropSvg(
+ url: string,
+ sampleSize: number,
+ trimX: number,
+ trimY: number,
+ trimWidth: number,
+ trimHeight: number,
+): Promise<{ dataUrl: string; width: number; height: number }> {
+ const img = await loadImage(url);
+
+ // Draw at full sample size
+ const srcCanvas = document.createElement("canvas");
+ srcCanvas.width = sampleSize;
+ srcCanvas.height = sampleSize;
+ const srcCtx = srcCanvas.getContext("2d")!;
+ srcCtx.drawImage(img, 0, 0, sampleSize, sampleSize);
+
+ // Crop to trim bounds
+ const dstCanvas = document.createElement("canvas");
+ dstCanvas.width = trimWidth;
+ dstCanvas.height = trimHeight;
+ const dstCtx = dstCanvas.getContext("2d")!;
+ dstCtx.drawImage(srcCanvas, trimX, trimY, trimWidth, trimHeight, 0, 0, trimWidth, trimHeight);
+
+ return {
+ dataUrl: dstCanvas.toDataURL("image/png"),
+ width: dstCanvas.width,
+ height: dstCanvas.height,
+ };
+}
+
+function getCursorAsset(key: CursorAssetKey): LoadedCursorAsset {
+ const asset = loadedCursorAssets[key];
+ if (!asset) {
+ throw new Error(`Missing cursor asset for ${key}`);
+ }
+
+ return asset;
+}
+
+function getAvailableCursorKeys(): CursorAssetKey[] {
+ const loadedKeys = Object.keys(loadedCursorAssets) as CursorAssetKey[];
+ return loadedKeys.length > 0 ? loadedKeys : ["arrow"];
+}
+
+export async function preloadCursorAssets() {
+ if (!cursorAssetsPromise) {
+ cursorAssetsPromise = (async () => {
+ let systemCursors: Record = {};
+
+ try {
+ const api = window.electronAPI as Record;
+ if (typeof api.getSystemCursorAssets === "function") {
+ const result = await (
+ api.getSystemCursorAssets as () => Promise<{
+ success: boolean;
+ cursors?: Record;
+ }>
+ )();
+ if (result.success && result.cursors) {
+ systemCursors = result.cursors;
+ }
+ }
+ } catch (error) {
+ console.warn("[CursorRenderer] Failed to fetch system cursor assets:", error);
+ }
+
+ const entries = await Promise.all(
+ SUPPORTED_CURSOR_KEYS.map(async (key) => {
+ const systemAsset = systemCursors[key];
+ const uploadedAsset = uploadedCursorAssets[key];
+ const assetUrl = uploadedAsset?.url ?? systemAsset?.dataUrl;
+
+ if (!assetUrl) {
+ console.warn(`[CursorRenderer] No cursor image for: ${key}`);
+ return null;
+ }
+
+ try {
+ let finalUrl: string;
+ let width: number;
+ let height: number;
+ let normalizedAnchor: { x: number; y: number };
+
+ if (uploadedAsset) {
+ const { trim, fallbackAnchor } = uploadedAsset;
+ const rasterized = await rasterizeAndCropSvg(
+ assetUrl,
+ UPLOADED_CURSOR_SAMPLE_SIZE,
+ trim.x,
+ trim.y,
+ trim.width,
+ trim.height,
+ );
+ finalUrl = rasterized.dataUrl;
+ width = rasterized.width;
+ height = rasterized.height;
+ normalizedAnchor = {
+ x: clamp((fallbackAnchor.x * trim.width) / width, 0, 1),
+ y: clamp((fallbackAnchor.y * trim.height) / height, 0, 1),
+ };
+ } else {
+ finalUrl = assetUrl;
+ const img = await loadImage(finalUrl);
+ width = img.naturalWidth;
+ height = img.naturalHeight;
+ normalizedAnchor = getNormalizedAnchor(systemAsset, { x: 0, y: 0 });
+ }
+
+ await Assets.load(finalUrl);
+ const image = await loadImage(finalUrl);
+ const texture = Texture.from(finalUrl);
+
+ return [
+ key,
+ {
+ texture,
+ image,
+ aspectRatio: height > 0 ? width / height : 1,
+ anchorX: normalizedAnchor.x,
+ anchorY: normalizedAnchor.y,
+ } satisfies LoadedCursorAsset,
+ ] as const;
+ } catch (error) {
+ console.warn(`[CursorRenderer] Failed to load cursor image for: ${key}`, error);
+ return null;
+ }
+ }),
+ );
+
+ loadedCursorAssets = Object.fromEntries(
+ entries.filter(Boolean).map((entry) => entry!),
+ ) as Partial>;
+
+ if (!loadedCursorAssets.arrow) {
+ throw new Error("Failed to initialize the fallback arrow cursor asset");
+ }
+ })();
+ }
+
+ return cursorAssetsPromise;
+}
+
+/**
+ * Interpolates cursor position from telemetry samples at a given time.
+ * Uses linear interpolation between the two nearest samples.
+ */
+export function interpolateCursorPosition(
+ samples: CursorTelemetryPoint[],
+ timeMs: number,
+): { cx: number; cy: number } | null {
+ if (!samples || samples.length === 0) return null;
+
+ if (timeMs <= samples[0].timeMs) {
+ return { cx: samples[0].cx, cy: samples[0].cy };
+ }
+
+ if (timeMs >= samples[samples.length - 1].timeMs) {
+ return { cx: samples[samples.length - 1].cx, cy: samples[samples.length - 1].cy };
+ }
+
+ let lo = 0;
+ let hi = samples.length - 1;
+ while (lo < hi - 1) {
+ const mid = (lo + hi) >> 1;
+ if (samples[mid].timeMs <= timeMs) {
+ lo = mid;
+ } else {
+ hi = mid;
+ }
+ }
+
+ const a = samples[lo];
+ const b = samples[hi];
+ const span = b.timeMs - a.timeMs;
+ if (span <= 0) return { cx: a.cx, cy: a.cy };
+
+ const t = (timeMs - a.timeMs) / span;
+ return {
+ cx: a.cx + (b.cx - a.cx) * t,
+ cy: a.cy + (b.cy - a.cy) * t,
+ };
+}
+
+function findLatestSample(samples: CursorTelemetryPoint[], timeMs: number) {
+ if (samples.length === 0) return null;
+
+ let lo = 0;
+ let hi = samples.length - 1;
+ while (lo < hi) {
+ const mid = Math.ceil((lo + hi) / 2);
+ if (samples[mid].timeMs <= timeMs) {
+ lo = mid;
+ } else {
+ hi = mid - 1;
+ }
+ }
+
+ return samples[lo]?.timeMs <= timeMs ? samples[lo] : null;
+}
+
+function findLatestInteractionSample(samples: CursorTelemetryPoint[], timeMs: number) {
+ for (let index = samples.length - 1; index >= 0; index -= 1) {
+ const sample = samples[index];
+ if (sample.timeMs > timeMs) {
+ continue;
+ }
+
+ if (
+ sample.interactionType === "click" ||
+ sample.interactionType === "double-click" ||
+ sample.interactionType === "right-click" ||
+ sample.interactionType === "middle-click"
+ ) {
+ return sample;
+ }
+ }
+
+ return null;
+}
+
+function findLatestStableCursorType(samples: CursorTelemetryPoint[], timeMs: number) {
+ // Binary search to find position at timeMs, then scan backwards
+ let lo = 0;
+ let hi = samples.length - 1;
+ while (lo < hi) {
+ const mid = Math.ceil((lo + hi) / 2);
+ if (samples[mid].timeMs <= timeMs) {
+ lo = mid;
+ } else {
+ hi = mid - 1;
+ }
+ }
+
+ // Scan backwards from the position to find a sample with cursorType
+ // Skip click events only (not mouseup) to avoid transient re-type during clicks
+ for (let index = lo; index >= 0; index -= 1) {
+ const sample = samples[index];
+ if (sample.timeMs > timeMs) {
+ continue;
+ }
+
+ if (!sample.cursorType) {
+ continue;
+ }
+
+ if (
+ sample.interactionType === "click" ||
+ sample.interactionType === "double-click" ||
+ sample.interactionType === "right-click" ||
+ sample.interactionType === "middle-click"
+ ) {
+ continue;
+ }
+
+ return sample.cursorType;
+ }
+
+ return findLatestSample(samples, timeMs)?.cursorType ?? "arrow";
+}
+
+function getCursorViewportScale(viewport: CursorViewportRect) {
+ return Math.max(MIN_CURSOR_VIEWPORT_SCALE, viewport.width / REFERENCE_WIDTH);
+}
+
+function getCursorVisualState(samples: CursorTelemetryPoint[], timeMs: number) {
+ const latestClick = findLatestInteractionSample(samples, timeMs);
+ const interactionType = latestClick?.interactionType;
+ const ageMs = latestClick ? Math.max(0, timeMs - latestClick.timeMs) : Number.POSITIVE_INFINITY;
+ const isClickEvent =
+ interactionType === "click" ||
+ interactionType === "double-click" ||
+ interactionType === "right-click" ||
+ interactionType === "middle-click";
+ const clickBounceProgress =
+ latestClick && isClickEvent && ageMs <= CLICK_ANIMATION_MS ? 1 - ageMs / CLICK_ANIMATION_MS : 0;
+
+ return {
+ cursorType: findLatestStableCursorType(samples, timeMs),
+ clickBounceProgress,
+ clickProgress:
+ latestClick && isClickEvent && ageMs <= CLICK_RING_FADE_MS
+ ? 1 - ageMs / CLICK_RING_FADE_MS
+ : 0,
+ };
+}
+
+/**
+ * Manages a smoothed cursor state that chases the interpolated target.
+ */
+export class SmoothedCursorState {
+ public x = 0.5;
+ public y = 0.5;
+ public trail: Array<{ x: number; y: number }> = [];
+ private smoothingFactor: number;
+ private trailLength: number;
+ private initialized = false;
+ private lastTimeMs: number | null = null;
+ private xSpring = createSpringState(0.5);
+ private ySpring = createSpringState(0.5);
+
+ constructor(config: Pick) {
+ this.smoothingFactor = config.smoothingFactor;
+ this.trailLength = config.trailLength;
+ }
+
+ update(targetX: number, targetY: number, timeMs: number): void {
+ if (!this.initialized) {
+ this.x = targetX;
+ this.y = targetY;
+ this.initialized = true;
+ this.lastTimeMs = timeMs;
+ this.xSpring.value = targetX;
+ this.ySpring.value = targetY;
+ this.xSpring.velocity = 0;
+ this.ySpring.velocity = 0;
+ this.xSpring.initialized = true;
+ this.ySpring.initialized = true;
+ this.trail = [];
+ return;
+ }
+
+ if (this.smoothingFactor <= 0 || (this.lastTimeMs !== null && timeMs < this.lastTimeMs)) {
+ this.snapTo(targetX, targetY, timeMs);
+ return;
+ }
+
+ this.trail.unshift({ x: this.x, y: this.y });
+ if (this.trail.length > this.trailLength) {
+ this.trail.length = this.trailLength;
+ }
+
+ const deltaMs = this.lastTimeMs === null ? 1000 / 60 : Math.max(1, timeMs - this.lastTimeMs);
+ this.lastTimeMs = timeMs;
+
+ const springConfig = getCursorSpringConfig(this.smoothingFactor);
+ this.x = stepSpringValue(this.xSpring, targetX, deltaMs, springConfig);
+ this.y = stepSpringValue(this.ySpring, targetY, deltaMs, springConfig);
+ }
+
+ setSmoothingFactor(smoothingFactor: number): void {
+ this.smoothingFactor = smoothingFactor;
+ }
+
+ snapTo(targetX: number, targetY: number, timeMs: number): void {
+ this.x = targetX;
+ this.y = targetY;
+ this.initialized = true;
+ this.lastTimeMs = timeMs;
+ this.xSpring.value = targetX;
+ this.ySpring.value = targetY;
+ this.xSpring.velocity = 0;
+ this.ySpring.velocity = 0;
+ this.xSpring.initialized = true;
+ this.ySpring.initialized = true;
+ this.trail = [];
+ }
+
+ reset(): void {
+ this.initialized = false;
+ this.lastTimeMs = null;
+ this.trail = [];
+ resetSpringState(this.xSpring, this.x);
+ resetSpringState(this.ySpring, this.y);
+ }
+}
+
+function drawClickRing(graphics: Graphics, px: number, py: number, h: number, progress: number) {
+ void graphics;
+ void px;
+ void py;
+ void h;
+ void progress;
+}
+
+export class PixiCursorOverlay {
+ public readonly container: Container;
+ private clickRingGraphics: Graphics;
+ private cursorShadowSprites: Partial>;
+ private cursorShadowFilters: Partial>;
+ private cursorSprites: Partial>;
+ private cursorMotionBlurFilter: MotionBlurFilter;
+ private state: SmoothedCursorState;
+ private config: CursorRenderConfig;
+ private lastRenderedPoint: { px: number; py: number } | null = null;
+ private lastRenderedTimeMs: number | null = null;
+
+ constructor(config: Partial = {}) {
+ this.config = { ...DEFAULT_CURSOR_CONFIG, ...config };
+ this.state = new SmoothedCursorState(this.config);
+
+ this.container = new Container();
+ this.container.label = "cursor-overlay";
+
+ this.clickRingGraphics = new Graphics();
+ this.cursorShadowSprites = {};
+ this.cursorShadowFilters = {};
+ this.cursorSprites = {};
+ for (const key of getAvailableCursorKeys()) {
+ const asset = getCursorAsset(key);
+ const shadowSprite = new Sprite(asset.texture);
+ shadowSprite.anchor.set(asset.anchorX, asset.anchorY);
+ shadowSprite.visible = false;
+ shadowSprite.tint = CURSOR_SHADOW_COLOR;
+ shadowSprite.alpha = CURSOR_SHADOW_ALPHA;
+ const shadowFilter = new BlurFilter();
+ shadowFilter.blur = CURSOR_SHADOW_BLUR;
+ shadowFilter.quality = 4;
+ shadowFilter.padding = CURSOR_SHADOW_PADDING;
+ shadowSprite.filters = [shadowFilter];
+ this.cursorShadowSprites[key] = shadowSprite;
+ this.cursorShadowFilters[key] = shadowFilter;
+
+ const sprite = new Sprite(asset.texture);
+ sprite.anchor.set(asset.anchorX, asset.anchorY);
+ sprite.visible = false;
+ this.cursorSprites[key] = sprite;
+ }
+
+ this.cursorMotionBlurFilter = new MotionBlurFilter([0, 0], 5, 0);
+ this.container.filters = null;
+
+ this.container.addChild(
+ this.clickRingGraphics,
+ ...Object.values(this.cursorShadowSprites),
+ ...Object.values(this.cursorSprites),
+ );
+ this.setMotionBlur(this.config.motionBlur);
+ }
+
+ setDotRadius(dotRadius: number) {
+ this.config.dotRadius = dotRadius;
+ }
+
+ setSmoothingFactor(smoothingFactor: number) {
+ this.config.smoothingFactor = smoothingFactor;
+ this.state.setSmoothingFactor(smoothingFactor);
+ }
+
+ setMotionBlur(motionBlur: number) {
+ this.config.motionBlur = Math.max(0, motionBlur);
+ this.container.filters = this.config.motionBlur > 0 ? [this.cursorMotionBlurFilter] : null;
+ if (this.config.motionBlur <= 0) {
+ this.cursorMotionBlurFilter.velocity = { x: 0, y: 0 };
+ this.cursorMotionBlurFilter.kernelSize = 5;
+ this.cursorMotionBlurFilter.offset = 0;
+ }
+ }
+
+ setClickBounce(clickBounce: number) {
+ this.config.clickBounce = Math.max(0, clickBounce);
+ }
+
+ update(
+ samples: CursorTelemetryPoint[],
+ timeMs: number,
+ viewport: CursorViewportRect,
+ visible: boolean,
+ freeze = false,
+ ): void {
+ if (!visible || samples.length === 0 || viewport.width <= 0 || viewport.height <= 0) {
+ this.container.visible = false;
+ this.lastRenderedPoint = null;
+ this.lastRenderedTimeMs = null;
+ this.cursorMotionBlurFilter.velocity = { x: 0, y: 0 };
+ return;
+ }
+
+ const target = interpolateCursorPosition(samples, timeMs);
+ if (!target) {
+ this.container.visible = false;
+ return;
+ }
+
+ const sameFrameTime =
+ this.lastRenderedTimeMs !== null && Math.abs(this.lastRenderedTimeMs - timeMs) < 0.0001;
+ const hasTimeDiscontinuity =
+ this.lastRenderedTimeMs !== null &&
+ Math.abs(timeMs - this.lastRenderedTimeMs) > CURSOR_TIME_DISCONTINUITY_MS;
+
+ if (freeze || hasTimeDiscontinuity) {
+ if (!sameFrameTime || !this.lastRenderedPoint) {
+ this.state.snapTo(target.cx, target.cy, timeMs);
+ }
+ } else {
+ this.state.update(target.cx, target.cy, timeMs);
+ }
+ this.container.visible = true;
+
+ const px = viewport.x + this.state.x * viewport.width;
+ const py = viewport.y + this.state.y * viewport.height;
+ const h = this.config.dotRadius * getCursorViewportScale(viewport);
+ const { cursorType, clickBounceProgress, clickProgress } = getCursorVisualState(
+ samples,
+ timeMs,
+ );
+ const spriteKey = (cursorType in this.cursorSprites ? cursorType : "arrow") as CursorAssetKey;
+ const asset = getCursorAsset(spriteKey);
+ const shadowSprite = this.cursorShadowSprites[spriteKey] ?? this.cursorShadowSprites.arrow!;
+ const sprite = this.cursorSprites[spriteKey] ?? this.cursorSprites.arrow!;
+ const bounceScale = Math.max(
+ 0.72,
+ 1 - Math.sin(clickBounceProgress * Math.PI) * (0.08 * this.config.clickBounce),
+ );
+ const scaledH = h;
+
+ this.clickRingGraphics.clear();
+ drawClickRing(this.clickRingGraphics, px, py, h, clickProgress);
+
+ for (const [key, currentShadowSprite] of Object.entries(this.cursorShadowSprites) as Array<
+ [CursorAssetKey, Sprite]
+ >) {
+ currentShadowSprite.visible = key === spriteKey;
+ }
+
+ for (const [key, currentSprite] of Object.entries(this.cursorSprites) as Array<
+ [CursorAssetKey, Sprite]
+ >) {
+ currentSprite.visible = key === spriteKey;
+ }
+
+ if (shadowSprite) {
+ shadowSprite.height = scaledH * bounceScale;
+ shadowSprite.width = scaledH * bounceScale * asset.aspectRatio;
+ shadowSprite.position.set(px + CURSOR_SHADOW_OFFSET_X, py + CURSOR_SHADOW_OFFSET_Y);
+ }
+
+ if (sprite) {
+ sprite.alpha = this.config.dotAlpha;
+ sprite.height = scaledH * bounceScale;
+ sprite.width = scaledH * bounceScale * asset.aspectRatio;
+ sprite.position.set(px, py);
+ }
+
+ this.applyCursorMotionBlur(px, py, timeMs, freeze);
+ this.lastRenderedPoint = { px, py };
+ this.lastRenderedTimeMs = timeMs;
+ }
+
+ private applyCursorMotionBlur(px: number, py: number, timeMs: number, freeze: boolean) {
+ if (
+ freeze ||
+ this.config.motionBlur <= 0 ||
+ !this.lastRenderedPoint ||
+ this.lastRenderedTimeMs === null
+ ) {
+ this.cursorMotionBlurFilter.velocity = { x: 0, y: 0 };
+ this.cursorMotionBlurFilter.kernelSize = 5;
+ this.cursorMotionBlurFilter.offset = 0;
+ return;
+ }
+
+ const deltaMs = Math.max(1, timeMs - this.lastRenderedTimeMs);
+ const dx = px - this.lastRenderedPoint.px;
+ const dy = py - this.lastRenderedPoint.py;
+ const velocityScale =
+ (1000 / deltaMs) * this.config.motionBlur * CURSOR_MOTION_BLUR_BASE_MULTIPLIER;
+ const velocity = {
+ x: dx * velocityScale,
+ y: dy * velocityScale,
+ };
+ const magnitude = Math.hypot(velocity.x, velocity.y);
+
+ this.cursorMotionBlurFilter.velocity = magnitude > 0.05 ? velocity : { x: 0, y: 0 };
+ this.cursorMotionBlurFilter.kernelSize = magnitude > 3 ? 9 : magnitude > 1 ? 7 : 5;
+ this.cursorMotionBlurFilter.offset = magnitude > 0.5 ? -0.25 : 0;
+ }
+
+ reset(): void {
+ this.state.reset();
+ this.clickRingGraphics.clear();
+ for (const shadowSprite of Object.values(this.cursorShadowSprites)) {
+ shadowSprite.visible = false;
+ shadowSprite.scale.set(1);
+ }
+ for (const sprite of Object.values(this.cursorSprites)) {
+ sprite.visible = false;
+ sprite.scale.set(1);
+ }
+ this.container.visible = false;
+ this.lastRenderedPoint = null;
+ this.lastRenderedTimeMs = null;
+ this.cursorMotionBlurFilter.velocity = { x: 0, y: 0 };
+ this.cursorMotionBlurFilter.kernelSize = 5;
+ this.cursorMotionBlurFilter.offset = 0;
+ }
+
+ destroy(): void {
+ this.clickRingGraphics.destroy();
+ for (const shadowFilter of Object.values(this.cursorShadowFilters)) {
+ shadowFilter.destroy();
+ }
+ this.cursorMotionBlurFilter.destroy();
+ this.container.destroy({ children: true });
+ }
+}
+
+export function drawCursorOnCanvas(
+ ctx: CanvasRenderingContext2D,
+ samples: CursorTelemetryPoint[],
+ timeMs: number,
+ viewport: CursorViewportRect,
+ smoothedState: SmoothedCursorState,
+ config: CursorRenderConfig = DEFAULT_CURSOR_CONFIG,
+): void {
+ if (samples.length === 0 || viewport.width <= 0 || viewport.height <= 0) return;
+
+ const target = interpolateCursorPosition(samples, timeMs);
+ if (!target) return;
+
+ smoothedState.update(target.cx, target.cy, timeMs);
+
+ const px = viewport.x + smoothedState.x * viewport.width;
+ const py = viewport.y + smoothedState.y * viewport.height;
+ const h = config.dotRadius * getCursorViewportScale(viewport);
+ const { cursorType, clickBounceProgress } = getCursorVisualState(samples, timeMs);
+ const spriteKey = (
+ cursorType && loadedCursorAssets[cursorType] ? cursorType : "arrow"
+ ) as CursorAssetKey;
+ const asset = getCursorAsset(spriteKey);
+ const bounceScale = Math.max(
+ 0.72,
+ 1 - Math.sin(clickBounceProgress * Math.PI) * (0.08 * config.clickBounce),
+ );
+
+ ctx.save();
+ ctx.filter = CURSOR_SVG_DROP_SHADOW_FILTER;
+
+ const drawHeight = h * bounceScale;
+ const drawWidth = drawHeight * asset.aspectRatio;
+ const hotspotX = asset.anchorX * drawWidth;
+ const hotspotY = asset.anchorY * drawHeight;
+ ctx.globalAlpha = config.dotAlpha;
+ ctx.drawImage(asset.image, px - hotspotX, py - hotspotY, drawWidth, drawHeight);
+
+ ctx.restore();
+}
diff --git a/src/components/video-editor/videoPlayback/motionSmoothing.ts b/src/components/video-editor/videoPlayback/motionSmoothing.ts
new file mode 100644
index 0000000..e2cec6f
--- /dev/null
+++ b/src/components/video-editor/videoPlayback/motionSmoothing.ts
@@ -0,0 +1,149 @@
+import { spring } from "motion";
+
+export interface SpringState {
+ value: number;
+ velocity: number;
+ initialized: boolean;
+}
+
+export interface SpringConfig {
+ stiffness: number;
+ damping: number;
+ mass: number;
+ restDelta?: number;
+ restSpeed?: number;
+}
+
+const CURSOR_SMOOTHING_MIN = 0;
+const CURSOR_SMOOTHING_MAX = 2;
+const CURSOR_SMOOTHING_LEGACY_MAX = 0.5;
+
+export function createSpringState(initialValue = 0): SpringState {
+ return {
+ value: initialValue,
+ velocity: 0,
+ initialized: false,
+ };
+}
+
+export function resetSpringState(state: SpringState, initialValue?: number) {
+ if (typeof initialValue === "number") {
+ state.value = initialValue;
+ }
+
+ state.velocity = 0;
+ state.initialized = false;
+}
+
+export function clampDeltaMs(deltaMs: number, fallbackMs = 1000 / 60) {
+ if (!Number.isFinite(deltaMs) || deltaMs <= 0) {
+ return fallbackMs;
+ }
+
+ return Math.min(80, Math.max(1, deltaMs));
+}
+
+export function stepSpringValue(
+ state: SpringState,
+ target: number,
+ deltaMs: number,
+ config: SpringConfig,
+) {
+ const safeDeltaMs = clampDeltaMs(deltaMs);
+
+ if (!state.initialized || !Number.isFinite(state.value)) {
+ state.value = target;
+ state.velocity = 0;
+ state.initialized = true;
+ return state.value;
+ }
+
+ const restDelta = config.restDelta ?? 0.0005;
+ const restSpeed = config.restSpeed ?? 0.02;
+
+ if (Math.abs(target - state.value) <= restDelta && Math.abs(state.velocity) <= restSpeed) {
+ state.value = target;
+ state.velocity = 0;
+ return state.value;
+ }
+
+ const previousValue = state.value;
+ const generator = spring({
+ keyframes: [state.value, target],
+ velocity: state.velocity,
+ stiffness: config.stiffness,
+ damping: config.damping,
+ mass: config.mass,
+ restDelta,
+ restSpeed,
+ });
+
+ const result = generator.next(safeDeltaMs);
+ state.value = result.done ? target : result.value;
+ state.velocity = ((state.value - previousValue) / safeDeltaMs) * 1000;
+
+ if (result.done) {
+ state.velocity = 0;
+ }
+
+ return state.value;
+}
+
+export function getCursorSpringConfig(smoothingFactor: number): SpringConfig {
+ const clamped = Math.min(CURSOR_SMOOTHING_MAX, Math.max(CURSOR_SMOOTHING_MIN, smoothingFactor));
+
+ if (clamped <= 0) {
+ return {
+ stiffness: 1000,
+ damping: 100,
+ mass: 1,
+ restDelta: 0.0001,
+ restSpeed: 0.001,
+ };
+ }
+
+ if (clamped <= CURSOR_SMOOTHING_LEGACY_MAX) {
+ const legacyNormalized = Math.min(
+ 1,
+ Math.max(
+ 0,
+ (clamped - CURSOR_SMOOTHING_MIN) / (CURSOR_SMOOTHING_LEGACY_MAX - CURSOR_SMOOTHING_MIN),
+ ),
+ );
+
+ return {
+ stiffness: 760 - legacyNormalized * 420,
+ damping: 34 + legacyNormalized * 24,
+ mass: 0.55 + legacyNormalized * 0.45,
+ restDelta: 0.0002,
+ restSpeed: 0.01,
+ };
+ }
+
+ const extendedNormalized = Math.min(
+ 1,
+ Math.max(
+ 0,
+ (clamped - CURSOR_SMOOTHING_LEGACY_MAX) /
+ (CURSOR_SMOOTHING_MAX - CURSOR_SMOOTHING_LEGACY_MAX),
+ ),
+ );
+
+ return {
+ stiffness: 340 - extendedNormalized * 180,
+ damping: 58 + extendedNormalized * 22,
+ mass: 1 + extendedNormalized * 0.35,
+ restDelta: 0.0002,
+ restSpeed: 0.01,
+ };
+}
+
+export function getZoomSpringConfig(): SpringConfig {
+ return {
+ stiffness: 320,
+ damping: 40,
+ mass: 0.92,
+ restDelta: 0.0005,
+ restSpeed: 0.015,
+ };
+}
diff --git a/src/components/video-editor/videoPlayback/uploadedCursorAssets.ts b/src/components/video-editor/videoPlayback/uploadedCursorAssets.ts
new file mode 100644
index 0000000..4a0cd29
--- /dev/null
+++ b/src/components/video-editor/videoPlayback/uploadedCursorAssets.ts
@@ -0,0 +1,70 @@
+import crosshairUrl from "../../../assets/cursors/Cursor=Cross.svg";
+import arrowUrl from "../../../assets/cursors/Cursor=Default.svg";
+import closedHandUrl from "../../../assets/cursors/Cursor=Hand-(Grabbing).svg";
+import openHandUrl from "../../../assets/cursors/Cursor=Hand-(Open).svg";
+import pointerUrl from "../../../assets/cursors/Cursor=Hand-(Pointing).svg";
+import resizeNsUrl from "../../../assets/cursors/Cursor=Resize-North-South.svg";
+import resizeEwUrl from "../../../assets/cursors/Cursor=Resize-West-East.svg";
+import textUrl from "../../../assets/cursors/Cursor=Text-Cursor.svg";
+import type { CursorTelemetryPoint } from "../types";
+
+type CursorAssetKey = NonNullable;
+
+export type UploadedCursorAsset = {
+ url: string;
+ trim: {
+ x: number;
+ y: number;
+ width: number;
+ height: number;
+ };
+ fallbackAnchor: {
+ x: number;
+ y: number;
+ };
+};
+
+export const UPLOADED_CURSOR_SAMPLE_SIZE = 1024;
+
+export const uploadedCursorAssets: Partial> = {
+ arrow: {
+ url: arrowUrl,
+ trim: { x: 480, y: 435, width: 333, height: 553 },
+ fallbackAnchor: { x: 0.18, y: 0.1 },
+ },
+ text: {
+ url: textUrl,
+ trim: { x: 404, y: 192, width: 247, height: 596 },
+ fallbackAnchor: { x: 0.5, y: 0.5 },
+ },
+ pointer: {
+ url: pointerUrl,
+ trim: { x: 352, y: 441, width: 466, height: 583 },
+ fallbackAnchor: { x: 0.37, y: 0.08 },
+ },
+ crosshair: {
+ url: crosshairUrl,
+ trim: { x: 288, y: 288, width: 480, height: 480 },
+ fallbackAnchor: { x: 0.5, y: 0.5 },
+ },
+ "open-hand": {
+ url: openHandUrl,
+ trim: { x: 288, y: 188, width: 512, height: 580 },
+ fallbackAnchor: { x: 0.5, y: 0.28 },
+ },
+ "closed-hand": {
+ url: closedHandUrl,
+ trim: { x: 344, y: 365, width: 432, height: 403 },
+ fallbackAnchor: { x: 0.5, y: 0.28 },
+ },
+ "resize-ew": {
+ url: resizeEwUrl,
+ trim: { x: 187, y: 384, width: 669, height: 270 },
+ fallbackAnchor: { x: 0.5, y: 0.5 },
+ },
+ "resize-ns": {
+ url: resizeNsUrl,
+ trim: { x: 376, y: 178, width: 271, height: 669 },
+ fallbackAnchor: { x: 0.5, y: 0.5 },
+ },
+};
From 44f59bfa8901cec17cb34bfcaf03b0be464d3bd6 Mon Sep 17 00:00:00 2001
From: Etienne Lescot
Date: Mon, 16 Mar 2026 10:41:35 +0100
Subject: [PATCH 04/92] feat: add unified native bridge foundation
---
docs/architecture/native-bridge.md | 39 +++
electron/electron-env.d.ts | 3 +
electron/ipc/nativeBridge.ts | 229 ++++++++++++++++++
electron/native-bridge/cursor/adapter.ts | 20 ++
.../cursor/telemetryCursorAdapter.ts | 48 ++++
.../native-bridge/services/cursorService.ts | 46 ++++
.../native-bridge/services/projectService.ts | 80 ++++++
.../native-bridge/services/systemService.ts | 43 ++++
electron/native-bridge/store.ts | 88 +++++++
electron/preload.ts | 5 +-
src/components/launch/LaunchWindow.tsx | 5 +-
src/hooks/useScreenRecorder.ts | 3 +-
src/native/client.ts | 133 ++++++++++
src/native/contracts.ts | 209 ++++++++++++++++
src/native/hooks/useCursorRecordingData.ts | 61 +++++
src/native/hooks/useCursorTelemetry.ts | 61 +++++
src/native/index.ts | 4 +
17 files changed, 1072 insertions(+), 5 deletions(-)
create mode 100644 docs/architecture/native-bridge.md
create mode 100644 electron/ipc/nativeBridge.ts
create mode 100644 electron/native-bridge/cursor/adapter.ts
create mode 100644 electron/native-bridge/cursor/telemetryCursorAdapter.ts
create mode 100644 electron/native-bridge/services/cursorService.ts
create mode 100644 electron/native-bridge/services/projectService.ts
create mode 100644 electron/native-bridge/services/systemService.ts
create mode 100644 electron/native-bridge/store.ts
create mode 100644 src/native/client.ts
create mode 100644 src/native/contracts.ts
create mode 100644 src/native/hooks/useCursorRecordingData.ts
create mode 100644 src/native/hooks/useCursorTelemetry.ts
create mode 100644 src/native/index.ts
diff --git a/docs/architecture/native-bridge.md b/docs/architecture/native-bridge.md
new file mode 100644
index 0000000..ef320f7
--- /dev/null
+++ b/docs/architecture/native-bridge.md
@@ -0,0 +1,39 @@
+# Native Bridge Architecture
+
+## Goal
+
+Provide a single, resilient source of truth for platform-native capabilities while keeping Electron transport thin and renderer APIs unified.
+
+## Layers
+
+1. Native adapters
+Platform-specific providers implement stable domain interfaces such as cursor telemetry or system asset discovery.
+
+2. Main-process services
+Services orchestrate adapters, own runtime state, and expose domain-level operations.
+
+3. Unified IPC transport
+Renderer code talks to a single `native-bridge:invoke` channel using versioned contracts.
+
+4. Renderer client
+React code should consume `src/native/client.ts` rather than binding directly to ad hoc Electron APIs.
+
+## Principles
+
+- Single source of truth: runtime-native state lives in the Electron main process.
+- Capability-first: renderer can query support before attempting native behavior.
+- Versioned contracts: requests and responses are explicit and evolve predictably.
+- Resilience: every response uses a consistent result envelope with stable error codes.
+
+## Current rollout
+
+This repository now contains the initial scaffold:
+
+- shared contracts in `src/native/contracts.ts`
+- renderer SDK in `src/native/client.ts`
+- main-process state store in `electron/native-bridge/store.ts`
+- cursor telemetry adapter in `electron/native-bridge/cursor/telemetryCursorAdapter.ts`
+- domain services in `electron/native-bridge/services/*`
+- unified handler registration in `electron/ipc/nativeBridge.ts`
+
+The legacy `window.electronAPI` surface still exists for backward compatibility. New native-facing features should prefer the unified bridge client.
\ No newline at end of file
diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts
index 1d528cd..686575d 100644
--- a/electron/electron-env.d.ts
+++ b/electron/electron-env.d.ts
@@ -24,6 +24,9 @@ declare namespace NodeJS {
// Used in Renderer process, expose in `preload.ts`
interface Window {
electronAPI: {
+ invokeNativeBridge: (
+ request: import("../src/native/contracts").NativeBridgeRequest,
+ ) => Promise>;
getSources: (opts: Electron.SourcesOptions) => Promise;
switchToEditor: () => Promise;
switchToHud: () => Promise;
diff --git a/electron/ipc/nativeBridge.ts b/electron/ipc/nativeBridge.ts
new file mode 100644
index 0000000..ba6258a
--- /dev/null
+++ b/electron/ipc/nativeBridge.ts
@@ -0,0 +1,229 @@
+import { ipcMain } from "electron";
+import {
+ NATIVE_BRIDGE_CHANNEL,
+ NATIVE_BRIDGE_VERSION,
+ type NativeBridgeErrorCode,
+ type NativeBridgeRequest,
+ type NativeBridgeResponse,
+ type NativePlatform,
+ type ProjectFileResult,
+ type ProjectPathResult,
+} from "../../src/native/contracts";
+import type { CursorTelemetryLoadResult } from "../native-bridge/cursor/adapter";
+import { TelemetryCursorAdapter } from "../native-bridge/cursor/telemetryCursorAdapter";
+import { CursorService } from "../native-bridge/services/cursorService";
+import { ProjectService } from "../native-bridge/services/projectService";
+import { SystemService } from "../native-bridge/services/systemService";
+import { NativeBridgeStateStore } from "../native-bridge/store";
+
+export interface NativeBridgeContext {
+ getPlatform: () => NodeJS.Platform;
+ getCurrentProjectPath: () => string | null;
+ getCurrentVideoPath: () => string | null;
+ saveProjectFile: (
+ projectData: unknown,
+ suggestedName?: string,
+ existingProjectPath?: string,
+ ) => Promise;
+ loadProjectFile: () => Promise;
+ loadCurrentProjectFile: () => Promise;
+ setCurrentVideoPath: (path: string) => ProjectPathResult;
+ getCurrentVideoPathResult: () => ProjectPathResult;
+ clearCurrentVideoPath: () => ProjectPathResult;
+ resolveAssetBasePath: () => string | null;
+ resolveVideoPath: (videoPath?: string | null) => string | null;
+ loadCursorRecordingData: (
+ videoPath: string,
+ ) => Promise;
+ loadCursorTelemetry: (videoPath: string) => Promise;
+}
+
+function normalizePlatform(platform: NodeJS.Platform): NativePlatform {
+ if (platform === "darwin" || platform === "win32") {
+ return platform;
+ }
+
+ return "linux";
+}
+
+function createMeta(requestId?: string) {
+ return {
+ version: NATIVE_BRIDGE_VERSION,
+ requestId: requestId || `native-${Date.now()}`,
+ timestampMs: Date.now(),
+ } as const;
+}
+
+function createSuccessResponse(requestId: string | undefined, data: TData) {
+ return {
+ ok: true,
+ data,
+ meta: createMeta(requestId),
+ } satisfies NativeBridgeResponse;
+}
+
+function createErrorResponse(
+ requestId: string | undefined,
+ code: NativeBridgeErrorCode,
+ message: string,
+ retryable = false,
+) {
+ return {
+ ok: false,
+ error: {
+ code,
+ message,
+ retryable,
+ },
+ meta: createMeta(requestId),
+ } satisfies NativeBridgeResponse;
+}
+
+function isBridgeRequest(value: unknown): value is NativeBridgeRequest {
+ if (!value || typeof value !== "object") {
+ return false;
+ }
+
+ const candidate = value as Partial;
+ return typeof candidate.domain === "string" && typeof candidate.action === "string";
+}
+
+export function registerNativeBridgeHandlers(context: NativeBridgeContext) {
+ ipcMain.removeHandler(NATIVE_BRIDGE_CHANNEL);
+
+ const platform = normalizePlatform(context.getPlatform());
+ const store = new NativeBridgeStateStore(platform);
+ const projectService = new ProjectService({
+ store,
+ getCurrentProjectPath: context.getCurrentProjectPath,
+ getCurrentVideoPath: context.getCurrentVideoPath,
+ saveProjectFile: context.saveProjectFile,
+ loadProjectFile: context.loadProjectFile,
+ loadCurrentProjectFile: context.loadCurrentProjectFile,
+ setCurrentVideoPath: context.setCurrentVideoPath,
+ getCurrentVideoPathResult: context.getCurrentVideoPathResult,
+ clearCurrentVideoPath: context.clearCurrentVideoPath,
+ });
+ const cursorService = new CursorService({
+ store,
+ adapter: new TelemetryCursorAdapter({
+ loadRecordingData: context.loadCursorRecordingData,
+ resolveVideoPath: context.resolveVideoPath,
+ loadTelemetry: context.loadCursorTelemetry,
+ }),
+ });
+ const systemService = new SystemService({
+ store,
+ getPlatform: () => platform,
+ getAssetBasePath: context.resolveAssetBasePath,
+ getCursorCapabilities: () => cursorService.getCapabilities(),
+ });
+
+ ipcMain.handle(NATIVE_BRIDGE_CHANNEL, async (_, request: unknown) => {
+ if (!isBridgeRequest(request)) {
+ return createErrorResponse(undefined, "INVALID_REQUEST", "Invalid native bridge request.");
+ }
+
+ const requestId = request.requestId;
+ const domain = request.domain as string;
+
+ try {
+ switch (request.domain) {
+ case "system": {
+ const action = request.action as string;
+ switch (request.action) {
+ case "getPlatform":
+ return createSuccessResponse(requestId, systemService.getPlatform());
+ case "getAssetBasePath":
+ return createSuccessResponse(requestId, systemService.getAssetBasePath());
+ case "getCapabilities":
+ return createSuccessResponse(requestId, await systemService.getCapabilities());
+ default:
+ return createErrorResponse(
+ requestId,
+ "UNSUPPORTED_ACTION",
+ `Unsupported system action: ${action}`,
+ );
+ }
+ }
+
+ case "project": {
+ const action = request.action as string;
+ switch (request.action) {
+ case "getCurrentContext":
+ return createSuccessResponse(requestId, projectService.getCurrentContext());
+ case "saveProjectFile":
+ return createSuccessResponse(
+ requestId,
+ await projectService.saveProjectFile(
+ request.payload.projectData,
+ request.payload.suggestedName,
+ request.payload.existingProjectPath,
+ ),
+ );
+ case "loadProjectFile":
+ return createSuccessResponse(requestId, await projectService.loadProjectFile());
+ case "loadCurrentProjectFile":
+ return createSuccessResponse(
+ requestId,
+ await projectService.loadCurrentProjectFile(),
+ );
+ case "setCurrentVideoPath":
+ return createSuccessResponse(
+ requestId,
+ projectService.setCurrentVideoPath(request.payload.path),
+ );
+ case "getCurrentVideoPath":
+ return createSuccessResponse(requestId, projectService.getCurrentVideoPath());
+ case "clearCurrentVideoPath":
+ return createSuccessResponse(requestId, projectService.clearCurrentVideoPath());
+ default:
+ return createErrorResponse(
+ requestId,
+ "UNSUPPORTED_ACTION",
+ `Unsupported project action: ${action}`,
+ );
+ }
+ }
+
+ case "cursor": {
+ const action = request.action as string;
+ switch (request.action) {
+ case "getCapabilities":
+ return createSuccessResponse(requestId, await cursorService.getCapabilities());
+ case "getTelemetry":
+ return createSuccessResponse(
+ requestId,
+ await cursorService.getTelemetry(request.payload?.videoPath),
+ );
+ case "getRecordingData":
+ return createSuccessResponse(
+ requestId,
+ await cursorService.getRecordingData(request.payload?.videoPath),
+ );
+ default:
+ return createErrorResponse(
+ requestId,
+ "UNSUPPORTED_ACTION",
+ `Unsupported cursor action: ${action}`,
+ );
+ }
+ }
+
+ default:
+ return createErrorResponse(
+ requestId,
+ "UNSUPPORTED_ACTION",
+ `Unsupported bridge domain: ${domain}`,
+ );
+ }
+ } catch (error) {
+ return createErrorResponse(
+ requestId,
+ "INTERNAL_ERROR",
+ error instanceof Error ? error.message : "Unknown native bridge error.",
+ true,
+ );
+ }
+ });
+}
diff --git a/electron/native-bridge/cursor/adapter.ts b/electron/native-bridge/cursor/adapter.ts
new file mode 100644
index 0000000..cdb88e2
--- /dev/null
+++ b/electron/native-bridge/cursor/adapter.ts
@@ -0,0 +1,20 @@
+import type {
+ CursorCapabilities,
+ CursorProviderKind,
+ CursorRecordingData,
+ CursorTelemetryPoint,
+} from "../../../src/native/contracts";
+
+export interface CursorTelemetryLoadResult {
+ success: boolean;
+ samples: CursorTelemetryPoint[];
+ message?: string;
+ error?: string;
+}
+
+export interface CursorNativeAdapter {
+ readonly kind: CursorProviderKind;
+ getCapabilities(): Promise;
+ getRecordingData(videoPath?: string | null): Promise;
+ getTelemetry(videoPath?: string | null): Promise;
+}
diff --git a/electron/native-bridge/cursor/telemetryCursorAdapter.ts b/electron/native-bridge/cursor/telemetryCursorAdapter.ts
new file mode 100644
index 0000000..d083995
--- /dev/null
+++ b/electron/native-bridge/cursor/telemetryCursorAdapter.ts
@@ -0,0 +1,48 @@
+import type { CursorCapabilities, CursorRecordingData } from "../../../src/native/contracts";
+import type { CursorNativeAdapter, CursorTelemetryLoadResult } from "./adapter";
+
+interface TelemetryCursorAdapterOptions {
+ loadRecordingData: (videoPath: string) => Promise;
+ resolveVideoPath: (videoPath?: string | null) => string | null;
+ loadTelemetry: (videoPath: string) => Promise;
+}
+
+export class TelemetryCursorAdapter implements CursorNativeAdapter {
+ readonly kind = "none" as const;
+
+ constructor(private readonly options: TelemetryCursorAdapterOptions) {}
+
+ async getCapabilities(): Promise {
+ return {
+ telemetry: true,
+ systemAssets: false,
+ provider: this.kind,
+ };
+ }
+
+ async getRecordingData(videoPath?: string | null): Promise {
+ const resolvedVideoPath = this.options.resolveVideoPath(videoPath);
+ if (!resolvedVideoPath) {
+ return {
+ version: 2,
+ provider: this.kind,
+ samples: [],
+ assets: [],
+ };
+ }
+
+ return this.options.loadRecordingData(resolvedVideoPath);
+ }
+
+ async getTelemetry(videoPath?: string | null) {
+ const resolvedVideoPath = this.options.resolveVideoPath(videoPath);
+ if (!resolvedVideoPath) {
+ return {
+ success: true,
+ samples: [],
+ } satisfies CursorTelemetryLoadResult;
+ }
+
+ return this.options.loadTelemetry(resolvedVideoPath);
+ }
+}
diff --git a/electron/native-bridge/services/cursorService.ts b/electron/native-bridge/services/cursorService.ts
new file mode 100644
index 0000000..e3e9a25
--- /dev/null
+++ b/electron/native-bridge/services/cursorService.ts
@@ -0,0 +1,46 @@
+import type {
+ CursorCapabilities,
+ CursorRecordingData,
+ CursorTelemetryPoint,
+} from "../../../src/native/contracts";
+import type { CursorNativeAdapter } from "../cursor/adapter";
+import type { NativeBridgeStateStore } from "../store";
+
+interface CursorServiceOptions {
+ store: NativeBridgeStateStore;
+ adapter: CursorNativeAdapter;
+}
+
+export class CursorService {
+ constructor(private readonly options: CursorServiceOptions) {}
+
+ async getCapabilities(): Promise {
+ const capabilities = await this.options.adapter.getCapabilities();
+ this.options.store.setCursorCapabilities(capabilities);
+ return capabilities;
+ }
+
+ async getTelemetry(videoPath?: string | null): Promise {
+ const result = await this.options.adapter.getTelemetry(videoPath);
+ if (!result.success) {
+ throw new Error(result.message || result.error || "Failed to load cursor telemetry");
+ }
+
+ const resolvedVideoPath = videoPath ?? this.options.store.getState().project.currentVideoPath;
+ if (resolvedVideoPath) {
+ this.options.store.markCursorTelemetryLoaded(resolvedVideoPath, result.samples.length);
+ }
+
+ return result.samples;
+ }
+
+ async getRecordingData(videoPath?: string | null): Promise {
+ const data = await this.options.adapter.getRecordingData(videoPath);
+ const resolvedVideoPath = videoPath ?? this.options.store.getState().project.currentVideoPath;
+ if (resolvedVideoPath) {
+ this.options.store.markCursorTelemetryLoaded(resolvedVideoPath, data.samples.length);
+ }
+
+ return data;
+ }
+}
diff --git a/electron/native-bridge/services/projectService.ts b/electron/native-bridge/services/projectService.ts
new file mode 100644
index 0000000..e8d1cd5
--- /dev/null
+++ b/electron/native-bridge/services/projectService.ts
@@ -0,0 +1,80 @@
+import type {
+ ProjectContext,
+ ProjectFileResult,
+ ProjectPathResult,
+} from "../../../src/native/contracts";
+import type { NativeBridgeStateStore } from "../store";
+
+interface ProjectServiceOptions {
+ store: NativeBridgeStateStore;
+ getCurrentProjectPath: () => string | null;
+ getCurrentVideoPath: () => string | null;
+ saveProjectFile: (
+ projectData: unknown,
+ suggestedName?: string,
+ existingProjectPath?: string,
+ ) => Promise;
+ loadProjectFile: () => Promise;
+ loadCurrentProjectFile: () => Promise;
+ setCurrentVideoPath: (path: string) => ProjectPathResult;
+ getCurrentVideoPathResult: () => ProjectPathResult;
+ clearCurrentVideoPath: () => ProjectPathResult;
+}
+
+export class ProjectService {
+ constructor(private readonly options: ProjectServiceOptions) {}
+
+ getCurrentContext(): ProjectContext {
+ const context = {
+ currentProjectPath: this.options.getCurrentProjectPath(),
+ currentVideoPath: this.options.getCurrentVideoPath(),
+ };
+
+ this.options.store.setProjectContext(context);
+ return context;
+ }
+
+ async saveProjectFile(
+ projectData: unknown,
+ suggestedName?: string,
+ existingProjectPath?: string,
+ ) {
+ const result = await this.options.saveProjectFile(
+ projectData,
+ suggestedName,
+ existingProjectPath,
+ );
+ this.getCurrentContext();
+ return result;
+ }
+
+ async loadProjectFile() {
+ const result = await this.options.loadProjectFile();
+ this.getCurrentContext();
+ return result;
+ }
+
+ async loadCurrentProjectFile() {
+ const result = await this.options.loadCurrentProjectFile();
+ this.getCurrentContext();
+ return result;
+ }
+
+ setCurrentVideoPath(path: string) {
+ const result = this.options.setCurrentVideoPath(path);
+ this.getCurrentContext();
+ return result;
+ }
+
+ getCurrentVideoPath() {
+ const result = this.options.getCurrentVideoPathResult();
+ this.getCurrentContext();
+ return result;
+ }
+
+ clearCurrentVideoPath() {
+ const result = this.options.clearCurrentVideoPath();
+ this.getCurrentContext();
+ return result;
+ }
+}
diff --git a/electron/native-bridge/services/systemService.ts b/electron/native-bridge/services/systemService.ts
new file mode 100644
index 0000000..50eff28
--- /dev/null
+++ b/electron/native-bridge/services/systemService.ts
@@ -0,0 +1,43 @@
+import type {
+ CursorCapabilities,
+ NativePlatform,
+ SystemCapabilities,
+} from "../../../src/native/contracts";
+import { NATIVE_BRIDGE_VERSION } from "../../../src/native/contracts";
+import type { NativeBridgeStateStore } from "../store";
+
+interface SystemServiceOptions {
+ store: NativeBridgeStateStore;
+ getPlatform: () => NativePlatform;
+ getAssetBasePath: () => string | null;
+ getCursorCapabilities: () => Promise;
+}
+
+export class SystemService {
+ constructor(private readonly options: SystemServiceOptions) {}
+
+ getPlatform() {
+ return this.options.getPlatform();
+ }
+
+ getAssetBasePath() {
+ return this.options.getAssetBasePath();
+ }
+
+ async getCapabilities(): Promise {
+ const platform = this.getPlatform();
+ const cursorCapabilities = await this.options.getCursorCapabilities();
+
+ const capabilities: SystemCapabilities = {
+ bridgeVersion: NATIVE_BRIDGE_VERSION,
+ platform,
+ cursor: cursorCapabilities,
+ project: {
+ currentContext: true,
+ },
+ };
+
+ this.options.store.setSystemCapabilities(capabilities);
+ return capabilities;
+ }
+}
diff --git a/electron/native-bridge/store.ts b/electron/native-bridge/store.ts
new file mode 100644
index 0000000..dcdbed1
--- /dev/null
+++ b/electron/native-bridge/store.ts
@@ -0,0 +1,88 @@
+import type {
+ CursorCapabilities,
+ NativePlatform,
+ ProjectContext,
+ SystemCapabilities,
+} from "../../src/native/contracts";
+
+export interface NativeBridgeState {
+ system: {
+ platform: NativePlatform;
+ capabilities: SystemCapabilities | null;
+ };
+ project: ProjectContext;
+ cursor: {
+ capabilities: CursorCapabilities | null;
+ lastTelemetryLoad: {
+ videoPath: string;
+ sampleCount: number;
+ loadedAt: number;
+ } | null;
+ };
+}
+
+export class NativeBridgeStateStore {
+ private state: NativeBridgeState;
+
+ constructor(platform: NativePlatform) {
+ this.state = {
+ system: {
+ platform,
+ capabilities: null,
+ },
+ project: {
+ currentProjectPath: null,
+ currentVideoPath: null,
+ },
+ cursor: {
+ capabilities: null,
+ lastTelemetryLoad: null,
+ },
+ };
+ }
+
+ getState() {
+ return this.state;
+ }
+
+ setProjectContext(project: ProjectContext) {
+ this.state = {
+ ...this.state,
+ project,
+ };
+ }
+
+ setSystemCapabilities(capabilities: SystemCapabilities) {
+ this.state = {
+ ...this.state,
+ system: {
+ ...this.state.system,
+ capabilities,
+ },
+ };
+ }
+
+ setCursorCapabilities(capabilities: CursorCapabilities) {
+ this.state = {
+ ...this.state,
+ cursor: {
+ ...this.state.cursor,
+ capabilities,
+ },
+ };
+ }
+
+ markCursorTelemetryLoaded(videoPath: string, sampleCount: number) {
+ this.state = {
+ ...this.state,
+ cursor: {
+ ...this.state.cursor,
+ lastTelemetryLoad: {
+ videoPath,
+ sampleCount,
+ loadedAt: Date.now(),
+ },
+ },
+ };
+ }
+}
diff --git a/electron/preload.ts b/electron/preload.ts
index 5980b4c..5ff110d 100644
--- a/electron/preload.ts
+++ b/electron/preload.ts
@@ -1,5 +1,5 @@
import { contextBridge, ipcRenderer } from "electron";
-import type { RecordingSession, StoreRecordedSessionInput } from "../src/lib/recordingSession";
+import { NATIVE_BRIDGE_CHANNEL, type NativeBridgeRequest } from "../src/native/contracts";
// Asset base URL is passed from the main process via webPreferences.additionalArguments
// (see windows.ts). Sandboxed preloads cannot import node:path / node:url, so we
@@ -10,6 +10,9 @@ const assetBaseUrl = assetBaseUrlArg ? assetBaseUrlArg.slice(ASSET_BASE_URL_ARG_
contextBridge.exposeInMainWorld("electronAPI", {
assetBaseUrl,
+ invokeNativeBridge: (request: NativeBridgeRequest) => {
+ return ipcRenderer.invoke(NATIVE_BRIDGE_CHANNEL, request) as Promise;
+ },
hudOverlayHide: () => {
ipcRenderer.send("hud-overlay-hide");
},
diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx
index 6a14fc0..260f4cb 100644
--- a/src/components/launch/LaunchWindow.tsx
+++ b/src/components/launch/LaunchWindow.tsx
@@ -20,6 +20,7 @@ import {
import { RxDragHandleDots2 } from "react-icons/rx";
import { useI18n, useScopedT } from "@/contexts/I18nContext";
import { getAvailableLocales, getLocaleName } from "@/i18n/loader";
+import { nativeBridgeClient } from "@/native";
import { useAudioLevelMeter } from "../../hooks/useAudioLevelMeter";
import { useCameraDevices } from "../../hooks/useCameraDevices";
import { useMicrophoneDevices } from "../../hooks/useMicrophoneDevices";
@@ -293,13 +294,13 @@ export function LaunchWindow() {
}
if (result.success && result.path) {
- await window.electronAPI.setCurrentVideoPath(result.path);
+ await nativeBridgeClient.project.setCurrentVideoPath(result.path);
await window.electronAPI.switchToEditor();
}
};
const openProjectFile = async () => {
- const result = await window.electronAPI.loadProjectFile();
+ const result = await nativeBridgeClient.project.loadProjectFile();
if (result.canceled || !result.success) return;
await window.electronAPI.switchToEditor();
};
diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts
index f14be62..c1bcc2d 100644
--- a/src/hooks/useScreenRecorder.ts
+++ b/src/hooks/useScreenRecorder.ts
@@ -1,8 +1,7 @@
import { fixWebmDuration } from "@fix-webm-duration/fix";
import { useCallback, useEffect, useRef, useState } from "react";
import { toast } from "sonner";
-import { useScopedT } from "@/contexts/I18nContext";
-import { requestCameraAccess } from "@/lib/requestCameraAccess";
+import { nativeBridgeClient } from "@/native";
const TARGET_FRAME_RATE = 60;
const MIN_FRAME_RATE = 30;
diff --git a/src/native/client.ts b/src/native/client.ts
new file mode 100644
index 0000000..3f53ce4
--- /dev/null
+++ b/src/native/client.ts
@@ -0,0 +1,133 @@
+import {
+ type CursorCapabilities,
+ type CursorRecordingData,
+ type CursorTelemetryPoint,
+ NATIVE_BRIDGE_CHANNEL,
+ type NativeBridgeRequest,
+ type NativeBridgeResponse,
+ type NativePlatform,
+ type ProjectContext,
+ type ProjectFileResult,
+ type ProjectPathResult,
+ type SystemCapabilities,
+} from "./contracts";
+
+function createRequestId() {
+ if (typeof crypto !== "undefined" && typeof crypto.randomUUID === "function") {
+ return crypto.randomUUID();
+ }
+
+ return `req-${Date.now()}-${Math.random().toString(16).slice(2)}`;
+}
+
+function getElectronBridge() {
+ if (!window.electronAPI?.invokeNativeBridge) {
+ throw new Error(
+ `Native bridge unavailable. Expected ${NATIVE_BRIDGE_CHANNEL} transport in preload.`,
+ );
+ }
+
+ return window.electronAPI.invokeNativeBridge;
+}
+
+export async function invokeNativeBridge(
+ request: NativeBridgeRequest,
+): Promise> {
+ const invoke = getElectronBridge();
+ return invoke({
+ ...request,
+ requestId: request.requestId ?? createRequestId(),
+ });
+}
+
+export async function requireNativeBridgeData(request: NativeBridgeRequest): Promise {
+ const response = await invokeNativeBridge(request);
+ if (!response.ok) {
+ throw new Error(response.error.message);
+ }
+
+ return response.data;
+}
+
+export const nativeBridgeClient = {
+ rawInvoke: invokeNativeBridge,
+ system: {
+ getPlatform: () =>
+ requireNativeBridgeData({
+ domain: "system",
+ action: "getPlatform",
+ }),
+ getAssetBasePath: () =>
+ requireNativeBridgeData({
+ domain: "system",
+ action: "getAssetBasePath",
+ }),
+ getCapabilities: () =>
+ requireNativeBridgeData({
+ domain: "system",
+ action: "getCapabilities",
+ }),
+ },
+ project: {
+ getCurrentContext: () =>
+ requireNativeBridgeData({
+ domain: "project",
+ action: "getCurrentContext",
+ }),
+ saveProjectFile: (projectData: unknown, suggestedName?: string, existingProjectPath?: string) =>
+ requireNativeBridgeData({
+ domain: "project",
+ action: "saveProjectFile",
+ payload: {
+ projectData,
+ suggestedName,
+ existingProjectPath,
+ },
+ }),
+ loadProjectFile: () =>
+ requireNativeBridgeData({
+ domain: "project",
+ action: "loadProjectFile",
+ }),
+ loadCurrentProjectFile: () =>
+ requireNativeBridgeData({
+ domain: "project",
+ action: "loadCurrentProjectFile",
+ }),
+ setCurrentVideoPath: (path: string) =>
+ requireNativeBridgeData({
+ domain: "project",
+ action: "setCurrentVideoPath",
+ payload: { path },
+ }),
+ getCurrentVideoPath: () =>
+ requireNativeBridgeData({
+ domain: "project",
+ action: "getCurrentVideoPath",
+ }),
+ clearCurrentVideoPath: () =>
+ requireNativeBridgeData({
+ domain: "project",
+ action: "clearCurrentVideoPath",
+ }),
+ },
+ cursor: {
+ getCapabilities: () =>
+ requireNativeBridgeData({
+ domain: "cursor",
+ action: "getCapabilities",
+ }),
+ getRecordingData: (videoPath?: string) =>
+ requireNativeBridgeData({
+ domain: "cursor",
+ action: "getRecordingData",
+ payload: videoPath ? { videoPath } : {},
+ }),
+ getTelemetry: (videoPath?: string) =>
+ requireNativeBridgeData({
+ domain: "cursor",
+ action: "getTelemetry",
+ payload: videoPath ? { videoPath } : {},
+ }),
+ },
+};
diff --git a/src/native/contracts.ts b/src/native/contracts.ts
new file mode 100644
index 0000000..73d53db
--- /dev/null
+++ b/src/native/contracts.ts
@@ -0,0 +1,209 @@
+export const NATIVE_BRIDGE_CHANNEL = "native-bridge:invoke";
+export const NATIVE_BRIDGE_VERSION = 1;
+
+export type NativePlatform = "darwin" | "win32" | "linux";
+export type CursorProviderKind = "native" | "none";
+
+export interface CursorTelemetryPoint {
+ timeMs: number;
+ cx: number;
+ cy: number;
+}
+
+export interface CursorRecordingSample extends CursorTelemetryPoint {
+ assetId?: string | null;
+ visible?: boolean;
+}
+
+export interface NativeCursorAsset {
+ id: string;
+ platform: NativePlatform;
+ imageDataUrl: string;
+ width: number;
+ height: number;
+ hotspotX: number;
+ hotspotY: number;
+ scaleFactor?: number;
+}
+
+export interface CursorRecordingData {
+ version: number;
+ provider: CursorProviderKind;
+ samples: CursorRecordingSample[];
+ assets: NativeCursorAsset[];
+}
+
+export interface CursorCapabilities {
+ telemetry: boolean;
+ systemAssets: boolean;
+ provider: CursorProviderKind;
+}
+
+export interface SystemCapabilities {
+ bridgeVersion: typeof NATIVE_BRIDGE_VERSION;
+ platform: NativePlatform;
+ cursor: CursorCapabilities;
+ project: {
+ currentContext: boolean;
+ };
+}
+
+export interface ProjectContext {
+ currentProjectPath: string | null;
+ currentVideoPath: string | null;
+}
+
+export interface ProjectPathResult {
+ success: boolean;
+ path?: string;
+ message?: string;
+ canceled?: boolean;
+ error?: string;
+}
+
+export interface ProjectFileResult {
+ success: boolean;
+ path?: string;
+ project?: unknown;
+ message?: string;
+ canceled?: boolean;
+ error?: string;
+}
+
+export type NativeBridgeErrorCode =
+ | "INVALID_REQUEST"
+ | "UNSUPPORTED_ACTION"
+ | "NOT_FOUND"
+ | "UNAVAILABLE"
+ | "INTERNAL_ERROR";
+
+export interface NativeBridgeError {
+ code: NativeBridgeErrorCode;
+ message: string;
+ retryable: boolean;
+}
+
+export interface NativeBridgeMeta {
+ version: typeof NATIVE_BRIDGE_VERSION;
+ requestId: string;
+ timestampMs: number;
+}
+
+export interface NativeBridgeSuccess {
+ ok: true;
+ data: TData;
+ meta: NativeBridgeMeta;
+}
+
+export interface NativeBridgeFailure {
+ ok: false;
+ error: NativeBridgeError;
+ meta: NativeBridgeMeta;
+}
+
+export type NativeBridgeResponse =
+ | NativeBridgeSuccess
+ | NativeBridgeFailure;
+
+type EmptyPayload = Record;
+
+export type NativeBridgeRequest =
+ | {
+ domain: "system";
+ action: "getPlatform";
+ payload?: EmptyPayload;
+ requestId?: string;
+ }
+ | {
+ domain: "system";
+ action: "getAssetBasePath";
+ payload?: EmptyPayload;
+ requestId?: string;
+ }
+ | {
+ domain: "system";
+ action: "getCapabilities";
+ payload?: EmptyPayload;
+ requestId?: string;
+ }
+ | {
+ domain: "project";
+ action: "getCurrentContext";
+ payload?: EmptyPayload;
+ requestId?: string;
+ }
+ | {
+ domain: "project";
+ action: "saveProjectFile";
+ payload: {
+ projectData: unknown;
+ suggestedName?: string;
+ existingProjectPath?: string;
+ };
+ requestId?: string;
+ }
+ | {
+ domain: "project";
+ action: "loadProjectFile";
+ payload?: EmptyPayload;
+ requestId?: string;
+ }
+ | {
+ domain: "project";
+ action: "loadCurrentProjectFile";
+ payload?: EmptyPayload;
+ requestId?: string;
+ }
+ | {
+ domain: "project";
+ action: "setCurrentVideoPath";
+ payload: {
+ path: string;
+ };
+ requestId?: string;
+ }
+ | {
+ domain: "project";
+ action: "getCurrentVideoPath";
+ payload?: EmptyPayload;
+ requestId?: string;
+ }
+ | {
+ domain: "project";
+ action: "clearCurrentVideoPath";
+ payload?: EmptyPayload;
+ requestId?: string;
+ }
+ | {
+ domain: "cursor";
+ action: "getCapabilities";
+ payload?: EmptyPayload;
+ requestId?: string;
+ }
+ | {
+ domain: "cursor";
+ action: "getTelemetry";
+ payload?: {
+ videoPath?: string;
+ };
+ requestId?: string;
+ }
+ | {
+ domain: "cursor";
+ action: "getRecordingData";
+ payload?: {
+ videoPath?: string;
+ };
+ requestId?: string;
+ };
+
+export type NativeBridgeEventName =
+ | "project.contextChanged"
+ | "cursor.providerChanged"
+ | "cursor.telemetryLoaded";
+
+export interface NativeBridgeEvent {
+ name: NativeBridgeEventName;
+ payload: TPayload;
+ meta: NativeBridgeMeta;
+}
diff --git a/src/native/hooks/useCursorRecordingData.ts b/src/native/hooks/useCursorRecordingData.ts
new file mode 100644
index 0000000..6b3451a
--- /dev/null
+++ b/src/native/hooks/useCursorRecordingData.ts
@@ -0,0 +1,61 @@
+import { useEffect, useState } from "react";
+import type { CursorRecordingData } from "@/native/contracts";
+import { nativeBridgeClient } from "../client";
+
+interface UseCursorRecordingDataResult {
+ data: CursorRecordingData | null;
+ loading: boolean;
+ error: string | null;
+}
+
+export function useCursorRecordingData(videoPath: string | null): UseCursorRecordingDataResult {
+ const [data, setData] = useState(null);
+ const [loading, setLoading] = useState(false);
+ const [error, setError] = useState(null);
+
+ useEffect(() => {
+ let cancelled = false;
+
+ async function loadCursorRecordingData() {
+ if (!videoPath) {
+ setData(null);
+ setLoading(false);
+ setError(null);
+ return;
+ }
+
+ setLoading(true);
+ setError(null);
+
+ try {
+ const nextData = await nativeBridgeClient.cursor.getRecordingData(videoPath);
+ if (!cancelled) {
+ setData(nextData);
+ }
+ } catch (nextError) {
+ if (!cancelled) {
+ setData(null);
+ setError(
+ nextError instanceof Error ? nextError.message : "Failed to load cursor recording data",
+ );
+ }
+ } finally {
+ if (!cancelled) {
+ setLoading(false);
+ }
+ }
+ }
+
+ loadCursorRecordingData();
+
+ return () => {
+ cancelled = true;
+ };
+ }, [videoPath]);
+
+ return {
+ data,
+ loading,
+ error,
+ };
+}
diff --git a/src/native/hooks/useCursorTelemetry.ts b/src/native/hooks/useCursorTelemetry.ts
new file mode 100644
index 0000000..1617680
--- /dev/null
+++ b/src/native/hooks/useCursorTelemetry.ts
@@ -0,0 +1,61 @@
+import { useEffect, useState } from "react";
+import type { CursorTelemetryPoint } from "@/components/video-editor/types";
+import { nativeBridgeClient } from "../client";
+
+interface UseCursorTelemetryResult {
+ samples: CursorTelemetryPoint[];
+ loading: boolean;
+ error: string | null;
+}
+
+export function useCursorTelemetry(videoPath: string | null): UseCursorTelemetryResult {
+ const [samples, setSamples] = useState([]);
+ const [loading, setLoading] = useState(false);
+ const [error, setError] = useState(null);
+
+ useEffect(() => {
+ let cancelled = false;
+
+ async function loadCursorTelemetry() {
+ if (!videoPath) {
+ setSamples([]);
+ setLoading(false);
+ setError(null);
+ return;
+ }
+
+ setLoading(true);
+ setError(null);
+
+ try {
+ const nextSamples = await nativeBridgeClient.cursor.getTelemetry(videoPath);
+ if (!cancelled) {
+ setSamples(nextSamples);
+ }
+ } catch (nextError) {
+ if (!cancelled) {
+ setSamples([]);
+ setError(
+ nextError instanceof Error ? nextError.message : "Failed to load cursor telemetry",
+ );
+ }
+ } finally {
+ if (!cancelled) {
+ setLoading(false);
+ }
+ }
+ }
+
+ loadCursorTelemetry();
+
+ return () => {
+ cancelled = true;
+ };
+ }, [videoPath]);
+
+ return {
+ samples,
+ loading,
+ error,
+ };
+}
diff --git a/src/native/index.ts b/src/native/index.ts
new file mode 100644
index 0000000..817d1cf
--- /dev/null
+++ b/src/native/index.ts
@@ -0,0 +1,4 @@
+export * from "./client";
+export * from "./contracts";
+export * from "./hooks/useCursorRecordingData";
+export * from "./hooks/useCursorTelemetry";
From 248ebabcf10b0e44b1edae6284d9464862078439 Mon Sep 17 00:00:00 2001
From: Etienne Lescot
Date: Mon, 16 Mar 2026 10:41:46 +0100
Subject: [PATCH 05/92] feat: add windows native cursor capture and rendering
---
electron/ipc/handlers.ts | 999 ++++++------------
.../native-bridge/cursor/recording/factory.ts | 29 +
.../native-bridge/cursor/recording/session.ts | 6 +
.../recording/telemetryRecordingSession.ts | 62 ++
.../windowsNativeRecordingSession.ts | 326 ++++++
src/components/video-editor/VideoEditor.tsx | 64 +-
src/components/video-editor/VideoPlayback.tsx | 81 +-
.../video-editor/timeline/TimelineEditor.tsx | 12 +-
src/lib/cursor/nativeCursor.ts | 101 ++
src/lib/exporter/frameRenderer.ts | 69 +-
src/lib/exporter/gifExporter.ts | 3 +
src/lib/exporter/videoExporter.ts | 3 +
12 files changed, 1035 insertions(+), 720 deletions(-)
create mode 100644 electron/native-bridge/cursor/recording/factory.ts
create mode 100644 electron/native-bridge/cursor/recording/session.ts
create mode 100644 electron/native-bridge/cursor/recording/telemetryRecordingSession.ts
create mode 100644 electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts
create mode 100644 src/lib/cursor/nativeCursor.ts
diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts
index 431cda1..1d73a9b 100644
--- a/electron/ipc/handlers.ts
+++ b/electron/ipc/handlers.ts
@@ -2,7 +2,7 @@ import fs from "node:fs/promises";
import { createRequire } from "node:module";
import os from "node:os";
import path from "node:path";
-import { fileURLToPath } from "node:url";
+import { fileURLToPath, pathToFileURL } from "node:url";
const nodeRequire = createRequire(import.meta.url);
@@ -27,8 +27,18 @@ import {
type RecordingSession,
type StoreRecordedSessionInput,
} from "../../src/lib/recordingSession";
+import type {
+ CursorRecordingData,
+ CursorRecordingSample,
+ NativeCursorAsset,
+ ProjectFileResult,
+ ProjectPathResult,
+} from "../../src/native/contracts";
import { mainT } from "../i18n";
import { RECORDINGS_DIR } from "../main";
+import { createCursorRecordingSession } from "../native-bridge/cursor/recording/factory";
+import type { CursorRecordingSession } from "../native-bridge/cursor/recording/session";
+import { registerNativeBridgeHandlers } from "./nativeBridge";
const PROJECT_FILE_EXTENSION = "openscreen";
const SHORTCUTS_FILE = path.join(app.getPath("userData"), "shortcuts.json");
@@ -193,7 +203,7 @@ type SelectedSource = {
let selectedSource: SelectedSource | null = null;
let currentProjectPath: string | null = null;
-let currentRecordingSession: RecordingSession | null = null;
+let currentVideoPath: string | null = null;
function normalizePath(filePath: string) {
return path.resolve(filePath);
@@ -227,450 +237,180 @@ function isTrustedProjectPath(filePath?: string | null) {
return normalizePath(filePath) === normalizePath(currentProjectPath);
}
-function setCurrentRecordingSessionState(session: RecordingSession | null) {
- currentRecordingSession = session;
-}
-
-function getSessionManifestPathForVideo(videoPath: string) {
- const parsed = path.parse(videoPath);
- const baseName = parsed.name.endsWith("-webcam")
- ? parsed.name.slice(0, -"-webcam".length)
- : parsed.name;
- return path.join(parsed.dir, `${baseName}${RECORDING_SESSION_SUFFIX}`);
-}
-
-async function loadRecordedSessionForVideoPath(
- videoPath: string,
-): Promise {
- const normalizedVideoPath = normalizeVideoSourcePath(videoPath);
- if (!normalizedVideoPath) {
- return null;
- }
-
- try {
- const manifestPath = getSessionManifestPathForVideo(normalizedVideoPath);
- const content = await fs.readFile(manifestPath, "utf-8");
- const session = normalizeRecordingSession(JSON.parse(content));
- if (!session) {
- return null;
- }
-
- const normalizedSession: RecordingSession = {
- ...session,
- screenVideoPath: normalizeVideoSourcePath(session.screenVideoPath) ?? session.screenVideoPath,
- ...(session.webcamVideoPath
- ? {
- webcamVideoPath:
- normalizeVideoSourcePath(session.webcamVideoPath) ?? session.webcamVideoPath,
- }
- : {}),
- };
-
- const targetPath = normalizePath(normalizedVideoPath);
- const screenMatches = normalizePath(normalizedSession.screenVideoPath) === targetPath;
- const webcamMatches = normalizedSession.webcamVideoPath
- ? normalizePath(normalizedSession.webcamVideoPath) === targetPath
- : false;
-
- return screenMatches || webcamMatches ? normalizedSession : null;
- } catch {
- return null;
- }
-}
-
-async function storeRecordedSessionFiles(payload: StoreRecordedSessionInput) {
- const createdAt =
- typeof payload.createdAt === "number" && Number.isFinite(payload.createdAt)
- ? payload.createdAt
- : Date.now();
- const screenVideoPath = resolveRecordingOutputPath(payload.screen.fileName);
- await fs.writeFile(screenVideoPath, Buffer.from(payload.screen.videoData));
-
- let webcamVideoPath: string | undefined;
- if (payload.webcam) {
- webcamVideoPath = resolveRecordingOutputPath(payload.webcam.fileName);
- await fs.writeFile(webcamVideoPath, Buffer.from(payload.webcam.videoData));
- }
-
- const session: RecordingSession = webcamVideoPath
- ? { screenVideoPath, webcamVideoPath, createdAt }
- : { screenVideoPath, createdAt };
- setCurrentRecordingSessionState(session);
- currentProjectPath = null;
-
- const telemetryPath = `${screenVideoPath}.cursor.json`;
- const pendingBatch = cursorTelemetryBuffer.takeNextBatch();
- const pendingClicks = takeCursorClickTimestamps();
- if ((pendingBatch && pendingBatch.samples.length > 0) || pendingClicks.length > 0) {
- try {
- await fs.writeFile(
- telemetryPath,
- JSON.stringify(
- {
- version: CURSOR_TELEMETRY_VERSION,
- samples: pendingBatch?.samples ?? [],
- clicks: pendingClicks,
- },
- null,
- 2,
- ),
- "utf-8",
- );
- } catch (err) {
- if (pendingBatch) cursorTelemetryBuffer.prependBatch(pendingBatch);
- throw err;
- }
- }
-
- const sessionManifestPath = path.join(
- RECORDINGS_DIR,
- `${path.parse(payload.screen.fileName).name}${RECORDING_SESSION_SUFFIX}`,
- );
- await fs.writeFile(sessionManifestPath, JSON.stringify(session, null, 2), "utf-8");
-
- return {
- success: true,
- path: screenVideoPath,
- session,
- message: "Recording session stored successfully",
- };
-}
-
-const CURSOR_TELEMETRY_VERSION = 1;
+const CURSOR_TELEMETRY_VERSION = 2;
const CURSOR_SAMPLE_INTERVAL_MS = 100;
const MAX_CURSOR_SAMPLES = 60 * 60 * 10; // 1 hour @ 10Hz
-let cursorCaptureInterval: NodeJS.Timeout | null = null;
-let cursorCaptureStartTimeMs = 0;
-const cursorTelemetryBuffer = createCursorTelemetryBuffer({
- maxActiveSamples: MAX_CURSOR_SAMPLES,
-});
-
-// Mouse click timestamps (macOS only — uiohook-napi behind Accessibility).
-const MAX_CURSOR_CLICKS = 60 * 60 * 60; // ~1 click/sec for an hour
-let cursorClickTimestampsMs: number[] = [];
-let uioHookInstance: {
- start: () => void;
- stop: () => void;
- on: (...a: unknown[]) => void;
- off?: (...a: unknown[]) => void;
- removeListener?: (...a: unknown[]) => void;
-} | null = null;
-let uioHookMouseDownHandler: ((event: { time?: number }) => void) | null = null;
-let uioHookFailureLogged = false;
+let cursorRecordingSession: CursorRecordingSession | null = null;
+let pendingCursorRecordingData: CursorRecordingData | null = null;
function clamp(value: number, min: number, max: number) {
return Math.min(max, Math.max(min, value));
}
-function loadUioHookForClicks(): typeof uioHookInstance {
- try {
- // Dynamic require + try/catch so a broken native binary doesn't crash startup.
- const mod = nodeRequire("uiohook-napi");
- const candidate = mod.uIOhook ?? mod.default?.uIOhook ?? mod.uiohook ?? mod.default;
- if (candidate && typeof candidate.start === "function" && typeof candidate.on === "function") {
- return candidate;
- }
- return null;
- } catch (error) {
- if (!uioHookFailureLogged) {
- uioHookFailureLogged = true;
- console.warn("[clickCapture] uiohook-napi unavailable:", error);
- }
+function normalizeCursorSample(sample: unknown): CursorRecordingSample | null {
+ if (!sample || typeof sample !== "object") {
return null;
}
-}
-function startClickCapture() {
- if (process.platform !== "darwin") return;
- if (uioHookInstance) return;
-
- // Passive check — the prompt fires from the renderer when the user toggles
- // "Only on clicks" so it doesn't stack with the screen-recording prompt.
- try {
- if (!systemPreferences.isTrustedAccessibilityClient(false)) {
- if (!uioHookFailureLogged) {
- uioHookFailureLogged = true;
- console.warn(
- "[clickCapture] Accessibility permission not granted — click capture disabled.",
- );
- }
- return;
- }
- } catch {
- // fall through; uiohook will fail defensively below
- }
-
- const hook = loadUioHookForClicks();
- if (!hook) return;
-
- uioHookMouseDownHandler = (event) => {
- const elapsed = Math.max(0, Date.now() - cursorCaptureStartTimeMs);
- void event;
- if (cursorClickTimestampsMs.length >= MAX_CURSOR_CLICKS) return;
- cursorClickTimestampsMs.push(elapsed);
+ const point = sample as Partial;
+ return {
+ timeMs:
+ typeof point.timeMs === "number" && Number.isFinite(point.timeMs)
+ ? Math.max(0, point.timeMs)
+ : 0,
+ cx: typeof point.cx === "number" && Number.isFinite(point.cx) ? clamp(point.cx, 0, 1) : 0.5,
+ cy: typeof point.cy === "number" && Number.isFinite(point.cy) ? clamp(point.cy, 0, 1) : 0.5,
+ assetId: typeof point.assetId === "string" ? point.assetId : null,
+ visible: typeof point.visible === "boolean" ? point.visible : true,
};
+}
+function normalizeCursorAsset(asset: unknown): NativeCursorAsset | null {
+ if (!asset || typeof asset !== "object") {
+ return null;
+ }
+
+ const candidate = asset as Partial;
+ if (typeof candidate.id !== "string" || typeof candidate.imageDataUrl !== "string") {
+ return null;
+ }
+
+ return {
+ id: candidate.id,
+ platform:
+ candidate.platform === "win32" ? "win32" : process.platform === "darwin" ? "darwin" : "linux",
+ imageDataUrl: candidate.imageDataUrl,
+ width:
+ typeof candidate.width === "number" && Number.isFinite(candidate.width)
+ ? Math.max(1, Math.round(candidate.width))
+ : 1,
+ height:
+ typeof candidate.height === "number" && Number.isFinite(candidate.height)
+ ? Math.max(1, Math.round(candidate.height))
+ : 1,
+ hotspotX:
+ typeof candidate.hotspotX === "number" && Number.isFinite(candidate.hotspotX)
+ ? Math.max(0, Math.round(candidate.hotspotX))
+ : 0,
+ hotspotY:
+ typeof candidate.hotspotY === "number" && Number.isFinite(candidate.hotspotY)
+ ? Math.max(0, Math.round(candidate.hotspotY))
+ : 0,
+ scaleFactor:
+ typeof candidate.scaleFactor === "number" && Number.isFinite(candidate.scaleFactor)
+ ? Math.max(0.1, candidate.scaleFactor)
+ : undefined,
+ };
+}
+
+async function readCursorRecordingFile(targetVideoPath: string): Promise {
+ const telemetryPath = `${targetVideoPath}.cursor.json`;
try {
- hook.on("mousedown", uioHookMouseDownHandler);
- hook.start();
- uioHookInstance = hook;
+ const content = await fs.readFile(telemetryPath, "utf-8");
+ const parsed = JSON.parse(content);
+ const rawSamples = Array.isArray(parsed)
+ ? parsed
+ : Array.isArray(parsed?.samples)
+ ? parsed.samples
+ : [];
+ const rawAssets = Array.isArray(parsed?.assets) ? parsed.assets : [];
+
+ const samples = rawSamples
+ .map((sample: unknown) => normalizeCursorSample(sample))
+ .filter((sample: CursorRecordingSample | null): sample is CursorRecordingSample =>
+ Boolean(sample),
+ )
+ .sort((a: CursorRecordingSample, b: CursorRecordingSample) => a.timeMs - b.timeMs);
+
+ const assets = rawAssets
+ .map((asset: unknown) => normalizeCursorAsset(asset))
+ .filter((asset: NativeCursorAsset | null): asset is NativeCursorAsset => Boolean(asset));
+
+ return {
+ version:
+ typeof parsed?.version === "number" && Number.isFinite(parsed.version) ? parsed.version : 1,
+ provider: parsed?.provider === "native" ? "native" : "none",
+ samples,
+ assets,
+ };
} catch (error) {
- if (!uioHookFailureLogged) {
- uioHookFailureLogged = true;
- console.warn("[clickCapture] failed to start uiohook:", error);
+ const nodeError = error as NodeJS.ErrnoException;
+ if (nodeError.code === "ENOENT") {
+ return {
+ version: CURSOR_TELEMETRY_VERSION,
+ provider: "none",
+ samples: [],
+ assets: [],
+ };
}
- uioHookMouseDownHandler = null;
+
+ console.error("Failed to load cursor telemetry:", error);
+ throw error;
}
}
-function stopClickCapture() {
- if (!uioHookInstance) return;
+async function readCursorTelemetryFile(targetVideoPath: string) {
try {
- if (uioHookMouseDownHandler) {
- if (typeof uioHookInstance.off === "function") {
- uioHookInstance.off("mousedown", uioHookMouseDownHandler);
- } else if (typeof uioHookInstance.removeListener === "function") {
- uioHookInstance.removeListener("mousedown", uioHookMouseDownHandler);
- }
- }
- uioHookInstance.stop();
+ const recordingData = await readCursorRecordingFile(targetVideoPath);
+ return {
+ success: true,
+ samples: recordingData.samples.map((sample) => ({
+ timeMs: sample.timeMs,
+ cx: sample.cx,
+ cy: sample.cy,
+ })),
+ };
} catch (error) {
- console.warn("[clickCapture] failed to stop uiohook:", error);
+ console.error("Failed to load cursor telemetry:", error);
+ return {
+ success: false,
+ message: "Failed to load cursor telemetry",
+ error: String(error),
+ samples: [],
+ };
}
- uioHookInstance = null;
- uioHookMouseDownHandler = null;
}
-function takeCursorClickTimestamps(): number[] {
- const out = cursorClickTimestampsMs;
- cursorClickTimestampsMs = [];
- return out;
-}
-
-function stopCursorCapture() {
- if (cursorCaptureInterval) {
- clearInterval(cursorCaptureInterval);
- cursorCaptureInterval = null;
+function resolveAssetBasePath() {
+ try {
+ if (app.isPackaged) {
+ const assetPath = path.join(process.resourcesPath, "assets");
+ return pathToFileURL(`${assetPath}${path.sep}`).toString();
+ }
+ const assetPath = path.join(app.getAppPath(), "public", "assets");
+ return pathToFileURL(`${assetPath}${path.sep}`).toString();
+ } catch (err) {
+ console.error("Failed to resolve asset base path:", err);
+ return null;
}
- stopClickCapture();
}
-function sampleCursorPoint() {
+function getSelectedSourceBounds() {
const cursor = screen.getCursorScreenPoint();
const sourceDisplayId = Number(selectedSource?.display_id);
const sourceDisplay = Number.isFinite(sourceDisplayId)
? (screen.getAllDisplays().find((display) => display.id === sourceDisplayId) ?? null)
: null;
- const display = sourceDisplay ?? screen.getDisplayNearestPoint(cursor);
- const bounds = display.bounds;
- const width = Math.max(1, bounds.width);
- const height = Math.max(1, bounds.height);
-
- const cx = clamp((cursor.x - bounds.x) / width, 0, 1);
- const cy = clamp((cursor.y - bounds.y) / height, 0, 1);
-
- cursorTelemetryBuffer.push({
- timeMs: Math.max(0, Date.now() - cursorCaptureStartTimeMs),
- cx,
- cy,
- });
+ return (sourceDisplay ?? screen.getDisplayNearestPoint(cursor)).bounds;
}
export function registerIpcHandlers(
createEditorWindow: () => void,
createSourceSelectorWindow: () => BrowserWindow,
- createCountdownOverlayWindow: () => BrowserWindow,
getMainWindow: () => BrowserWindow | null,
getSourceSelectorWindow: () => BrowserWindow | null,
- getCountdownOverlayWindow: () => BrowserWindow | null,
onRecordingStateChange?: (recording: boolean, sourceName: string) => void,
- switchToHud?: () => void,
) {
- const supportsWindowOpacity = process.platform !== "linux";
- const countdownOverlayState = {
- visible: false,
- value: null as number | null,
- activeRunId: null as number | null,
- hideCommitId: 0,
- hideCommitTimer: null as ReturnType | null,
- };
- const COUNTDOWN_OVERLAY_HIDE_DEBOUNCE_MS = 1200;
-
- const clearCountdownOverlayHideCommit = () => {
- if (countdownOverlayState.hideCommitTimer) {
- clearTimeout(countdownOverlayState.hideCommitTimer);
- countdownOverlayState.hideCommitTimer = null;
- }
- };
-
- const commitCountdownOverlayHide = (win: BrowserWindow, hideCommitId: number) => {
- if (win.isDestroyed()) {
- return;
- }
-
- if (countdownOverlayState.visible || countdownOverlayState.hideCommitId !== hideCommitId) {
- return;
- }
-
- win.hide();
- if (supportsWindowOpacity) {
- // Reset baseline opacity for the next show cycle.
- win.setOpacity(1);
- }
- };
-
- const flushCountdownOverlayState = (win: BrowserWindow) => {
- if (win.isDestroyed()) {
- return;
- }
-
- clearCountdownOverlayHideCommit();
- win.webContents.send("countdown-overlay-value", countdownOverlayState.value);
- if (!countdownOverlayState.visible) {
- return;
- }
-
- if (win.isVisible()) {
- if (supportsWindowOpacity) {
- win.setOpacity(1);
- }
- return;
- }
-
- setTimeout(() => {
- if (!win.isDestroyed() && countdownOverlayState.visible && !win.isVisible()) {
- if (supportsWindowOpacity) {
- win.setOpacity(0);
- }
- win.showInactive();
-
- if (supportsWindowOpacity) {
- setTimeout(() => {
- if (!win.isDestroyed() && countdownOverlayState.visible && win.isVisible()) {
- win.setOpacity(1);
- }
- }, 0);
- }
- }
- }, 16);
- };
-
- ipcMain.handle("countdown-overlay-show", (_, value: number, runId: number) => {
- countdownOverlayState.activeRunId = runId;
- countdownOverlayState.visible = true;
- countdownOverlayState.value = value;
-
- const win = getCountdownOverlayWindow() ?? createCountdownOverlayWindow();
- if (win.isDestroyed()) {
- return;
- }
-
- if (win.webContents.isLoading()) {
- win.webContents.once("did-finish-load", () => {
- if (!win.isDestroyed()) {
- flushCountdownOverlayState(win);
- }
- });
- } else {
- flushCountdownOverlayState(win);
- }
- });
-
- ipcMain.handle("countdown-overlay-set-value", (_, value: number, runId: number) => {
- if (countdownOverlayState.activeRunId !== runId || !countdownOverlayState.visible) {
- return;
- }
-
- countdownOverlayState.value = value;
-
- const win = getCountdownOverlayWindow();
- if (!win || win.isDestroyed()) {
- return;
- }
-
- if (win.webContents.isLoading()) {
- return;
- }
-
- win.webContents.send("countdown-overlay-value", value);
- });
-
- ipcMain.handle("countdown-overlay-hide", (_, runId: number) => {
- if (countdownOverlayState.activeRunId !== runId) {
- return;
- }
-
- countdownOverlayState.visible = false;
- countdownOverlayState.hideCommitId += 1;
- const hideCommitId = countdownOverlayState.hideCommitId;
- clearCountdownOverlayHideCommit();
-
- const win = getCountdownOverlayWindow();
- if (!win || win.isDestroyed()) {
- countdownOverlayState.value = null;
- return;
- }
-
- if (supportsWindowOpacity) {
- // Hide visually immediately to avoid hide/show compositor flashes on rapid restart.
- win.setOpacity(0);
- }
-
- countdownOverlayState.value = null;
- if (!win.webContents.isLoading()) {
- win.webContents.send("countdown-overlay-value", countdownOverlayState.value);
- }
-
- if (!supportsWindowOpacity) {
- win.hide();
- return;
- }
-
- countdownOverlayState.hideCommitTimer = setTimeout(() => {
- countdownOverlayState.hideCommitTimer = null;
- commitCountdownOverlayHide(win, hideCommitId);
- }, COUNTDOWN_OVERLAY_HIDE_DEBOUNCE_MS);
- });
-
- ipcMain.handle("switch-to-hud", () => {
- if (switchToHud) switchToHud();
- });
- ipcMain.handle("start-new-recording", () => {
- try {
- setCurrentRecordingSessionState(null);
- if (switchToHud) {
- switchToHud();
- }
- return { success: true };
- } catch (error) {
- console.error("Failed to start new recording:", error);
- return { success: false, error: String(error) };
- }
- });
-
ipcMain.handle("get-sources", async (_, opts) => {
- const ownWindowSourceIds = new Set(
- BrowserWindow.getAllWindows()
- .map((win) => {
- try {
- return win.getMediaSourceId();
- } catch {
- return null;
- }
- })
- .filter((id): id is string => Boolean(id)),
- );
const sources = await desktopCapturer.getSources(opts);
- return sources
- .filter((source) => !ownWindowSourceIds.has(source.id))
- .map((source) => ({
- id: source.id,
- name: source.name,
- display_id: source.display_id,
- thumbnail: source.thumbnail ? source.thumbnail.toDataURL() : null,
- appIcon: source.appIcon ? source.appIcon.toDataURL() : null,
- }));
+ return sources.map((source) => ({
+ id: source.id,
+ name: source.name,
+ display_id: source.display_id,
+ thumbnail: source.thumbnail ? source.thumbnail.toDataURL() : null,
+ appIcon: source.appIcon ? source.appIcon.toDataURL() : null,
+ }));
});
ipcMain.handle("select-source", (_, source: SelectedSource) => {
@@ -779,7 +519,25 @@ export function registerIpcHandlers(
ipcMain.handle("store-recorded-session", async (_, payload: StoreRecordedSessionInput) => {
try {
- return await storeRecordedSessionFiles(payload);
+ const videoPath = path.join(RECORDINGS_DIR, fileName);
+ await fs.writeFile(videoPath, Buffer.from(videoData));
+ currentProjectPath = null;
+
+ const telemetryPath = `${videoPath}.cursor.json`;
+ if (pendingCursorRecordingData && pendingCursorRecordingData.samples.length > 0) {
+ await fs.writeFile(
+ telemetryPath,
+ JSON.stringify(pendingCursorRecordingData, null, 2),
+ "utf-8",
+ );
+ }
+ pendingCursorRecordingData = null;
+
+ return {
+ success: true,
+ path: videoPath,
+ message: "Video stored successfully",
+ };
} catch (error) {
console.error("Failed to store recording session:", error);
return {
@@ -821,24 +579,7 @@ export function registerIpcHandlers(
return { success: false, message: "No recorded video found" };
}
- // Sort by most recently modified to reliably get the latest recording.
- // Lexicographic sort is unreliable (e.g. recording-9.webm > recording-10.webm).
- let latestVideo: string | null = null;
- let latestMtimeMs = 0;
- for (const file of videoFiles) {
- try {
- const stat = await fs.stat(path.join(RECORDINGS_DIR, file));
- if (stat.mtimeMs > latestMtimeMs) {
- latestMtimeMs = stat.mtimeMs;
- latestVideo = file;
- }
- } catch {
- // Skip inaccessible files.
- }
- }
- if (!latestVideo) {
- return { success: false, message: "No recorded video found" };
- }
+ const latestVideo = videoFiles.sort().reverse()[0];
const videoPath = path.join(RECORDINGS_DIR, latestVideo);
return { success: true, path: videoPath };
@@ -848,54 +589,38 @@ export function registerIpcHandlers(
}
});
- ipcMain.handle("read-binary-file", async (_, inputPath: string) => {
- try {
- const normalizedPath = normalizeVideoSourcePath(inputPath);
- if (!normalizedPath) {
- return { success: false, message: "Invalid file path" };
- }
-
- if (!isPathAllowed(normalizedPath)) {
- console.warn(
- "[read-binary-file] Rejected path outside allowed directories:",
- normalizedPath,
- );
- return { success: false, message: "Access denied: path outside allowed directories" };
- }
-
- const data = await fs.readFile(normalizedPath);
- return {
- success: true,
- data: data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength),
- path: normalizedPath,
- };
- } catch (error) {
- console.error("Failed to read binary file:", error);
- return {
- success: false,
- message: "Failed to read binary file",
- error: String(error),
- };
- }
- });
-
- ipcMain.handle("set-recording-state", (_, recording: boolean, recordingId?: number) => {
+ ipcMain.handle("set-recording-state", async (_, recording: boolean) => {
if (recording) {
- stopCursorCapture();
- // The renderer is the source of truth for the recording id (it
- // uses the same id as the saved fileName). Fall back to a
- // timestamp only if the renderer didn't supply one, so the
- // buffer always has a stable key per session.
- const id = typeof recordingId === "number" ? recordingId : Date.now();
- cursorTelemetryBuffer.startSession(id);
- cursorCaptureStartTimeMs = Date.now();
- cursorClickTimestampsMs = [];
- startClickCapture();
- sampleCursorPoint();
- cursorCaptureInterval = setInterval(sampleCursorPoint, CURSOR_SAMPLE_INTERVAL_MS);
+ if (cursorRecordingSession) {
+ pendingCursorRecordingData = await cursorRecordingSession.stop();
+ cursorRecordingSession = null;
+ }
+
+ pendingCursorRecordingData = null;
+ cursorRecordingSession = createCursorRecordingSession({
+ getDisplayBounds: getSelectedSourceBounds,
+ maxSamples: MAX_CURSOR_SAMPLES,
+ platform: process.platform,
+ sampleIntervalMs: CURSOR_SAMPLE_INTERVAL_MS,
+ });
+
+ try {
+ await cursorRecordingSession.start();
+ } catch (error) {
+ console.error("Failed to start cursor recording session:", error);
+ cursorRecordingSession = null;
+ }
} else {
- stopCursorCapture();
- cursorTelemetryBuffer.endSession();
+ if (cursorRecordingSession) {
+ try {
+ pendingCursorRecordingData = await cursorRecordingSession.stop();
+ } catch (error) {
+ console.error("Failed to stop cursor recording session:", error);
+ pendingCursorRecordingData = null;
+ } finally {
+ cursorRecordingSession = null;
+ }
+ }
}
const source = selectedSource || { name: "Screen" };
@@ -904,10 +629,6 @@ export function registerIpcHandlers(
}
});
- ipcMain.handle("discard-cursor-telemetry", (_, recordingId: number) => {
- cursorTelemetryBuffer.discardBatch(recordingId);
- });
-
ipcMain.handle("get-cursor-telemetry", async (_, videoPath?: string) => {
const targetVideoPath = normalizeVideoSourcePath(
videoPath ?? currentRecordingSession?.screenVideoPath,
@@ -916,85 +637,12 @@ export function registerIpcHandlers(
return { success: true, samples: [] };
}
- if (!isPathAllowed(targetVideoPath)) {
- console.warn(
- "[get-cursor-telemetry] Rejected path outside allowed directories:",
- targetVideoPath,
- );
- return { success: true, samples: [] };
- }
-
- const telemetryPath = `${targetVideoPath}.cursor.json`;
- try {
- const content = await fs.readFile(telemetryPath, "utf-8");
- const parsed = JSON.parse(content);
- const rawSamples = Array.isArray(parsed)
- ? parsed
- : Array.isArray(parsed?.samples)
- ? parsed.samples
- : [];
-
- const samples: CursorTelemetryPoint[] = rawSamples
- .filter((sample: unknown) => Boolean(sample && typeof sample === "object"))
- .map((sample: unknown) => {
- const point = sample as Partial;
- return {
- timeMs:
- typeof point.timeMs === "number" && Number.isFinite(point.timeMs)
- ? Math.max(0, point.timeMs)
- : 0,
- cx:
- typeof point.cx === "number" && Number.isFinite(point.cx)
- ? clamp(point.cx, 0, 1)
- : 0.5,
- cy:
- typeof point.cy === "number" && Number.isFinite(point.cy)
- ? clamp(point.cy, 0, 1)
- : 0.5,
- };
- })
- .sort((a: CursorTelemetryPoint, b: CursorTelemetryPoint) => a.timeMs - b.timeMs);
-
- const rawClicks = Array.isArray(parsed?.clicks) ? parsed.clicks : [];
- const clicks: number[] = rawClicks
- .map((value: unknown) =>
- typeof value === "number" && Number.isFinite(value) ? Math.max(0, value) : null,
- )
- .filter((v: number | null): v is number => v !== null)
- .sort((a: number, b: number) => a - b);
-
- return { success: true, samples, clicks };
- } catch (error) {
- const nodeError = error as NodeJS.ErrnoException;
- if (nodeError.code === "ENOENT") {
- return { success: true, samples: [], clicks: [] };
- }
- console.error("Failed to load cursor telemetry:", error);
- return {
- success: false,
- message: "Failed to load cursor telemetry",
- error: String(error),
- samples: [],
- clicks: [],
- };
- }
+ return readCursorTelemetryFile(targetVideoPath);
});
ipcMain.handle("open-external-url", async (_, url: string) => {
try {
- const ALLOWED_SCHEMES = ["http:", "https:", "mailto:"];
- let parsed: URL;
- try {
- parsed = new URL(url);
- } catch {
- return { success: false, error: "Invalid URL" };
- }
-
- if (!ALLOWED_SCHEMES.includes(parsed.protocol)) {
- return { success: false, error: `Unsupported URL scheme: ${parsed.protocol}` };
- }
-
- await shell.openExternal(parsed.toString());
+ await shell.openExternal(url);
return { success: true };
} catch (error) {
console.error("Failed to open URL:", error);
@@ -1002,15 +650,10 @@ export function registerIpcHandlers(
}
});
- /**
- * Handles saving an exported video file.
- * Shows a save dialog, normalizes the file path for the current OS,
- * ensures the directory exists, and writes the video data.
- * @param _ - Unused event parameter.
- * @param videoData - The exported video as an ArrayBuffer.
- * @param fileName - Suggested filename for the save dialog.
- * @returns Object with success status, optional file path, and error details.
- */
+ // Return base path for assets so renderer can resolve file:// paths in production
+ ipcMain.handle("get-asset-base-path", () => {
+ return resolveAssetBasePath();
+ });
ipcMain.handle("pick-export-save-path", async (_, fileName: string, exportFolder?: string) => {
try {
@@ -1082,7 +725,7 @@ export function registerIpcHandlers(
return {
success: true,
- path: normalizedPath,
+ path: result.filePath,
message: "Video exported successfully",
};
} catch (error) {
@@ -1094,6 +737,7 @@ export function registerIpcHandlers(
};
}
});
+
ipcMain.handle("open-video-file-picker", async () => {
try {
const dialogOptions = buildDialogOptions(
@@ -1117,17 +761,10 @@ export function registerIpcHandlers(
return { success: false, canceled: true };
}
- const approvedPath = await approveReadableVideoPath(result.filePaths[0]);
- if (!approvedPath) {
- return {
- success: false,
- message: "Selected file is not a supported video",
- };
- }
currentProjectPath = null;
return {
success: true,
- path: approvedPath,
+ path: result.filePaths[0],
};
} catch (error) {
console.error("Failed to open file picker:", error);
@@ -1166,75 +803,87 @@ export function registerIpcHandlers(
ipcMain.handle(
"save-project-file",
async (_, projectData: unknown, suggestedName?: string, existingProjectPath?: string) => {
- try {
- const trustedExistingProjectPath = isTrustedProjectPath(existingProjectPath)
- ? existingProjectPath
- : null;
-
- if (trustedExistingProjectPath) {
- await fs.writeFile(
- trustedExistingProjectPath,
- JSON.stringify(projectData, null, 2),
- "utf-8",
- );
- currentProjectPath = trustedExistingProjectPath;
- return {
- success: true,
- path: trustedExistingProjectPath,
- message: "Project saved successfully",
- };
- }
-
- const safeName = (suggestedName || `project-${Date.now()}`).replace(/[^a-zA-Z0-9-_]/g, "_");
- const defaultName = safeName.endsWith(`.${PROJECT_FILE_EXTENSION}`)
- ? safeName
- : `${safeName}.${PROJECT_FILE_EXTENSION}`;
-
- const dialogOptions = buildDialogOptions(
- {
- title: mainT("dialogs", "fileDialogs.saveProject"),
- defaultPath: path.join(RECORDINGS_DIR, defaultName),
- filters: [
- {
- name: mainT("dialogs", "fileDialogs.openscreenProject"),
- extensions: [PROJECT_FILE_EXTENSION],
- },
- { name: "JSON", extensions: ["json"] },
- ],
- properties: ["createDirectory", "showOverwriteConfirmation"],
- },
- getMainWindow(),
- );
- const result = await dialog.showSaveDialog(dialogOptions);
-
- if (result.canceled || !result.filePath) {
- return {
- success: false,
- canceled: true,
- message: "Save project canceled",
- };
- }
-
- await fs.writeFile(result.filePath, JSON.stringify(projectData, null, 2), "utf-8");
- currentProjectPath = result.filePath;
-
- return {
- success: true,
- path: result.filePath,
- message: "Project saved successfully",
- };
- } catch (error) {
- console.error("Failed to save project file:", error);
- return {
- success: false,
- message: "Failed to save project file",
- error: String(error),
- };
- }
+ return saveProjectFile(projectData, suggestedName, existingProjectPath);
},
);
+ async function saveProjectFile(
+ projectData: unknown,
+ suggestedName?: string,
+ existingProjectPath?: string,
+ ): Promise {
+ try {
+ const trustedExistingProjectPath = isTrustedProjectPath(existingProjectPath)
+ ? existingProjectPath
+ : null;
+
+ if (trustedExistingProjectPath) {
+ await fs.writeFile(
+ trustedExistingProjectPath,
+ JSON.stringify(projectData, null, 2),
+ "utf-8",
+ );
+ currentProjectPath = trustedExistingProjectPath;
+ return {
+ success: true,
+ path: trustedExistingProjectPath,
+ message: "Project saved successfully",
+ };
+ }
+
+ const safeName = (suggestedName || `project-${Date.now()}`).replace(/[^a-zA-Z0-9-_]/g, "_");
+ const defaultName = safeName.endsWith(`.${PROJECT_FILE_EXTENSION}`)
+ ? safeName
+ : `${safeName}.${PROJECT_FILE_EXTENSION}`;
+
+ const dialogOptions = buildDialogOptions(
+ {
+ title: mainT("dialogs", "fileDialogs.saveProject"),
+ defaultPath: path.join(RECORDINGS_DIR, defaultName),
+ filters: [
+ {
+ name: mainT("dialogs", "fileDialogs.openscreenProject"),
+ extensions: [PROJECT_FILE_EXTENSION],
+ },
+ { name: "JSON", extensions: ["json"] },
+ ],
+ properties: ["createDirectory", "showOverwriteConfirmation"],
+ },
+ getMainWindow(),
+ );
+ const result = await dialog.showSaveDialog(dialogOptions);
+
+ if (result.canceled || !result.filePath) {
+ return {
+ success: false,
+ canceled: true,
+ message: "Save project canceled",
+ };
+ }
+
+ await fs.writeFile(result.filePath, JSON.stringify(projectData, null, 2), "utf-8");
+ currentProjectPath = result.filePath;
+
+ return {
+ success: true,
+ path: result.filePath,
+ message: "Project saved successfully",
+ };
+ } catch (error) {
+ console.error("Failed to save project file:", error);
+ return {
+ success: false,
+ message: "Failed to save project file",
+ error: String(error),
+ };
+ }
+ }
+
ipcMain.handle("load-project-file", async () => {
+ return loadProjectFile();
+ });
+
+ async function loadProjectFile(): Promise {
try {
const dialogOptions = buildDialogOptions(
{
@@ -1261,9 +910,19 @@ export function registerIpcHandlers(
const filePath = result.filePaths[0];
const content = await fs.readFile(filePath, "utf-8");
const project = JSON.parse(content);
- const session = await getApprovedProjectSession(project, filePath);
currentProjectPath = filePath;
- setCurrentRecordingSessionState(session);
+ if (project && typeof project === "object") {
+ const rawProject = project as { media?: unknown; videoPath?: unknown };
+ const media =
+ normalizeProjectMedia(rawProject.media) ??
+ (typeof rawProject.videoPath === "string"
+ ? {
+ screenVideoPath:
+ normalizeVideoSourcePath(rawProject.videoPath) ?? rawProject.videoPath,
+ }
+ : null);
+ setCurrentRecordingSessionState(media ? { ...media, createdAt: Date.now() } : null);
+ }
return {
success: true,
@@ -1278,9 +937,13 @@ export function registerIpcHandlers(
error: String(error),
};
}
- });
+ }
ipcMain.handle("load-current-project-file", async () => {
+ return loadCurrentProjectFile();
+ });
+
+ async function loadCurrentProjectFile(): Promise {
try {
if (!currentProjectPath) {
return { success: false, message: "No active project" };
@@ -1288,8 +951,18 @@ export function registerIpcHandlers(
const content = await fs.readFile(currentProjectPath, "utf-8");
const project = JSON.parse(content);
- const session = await getApprovedProjectSession(project, currentProjectPath);
- setCurrentRecordingSessionState(session);
+ if (project && typeof project === "object") {
+ const rawProject = project as { media?: unknown; videoPath?: unknown };
+ const media =
+ normalizeProjectMedia(rawProject.media) ??
+ (typeof rawProject.videoPath === "string"
+ ? {
+ screenVideoPath:
+ normalizeVideoSourcePath(rawProject.videoPath) ?? rawProject.videoPath,
+ }
+ : null);
+ setCurrentRecordingSessionState(media ? { ...media, createdAt: Date.now() } : null);
+ }
return {
success: true,
path: currentProjectPath,
@@ -1303,55 +976,35 @@ export function registerIpcHandlers(
error: String(error),
};
}
- });
- ipcMain.handle("set-current-recording-session", (_, session: RecordingSession | null) => {
- const normalized = normalizeRecordingSession(session);
- setCurrentRecordingSessionState(normalized);
- currentProjectPath = null;
- return { success: true, session: normalized ?? undefined };
+ }
+
+ ipcMain.handle("set-current-video-path", (_, path: string) => {
+ return setCurrentVideoPath(path);
});
- ipcMain.handle("get-current-recording-session", () => {
- return currentRecordingSession
- ? { success: true, session: currentRecordingSession }
- : { success: false };
- });
-
- ipcMain.handle("set-current-video-path", async (_, path: string) => {
- const normalizedPath = normalizeVideoSourcePath(path);
- if (!normalizedPath || !isPathAllowed(normalizedPath)) {
- return { success: false, message: "Video path has not been approved" };
- }
-
- const restoredSession = await loadRecordedSessionForVideoPath(normalizedPath);
- if (restoredSession) {
- // Approve all media paths from the restored session so they can be read later
- approveFilePath(restoredSession.screenVideoPath);
- if (restoredSession.webcamVideoPath) {
- approveFilePath(restoredSession.webcamVideoPath);
- }
- setCurrentRecordingSessionState(restoredSession);
- } else {
- setCurrentRecordingSessionState({
- screenVideoPath: normalizedPath,
- createdAt: Date.now(),
- });
- }
+ function setCurrentVideoPath(path: string): ProjectPathResult {
+ currentVideoPath = normalizeVideoSourcePath(path) ?? path;
currentProjectPath = null;
return { success: true };
- });
+ }
ipcMain.handle("get-current-video-path", () => {
- return currentRecordingSession?.screenVideoPath
- ? { success: true, path: currentRecordingSession.screenVideoPath }
- : { success: false };
+ return getCurrentVideoPathResult();
});
+ function getCurrentVideoPathResult(): ProjectPathResult {
+ return currentVideoPath ? { success: true, path: currentVideoPath } : { success: false };
+ }
+
ipcMain.handle("clear-current-video-path", () => {
- setCurrentRecordingSessionState(null);
- return { success: true };
+ return clearCurrentVideoPath();
});
+ function clearCurrentVideoPath(): ProjectPathResult {
+ currentVideoPath = null;
+ return { success: true };
+ }
+
ipcMain.handle("get-platform", () => {
return process.platform;
});
diff --git a/electron/native-bridge/cursor/recording/factory.ts b/electron/native-bridge/cursor/recording/factory.ts
new file mode 100644
index 0000000..fe92991
--- /dev/null
+++ b/electron/native-bridge/cursor/recording/factory.ts
@@ -0,0 +1,29 @@
+import type { Rectangle } from "electron";
+import type { CursorRecordingSession } from "./session";
+import { TelemetryRecordingSession } from "./telemetryRecordingSession";
+import { WindowsNativeRecordingSession } from "./windowsNativeRecordingSession";
+
+interface CreateCursorRecordingSessionOptions {
+ getDisplayBounds: () => Rectangle | null;
+ maxSamples: number;
+ platform: NodeJS.Platform;
+ sampleIntervalMs: number;
+}
+
+export function createCursorRecordingSession(
+ options: CreateCursorRecordingSessionOptions,
+): CursorRecordingSession {
+ if (options.platform === "win32") {
+ return new WindowsNativeRecordingSession({
+ getDisplayBounds: options.getDisplayBounds,
+ maxSamples: options.maxSamples,
+ sampleIntervalMs: options.sampleIntervalMs,
+ });
+ }
+
+ return new TelemetryRecordingSession({
+ getDisplayBounds: options.getDisplayBounds,
+ maxSamples: options.maxSamples,
+ sampleIntervalMs: options.sampleIntervalMs,
+ });
+}
diff --git a/electron/native-bridge/cursor/recording/session.ts b/electron/native-bridge/cursor/recording/session.ts
new file mode 100644
index 0000000..9cebe9f
--- /dev/null
+++ b/electron/native-bridge/cursor/recording/session.ts
@@ -0,0 +1,6 @@
+import type { CursorRecordingData } from "../../../../src/native/contracts";
+
+export interface CursorRecordingSession {
+ start(): Promise;
+ stop(): Promise;
+}
diff --git a/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts b/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts
new file mode 100644
index 0000000..dd42871
--- /dev/null
+++ b/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts
@@ -0,0 +1,62 @@
+import { type Rectangle, screen } from "electron";
+import type { CursorRecordingData, CursorRecordingSample } from "../../../../src/native/contracts";
+import type { CursorRecordingSession } from "./session";
+
+interface TelemetryRecordingSessionOptions {
+ getDisplayBounds: () => Rectangle | null;
+ maxSamples: number;
+ sampleIntervalMs: number;
+}
+
+function clamp(value: number, min: number, max: number) {
+ return Math.min(max, Math.max(min, value));
+}
+
+export class TelemetryRecordingSession implements CursorRecordingSession {
+ private samples: CursorRecordingSample[] = [];
+ private interval: NodeJS.Timeout | null = null;
+ private startTimeMs = 0;
+
+ constructor(private readonly options: TelemetryRecordingSessionOptions) {}
+
+ async start(): Promise {
+ this.samples = [];
+ this.startTimeMs = Date.now();
+ this.captureSample();
+ this.interval = setInterval(() => {
+ this.captureSample();
+ }, this.options.sampleIntervalMs);
+ }
+
+ async stop(): Promise {
+ if (this.interval) {
+ clearInterval(this.interval);
+ this.interval = null;
+ }
+
+ return {
+ version: 2,
+ provider: "none",
+ samples: this.samples,
+ assets: [],
+ };
+ }
+
+ private captureSample() {
+ const cursor = screen.getCursorScreenPoint();
+ const display = this.options.getDisplayBounds() ?? screen.getDisplayNearestPoint(cursor).bounds;
+ const width = Math.max(1, display.width);
+ const height = Math.max(1, display.height);
+
+ this.samples.push({
+ timeMs: Math.max(0, Date.now() - this.startTimeMs),
+ cx: clamp((cursor.x - display.x) / width, 0, 1),
+ cy: clamp((cursor.y - display.y) / height, 0, 1),
+ visible: true,
+ });
+
+ if (this.samples.length > this.options.maxSamples) {
+ this.samples.shift();
+ }
+ }
+}
diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts
new file mode 100644
index 0000000..a0540ed
--- /dev/null
+++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts
@@ -0,0 +1,326 @@
+import { type ChildProcessByStdio, spawn } from "node:child_process";
+import type { Readable } from "node:stream";
+import { type Rectangle, screen } from "electron";
+import type {
+ CursorRecordingData,
+ CursorRecordingSample,
+ NativeCursorAsset,
+} from "../../../../src/native/contracts";
+import type { CursorRecordingSession } from "./session";
+
+interface WindowsCursorSampleEvent {
+ type: "sample";
+ timestampMs: number;
+ x: number;
+ y: number;
+ visible: boolean;
+ handle: string | null;
+ asset?: WindowsCursorAssetPayload;
+}
+
+interface WindowsCursorReadyEvent {
+ type: "ready";
+ timestampMs: number;
+}
+
+interface WindowsCursorErrorEvent {
+ type: "error";
+ timestampMs: number;
+ message: string;
+}
+
+interface WindowsCursorAssetPayload {
+ id: string;
+ imageDataUrl: string;
+ width: number;
+ height: number;
+ hotspotX: number;
+ hotspotY: number;
+}
+
+type WindowsCursorEvent =
+ | WindowsCursorSampleEvent
+ | WindowsCursorReadyEvent
+ | WindowsCursorErrorEvent;
+
+interface WindowsNativeRecordingSessionOptions {
+ getDisplayBounds: () => Rectangle | null;
+ maxSamples: number;
+ sampleIntervalMs: number;
+}
+
+function clamp(value: number, min: number, max: number) {
+ return Math.min(max, Math.max(min, value));
+}
+
+function buildPowerShellCommand(sampleIntervalMs: number) {
+ const script = String.raw`
+$ErrorActionPreference = 'Stop'
+Add-Type -AssemblyName System.Drawing
+
+$source = @"
+using System;
+using System.Runtime.InteropServices;
+
+public static class OpenScreenCursorInterop {
+ [StructLayout(LayoutKind.Sequential)]
+ public struct POINT {
+ public int X;
+ public int Y;
+ }
+
+ [StructLayout(LayoutKind.Sequential)]
+ public struct CURSORINFO {
+ public int cbSize;
+ public int flags;
+ public IntPtr hCursor;
+ public POINT ptScreenPos;
+ }
+
+ [StructLayout(LayoutKind.Sequential)]
+ public struct ICONINFO {
+ [MarshalAs(UnmanagedType.Bool)]
+ public bool fIcon;
+ public int xHotspot;
+ public int yHotspot;
+ public IntPtr hbmMask;
+ public IntPtr hbmColor;
+ }
+
+ [DllImport("user32.dll", SetLastError = true)]
+ [return: MarshalAs(UnmanagedType.Bool)]
+ public static extern bool GetCursorInfo(ref CURSORINFO pci);
+
+ [DllImport("user32.dll", SetLastError = true)]
+ public static extern IntPtr CopyIcon(IntPtr hIcon);
+
+ [DllImport("user32.dll", SetLastError = true)]
+ [return: MarshalAs(UnmanagedType.Bool)]
+ public static extern bool DestroyIcon(IntPtr hIcon);
+
+ [DllImport("user32.dll", SetLastError = true)]
+ [return: MarshalAs(UnmanagedType.Bool)]
+ public static extern bool GetIconInfo(IntPtr hIcon, out ICONINFO piconinfo);
+
+ [DllImport("gdi32.dll", SetLastError = true)]
+ [return: MarshalAs(UnmanagedType.Bool)]
+ public static extern bool DeleteObject(IntPtr hObject);
+}
+"@
+
+Add-Type -TypeDefinition $source
+
+function Write-JsonLine($payload) {
+ [Console]::Out.WriteLine(($payload | ConvertTo-Json -Compress -Depth 6))
+}
+
+function Get-CursorAsset($cursorHandle, $cursorId) {
+ $copiedHandle = [OpenScreenCursorInterop]::CopyIcon($cursorHandle)
+ if ($copiedHandle -eq [IntPtr]::Zero) {
+ return $null
+ }
+
+ $iconInfo = New-Object OpenScreenCursorInterop+ICONINFO
+ $hasIconInfo = [OpenScreenCursorInterop]::GetIconInfo($copiedHandle, [ref]$iconInfo)
+
+ try {
+ $icon = [System.Drawing.Icon]::FromHandle($copiedHandle)
+ $bitmap = New-Object System.Drawing.Bitmap $icon.Width, $icon.Height, ([System.Drawing.Imaging.PixelFormat]::Format32bppArgb)
+ $graphics = [System.Drawing.Graphics]::FromImage($bitmap)
+ $memoryStream = New-Object System.IO.MemoryStream
+
+ try {
+ $graphics.Clear([System.Drawing.Color]::Transparent)
+ $graphics.DrawIcon($icon, 0, 0)
+ $bitmap.Save($memoryStream, [System.Drawing.Imaging.ImageFormat]::Png)
+ $base64 = [System.Convert]::ToBase64String($memoryStream.ToArray())
+
+ return @{
+ id = $cursorId
+ imageDataUrl = "data:image/png;base64,$base64"
+ width = $bitmap.Width
+ height = $bitmap.Height
+ hotspotX = if ($hasIconInfo) { $iconInfo.xHotspot } else { 0 }
+ hotspotY = if ($hasIconInfo) { $iconInfo.yHotspot } else { 0 }
+ }
+ }
+ finally {
+ $memoryStream.Dispose()
+ $graphics.Dispose()
+ $bitmap.Dispose()
+ $icon.Dispose()
+ }
+ }
+ finally {
+ if ($hasIconInfo) {
+ if ($iconInfo.hbmMask -ne [IntPtr]::Zero) {
+ [OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmMask) | Out-Null
+ }
+ if ($iconInfo.hbmColor -ne [IntPtr]::Zero) {
+ [OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmColor) | Out-Null
+ }
+ }
+ [OpenScreenCursorInterop]::DestroyIcon($copiedHandle) | Out-Null
+ }
+}
+
+Write-JsonLine @{ type = 'ready'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() }
+
+$lastCursorId = $null
+while ($true) {
+ $cursorInfo = New-Object OpenScreenCursorInterop+CURSORINFO
+ $cursorInfo.cbSize = [Runtime.InteropServices.Marshal]::SizeOf([type][OpenScreenCursorInterop+CURSORINFO])
+
+ if (-not [OpenScreenCursorInterop]::GetCursorInfo([ref]$cursorInfo)) {
+ Write-JsonLine @{ type = 'error'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds(); message = 'GetCursorInfo failed' }
+ Start-Sleep -Milliseconds ${sampleIntervalMs}
+ continue
+ }
+
+ $visible = ($cursorInfo.flags -band 1) -ne 0
+ $cursorId = if ($cursorInfo.hCursor -eq [IntPtr]::Zero) { $null } else { ('0x{0:X}' -f $cursorInfo.hCursor.ToInt64()) }
+ $asset = $null
+
+ if ($visible -and $cursorId -and $cursorId -ne $lastCursorId) {
+ $asset = Get-CursorAsset -cursorHandle $cursorInfo.hCursor -cursorId $cursorId
+ $lastCursorId = $cursorId
+ }
+
+ Write-JsonLine @{
+ type = 'sample'
+ timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds()
+ x = $cursorInfo.ptScreenPos.X
+ y = $cursorInfo.ptScreenPos.Y
+ visible = $visible
+ handle = $cursorId
+ asset = $asset
+ }
+
+ Start-Sleep -Milliseconds ${sampleIntervalMs}
+}
+`;
+
+ return Buffer.from(script, "utf16le").toString("base64");
+}
+
+export class WindowsNativeRecordingSession implements CursorRecordingSession {
+ private assets = new Map();
+ private samples: CursorRecordingSample[] = [];
+ private process: ChildProcessByStdio | null = null;
+ private lineBuffer = "";
+ private startTimeMs = 0;
+
+ constructor(private readonly options: WindowsNativeRecordingSessionOptions) {}
+
+ async start(): Promise {
+ this.assets.clear();
+ this.samples = [];
+ this.lineBuffer = "";
+ this.startTimeMs = Date.now();
+
+ const encodedCommand = buildPowerShellCommand(this.options.sampleIntervalMs);
+ const child = spawn(
+ "powershell.exe",
+ [
+ "-NoLogo",
+ "-NoProfile",
+ "-NonInteractive",
+ "-ExecutionPolicy",
+ "Bypass",
+ "-EncodedCommand",
+ encodedCommand,
+ ],
+ {
+ stdio: ["ignore", "pipe", "pipe"],
+ windowsHide: true,
+ },
+ );
+
+ this.process = child;
+ child.stdout.setEncoding("utf8");
+ child.stdout.on("data", (chunk: string) => {
+ this.handleStdoutChunk(chunk);
+ });
+ child.stderr.setEncoding("utf8");
+ child.stderr.on("data", (chunk: string) => {
+ console.error("[cursor-native]", chunk.trim());
+ });
+ }
+
+ async stop(): Promise {
+ const child = this.process;
+ this.process = null;
+
+ if (child && !child.killed) {
+ child.kill();
+ }
+
+ return {
+ version: 2,
+ provider: this.assets.size > 0 ? "native" : "none",
+ samples: this.samples,
+ assets: [...this.assets.values()],
+ };
+ }
+
+ private handleStdoutChunk(chunk: string) {
+ this.lineBuffer += chunk;
+ const lines = this.lineBuffer.split(/\r?\n/);
+ this.lineBuffer = lines.pop() ?? "";
+
+ for (const line of lines) {
+ const trimmedLine = line.trim();
+ if (!trimmedLine) {
+ continue;
+ }
+
+ try {
+ const payload = JSON.parse(trimmedLine) as WindowsCursorEvent;
+ this.handleEvent(payload);
+ } catch (error) {
+ console.error("Failed to parse Windows cursor helper output:", error, trimmedLine);
+ }
+ }
+ }
+
+ private handleEvent(payload: WindowsCursorEvent) {
+ if (payload.type === "error") {
+ console.error("Windows cursor helper error:", payload.message);
+ return;
+ }
+
+ if (payload.type === "ready") {
+ return;
+ }
+
+ if (payload.asset?.id && !this.assets.has(payload.asset.id)) {
+ const assetDisplay = screen.getDisplayNearestPoint({ x: payload.x, y: payload.y });
+ this.assets.set(payload.asset.id, {
+ id: payload.asset.id,
+ platform: "win32",
+ imageDataUrl: payload.asset.imageDataUrl,
+ width: payload.asset.width,
+ height: payload.asset.height,
+ hotspotX: payload.asset.hotspotX,
+ hotspotY: payload.asset.hotspotY,
+ scaleFactor: assetDisplay.scaleFactor,
+ });
+ }
+
+ const bounds = this.options.getDisplayBounds() ?? screen.getPrimaryDisplay().bounds;
+ const width = Math.max(1, bounds.width);
+ const height = Math.max(1, bounds.height);
+
+ this.samples.push({
+ timeMs: Math.max(0, payload.timestampMs - this.startTimeMs),
+ cx: clamp((payload.x - bounds.x) / width, 0, 1),
+ cy: clamp((payload.y - bounds.y) / height, 0, 1),
+ assetId: payload.handle,
+ visible: payload.visible,
+ });
+
+ if (this.samples.length > this.options.maxSamples) {
+ this.samples.shift();
+ }
+ }
+}
diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx
index 2612b68..1ed8938 100644
--- a/src/components/video-editor/VideoEditor.tsx
+++ b/src/components/video-editor/VideoEditor.tsx
@@ -43,6 +43,7 @@ import {
getNativeAspectRatioValue,
isPortraitAspectRatio,
} from "@/utils/aspectRatioUtils";
+import { nativeBridgeClient, useCursorRecordingData, useCursorTelemetry } from "@/native";
import { ExportDialog } from "./ExportDialog";
import PlaybackControls from "./PlaybackControls";
import {
@@ -61,7 +62,6 @@ import TimelineEditor from "./timeline/TimelineEditor";
import {
type AnnotationRegion,
type BlurData,
- type CursorTelemetryPoint,
clampFocusToDepth,
DEFAULT_ANNOTATION_POSITION,
DEFAULT_ANNOTATION_SIZE,
@@ -133,8 +133,6 @@ export default function VideoEditor() {
currentTimeRef.current = currentTime;
const durationRef = useRef(duration);
durationRef.current = duration;
- const [cursorTelemetry, setCursorTelemetry] = useState([]);
- const [cursorClickTimestamps, setCursorClickTimestamps] = useState([]);
const [selectedZoomId, setSelectedZoomId] = useState(null);
const [selectedTrimId, setSelectedTrimId] = useState(null);
const [selectedSpeedId, setSelectedSpeedId] = useState(null);
@@ -220,6 +218,13 @@ export default function VideoEditor() {
const project = candidate;
const sourcePath = project.videoPath;
const normalizedEditor = normalizeProjectEditor(project.editor);
+ const inferredDurationMs = Math.max(
+ 0,
+ ...normalizedEditor.zoomRegions.map((region) => region.endMs),
+ ...normalizedEditor.trimRegions.map((region) => region.endMs),
+ ...normalizedEditor.speedRegions.map((region) => region.endMs),
+ ...normalizedEditor.annotationRegions.map((region) => region.endMs),
+ );
try {
videoPlaybackRef.current?.pause();
@@ -228,7 +233,7 @@ export default function VideoEditor() {
}
setIsPlaying(false);
setCurrentTime(0);
- setDuration(0);
+ setDuration(inferredDurationMs > 0 ? inferredDurationMs / 1000 : 0);
setError(null);
setVideoSourcePath(sourcePath);
@@ -357,7 +362,7 @@ export default function VideoEditor() {
useEffect(() => {
async function loadInitialData() {
try {
- const currentProjectResult = await window.electronAPI.loadCurrentProjectFile();
+ const currentProjectResult = await nativeBridgeClient.project.loadCurrentProjectFile();
if (currentProjectResult.success && currentProjectResult.project) {
const restored = await applyLoadedProject(
currentProjectResult.project,
@@ -394,7 +399,7 @@ export default function VideoEditor() {
return;
}
- const result = await window.electronAPI.getCurrentVideoPath();
+ const result = await nativeBridgeClient.project.getCurrentVideoPath();
if (result.success && result.path) {
setVideoSourcePath(result.path);
setVideoPath(toFileUrl(result.path));
@@ -483,7 +488,7 @@ export default function VideoEditor() {
// Match the normalization path used by `currentProjectSnapshot` so the
// post-save baseline compares equal and `hasUnsavedChanges` clears.
const projectSnapshot = createProjectSnapshot(currentProjectMedia, editorState);
- const result = await window.electronAPI.saveProjectFile(
+ const result = await nativeBridgeClient.project.saveProjectFile(
projectData,
fileNameBase,
forceSaveAs ? undefined : (currentProjectPath ?? undefined),
@@ -589,7 +594,7 @@ export default function VideoEditor() {
}, []);
const handleLoadProject = useCallback(async () => {
- const result = await window.electronAPI.loadProjectFile();
+ const result = await nativeBridgeClient.project.loadProjectFile();
if (result.canceled) {
return;
@@ -622,40 +627,16 @@ export default function VideoEditor() {
}, [handleLoadProject, handleSaveProject, handleSaveProjectAs]);
useEffect(() => {
- let mounted = true;
-
- async function loadCursorTelemetry() {
- const sourcePath = currentProjectMedia?.screenVideoPath ?? null;
-
- if (!sourcePath) {
- if (mounted) {
- setCursorTelemetry([]);
- setCursorClickTimestamps([]);
- }
- return;
- }
-
- try {
- const result = await window.electronAPI.getCursorTelemetry(sourcePath);
- if (mounted) {
- setCursorTelemetry(result.success ? result.samples : []);
- setCursorClickTimestamps(result.success ? (result.clicks ?? []) : []);
- }
- } catch (telemetryError) {
- console.warn("Unable to load cursor telemetry:", telemetryError);
- if (mounted) {
- setCursorTelemetry([]);
- setCursorClickTimestamps([]);
- }
- }
+ if (cursorTelemetryError) {
+ console.warn("Unable to load cursor telemetry:", cursorTelemetryError);
}
+ }, [cursorTelemetryError]);
- loadCursorTelemetry();
-
- return () => {
- mounted = false;
- };
- }, [currentProjectMedia]);
+ useEffect(() => {
+ if (cursorRecordingDataError) {
+ console.warn("Unable to load cursor recording data:", cursorRecordingDataError);
+ }
+ }, [cursorRecordingDataError]);
function togglePlayPause() {
const playback = videoPlaybackRef.current;
@@ -1495,6 +1476,7 @@ export default function VideoEditor() {
padding,
videoPadding: padding,
cropRegion,
+ cursorRecordingData,
annotationRegions,
webcamLayoutPreset,
webcamMaskShape,
@@ -1636,6 +1618,7 @@ export default function VideoEditor() {
borderRadius,
padding,
cropRegion,
+ cursorRecordingData,
annotationRegions,
webcamLayoutPreset,
webcamMaskShape,
@@ -1715,6 +1698,7 @@ export default function VideoEditor() {
borderRadius,
padding,
cropRegion,
+ cursorRecordingData,
annotationRegions,
isPlaying,
aspectRatio,
diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx
index a053c3a..12b1c25 100644
--- a/src/components/video-editor/VideoPlayback.tsx
+++ b/src/components/video-editor/VideoPlayback.tsx
@@ -27,6 +27,12 @@ import {
} from "@/lib/compositeLayout";
import { classifyWallpaper, DEFAULT_WALLPAPER, resolveImageWallpaperUrl } from "@/lib/wallpaper";
import { getCssClipPath } from "@/lib/webcamMaskShapes";
+import {
+ getNativeCursorDisplayMetrics,
+ projectNativeCursorToStage,
+ resolveActiveNativeCursorFrame,
+} from "@/lib/cursor/nativeCursor";
+import type { CursorRecordingData } from "@/native/contracts";
import {
type AspectRatio,
formatAspectRatioForCSS,
@@ -123,6 +129,7 @@ interface VideoPlaybackProps {
trimRegions?: TrimRegion[];
speedRegions?: SpeedRegion[];
aspectRatio: AspectRatio;
+ cursorRecordingData?: CursorRecordingData | null;
annotationRegions?: AnnotationRegion[];
selectedAnnotationId?: string | null;
onSelectAnnotation?: (id: string | null) => void;
@@ -155,6 +162,22 @@ export interface VideoPlaybackRef {
pause: () => void;
}
+function getResolvedVideoDuration(video: HTMLVideoElement): number | null {
+ if (Number.isFinite(video.duration) && video.duration > 0) {
+ return video.duration;
+ }
+
+ if (video.seekable.length > 0) {
+ const lastRangeIndex = video.seekable.length - 1;
+ const seekableEnd = video.seekable.end(lastRangeIndex);
+ if (Number.isFinite(seekableEnd) && seekableEnd > 0) {
+ return seekableEnd;
+ }
+ }
+
+ return null;
+}
+
const VideoPlayback = forwardRef(
(
{
@@ -188,6 +211,7 @@ const VideoPlayback = forwardRef(
trimRegions = [],
speedRegions = [],
aspectRatio,
+ cursorRecordingData,
annotationRegions = [],
selectedAnnotationId,
onSelectAnnotation,
@@ -843,6 +867,8 @@ const VideoPlayback = forwardRef(
useEffect(() => {
if (!videoPath) {
+ lastResolvedDurationRef.current = null;
+ isResolvingDurationRef.current = false;
setVideoReady(false);
return;
}
@@ -853,11 +879,18 @@ const VideoPlayback = forwardRef(
video.currentTime = 0;
allowPlaybackRef.current = false;
lockedVideoDimensionsRef.current = null;
+ lastResolvedDurationRef.current = null;
+ isResolvingDurationRef.current = false;
+ if (durationResolutionTimeoutRef.current) {
+ clearTimeout(durationResolutionTimeoutRef.current);
+ durationResolutionTimeoutRef.current = null;
+ }
setVideoReady(false);
if (videoReadyRafRef.current) {
cancelAnimationFrame(videoReadyRafRef.current);
videoReadyRafRef.current = null;
}
+ video.load();
}, [videoPath]);
useEffect(() => {
@@ -1299,8 +1332,12 @@ const VideoPlayback = forwardRef(
const handleLoadedMetadata = (e: React.SyntheticEvent) => {
const video = e.currentTarget;
- onDurationChange(video.duration);
- video.currentTime = 0;
+ const hasResolvedDuration = syncResolvedDuration(video);
+ if (!hasResolvedDuration) {
+ forceResolveDuration(video);
+ } else {
+ video.currentTime = 0;
+ }
video.pause();
allowPlaybackRef.current = false;
currentTimeRef.current = 0;
@@ -1313,6 +1350,9 @@ const VideoPlayback = forwardRef(
const waitForRenderableFrame = () => {
const hasDimensions = video.videoWidth > 0 && video.videoHeight > 0;
const hasData = video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA;
+ if (!syncResolvedDuration(video)) {
+ forceResolveDuration(video);
+ }
if (hasDimensions && hasData) {
videoReadyRafRef.current = null;
setVideoReady(true);
@@ -1412,6 +1452,10 @@ const VideoPlayback = forwardRef(
window.clearTimeout(scrubEndTimerRef.current);
scrubEndTimerRef.current = null;
}
+ if (durationResolutionTimeoutRef.current) {
+ clearTimeout(durationResolutionTimeoutRef.current);
+ durationResolutionTimeoutRef.current = null;
+ }
};
}, []);
@@ -1527,6 +1571,22 @@ const VideoPlayback = forwardRef(
className="absolute rounded-md border border-[#34B27B]/80 bg-[#34B27B]/20 shadow-[0_0_0_1px_rgba(52,178,123,0.35)]"
style={{ display: "none", pointerEvents: "none" }}
/>
+ {activeNativeCursor && nativeCursorStyle ? (
+ (
ref={videoRef}
src={videoPath}
className="hidden"
- preload="metadata"
+ preload="auto"
+ muted
playsInline
onLoadedMetadata={handleLoadedMetadata}
onDurationChange={(e) => {
- onDurationChange(e.currentTarget.duration);
+ if (!syncResolvedDuration(e.currentTarget)) {
+ forceResolveDuration(e.currentTarget);
+ }
+ }}
+ onLoadedData={(e) => {
+ if (!syncResolvedDuration(e.currentTarget)) {
+ forceResolveDuration(e.currentTarget);
+ }
+ }}
+ onCanPlay={(e) => {
+ if (!syncResolvedDuration(e.currentTarget)) {
+ forceResolveDuration(e.currentTarget);
+ }
}}
onError={() => onError("Failed to load video")}
/>
diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx
index feac886..20faf65 100644
--- a/src/components/video-editor/timeline/TimelineEditor.tsx
+++ b/src/components/video-editor/timeline/TimelineEditor.tsx
@@ -51,6 +51,7 @@ const SUGGESTION_SPACING_MS = 1800;
interface TimelineEditorProps {
videoDuration: number;
+ hasVideoSource?: boolean;
currentTime: number;
onSeek?: (time: number) => void;
cursorTelemetry?: CursorTelemetryPoint[];
@@ -766,6 +767,7 @@ function Timeline({
export default function TimelineEditor({
videoDuration,
+ hasVideoSource = false,
currentTime,
onSeek,
cursorTelemetry = [],
@@ -1439,8 +1441,14 @@ export default function TimelineEditor({
-
{t("emptyState.noVideo")}
-
{t("emptyState.dragAndDrop")}
+
+ {hasVideoSource ? "Loading Timeline" : "No Video Loaded"}
+
+
+ {hasVideoSource
+ ? "Video opened, waiting for duration metadata"
+ : "Drag and drop a video to start editing"}
+
);
diff --git a/src/lib/cursor/nativeCursor.ts b/src/lib/cursor/nativeCursor.ts
new file mode 100644
index 0000000..23e187b
--- /dev/null
+++ b/src/lib/cursor/nativeCursor.ts
@@ -0,0 +1,101 @@
+import { type Container, Point } from "pixi.js";
+import type { CropRegion } from "@/components/video-editor/types";
+import type {
+ CursorRecordingData,
+ CursorRecordingSample,
+ NativeCursorAsset,
+} from "@/native/contracts";
+
+export interface ActiveNativeCursorFrame {
+ asset: NativeCursorAsset;
+ sample: CursorRecordingSample;
+}
+
+interface ProjectNativeCursorOptions {
+ cameraContainer: Container;
+ cropRegion: CropRegion;
+ maskRect: { width: number; height: number };
+ videoContainerPosition: { x: number; y: number };
+ sample: CursorRecordingSample;
+}
+
+function clamp(value: number, min: number, max: number) {
+ return Math.min(max, Math.max(min, value));
+}
+
+function getCroppedCursorPosition(sample: CursorRecordingSample, cropRegion: CropRegion) {
+ if (cropRegion.width <= 0 || cropRegion.height <= 0) {
+ return null;
+ }
+
+ const croppedCx = (sample.cx - cropRegion.x) / cropRegion.width;
+ const croppedCy = (sample.cy - cropRegion.y) / cropRegion.height;
+
+ if (croppedCx < 0 || croppedCx > 1 || croppedCy < 0 || croppedCy > 1) {
+ return null;
+ }
+
+ return {
+ cx: clamp(croppedCx, 0, 1),
+ cy: clamp(croppedCy, 0, 1),
+ };
+}
+
+export function resolveActiveNativeCursorFrame(
+ recordingData: CursorRecordingData | null | undefined,
+ timeMs: number,
+): ActiveNativeCursorFrame | null {
+ if (!recordingData || recordingData.provider !== "native" || recordingData.assets.length === 0) {
+ return null;
+ }
+
+ for (let index = recordingData.samples.length - 1; index >= 0; index -= 1) {
+ const sample = recordingData.samples[index];
+ if (sample.timeMs > timeMs) {
+ continue;
+ }
+
+ if (sample.visible === false || !sample.assetId) {
+ return null;
+ }
+
+ const asset = recordingData.assets.find((candidate) => candidate.id === sample.assetId);
+ if (!asset) {
+ return null;
+ }
+
+ return { sample, asset };
+ }
+
+ return null;
+}
+
+export function projectNativeCursorToStage({
+ cameraContainer,
+ cropRegion,
+ maskRect,
+ videoContainerPosition,
+ sample,
+}: ProjectNativeCursorOptions) {
+ const croppedPosition = getCroppedCursorPosition(sample, cropRegion);
+ if (!croppedPosition) {
+ return null;
+ }
+
+ const localPoint = new Point(
+ videoContainerPosition.x + croppedPosition.cx * maskRect.width,
+ videoContainerPosition.y + croppedPosition.cy * maskRect.height,
+ );
+
+ return cameraContainer.toGlobal(localPoint);
+}
+
+export function getNativeCursorDisplayMetrics(asset: NativeCursorAsset, deviceScaleFactor: number) {
+ const scaleFactor = asset.scaleFactor ?? deviceScaleFactor ?? 1;
+ return {
+ width: asset.width / scaleFactor,
+ height: asset.height / scaleFactor,
+ hotspotX: asset.hotspotX / scaleFactor,
+ hotspotY: asset.hotspotY / scaleFactor,
+ };
+}
diff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.ts
index 017af83..f13735d 100644
--- a/src/lib/exporter/frameRenderer.ts
+++ b/src/lib/exporter/frameRenderer.ts
@@ -56,8 +56,14 @@ import {
type Size,
type StyledRenderRect,
} from "@/lib/compositeLayout";
+import {
+ getNativeCursorDisplayMetrics,
+ projectNativeCursorToStage,
+ resolveActiveNativeCursorFrame,
+} from "@/lib/cursor/nativeCursor";
import { BackgroundLoadError, classifyWallpaper, resolveImageWallpaperUrl } from "@/lib/wallpaper";
import { drawCanvasClipPath } from "@/lib/webcamMaskShapes";
+import type { CursorRecordingData, NativeCursorAsset } from "@/native/contracts";
import { renderAnnotations } from "./annotationRenderer";
import {
getLinearGradientPoints,
@@ -79,6 +85,7 @@ interface FrameRenderConfig {
borderRadius?: number;
padding?: number;
cropRegion: CropRegion;
+ cursorRecordingData?: CursorRecordingData | null;
videoWidth: number;
videoHeight: number;
webcamSize?: Size | null;
@@ -136,6 +143,7 @@ export class FrameRenderer {
private rasterCtx: CanvasRenderingContext2D | null = null;
private threeDPass: ThreeDPass | null = null;
private currentRotation3D: Rotation3D = { ...DEFAULT_ROTATION_3D };
+ private cursorImageCache = new Map
();
private config: FrameRenderConfig;
private animationState: AnimationState;
private layoutCache: LayoutCache | null = null;
@@ -468,6 +476,8 @@ export class FrameRenderer {
}
}
+ await this.drawNativeCursor(timeMs);
+
// Render annotations on top of foreground (so they rotate with recording).
if (
this.config.annotationRegions &&
@@ -543,7 +553,63 @@ export class FrameRenderer {
}
}
- private updateLayout(webcamFrame?: VideoFrame | null): void {
+ private async drawNativeCursor(timeMs: number) {
+ if (!this.compositeCtx || !this.cameraContainer || !this.videoContainer || !this.layoutCache) {
+ return;
+ }
+
+ const activeNativeCursor = resolveActiveNativeCursorFrame(
+ this.config.cursorRecordingData,
+ timeMs,
+ );
+ if (!activeNativeCursor) {
+ return;
+ }
+
+ const projectedPoint = projectNativeCursorToStage({
+ cameraContainer: this.cameraContainer,
+ cropRegion: this.config.cropRegion,
+ maskRect: this.layoutCache.maskRect,
+ videoContainerPosition: {
+ x: this.videoContainer.x,
+ y: this.videoContainer.y,
+ },
+ sample: activeNativeCursor.sample,
+ });
+ if (!projectedPoint) {
+ return;
+ }
+
+ const image = await this.getCursorImage(activeNativeCursor.asset);
+ const metrics = getNativeCursorDisplayMetrics(activeNativeCursor.asset, 1);
+
+ this.compositeCtx.drawImage(
+ image,
+ projectedPoint.x - metrics.hotspotX,
+ projectedPoint.y - metrics.hotspotY,
+ metrics.width,
+ metrics.height,
+ );
+ }
+
+ private async getCursorImage(asset: NativeCursorAsset) {
+ const cachedImage = this.cursorImageCache.get(asset.id);
+ if (cachedImage) {
+ return cachedImage;
+ }
+
+ const image = new Image();
+ await new Promise((resolve, reject) => {
+ image.onload = () => resolve();
+ image.onerror = () => reject(new Error(`Failed to load cursor asset ${asset.id}`));
+ image.src = asset.imageDataUrl;
+ });
+
+ this.cursorImageCache.set(asset.id, image);
+ return image;
+ }
+
+ private updateLayout(): void {
if (!this.app || !this.videoSprite || !this.maskGraphics || !this.videoContainer) return;
const { width, height } = this.config;
@@ -999,5 +1065,6 @@ export class FrameRenderer {
this.threeDPass.destroy();
this.threeDPass = null;
}
+ this.cursorImageCache.clear();
}
}
diff --git a/src/lib/exporter/gifExporter.ts b/src/lib/exporter/gifExporter.ts
index 0d7a432..02564db 100644
--- a/src/lib/exporter/gifExporter.ts
+++ b/src/lib/exporter/gifExporter.ts
@@ -11,6 +11,7 @@ import type {
import { BackgroundLoadError } from "@/lib/wallpaper";
import { getPlatform } from "@/utils/platformUtils";
import { AsyncVideoFrameQueue } from "./asyncVideoFrameQueue";
+import type { CursorRecordingData } from "@/native/contracts";
import { FrameRenderer } from "./frameRenderer";
import { StreamingVideoDecoder } from "./streamingDecoder";
import type {
@@ -47,6 +48,7 @@ interface GifExporterConfig {
webcamMaskShape?: import("@/components/video-editor/types").WebcamMaskShape;
webcamSizePreset?: WebcamSizePreset;
webcamPosition?: { cx: number; cy: number } | null;
+ cursorRecordingData?: CursorRecordingData | null;
annotationRegions?: AnnotationRegion[];
previewWidth?: number;
previewHeight?: number;
@@ -151,6 +153,7 @@ export class GifExporter {
borderRadius: this.config.borderRadius,
padding: this.config.padding,
cropRegion: this.config.cropRegion,
+ cursorRecordingData: this.config.cursorRecordingData,
videoWidth: videoInfo.width,
videoHeight: videoInfo.height,
webcamSize: webcamInfo ? { width: webcamInfo.width, height: webcamInfo.height } : null,
diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts
index e064ba7..edddd05 100644
--- a/src/lib/exporter/videoExporter.ts
+++ b/src/lib/exporter/videoExporter.ts
@@ -10,6 +10,7 @@ import type {
import { BackgroundLoadError } from "@/lib/wallpaper";
import { getPlatform } from "@/utils/platformUtils";
import { AsyncVideoFrameQueue } from "./asyncVideoFrameQueue";
+import type { CursorRecordingData } from "@/native/contracts";
import { AudioProcessor } from "./audioEncoder";
import { FrameRenderer } from "./frameRenderer";
import { VideoMuxer } from "./muxer";
@@ -38,6 +39,7 @@ interface VideoExporterConfig extends ExportConfig {
webcamMaskShape?: import("@/components/video-editor/types").WebcamMaskShape;
webcamSizePreset?: WebcamSizePreset;
webcamPosition?: { cx: number; cy: number } | null;
+ cursorRecordingData?: CursorRecordingData | null;
annotationRegions?: AnnotationRegion[];
previewWidth?: number;
previewHeight?: number;
@@ -146,6 +148,7 @@ export class VideoExporter {
borderRadius: this.config.borderRadius,
padding: this.config.padding,
cropRegion: this.config.cropRegion,
+ cursorRecordingData: this.config.cursorRecordingData,
videoWidth: videoInfo.width,
videoHeight: videoInfo.height,
webcamSize: webcamInfo ? { width: webcamInfo.width, height: webcamInfo.height } : null,
From e9650225bade7f4f2eca86f29c5f5f03145492df Mon Sep 17 00:00:00 2001
From: Etienne Lescot
Date: Thu, 26 Mar 2026 11:16:41 +0100
Subject: [PATCH 06/92] feat: add cursor overlay pipeline for high-fidelity
cursor recording and playback
- Implement native bridge for Windows cursor capture via PowerShell/C#
- Add cursor-free capture using getDisplayMedia with setDisplayMediaRequestHandler
- Update video player and exporters to support native cursor telemetry
- Enable system audio capture on Windows via WASAPI loopback
- Add interpolation for smoother cursor movement in playback and export
- Improve cursor scaling and visibility handling in editor and playback
---
electron/ipc/handlers.ts | 123 ++++--
electron/main.ts | 2 +-
.../native-bridge/cursor/recording/factory.ts | 2 +
.../windowsNativeRecordingSession.script.ts | 216 +++++++++++
.../windowsNativeRecordingSession.ts | 352 ++++++++----------
.../windowsNativeRecordingSession.types.ts | 49 +++
src/components/launch/LaunchWindow.tsx | 5 +
src/components/video-editor/VideoEditor.tsx | 2 +
src/components/video-editor/VideoPlayback.tsx | 81 +++-
src/hooks/useScreenRecorder.ts | 50 +--
src/lib/cursor/nativeCursor.ts | 78 +++-
src/lib/exporter/frameRenderer.ts | 19 +-
src/lib/exporter/gifExporter.ts | 2 +
src/lib/exporter/videoExporter.ts | 2 +
14 files changed, 686 insertions(+), 297 deletions(-)
create mode 100644 electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts
create mode 100644 electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts
diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts
index 1d73a9b..d50cab3 100644
--- a/electron/ipc/handlers.ts
+++ b/electron/ipc/handlers.ts
@@ -4,8 +4,6 @@ import os from "node:os";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
-const nodeRequire = createRequire(import.meta.url);
-
import {
app,
BrowserWindow,
@@ -16,10 +14,7 @@ import {
shell,
systemPreferences,
} from "electron";
-import {
- type CursorTelemetryPoint,
- createCursorTelemetryBuffer,
-} from "../../src/lib/cursorTelemetryBuffer";
+import type { DesktopCapturerSource } from "electron";
import {
normalizeProjectMedia,
normalizeRecordingSession,
@@ -198,11 +193,24 @@ async function getApprovedProjectSession(
type SelectedSource = {
name: string;
+ id?: string;
+ display_id?: string;
[key: string]: unknown;
};
let selectedSource: SelectedSource | null = null;
+let selectedDesktopSource: DesktopCapturerSource | null = null;
+let lastEnumeratedSources = new Map();
let currentProjectPath: string | null = null;
+let currentRecordingSession: RecordingSession | null = null;
+
+/**
+ * Returns the cached DesktopCapturerSource set when the user picked a source.
+ * Used by setDisplayMediaRequestHandler in main.ts for cursor-free capture.
+ */
+export function getSelectedDesktopSource(): DesktopCapturerSource | null {
+ return selectedDesktopSource;
+}
let currentVideoPath: string | null = null;
function normalizePath(filePath: string) {
@@ -238,16 +246,12 @@ function isTrustedProjectPath(filePath?: string | null) {
}
const CURSOR_TELEMETRY_VERSION = 2;
-const CURSOR_SAMPLE_INTERVAL_MS = 100;
-const MAX_CURSOR_SAMPLES = 60 * 60 * 10; // 1 hour @ 10Hz
+const CURSOR_SAMPLE_INTERVAL_MS = 33;
+const MAX_CURSOR_SAMPLES = 60 * 60 * 30; // 1 hour @ 30Hz
let cursorRecordingSession: CursorRecordingSession | null = null;
let pendingCursorRecordingData: CursorRecordingData | null = null;
-function clamp(value: number, min: number, max: number) {
- return Math.min(max, Math.max(min, value));
-}
-
function normalizeCursorSample(sample: unknown): CursorRecordingSample | null {
if (!sample || typeof sample !== "object") {
return null;
@@ -259,8 +263,8 @@ function normalizeCursorSample(sample: unknown): CursorRecordingSample | null {
typeof point.timeMs === "number" && Number.isFinite(point.timeMs)
? Math.max(0, point.timeMs)
: 0,
- cx: typeof point.cx === "number" && Number.isFinite(point.cx) ? clamp(point.cx, 0, 1) : 0.5,
- cy: typeof point.cy === "number" && Number.isFinite(point.cy) ? clamp(point.cy, 0, 1) : 0.5,
+ cx: typeof point.cx === "number" && Number.isFinite(point.cx) ? point.cx : 0.5,
+ cy: typeof point.cy === "number" && Number.isFinite(point.cy) ? point.cy : 0.5,
assetId: typeof point.assetId === "string" ? point.assetId : null,
visible: typeof point.visible === "boolean" ? point.visible : true,
};
@@ -395,6 +399,55 @@ function getSelectedSourceBounds() {
return (sourceDisplay ?? screen.getDisplayNearestPoint(cursor)).bounds;
}
+function getSelectedSourceId() {
+ return typeof selectedSource?.id === "string" ? selectedSource.id : null;
+}
+
+function setCurrentRecordingSessionState(session: RecordingSession | null) {
+ currentRecordingSession = session;
+ currentVideoPath = session?.screenVideoPath ?? null;
+}
+
+async function storeRecordedSessionFiles(payload: StoreRecordedSessionInput) {
+ const createdAt =
+ typeof payload.createdAt === "number" && Number.isFinite(payload.createdAt)
+ ? payload.createdAt
+ : Date.now();
+ const screenVideoPath = resolveRecordingOutputPath(payload.screen.fileName);
+ await fs.writeFile(screenVideoPath, Buffer.from(payload.screen.videoData));
+
+ let webcamVideoPath: string | undefined;
+ if (payload.webcam) {
+ webcamVideoPath = resolveRecordingOutputPath(payload.webcam.fileName);
+ await fs.writeFile(webcamVideoPath, Buffer.from(payload.webcam.videoData));
+ }
+
+ const session: RecordingSession = webcamVideoPath
+ ? { screenVideoPath, webcamVideoPath, createdAt }
+ : { screenVideoPath, createdAt };
+ setCurrentRecordingSessionState(session);
+ currentProjectPath = null;
+
+ const telemetryPath = `${screenVideoPath}.cursor.json`;
+ if (pendingCursorRecordingData && pendingCursorRecordingData.samples.length > 0) {
+ await fs.writeFile(telemetryPath, JSON.stringify(pendingCursorRecordingData, null, 2), "utf-8");
+ }
+ pendingCursorRecordingData = null;
+
+ const sessionManifestPath = path.join(
+ RECORDINGS_DIR,
+ `${path.parse(payload.screen.fileName).name}${RECORDING_SESSION_SUFFIX}`,
+ );
+ await fs.writeFile(sessionManifestPath, JSON.stringify(session, null, 2), "utf-8");
+
+ return {
+ success: true,
+ path: screenVideoPath,
+ session,
+ message: "Recording session stored successfully",
+ };
+}
+
export function registerIpcHandlers(
createEditorWindow: () => void,
createSourceSelectorWindow: () => BrowserWindow,
@@ -404,6 +457,7 @@ export function registerIpcHandlers(
) {
ipcMain.handle("get-sources", async (_, opts) => {
const sources = await desktopCapturer.getSources(opts);
+ lastEnumeratedSources = new Map(sources.map((source) => [source.id, source]));
return sources.map((source) => ({
id: source.id,
name: source.name,
@@ -413,8 +467,26 @@ export function registerIpcHandlers(
}));
});
- ipcMain.handle("select-source", (_, source: SelectedSource) => {
+ ipcMain.handle("select-source", async (_, source: SelectedSource) => {
selectedSource = source;
+ // Reuse the exact source object returned during enumeration to avoid
+ // Windows window-source id mismatches across separate getSources() calls.
+ selectedDesktopSource =
+ typeof source.id === "string" ? lastEnumeratedSources.get(source.id) ?? null : null;
+
+ if (!selectedDesktopSource && typeof source.id === "string") {
+ try {
+ const sources = await desktopCapturer.getSources({
+ types: ["screen", "window"],
+ thumbnailSize: { width: 0, height: 0 },
+ fetchWindowIcons: true,
+ });
+ lastEnumeratedSources = new Map(sources.map((candidate) => [candidate.id, candidate]));
+ selectedDesktopSource = lastEnumeratedSources.get(source.id) ?? null;
+ } catch {
+ selectedDesktopSource = null;
+ }
+ }
const sourceSelectorWin = getSourceSelectorWindow();
if (sourceSelectorWin) {
sourceSelectorWin.close();
@@ -519,25 +591,7 @@ export function registerIpcHandlers(
ipcMain.handle("store-recorded-session", async (_, payload: StoreRecordedSessionInput) => {
try {
- const videoPath = path.join(RECORDINGS_DIR, fileName);
- await fs.writeFile(videoPath, Buffer.from(videoData));
- currentProjectPath = null;
-
- const telemetryPath = `${videoPath}.cursor.json`;
- if (pendingCursorRecordingData && pendingCursorRecordingData.samples.length > 0) {
- await fs.writeFile(
- telemetryPath,
- JSON.stringify(pendingCursorRecordingData, null, 2),
- "utf-8",
- );
- }
- pendingCursorRecordingData = null;
-
- return {
- success: true,
- path: videoPath,
- message: "Video stored successfully",
- };
+ return await storeRecordedSessionFiles(payload);
} catch (error) {
console.error("Failed to store recording session:", error);
return {
@@ -602,6 +656,7 @@ export function registerIpcHandlers(
maxSamples: MAX_CURSOR_SAMPLES,
platform: process.platform,
sampleIntervalMs: CURSOR_SAMPLE_INTERVAL_MS,
+ sourceId: getSelectedSourceId(),
});
try {
diff --git a/electron/main.ts b/electron/main.ts
index 4e443a9..1db4740 100644
--- a/electron/main.ts
+++ b/electron/main.ts
@@ -13,7 +13,7 @@ import {
Tray,
} from "electron";
import { mainT, setMainLocale } from "./i18n";
-import { registerIpcHandlers } from "./ipc/handlers";
+import { getSelectedDesktopSource, registerIpcHandlers } from "./ipc/handlers";
import {
createCountdownOverlayWindow,
createEditorWindow,
diff --git a/electron/native-bridge/cursor/recording/factory.ts b/electron/native-bridge/cursor/recording/factory.ts
index fe92991..4e0f75c 100644
--- a/electron/native-bridge/cursor/recording/factory.ts
+++ b/electron/native-bridge/cursor/recording/factory.ts
@@ -8,6 +8,7 @@ interface CreateCursorRecordingSessionOptions {
maxSamples: number;
platform: NodeJS.Platform;
sampleIntervalMs: number;
+ sourceId?: string | null;
}
export function createCursorRecordingSession(
@@ -18,6 +19,7 @@ export function createCursorRecordingSession(
getDisplayBounds: options.getDisplayBounds,
maxSamples: options.maxSamples,
sampleIntervalMs: options.sampleIntervalMs,
+ sourceId: options.sourceId,
});
}
diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts
new file mode 100644
index 0000000..b7a11cb
--- /dev/null
+++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts
@@ -0,0 +1,216 @@
+export function parseWindowHandleFromSourceId(sourceId?: string | null) {
+ if (!sourceId?.startsWith("window:")) {
+ return null;
+ }
+
+ const handlePart = sourceId.split(":")[1];
+ if (!handlePart || !/^\d+$/.test(handlePart)) {
+ return null;
+ }
+
+ return handlePart;
+}
+
+export function buildPowerShellCommand(sampleIntervalMs: number, windowHandle?: string | null) {
+ const script = String.raw`
+$ErrorActionPreference = 'Stop'
+Add-Type -AssemblyName System.Drawing
+
+$targetWindowHandle = ${windowHandle ? `'${windowHandle}'` : '$null'}
+
+$source = @"
+using System;
+using System.Runtime.InteropServices;
+
+public static class OpenScreenCursorInterop {
+ [StructLayout(LayoutKind.Sequential)]
+ public struct POINT {
+ public int X;
+ public int Y;
+ }
+
+ [StructLayout(LayoutKind.Sequential)]
+ public struct CURSORINFO {
+ public int cbSize;
+ public int flags;
+ public IntPtr hCursor;
+ public POINT ptScreenPos;
+ }
+
+ [StructLayout(LayoutKind.Sequential)]
+ public struct ICONINFO {
+ [MarshalAs(UnmanagedType.Bool)]
+ public bool fIcon;
+ public int xHotspot;
+ public int yHotspot;
+ public IntPtr hbmMask;
+ public IntPtr hbmColor;
+ }
+
+ [StructLayout(LayoutKind.Sequential)]
+ public struct RECT {
+ public int Left;
+ public int Top;
+ public int Right;
+ public int Bottom;
+ }
+
+ [DllImport("user32.dll", SetLastError = true)]
+ [return: MarshalAs(UnmanagedType.Bool)]
+ public static extern bool GetCursorInfo(ref CURSORINFO pci);
+
+ [DllImport("user32.dll", SetLastError = true)]
+ [return: MarshalAs(UnmanagedType.Bool)]
+ public static extern bool GetWindowRect(IntPtr hWnd, out RECT lpRect);
+
+ [DllImport("user32.dll", SetLastError = true)]
+ [return: MarshalAs(UnmanagedType.Bool)]
+ public static extern bool IsWindow(IntPtr hWnd);
+
+ [DllImport("user32.dll", SetLastError = true)]
+ public static extern IntPtr CopyIcon(IntPtr hIcon);
+
+ [DllImport("user32.dll", SetLastError = true)]
+ [return: MarshalAs(UnmanagedType.Bool)]
+ public static extern bool DestroyIcon(IntPtr hIcon);
+
+ [DllImport("user32.dll", SetLastError = true)]
+ [return: MarshalAs(UnmanagedType.Bool)]
+ public static extern bool GetIconInfo(IntPtr hIcon, out ICONINFO piconinfo);
+
+ [DllImport("gdi32.dll", SetLastError = true)]
+ [return: MarshalAs(UnmanagedType.Bool)]
+ public static extern bool DeleteObject(IntPtr hObject);
+}
+"@
+
+Add-Type -TypeDefinition $source
+
+function Write-JsonLine($payload) {
+ [Console]::Out.WriteLine(($payload | ConvertTo-Json -Compress -Depth 6))
+}
+
+function Get-TargetBounds() {
+ if ([string]::IsNullOrWhiteSpace($targetWindowHandle)) {
+ return $null
+ }
+
+ try {
+ $handleValue = [int64]::Parse($targetWindowHandle)
+ $windowHandle = [IntPtr]::new($handleValue)
+ if (-not [OpenScreenCursorInterop]::IsWindow($windowHandle)) {
+ return $null
+ }
+
+ $rect = New-Object OpenScreenCursorInterop+RECT
+ if (-not [OpenScreenCursorInterop]::GetWindowRect($windowHandle, [ref]$rect)) {
+ return $null
+ }
+
+ $width = $rect.Right - $rect.Left
+ $height = $rect.Bottom - $rect.Top
+ if ($width -le 0 -or $height -le 0) {
+ return $null
+ }
+
+ return @{
+ x = $rect.Left
+ y = $rect.Top
+ width = $width
+ height = $height
+ }
+ }
+ catch {
+ return $null
+ }
+}
+
+function Get-CursorAsset($cursorHandle, $cursorId) {
+ $copiedHandle = [OpenScreenCursorInterop]::CopyIcon($cursorHandle)
+ if ($copiedHandle -eq [IntPtr]::Zero) {
+ return $null
+ }
+
+ $iconInfo = New-Object OpenScreenCursorInterop+ICONINFO
+ $hasIconInfo = [OpenScreenCursorInterop]::GetIconInfo($copiedHandle, [ref]$iconInfo)
+
+ try {
+ $icon = [System.Drawing.Icon]::FromHandle($copiedHandle)
+ $bitmap = New-Object System.Drawing.Bitmap $icon.Width, $icon.Height, ([System.Drawing.Imaging.PixelFormat]::Format32bppArgb)
+ $graphics = [System.Drawing.Graphics]::FromImage($bitmap)
+ $memoryStream = New-Object System.IO.MemoryStream
+
+ try {
+ $graphics.Clear([System.Drawing.Color]::Transparent)
+ $graphics.DrawIcon($icon, 0, 0)
+ $bitmap.Save($memoryStream, [System.Drawing.Imaging.ImageFormat]::Png)
+ $base64 = [System.Convert]::ToBase64String($memoryStream.ToArray())
+
+ return @{
+ id = $cursorId
+ imageDataUrl = "data:image/png;base64,$base64"
+ width = $bitmap.Width
+ height = $bitmap.Height
+ hotspotX = if ($hasIconInfo) { $iconInfo.xHotspot } else { 0 }
+ hotspotY = if ($hasIconInfo) { $iconInfo.yHotspot } else { 0 }
+ }
+ }
+ finally {
+ $memoryStream.Dispose()
+ $graphics.Dispose()
+ $bitmap.Dispose()
+ $icon.Dispose()
+ }
+ }
+ finally {
+ if ($hasIconInfo) {
+ if ($iconInfo.hbmMask -ne [IntPtr]::Zero) {
+ [OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmMask) | Out-Null
+ }
+ if ($iconInfo.hbmColor -ne [IntPtr]::Zero) {
+ [OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmColor) | Out-Null
+ }
+ }
+ [OpenScreenCursorInterop]::DestroyIcon($copiedHandle) | Out-Null
+ }
+}
+
+Write-JsonLine @{ type = 'ready'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() }
+
+$lastCursorId = $null
+while ($true) {
+ $cursorInfo = New-Object OpenScreenCursorInterop+CURSORINFO
+ $cursorInfo.cbSize = [Runtime.InteropServices.Marshal]::SizeOf([type][OpenScreenCursorInterop+CURSORINFO])
+
+ if (-not [OpenScreenCursorInterop]::GetCursorInfo([ref]$cursorInfo)) {
+ Write-JsonLine @{ type = 'error'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds(); message = 'GetCursorInfo failed' }
+ Start-Sleep -Milliseconds ${sampleIntervalMs}
+ continue
+ }
+
+ $visible = ($cursorInfo.flags -band 1) -ne 0
+ $cursorId = if ($cursorInfo.hCursor -eq [IntPtr]::Zero) { $null } else { ('0x{0:X}' -f $cursorInfo.hCursor.ToInt64()) }
+ $asset = $null
+
+ if ($visible -and $cursorId -and $cursorId -ne $lastCursorId) {
+ $asset = Get-CursorAsset -cursorHandle $cursorInfo.hCursor -cursorId $cursorId
+ $lastCursorId = $cursorId
+ }
+
+ Write-JsonLine @{
+ type = 'sample'
+ timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds()
+ x = $cursorInfo.ptScreenPos.X
+ y = $cursorInfo.ptScreenPos.Y
+ visible = $visible
+ handle = $cursorId
+ bounds = Get-TargetBounds
+ asset = $asset
+ }
+
+ Start-Sleep -Milliseconds ${sampleIntervalMs}
+}
+`;
+
+ return Buffer.from(script, "utf16le").toString("base64");
+}
diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts
index a0540ed..d5e43d7 100644
--- a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts
+++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts
@@ -1,206 +1,23 @@
import { type ChildProcessByStdio, spawn } from "node:child_process";
import type { Readable } from "node:stream";
-import { type Rectangle, screen } from "electron";
+import { screen } from "electron";
import type {
CursorRecordingData,
CursorRecordingSample,
NativeCursorAsset,
} from "../../../../src/native/contracts";
import type { CursorRecordingSession } from "./session";
+import { buildPowerShellCommand, parseWindowHandleFromSourceId } from "./windowsNativeRecordingSession.script";
+import type {
+ WindowsCursorEvent,
+ WindowsNativeRecordingSessionOptions,
+} from "./windowsNativeRecordingSession.types";
-interface WindowsCursorSampleEvent {
- type: "sample";
- timestampMs: number;
- x: number;
- y: number;
- visible: boolean;
- handle: string | null;
- asset?: WindowsCursorAssetPayload;
-}
+const READY_TIMEOUT_MS = 5_000;
-interface WindowsCursorReadyEvent {
- type: "ready";
- timestampMs: number;
-}
-
-interface WindowsCursorErrorEvent {
- type: "error";
- timestampMs: number;
- message: string;
-}
-
-interface WindowsCursorAssetPayload {
- id: string;
- imageDataUrl: string;
- width: number;
- height: number;
- hotspotX: number;
- hotspotY: number;
-}
-
-type WindowsCursorEvent =
- | WindowsCursorSampleEvent
- | WindowsCursorReadyEvent
- | WindowsCursorErrorEvent;
-
-interface WindowsNativeRecordingSessionOptions {
- getDisplayBounds: () => Rectangle | null;
- maxSamples: number;
- sampleIntervalMs: number;
-}
-
-function clamp(value: number, min: number, max: number) {
- return Math.min(max, Math.max(min, value));
-}
-
-function buildPowerShellCommand(sampleIntervalMs: number) {
- const script = String.raw`
-$ErrorActionPreference = 'Stop'
-Add-Type -AssemblyName System.Drawing
-
-$source = @"
-using System;
-using System.Runtime.InteropServices;
-
-public static class OpenScreenCursorInterop {
- [StructLayout(LayoutKind.Sequential)]
- public struct POINT {
- public int X;
- public int Y;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct CURSORINFO {
- public int cbSize;
- public int flags;
- public IntPtr hCursor;
- public POINT ptScreenPos;
- }
-
- [StructLayout(LayoutKind.Sequential)]
- public struct ICONINFO {
- [MarshalAs(UnmanagedType.Bool)]
- public bool fIcon;
- public int xHotspot;
- public int yHotspot;
- public IntPtr hbmMask;
- public IntPtr hbmColor;
- }
-
- [DllImport("user32.dll", SetLastError = true)]
- [return: MarshalAs(UnmanagedType.Bool)]
- public static extern bool GetCursorInfo(ref CURSORINFO pci);
-
- [DllImport("user32.dll", SetLastError = true)]
- public static extern IntPtr CopyIcon(IntPtr hIcon);
-
- [DllImport("user32.dll", SetLastError = true)]
- [return: MarshalAs(UnmanagedType.Bool)]
- public static extern bool DestroyIcon(IntPtr hIcon);
-
- [DllImport("user32.dll", SetLastError = true)]
- [return: MarshalAs(UnmanagedType.Bool)]
- public static extern bool GetIconInfo(IntPtr hIcon, out ICONINFO piconinfo);
-
- [DllImport("gdi32.dll", SetLastError = true)]
- [return: MarshalAs(UnmanagedType.Bool)]
- public static extern bool DeleteObject(IntPtr hObject);
-}
-"@
-
-Add-Type -TypeDefinition $source
-
-function Write-JsonLine($payload) {
- [Console]::Out.WriteLine(($payload | ConvertTo-Json -Compress -Depth 6))
-}
-
-function Get-CursorAsset($cursorHandle, $cursorId) {
- $copiedHandle = [OpenScreenCursorInterop]::CopyIcon($cursorHandle)
- if ($copiedHandle -eq [IntPtr]::Zero) {
- return $null
- }
-
- $iconInfo = New-Object OpenScreenCursorInterop+ICONINFO
- $hasIconInfo = [OpenScreenCursorInterop]::GetIconInfo($copiedHandle, [ref]$iconInfo)
-
- try {
- $icon = [System.Drawing.Icon]::FromHandle($copiedHandle)
- $bitmap = New-Object System.Drawing.Bitmap $icon.Width, $icon.Height, ([System.Drawing.Imaging.PixelFormat]::Format32bppArgb)
- $graphics = [System.Drawing.Graphics]::FromImage($bitmap)
- $memoryStream = New-Object System.IO.MemoryStream
-
- try {
- $graphics.Clear([System.Drawing.Color]::Transparent)
- $graphics.DrawIcon($icon, 0, 0)
- $bitmap.Save($memoryStream, [System.Drawing.Imaging.ImageFormat]::Png)
- $base64 = [System.Convert]::ToBase64String($memoryStream.ToArray())
-
- return @{
- id = $cursorId
- imageDataUrl = "data:image/png;base64,$base64"
- width = $bitmap.Width
- height = $bitmap.Height
- hotspotX = if ($hasIconInfo) { $iconInfo.xHotspot } else { 0 }
- hotspotY = if ($hasIconInfo) { $iconInfo.yHotspot } else { 0 }
- }
- }
- finally {
- $memoryStream.Dispose()
- $graphics.Dispose()
- $bitmap.Dispose()
- $icon.Dispose()
- }
- }
- finally {
- if ($hasIconInfo) {
- if ($iconInfo.hbmMask -ne [IntPtr]::Zero) {
- [OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmMask) | Out-Null
- }
- if ($iconInfo.hbmColor -ne [IntPtr]::Zero) {
- [OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmColor) | Out-Null
- }
- }
- [OpenScreenCursorInterop]::DestroyIcon($copiedHandle) | Out-Null
- }
-}
-
-Write-JsonLine @{ type = 'ready'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() }
-
-$lastCursorId = $null
-while ($true) {
- $cursorInfo = New-Object OpenScreenCursorInterop+CURSORINFO
- $cursorInfo.cbSize = [Runtime.InteropServices.Marshal]::SizeOf([type][OpenScreenCursorInterop+CURSORINFO])
-
- if (-not [OpenScreenCursorInterop]::GetCursorInfo([ref]$cursorInfo)) {
- Write-JsonLine @{ type = 'error'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds(); message = 'GetCursorInfo failed' }
- Start-Sleep -Milliseconds ${sampleIntervalMs}
- continue
- }
-
- $visible = ($cursorInfo.flags -band 1) -ne 0
- $cursorId = if ($cursorInfo.hCursor -eq [IntPtr]::Zero) { $null } else { ('0x{0:X}' -f $cursorInfo.hCursor.ToInt64()) }
- $asset = $null
-
- if ($visible -and $cursorId -and $cursorId -ne $lastCursorId) {
- $asset = Get-CursorAsset -cursorHandle $cursorInfo.hCursor -cursorId $cursorId
- $lastCursorId = $cursorId
- }
-
- Write-JsonLine @{
- type = 'sample'
- timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds()
- x = $cursorInfo.ptScreenPos.X
- y = $cursorInfo.ptScreenPos.Y
- visible = $visible
- handle = $cursorId
- asset = $asset
- }
-
- Start-Sleep -Milliseconds ${sampleIntervalMs}
-}
-`;
-
- return Buffer.from(script, "utf16le").toString("base64");
+interface NormalizedSample {
+ sample: CursorRecordingSample;
+ withinBounds: boolean;
}
export class WindowsNativeRecordingSession implements CursorRecordingSession {
@@ -209,6 +26,11 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession {
private process: ChildProcessByStdio | null = null;
private lineBuffer = "";
private startTimeMs = 0;
+ private readyResolve: (() => void) | null = null;
+ private readyReject: ((error: Error) => void) | null = null;
+ private readyTimer: NodeJS.Timeout | null = null;
+ private sampleCount = 0;
+ private outOfBoundsSampleCount = 0;
constructor(private readonly options: WindowsNativeRecordingSessionOptions) {}
@@ -217,8 +39,13 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession {
this.samples = [];
this.lineBuffer = "";
this.startTimeMs = Date.now();
+ this.sampleCount = 0;
+ this.outOfBoundsSampleCount = 0;
- const encodedCommand = buildPowerShellCommand(this.options.sampleIntervalMs);
+ const encodedCommand = buildPowerShellCommand(
+ this.options.sampleIntervalMs,
+ parseWindowHandleFromSourceId(this.options.sourceId),
+ );
const child = spawn(
"powershell.exe",
[
@@ -237,24 +64,58 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession {
);
this.process = child;
+ this.logDiagnostic("spawn", {
+ pid: child.pid ?? null,
+ sampleIntervalMs: this.options.sampleIntervalMs,
+ sourceId: this.options.sourceId ?? null,
+ windowHandle: parseWindowHandleFromSourceId(this.options.sourceId),
+ });
+
child.stdout.setEncoding("utf8");
child.stdout.on("data", (chunk: string) => {
this.handleStdoutChunk(chunk);
});
child.stderr.setEncoding("utf8");
child.stderr.on("data", (chunk: string) => {
- console.error("[cursor-native]", chunk.trim());
+ const message = chunk.trim();
+ if (message) {
+ this.logDiagnostic("stderr", { message });
+ }
+ console.error("[cursor-native]", message);
});
+ child.once("exit", (code, signal) => {
+ this.logDiagnostic("exit", {
+ code,
+ signal,
+ sampleCount: this.sampleCount,
+ assetCount: this.assets.size,
+ outOfBoundsSampleCount: this.outOfBoundsSampleCount,
+ });
+ this.rejectReady(new Error(`Windows cursor helper exited before ready (code=${code}, signal=${signal})`));
+ });
+ child.once("error", (error) => {
+ this.logDiagnostic("process-error", { message: error.message });
+ this.rejectReady(error);
+ });
+
+ await this.waitUntilReady();
}
async stop(): Promise {
const child = this.process;
this.process = null;
+ this.clearReadyState();
if (child && !child.killed) {
child.kill();
}
+ this.logDiagnostic("stop", {
+ sampleCount: this.sampleCount,
+ assetCount: this.assets.size,
+ outOfBoundsSampleCount: this.outOfBoundsSampleCount,
+ });
+
return {
version: 2,
provider: this.assets.size > 0 ? "native" : "none",
@@ -285,11 +146,14 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession {
private handleEvent(payload: WindowsCursorEvent) {
if (payload.type === "error") {
+ this.logDiagnostic("helper-error", { message: payload.message });
console.error("Windows cursor helper error:", payload.message);
return;
}
if (payload.type === "ready") {
+ this.logDiagnostic("ready", { timestampMs: payload.timestampMs });
+ this.resolveReady();
return;
}
@@ -305,22 +169,100 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession {
hotspotY: payload.asset.hotspotY,
scaleFactor: assetDisplay.scaleFactor,
});
+ this.logDiagnostic("asset", {
+ id: payload.asset.id,
+ width: payload.asset.width,
+ height: payload.asset.height,
+ hotspotX: payload.asset.hotspotX,
+ hotspotY: payload.asset.hotspotY,
+ scaleFactor: assetDisplay.scaleFactor,
+ });
}
- const bounds = this.options.getDisplayBounds() ?? screen.getPrimaryDisplay().bounds;
- const width = Math.max(1, bounds.width);
- const height = Math.max(1, bounds.height);
+ const normalized = this.normalizeSample(payload);
+ this.sampleCount += 1;
+ if (!normalized.withinBounds) {
+ this.outOfBoundsSampleCount += 1;
+ }
- this.samples.push({
- timeMs: Math.max(0, payload.timestampMs - this.startTimeMs),
- cx: clamp((payload.x - bounds.x) / width, 0, 1),
- cy: clamp((payload.y - bounds.y) / height, 0, 1),
- assetId: payload.handle,
- visible: payload.visible,
- });
+ this.samples.push(normalized.sample);
if (this.samples.length > this.options.maxSamples) {
this.samples.shift();
}
}
+
+ private normalizeSample(payload: Extract): NormalizedSample {
+ const bounds = payload.bounds ?? this.options.getDisplayBounds() ?? screen.getPrimaryDisplay().bounds;
+ const width = Math.max(1, bounds.width);
+ const height = Math.max(1, bounds.height);
+ const normalizedX = (payload.x - bounds.x) / width;
+ const normalizedY = (payload.y - bounds.y) / height;
+ const withinBounds = normalizedX >= 0 && normalizedX <= 1 && normalizedY >= 0 && normalizedY <= 1;
+
+ if (this.sampleCount === 0 || (!withinBounds && this.outOfBoundsSampleCount === 0)) {
+ this.logDiagnostic("sample", {
+ rawX: payload.x,
+ rawY: payload.y,
+ normalizedX,
+ normalizedY,
+ visible: payload.visible,
+ withinBounds,
+ bounds,
+ handle: payload.handle,
+ });
+ }
+
+ return {
+ withinBounds,
+ sample: {
+ timeMs: Math.max(0, payload.timestampMs - this.startTimeMs),
+ cx: normalizedX,
+ cy: normalizedY,
+ assetId: payload.handle,
+ visible: payload.visible && withinBounds,
+ },
+ };
+ }
+
+ private waitUntilReady() {
+ return new Promise((resolve, reject) => {
+ this.readyResolve = resolve;
+ this.readyReject = reject;
+ this.readyTimer = setTimeout(() => {
+ this.rejectReady(new Error("Timed out waiting for Windows cursor helper readiness"));
+ }, READY_TIMEOUT_MS);
+ });
+ }
+
+ private resolveReady() {
+ const resolve = this.readyResolve;
+ this.clearReadyState();
+ resolve?.();
+ }
+
+ private rejectReady(error: Error) {
+ const reject = this.readyReject;
+ this.clearReadyState();
+ reject?.(error);
+ }
+
+ private clearReadyState() {
+ if (this.readyTimer) {
+ clearTimeout(this.readyTimer);
+ this.readyTimer = null;
+ }
+ this.readyResolve = null;
+ this.readyReject = null;
+ }
+
+ private logDiagnostic(event: string, data: Record) {
+ console.info(
+ "[cursor-native][win32]",
+ JSON.stringify({
+ event,
+ ...data,
+ }),
+ );
+ }
}
diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts
new file mode 100644
index 0000000..6efd59d
--- /dev/null
+++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts
@@ -0,0 +1,49 @@
+import type { Rectangle } from "electron";
+
+export interface WindowsCursorSampleEvent {
+ type: "sample";
+ timestampMs: number;
+ x: number;
+ y: number;
+ visible: boolean;
+ handle: string | null;
+ bounds?: {
+ x: number;
+ y: number;
+ width: number;
+ height: number;
+ } | null;
+ asset?: WindowsCursorAssetPayload;
+}
+
+export interface WindowsCursorReadyEvent {
+ type: "ready";
+ timestampMs: number;
+}
+
+export interface WindowsCursorErrorEvent {
+ type: "error";
+ timestampMs: number;
+ message: string;
+}
+
+export interface WindowsCursorAssetPayload {
+ id: string;
+ imageDataUrl: string;
+ width: number;
+ height: number;
+ hotspotX: number;
+ hotspotY: number;
+}
+
+export type WindowsCursorEvent =
+ | WindowsCursorSampleEvent
+ | WindowsCursorReadyEvent
+ | WindowsCursorErrorEvent;
+
+export interface WindowsNativeRecordingSessionOptions {
+ getDisplayBounds: () => Rectangle | null;
+ maxSamples: number;
+ sampleIntervalMs: number;
+ sourceId?: string | null;
+}
diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx
index 260f4cb..57f79b3 100644
--- a/src/components/launch/LaunchWindow.tsx
+++ b/src/components/launch/LaunchWindow.tsx
@@ -259,6 +259,8 @@ export function LaunchWindow() {
const [selectedSource, setSelectedSource] = useState("Screen");
const [hasSelectedSource, setHasSelectedSource] = useState(false);
+ const [, setHudPointerDownCount] = useState(0);
+ const [, setRecordPointerDownCount] = useState(0);
useEffect(() => {
const checkSelectedSource = async () => {
@@ -541,6 +543,9 @@ export function LaunchWindow() {
onClick={toggleMicrophone}
disabled={recording}
title={microphoneEnabled ? t("audio.disableMicrophone") : t("audio.enableMicrophone")}
+ onPointerDown={() => {
+ setRecordPointerDownCount((count) => count + 1);
+ }}
>
{microphoneEnabled
? getIcon("micOn", "text-green-400")
diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx
index 1ed8938..842767a 100644
--- a/src/components/video-editor/VideoEditor.tsx
+++ b/src/components/video-editor/VideoEditor.tsx
@@ -1477,6 +1477,7 @@ export default function VideoEditor() {
videoPadding: padding,
cropRegion,
cursorRecordingData,
+ cursorScale: showCursor ? cursorSize : 0,
annotationRegions,
webcamLayoutPreset,
webcamMaskShape,
@@ -1619,6 +1620,7 @@ export default function VideoEditor() {
padding,
cropRegion,
cursorRecordingData,
+ cursorScale: showCursor ? cursorSize : 0,
annotationRegions,
webcamLayoutPreset,
webcamMaskShape,
diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx
index 12b1c25..32db9d2 100644
--- a/src/components/video-editor/VideoPlayback.tsx
+++ b/src/components/video-editor/VideoPlayback.tsx
@@ -29,8 +29,9 @@ import { classifyWallpaper, DEFAULT_WALLPAPER, resolveImageWallpaperUrl } from "
import { getCssClipPath } from "@/lib/webcamMaskShapes";
import {
getNativeCursorDisplayMetrics,
+ hasNativeCursorRecordingData,
projectNativeCursorToStage,
- resolveActiveNativeCursorFrame,
+ resolveInterpolatedNativeCursorFrame,
} from "@/lib/cursor/nativeCursor";
import type { CursorRecordingData } from "@/native/contracts";
import {
@@ -635,6 +636,18 @@ const VideoPlayback = forwardRef(
showCursorRef.current = showCursor;
}, [showCursor]);
+ useEffect(() => {
+ hasNativeCursorRecordingRef.current = hasNativeCursorRecording;
+ }, [hasNativeCursorRecording]);
+
+ useEffect(() => {
+ cursorRecordingDataRef.current = cursorRecordingData;
+ }, [cursorRecordingData]);
+
+ useEffect(() => {
+ cropRegionRef.current = cropRegion;
+ }, [cropRegion]);
+
useEffect(() => {
cursorSizeRef.current = cursorSize;
}, [cursorSize]);
@@ -1273,16 +1286,69 @@ const VideoPlayback = forwardRef(
// Update cursor overlay
const cursorOverlay = cursorOverlayRef.current;
if (cursorOverlay) {
- const timeMs = currentTimeRef.current;
+ const timeMs = currentTimeRef.current; // already in ms
cursorOverlay.update(
cursorTelemetryRef.current,
timeMs,
baseMaskRef.current,
- showCursorRef.current,
+ showCursorRef.current && !hasNativeCursorRecordingRef.current,
!isPlayingRef.current || isSeekingRef.current,
);
}
+ // Update native cursor image position at ticker rate (60fps)
+ const nativeCursorImg = nativeCursorImgRef.current;
+ if (nativeCursorImg) {
+ const cameraContainerRc = cameraContainerRef.current;
+ const videoContainerRc = videoContainerRef.current;
+ if (
+ hasNativeCursorRecordingRef.current &&
+ showCursorRef.current &&
+ cameraContainerRc &&
+ videoContainerRc
+ ) {
+ const timeMs = currentTimeRef.current; // already in ms
+ const frame = resolveInterpolatedNativeCursorFrame(
+ cursorRecordingDataRef.current,
+ timeMs,
+ );
+ if (frame) {
+ const projectedPoint = projectNativeCursorToStage({
+ cameraContainer: cameraContainerRc,
+ cropRegion: cropRegionRef.current ?? { x: 0, y: 0, width: 1, height: 1 },
+ maskRect: baseMaskRef.current,
+ videoContainerPosition: {
+ x: videoContainerRc.x,
+ y: videoContainerRc.y,
+ },
+ sample: frame.sample,
+ });
+ if (projectedPoint) {
+ const metrics = getNativeCursorDisplayMetrics(
+ frame.asset,
+ window.devicePixelRatio || 1,
+ );
+ const scale = Math.max(0, cursorSizeRef.current);
+ if (nativeCursorImg.dataset.cursorId !== frame.asset.id) {
+ nativeCursorImg.src = frame.asset.imageDataUrl;
+ nativeCursorImg.dataset.cursorId = frame.asset.id;
+ }
+ nativeCursorImg.style.left = `${projectedPoint.x - metrics.hotspotX * scale}px`;
+ nativeCursorImg.style.top = `${projectedPoint.y - metrics.hotspotY * scale}px`;
+ nativeCursorImg.style.width = `${metrics.width * scale}px`;
+ nativeCursorImg.style.height = `${metrics.height * scale}px`;
+ nativeCursorImg.style.display = "block";
+ } else {
+ nativeCursorImg.style.display = "none";
+ }
+ } else {
+ nativeCursorImg.style.display = "none";
+ }
+ } else {
+ nativeCursorImg.style.display = "none";
+ }
+ }
+
const composite3D = composite3DRef.current;
const outerWrapper = outerWrapperRef.current;
if (composite3D && outerWrapper) {
@@ -1571,17 +1637,14 @@ const VideoPlayback = forwardRef(
className="absolute rounded-md border border-[#34B27B]/80 bg-[#34B27B]/20 shadow-[0_0_0_1px_rgba(52,178,123,0.35)]"
style={{ display: "none", pointerEvents: "none" }}
/>
- {activeNativeCursor && nativeCursorStyle ? (
+ {hasNativeCursorRecording ? (
Date: Sun, 3 May 2026 16:53:04 +0200
Subject: [PATCH 07/92] fix: restore cursor pipeline build after rebase
---
.gitignore | 3 ++
electron/ipc/handlers.ts | 44 ++++++++++++++++++-
electron/preload.ts | 1 +
src/components/video-editor/SettingsPanel.tsx | 2 -
src/components/video-editor/VideoEditor.tsx | 11 +++--
src/components/video-editor/VideoPlayback.tsx | 4 +-
src/hooks/useScreenRecorder.ts | 3 +-
src/lib/exporter/frameRenderer.ts | 2 +-
8 files changed, 60 insertions(+), 10 deletions(-)
diff --git a/.gitignore b/.gitignore
index 771c4bd..494da30 100644
--- a/.gitignore
+++ b/.gitignore
@@ -43,3 +43,6 @@ __screenshots__/
result
result-*
.direnv/
+
+#kilocode
+.kilo/
\ No newline at end of file
diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts
index d50cab3..3a4dd3b 100644
--- a/electron/ipc/handlers.ts
+++ b/electron/ipc/handlers.ts
@@ -451,9 +451,12 @@ async function storeRecordedSessionFiles(payload: StoreRecordedSessionInput) {
export function registerIpcHandlers(
createEditorWindow: () => void,
createSourceSelectorWindow: () => BrowserWindow,
+ _createCountdownOverlayWindow: () => BrowserWindow,
getMainWindow: () => BrowserWindow | null,
getSourceSelectorWindow: () => BrowserWindow | null,
+ _getCountdownOverlayWindow?: () => BrowserWindow | null,
onRecordingStateChange?: (recording: boolean, sourceName: string) => void,
+ _switchToHud?: () => void,
) {
ipcMain.handle("get-sources", async (_, opts) => {
const sources = await desktopCapturer.getSources(opts);
@@ -472,7 +475,7 @@ export function registerIpcHandlers(
// Reuse the exact source object returned during enumeration to avoid
// Windows window-source id mismatches across separate getSources() calls.
selectedDesktopSource =
- typeof source.id === "string" ? lastEnumeratedSources.get(source.id) ?? null : null;
+ typeof source.id === "string" ? (lastEnumeratedSources.get(source.id) ?? null) : null;
if (!selectedDesktopSource && typeof source.id === "string") {
try {
@@ -602,6 +605,45 @@ export function registerIpcHandlers(
}
});
+ async function storeRecordedSessionFiles(payload: StoreRecordedSessionInput) {
+ const createdAt =
+ typeof payload.createdAt === "number" && Number.isFinite(payload.createdAt)
+ ? payload.createdAt
+ : Date.now();
+ const screenVideoPath = path.join(RECORDINGS_DIR, payload.screen.fileName);
+ await fs.writeFile(screenVideoPath, Buffer.from(payload.screen.videoData));
+
+ let webcamVideoPath: string | undefined;
+ if (payload.webcam) {
+ webcamVideoPath = path.join(RECORDINGS_DIR, payload.webcam.fileName);
+ await fs.writeFile(webcamVideoPath, Buffer.from(payload.webcam.videoData));
+ }
+
+ const session: RecordingSession = webcamVideoPath
+ ? { screenVideoPath, webcamVideoPath, createdAt }
+ : { screenVideoPath, createdAt };
+ setCurrentRecordingSessionState(session);
+ currentVideoPath = screenVideoPath;
+ currentProjectPath = null;
+
+ const telemetryPath = `${screenVideoPath}.cursor.json`;
+ if (pendingCursorRecordingData && pendingCursorRecordingData.samples.length > 0) {
+ await fs.writeFile(
+ telemetryPath,
+ JSON.stringify(pendingCursorRecordingData, null, 2),
+ "utf-8",
+ );
+ }
+ pendingCursorRecordingData = null;
+
+ return {
+ success: true,
+ path: screenVideoPath,
+ session,
+ message: "Recording session stored successfully",
+ };
+ }
+
ipcMain.handle("store-recorded-video", async (_, videoData: ArrayBuffer, fileName: string) => {
try {
return await storeRecordedSessionFiles({
diff --git a/electron/preload.ts b/electron/preload.ts
index 5ff110d..5345aab 100644
--- a/electron/preload.ts
+++ b/electron/preload.ts
@@ -1,4 +1,5 @@
import { contextBridge, ipcRenderer } from "electron";
+import type { RecordingSession, StoreRecordedSessionInput } from "../src/lib/recordingSession";
import { NATIVE_BRIDGE_CHANNEL, type NativeBridgeRequest } from "../src/native/contracts";
// Asset base URL is passed from the main process via webPreferences.additionalArguments
diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx
index 110f025..ad74239 100644
--- a/src/components/video-editor/SettingsPanel.tsx
+++ b/src/components/video-editor/SettingsPanel.tsx
@@ -435,8 +435,6 @@ export function SettingsPanel({
const [selectedColor, setSelectedColor] = useState("#ADADAD");
const [gradient, setGradient] = useState(GRADIENTS[0]);
- const [showCropModal, setShowCropModal] = useState(false);
- const cropSnapshotRef = useRef(null);
const [cropAspectLocked, setCropAspectLocked] = useState(false);
const [cropAspectRatio, setCropAspectRatio] = useState("");
const isPortraitCanvas = isPortraitAspectRatio(aspectRatio);
diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx
index 842767a..5394eee 100644
--- a/src/components/video-editor/VideoEditor.tsx
+++ b/src/components/video-editor/VideoEditor.tsx
@@ -38,12 +38,12 @@ import {
saveUserPreferences,
} from "@/lib/userPreferences";
import { BackgroundLoadError } from "@/lib/wallpaper";
+import { nativeBridgeClient, useCursorRecordingData, useCursorTelemetry } from "@/native";
import {
getAspectRatioValue,
getNativeAspectRatioValue,
isPortraitAspectRatio,
} from "@/utils/aspectRatioUtils";
-import { nativeBridgeClient, useCursorRecordingData, useCursorTelemetry } from "@/native";
import { ExportDialog } from "./ExportDialog";
import PlaybackControls from "./PlaybackControls";
import {
@@ -216,7 +216,12 @@ export default function VideoEditor() {
}
const project = candidate;
- const sourcePath = project.videoPath;
+ const projectMedia = resolveProjectMedia(project);
+ if (!projectMedia) {
+ return false;
+ }
+ const sourcePath = projectMedia.screenVideoPath;
+ const webcamSourcePath = projectMedia.webcamVideoPath ?? null;
const normalizedEditor = normalizeProjectEditor(project.editor);
const inferredDurationMs = Math.max(
0,
@@ -405,7 +410,7 @@ export default function VideoEditor() {
setVideoPath(toFileUrl(result.path));
setCurrentProjectPath(null);
setLastSavedSnapshot(
- createProjectSnapshot({ screenVideoPath: sourcePath }, INITIAL_EDITOR_STATE),
+ createProjectSnapshot({ screenVideoPath: result.path }, INITIAL_EDITOR_STATE),
);
} else {
setError("No video to load. Please record or select a video.");
diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx
index 32db9d2..d7b3836 100644
--- a/src/components/video-editor/VideoPlayback.tsx
+++ b/src/components/video-editor/VideoPlayback.tsx
@@ -25,14 +25,14 @@ import {
type WebcamLayoutPreset,
type WebcamSizePreset,
} from "@/lib/compositeLayout";
-import { classifyWallpaper, DEFAULT_WALLPAPER, resolveImageWallpaperUrl } from "@/lib/wallpaper";
-import { getCssClipPath } from "@/lib/webcamMaskShapes";
import {
getNativeCursorDisplayMetrics,
hasNativeCursorRecordingData,
projectNativeCursorToStage,
resolveInterpolatedNativeCursorFrame,
} from "@/lib/cursor/nativeCursor";
+import { classifyWallpaper, DEFAULT_WALLPAPER, resolveImageWallpaperUrl } from "@/lib/wallpaper";
+import { getCssClipPath } from "@/lib/webcamMaskShapes";
import type { CursorRecordingData } from "@/native/contracts";
import {
type AspectRatio,
diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts
index 67b0d96..8aa673e 100644
--- a/src/hooks/useScreenRecorder.ts
+++ b/src/hooks/useScreenRecorder.ts
@@ -1,7 +1,8 @@
import { fixWebmDuration } from "@fix-webm-duration/fix";
import { useCallback, useEffect, useRef, useState } from "react";
import { toast } from "sonner";
-import { nativeBridgeClient } from "@/native";
+import { useScopedT } from "@/contexts/I18nContext";
+import { requestCameraAccess } from "@/lib/requestCameraAccess";
const TARGET_FRAME_RATE = 60;
const MIN_FRAME_RATE = 30;
diff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.ts
index 6f605b0..c43908f 100644
--- a/src/lib/exporter/frameRenderer.ts
+++ b/src/lib/exporter/frameRenderer.ts
@@ -614,7 +614,7 @@ export class FrameRenderer {
return image;
}
- private updateLayout(): void {
+ private updateLayout(webcamFrame?: VideoFrame | null): void {
if (!this.app || !this.videoSprite || !this.maskGraphics || !this.videoContainer) return;
const { width, height } = this.config;
From bb0dec73443064856b366aa499dd1ef4ed78aab8 Mon Sep 17 00:00:00 2001
From: EtienneLescot
Date: Tue, 5 May 2026 10:16:01 +0200
Subject: [PATCH 08/92] feat: add windows cursor preview diagnostics
---
README.md | 3 +
docs/testing/windows-native-cursor.md | 85 ++
electron/ipc/handlers.ts | 16 +
.../windowsNativeRecordingSession.script.ts | 41 +-
.../windowsNativeRecordingSession.ts | 21 +-
.../windowsNativeRecordingSession.types.ts | 3 +
package.json | 12 +-
scripts/capture-openscreen-preview.mjs | 262 ++++
scripts/test-windows-native-cursor.mjs | 1113 +++++++++++++++++
src/components/video-editor/VideoPlayback.tsx | 19 +-
src/lib/cursor/nativeCursor.ts | 128 ++
src/lib/exporter/frameRenderer.ts | 22 +-
src/native/contracts.ts | 17 +
13 files changed, 1713 insertions(+), 29 deletions(-)
create mode 100644 docs/testing/windows-native-cursor.md
create mode 100644 scripts/capture-openscreen-preview.mjs
create mode 100644 scripts/test-windows-native-cursor.mjs
diff --git a/README.md b/README.md
index 7009a22..0403afe 100644
--- a/README.md
+++ b/README.md
@@ -171,6 +171,9 @@ See the documentation here:
[OpenScreen Docs](https://deepwiki.com/siddharthvaddem/openscreen)
Refresh if outdated.
+Developer notes:
+- [Windows native cursor test pipeline](docs/testing/windows-native-cursor.md)
+
## Contributing
Contributions are welcome - please **include screenshots or a short video** for any UI change or new user-facing feature. If it touches what users see or do, show it. Skip only when it genuinely doesn't apply. PRs that don't follow this will be closed.
diff --git a/docs/testing/windows-native-cursor.md b/docs/testing/windows-native-cursor.md
new file mode 100644
index 0000000..1abe04e
--- /dev/null
+++ b/docs/testing/windows-native-cursor.md
@@ -0,0 +1,85 @@
+# Windows native cursor test pipeline
+
+This branch includes two Windows-focused diagnostics for fast iteration on native cursor capture and rendering. They are intentionally local developer tools: they create short videos and JSON reports so cursor changes can be inspected without doing a full manual record/edit/export cycle.
+
+## Native sampler diagnostic
+
+```powershell
+npm run test:cursor-native:win
+```
+
+This script does not launch OpenScreen. It:
+
+- starts a Windows `GetCursorInfo` sampler
+- moves the real OS pointer with `SetCursorPos`
+- captures native cursor handles, hotspots, assets, and standard `IDC_*` cursor types
+- writes normalized `CursorRecordingData`
+- generates an abstract preview video
+- generates a real-screen preview video using screenshots of the current desktop
+
+The output directory is printed in the command result, for example:
+
+```text
+C:\Users\\AppData\Local\Temp\openscreen-cursor-native-...
+```
+
+Useful files:
+
+- `report.json`: sample counts, asset counts, cursor handles, and generated artifact paths
+- `cursor-recording-data.json`: sidecar-compatible cursor data
+- `preview.webm`: abstract path/asset/hotspot preview
+- `real-capture-preview.webm`: real desktop screenshot background with reconstructed cursor overlay
+- `assets/*.png`: raw cursor bitmaps captured from Windows
+
+Environment overrides:
+
+```powershell
+$env:CURSOR_TEST_DURATION_MS = "3000"
+$env:CURSOR_TEST_SAMPLE_INTERVAL_MS = "16"
+$env:CURSOR_TEST_SCREEN_FRAME_INTERVAL_MS = "80"
+$env:CURSOR_TEST_OUTPUT_DIR = "C:\temp\openscreen-cursor-test"
+npm run test:cursor-native:win
+```
+
+## OpenScreen preview capture
+
+```powershell
+npm run capture:openscreen-preview
+```
+
+This script launches the real Electron app, injects a fixture video plus cursor sidecar data, opens the editor, captures frames from the actual OpenScreen preview UI, and encodes them into a WebM.
+
+By default it uses the latest `cursor-recording-data.json` generated by `npm run test:cursor-native:win`. To force a specific sidecar:
+
+```powershell
+$env:CURSOR_RECORDING_DATA_PATH = "C:\path\to\cursor-recording-data.json"
+npm run capture:openscreen-preview
+```
+
+Useful environment overrides:
+
+```powershell
+$env:OPENSCREEN_PREVIEW_SKIP_BUILD = "true"
+$env:OPENSCREEN_PREVIEW_FRAME_COUNT = "120"
+$env:OPENSCREEN_PREVIEW_FPS = "30"
+$env:OPENSCREEN_PREVIEW_OUTPUT_DIR = "C:\temp\openscreen-preview"
+npm run capture:openscreen-preview
+```
+
+Useful files:
+
+- `openscreen-preview.webm`: video of the real OpenScreen editor preview
+- `frames/*.png`: captured preview frames
+- `report.json`: fixture paths, source sidecar, frame count, and output path
+
+## What these tests validate
+
+Together, the scripts make it quick to inspect:
+
+- whether Windows cursor samples are visible and continuous
+- whether native hotspots stay anchored when scaling to `3x`
+- whether standard Windows cursors are recognized via `IDC_*`
+- whether high-quality SVG cursor replacements follow the native hotspot
+- whether the real OpenScreen preview renders the same cursor behavior as the diagnostic pipeline
+
+They are not a full substitute for an end-to-end manual recording pass. Before shipping cursor changes, also test a real capture session and export from the packaged app.
diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts
index 3a4dd3b..c5a1269 100644
--- a/electron/ipc/handlers.ts
+++ b/electron/ipc/handlers.ts
@@ -267,6 +267,7 @@ function normalizeCursorSample(sample: unknown): CursorRecordingSample | null {
cy: typeof point.cy === "number" && Number.isFinite(point.cy) ? point.cy : 0.5,
assetId: typeof point.assetId === "string" ? point.assetId : null,
visible: typeof point.visible === "boolean" ? point.visible : true,
+ cursorType: typeof point.cursorType === "string" ? point.cursorType : null,
};
}
@@ -305,6 +306,7 @@ function normalizeCursorAsset(asset: unknown): NativeCursorAsset | null {
typeof candidate.scaleFactor === "number" && Number.isFinite(candidate.scaleFactor)
? Math.max(0.1, candidate.scaleFactor)
: undefined,
+ cursorType: typeof candidate.cursorType === "string" ? candidate.cursorType : null,
};
}
@@ -1079,6 +1081,20 @@ export function registerIpcHandlers(
return setCurrentVideoPath(path);
});
+ ipcMain.handle("set-current-recording-session", (_, session: RecordingSession | null) => {
+ const normalizedSession = normalizeRecordingSession(session);
+ setCurrentRecordingSessionState(normalizedSession);
+ currentVideoPath = normalizedSession?.screenVideoPath ?? null;
+ currentProjectPath = null;
+ return { success: true, session: currentRecordingSession };
+ });
+
+ ipcMain.handle("get-current-recording-session", () => {
+ return currentRecordingSession
+ ? { success: true, session: currentRecordingSession }
+ : { success: false };
+ });
+
function setCurrentVideoPath(path: string): ProjectPathResult {
currentVideoPath = normalizeVideoSourcePath(path) ?? path;
currentProjectPath = null;
diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts
index b7a11cb..5607134 100644
--- a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts
+++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts
@@ -16,7 +16,7 @@ export function buildPowerShellCommand(sampleIntervalMs: number, windowHandle?:
$ErrorActionPreference = 'Stop'
Add-Type -AssemblyName System.Drawing
-$targetWindowHandle = ${windowHandle ? `'${windowHandle}'` : '$null'}
+$targetWindowHandle = ${windowHandle ? `'${windowHandle}'` : "$null"}
$source = @"
using System;
@@ -59,6 +59,9 @@ public static class OpenScreenCursorInterop {
[return: MarshalAs(UnmanagedType.Bool)]
public static extern bool GetCursorInfo(ref CURSORINFO pci);
+ [DllImport("user32.dll", SetLastError = true)]
+ public static extern IntPtr LoadCursor(IntPtr hInstance, IntPtr lpCursorName);
+
[DllImport("user32.dll", SetLastError = true)]
[return: MarshalAs(UnmanagedType.Bool)]
public static extern bool GetWindowRect(IntPtr hWnd, out RECT lpRect);
@@ -86,6 +89,37 @@ public static class OpenScreenCursorInterop {
Add-Type -TypeDefinition $source
+$standardCursors = @{
+ arrow = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32512))
+ text = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32513))
+ wait = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32514))
+ crosshair = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32515))
+ 'up-arrow' = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32516))
+ 'resize-nwse' = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32642))
+ 'resize-nesw' = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32643))
+ 'resize-ew' = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32644))
+ 'resize-ns' = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32645))
+ move = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32646))
+ 'not-allowed' = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32648))
+ pointer = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32649))
+ 'app-starting' = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32650))
+ help = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32651))
+}
+
+function Get-StandardCursorType($cursorHandle) {
+ if ($cursorHandle -eq [IntPtr]::Zero) {
+ return $null
+ }
+
+ foreach ($entry in $standardCursors.GetEnumerator()) {
+ if ($entry.Value -eq $cursorHandle) {
+ return $entry.Key
+ }
+ }
+
+ return $null
+}
+
function Write-JsonLine($payload) {
[Console]::Out.WriteLine(($payload | ConvertTo-Json -Compress -Depth 6))
}
@@ -190,10 +224,14 @@ while ($true) {
$visible = ($cursorInfo.flags -band 1) -ne 0
$cursorId = if ($cursorInfo.hCursor -eq [IntPtr]::Zero) { $null } else { ('0x{0:X}' -f $cursorInfo.hCursor.ToInt64()) }
+ $cursorType = Get-StandardCursorType $cursorInfo.hCursor
$asset = $null
if ($visible -and $cursorId -and $cursorId -ne $lastCursorId) {
$asset = Get-CursorAsset -cursorHandle $cursorInfo.hCursor -cursorId $cursorId
+ if ($asset -and $cursorType) {
+ $asset.cursorType = $cursorType
+ }
$lastCursorId = $cursorId
}
@@ -204,6 +242,7 @@ while ($true) {
y = $cursorInfo.ptScreenPos.Y
visible = $visible
handle = $cursorId
+ cursorType = $cursorType
bounds = Get-TargetBounds
asset = $asset
}
diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts
index d5e43d7..632a74d 100644
--- a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts
+++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts
@@ -7,7 +7,10 @@ import type {
NativeCursorAsset,
} from "../../../../src/native/contracts";
import type { CursorRecordingSession } from "./session";
-import { buildPowerShellCommand, parseWindowHandleFromSourceId } from "./windowsNativeRecordingSession.script";
+import {
+ buildPowerShellCommand,
+ parseWindowHandleFromSourceId,
+} from "./windowsNativeRecordingSession.script";
import type {
WindowsCursorEvent,
WindowsNativeRecordingSessionOptions,
@@ -91,7 +94,9 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession {
assetCount: this.assets.size,
outOfBoundsSampleCount: this.outOfBoundsSampleCount,
});
- this.rejectReady(new Error(`Windows cursor helper exited before ready (code=${code}, signal=${signal})`));
+ this.rejectReady(
+ new Error(`Windows cursor helper exited before ready (code=${code}, signal=${signal})`),
+ );
});
child.once("error", (error) => {
this.logDiagnostic("process-error", { message: error.message });
@@ -168,6 +173,7 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession {
hotspotX: payload.asset.hotspotX,
hotspotY: payload.asset.hotspotY,
scaleFactor: assetDisplay.scaleFactor,
+ cursorType: payload.asset.cursorType ?? payload.cursorType ?? null,
});
this.logDiagnostic("asset", {
id: payload.asset.id,
@@ -192,13 +198,17 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession {
}
}
- private normalizeSample(payload: Extract): NormalizedSample {
- const bounds = payload.bounds ?? this.options.getDisplayBounds() ?? screen.getPrimaryDisplay().bounds;
+ private normalizeSample(
+ payload: Extract,
+ ): NormalizedSample {
+ const bounds =
+ payload.bounds ?? this.options.getDisplayBounds() ?? screen.getPrimaryDisplay().bounds;
const width = Math.max(1, bounds.width);
const height = Math.max(1, bounds.height);
const normalizedX = (payload.x - bounds.x) / width;
const normalizedY = (payload.y - bounds.y) / height;
- const withinBounds = normalizedX >= 0 && normalizedX <= 1 && normalizedY >= 0 && normalizedY <= 1;
+ const withinBounds =
+ normalizedX >= 0 && normalizedX <= 1 && normalizedY >= 0 && normalizedY <= 1;
if (this.sampleCount === 0 || (!withinBounds && this.outOfBoundsSampleCount === 0)) {
this.logDiagnostic("sample", {
@@ -221,6 +231,7 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession {
cy: normalizedY,
assetId: payload.handle,
visible: payload.visible && withinBounds,
+ cursorType: payload.cursorType ?? payload.asset?.cursorType ?? null,
},
};
}
diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts
index 6efd59d..fdc4ab9 100644
--- a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts
+++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts
@@ -1,4 +1,5 @@
import type { Rectangle } from "electron";
+import type { NativeCursorType } from "../../../../src/native/contracts";
export interface WindowsCursorSampleEvent {
type: "sample";
@@ -7,6 +8,7 @@ export interface WindowsCursorSampleEvent {
y: number;
visible: boolean;
handle: string | null;
+ cursorType?: NativeCursorType | null;
bounds?: {
x: number;
y: number;
@@ -34,6 +36,7 @@ export interface WindowsCursorAssetPayload {
height: number;
hotspotX: number;
hotspotY: number;
+ cursorType?: NativeCursorType | null;
}
export type WindowsCursorEvent =
diff --git a/package.json b/package.json
index 2ccb0b3..f81d99b 100644
--- a/package.json
+++ b/package.json
@@ -19,12 +19,14 @@
"lint:fix": "biome check --write .",
"format": "biome format --write .",
"i18n:check": "node scripts/i18n-check.mjs",
- "preview": "vite preview",
- "build:mac": "tsc && vite build && electron-builder --mac",
- "build:win": "tsc && vite build && electron-builder --win --config.npmRebuild=false",
- "build:linux": "tsc && vite build && electron-builder --linux AppImage deb pacman --config.npmRebuild=false",
- "test": "vitest --run",
+ "preview": "vite preview",
+ "build:mac": "tsc && vite build && electron-builder --mac",
+ "build:win": "tsc && vite build && electron-builder --win --config.npmRebuild=false",
+ "build:linux": "tsc && vite build && electron-builder --linux AppImage deb pacman --config.npmRebuild=false",
+ "test": "vitest --run",
"test:watch": "vitest",
+ "test:cursor-native:win": "node scripts/test-windows-native-cursor.mjs",
+ "capture:openscreen-preview": "node scripts/capture-openscreen-preview.mjs",
"build-vite": "tsc && vite build",
"test:browser": "vitest --config vitest.browser.config.ts --run",
"test:browser:install": "playwright install --with-deps chromium-headless-shell",
diff --git a/scripts/capture-openscreen-preview.mjs b/scripts/capture-openscreen-preview.mjs
new file mode 100644
index 0000000..6c9b6eb
--- /dev/null
+++ b/scripts/capture-openscreen-preview.mjs
@@ -0,0 +1,262 @@
+import { spawn } from "node:child_process";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { chromium, _electron as electron } from "@playwright/test";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const ROOT = path.join(__dirname, "..");
+const MAIN_JS = path.join(ROOT, "dist-electron", "main.js");
+const TEST_VIDEO = path.join(ROOT, "tests", "fixtures", "sample.webm");
+const OUTPUT_DIR =
+ process.env.OPENSCREEN_PREVIEW_OUTPUT_DIR ??
+ path.join(os.tmpdir(), `openscreen-real-preview-${Date.now()}`);
+const FRAME_COUNT = Number(process.env.OPENSCREEN_PREVIEW_FRAME_COUNT ?? 90);
+const FPS = Number(process.env.OPENSCREEN_PREVIEW_FPS ?? 30);
+
+function findLatestCursorRecordingData() {
+ const explicit = process.env.CURSOR_RECORDING_DATA_PATH;
+ if (explicit) {
+ if (!fs.existsSync(explicit)) {
+ throw new Error(`CURSOR_RECORDING_DATA_PATH does not exist: ${explicit}`);
+ }
+ return explicit;
+ }
+
+ const tempDir = os.tmpdir();
+ const candidates = fs
+ .readdirSync(tempDir, { withFileTypes: true })
+ .filter((entry) => entry.isDirectory() && entry.name.startsWith("openscreen-cursor-native-"))
+ .map((entry) => path.join(tempDir, entry.name, "cursor-recording-data.json"))
+ .filter((candidate) => fs.existsSync(candidate))
+ .map((candidate) => ({ path: candidate, mtimeMs: fs.statSync(candidate).mtimeMs }))
+ .sort((a, b) => b.mtimeMs - a.mtimeMs);
+
+ if (!candidates[0]) {
+ throw new Error(
+ "No cursor-recording-data.json found. Run npm run test:cursor-native:win first.",
+ );
+ }
+
+ return candidates[0].path;
+}
+
+function findPlaywrightChromiumExecutable(defaultPath) {
+ if (fs.existsSync(defaultPath)) {
+ return defaultPath;
+ }
+
+ const baseDir = path.join(process.env.LOCALAPPDATA ?? "", "ms-playwright");
+ if (!baseDir || !fs.existsSync(baseDir)) {
+ return defaultPath;
+ }
+
+ const candidates = fs
+ .readdirSync(baseDir, { withFileTypes: true })
+ .filter((entry) => entry.isDirectory() && entry.name.startsWith("chromium-"))
+ .map((entry) => path.join(baseDir, entry.name, "chrome-win64", "chrome.exe"))
+ .filter((candidate) => fs.existsSync(candidate))
+ .sort()
+ .reverse();
+
+ return candidates[0] ?? defaultPath;
+}
+
+function ensureBuildExists() {
+ if (!fs.existsSync(MAIN_JS)) {
+ throw new Error(`Missing ${MAIN_JS}. Run npm run build-vite first.`);
+ }
+ if (!fs.existsSync(path.join(ROOT, "dist", "index.html"))) {
+ throw new Error(`Missing renderer build. Run npm run build-vite first.`);
+ }
+}
+
+function runNpmBuildViteIfRequested() {
+ if (process.env.OPENSCREEN_PREVIEW_SKIP_BUILD === "true") {
+ ensureBuildExists();
+ return Promise.resolve();
+ }
+
+ return new Promise((resolve, reject) => {
+ const child = spawn("cmd.exe", ["/d", "/s", "/c", "npm run build-vite"], {
+ cwd: ROOT,
+ stdio: "inherit",
+ });
+ child.once("error", reject);
+ child.once("exit", (code) => {
+ if (code === 0) resolve();
+ else reject(new Error(`npm run build-vite failed with code ${code}`));
+ });
+ });
+}
+
+async function encodeFramesToWebm(framePaths, outputPath) {
+ const frameData = framePaths.map((framePath) => ({
+ src: `data:image/png;base64,${fs.readFileSync(framePath).toString("base64")}`,
+ }));
+ const html = `
+
+
+OpenScreen native cursor diagnostic
+
+
+
+
+OpenScreen native cursor diagnostic
+
+
${report.sampleCount} samples
+
${report.assetCount} assets
+
${report.uniquePositionCount} positions
+
${report.errorCount} errors
+
The red cross is the captured native hotspot. Native bitmaps are drawn at 1x, 2x, and 3x. The last cursor is a crisp vector 3x replacement anchored on the same hotspot.
+
+
+
+
+`;
+}
+
+function readScreenFrames(outputDir, recordingStartTimestampMs) {
+ const framesJsonPath = path.join(outputDir, "screen-frames", "frames.json");
+ if (!fs.existsSync(framesJsonPath)) {
+ return [];
+ }
+
+ const rawFrames = JSON.parse(fs.readFileSync(framesJsonPath, "utf8").replace(/^\uFEFF/, ""));
+ const frames = Array.isArray(rawFrames) ? rawFrames : [rawFrames];
+
+ return frames
+ .filter((frame) => frame?.path && fs.existsSync(frame.path))
+ .map((frame) => ({
+ ...frame,
+ timeMs: Math.max(0, frame.timestampMs - recordingStartTimestampMs),
+ imageDataUrl: `data:image/png;base64,${fs.readFileSync(frame.path).toString("base64")}`,
+ }));
+}
+
+function buildRealCaptureHtml(report, recordingData, screenFrames) {
+ return `
+
+
+OpenScreen native cursor real capture diagnostic
+
+
+
+
+Real screen capture + reconstructed native cursor
+Background frames are real Windows screenshots. Native bitmaps are reconstructed at 1x, 2x, and 3x; the last cursor is a crisp vector 3x replacement. The red cross marks the recorded hotspot.
+
+
+
+
+
+`;
+}
+
+function findPlaywrightChromiumExecutable(defaultPath) {
+ if (fs.existsSync(defaultPath)) {
+ return defaultPath;
+ }
+
+ const baseDir = path.join(process.env.LOCALAPPDATA ?? "", "ms-playwright");
+ if (!baseDir || !fs.existsSync(baseDir)) {
+ return defaultPath;
+ }
+
+ const candidates = fs
+ .readdirSync(baseDir, { withFileTypes: true })
+ .filter((entry) => entry.isDirectory() && entry.name.startsWith("chromium-"))
+ .map((entry) => path.join(baseDir, entry.name, "chrome-win64", "chrome.exe"))
+ .filter((candidate) => fs.existsSync(candidate))
+ .sort()
+ .reverse();
+
+ return candidates[0] ?? defaultPath;
+}
+
+async function writePreviewVideo(reportPath, outputPath) {
+ const { chromium } = await import("playwright");
+ const browser = await chromium.launch({
+ executablePath: findPlaywrightChromiumExecutable(chromium.executablePath()),
+ headless: true,
+ });
+ try {
+ const page = await browser.newPage({ viewport: { width: 1180, height: 760 } });
+ await page.goto(`file:///${reportPath.replaceAll("\\", "/")}`);
+ const base64 = await page.evaluate(() => window.__exportWebm());
+ fs.writeFileSync(outputPath, Buffer.from(base64, "base64"));
+ } finally {
+ await browser.close();
+ }
+}
+
+function assertReport(report) {
+ const failures = [];
+ if (report.sampleCount < Math.floor(DURATION_MS / SAMPLE_INTERVAL_MS / 3)) {
+ failures.push(`Too few samples: ${report.sampleCount}.`);
+ }
+ if (report.visibleSampleCount === 0) {
+ failures.push("No visible cursor samples were captured.");
+ }
+ if (report.assetCount === 0) {
+ failures.push("No cursor asset PNG was captured.");
+ }
+ if (report.uniquePositionCount < 4) {
+ failures.push(`Cursor movement was not observed enough times: ${report.uniquePositionCount}.`);
+ }
+ if (report.errorCount > 0) {
+ failures.push(`Sampler reported ${report.errorCount} error event(s).`);
+ }
+
+ if (failures.length > 0) {
+ throw new Error(failures.join(" "));
+ }
+}
+
+fs.mkdirSync(OUTPUT_DIR, { recursive: true });
+
+const events = [];
+const assets = new Map();
+let lineBuffer = "";
+let stoppingSampler = false;
+const sampler = spawnPowerShell(buildSamplerScript(), {
+ onStdout: (chunk) => {
+ lineBuffer += chunk;
+ const lines = lineBuffer.split(/\r?\n/);
+ lineBuffer = lines.pop() ?? "";
+
+ for (const line of lines) {
+ const trimmed = line.trim();
+ if (!trimmed) {
+ continue;
+ }
+
+ const event = JSON.parse(trimmed);
+ events.push(event);
+ if (event.asset?.id && !assets.has(event.asset.id)) {
+ assets.set(event.asset.id, event.asset);
+ }
+ }
+ },
+ onStderr: (chunk) => {
+ if (!stoppingSampler && !chunk.startsWith("#< CLIXML")) {
+ process.stderr.write(`[cursor-native-test] ${chunk}`);
+ }
+ },
+});
+let screenRecorder = null;
+
+try {
+ await waitForReady(events);
+ screenRecorder = spawnPowerShell(buildScreenRecorderScript(OUTPUT_DIR, DURATION_MS), {
+ onStderr: (chunk) => {
+ if (!chunk.startsWith("#< CLIXML") && !chunk.startsWith(" setTimeout(resolve, 150));
+ await runPowerShell(buildMousePathScript(DURATION_MS));
+ await new Promise((resolve) => setTimeout(resolve, Math.max(250, SAMPLE_INTERVAL_MS * 3)));
+ await screenRecorder.done;
+} finally {
+ if (!sampler.child.killed) {
+ stoppingSampler = true;
+ sampler.child.kill();
+ }
+ if (screenRecorder && !screenRecorder.child.killed) {
+ screenRecorder.child.kill();
+ }
+}
+
+const samples = events.filter((event) => event.type === "sample");
+const errors = events.filter((event) => event.type === "error");
+const recordingStartTimestampMs = samples[0]?.timestampMs ?? Date.now();
+const uniquePositions = new Set(samples.map((sample) => `${sample.x},${sample.y}`));
+const report = {
+ outputDir: OUTPUT_DIR,
+ sampleIntervalMs: SAMPLE_INTERVAL_MS,
+ durationMs: DURATION_MS,
+ eventCount: events.length,
+ sampleCount: samples.length,
+ visibleSampleCount: samples.filter((sample) => sample.visible).length,
+ assetCount: assets.size,
+ uniqueCursorHandleCount: new Set(samples.map((sample) => sample.handle).filter(Boolean)).size,
+ uniquePositionCount: uniquePositions.size,
+ errorCount: errors.length,
+ firstSample: samples[0] ?? null,
+ lastSample: samples.at(-1) ?? null,
+ assets: [...assets.values()].map((asset) => ({
+ id: asset.id,
+ width: asset.width,
+ height: asset.height,
+ hotspotX: asset.hotspotX,
+ hotspotY: asset.hotspotY,
+ })),
+};
+const recordingData = toRecordingData(samples, assets);
+const screenFrames = readScreenFrames(OUTPUT_DIR, recordingStartTimestampMs);
+const reportHtmlPath = path.join(OUTPUT_DIR, "report.html");
+const previewVideoPath = path.join(OUTPUT_DIR, "preview.webm");
+const realCaptureHtmlPath = path.join(OUTPUT_DIR, "real-capture-report.html");
+const realCaptureVideoPath = path.join(OUTPUT_DIR, "real-capture-preview.webm");
+
+writeAssets(assets, OUTPUT_DIR);
+fs.writeFileSync(path.join(OUTPUT_DIR, "events.json"), JSON.stringify(events, null, 2));
+fs.writeFileSync(
+ path.join(OUTPUT_DIR, "cursor-recording-data.json"),
+ JSON.stringify(recordingData, null, 2),
+);
+fs.writeFileSync(path.join(OUTPUT_DIR, "report.json"), JSON.stringify(report, null, 2));
+fs.writeFileSync(reportHtmlPath, buildVisualReportHtml(report, recordingData));
+if (screenFrames.length > 0) {
+ fs.writeFileSync(realCaptureHtmlPath, buildRealCaptureHtml(report, recordingData, screenFrames));
+ report.screenFrameCount = screenFrames.length;
+}
+
+try {
+ await writePreviewVideo(reportHtmlPath, previewVideoPath);
+ report.previewVideoPath = previewVideoPath;
+} catch (error) {
+ report.previewVideoError = error instanceof Error ? error.message : String(error);
+}
+
+if (screenFrames.length > 0) {
+ try {
+ await writePreviewVideo(realCaptureHtmlPath, realCaptureVideoPath);
+ report.realCaptureVideoPath = realCaptureVideoPath;
+ } catch (error) {
+ report.realCaptureVideoError = error instanceof Error ? error.message : String(error);
+ }
+}
+
+fs.writeFileSync(path.join(OUTPUT_DIR, "report.json"), JSON.stringify(report, null, 2));
+
+assertReport(report);
+
+console.log(JSON.stringify(report, null, 2));
diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx
index d7b3836..0586e54 100644
--- a/src/components/video-editor/VideoPlayback.tsx
+++ b/src/components/video-editor/VideoPlayback.tsx
@@ -26,10 +26,10 @@ import {
type WebcamSizePreset,
} from "@/lib/compositeLayout";
import {
- getNativeCursorDisplayMetrics,
hasNativeCursorRecordingData,
projectNativeCursorToStage,
resolveInterpolatedNativeCursorFrame,
+ resolveNativeCursorRenderAsset,
} from "@/lib/cursor/nativeCursor";
import { classifyWallpaper, DEFAULT_WALLPAPER, resolveImageWallpaperUrl } from "@/lib/wallpaper";
import { getCssClipPath } from "@/lib/webcamMaskShapes";
@@ -1324,19 +1324,20 @@ const VideoPlayback = forwardRef(
sample: frame.sample,
});
if (projectedPoint) {
- const metrics = getNativeCursorDisplayMetrics(
+ const renderAsset = resolveNativeCursorRenderAsset(
frame.asset,
window.devicePixelRatio || 1,
+ frame.sample,
);
const scale = Math.max(0, cursorSizeRef.current);
- if (nativeCursorImg.dataset.cursorId !== frame.asset.id) {
- nativeCursorImg.src = frame.asset.imageDataUrl;
- nativeCursorImg.dataset.cursorId = frame.asset.id;
+ if (nativeCursorImg.dataset.cursorId !== renderAsset.id) {
+ nativeCursorImg.src = renderAsset.imageDataUrl;
+ nativeCursorImg.dataset.cursorId = renderAsset.id;
}
- nativeCursorImg.style.left = `${projectedPoint.x - metrics.hotspotX * scale}px`;
- nativeCursorImg.style.top = `${projectedPoint.y - metrics.hotspotY * scale}px`;
- nativeCursorImg.style.width = `${metrics.width * scale}px`;
- nativeCursorImg.style.height = `${metrics.height * scale}px`;
+ nativeCursorImg.style.left = `${projectedPoint.x - renderAsset.hotspotX * scale}px`;
+ nativeCursorImg.style.top = `${projectedPoint.y - renderAsset.hotspotY * scale}px`;
+ nativeCursorImg.style.width = `${renderAsset.width * scale}px`;
+ nativeCursorImg.style.height = `${renderAsset.height * scale}px`;
nativeCursorImg.style.display = "block";
} else {
nativeCursorImg.style.display = "none";
diff --git a/src/lib/cursor/nativeCursor.ts b/src/lib/cursor/nativeCursor.ts
index b32bd9e..d6ae220 100644
--- a/src/lib/cursor/nativeCursor.ts
+++ b/src/lib/cursor/nativeCursor.ts
@@ -1,9 +1,20 @@
import { type Container, Point } from "pixi.js";
+import crosshairUrl from "@/assets/cursors/Cursor=Cross.svg";
+import arrowUrl from "@/assets/cursors/Cursor=Default.svg";
+import pointerUrl from "@/assets/cursors/Cursor=Hand-(Pointing).svg";
+import notAllowedUrl from "@/assets/cursors/Cursor=Menu.svg";
+import moveUrl from "@/assets/cursors/Cursor=Move.svg";
+import resizeNeswUrl from "@/assets/cursors/Cursor=Resize-North-East-South-West.svg";
+import resizeNsUrl from "@/assets/cursors/Cursor=Resize-North-South.svg";
+import resizeNwseUrl from "@/assets/cursors/Cursor=Resize-North-West-South-East.svg";
+import resizeEwUrl from "@/assets/cursors/Cursor=Resize-West-East.svg";
+import textUrl from "@/assets/cursors/Cursor=Text-Cursor.svg";
import type { CropRegion } from "@/components/video-editor/types";
import type {
CursorRecordingData,
CursorRecordingSample,
NativeCursorAsset,
+ NativeCursorType,
} from "@/native/contracts";
export interface ActiveNativeCursorFrame {
@@ -23,6 +34,87 @@ function clamp(value: number, min: number, max: number) {
return Math.min(max, Math.max(min, value));
}
+interface PrettyNativeCursorAsset {
+ imageDataUrl: string;
+ width: number;
+ height: number;
+ hotspotX: number;
+ hotspotY: number;
+}
+
+const PRETTY_NATIVE_CURSOR_ASSETS: Partial> = {
+ arrow: {
+ imageDataUrl: arrowUrl,
+ width: 32,
+ height: 32,
+ hotspotX: 5.8,
+ hotspotY: 3.2,
+ },
+ text: {
+ imageDataUrl: textUrl,
+ width: 32,
+ height: 32,
+ hotspotX: 16,
+ hotspotY: 16,
+ },
+ pointer: {
+ imageDataUrl: pointerUrl,
+ width: 32,
+ height: 32,
+ hotspotX: 11.8,
+ hotspotY: 2.6,
+ },
+ crosshair: {
+ imageDataUrl: crosshairUrl,
+ width: 32,
+ height: 32,
+ hotspotX: 16,
+ hotspotY: 16,
+ },
+ "resize-ew": {
+ imageDataUrl: resizeEwUrl,
+ width: 32,
+ height: 32,
+ hotspotX: 16,
+ hotspotY: 16,
+ },
+ "resize-ns": {
+ imageDataUrl: resizeNsUrl,
+ width: 32,
+ height: 32,
+ hotspotX: 16,
+ hotspotY: 16,
+ },
+ "resize-nesw": {
+ imageDataUrl: resizeNeswUrl,
+ width: 32,
+ height: 32,
+ hotspotX: 16,
+ hotspotY: 16,
+ },
+ "resize-nwse": {
+ imageDataUrl: resizeNwseUrl,
+ width: 32,
+ height: 32,
+ hotspotX: 16,
+ hotspotY: 16,
+ },
+ move: {
+ imageDataUrl: moveUrl,
+ width: 32,
+ height: 32,
+ hotspotX: 16,
+ hotspotY: 16,
+ },
+ "not-allowed": {
+ imageDataUrl: notAllowedUrl,
+ width: 32,
+ height: 32,
+ hotspotX: 16,
+ hotspotY: 16,
+ },
+};
+
export function hasNativeCursorRecordingData(
recordingData: CursorRecordingData | null | undefined,
): recordingData is CursorRecordingData {
@@ -169,3 +261,39 @@ export function getNativeCursorDisplayMetrics(asset: NativeCursorAsset, deviceSc
hotspotY: asset.hotspotY / scaleFactor,
};
}
+
+export function resolvePrettyNativeCursorAsset(
+ asset: NativeCursorAsset,
+ sample?: CursorRecordingSample,
+) {
+ const cursorType = sample?.cursorType ?? asset.cursorType ?? null;
+ return cursorType ? (PRETTY_NATIVE_CURSOR_ASSETS[cursorType] ?? null) : null;
+}
+
+export function resolveNativeCursorRenderAsset(
+ asset: NativeCursorAsset,
+ deviceScaleFactor: number,
+ sample?: CursorRecordingSample,
+) {
+ const prettyAsset = resolvePrettyNativeCursorAsset(asset, sample);
+ if (prettyAsset) {
+ return {
+ id: `pretty:${sample?.cursorType ?? asset.cursorType}`,
+ imageDataUrl: prettyAsset.imageDataUrl,
+ width: prettyAsset.width,
+ height: prettyAsset.height,
+ hotspotX: prettyAsset.hotspotX,
+ hotspotY: prettyAsset.hotspotY,
+ };
+ }
+
+ const metrics = getNativeCursorDisplayMetrics(asset, deviceScaleFactor);
+ return {
+ id: asset.id,
+ imageDataUrl: asset.imageDataUrl,
+ width: metrics.width,
+ height: metrics.height,
+ hotspotX: metrics.hotspotX,
+ hotspotY: metrics.hotspotY,
+ };
+}
diff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.ts
index c43908f..a1e20cc 100644
--- a/src/lib/exporter/frameRenderer.ts
+++ b/src/lib/exporter/frameRenderer.ts
@@ -57,13 +57,13 @@ import {
type StyledRenderRect,
} from "@/lib/compositeLayout";
import {
- getNativeCursorDisplayMetrics,
projectNativeCursorToStage,
resolveInterpolatedNativeCursorFrame,
+ resolveNativeCursorRenderAsset,
} from "@/lib/cursor/nativeCursor";
import { BackgroundLoadError, classifyWallpaper, resolveImageWallpaperUrl } from "@/lib/wallpaper";
import { drawCanvasClipPath } from "@/lib/webcamMaskShapes";
-import type { CursorRecordingData, NativeCursorAsset } from "@/native/contracts";
+import type { CursorRecordingData } from "@/native/contracts";
import { renderAnnotations } from "./annotationRenderer";
import {
getLinearGradientPoints,
@@ -585,19 +585,23 @@ export class FrameRenderer {
return;
}
- const image = await this.getCursorImage(activeNativeCursor.asset);
- const metrics = getNativeCursorDisplayMetrics(activeNativeCursor.asset, 1);
+ const renderAsset = resolveNativeCursorRenderAsset(
+ activeNativeCursor.asset,
+ 1,
+ activeNativeCursor.sample,
+ );
+ const image = await this.getCursorImage(renderAsset);
const scale = Math.max(0, this.config.cursorScale ?? 1);
this.compositeCtx.drawImage(
image,
- projectedPoint.x - metrics.hotspotX * scale,
- projectedPoint.y - metrics.hotspotY * scale,
- metrics.width * scale,
- metrics.height * scale,
+ projectedPoint.x - renderAsset.hotspotX * scale,
+ projectedPoint.y - renderAsset.hotspotY * scale,
+ renderAsset.width * scale,
+ renderAsset.height * scale,
);
}
- private async getCursorImage(asset: NativeCursorAsset) {
+ private async getCursorImage(asset: { id: string; imageDataUrl: string }) {
const cachedImage = this.cursorImageCache.get(asset.id);
if (cachedImage) {
return cachedImage;
diff --git a/src/native/contracts.ts b/src/native/contracts.ts
index 73d53db..a3c9087 100644
--- a/src/native/contracts.ts
+++ b/src/native/contracts.ts
@@ -3,6 +3,21 @@ export const NATIVE_BRIDGE_VERSION = 1;
export type NativePlatform = "darwin" | "win32" | "linux";
export type CursorProviderKind = "native" | "none";
+export type NativeCursorType =
+ | "arrow"
+ | "text"
+ | "pointer"
+ | "crosshair"
+ | "resize-ew"
+ | "resize-ns"
+ | "resize-nesw"
+ | "resize-nwse"
+ | "move"
+ | "not-allowed"
+ | "wait"
+ | "app-starting"
+ | "help"
+ | "up-arrow";
export interface CursorTelemetryPoint {
timeMs: number;
@@ -13,6 +28,7 @@ export interface CursorTelemetryPoint {
export interface CursorRecordingSample extends CursorTelemetryPoint {
assetId?: string | null;
visible?: boolean;
+ cursorType?: NativeCursorType | null;
}
export interface NativeCursorAsset {
@@ -24,6 +40,7 @@ export interface NativeCursorAsset {
hotspotX: number;
hotspotY: number;
scaleFactor?: number;
+ cursorType?: NativeCursorType | null;
}
export interface CursorRecordingData {
From ef36da4a4f0f859eb0b4d451f05011a6ee4c21e4 Mon Sep 17 00:00:00 2001
From: EtienneLescot
Date: Tue, 5 May 2026 10:41:34 +0200
Subject: [PATCH 09/92] feat: complete windows cursor assets
---
electron/ipc/handlers.ts | 80 +++-------------------
src/assets/cursors/Cursor=App-Starting.svg | 8 +++
src/assets/cursors/Cursor=Help.svg | 8 +++
src/assets/cursors/Cursor=Not-Allowed.svg | 7 ++
src/assets/cursors/Cursor=Up-Arrow.svg | 4 ++
src/assets/cursors/Cursor=Wait.svg | 6 ++
src/lib/cursor/nativeCursor.ts | 34 ++++++++-
7 files changed, 77 insertions(+), 70 deletions(-)
create mode 100644 src/assets/cursors/Cursor=App-Starting.svg
create mode 100644 src/assets/cursors/Cursor=Help.svg
create mode 100644 src/assets/cursors/Cursor=Not-Allowed.svg
create mode 100644 src/assets/cursors/Cursor=Up-Arrow.svg
create mode 100644 src/assets/cursors/Cursor=Wait.svg
diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts
index c5a1269..7b16f2f 100644
--- a/electron/ipc/handlers.ts
+++ b/electron/ipc/handlers.ts
@@ -3,7 +3,7 @@ import { createRequire } from "node:module";
import os from "node:os";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
-
+import type { DesktopCapturerSource } from "electron";
import {
app,
BrowserWindow,
@@ -14,7 +14,6 @@ import {
shell,
systemPreferences,
} from "electron";
-import type { DesktopCapturerSource } from "electron";
import {
normalizeProjectMedia,
normalizeRecordingSession,
@@ -410,46 +409,6 @@ function setCurrentRecordingSessionState(session: RecordingSession | null) {
currentVideoPath = session?.screenVideoPath ?? null;
}
-async function storeRecordedSessionFiles(payload: StoreRecordedSessionInput) {
- const createdAt =
- typeof payload.createdAt === "number" && Number.isFinite(payload.createdAt)
- ? payload.createdAt
- : Date.now();
- const screenVideoPath = resolveRecordingOutputPath(payload.screen.fileName);
- await fs.writeFile(screenVideoPath, Buffer.from(payload.screen.videoData));
-
- let webcamVideoPath: string | undefined;
- if (payload.webcam) {
- webcamVideoPath = resolveRecordingOutputPath(payload.webcam.fileName);
- await fs.writeFile(webcamVideoPath, Buffer.from(payload.webcam.videoData));
- }
-
- const session: RecordingSession = webcamVideoPath
- ? { screenVideoPath, webcamVideoPath, createdAt }
- : { screenVideoPath, createdAt };
- setCurrentRecordingSessionState(session);
- currentProjectPath = null;
-
- const telemetryPath = `${screenVideoPath}.cursor.json`;
- if (pendingCursorRecordingData && pendingCursorRecordingData.samples.length > 0) {
- await fs.writeFile(telemetryPath, JSON.stringify(pendingCursorRecordingData, null, 2), "utf-8");
- }
- pendingCursorRecordingData = null;
-
- const sessionManifestPath = path.join(
- RECORDINGS_DIR,
- `${path.parse(payload.screen.fileName).name}${RECORDING_SESSION_SUFFIX}`,
- );
- await fs.writeFile(sessionManifestPath, JSON.stringify(session, null, 2), "utf-8");
-
- return {
- success: true,
- path: screenVideoPath,
- session,
- message: "Recording session stored successfully",
- };
-}
-
export function registerIpcHandlers(
createEditorWindow: () => void,
createSourceSelectorWindow: () => BrowserWindow,
@@ -612,12 +571,12 @@ export function registerIpcHandlers(
typeof payload.createdAt === "number" && Number.isFinite(payload.createdAt)
? payload.createdAt
: Date.now();
- const screenVideoPath = path.join(RECORDINGS_DIR, payload.screen.fileName);
+ const screenVideoPath = resolveRecordingOutputPath(payload.screen.fileName);
await fs.writeFile(screenVideoPath, Buffer.from(payload.screen.videoData));
let webcamVideoPath: string | undefined;
if (payload.webcam) {
- webcamVideoPath = path.join(RECORDINGS_DIR, payload.webcam.fileName);
+ webcamVideoPath = resolveRecordingOutputPath(payload.webcam.fileName);
await fs.writeFile(webcamVideoPath, Buffer.from(payload.webcam.videoData));
}
@@ -625,7 +584,6 @@ export function registerIpcHandlers(
? { screenVideoPath, webcamVideoPath, createdAt }
: { screenVideoPath, createdAt };
setCurrentRecordingSessionState(session);
- currentVideoPath = screenVideoPath;
currentProjectPath = null;
const telemetryPath = `${screenVideoPath}.cursor.json`;
@@ -638,6 +596,12 @@ export function registerIpcHandlers(
}
pendingCursorRecordingData = null;
+ const sessionManifestPath = path.join(
+ RECORDINGS_DIR,
+ `${path.parse(payload.screen.fileName).name}${RECORDING_SESSION_SUFFIX}`,
+ );
+ await fs.writeFile(sessionManifestPath, JSON.stringify(session, null, 2), "utf-8");
+
return {
success: true,
path: screenVideoPath,
@@ -1010,18 +974,7 @@ export function registerIpcHandlers(
const content = await fs.readFile(filePath, "utf-8");
const project = JSON.parse(content);
currentProjectPath = filePath;
- if (project && typeof project === "object") {
- const rawProject = project as { media?: unknown; videoPath?: unknown };
- const media =
- normalizeProjectMedia(rawProject.media) ??
- (typeof rawProject.videoPath === "string"
- ? {
- screenVideoPath:
- normalizeVideoSourcePath(rawProject.videoPath) ?? rawProject.videoPath,
- }
- : null);
- setCurrentRecordingSessionState(media ? { ...media, createdAt: Date.now() } : null);
- }
+ setCurrentRecordingSessionState(await getApprovedProjectSession(project, filePath));
return {
success: true,
@@ -1050,18 +1003,7 @@ export function registerIpcHandlers(
const content = await fs.readFile(currentProjectPath, "utf-8");
const project = JSON.parse(content);
- if (project && typeof project === "object") {
- const rawProject = project as { media?: unknown; videoPath?: unknown };
- const media =
- normalizeProjectMedia(rawProject.media) ??
- (typeof rawProject.videoPath === "string"
- ? {
- screenVideoPath:
- normalizeVideoSourcePath(rawProject.videoPath) ?? rawProject.videoPath,
- }
- : null);
- setCurrentRecordingSessionState(media ? { ...media, createdAt: Date.now() } : null);
- }
+ setCurrentRecordingSessionState(await getApprovedProjectSession(project, currentProjectPath));
return {
success: true,
path: currentProjectPath,
diff --git a/src/assets/cursors/Cursor=App-Starting.svg b/src/assets/cursors/Cursor=App-Starting.svg
new file mode 100644
index 0000000..7a10d40
--- /dev/null
+++ b/src/assets/cursors/Cursor=App-Starting.svg
@@ -0,0 +1,8 @@
+
+
diff --git a/src/assets/cursors/Cursor=Help.svg b/src/assets/cursors/Cursor=Help.svg
new file mode 100644
index 0000000..d187c52
--- /dev/null
+++ b/src/assets/cursors/Cursor=Help.svg
@@ -0,0 +1,8 @@
+
+
diff --git a/src/assets/cursors/Cursor=Not-Allowed.svg b/src/assets/cursors/Cursor=Not-Allowed.svg
new file mode 100644
index 0000000..8b2c3f8
--- /dev/null
+++ b/src/assets/cursors/Cursor=Not-Allowed.svg
@@ -0,0 +1,7 @@
+
+
diff --git a/src/assets/cursors/Cursor=Up-Arrow.svg b/src/assets/cursors/Cursor=Up-Arrow.svg
new file mode 100644
index 0000000..b742e70
--- /dev/null
+++ b/src/assets/cursors/Cursor=Up-Arrow.svg
@@ -0,0 +1,4 @@
+
+
diff --git a/src/assets/cursors/Cursor=Wait.svg b/src/assets/cursors/Cursor=Wait.svg
new file mode 100644
index 0000000..2b56934
--- /dev/null
+++ b/src/assets/cursors/Cursor=Wait.svg
@@ -0,0 +1,6 @@
+
+
diff --git a/src/lib/cursor/nativeCursor.ts b/src/lib/cursor/nativeCursor.ts
index d6ae220..30f7a46 100644
--- a/src/lib/cursor/nativeCursor.ts
+++ b/src/lib/cursor/nativeCursor.ts
@@ -1,14 +1,18 @@
import { type Container, Point } from "pixi.js";
+import appStartingUrl from "@/assets/cursors/Cursor=App-Starting.svg";
import crosshairUrl from "@/assets/cursors/Cursor=Cross.svg";
import arrowUrl from "@/assets/cursors/Cursor=Default.svg";
import pointerUrl from "@/assets/cursors/Cursor=Hand-(Pointing).svg";
-import notAllowedUrl from "@/assets/cursors/Cursor=Menu.svg";
+import helpUrl from "@/assets/cursors/Cursor=Help.svg";
import moveUrl from "@/assets/cursors/Cursor=Move.svg";
+import notAllowedUrl from "@/assets/cursors/Cursor=Not-Allowed.svg";
import resizeNeswUrl from "@/assets/cursors/Cursor=Resize-North-East-South-West.svg";
import resizeNsUrl from "@/assets/cursors/Cursor=Resize-North-South.svg";
import resizeNwseUrl from "@/assets/cursors/Cursor=Resize-North-West-South-East.svg";
import resizeEwUrl from "@/assets/cursors/Cursor=Resize-West-East.svg";
import textUrl from "@/assets/cursors/Cursor=Text-Cursor.svg";
+import upArrowUrl from "@/assets/cursors/Cursor=Up-Arrow.svg";
+import waitUrl from "@/assets/cursors/Cursor=Wait.svg";
import type { CropRegion } from "@/components/video-editor/types";
import type {
CursorRecordingData,
@@ -113,6 +117,34 @@ const PRETTY_NATIVE_CURSOR_ASSETS: Partial
Date: Tue, 5 May 2026 10:50:20 +0200
Subject: [PATCH 10/92] fix: avoid unsupported display media min constraint
---
src/hooks/useScreenRecorder.ts | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts
index 8aa673e..7cd86a7 100644
--- a/src/hooks/useScreenRecorder.ts
+++ b/src/hooks/useScreenRecorder.ts
@@ -5,7 +5,6 @@ import { useScopedT } from "@/contexts/I18nContext";
import { requestCameraAccess } from "@/lib/requestCameraAccess";
const TARGET_FRAME_RATE = 60;
-const MIN_FRAME_RATE = 30;
const TARGET_WIDTH = 3840;
const TARGET_HEIGHT = 2160;
const FOUR_K_PIXELS = TARGET_WIDTH * TARGET_HEIGHT;
@@ -585,7 +584,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
cursor: "never",
width: { max: TARGET_WIDTH },
height: { max: TARGET_HEIGHT },
- frameRate: { ideal: TARGET_FRAME_RATE, min: MIN_FRAME_RATE },
+ frameRate: { ideal: TARGET_FRAME_RATE },
} as MediaTrackConstraints,
audio: systemAudioEnabled,
} as DisplayMediaStreamOptions);
From 87240a919ee1aa25426e0795fde2dee4c9586ab0 Mon Sep 17 00:00:00 2001
From: EtienneLescot
Date: Tue, 5 May 2026 11:04:08 +0200
Subject: [PATCH 11/92] fix: align native cursor preview and export
---
electron/ipc/handlers.ts | 31 ++++++++-
.../native-bridge/cursor/recording/factory.ts | 3 +
.../recording/telemetryRecordingSession.ts | 3 +-
.../windowsNativeRecordingSession.ts | 2 +-
.../windowsNativeRecordingSession.types.ts | 1 +
scripts/capture-openscreen-preview.mjs | 4 --
src/components/video-editor/VideoPlayback.tsx | 64 ++++++++-----------
src/hooks/useScreenRecorder.ts | 6 +-
src/lib/cursor/nativeCursor.ts | 19 ++++--
9 files changed, 82 insertions(+), 51 deletions(-)
diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts
index 7b16f2f..4c306ee 100644
--- a/electron/ipc/handlers.ts
+++ b/electron/ipc/handlers.ts
@@ -651,7 +651,7 @@ export function registerIpcHandlers(
}
});
- ipcMain.handle("set-recording-state", async (_, recording: boolean) => {
+ ipcMain.handle("set-recording-state", async (_, recording: boolean, recordingId?: number) => {
if (recording) {
if (cursorRecordingSession) {
pendingCursorRecordingData = await cursorRecordingSession.stop();
@@ -665,6 +665,8 @@ export function registerIpcHandlers(
platform: process.platform,
sampleIntervalMs: CURSOR_SAMPLE_INTERVAL_MS,
sourceId: getSelectedSourceId(),
+ startTimeMs:
+ typeof recordingId === "number" && Number.isFinite(recordingId) ? recordingId : undefined,
});
try {
@@ -824,6 +826,7 @@ export function registerIpcHandlers(
return { success: false, canceled: true };
}
+ approveFilePath(result.filePaths[0]);
currentProjectPath = null;
return {
success: true,
@@ -863,6 +866,32 @@ export function registerIpcHandlers(
}
});
+ ipcMain.handle("read-binary-file", async (_, filePath: string) => {
+ try {
+ const normalizedPath = await approveReadableVideoPath(filePath);
+ if (!normalizedPath) {
+ return {
+ success: false,
+ message: "File path is not approved or is not a supported video file",
+ };
+ }
+
+ const data = await fs.readFile(normalizedPath);
+ return {
+ success: true,
+ data: data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength),
+ path: normalizedPath,
+ };
+ } catch (error) {
+ console.error("Failed to read binary file:", error);
+ return {
+ success: false,
+ message: "Failed to read binary file",
+ error: String(error),
+ };
+ }
+ });
+
ipcMain.handle(
"save-project-file",
async (_, projectData: unknown, suggestedName?: string, existingProjectPath?: string) => {
diff --git a/electron/native-bridge/cursor/recording/factory.ts b/electron/native-bridge/cursor/recording/factory.ts
index 4e0f75c..52d6079 100644
--- a/electron/native-bridge/cursor/recording/factory.ts
+++ b/electron/native-bridge/cursor/recording/factory.ts
@@ -9,6 +9,7 @@ interface CreateCursorRecordingSessionOptions {
platform: NodeJS.Platform;
sampleIntervalMs: number;
sourceId?: string | null;
+ startTimeMs?: number;
}
export function createCursorRecordingSession(
@@ -20,6 +21,7 @@ export function createCursorRecordingSession(
maxSamples: options.maxSamples,
sampleIntervalMs: options.sampleIntervalMs,
sourceId: options.sourceId,
+ startTimeMs: options.startTimeMs,
});
}
@@ -27,5 +29,6 @@ export function createCursorRecordingSession(
getDisplayBounds: options.getDisplayBounds,
maxSamples: options.maxSamples,
sampleIntervalMs: options.sampleIntervalMs,
+ startTimeMs: options.startTimeMs,
});
}
diff --git a/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts b/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts
index dd42871..e719d8e 100644
--- a/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts
+++ b/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts
@@ -6,6 +6,7 @@ interface TelemetryRecordingSessionOptions {
getDisplayBounds: () => Rectangle | null;
maxSamples: number;
sampleIntervalMs: number;
+ startTimeMs?: number;
}
function clamp(value: number, min: number, max: number) {
@@ -21,7 +22,7 @@ export class TelemetryRecordingSession implements CursorRecordingSession {
async start(): Promise {
this.samples = [];
- this.startTimeMs = Date.now();
+ this.startTimeMs = this.options.startTimeMs ?? Date.now();
this.captureSample();
this.interval = setInterval(() => {
this.captureSample();
diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts
index 632a74d..8075fe3 100644
--- a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts
+++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts
@@ -41,7 +41,7 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession {
this.assets.clear();
this.samples = [];
this.lineBuffer = "";
- this.startTimeMs = Date.now();
+ this.startTimeMs = this.options.startTimeMs ?? Date.now();
this.sampleCount = 0;
this.outOfBoundsSampleCount = 0;
diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts
index fdc4ab9..5afc012 100644
--- a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts
+++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts
@@ -49,4 +49,5 @@ export interface WindowsNativeRecordingSessionOptions {
maxSamples: number;
sampleIntervalMs: number;
sourceId?: string | null;
+ startTimeMs?: number;
}
diff --git a/scripts/capture-openscreen-preview.mjs b/scripts/capture-openscreen-preview.mjs
index 6c9b6eb..25f86db 100644
--- a/scripts/capture-openscreen-preview.mjs
+++ b/scripts/capture-openscreen-preview.mjs
@@ -214,10 +214,6 @@ try {
await editorWindow.waitForLoadState("domcontentloaded");
await editorWindow.waitForSelector("video", { state: "attached", timeout: 30_000 });
await editorWindow.waitForSelector("canvas", { state: "attached", timeout: 30_000 });
- await editorWindow.waitForSelector('img[aria-hidden="true"]', {
- state: "attached",
- timeout: 30_000,
- });
await editorWindow.setViewportSize({ width: 1280, height: 800 });
await editorWindow.evaluate(async () => {
diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx
index 0586e54..840101d 100644
--- a/src/components/video-editor/VideoPlayback.tsx
+++ b/src/components/video-editor/VideoPlayback.tsx
@@ -27,7 +27,7 @@ import {
} from "@/lib/compositeLayout";
import {
hasNativeCursorRecordingData,
- projectNativeCursorToStage,
+ projectNativeCursorToLocal,
resolveInterpolatedNativeCursorFrame,
resolveNativeCursorRenderAsset,
} from "@/lib/cursor/nativeCursor";
@@ -841,6 +841,12 @@ const VideoPlayback = forwardRef(
cursorContainerRef.current = cursorContainer;
cameraContainer.addChild(cursorContainer);
+ const nativeCursorSprite = new Sprite(Texture.EMPTY);
+ nativeCursorSprite.visible = false;
+ nativeCursorSprite.eventMode = "none";
+ nativeCursorSpriteRef.current = nativeCursorSprite;
+ cursorContainer.addChild(nativeCursorSprite);
+
// Cursor overlay - rendered above the masked video
if (cursorOverlayEnabled) {
const cursorOverlay = new PixiCursorOverlay({
@@ -863,6 +869,8 @@ const VideoPlayback = forwardRef(
cursorOverlayRef.current.destroy();
cursorOverlayRef.current = null;
}
+ nativeCursorSpriteRef.current = null;
+ nativeCursorTextureIdRef.current = null;
if (app && app.renderer) {
app.destroy(true, {
children: true,
@@ -1296,25 +1304,18 @@ const VideoPlayback = forwardRef(
);
}
- // Update native cursor image position at ticker rate (60fps)
- const nativeCursorImg = nativeCursorImgRef.current;
- if (nativeCursorImg) {
- const cameraContainerRc = cameraContainerRef.current;
+ // Update native cursor sprite in the same PIXI coordinate space as the video.
+ const nativeCursorSprite = nativeCursorSpriteRef.current;
+ if (nativeCursorSprite) {
const videoContainerRc = videoContainerRef.current;
- if (
- hasNativeCursorRecordingRef.current &&
- showCursorRef.current &&
- cameraContainerRc &&
- videoContainerRc
- ) {
+ if (hasNativeCursorRecordingRef.current && showCursorRef.current && videoContainerRc) {
const timeMs = currentTimeRef.current; // already in ms
const frame = resolveInterpolatedNativeCursorFrame(
cursorRecordingDataRef.current,
timeMs,
);
if (frame) {
- const projectedPoint = projectNativeCursorToStage({
- cameraContainer: cameraContainerRc,
+ const projectedPoint = projectNativeCursorToLocal({
cropRegion: cropRegionRef.current ?? { x: 0, y: 0, width: 1, height: 1 },
maskRect: baseMaskRef.current,
videoContainerPosition: {
@@ -1330,23 +1331,25 @@ const VideoPlayback = forwardRef(
frame.sample,
);
const scale = Math.max(0, cursorSizeRef.current);
- if (nativeCursorImg.dataset.cursorId !== renderAsset.id) {
- nativeCursorImg.src = renderAsset.imageDataUrl;
- nativeCursorImg.dataset.cursorId = renderAsset.id;
+ if (nativeCursorTextureIdRef.current !== renderAsset.id) {
+ nativeCursorSprite.texture = Texture.from(renderAsset.imageDataUrl);
+ nativeCursorTextureIdRef.current = renderAsset.id;
}
- nativeCursorImg.style.left = `${projectedPoint.x - renderAsset.hotspotX * scale}px`;
- nativeCursorImg.style.top = `${projectedPoint.y - renderAsset.hotspotY * scale}px`;
- nativeCursorImg.style.width = `${renderAsset.width * scale}px`;
- nativeCursorImg.style.height = `${renderAsset.height * scale}px`;
- nativeCursorImg.style.display = "block";
+ nativeCursorSprite.position.set(
+ projectedPoint.x - renderAsset.hotspotX * scale,
+ projectedPoint.y - renderAsset.hotspotY * scale,
+ );
+ nativeCursorSprite.width = renderAsset.width * scale;
+ nativeCursorSprite.height = renderAsset.height * scale;
+ nativeCursorSprite.visible = true;
} else {
- nativeCursorImg.style.display = "none";
+ nativeCursorSprite.visible = false;
}
} else {
- nativeCursorImg.style.display = "none";
+ nativeCursorSprite.visible = false;
}
} else {
- nativeCursorImg.style.display = "none";
+ nativeCursorSprite.visible = false;
}
}
@@ -1638,19 +1641,6 @@ const VideoPlayback = forwardRef(
className="absolute rounded-md border border-[#34B27B]/80 bg-[#34B27B]/20 shadow-[0_0_0_1px_rgba(52,178,123,0.35)]"
style={{ display: "none", pointerEvents: "none" }}
/>
- {hasNativeCursorRecording ? (
-
Date: Tue, 5 May 2026 11:38:50 +0200
Subject: [PATCH 12/92] fix: restore native cursor preview and export
---
src/components/video-editor/VideoPlayback.tsx | 127 ++++++++++++------
src/lib/cursor/nativeCursor.ts | 26 +++-
src/lib/exporter/frameRenderer.ts | 24 ++--
3 files changed, 116 insertions(+), 61 deletions(-)
diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx
index 840101d..432958a 100644
--- a/src/components/video-editor/VideoPlayback.tsx
+++ b/src/components/video-editor/VideoPlayback.tsx
@@ -28,6 +28,7 @@ import {
import {
hasNativeCursorRecordingData,
projectNativeCursorToLocal,
+ projectNativeCursorToStage,
resolveInterpolatedNativeCursorFrame,
resolveNativeCursorRenderAsset,
} from "@/lib/cursor/nativeCursor";
@@ -243,7 +244,6 @@ const VideoPlayback = forwardRef(
const videoSpriteRef = useRef(null);
const videoContainerRef = useRef(null);
const cameraContainerRef = useRef(null);
- const cursorContainerRef = useRef(null);
const timeUpdateAnimationRef = useRef(null);
const [pixiReady, setPixiReady] = useState(false);
const [videoReady, setVideoReady] = useState(false);
@@ -836,17 +836,6 @@ const VideoPlayback = forwardRef(
videoContainerRef.current = videoContainer;
cameraContainer.addChild(videoContainer);
- // Cursor container - rendered above video
- const cursorContainer = new Container();
- cursorContainerRef.current = cursorContainer;
- cameraContainer.addChild(cursorContainer);
-
- const nativeCursorSprite = new Sprite(Texture.EMPTY);
- nativeCursorSprite.visible = false;
- nativeCursorSprite.eventMode = "none";
- nativeCursorSpriteRef.current = nativeCursorSprite;
- cursorContainer.addChild(nativeCursorSprite);
-
// Cursor overlay - rendered above the masked video
if (cursorOverlayEnabled) {
const cursorOverlay = new PixiCursorOverlay({
@@ -856,7 +845,6 @@ const VideoPlayback = forwardRef(
clickBounce: cursorClickBounceRef.current,
});
cursorOverlayRef.current = cursorOverlay;
- cursorContainer.addChild(cursorOverlay.container);
}
setPixiReady(true);
@@ -871,6 +859,7 @@ const VideoPlayback = forwardRef(
}
nativeCursorSpriteRef.current = null;
nativeCursorTextureIdRef.current = null;
+ nativeCursorImageIdRef.current = null;
if (app && app.renderer) {
app.destroy(true, {
children: true,
@@ -881,7 +870,6 @@ const VideoPlayback = forwardRef(
appRef.current = null;
cameraContainerRef.current = null;
videoContainerRef.current = null;
- cursorContainerRef.current = null;
videoSpriteRef.current = null;
};
}, []);
@@ -920,9 +908,8 @@ const VideoPlayback = forwardRef(
const video = videoRef.current;
const app = appRef.current;
const videoContainer = videoContainerRef.current;
- const cursorContainer = cursorContainerRef.current;
- if (!video || !app || !videoContainer || !cursorContainer) return;
+ if (!video || !app || !videoContainer) return;
if (video.videoWidth === 0 || video.videoHeight === 0) return;
const source = VideoSource.from(video);
@@ -942,8 +929,12 @@ const VideoPlayback = forwardRef(
videoContainer.addChild(maskGraphics);
videoContainer.mask = maskGraphics;
maskGraphicsRef.current = maskGraphics;
+ const nativeCursorSprite = new Sprite(Texture.EMPTY);
+ nativeCursorSprite.visible = false;
+ nativeCursorSprite.eventMode = "none";
+ nativeCursorSpriteRef.current = nativeCursorSprite;
if (cursorOverlayRef.current) {
- cursorContainer.addChild(cursorOverlayRef.current.container);
+ videoContainer.addChild(cursorOverlayRef.current.container);
}
const cursorHighlightGraphics = new Graphics();
@@ -951,6 +942,7 @@ const VideoPlayback = forwardRef(
videoContainer.addChild(cursorHighlightGraphics);
cursorHighlightGraphicsRef.current = cursorHighlightGraphics;
drawCursorHighlightGraphics(cursorHighlightGraphics, cursorHighlightRef.current);
+ videoContainer.addChild(nativeCursorSprite);
animationStateRef.current = {
scale: 1,
@@ -1019,6 +1011,12 @@ const VideoPlayback = forwardRef(
cursorHighlightGraphicsRef.current.destroy();
cursorHighlightGraphicsRef.current = null;
}
+ if (nativeCursorSpriteRef.current) {
+ videoContainer.removeChild(nativeCursorSpriteRef.current);
+ nativeCursorSpriteRef.current.destroy();
+ nativeCursorSpriteRef.current = null;
+ nativeCursorTextureIdRef.current = null;
+ }
videoContainer.mask = null;
maskGraphicsRef.current = null;
if (blurFilterRef.current) {
@@ -1304,53 +1302,88 @@ const VideoPlayback = forwardRef(
);
}
- // Update native cursor sprite in the same PIXI coordinate space as the video.
+ // Keep the native cursor preview in the same transformed coordinate space as PIXI.
const nativeCursorSprite = nativeCursorSpriteRef.current;
- if (nativeCursorSprite) {
- const videoContainerRc = videoContainerRef.current;
- if (hasNativeCursorRecordingRef.current && showCursorRef.current && videoContainerRc) {
+ const nativeCursorImage = nativeCursorImageRef.current;
+ const hideNativeCursorPreview = () => {
+ if (nativeCursorSprite) {
+ nativeCursorSprite.visible = false;
+ }
+ if (nativeCursorImage) {
+ nativeCursorImage.style.display = "none";
+ }
+ };
+ if (nativeCursorImage) {
+ if (hasNativeCursorRecordingRef.current && showCursorRef.current) {
const timeMs = currentTimeRef.current; // already in ms
const frame = resolveInterpolatedNativeCursorFrame(
cursorRecordingDataRef.current,
timeMs,
);
if (frame) {
- const projectedPoint = projectNativeCursorToLocal({
- cropRegion: cropRegionRef.current ?? { x: 0, y: 0, width: 1, height: 1 },
+ const cameraContainer = cameraContainerRef.current;
+ const videoContainer = videoContainerRef.current;
+ const cropRegionValue = cropRegionRef.current ?? { x: 0, y: 0, width: 1, height: 1 };
+ const projectedLocalPoint = projectNativeCursorToLocal({
+ cropRegion: cropRegionValue,
maskRect: baseMaskRef.current,
- videoContainerPosition: {
- x: videoContainerRc.x,
- y: videoContainerRc.y,
- },
sample: frame.sample,
});
- if (projectedPoint) {
+ const projectedStagePoint =
+ cameraContainer && videoContainer
+ ? projectNativeCursorToStage({
+ cameraContainer,
+ cropRegion: cropRegionValue,
+ maskRect: baseMaskRef.current,
+ videoContainerPosition: {
+ x: videoContainer.x,
+ y: videoContainer.y,
+ },
+ sample: frame.sample,
+ })
+ : null;
+ if (projectedLocalPoint && projectedStagePoint) {
const renderAsset = resolveNativeCursorRenderAsset(
frame.asset,
window.devicePixelRatio || 1,
frame.sample,
);
const scale = Math.max(0, cursorSizeRef.current);
- if (nativeCursorTextureIdRef.current !== renderAsset.id) {
- nativeCursorSprite.texture = Texture.from(renderAsset.imageDataUrl);
- nativeCursorTextureIdRef.current = renderAsset.id;
+ const transformedScale = scale * Math.abs(cameraContainer?.scale.x || 1);
+ if (nativeCursorImageIdRef.current !== renderAsset.id) {
+ nativeCursorImage.src = renderAsset.imageDataUrl;
+ nativeCursorImageIdRef.current = renderAsset.id;
+ }
+ nativeCursorImage.style.display = "block";
+ nativeCursorImage.style.width = `${renderAsset.width * transformedScale}px`;
+ nativeCursorImage.style.height = `${renderAsset.height * transformedScale}px`;
+ nativeCursorImage.style.transform = `translate3d(${
+ projectedStagePoint.x - renderAsset.hotspotX * transformedScale
+ }px, ${projectedStagePoint.y - renderAsset.hotspotY * transformedScale}px, 0)`;
+ if (nativeCursorSprite) {
+ nativeCursorSprite.visible = false;
+ if (nativeCursorTextureIdRef.current !== renderAsset.id) {
+ nativeCursorSprite.texture = Texture.from(renderAsset.imageDataUrl);
+ nativeCursorTextureIdRef.current = renderAsset.id;
+ }
+ nativeCursorSprite.position.set(
+ projectedLocalPoint.x - renderAsset.hotspotX * scale,
+ projectedLocalPoint.y - renderAsset.hotspotY * scale,
+ );
+ nativeCursorSprite.width = renderAsset.width * scale;
+ nativeCursorSprite.height = renderAsset.height * scale;
}
- nativeCursorSprite.position.set(
- projectedPoint.x - renderAsset.hotspotX * scale,
- projectedPoint.y - renderAsset.hotspotY * scale,
- );
- nativeCursorSprite.width = renderAsset.width * scale;
- nativeCursorSprite.height = renderAsset.height * scale;
- nativeCursorSprite.visible = true;
} else {
- nativeCursorSprite.visible = false;
+ hideNativeCursorPreview();
}
} else {
- nativeCursorSprite.visible = false;
+ hideNativeCursorPreview();
}
} else {
- nativeCursorSprite.visible = false;
+ hideNativeCursorPreview();
}
+ } else {
+ hideNativeCursorPreview();
}
const composite3D = composite3DRef.current;
@@ -1584,6 +1617,18 @@ const VideoPlayback = forwardRef(
: "none",
}}
/>
+
Date: Tue, 5 May 2026 16:07:07 +0200
Subject: [PATCH 13/92] feat: add native Windows recorder helper
---
.gitignore | 98 ++--
.../windows-native-recorder-roadmap.md | 202 +++++++
docs/testing/windows-native-cursor.md | 34 ++
electron-builder.json5 | 18 +-
electron/electron-env.d.ts | 18 +
electron/ipc/handlers.ts | 543 ++++++++++++++++--
electron/native/README.md | 50 ++
electron/native/wgc-capture/CMakeLists.txt | 45 ++
electron/native/wgc-capture/src/main.cpp | 433 ++++++++++++++
.../native/wgc-capture/src/mf_encoder.cpp | 317 ++++++++++
electron/native/wgc-capture/src/mf_encoder.h | 63 ++
.../native/wgc-capture/src/monitor_utils.cpp | 88 +++
.../native/wgc-capture/src/monitor_utils.h | 14 +
.../src/wasapi_loopback_capture.cpp | 205 +++++++
.../wgc-capture/src/wasapi_loopback_capture.h | 47 ++
.../native/wgc-capture/src/wgc_session.cpp | 223 +++++++
electron/native/wgc-capture/src/wgc_session.h | 55 ++
electron/preload.ts | 10 +
package.json | 13 +-
scripts/build-windows-wgc-helper.mjs | 112 ++++
scripts/test-windows-wgc-helper.mjs | 167 ++++++
src/components/video-editor/VideoPlayback.tsx | 1 -
src/hooks/useScreenRecorder.ts | 167 +++++-
src/lib/cursor/nativeCursor.ts | 18 +-
src/lib/exporter/audioEncoder.ts | 28 +-
src/lib/exporter/muxer.ts | 2 +-
src/lib/nativeWindowsRecording.ts | 41 ++
27 files changed, 2873 insertions(+), 139 deletions(-)
create mode 100644 docs/engineering/windows-native-recorder-roadmap.md
create mode 100644 electron/native/README.md
create mode 100644 electron/native/wgc-capture/CMakeLists.txt
create mode 100644 electron/native/wgc-capture/src/main.cpp
create mode 100644 electron/native/wgc-capture/src/mf_encoder.cpp
create mode 100644 electron/native/wgc-capture/src/mf_encoder.h
create mode 100644 electron/native/wgc-capture/src/monitor_utils.cpp
create mode 100644 electron/native/wgc-capture/src/monitor_utils.h
create mode 100644 electron/native/wgc-capture/src/wasapi_loopback_capture.cpp
create mode 100644 electron/native/wgc-capture/src/wasapi_loopback_capture.h
create mode 100644 electron/native/wgc-capture/src/wgc_session.cpp
create mode 100644 electron/native/wgc-capture/src/wgc_session.h
create mode 100644 scripts/build-windows-wgc-helper.mjs
create mode 100644 scripts/test-windows-wgc-helper.mjs
create mode 100644 src/lib/nativeWindowsRecording.ts
diff --git a/.gitignore b/.gitignore
index 494da30..84a5a1c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,48 +1,52 @@
-# Logs
-logs
-*.log
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-pnpm-debug.log*
-lerna-debug.log*
-
-node_modules
-dist
-dist-electron
-dist-ssr
-*.local
-.env
-
-# Editor directories and files
-.vscode/*
-.zed/
-!.vscode/extensions.json
-.idea
-.DS_Store
-*.suo
-*.ntvs*
-*.njsproj
-*.sln
-*.sw?
-release/**
-*.kiro/
-.claude/
-# npx electron-builder --mac --win
-
-# Playwright
-test-results
-playwright-report/
-
-# Vitest browser mode screenshots
-__screenshots__/
-
-# shell files
-/shell.sh
-# Nix
-result
-result-*
-.direnv/
-
-#kilocode
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-electron
+dist-ssr
+*.local
+.env
+
+# Native helper build outputs
+/electron/native/wgc-capture/build/
+/electron/native/bin/
+
+# Editor directories and files
+.vscode/*
+.zed/
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
+release/**
+*.kiro/
+.claude/
+# npx electron-builder --mac --win
+
+# Playwright
+test-results
+playwright-report/
+
+# Vitest browser mode screenshots
+__screenshots__/
+
+# shell files
+/shell.sh
+# Nix
+result
+result-*
+.direnv/
+
+#kilocode
.kilo/
\ No newline at end of file
diff --git a/docs/engineering/windows-native-recorder-roadmap.md b/docs/engineering/windows-native-recorder-roadmap.md
new file mode 100644
index 0000000..c6a0a06
--- /dev/null
+++ b/docs/engineering/windows-native-recorder-roadmap.md
@@ -0,0 +1,202 @@
+# Windows Native Recorder Roadmap
+
+OpenScreen's Windows recorder should be owned by one native backend. Electron capture can remain available for non-Windows platforms and temporary developer diagnostics, but Windows production recording should not silently fall back to `getDisplayMedia` / `MediaRecorder`.
+
+## Goals
+
+- Capture displays and windows through Windows Graphics Capture (WGC).
+- Render the native Windows cursor as OpenScreen's high-quality scalable cursor overlay.
+- Capture system audio through WASAPI loopback.
+- Capture microphone audio through WASAPI.
+- Mix system audio and microphone audio into the primary screen recording.
+- Capture webcam video natively and keep it as a separate editable OpenScreen media stream.
+- Keep preview/export aligned because screen video, audio, webcam, and cursor share one native timing origin.
+- Keep exported MP4s Windows-friendly: H.264 video plus AAC audio. Opus-in-MP4 is not an acceptable Windows export target.
+- Package the native helper with the Windows app.
+
+## Non-Goals
+
+- Replacing the editor/export pipeline.
+- Flattening webcam into the screen recording. The editor currently treats webcam as editable picture-in-picture media, so the native recorder should preserve a separate `webcamVideoPath`.
+- Adding a native fallback for macOS or Linux in this branch.
+
+## Target Architecture
+
+The renderer keeps the existing recording controls. On Windows, `useScreenRecorder` sends a complete recording request to Electron and does not assemble Windows `MediaStream` tracks with `MediaRecorder`.
+
+Electron owns the native recording session:
+
+- resolves the selected source;
+- resolves output paths;
+- starts cursor sampling;
+- starts the helper process;
+- sends pause/resume/stop/cancel commands;
+- writes `RecordingSession` manifests;
+- reports explicit errors when a Windows-native capability is unavailable.
+
+The helper owns Windows media capture:
+
+- WGC screen/window frames;
+- WASAPI system loopback;
+- WASAPI microphone input;
+- Media Foundation webcam capture;
+- Media Foundation encoding/muxing;
+- stream timestamp normalization.
+
+## Helper Contract V2
+
+The helper receives a single JSON argument:
+
+```json
+{
+ "schemaVersion": 2,
+ "recordingId": 1234567890,
+ "source": {
+ "type": "display",
+ "sourceId": "screen:0:0",
+ "displayId": 123,
+ "windowHandle": null,
+ "bounds": { "x": 0, "y": 0, "width": 1920, "height": 1080 }
+ },
+ "video": {
+ "fps": 60,
+ "width": 1920,
+ "height": 1080,
+ "bitrate": 18000000
+ },
+ "audio": {
+ "system": { "enabled": true },
+ "microphone": { "enabled": true, "deviceId": "default", "gain": 1.4 }
+ },
+ "webcam": {
+ "enabled": true,
+ "deviceId": "default",
+ "width": 1280,
+ "height": 720,
+ "fps": 30,
+ "bitrate": 18000000
+ },
+ "outputs": {
+ "screenPath": "C:\\Users\\me\\recording-123.mp4",
+ "webcamPath": "C:\\Users\\me\\recording-123-webcam.mp4",
+ "manifestPath": "C:\\Users\\me\\recording-123.session.json"
+ }
+}
+```
+
+The helper emits newline-delimited JSON events to stdout:
+
+```json
+{ "event": "ready", "schemaVersion": 2 }
+{ "event": "recording-started", "timestampMs": 1234567890 }
+{ "event": "warning", "code": "audio-device-unavailable", "message": "..." }
+{ "event": "recording-stopped", "screenPath": "...", "webcamPath": "..." }
+{ "event": "error", "code": "unsupported-window-source", "message": "..." }
+```
+
+During migration, Electron also accepts the current textual helper messages so existing display-only smoke tests keep working.
+
+## Implementation Phases
+
+### 1. Native Session Boundary
+
+- Add a structured Windows native recording request type.
+- Pass source kind, audio flags, microphone device, webcam flags, and output paths into the helper.
+- On Windows, do not silently fall back to Electron capture. If the helper is unavailable or a native feature is missing, show a clear error.
+- Keep Electron fallback only for non-Windows and optional developer diagnostics.
+
+Acceptance:
+
+- Display-only recording still works.
+- Enabling an unsupported native feature returns an explicit native error instead of recording through Electron.
+
+### 2. WASAPI System Audio
+
+Status: initial implementation landed. The helper captures the default render endpoint with WASAPI loopback, passes the runtime mix format into `MFEncoder`, and muxes AAC audio into the primary MP4. Long-run drift correction and explicit silence insertion remain follow-up hardening work.
+
+- Add `WasapiLoopbackCapture`.
+- Capture the default render endpoint in shared loopback mode.
+- Keep `WasapiLoopbackCapture` responsible only for device activation, packet capture, and packet timestamps.
+- Keep `MFEncoder` responsible for all Media Foundation stream definitions and muxing.
+- Feed the endpoint mix format into `MFEncoder` as the single source of truth for audio stream shape: sample rate, channel count, bits per sample, block alignment, average bytes/sec, and subtype (`PCM` or `Float`).
+- Encode the primary screen MP4 with H.264 video and AAC audio through one `IMFSinkWriter`.
+- Timestamp audio from the captured frame count in 100ns units. The first implementation uses the WASAPI packet timeline; later drift correction will add explicit silence or resampling if long recordings show measurable clock skew.
+- Treat microphone mixing as a later phase. System loopback must land first without introducing renderer-side audio code.
+
+Acceptance:
+
+- Screen MP4 has an AAC audio track when system audio is enabled.
+- A 5-minute recording has audio/video duration drift below one frame.
+
+SSOT rules for this phase:
+
+- `src/lib/nativeWindowsRecording.ts` is the renderer/main TypeScript request contract.
+- `docs/engineering/windows-native-recorder-roadmap.md` is the feature-level contract and phase checklist.
+- `WgcSession::captureWidth()/captureHeight()` is the encoded screen frame size until a dedicated native scaling stage exists.
+- `WasapiLoopbackCapture::inputFormat()` is the runtime audio format source used by `MFEncoder`.
+- No duplicated hard-coded audio format assumptions in `main.cpp`.
+
+### 3. WASAPI Microphone
+
+- Add microphone device enumeration and stable device-id mapping.
+- Capture selected/default microphone through WASAPI.
+- Apply OpenScreen's current mic gain policy.
+- Mix microphone and system audio before AAC encoding.
+
+Acceptance:
+
+- Mic-only, system-only, and mixed audio recordings produce a valid AAC track.
+- Device unplug/permission failure produces an explicit error or warning.
+
+### 4. Webcam Capture
+
+- Add Media Foundation webcam source reader.
+- Select 1280x720/30fps or nearest supported format.
+- Encode webcam to `recording--webcam.mp4`.
+- Synchronize webcam timestamps to the native session clock.
+- Store `webcamVideoPath` in the OpenScreen session manifest.
+
+Acceptance:
+
+- Editor loads the native screen recording and the native webcam recording.
+- Webcam layout controls behave the same as today.
+
+### 5. Native Window Capture
+
+- Resolve Electron `window:*` selections to an `HWND`.
+- Use WGC `CreateForWindow(HWND)`.
+- Handle window close, minimize, resize, DPI scaling, and monitor moves.
+- Return clear errors for unsupported protected windows.
+
+Acceptance:
+
+- Capturing a normal app window works with cursor/audio/mic/webcam.
+- Window resize and movement do not corrupt the recording.
+
+### 6. Runtime Controls
+
+- Add pause/resume commands to the helper.
+- Add cancel command that removes partial screen/webcam outputs.
+- Keep restart as stop-discard-start from Electron until the helper supports a native restart event.
+
+Acceptance:
+
+- Pause/resume keeps preview duration coherent.
+- Cancel leaves no stale media/session/cursor files.
+
+### 7. Test Pipeline
+
+- `npm run test:wgc-helper:win`: display-only helper smoke test.
+- `npm run test:wgc-audio:win`: validates AAC track presence and duration.
+- `npm run test:wgc-window:win`: captures a fixture window by HWND.
+- `npm run test:wgc-webcam:win`: validates webcam output when a webcam is available, otherwise skips explicitly.
+- Packaging check: confirms the helper is in `app.asar.unpacked`.
+- Export check: exported MP4s generated from native recordings keep an AAC audio track when the source has audio.
+
+## Ship Criteria
+
+- Windows display capture works with cursor, system audio, microphone, and webcam.
+- Windows window capture works with cursor, system audio, microphone, and webcam.
+- Preview and export show no cursor position drift.
+- Preview and export show no measurable audio/video/webcam drift.
+- Windows production builds do not depend on Electron capture fallback.
diff --git a/docs/testing/windows-native-cursor.md b/docs/testing/windows-native-cursor.md
index 1abe04e..23c57a9 100644
--- a/docs/testing/windows-native-cursor.md
+++ b/docs/testing/windows-native-cursor.md
@@ -83,3 +83,37 @@ Together, the scripts make it quick to inspect:
- whether the real OpenScreen preview renders the same cursor behavior as the diagnostic pipeline
They are not a full substitute for an end-to-end manual recording pass. Before shipping cursor changes, also test a real capture session and export from the packaged app.
+
+## Native Windows capture backend
+
+The app now routes Windows recordings through an external WGC helper instead of Electron `getDisplayMedia`. This is meant to remove the coordinate and clock split that made the reconstructed cursor drift in the preview/export path.
+
+Current native availability rules:
+
+- Windows 10 build 19041 or newer
+- a helper executable is available
+
+The helper currently implements display video capture and system audio loopback. Window capture, microphone audio, and webcam capture are part of the native recorder roadmap and fail explicitly instead of silently falling back to Electron capture on Windows.
+
+Build OpenScreen's helper locally:
+
+```powershell
+npm run build:native:win
+```
+
+Smoke-test the helper directly:
+
+```powershell
+npm run test:wgc-helper:win
+npm run test:wgc-audio:win
+```
+
+For local diagnostics with another compatible helper, point OpenScreen at that executable:
+
+```powershell
+$env:OPENSCREEN_WGC_CAPTURE_EXE = "C:\path\to\wgc-capture.exe"
+npm run build-vite
+npm run dev
+```
+
+The helper receives one JSON config argument, emits JSON lifecycle events, prints the legacy `Recording started` marker, accepts `stop` on stdin, and prints `Recording stopped. Output path: `. See `electron/native/README.md` for the exact contract and build output paths.
diff --git a/electron-builder.json5 b/electron-builder.json5
index 372cdf7..c11dc56 100644
--- a/electron-builder.json5
+++ b/electron-builder.json5
@@ -4,10 +4,11 @@
"appId": "com.siddharthvaddem.openscreen",
"asar": true,
// .node binaries can't be dlopen'd from inside an asar — must live unpacked.
- "asarUnpack": [
- "node_modules/uiohook-napi/**/*",
- "**/*.node"
- ],
+ "asarUnpack": [
+ "node_modules/uiohook-napi/**/*",
+ "**/*.node",
+ "electron/native/bin/**"
+ ],
"productName": "Openscreen",
"npmRebuild": true,
"buildDependenciesFromSource": true,
@@ -15,10 +16,11 @@
"directories": {
"output": "release/${version}"
},
- "files": [
- "dist",
- "dist-electron",
- "!*.png",
+ "files": [
+ "dist",
+ "dist-electron",
+ "electron/native/bin/**/*",
+ "!*.png",
"!preview*.png",
"!*.md",
"!README.md",
diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts
index 686575d..eb28420 100644
--- a/electron/electron-env.d.ts
+++ b/electron/electron-env.d.ts
@@ -72,6 +72,24 @@ interface Window {
error?: string;
}>;
setRecordingState: (recording: boolean, recordingId?: number) => Promise;
+ isNativeWindowsCaptureAvailable: () => Promise<{
+ success: boolean;
+ available: boolean;
+ helperPath?: string;
+ reason?: string;
+ error?: string;
+ }>;
+ startNativeWindowsRecording: (
+ request: import("../src/lib/nativeWindowsRecording").NativeWindowsRecordingRequest,
+ ) => Promise;
+ stopNativeWindowsRecording: (discard?: boolean) => Promise<{
+ success: boolean;
+ path?: string;
+ session?: import("../src/lib/recordingSession").RecordingSession;
+ message?: string;
+ discarded?: boolean;
+ error?: string;
+ }>;
discardCursorTelemetry: (recordingId: number) => Promise;
getCursorTelemetry: (videoPath?: string) => Promise<{
success: boolean;
diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts
index 4c306ee..de3b6b5 100644
--- a/electron/ipc/handlers.ts
+++ b/electron/ipc/handlers.ts
@@ -1,3 +1,5 @@
+import { type ChildProcessWithoutNullStreams, spawn } from "node:child_process";
+import { constants as fsConstants } from "node:fs";
import fs from "node:fs/promises";
import { createRequire } from "node:module";
import os from "node:os";
@@ -14,6 +16,7 @@ import {
shell,
systemPreferences,
} from "electron";
+import type { NativeWindowsRecordingRequest } from "../../src/lib/nativeWindowsRecording";
import {
normalizeProjectMedia,
normalizeRecordingSession,
@@ -36,6 +39,7 @@ import { registerNativeBridgeHandlers } from "./nativeBridge";
const PROJECT_FILE_EXTENSION = "openscreen";
const SHORTCUTS_FILE = path.join(app.getPath("userData"), "shortcuts.json");
+const RECORDING_FILE_PREFIX = "recording-";
const RECORDING_SESSION_SUFFIX = ".session.json";
const ALLOWED_IMPORT_VIDEO_EXTENSIONS = new Set([".webm", ".mp4", ".mov", ".avi", ".mkv"]);
@@ -250,6 +254,12 @@ const MAX_CURSOR_SAMPLES = 60 * 60 * 30; // 1 hour @ 30Hz
let cursorRecordingSession: CursorRecordingSession | null = null;
let pendingCursorRecordingData: CursorRecordingData | null = null;
+let nativeWindowsCaptureProcess: ChildProcessWithoutNullStreams | null = null;
+let nativeWindowsCaptureOutput = "";
+let nativeWindowsCaptureTargetPath: string | null = null;
+let nativeWindowsCaptureWebcamTargetPath: string | null = null;
+let nativeWindowsCaptureRecordingId: number | null = null;
+let nativeWindowsCursorOffsetMs = 0;
function normalizeCursorSample(sample: unknown): CursorRecordingSample | null {
if (!sample || typeof sample !== "object") {
@@ -404,6 +414,215 @@ function getSelectedSourceId() {
return typeof selectedSource?.id === "string" ? selectedSource.id : null;
}
+function getSelectedDisplay() {
+ const sourceDisplayId = Number(selectedSource?.display_id);
+ if (!Number.isFinite(sourceDisplayId)) {
+ return null;
+ }
+
+ return screen.getAllDisplays().find((display) => display.id === sourceDisplayId) ?? null;
+}
+
+function resolveUnpackedAppPath(...segments: string[]) {
+ const resolved = path.join(app.getAppPath(), ...segments);
+ if (app.isPackaged) {
+ return resolved.replace(/\.asar([/\\])/, ".asar.unpacked$1");
+ }
+
+ return resolved;
+}
+
+function getNativeWindowsCaptureHelperCandidates() {
+ const envPath = process.env.OPENSCREEN_WGC_CAPTURE_EXE?.trim();
+ const archTag = process.arch === "arm64" ? "win32-arm64" : "win32-x64";
+ return [
+ envPath,
+ resolveUnpackedAppPath(
+ "electron",
+ "native",
+ "wgc-capture",
+ "build",
+ "Release",
+ "wgc-capture.exe",
+ ),
+ resolveUnpackedAppPath("electron", "native", "wgc-capture", "build", "wgc-capture.exe"),
+ resolveUnpackedAppPath("electron", "native", "bin", archTag, "wgc-capture.exe"),
+ ].filter((candidate): candidate is string => Boolean(candidate));
+}
+
+async function findNativeWindowsCaptureHelperPath() {
+ if (process.platform !== "win32") {
+ return null;
+ }
+
+ for (const candidate of getNativeWindowsCaptureHelperCandidates()) {
+ try {
+ await fs.access(candidate, fsConstants.X_OK);
+ return candidate;
+ } catch {
+ // Try the next configured helper location.
+ }
+ }
+
+ return null;
+}
+
+function isWindowsGraphicsCaptureOsSupported() {
+ if (process.platform !== "win32") {
+ return false;
+ }
+
+ const [, , build] = process.getSystemVersion().split(".").map(Number);
+ return Number.isFinite(build) && build >= 19041;
+}
+
+async function startCursorRecording(recordingId?: number) {
+ if (cursorRecordingSession) {
+ pendingCursorRecordingData = await cursorRecordingSession.stop();
+ cursorRecordingSession = null;
+ }
+
+ pendingCursorRecordingData = null;
+ cursorRecordingSession = createCursorRecordingSession({
+ getDisplayBounds: getSelectedSourceBounds,
+ maxSamples: MAX_CURSOR_SAMPLES,
+ platform: process.platform,
+ sampleIntervalMs: CURSOR_SAMPLE_INTERVAL_MS,
+ sourceId: getSelectedSourceId(),
+ startTimeMs:
+ typeof recordingId === "number" && Number.isFinite(recordingId) ? recordingId : undefined,
+ });
+
+ try {
+ await cursorRecordingSession.start();
+ } catch (error) {
+ console.error("Failed to start cursor recording session:", error);
+ cursorRecordingSession = null;
+ }
+}
+
+async function stopCursorRecording() {
+ if (!cursorRecordingSession) {
+ return;
+ }
+
+ try {
+ pendingCursorRecordingData = await cursorRecordingSession.stop();
+ } catch (error) {
+ console.error("Failed to stop cursor recording session:", error);
+ pendingCursorRecordingData = null;
+ } finally {
+ cursorRecordingSession = null;
+ }
+}
+
+async function writePendingCursorTelemetry(videoPath: string) {
+ const telemetryPath = `${videoPath}.cursor.json`;
+ if (pendingCursorRecordingData && pendingCursorRecordingData.samples.length > 0) {
+ await fs.writeFile(telemetryPath, JSON.stringify(pendingCursorRecordingData, null, 2), "utf-8");
+ }
+ pendingCursorRecordingData = null;
+}
+
+function shiftPendingCursorTelemetry(offsetMs: number) {
+ if (!pendingCursorRecordingData || !Number.isFinite(offsetMs) || offsetMs <= 0) {
+ return;
+ }
+
+ pendingCursorRecordingData = {
+ ...pendingCursorRecordingData,
+ samples: pendingCursorRecordingData.samples
+ .map((sample) => ({
+ ...sample,
+ timeMs: Math.max(0, sample.timeMs - offsetMs),
+ }))
+ .sort((a, b) => a.timeMs - b.timeMs),
+ };
+}
+
+function waitForNativeWindowsCaptureStart(proc: ChildProcessWithoutNullStreams) {
+ return new Promise((resolve, reject) => {
+ const timer = setTimeout(() => {
+ cleanup();
+ reject(new Error("Timed out waiting for native Windows capture to start"));
+ }, 12000);
+
+ const onOutput = (chunk: Buffer) => {
+ nativeWindowsCaptureOutput += chunk.toString();
+ if (nativeWindowsCaptureOutput.includes("Recording started")) {
+ cleanup();
+ resolve();
+ }
+ };
+ const onError = (error: Error) => {
+ cleanup();
+ reject(error);
+ };
+ const onExit = (code: number | null) => {
+ cleanup();
+ reject(
+ new Error(
+ nativeWindowsCaptureOutput.trim() ||
+ `Native Windows capture exited before recording started (code=${code ?? "unknown"})`,
+ ),
+ );
+ };
+ const cleanup = () => {
+ clearTimeout(timer);
+ proc.stdout.off("data", onOutput);
+ proc.stderr.off("data", onOutput);
+ proc.off("error", onError);
+ proc.off("exit", onExit);
+ };
+
+ proc.stdout.on("data", onOutput);
+ proc.stderr.on("data", onOutput);
+ proc.once("error", onError);
+ proc.once("exit", onExit);
+ });
+}
+
+function waitForNativeWindowsCaptureStop(proc: ChildProcessWithoutNullStreams) {
+ return new Promise((resolve, reject) => {
+ const onOutput = (chunk: Buffer) => {
+ nativeWindowsCaptureOutput += chunk.toString();
+ };
+ const onClose = (code: number | null) => {
+ cleanup();
+ const match = nativeWindowsCaptureOutput.match(/Recording stopped\. Output path: (.+)/);
+ if (match?.[1]) {
+ resolve(match[1].trim());
+ return;
+ }
+ if (code === 0 && nativeWindowsCaptureTargetPath) {
+ resolve(nativeWindowsCaptureTargetPath);
+ return;
+ }
+ reject(
+ new Error(
+ nativeWindowsCaptureOutput.trim() ||
+ `Native Windows capture exited with code=${code ?? "unknown"}`,
+ ),
+ );
+ };
+ const onError = (error: Error) => {
+ cleanup();
+ reject(error);
+ };
+ const cleanup = () => {
+ proc.stdout.off("data", onOutput);
+ proc.stderr.off("data", onOutput);
+ proc.off("close", onClose);
+ proc.off("error", onError);
+ };
+
+ proc.stdout.on("data", onOutput);
+ proc.stderr.on("data", onOutput);
+ proc.once("close", onClose);
+ proc.once("error", onError);
+ });
+}
+
function setCurrentRecordingSessionState(session: RecordingSession | null) {
currentRecordingSession = session;
currentVideoPath = session?.screenVideoPath ?? null;
@@ -412,10 +631,10 @@ function setCurrentRecordingSessionState(session: RecordingSession | null) {
export function registerIpcHandlers(
createEditorWindow: () => void,
createSourceSelectorWindow: () => BrowserWindow,
- _createCountdownOverlayWindow: () => BrowserWindow,
+ createCountdownOverlayWindow: () => BrowserWindow,
getMainWindow: () => BrowserWindow | null,
getSourceSelectorWindow: () => BrowserWindow | null,
- _getCountdownOverlayWindow?: () => BrowserWindow | null,
+ getCountdownOverlayWindow?: () => BrowserWindow | null,
onRecordingStateChange?: (recording: boolean, sourceName: string) => void,
_switchToHud?: () => void,
) {
@@ -553,6 +772,282 @@ export function registerIpcHandlers(
createEditorWindow();
});
+ ipcMain.handle("countdown-overlay-show", async (_, value: number, runId: number) => {
+ const overlayWindow = getCountdownOverlayWindow?.() ?? createCountdownOverlayWindow();
+ if (overlayWindow.isDestroyed()) {
+ return;
+ }
+
+ if (!overlayWindow.isVisible()) {
+ overlayWindow.showInactive();
+ }
+
+ if (overlayWindow.webContents.isLoading()) {
+ await new Promise((resolve) => {
+ overlayWindow.webContents.once("did-finish-load", () => resolve());
+ });
+ }
+
+ overlayWindow.webContents.send("countdown-overlay-value", value, runId);
+ });
+
+ ipcMain.handle("countdown-overlay-set-value", (_, value: number, runId: number) => {
+ const overlayWindow = getCountdownOverlayWindow?.();
+ if (!overlayWindow || overlayWindow.isDestroyed()) {
+ return;
+ }
+
+ overlayWindow.webContents.send("countdown-overlay-value", value, runId);
+ });
+
+ ipcMain.handle("countdown-overlay-hide", (_, runId: number) => {
+ const overlayWindow = getCountdownOverlayWindow?.();
+ if (!overlayWindow || overlayWindow.isDestroyed()) {
+ return;
+ }
+
+ overlayWindow.webContents.send("countdown-overlay-value", null, runId);
+ overlayWindow.hide();
+ });
+
+ ipcMain.handle("is-native-windows-capture-available", async () => {
+ if (!isWindowsGraphicsCaptureOsSupported()) {
+ return { success: true, available: false, reason: "unsupported-os" };
+ }
+
+ const helperPath = await findNativeWindowsCaptureHelperPath();
+ return helperPath
+ ? { success: true, available: true, helperPath }
+ : { success: true, available: false, reason: "missing-helper" };
+ });
+
+ ipcMain.handle(
+ "start-native-windows-recording",
+ async (_, request: NativeWindowsRecordingRequest) => {
+ try {
+ if (!isWindowsGraphicsCaptureOsSupported()) {
+ return {
+ success: false,
+ error: "Windows Graphics Capture requires Windows 10 build 19041 or newer.",
+ };
+ }
+ if (nativeWindowsCaptureProcess) {
+ return { success: false, error: "Native Windows capture is already running." };
+ }
+
+ const helperPath = await findNativeWindowsCaptureHelperPath();
+ if (!helperPath) {
+ return { success: false, error: "Native Windows capture helper is not available." };
+ }
+
+ if (!request?.source?.sourceId) {
+ return {
+ success: false,
+ error: "Native Windows capture request is missing a source.",
+ };
+ }
+
+ const recordingId =
+ typeof request.recordingId === "number" && Number.isFinite(request.recordingId)
+ ? request.recordingId
+ : Date.now();
+ const outputPath = path.join(RECORDINGS_DIR, `${RECORDING_FILE_PREFIX}${recordingId}.mp4`);
+ const webcamOutputPath = path.join(
+ RECORDINGS_DIR,
+ `${RECORDING_FILE_PREFIX}${recordingId}-webcam.mp4`,
+ );
+ const sourceDisplay =
+ request.source.type === "display" && typeof request.source.displayId === "number"
+ ? (screen.getAllDisplays().find((display) => display.id === request.source.displayId) ??
+ null)
+ : getSelectedDisplay();
+ const bounds = sourceDisplay?.bounds ?? getSelectedSourceBounds();
+ const displayId =
+ typeof request.source.displayId === "number" && Number.isFinite(request.source.displayId)
+ ? request.source.displayId
+ : Number(selectedSource?.display_id);
+ const config = {
+ schemaVersion: 2,
+ recordingId,
+ outputPath,
+ sourceType: request.source.type,
+ sourceId: request.source.sourceId,
+ displayId: Number.isFinite(displayId) ? displayId : 0,
+ windowHandle: request.source.windowHandle ?? null,
+ fps: request.video.fps,
+ videoWidth: request.video.width,
+ videoHeight: request.video.height,
+ displayX: bounds.x,
+ displayY: bounds.y,
+ displayW: bounds.width,
+ displayH: bounds.height,
+ hasDisplayBounds: true,
+ captureSystemAudio: request.audio.system.enabled,
+ captureMic: request.audio.microphone.enabled,
+ microphoneDeviceId: request.audio.microphone.deviceId ?? null,
+ microphoneGain: request.audio.microphone.gain,
+ webcamEnabled: request.webcam.enabled,
+ webcamDeviceId: request.webcam.deviceId ?? null,
+ webcamWidth: request.webcam.width,
+ webcamHeight: request.webcam.height,
+ webcamFps: request.webcam.fps,
+ outputs: {
+ screenPath: outputPath,
+ webcamPath: webcamOutputPath,
+ },
+ source: {
+ type: request.source.type,
+ sourceId: request.source.sourceId,
+ displayId: Number.isFinite(displayId) ? displayId : null,
+ windowHandle: request.source.windowHandle ?? null,
+ bounds,
+ },
+ video: request.video,
+ audio: request.audio,
+ webcam: request.webcam,
+ };
+
+ console.info("[native-wgc] starting Windows capture", {
+ helperPath,
+ source: request.source,
+ audio: request.audio,
+ webcam: request.webcam,
+ bounds,
+ sourceId: selectedSource?.id ?? null,
+ usedDisplayMatch: Boolean(sourceDisplay),
+ outputPath,
+ });
+
+ await fs.mkdir(RECORDINGS_DIR, { recursive: true });
+ nativeWindowsCaptureOutput = "";
+ nativeWindowsCaptureTargetPath = outputPath;
+ nativeWindowsCaptureWebcamTargetPath = request.webcam.enabled ? webcamOutputPath : null;
+ nativeWindowsCaptureRecordingId = recordingId;
+ nativeWindowsCursorOffsetMs = 0;
+
+ const cursorStartTimeMs = Date.now();
+ await startCursorRecording(cursorStartTimeMs);
+ console.info("[native-wgc] cursor sampler ready", {
+ cursorStartTimeMs,
+ warmupMs: Date.now() - cursorStartTimeMs,
+ });
+
+ const proc = spawn(helperPath, [JSON.stringify(config)], {
+ cwd: RECORDINGS_DIR,
+ stdio: ["pipe", "pipe", "pipe"],
+ windowsHide: true,
+ });
+ nativeWindowsCaptureProcess = proc;
+
+ await waitForNativeWindowsCaptureStart(proc);
+ const captureStartedAtMs = Date.now();
+ nativeWindowsCursorOffsetMs = Math.max(0, captureStartedAtMs - cursorStartTimeMs);
+ console.info("[native-wgc] capture started", {
+ captureStartedAtMs,
+ cursorOffsetMs: nativeWindowsCursorOffsetMs,
+ });
+
+ const source = selectedSource || { name: "Screen" };
+ if (onRecordingStateChange) {
+ onRecordingStateChange(true, source.name);
+ }
+
+ return {
+ success: true,
+ recordingId,
+ path: outputPath,
+ helperPath,
+ };
+ } catch (error) {
+ console.error("Failed to start native Windows recording:", error);
+ nativeWindowsCaptureProcess?.kill();
+ nativeWindowsCaptureProcess = null;
+ nativeWindowsCaptureTargetPath = null;
+ nativeWindowsCaptureWebcamTargetPath = null;
+ nativeWindowsCaptureRecordingId = null;
+ nativeWindowsCursorOffsetMs = 0;
+ await stopCursorRecording();
+ return { success: false, error: String(error) };
+ }
+ },
+ );
+
+ ipcMain.handle("stop-native-windows-recording", async (_, discard?: boolean) => {
+ const proc = nativeWindowsCaptureProcess;
+ const preferredPath = nativeWindowsCaptureTargetPath;
+ const preferredWebcamPath = nativeWindowsCaptureWebcamTargetPath;
+ const recordingId = nativeWindowsCaptureRecordingId ?? Date.now();
+
+ if (!proc) {
+ return { success: false, error: "Native Windows capture is not running." };
+ }
+
+ try {
+ const stoppedPathPromise = waitForNativeWindowsCaptureStop(proc);
+ proc.stdin.write("stop\n");
+ const stoppedPath = await stoppedPathPromise;
+ const screenVideoPath = stoppedPath || preferredPath;
+ if (!screenVideoPath) {
+ throw new Error("Native Windows capture did not return an output path.");
+ }
+
+ await stopCursorRecording();
+ if (discard) {
+ pendingCursorRecordingData = null;
+ await Promise.all([
+ fs.rm(screenVideoPath, { force: true }),
+ preferredWebcamPath ? fs.rm(preferredWebcamPath, { force: true }) : Promise.resolve(),
+ fs.rm(`${screenVideoPath}.cursor.json`, { force: true }),
+ ]);
+ return { success: true, discarded: true };
+ }
+
+ shiftPendingCursorTelemetry(nativeWindowsCursorOffsetMs);
+ await writePendingCursorTelemetry(screenVideoPath);
+ let webcamVideoPath: string | undefined;
+ if (preferredWebcamPath) {
+ try {
+ await fs.access(preferredWebcamPath, fsConstants.R_OK);
+ webcamVideoPath = preferredWebcamPath;
+ } catch {
+ webcamVideoPath = undefined;
+ }
+ }
+ const session: RecordingSession = webcamVideoPath
+ ? { screenVideoPath, webcamVideoPath, createdAt: recordingId }
+ : { screenVideoPath, createdAt: recordingId };
+ setCurrentRecordingSessionState(session);
+ currentProjectPath = null;
+
+ const sessionManifestPath = path.join(
+ RECORDINGS_DIR,
+ `${path.parse(screenVideoPath).name}${RECORDING_SESSION_SUFFIX}`,
+ );
+ await fs.writeFile(sessionManifestPath, JSON.stringify(session, null, 2), "utf-8");
+
+ return {
+ success: true,
+ path: screenVideoPath,
+ session,
+ message: "Native Windows recording session stored successfully",
+ };
+ } catch (error) {
+ console.error("Failed to stop native Windows recording:", error);
+ await stopCursorRecording();
+ return { success: false, error: String(error) };
+ } finally {
+ nativeWindowsCaptureProcess = null;
+ nativeWindowsCaptureTargetPath = null;
+ nativeWindowsCaptureWebcamTargetPath = null;
+ nativeWindowsCaptureRecordingId = null;
+ nativeWindowsCursorOffsetMs = 0;
+ const source = selectedSource || { name: "Screen" };
+ if (onRecordingStateChange) {
+ onRecordingStateChange(false, source.name);
+ }
+ }
+ });
+
ipcMain.handle("store-recorded-session", async (_, payload: StoreRecordedSessionInput) => {
try {
return await storeRecordedSessionFiles(payload);
@@ -586,15 +1081,7 @@ export function registerIpcHandlers(
setCurrentRecordingSessionState(session);
currentProjectPath = null;
- const telemetryPath = `${screenVideoPath}.cursor.json`;
- if (pendingCursorRecordingData && pendingCursorRecordingData.samples.length > 0) {
- await fs.writeFile(
- telemetryPath,
- JSON.stringify(pendingCursorRecordingData, null, 2),
- "utf-8",
- );
- }
- pendingCursorRecordingData = null;
+ await writePendingCursorTelemetry(screenVideoPath);
const sessionManifestPath = path.join(
RECORDINGS_DIR,
@@ -653,39 +1140,9 @@ export function registerIpcHandlers(
ipcMain.handle("set-recording-state", async (_, recording: boolean, recordingId?: number) => {
if (recording) {
- if (cursorRecordingSession) {
- pendingCursorRecordingData = await cursorRecordingSession.stop();
- cursorRecordingSession = null;
- }
-
- pendingCursorRecordingData = null;
- cursorRecordingSession = createCursorRecordingSession({
- getDisplayBounds: getSelectedSourceBounds,
- maxSamples: MAX_CURSOR_SAMPLES,
- platform: process.platform,
- sampleIntervalMs: CURSOR_SAMPLE_INTERVAL_MS,
- sourceId: getSelectedSourceId(),
- startTimeMs:
- typeof recordingId === "number" && Number.isFinite(recordingId) ? recordingId : undefined,
- });
-
- try {
- await cursorRecordingSession.start();
- } catch (error) {
- console.error("Failed to start cursor recording session:", error);
- cursorRecordingSession = null;
- }
+ await startCursorRecording(recordingId);
} else {
- if (cursorRecordingSession) {
- try {
- pendingCursorRecordingData = await cursorRecordingSession.stop();
- } catch (error) {
- console.error("Failed to stop cursor recording session:", error);
- pendingCursorRecordingData = null;
- } finally {
- cursorRecordingSession = null;
- }
- }
+ await stopCursorRecording();
}
const source = selectedSource || { name: "Screen" };
diff --git a/electron/native/README.md b/electron/native/README.md
new file mode 100644
index 0000000..512517b
--- /dev/null
+++ b/electron/native/README.md
@@ -0,0 +1,50 @@
+# Native capture helpers
+
+Windows native recording is resolved from one of these locations:
+
+1. `OPENSCREEN_WGC_CAPTURE_EXE`, for local development and diagnostics.
+2. `electron/native/wgc-capture/build/wgc-capture.exe`, for a locally built Ninja helper.
+3. `electron/native/wgc-capture/build/Release/wgc-capture.exe`, for a locally built multi-config helper.
+4. `electron/native/bin/win32-x64/wgc-capture.exe` or `electron/native/bin/win32-arm64/wgc-capture.exe`, for packaged prebuilt helpers.
+
+Build the Windows helper with:
+
+```powershell
+npm run build:native:win
+```
+
+The build writes the CMake output to `electron/native/wgc-capture/build/wgc-capture.exe` and copies the redistributable binary to `electron/native/bin/win32-x64/wgc-capture.exe`.
+
+The helper contract is process-based: the app starts the process with one JSON argument and sends commands on stdin. `stop\n` finalizes the recording. During migration the helper prints both newline-delimited JSON events and the legacy text messages `Recording started` / `Recording stopped. Output path: `.
+
+Current V2 JSON shape:
+
+```json
+{
+ "schemaVersion": 2,
+ "recordingId": 123,
+ "sourceType": "display",
+ "sourceId": "screen:0:0",
+ "displayId": 1,
+ "outputPath": "C:\\path\\recording-123.mp4",
+ "videoWidth": 1920,
+ "videoHeight": 1080,
+ "fps": 60,
+ "captureSystemAudio": false,
+ "captureMic": false,
+ "webcamEnabled": false,
+ "outputs": {
+ "screenPath": "C:\\path\\recording-123.mp4",
+ "webcamPath": "C:\\path\\recording-123-webcam.mp4"
+ }
+}
+```
+
+The current helper implementation supports display video capture and system audio loopback. Microphone, webcam, and window capture now fail explicitly in the helper rather than silently falling back to Electron capture on Windows. See `docs/engineering/windows-native-recorder-roadmap.md` for the phased implementation plan.
+
+Smoke-test the helper with:
+
+```powershell
+npm run test:wgc-helper:win
+npm run test:wgc-audio:win
+```
diff --git a/electron/native/wgc-capture/CMakeLists.txt b/electron/native/wgc-capture/CMakeLists.txt
new file mode 100644
index 0000000..76999f7
--- /dev/null
+++ b/electron/native/wgc-capture/CMakeLists.txt
@@ -0,0 +1,45 @@
+cmake_minimum_required(VERSION 3.20)
+
+# The local Windows SDK image used by some contributors can miss gdi32.lib,
+# while CMake's default MSVC console template links it unconditionally. This
+# helper does not use GDI, so keep the standard library set minimal and explicit.
+set(CMAKE_CXX_STANDARD_LIBRARIES
+ "kernel32.lib user32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib comdlg32.lib advapi32.lib"
+ CACHE STRING "" FORCE)
+
+project(openscreen-wgc-capture LANGUAGES CXX)
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+
+add_executable(wgc-capture
+ src/main.cpp
+ src/mf_encoder.cpp
+ src/mf_encoder.h
+ src/monitor_utils.cpp
+ src/monitor_utils.h
+ src/wasapi_loopback_capture.cpp
+ src/wasapi_loopback_capture.h
+ src/wgc_session.cpp
+ src/wgc_session.h
+)
+
+target_compile_definitions(wgc-capture PRIVATE
+ NOMINMAX
+ WIN32_LEAN_AND_MEAN
+ _WIN32_WINNT=0x0A00
+)
+
+target_compile_options(wgc-capture PRIVATE /EHsc /W4 /utf-8)
+
+target_link_libraries(wgc-capture PRIVATE
+ d3d11
+ dxgi
+ mf
+ mfplat
+ mfreadwrite
+ mfuuid
+ runtimeobject
+ windowsapp
+)
diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp
new file mode 100644
index 0000000..39d5c62
--- /dev/null
+++ b/electron/native/wgc-capture/src/main.cpp
@@ -0,0 +1,433 @@
+#include "mf_encoder.h"
+#include "monitor_utils.h"
+#include "wasapi_loopback_capture.h"
+#include "wgc_session.h"
+
+#include
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+namespace {
+
+struct CaptureConfig {
+ int schemaVersion = 1;
+ int64_t displayId = 0;
+ int64_t recordingId = 0;
+ std::string sourceType = "display";
+ std::string sourceId;
+ std::string windowHandle;
+ std::string outputPath;
+ int fps = 60;
+ int width = 0;
+ int height = 0;
+ MonitorBounds bounds{};
+ bool hasDisplayBounds = false;
+ bool captureSystemAudio = false;
+ bool captureMic = false;
+ bool webcamEnabled = false;
+ std::string microphoneDeviceId;
+ double microphoneGain = 1.0;
+ std::string webcamDeviceId;
+ int webcamWidth = 0;
+ int webcamHeight = 0;
+ int webcamFps = 0;
+};
+
+std::wstring utf8ToWide(const std::string& value) {
+ if (value.empty()) {
+ return {};
+ }
+
+ const int size = MultiByteToWideChar(CP_UTF8, 0, value.data(), static_cast(value.size()), nullptr, 0);
+ std::wstring result(static_cast(size), L'\0');
+ MultiByteToWideChar(CP_UTF8, 0, value.data(), static_cast(value.size()), result.data(), size);
+ return result;
+}
+
+std::string jsonEscape(const std::string& value) {
+ std::string result;
+ result.reserve(value.size());
+ for (const char c : value) {
+ switch (c) {
+ case '\\':
+ result += "\\\\";
+ break;
+ case '"':
+ result += "\\\"";
+ break;
+ case '\n':
+ result += "\\n";
+ break;
+ case '\r':
+ result += "\\r";
+ break;
+ case '\t':
+ result += "\\t";
+ break;
+ default:
+ result.push_back(c);
+ break;
+ }
+ }
+ return result;
+}
+
+bool findBool(const std::string& json, const std::string& key, bool fallback) {
+ auto pos = json.find("\"" + key + "\"");
+ if (pos == std::string::npos) {
+ return fallback;
+ }
+ pos = json.find(':', pos);
+ if (pos == std::string::npos) {
+ return fallback;
+ }
+ pos += 1;
+ while (pos < json.size() && std::isspace(static_cast(json[pos]))) {
+ pos += 1;
+ }
+ if (json.compare(pos, 4, "true") == 0) {
+ return true;
+ }
+ if (json.compare(pos, 5, "false") == 0) {
+ return false;
+ }
+ return fallback;
+}
+
+int64_t findInt64(const std::string& json, const std::string& key, int64_t fallback) {
+ auto pos = json.find("\"" + key + "\"");
+ if (pos == std::string::npos) {
+ return fallback;
+ }
+ pos = json.find(':', pos);
+ if (pos == std::string::npos) {
+ return fallback;
+ }
+ pos += 1;
+ while (pos < json.size() && std::isspace(static_cast(json[pos]))) {
+ pos += 1;
+ }
+ try {
+ return std::stoll(json.substr(pos));
+ } catch (...) {
+ return fallback;
+ }
+}
+
+int findInt(const std::string& json, const std::string& key, int fallback) {
+ return static_cast(findInt64(json, key, fallback));
+}
+
+double findDouble(const std::string& json, const std::string& key, double fallback) {
+ auto pos = json.find("\"" + key + "\"");
+ if (pos == std::string::npos) {
+ return fallback;
+ }
+ pos = json.find(':', pos);
+ if (pos == std::string::npos) {
+ return fallback;
+ }
+ pos += 1;
+ while (pos < json.size() && std::isspace(static_cast(json[pos]))) {
+ pos += 1;
+ }
+ try {
+ return std::stod(json.substr(pos));
+ } catch (...) {
+ return fallback;
+ }
+}
+
+std::string findString(const std::string& json, const std::string& key) {
+ auto pos = json.find("\"" + key + "\"");
+ if (pos == std::string::npos) {
+ return {};
+ }
+ pos = json.find(':', pos);
+ if (pos == std::string::npos) {
+ return {};
+ }
+ pos += 1;
+ while (pos < json.size() && std::isspace(static_cast(json[pos]))) {
+ pos += 1;
+ }
+ if (pos >= json.size() || json[pos] != '"') {
+ return {};
+ }
+ pos += 1;
+
+ std::string result;
+ while (pos < json.size()) {
+ const char c = json[pos++];
+ if (c == '"') {
+ break;
+ }
+ if (c == '\\' && pos < json.size()) {
+ const char escaped = json[pos++];
+ switch (escaped) {
+ case '\\':
+ case '"':
+ case '/':
+ result.push_back(escaped);
+ break;
+ case 'n':
+ result.push_back('\n');
+ break;
+ case 'r':
+ result.push_back('\r');
+ break;
+ case 't':
+ result.push_back('\t');
+ break;
+ default:
+ result.push_back(escaped);
+ break;
+ }
+ continue;
+ }
+ result.push_back(c);
+ }
+ return result;
+}
+
+bool parseConfig(const std::string& json, CaptureConfig& config) {
+ config.schemaVersion = findInt(json, "schemaVersion", 1);
+ config.outputPath = findString(json, "screenPath");
+ if (config.outputPath.empty()) {
+ config.outputPath = findString(json, "outputPath");
+ }
+ if (config.outputPath.empty()) {
+ return false;
+ }
+
+ config.recordingId = findInt64(json, "recordingId", 0);
+ config.sourceType = findString(json, "sourceType");
+ if (config.sourceType.empty()) {
+ config.sourceType = "display";
+ }
+ config.sourceId = findString(json, "sourceId");
+ config.windowHandle = findString(json, "windowHandle");
+ config.displayId = findInt64(json, "displayId", 0);
+ config.fps = std::clamp(findInt(json, "fps", 60), 1, 120);
+ config.width = findInt(json, "videoWidth", findInt(json, "width", 0));
+ config.height = findInt(json, "videoHeight", findInt(json, "height", 0));
+ config.bounds.x = findInt(json, "displayX", 0);
+ config.bounds.y = findInt(json, "displayY", 0);
+ config.bounds.width = findInt(json, "displayW", 0);
+ config.bounds.height = findInt(json, "displayH", 0);
+ config.hasDisplayBounds = findBool(json, "hasDisplayBounds", false);
+ config.captureSystemAudio = findBool(json, "captureSystemAudio", false);
+ config.captureMic = findBool(json, "captureMic", false);
+ config.webcamEnabled = findBool(json, "webcamEnabled", false);
+ config.microphoneDeviceId = findString(json, "microphoneDeviceId");
+ config.microphoneGain = findDouble(json, "microphoneGain", 1.0);
+ config.webcamDeviceId = findString(json, "webcamDeviceId");
+ config.webcamWidth = findInt(json, "webcamWidth", 0);
+ config.webcamHeight = findInt(json, "webcamHeight", 0);
+ config.webcamFps = findInt(json, "webcamFps", 0);
+ return true;
+}
+
+void readStopCommands(std::atomic& stopRequested, std::condition_variable& cv) {
+ std::string line;
+ while (std::getline(std::cin, line)) {
+ if (line == "stop" || line == "q" || line == "quit") {
+ stopRequested = true;
+ cv.notify_all();
+ return;
+ }
+ }
+ stopRequested = true;
+ cv.notify_all();
+}
+
+} // namespace
+
+int main(int argc, char* argv[]) {
+ if (argc < 2) {
+ std::cerr << "ERROR: Missing JSON config argument" << std::endl;
+ return 1;
+ }
+
+ winrt::init_apartment(winrt::apartment_type::multi_threaded);
+
+ CaptureConfig config;
+ if (!parseConfig(argv[1], config)) {
+ std::cerr << "ERROR: Failed to parse config JSON" << std::endl;
+ return 1;
+ }
+
+ std::cout << "{\"event\":\"ready\",\"schemaVersion\":2}" << std::endl;
+
+ if (config.sourceType != "display") {
+ std::cerr << "ERROR: Native window capture is not implemented yet" << std::endl;
+ return 1;
+ }
+
+ if (config.captureMic) {
+ std::cerr << "ERROR: Microphone capture is not implemented in this helper yet" << std::endl;
+ return 1;
+ }
+
+ if (config.webcamEnabled) {
+ std::cerr << "ERROR: Native webcam capture is not implemented in this helper yet" << std::endl;
+ return 1;
+ }
+
+ HMONITOR monitor = findMonitorForCapture(
+ config.displayId,
+ config.hasDisplayBounds ? &config.bounds : nullptr);
+ if (!monitor) {
+ std::cerr << "ERROR: Could not resolve monitor" << std::endl;
+ return 1;
+ }
+
+ WgcSession session;
+ if (!session.initialize(monitor, config.fps)) {
+ std::cerr << "ERROR: Failed to initialize WGC session" << std::endl;
+ return 1;
+ }
+
+ // WGC owns the captured texture size. Encoding must use that exact size
+ // until a dedicated GPU scaling pass is introduced; CopyResource requires
+ // matching resource dimensions.
+ int width = session.captureWidth();
+ int height = session.captureHeight();
+ width = (std::max(2, width) / 2) * 2;
+ height = (std::max(2, height) / 2) * 2;
+
+ const int pixels = width * height;
+ const int bitrate = pixels >= 3840 * 2160 ? 45'000'000 : pixels >= 2560 * 1440 ? 28'000'000 : 18'000'000;
+
+ WasapiLoopbackCapture loopbackCapture;
+ const AudioInputFormat* audioFormat = nullptr;
+ if (config.captureSystemAudio) {
+ if (!loopbackCapture.initialize()) {
+ std::cerr << "ERROR: Failed to initialize WASAPI loopback capture" << std::endl;
+ return 1;
+ }
+ audioFormat = &loopbackCapture.inputFormat();
+ std::cout << "{\"event\":\"audio-format\",\"schemaVersion\":2,\"sampleRate\":"
+ << audioFormat->sampleRate << ",\"channels\":" << audioFormat->channels
+ << ",\"bitsPerSample\":" << audioFormat->bitsPerSample << "}" << std::endl;
+ }
+
+ MFEncoder encoder;
+ if (!encoder.initialize(
+ utf8ToWide(config.outputPath),
+ width,
+ height,
+ config.fps,
+ bitrate,
+ session.device(),
+ session.context(),
+ audioFormat)) {
+ std::cerr << "ERROR: Failed to initialize Media Foundation encoder" << std::endl;
+ return 1;
+ }
+
+ std::mutex mutex;
+ std::condition_variable cv;
+ std::atomic stopRequested = false;
+ std::atomic firstFrameWritten = false;
+ std::atomic encodeFailed = false;
+
+ session.setFrameCallback([&](ID3D11Texture2D* texture, int64_t timestampHns) {
+ if (stopRequested) {
+ return;
+ }
+
+ std::scoped_lock lock(mutex);
+ if (!encoder.writeFrame(texture, timestampHns)) {
+ encodeFailed = true;
+ stopRequested = true;
+ cv.notify_all();
+ return;
+ }
+ if (!firstFrameWritten.exchange(true)) {
+ cv.notify_all();
+ }
+ });
+
+ if (config.captureSystemAudio) {
+ if (!loopbackCapture.start([&](const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns) {
+ if (stopRequested) {
+ return;
+ }
+
+ if (!encoder.writeAudio(data, byteCount, timestampHns, durationHns)) {
+ encodeFailed = true;
+ stopRequested = true;
+ cv.notify_all();
+ }
+ })) {
+ std::cerr << "ERROR: Failed to start WASAPI loopback capture" << std::endl;
+ return 1;
+ }
+ }
+
+ if (!session.start()) {
+ loopbackCapture.stop();
+ std::cerr << "ERROR: Failed to start WGC session" << std::endl;
+ return 1;
+ }
+
+ std::thread stdinThread(readStopCommands, std::ref(stopRequested), std::ref(cv));
+
+ {
+ std::unique_lock lock(mutex);
+ const bool started = cv.wait_for(lock, std::chrono::seconds(10), [&] {
+ return firstFrameWritten.load() || stopRequested.load();
+ });
+ if (!started || !firstFrameWritten) {
+ stopRequested = true;
+ cv.notify_all();
+ if (stdinThread.joinable()) {
+ stdinThread.detach();
+ }
+ loopbackCapture.stop();
+ std::cerr << "ERROR: Timed out waiting for first WGC frame" << std::endl;
+ return 1;
+ }
+ }
+
+ std::cout << "{\"event\":\"recording-started\",\"schemaVersion\":2}" << std::endl;
+ std::cout << "Recording started" << std::endl;
+
+ {
+ std::unique_lock lock(mutex);
+ cv.wait(lock, [&] {
+ return stopRequested.load();
+ });
+ }
+
+ loopbackCapture.stop();
+ session.stop();
+ {
+ std::scoped_lock lock(mutex);
+ encoder.finalize();
+ }
+
+ if (stdinThread.joinable()) {
+ stdinThread.join();
+ }
+
+ if (encodeFailed) {
+ std::cerr << "ERROR: Failed to encode WGC frame" << std::endl;
+ return 1;
+ }
+
+ std::cout << "{\"event\":\"recording-stopped\",\"schemaVersion\":2,\"screenPath\":\""
+ << jsonEscape(config.outputPath) << "\"}" << std::endl;
+ std::cout << "Recording stopped. Output path: " << config.outputPath << std::endl;
+ return 0;
+}
diff --git a/electron/native/wgc-capture/src/mf_encoder.cpp b/electron/native/wgc-capture/src/mf_encoder.cpp
new file mode 100644
index 0000000..fc95fc2
--- /dev/null
+++ b/electron/native/wgc-capture/src/mf_encoder.cpp
@@ -0,0 +1,317 @@
+#include "mf_encoder.h"
+
+#include
+#include
+#include
+
+#include
+#include
+#include
+
+namespace {
+
+bool succeeded(HRESULT hr, const char* label) {
+ if (SUCCEEDED(hr)) {
+ return true;
+ }
+
+ std::cerr << "ERROR: " << label << " failed (hr=0x" << std::hex << hr << std::dec << ")"
+ << std::endl;
+ return false;
+}
+
+void setFrameSize(IMFMediaType* type, UINT32 width, UINT32 height) {
+ MFSetAttributeSize(type, MF_MT_FRAME_SIZE, width, height);
+}
+
+void setFrameRate(IMFMediaType* type, UINT32 fps) {
+ MFSetAttributeRatio(type, MF_MT_FRAME_RATE, fps, 1);
+}
+
+void setPixelAspectRatio(IMFMediaType* type) {
+ MFSetAttributeRatio(type, MF_MT_PIXEL_ASPECT_RATIO, 1, 1);
+}
+
+void setAudioFormat(IMFMediaType* type, UINT32 channels, UINT32 sampleRate, UINT32 bitsPerSample) {
+ type->SetUINT32(MF_MT_AUDIO_NUM_CHANNELS, channels);
+ type->SetUINT32(MF_MT_AUDIO_SAMPLES_PER_SECOND, sampleRate);
+ type->SetUINT32(MF_MT_AUDIO_BITS_PER_SAMPLE, bitsPerSample);
+}
+
+} // namespace
+
+MFEncoder::~MFEncoder() {
+ finalize();
+}
+
+bool MFEncoder::initialize(
+ const std::wstring& outputPath,
+ int width,
+ int height,
+ int fps,
+ int bitrate,
+ ID3D11Device* device,
+ ID3D11DeviceContext* context,
+ const AudioInputFormat* audioFormat) {
+ width_ = (std::max(2, width) / 2) * 2;
+ height_ = (std::max(2, height) / 2) * 2;
+ fps_ = std::max(1, fps);
+ device_ = device;
+ context_ = context;
+
+ if (!succeeded(MFStartup(MF_VERSION), "MFStartup")) {
+ return false;
+ }
+
+ Microsoft::WRL::ComPtr outputType;
+ if (!succeeded(MFCreateMediaType(&outputType), "MFCreateMediaType(output)")) {
+ return false;
+ }
+ outputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
+ outputType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264);
+ outputType->SetUINT32(MF_MT_AVG_BITRATE, static_cast(std::max(1, bitrate)));
+ outputType->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive);
+ setFrameSize(outputType.Get(), static_cast(width_), static_cast(height_));
+ setFrameRate(outputType.Get(), static_cast(fps_));
+ setPixelAspectRatio(outputType.Get());
+
+ if (!succeeded(MFCreateSinkWriterFromURL(outputPath.c_str(), nullptr, nullptr, &sinkWriter_),
+ "MFCreateSinkWriterFromURL")) {
+ return false;
+ }
+ if (!succeeded(sinkWriter_->AddStream(outputType.Get(), &videoStreamIndex_), "AddStream")) {
+ return false;
+ }
+
+ if (audioFormat && !configureAudioStream(*audioFormat)) {
+ return false;
+ }
+
+ Microsoft::WRL::ComPtr inputType;
+ if (!succeeded(MFCreateMediaType(&inputType), "MFCreateMediaType(input)")) {
+ return false;
+ }
+ inputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
+ inputType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_RGB32);
+ inputType->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive);
+ inputType->SetUINT32(MF_MT_DEFAULT_STRIDE, static_cast(width_ * 4));
+ setFrameSize(inputType.Get(), static_cast(width_), static_cast(height_));
+ setFrameRate(inputType.Get(), static_cast(fps_));
+ setPixelAspectRatio(inputType.Get());
+
+ if (!succeeded(sinkWriter_->SetInputMediaType(videoStreamIndex_, inputType.Get(), nullptr),
+ "SetInputMediaType")) {
+ return false;
+ }
+ if (!succeeded(sinkWriter_->BeginWriting(), "BeginWriting")) {
+ return false;
+ }
+
+ return true;
+}
+
+bool MFEncoder::configureAudioStream(const AudioInputFormat& audioFormat) {
+ if (!sinkWriter_) {
+ return false;
+ }
+ if (audioFormat.sampleRate == 0 || audioFormat.channels == 0 || audioFormat.blockAlign == 0) {
+ std::cerr << "ERROR: Invalid audio input format" << std::endl;
+ return false;
+ }
+
+ const UINT32 bitsPerSample = std::max(8, audioFormat.bitsPerSample);
+ const UINT32 aacBytesPerSecond = 24'000;
+
+ Microsoft::WRL::ComPtr outputType;
+ if (!succeeded(MFCreateMediaType(&outputType), "MFCreateMediaType(audio output)")) {
+ return false;
+ }
+ outputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Audio);
+ outputType->SetGUID(MF_MT_SUBTYPE, MFAudioFormat_AAC);
+ setAudioFormat(outputType.Get(), audioFormat.channels, audioFormat.sampleRate, 16);
+ outputType->SetUINT32(MF_MT_AUDIO_AVG_BYTES_PER_SECOND, aacBytesPerSecond);
+ outputType->SetUINT32(MF_MT_AAC_PAYLOAD_TYPE, 0);
+
+ if (!succeeded(sinkWriter_->AddStream(outputType.Get(), &audioStreamIndex_), "AddStream(audio)")) {
+ return false;
+ }
+
+ Microsoft::WRL::ComPtr inputType;
+ if (!succeeded(MFCreateMediaType(&inputType), "MFCreateMediaType(audio input)")) {
+ return false;
+ }
+ inputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Audio);
+ inputType->SetGUID(MF_MT_SUBTYPE, audioFormat.subtype);
+ setAudioFormat(inputType.Get(), audioFormat.channels, audioFormat.sampleRate, bitsPerSample);
+ inputType->SetUINT32(MF_MT_AUDIO_BLOCK_ALIGNMENT, audioFormat.blockAlign);
+ inputType->SetUINT32(MF_MT_AUDIO_AVG_BYTES_PER_SECOND, audioFormat.avgBytesPerSec);
+ inputType->SetUINT32(MF_MT_ALL_SAMPLES_INDEPENDENT, TRUE);
+
+ if (!succeeded(sinkWriter_->SetInputMediaType(audioStreamIndex_, inputType.Get(), nullptr),
+ "SetInputMediaType(audio)")) {
+ return false;
+ }
+
+ hasAudioStream_ = true;
+ return true;
+}
+
+bool MFEncoder::ensureStagingTexture(ID3D11Texture2D* texture) {
+ if (stagingTexture_) {
+ return true;
+ }
+
+ D3D11_TEXTURE2D_DESC desc{};
+ texture->GetDesc(&desc);
+ desc.Width = static_cast(width_);
+ desc.Height = static_cast(height_);
+ desc.MipLevels = 1;
+ desc.ArraySize = 1;
+ desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM;
+ desc.SampleDesc.Count = 1;
+ desc.SampleDesc.Quality = 0;
+ desc.Usage = D3D11_USAGE_STAGING;
+ desc.BindFlags = 0;
+ desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
+ desc.MiscFlags = 0;
+
+ return succeeded(device_->CreateTexture2D(&desc, nullptr, &stagingTexture_),
+ "CreateTexture2D(staging)");
+}
+
+bool MFEncoder::copyFrameToBuffer(ID3D11Texture2D* texture, BYTE* destination, DWORD destinationSize) {
+ if (!ensureStagingTexture(texture)) {
+ return false;
+ }
+
+ context_->CopyResource(stagingTexture_.Get(), texture);
+
+ D3D11_MAPPED_SUBRESOURCE mapped{};
+ if (!succeeded(context_->Map(stagingTexture_.Get(), 0, D3D11_MAP_READ, 0, &mapped), "Map")) {
+ return false;
+ }
+
+ const DWORD rowBytes = static_cast(width_ * 4);
+ const DWORD requiredBytes = rowBytes * static_cast(height_);
+ if (destinationSize < requiredBytes) {
+ context_->Unmap(stagingTexture_.Get(), 0);
+ std::cerr << "ERROR: Media Foundation buffer is too small" << std::endl;
+ return false;
+ }
+
+ auto* source = static_cast(mapped.pData);
+ for (int y = 0; y < height_; y += 1) {
+ std::memcpy(destination + rowBytes * y, source + mapped.RowPitch * y, rowBytes);
+ }
+
+ context_->Unmap(stagingTexture_.Get(), 0);
+ return true;
+}
+
+bool MFEncoder::writeFrame(ID3D11Texture2D* texture, int64_t timestampHns) {
+ std::scoped_lock writerLock(writerMutex_);
+ if (!sinkWriter_ || finalized_) {
+ return false;
+ }
+
+ if (firstTimestampHns_ < 0) {
+ firstTimestampHns_ = timestampHns;
+ }
+
+ int64_t sampleTime = timestampHns - firstTimestampHns_;
+ if (sampleTime <= lastTimestampHns_) {
+ sampleTime = lastTimestampHns_ + (10'000'000LL / fps_);
+ }
+ const int64_t sampleDuration = 10'000'000LL / fps_;
+ lastTimestampHns_ = sampleTime;
+
+ Microsoft::WRL::ComPtr buffer;
+ const DWORD frameBytes = static_cast(width_ * height_ * 4);
+ if (!succeeded(MFCreateMemoryBuffer(frameBytes, &buffer), "MFCreateMemoryBuffer")) {
+ return false;
+ }
+
+ BYTE* data = nullptr;
+ DWORD maxLength = 0;
+ DWORD currentLength = 0;
+ if (!succeeded(buffer->Lock(&data, &maxLength, ¤tLength), "IMFMediaBuffer::Lock")) {
+ return false;
+ }
+
+ const bool copied = copyFrameToBuffer(texture, data, maxLength);
+ buffer->Unlock();
+ if (!copied) {
+ return false;
+ }
+ buffer->SetCurrentLength(frameBytes);
+
+ Microsoft::WRL::ComPtr sample;
+ if (!succeeded(MFCreateSample(&sample), "MFCreateSample")) {
+ return false;
+ }
+ sample->AddBuffer(buffer.Get());
+ sample->SetSampleTime(sampleTime);
+ sample->SetSampleDuration(sampleDuration);
+
+ return succeeded(sinkWriter_->WriteSample(videoStreamIndex_, sample.Get()), "WriteSample");
+}
+
+bool MFEncoder::writeAudio(const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns) {
+ std::scoped_lock writerLock(writerMutex_);
+ if (!sinkWriter_ || finalized_ || !hasAudioStream_) {
+ return false;
+ }
+ if (!data || byteCount == 0 || durationHns <= 0) {
+ return true;
+ }
+
+ Microsoft::WRL::ComPtr buffer;
+ if (!succeeded(MFCreateMemoryBuffer(byteCount, &buffer), "MFCreateMemoryBuffer(audio)")) {
+ return false;
+ }
+
+ BYTE* destination = nullptr;
+ DWORD maxLength = 0;
+ DWORD currentLength = 0;
+ if (!succeeded(buffer->Lock(&destination, &maxLength, ¤tLength),
+ "IMFMediaBuffer::Lock(audio)")) {
+ return false;
+ }
+ if (maxLength < byteCount) {
+ buffer->Unlock();
+ std::cerr << "ERROR: Media Foundation audio buffer is too small" << std::endl;
+ return false;
+ }
+ std::memcpy(destination, data, byteCount);
+ buffer->Unlock();
+ buffer->SetCurrentLength(byteCount);
+
+ Microsoft::WRL::ComPtr sample;
+ if (!succeeded(MFCreateSample(&sample), "MFCreateSample(audio)")) {
+ return false;
+ }
+ sample->AddBuffer(buffer.Get());
+ sample->SetSampleTime(std::max(0, timestampHns));
+ sample->SetSampleDuration(durationHns);
+
+ return succeeded(sinkWriter_->WriteSample(audioStreamIndex_, sample.Get()), "WriteSample(audio)");
+}
+
+bool MFEncoder::finalize() {
+ std::scoped_lock writerLock(writerMutex_);
+ if (finalized_) {
+ return true;
+ }
+
+ finalized_ = true;
+ bool ok = true;
+ if (sinkWriter_) {
+ ok = succeeded(sinkWriter_->Finalize(), "SinkWriter::Finalize");
+ sinkWriter_.Reset();
+ }
+ stagingTexture_.Reset();
+ context_.Reset();
+ device_.Reset();
+ MFShutdown();
+ return ok;
+}
diff --git a/electron/native/wgc-capture/src/mf_encoder.h b/electron/native/wgc-capture/src/mf_encoder.h
new file mode 100644
index 0000000..b6db685
--- /dev/null
+++ b/electron/native/wgc-capture/src/mf_encoder.h
@@ -0,0 +1,63 @@
+#pragma once
+
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include
+#include
+#include
+
+struct AudioInputFormat {
+ GUID subtype = MFAudioFormat_PCM;
+ UINT32 sampleRate = 0;
+ UINT32 channels = 0;
+ UINT32 bitsPerSample = 0;
+ UINT32 blockAlign = 0;
+ UINT32 avgBytesPerSec = 0;
+};
+
+class MFEncoder {
+public:
+ MFEncoder() = default;
+ ~MFEncoder();
+
+ MFEncoder(const MFEncoder&) = delete;
+ MFEncoder& operator=(const MFEncoder&) = delete;
+
+ bool initialize(
+ const std::wstring& outputPath,
+ int width,
+ int height,
+ int fps,
+ int bitrate,
+ ID3D11Device* device,
+ ID3D11DeviceContext* context,
+ const AudioInputFormat* audioFormat = nullptr);
+ bool writeFrame(ID3D11Texture2D* texture, int64_t timestampHns);
+ bool writeAudio(const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns);
+ bool finalize();
+
+private:
+ bool ensureStagingTexture(ID3D11Texture2D* texture);
+ bool copyFrameToBuffer(ID3D11Texture2D* texture, BYTE* destination, DWORD destinationSize);
+ bool configureAudioStream(const AudioInputFormat& audioFormat);
+
+ Microsoft::WRL::ComPtr sinkWriter_;
+ Microsoft::WRL::ComPtr device_;
+ Microsoft::WRL::ComPtr context_;
+ Microsoft::WRL::ComPtr stagingTexture_;
+ std::mutex writerMutex_;
+ DWORD videoStreamIndex_ = 0;
+ DWORD audioStreamIndex_ = 0;
+ bool hasAudioStream_ = false;
+ int width_ = 0;
+ int height_ = 0;
+ int fps_ = 60;
+ int64_t firstTimestampHns_ = -1;
+ int64_t lastTimestampHns_ = -1;
+ bool finalized_ = false;
+};
diff --git a/electron/native/wgc-capture/src/monitor_utils.cpp b/electron/native/wgc-capture/src/monitor_utils.cpp
new file mode 100644
index 0000000..f83e77d
--- /dev/null
+++ b/electron/native/wgc-capture/src/monitor_utils.cpp
@@ -0,0 +1,88 @@
+#include "monitor_utils.h"
+
+#include
+#include
+#include
+
+namespace {
+
+struct MonitorCandidate {
+ HMONITOR monitor = nullptr;
+ RECT rect{};
+};
+
+std::vector enumerateMonitors() {
+ std::vector monitors;
+ EnumDisplayMonitors(
+ nullptr,
+ nullptr,
+ [](HMONITOR monitor, HDC, LPRECT rect, LPARAM userData) -> BOOL {
+ auto* result = reinterpret_cast*>(userData);
+ result->push_back({monitor, *rect});
+ return TRUE;
+ },
+ reinterpret_cast(&monitors));
+ return monitors;
+}
+
+bool rectMatchesBounds(const RECT& rect, const MonitorBounds& bounds) {
+ return rect.left == bounds.x &&
+ rect.top == bounds.y &&
+ (rect.right - rect.left) == bounds.width &&
+ (rect.bottom - rect.top) == bounds.height;
+}
+
+int64_t overlapArea(const RECT& rect, const MonitorBounds& bounds) {
+ const LONG left = std::max(rect.left, bounds.x);
+ const LONG top = std::max(rect.top, bounds.y);
+ const LONG right = std::min(rect.right, bounds.x + bounds.width);
+ const LONG bottom = std::min(rect.bottom, bounds.y + bounds.height);
+ if (right <= left || bottom <= top) {
+ return 0;
+ }
+ return static_cast(right - left) * static_cast(bottom - top);
+}
+
+} // namespace
+
+HMONITOR findMonitorForCapture(int64_t displayId, const MonitorBounds* bounds) {
+ const auto monitors = enumerateMonitors();
+ if (monitors.empty()) {
+ return MonitorFromPoint({0, 0}, MONITOR_DEFAULTTOPRIMARY);
+ }
+
+ // Electron's display_id is not stable across all Windows capture backends.
+ // Bounds are the most reliable contract because they come from Electron's
+ // selected display and match the WGC monitor coordinate space.
+ if (bounds && bounds->width > 0 && bounds->height > 0) {
+ for (const auto& candidate : monitors) {
+ if (rectMatchesBounds(candidate.rect, *bounds)) {
+ return candidate.monitor;
+ }
+ }
+
+ HMONITOR bestMonitor = nullptr;
+ int64_t bestArea = 0;
+ for (const auto& candidate : monitors) {
+ const int64_t area = overlapArea(candidate.rect, *bounds);
+ if (area > bestArea) {
+ bestArea = area;
+ bestMonitor = candidate.monitor;
+ }
+ }
+ if (bestMonitor) {
+ return bestMonitor;
+ }
+ }
+
+ // Best-effort fallback for helpers invoked without bounds. Some callers pass
+ // zero-based ids while Win32 monitor handles are pointer values, so only use
+ // this when it exactly matches the HMONITOR value.
+ for (const auto& candidate : monitors) {
+ if (reinterpret_cast(candidate.monitor) == displayId) {
+ return candidate.monitor;
+ }
+ }
+
+ return MonitorFromPoint({0, 0}, MONITOR_DEFAULTTOPRIMARY);
+}
diff --git a/electron/native/wgc-capture/src/monitor_utils.h b/electron/native/wgc-capture/src/monitor_utils.h
new file mode 100644
index 0000000..11d5d83
--- /dev/null
+++ b/electron/native/wgc-capture/src/monitor_utils.h
@@ -0,0 +1,14 @@
+#pragma once
+
+#include
+
+#include
+
+struct MonitorBounds {
+ int x = 0;
+ int y = 0;
+ int width = 0;
+ int height = 0;
+};
+
+HMONITOR findMonitorForCapture(int64_t displayId, const MonitorBounds* bounds);
diff --git a/electron/native/wgc-capture/src/wasapi_loopback_capture.cpp b/electron/native/wgc-capture/src/wasapi_loopback_capture.cpp
new file mode 100644
index 0000000..e4f254e
--- /dev/null
+++ b/electron/native/wgc-capture/src/wasapi_loopback_capture.cpp
@@ -0,0 +1,205 @@
+#include "wasapi_loopback_capture.h"
+
+#include
+
+#include
+#include
+#include
+
+namespace {
+
+constexpr REFERENCE_TIME BufferDurationHns = 10'000'000;
+constexpr int64_t HnsPerSecond = 10'000'000;
+
+bool succeeded(HRESULT hr, const char* label) {
+ if (SUCCEEDED(hr)) {
+ return true;
+ }
+
+ std::cerr << "ERROR: " << label << " failed (hr=0x" << std::hex << hr << std::dec << ")"
+ << std::endl;
+ return false;
+}
+
+GUID audioSubtypeFromFormat(WAVEFORMATEX* format) {
+ if (format->wFormatTag == WAVE_FORMAT_IEEE_FLOAT) {
+ return MFAudioFormat_Float;
+ }
+ if (format->wFormatTag == WAVE_FORMAT_PCM) {
+ return MFAudioFormat_PCM;
+ }
+ if (format->wFormatTag == WAVE_FORMAT_EXTENSIBLE &&
+ format->cbSize >= sizeof(WAVEFORMATEXTENSIBLE) - sizeof(WAVEFORMATEX)) {
+ auto* extensible = reinterpret_cast(format);
+ if (extensible->SubFormat == KSDATAFORMAT_SUBTYPE_IEEE_FLOAT) {
+ return MFAudioFormat_Float;
+ }
+ if (extensible->SubFormat == KSDATAFORMAT_SUBTYPE_PCM) {
+ return MFAudioFormat_PCM;
+ }
+ }
+ return GUID_NULL;
+}
+
+} // namespace
+
+WasapiLoopbackCapture::~WasapiLoopbackCapture() {
+ stop();
+ if (mixFormat_) {
+ CoTaskMemFree(mixFormat_);
+ mixFormat_ = nullptr;
+ }
+}
+
+bool WasapiLoopbackCapture::initialize() {
+ HRESULT hr = CoCreateInstance(
+ __uuidof(MMDeviceEnumerator),
+ nullptr,
+ CLSCTX_ALL,
+ IID_PPV_ARGS(&deviceEnumerator_));
+ if (!succeeded(hr, "CoCreateInstance(MMDeviceEnumerator)")) {
+ return false;
+ }
+
+ hr = deviceEnumerator_->GetDefaultAudioEndpoint(eRender, eConsole, &device_);
+ if (!succeeded(hr, "GetDefaultAudioEndpoint(render)")) {
+ return false;
+ }
+
+ hr = device_->Activate(__uuidof(IAudioClient), CLSCTX_ALL, nullptr, &audioClient_);
+ if (!succeeded(hr, "IMMDevice::Activate(IAudioClient)")) {
+ return false;
+ }
+
+ hr = audioClient_->GetMixFormat(&mixFormat_);
+ if (!succeeded(hr, "IAudioClient::GetMixFormat") || !mixFormat_) {
+ return false;
+ }
+
+ if (!resolveInputFormat(mixFormat_)) {
+ std::cerr << "ERROR: Unsupported WASAPI loopback mix format" << std::endl;
+ return false;
+ }
+
+ hr = audioClient_->Initialize(
+ AUDCLNT_SHAREMODE_SHARED,
+ AUDCLNT_STREAMFLAGS_LOOPBACK,
+ BufferDurationHns,
+ 0,
+ mixFormat_,
+ nullptr);
+ if (!succeeded(hr, "IAudioClient::Initialize(loopback)")) {
+ return false;
+ }
+
+ hr = audioClient_->GetService(IID_PPV_ARGS(&captureClient_));
+ if (!succeeded(hr, "IAudioClient::GetService(IAudioCaptureClient)")) {
+ return false;
+ }
+
+ return true;
+}
+
+bool WasapiLoopbackCapture::resolveInputFormat(WAVEFORMATEX* mixFormat) {
+ const GUID subtype = audioSubtypeFromFormat(mixFormat);
+ if (subtype == GUID_NULL) {
+ return false;
+ }
+
+ inputFormat_.subtype = subtype;
+ inputFormat_.sampleRate = mixFormat->nSamplesPerSec;
+ inputFormat_.channels = mixFormat->nChannels;
+ inputFormat_.bitsPerSample = mixFormat->wBitsPerSample;
+ inputFormat_.blockAlign = mixFormat->nBlockAlign;
+ inputFormat_.avgBytesPerSec = mixFormat->nAvgBytesPerSec;
+ return inputFormat_.sampleRate > 0 && inputFormat_.channels > 0 && inputFormat_.blockAlign > 0;
+}
+
+bool WasapiLoopbackCapture::start(AudioCallback callback) {
+ if (!audioClient_ || !captureClient_ || !callback) {
+ return false;
+ }
+
+ callback_ = std::move(callback);
+ stopRequested_ = false;
+ writtenFrames_ = 0;
+
+ HRESULT hr = audioClient_->Start();
+ if (!succeeded(hr, "IAudioClient::Start")) {
+ return false;
+ }
+
+ thread_ = std::thread([this] {
+ captureLoop();
+ });
+ return true;
+}
+
+void WasapiLoopbackCapture::stop() {
+ stopRequested_ = true;
+ if (thread_.joinable()) {
+ thread_.join();
+ }
+ if (audioClient_) {
+ audioClient_->Stop();
+ }
+}
+
+const AudioInputFormat& WasapiLoopbackCapture::inputFormat() const {
+ return inputFormat_;
+}
+
+void WasapiLoopbackCapture::captureLoop() {
+ while (!stopRequested_) {
+ UINT32 packetFrames = 0;
+ HRESULT hr = captureClient_->GetNextPacketSize(&packetFrames);
+ if (FAILED(hr)) {
+ std::cerr << "ERROR: IAudioCaptureClient::GetNextPacketSize failed (hr=0x" << std::hex
+ << hr << std::dec << ")" << std::endl;
+ break;
+ }
+
+ while (packetFrames > 0 && !stopRequested_) {
+ BYTE* data = nullptr;
+ UINT32 framesAvailable = 0;
+ DWORD flags = 0;
+
+ hr = captureClient_->GetBuffer(&data, &framesAvailable, &flags, nullptr, nullptr);
+ if (FAILED(hr)) {
+ std::cerr << "ERROR: IAudioCaptureClient::GetBuffer failed (hr=0x" << std::hex
+ << hr << std::dec << ")" << std::endl;
+ break;
+ }
+
+ const DWORD byteCount = framesAvailable * inputFormat_.blockAlign;
+ const int64_t timestampHns =
+ static_cast((writtenFrames_ * HnsPerSecond) / inputFormat_.sampleRate);
+ const int64_t durationHns =
+ static_cast((static_cast(framesAvailable) * HnsPerSecond) /
+ inputFormat_.sampleRate);
+
+ if (byteCount > 0) {
+ if ((flags & AUDCLNT_BUFFERFLAGS_SILENT) != 0 || !data) {
+ silenceBuffer_.assign(byteCount, 0);
+ callback_(silenceBuffer_.data(), byteCount, timestampHns, durationHns);
+ } else {
+ callback_(data, byteCount, timestampHns, durationHns);
+ }
+ }
+
+ writtenFrames_ += framesAvailable;
+ captureClient_->ReleaseBuffer(framesAvailable);
+
+ hr = captureClient_->GetNextPacketSize(&packetFrames);
+ if (FAILED(hr)) {
+ std::cerr << "ERROR: IAudioCaptureClient::GetNextPacketSize failed (hr=0x"
+ << std::hex << hr << std::dec << ")" << std::endl;
+ packetFrames = 0;
+ break;
+ }
+ }
+
+ std::this_thread::sleep_for(std::chrono::milliseconds(5));
+ }
+
+}
diff --git a/electron/native/wgc-capture/src/wasapi_loopback_capture.h b/electron/native/wgc-capture/src/wasapi_loopback_capture.h
new file mode 100644
index 0000000..e6fb7e8
--- /dev/null
+++ b/electron/native/wgc-capture/src/wasapi_loopback_capture.h
@@ -0,0 +1,47 @@
+#pragma once
+
+#include "mf_encoder.h"
+
+#include
+#include
+#include
+#include
+
+#include
+#include
+#include
+#include
+#include
+
+class WasapiLoopbackCapture {
+public:
+ using AudioCallback = std::function;
+
+ WasapiLoopbackCapture() = default;
+ ~WasapiLoopbackCapture();
+
+ WasapiLoopbackCapture(const WasapiLoopbackCapture&) = delete;
+ WasapiLoopbackCapture& operator=(const WasapiLoopbackCapture&) = delete;
+
+ bool initialize();
+ bool start(AudioCallback callback);
+ void stop();
+
+ const AudioInputFormat& inputFormat() const;
+
+private:
+ void captureLoop();
+ bool resolveInputFormat(WAVEFORMATEX* mixFormat);
+
+ Microsoft::WRL::ComPtr deviceEnumerator_;
+ Microsoft::WRL::ComPtr device_;
+ Microsoft::WRL::ComPtr audioClient_;
+ Microsoft::WRL::ComPtr captureClient_;
+ WAVEFORMATEX* mixFormat_ = nullptr;
+ AudioInputFormat inputFormat_{};
+ AudioCallback callback_;
+ std::thread thread_;
+ std::atomic stopRequested_ = false;
+ std::vector silenceBuffer_;
+ uint64_t writtenFrames_ = 0;
+};
diff --git a/electron/native/wgc-capture/src/wgc_session.cpp b/electron/native/wgc-capture/src/wgc_session.cpp
new file mode 100644
index 0000000..c25444e
--- /dev/null
+++ b/electron/native/wgc-capture/src/wgc_session.cpp
@@ -0,0 +1,223 @@
+#include "wgc_session.h"
+
+#include
+#include
+#include
+#include
+
+#include
+
+namespace wf = winrt::Windows::Foundation;
+namespace wgcap = winrt::Windows::Graphics::Capture;
+namespace wgdx = winrt::Windows::Graphics::DirectX;
+namespace wgd3d = winrt::Windows::Graphics::DirectX::Direct3D11;
+
+extern "C" HRESULT __stdcall CreateDirect3D11DeviceFromDXGIDevice(
+ ::IDXGIDevice* dxgiDevice,
+ ::IInspectable** graphicsDevice);
+
+namespace {
+
+bool succeeded(HRESULT hr, const char* label) {
+ if (SUCCEEDED(hr)) {
+ return true;
+ }
+
+ std::cerr << "ERROR: " << label << " failed (hr=0x" << std::hex << hr << std::dec << ")"
+ << std::endl;
+ return false;
+}
+
+int64_t timeSpanToHns(wf::TimeSpan const& value) {
+ return value.count();
+}
+
+} // namespace
+
+WgcSession::~WgcSession() {
+ stop();
+}
+
+bool WgcSession::createD3DDevice() {
+ UINT flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT;
+#if defined(_DEBUG)
+ flags |= D3D11_CREATE_DEVICE_DEBUG;
+#endif
+
+ D3D_FEATURE_LEVEL featureLevels[] = {
+ D3D_FEATURE_LEVEL_11_1,
+ D3D_FEATURE_LEVEL_11_0,
+ D3D_FEATURE_LEVEL_10_1,
+ D3D_FEATURE_LEVEL_10_0,
+ };
+ D3D_FEATURE_LEVEL featureLevel{};
+
+ HRESULT hr = D3D11CreateDevice(
+ nullptr,
+ D3D_DRIVER_TYPE_HARDWARE,
+ nullptr,
+ flags,
+ featureLevels,
+ ARRAYSIZE(featureLevels),
+ D3D11_SDK_VERSION,
+ &d3dDevice_,
+ &featureLevel,
+ &d3dContext_);
+
+#if defined(_DEBUG)
+ if (FAILED(hr)) {
+ flags &= ~D3D11_CREATE_DEVICE_DEBUG;
+ hr = D3D11CreateDevice(
+ nullptr,
+ D3D_DRIVER_TYPE_HARDWARE,
+ nullptr,
+ flags,
+ featureLevels,
+ ARRAYSIZE(featureLevels),
+ D3D11_SDK_VERSION,
+ &d3dDevice_,
+ &featureLevel,
+ &d3dContext_);
+ }
+#endif
+
+ if (!succeeded(hr, "D3D11CreateDevice")) {
+ return false;
+ }
+
+ Microsoft::WRL::ComPtr dxgiDevice;
+ if (!succeeded(d3dDevice_.As(&dxgiDevice), "Query IDXGIDevice")) {
+ return false;
+ }
+
+ winrt::com_ptr<::IInspectable> inspectableDevice;
+ if (!succeeded(CreateDirect3D11DeviceFromDXGIDevice(dxgiDevice.Get(), inspectableDevice.put()),
+ "CreateDirect3D11DeviceFromDXGIDevice")) {
+ return false;
+ }
+
+ winrtDevice_ = inspectableDevice.as();
+ return true;
+}
+
+bool WgcSession::createCaptureItem(HMONITOR monitor) {
+ auto factory = winrt::get_activation_factory();
+ auto interop = factory.as();
+
+ wgcap::GraphicsCaptureItem item{nullptr};
+ HRESULT hr = interop->CreateForMonitor(
+ monitor,
+ winrt::guid_of(),
+ reinterpret_cast(winrt::put_abi(item)));
+ if (!succeeded(hr, "CreateForMonitor")) {
+ return false;
+ }
+
+ item_ = item;
+ const auto size = item_.Size();
+ width_ = static_cast(size.Width);
+ height_ = static_cast(size.Height);
+ return width_ > 0 && height_ > 0;
+}
+
+bool WgcSession::initialize(HMONITOR monitor, int fps) {
+ fps_ = fps > 0 ? fps : 60;
+ if (!createD3DDevice()) {
+ return false;
+ }
+ if (!createCaptureItem(monitor)) {
+ return false;
+ }
+
+ framePool_ = wgcap::Direct3D11CaptureFramePool::CreateFreeThreaded(
+ winrtDevice_,
+ wgdx::DirectXPixelFormat::B8G8R8A8UIntNormalized,
+ 2,
+ item_.Size());
+ session_ = framePool_.CreateCaptureSession(item_);
+
+ try {
+ session_.IsCursorCaptureEnabled(false);
+ } catch (...) {
+ // Older WGC builds can omit this property; callers still overlay their own cursor.
+ }
+
+ frameArrivedToken_ = framePool_.FrameArrived({this, &WgcSession::onFrameArrived});
+ return true;
+}
+
+void WgcSession::setFrameCallback(FrameCallback callback) {
+ std::scoped_lock lock(callbackMutex_);
+ frameCallback_ = std::move(callback);
+}
+
+bool WgcSession::start() {
+ if (!session_) {
+ return false;
+ }
+ session_.StartCapture();
+ started_ = true;
+ return true;
+}
+
+void WgcSession::stop() {
+ if (framePool_) {
+ framePool_.FrameArrived(frameArrivedToken_);
+ }
+ if (session_) {
+ session_.Close();
+ session_ = nullptr;
+ }
+ if (framePool_) {
+ framePool_.Close();
+ framePool_ = nullptr;
+ }
+ item_ = nullptr;
+ winrtDevice_ = nullptr;
+ d3dContext_.Reset();
+ d3dDevice_.Reset();
+ started_ = false;
+}
+
+void WgcSession::onFrameArrived(
+ wgcap::Direct3D11CaptureFramePool const& sender,
+ wf::IInspectable const&) {
+ auto frame = sender.TryGetNextFrame();
+ if (!frame) {
+ return;
+ }
+
+ auto surface = frame.Surface();
+ auto access = surface.as<::Windows::Graphics::DirectX::Direct3D11::IDirect3DDxgiInterfaceAccess>();
+ Microsoft::WRL::ComPtr texture;
+ HRESULT hr = access->GetInterface(__uuidof(ID3D11Texture2D), reinterpret_cast(texture.GetAddressOf()));
+ if (FAILED(hr) || !texture) {
+ return;
+ }
+
+ FrameCallback callback;
+ {
+ std::scoped_lock lock(callbackMutex_);
+ callback = frameCallback_;
+ }
+
+ if (callback) {
+ callback(texture.Get(), timeSpanToHns(frame.SystemRelativeTime()));
+ }
+}
+
+int WgcSession::captureWidth() const {
+ return width_;
+}
+
+int WgcSession::captureHeight() const {
+ return height_;
+}
+
+ID3D11Device* WgcSession::device() const {
+ return d3dDevice_.Get();
+}
+
+ID3D11DeviceContext* WgcSession::context() const {
+ return d3dContext_.Get();
+}
diff --git a/electron/native/wgc-capture/src/wgc_session.h b/electron/native/wgc-capture/src/wgc_session.h
new file mode 100644
index 0000000..8cfb050
--- /dev/null
+++ b/electron/native/wgc-capture/src/wgc_session.h
@@ -0,0 +1,55 @@
+#pragma once
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include
+#include
+
+class WgcSession {
+public:
+ using FrameCallback = std::function;
+
+ WgcSession() = default;
+ ~WgcSession();
+
+ WgcSession(const WgcSession&) = delete;
+ WgcSession& operator=(const WgcSession&) = delete;
+
+ bool initialize(HMONITOR monitor, int fps);
+ void setFrameCallback(FrameCallback callback);
+ bool start();
+ void stop();
+
+ int captureWidth() const;
+ int captureHeight() const;
+ ID3D11Device* device() const;
+ ID3D11DeviceContext* context() const;
+
+private:
+ bool createD3DDevice();
+ bool createCaptureItem(HMONITOR monitor);
+ void onFrameArrived(
+ winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool const& sender,
+ winrt::Windows::Foundation::IInspectable const&);
+
+ Microsoft::WRL::ComPtr d3dDevice_;
+ Microsoft::WRL::ComPtr d3dContext_;
+ winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice winrtDevice_{nullptr};
+ winrt::Windows::Graphics::Capture::GraphicsCaptureItem item_{nullptr};
+ winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool framePool_{nullptr};
+ winrt::Windows::Graphics::Capture::GraphicsCaptureSession session_{nullptr};
+ winrt::event_token frameArrivedToken_{};
+ FrameCallback frameCallback_;
+ std::mutex callbackMutex_;
+ int width_ = 0;
+ int height_ = 0;
+ int fps_ = 60;
+ bool started_ = false;
+};
diff --git a/electron/preload.ts b/electron/preload.ts
index 5345aab..2f9059f 100644
--- a/electron/preload.ts
+++ b/electron/preload.ts
@@ -1,4 +1,5 @@
import { contextBridge, ipcRenderer } from "electron";
+import type { NativeWindowsRecordingRequest } from "../src/lib/nativeWindowsRecording";
import type { RecordingSession, StoreRecordedSessionInput } from "../src/lib/recordingSession";
import { NATIVE_BRIDGE_CHANNEL, type NativeBridgeRequest } from "../src/native/contracts";
@@ -64,6 +65,15 @@ contextBridge.exposeInMainWorld("electronAPI", {
setRecordingState: (recording: boolean, recordingId?: number) => {
return ipcRenderer.invoke("set-recording-state", recording, recordingId);
},
+ isNativeWindowsCaptureAvailable: () => {
+ return ipcRenderer.invoke("is-native-windows-capture-available");
+ },
+ startNativeWindowsRecording: (request: NativeWindowsRecordingRequest) => {
+ return ipcRenderer.invoke("start-native-windows-recording", request);
+ },
+ stopNativeWindowsRecording: (discard?: boolean) => {
+ return ipcRenderer.invoke("stop-native-windows-recording", discard);
+ },
getCursorTelemetry: (videoPath?: string) => {
return ipcRenderer.invoke("get-cursor-telemetry", videoPath);
},
diff --git a/package.json b/package.json
index f81d99b..4311f6e 100644
--- a/package.json
+++ b/package.json
@@ -19,13 +19,16 @@
"lint:fix": "biome check --write .",
"format": "biome format --write .",
"i18n:check": "node scripts/i18n-check.mjs",
- "preview": "vite preview",
- "build:mac": "tsc && vite build && electron-builder --mac",
- "build:win": "tsc && vite build && electron-builder --win --config.npmRebuild=false",
- "build:linux": "tsc && vite build && electron-builder --linux AppImage deb pacman --config.npmRebuild=false",
- "test": "vitest --run",
+ "preview": "vite preview",
+ "build:mac": "tsc && vite build && electron-builder --mac",
+ "build:native:win": "node scripts/build-windows-wgc-helper.mjs",
+ "build:win": "npm run build:native:win && tsc && vite build && electron-builder --win --config.npmRebuild=false",
+ "build:linux": "tsc && vite build && electron-builder --linux AppImage deb pacman --config.npmRebuild=false",
+ "test": "vitest --run",
"test:watch": "vitest",
"test:cursor-native:win": "node scripts/test-windows-native-cursor.mjs",
+ "test:wgc-helper:win": "node scripts/test-windows-wgc-helper.mjs",
+ "test:wgc-audio:win": "node scripts/test-windows-wgc-helper.mjs --system-audio",
"capture:openscreen-preview": "node scripts/capture-openscreen-preview.mjs",
"build-vite": "tsc && vite build",
"test:browser": "vitest --config vitest.browser.config.ts --run",
diff --git a/scripts/build-windows-wgc-helper.mjs b/scripts/build-windows-wgc-helper.mjs
new file mode 100644
index 0000000..85da01e
--- /dev/null
+++ b/scripts/build-windows-wgc-helper.mjs
@@ -0,0 +1,112 @@
+import { spawn } from "node:child_process";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const ROOT = path.join(__dirname, "..");
+const SOURCE_DIR = path.join(ROOT, "electron", "native", "wgc-capture");
+const BUILD_DIR = path.join(SOURCE_DIR, "build");
+const COMPAT_LIB_DIR = path.join(BUILD_DIR, "compat-libs");
+const BIN_DIR = path.join(ROOT, "electron", "native", "bin", "win32-x64");
+const CMAKE = process.env.CMAKE_EXE ?? "cmake";
+
+function findVcVarsAll() {
+ const explicit = process.env.VCVARSALL;
+ if (explicit && fs.existsSync(explicit)) {
+ return explicit;
+ }
+
+ const roots = [
+ process.env.VSINSTALLDIR,
+ "C:\\Program Files\\Microsoft Visual Studio\\2022\\Community",
+ "C:\\Program Files\\Microsoft Visual Studio\\2022\\Professional",
+ "C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise",
+ "C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\BuildTools",
+ "C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\Community",
+ ];
+
+ for (const root of roots.filter(Boolean)) {
+ const candidate = path.join(root, "VC", "Auxiliary", "Build", "vcvarsall.bat");
+ if (fs.existsSync(candidate)) {
+ return candidate;
+ }
+ }
+
+ return null;
+}
+
+function run(command, args, options = {}) {
+ return new Promise((resolve, reject) => {
+ const child = spawn(command, args, {
+ cwd: ROOT,
+ stdio: "inherit",
+ windowsHide: true,
+ ...options,
+ });
+ child.once("error", reject);
+ child.once("exit", (code) => {
+ if (code === 0) {
+ resolve();
+ } else {
+ reject(new Error(`${command} ${args.join(" ")} failed with code ${code}`));
+ }
+ });
+ });
+}
+
+async function runInVsEnv(command) {
+ const vcvarsAll = findVcVarsAll();
+ if (!vcvarsAll) {
+ throw new Error(
+ "Could not find Visual Studio vcvarsall.bat. Install Visual Studio Build Tools with C++.",
+ );
+ }
+
+ const cmdPath = path.join(os.tmpdir(), `openscreen-build-wgc-${process.pid}-${Date.now()}.cmd`);
+ fs.writeFileSync(
+ cmdPath,
+ [
+ "@echo off",
+ `call "${vcvarsAll}" x64`,
+ "if errorlevel 1 exit /b %errorlevel%",
+ `if not exist "${COMPAT_LIB_DIR}" mkdir "${COMPAT_LIB_DIR}"`,
+ `for %%L in (gdi32.lib winspool.lib shell32.lib oleaut32.lib uuid.lib comdlg32.lib advapi32.lib) do if not exist "%WindowsSdkDir%Lib\\%WindowsSDKLibVersion%um\\x64\\%%L" copy /Y "%WindowsSdkDir%Lib\\%WindowsSDKLibVersion%um\\x64\\kernel32.Lib" "${COMPAT_LIB_DIR}\\%%L" >nul`,
+ "if errorlevel 1 exit /b %errorlevel%",
+ `set "LIB=${COMPAT_LIB_DIR};%LIB%"`,
+ command,
+ "exit /b %errorlevel%",
+ "",
+ ].join("\r\n"),
+ );
+ try {
+ await run("cmd.exe", ["/d", "/c", cmdPath]);
+ } finally {
+ fs.rmSync(cmdPath, { force: true });
+ }
+}
+
+if (process.platform !== "win32") {
+ console.log("Skipping WGC helper build: Windows-only.");
+ process.exit(0);
+}
+
+fs.mkdirSync(BUILD_DIR, { recursive: true });
+
+await runInVsEnv(
+ `"${CMAKE}" -S "${SOURCE_DIR}" -B "${BUILD_DIR}" -G Ninja -DCMAKE_BUILD_TYPE=Release`,
+);
+await runInVsEnv(`"${CMAKE}" --build "${BUILD_DIR}" --config Release`);
+
+const outputPath = path.join(BUILD_DIR, "wgc-capture.exe");
+if (!fs.existsSync(outputPath)) {
+ throw new Error(`WGC helper build completed but ${outputPath} was not found.`);
+}
+
+fs.mkdirSync(BIN_DIR, { recursive: true });
+const distributablePath = path.join(BIN_DIR, "wgc-capture.exe");
+fs.copyFileSync(outputPath, distributablePath);
+
+console.log(`Built ${outputPath}`);
+console.log(`Copied ${distributablePath}`);
diff --git a/scripts/test-windows-wgc-helper.mjs b/scripts/test-windows-wgc-helper.mjs
new file mode 100644
index 0000000..627c0ca
--- /dev/null
+++ b/scripts/test-windows-wgc-helper.mjs
@@ -0,0 +1,167 @@
+import { spawn, spawnSync } from "node:child_process";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const ROOT = path.join(__dirname, "..");
+const HELPER_PATH =
+ process.env.OPENSCREEN_WGC_CAPTURE_EXE ??
+ path.join(ROOT, "electron", "native", "bin", "win32-x64", "wgc-capture.exe");
+
+const DURATION_MS = Number(process.env.OPENSCREEN_WGC_TEST_DURATION_MS ?? 5000);
+const WITH_SYSTEM_AUDIO =
+ process.env.OPENSCREEN_WGC_TEST_SYSTEM_AUDIO === "true" ||
+ process.argv.includes("--system-audio");
+
+function runHelper(config) {
+ return new Promise((resolve, reject) => {
+ const child = spawn(HELPER_PATH, [JSON.stringify(config)], {
+ stdio: ["pipe", "pipe", "pipe"],
+ windowsHide: true,
+ });
+
+ let stdout = "";
+ let stderr = "";
+
+ child.stdout.on("data", (chunk) => {
+ stdout += chunk.toString();
+ });
+ child.stderr.on("data", (chunk) => {
+ stderr += chunk.toString();
+ });
+ child.once("error", reject);
+ child.once("exit", (code) => {
+ resolve({ code, stdout, stderr });
+ });
+
+ setTimeout(() => {
+ child.stdin.write("stop\n");
+ }, DURATION_MS);
+ });
+}
+
+function probeStreams(outputPath) {
+ const ffprobe = spawnSync(
+ "ffprobe",
+ ["-v", "error", "-show_streams", "-of", "json", outputPath],
+ { encoding: "utf8", windowsHide: true },
+ );
+ if (ffprobe.status !== 0) {
+ throw new Error(`ffprobe failed: ${ffprobe.stderr || ffprobe.stdout}`);
+ }
+ return JSON.parse(ffprobe.stdout).streams ?? [];
+}
+
+function measureFirstFrameLuma(outputPath) {
+ const ffmpeg = spawnSync(
+ "ffmpeg",
+ [
+ "-v",
+ "error",
+ "-i",
+ outputPath,
+ "-frames:v",
+ "1",
+ "-f",
+ "rawvideo",
+ "-pix_fmt",
+ "gray",
+ "pipe:1",
+ ],
+ { windowsHide: true, maxBuffer: 64 * 1024 * 1024 },
+ );
+ if (ffmpeg.status !== 0) {
+ throw new Error(`ffmpeg frame extraction failed: ${ffmpeg.stderr?.toString() ?? ""}`);
+ }
+ const data = ffmpeg.stdout;
+ if (!data || data.length === 0) {
+ throw new Error(`ffmpeg did not return frame data for ${outputPath}`);
+ }
+ let sum = 0;
+ let max = 0;
+ for (const value of data) {
+ sum += value;
+ if (value > max) {
+ max = value;
+ }
+ }
+ return { average: sum / data.length, max };
+}
+
+if (process.platform !== "win32") {
+ console.log("Skipping WGC helper smoke test: Windows-only.");
+ process.exit(0);
+}
+
+if (!fs.existsSync(HELPER_PATH)) {
+ throw new Error(`WGC helper not found at ${HELPER_PATH}. Run npm run build:native:win first.`);
+}
+
+const outputPath = path.join(
+ os.tmpdir(),
+ `openscreen-wgc-helper-${WITH_SYSTEM_AUDIO ? "audio" : "video"}-${Date.now()}.mp4`,
+);
+
+const config = {
+ schemaVersion: 2,
+ recordingId: Date.now(),
+ outputPath,
+ sourceType: "display",
+ sourceId: "screen:0:0",
+ displayId: 0,
+ fps: 30,
+ videoWidth: 1280,
+ videoHeight: 720,
+ displayX: 0,
+ displayY: 0,
+ displayW: 1920,
+ displayH: 1080,
+ hasDisplayBounds: true,
+ captureSystemAudio: WITH_SYSTEM_AUDIO,
+ captureMic: false,
+ webcamEnabled: false,
+ outputs: { screenPath: outputPath },
+};
+
+const result = await runHelper(config);
+if (result.code !== 0) {
+ throw new Error(`WGC helper exited with ${result.code}\n${result.stdout}\n${result.stderr}`);
+}
+if (!fs.existsSync(outputPath) || fs.statSync(outputPath).size === 0) {
+ throw new Error(`WGC helper did not produce a video at ${outputPath}`);
+}
+
+const streams = probeStreams(outputPath);
+const hasVideo = streams.some((stream) => stream.codec_type === "video");
+const hasAudio = streams.some((stream) => stream.codec_type === "audio");
+if (!hasVideo) {
+ throw new Error(`WGC helper output has no video stream: ${outputPath}`);
+}
+if (WITH_SYSTEM_AUDIO && !hasAudio) {
+ throw new Error(`WGC helper output has no audio stream: ${outputPath}`);
+}
+const frameLuma = measureFirstFrameLuma(outputPath);
+if (frameLuma.average < 1 && frameLuma.max < 5) {
+ throw new Error(`WGC helper output first frame is black: ${outputPath}`);
+}
+
+console.log(
+ JSON.stringify(
+ {
+ success: true,
+ outputPath,
+ bytes: fs.statSync(outputPath).size,
+ streams: streams.map((stream) => ({
+ index: stream.index,
+ codecType: stream.codec_type,
+ codecName: stream.codec_name,
+ duration: stream.duration,
+ })),
+ firstFrameLuma: frameLuma,
+ },
+ null,
+ 2,
+ ),
+);
diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx
index 432958a..3e87660 100644
--- a/src/components/video-editor/VideoPlayback.tsx
+++ b/src/components/video-editor/VideoPlayback.tsx
@@ -1832,7 +1832,6 @@ const VideoPlayback = forwardRef(
src={videoPath}
className="hidden"
preload="auto"
- muted
playsInline
onLoadedMetadata={handleLoadedMetadata}
onDurationChange={(e) => {
diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts
index 717a6cd..1ae9d22 100644
--- a/src/hooks/useScreenRecorder.ts
+++ b/src/hooks/useScreenRecorder.ts
@@ -2,6 +2,7 @@ import { fixWebmDuration } from "@fix-webm-duration/fix";
import { useCallback, useEffect, useRef, useState } from "react";
import { toast } from "sonner";
import { useScopedT } from "@/contexts/I18nContext";
+import type { NativeWindowsRecordingRequest } from "@/lib/nativeWindowsRecording";
import { requestCameraAccess } from "@/lib/requestCameraAccess";
const TARGET_FRAME_RATE = 60;
@@ -62,6 +63,11 @@ type RecorderHandle = {
recordedBlobPromise: Promise;
};
+type NativeWindowsRecordingHandle = {
+ recordingId: number;
+ finalizing: boolean;
+};
+
function createRecorderHandle(stream: MediaStream, options: MediaRecorderOptions): RecorderHandle {
const recorder = new MediaRecorder(stream, options);
const chunks: Blob[] = [];
@@ -96,6 +102,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
const [webcamEnabled, setWebcamEnabledState] = useState(false);
const screenRecorder = useRef(null);
const webcamRecorder = useRef(null);
+ const nativeWindowsRecording = useRef(null);
const stream = useRef(null);
const screenStream = useRef(null);
const microphoneStream = useRef(null);
@@ -365,7 +372,58 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
[teardownMedia],
);
+ const finalizeNativeWindowsRecording = useCallback(async (discard = false) => {
+ const activeNativeRecording = nativeWindowsRecording.current;
+ if (!activeNativeRecording || activeNativeRecording.finalizing) {
+ return false;
+ }
+
+ activeNativeRecording.finalizing = true;
+ nativeWindowsRecording.current = null;
+ setRecording(false);
+ setPaused(false);
+ setElapsedSeconds(0);
+ accumulatedDurationMs.current = 0;
+ segmentStartedAt.current = null;
+
+ try {
+ const result = await window.electronAPI.stopNativeWindowsRecording(discard);
+ if (discard || result.discarded) {
+ return true;
+ }
+ if (!result.success) {
+ console.error("Failed to stop native Windows recording:", result.error);
+ toast.error(result.error ?? "Failed to stop native Windows recording");
+ return true;
+ }
+
+ if (result.session) {
+ await window.electronAPI.setCurrentRecordingSession(result.session);
+ } else if (result.path) {
+ await window.electronAPI.setCurrentVideoPath(result.path);
+ }
+
+ await window.electronAPI.switchToEditor();
+ return true;
+ } catch (error) {
+ console.error("Error saving native Windows recording:", error);
+ toast.error(
+ error instanceof Error ? error.message : "Failed to save native Windows recording",
+ );
+ return true;
+ } finally {
+ if (discardRecordingId.current === activeNativeRecording.recordingId) {
+ discardRecordingId.current = null;
+ }
+ }
+ }, []);
+
const stopRecording = useRef(() => {
+ if (nativeWindowsRecording.current) {
+ void finalizeNativeWindowsRecording(false);
+ return;
+ }
+
const activeScreenRecorder = screenRecorder.current;
if (!activeScreenRecorder) {
return;
@@ -431,6 +489,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
allowAutoFinalize.current = false;
restarting.current = false;
discardRecordingId.current = null;
+ if (nativeWindowsRecording.current) {
+ void finalizeNativeWindowsRecording(true);
+ }
if (
screenRecorder.current?.recorder.state === "recording" ||
@@ -456,7 +517,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
webcamRecorder.current = null;
teardownMedia();
};
- }, [teardownMedia, safeHideCountdownOverlay]);
+ }, [teardownMedia, safeHideCountdownOverlay, finalizeNativeWindowsRecording]);
const safeShowCountdownOverlay = async (value: number, runId: number) => {
try {
@@ -486,6 +547,85 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
const isCountdownRunActive = (runId?: number) =>
runId === undefined || countdownRunId.current === runId;
+ const startNativeWindowsRecordingIfAvailable = async (
+ selectedSource: ProcessedDesktopSource,
+ countdownRunToken?: number,
+ ) => {
+ try {
+ const platform = await window.electronAPI.getPlatform();
+ if (platform !== "win32") {
+ return false;
+ }
+
+ const availability = await window.electronAPI.isNativeWindowsCaptureAvailable();
+ if (!availability.success || !availability.available) {
+ throw new Error(
+ availability.reason === "missing-helper"
+ ? "Native Windows capture helper is not available."
+ : (availability.error ?? "Native Windows capture is not available."),
+ );
+ }
+
+ if (!isCountdownRunActive(countdownRunToken)) {
+ return true;
+ }
+
+ const activeRecordingId = Date.now();
+ const displayId = Number(selectedSource.display_id);
+ const sourceType = selectedSource.id.startsWith("window:") ? "window" : "display";
+ const request: NativeWindowsRecordingRequest = {
+ recordingId: activeRecordingId,
+ source: {
+ type: sourceType,
+ sourceId: selectedSource.id,
+ ...(Number.isFinite(displayId) ? { displayId } : {}),
+ },
+ video: {
+ fps: TARGET_FRAME_RATE,
+ width: TARGET_WIDTH,
+ height: TARGET_HEIGHT,
+ },
+ audio: {
+ system: {
+ enabled: systemAudioEnabled,
+ },
+ microphone: {
+ enabled: microphoneEnabled,
+ deviceId: microphoneDeviceId,
+ gain: MIC_GAIN_BOOST,
+ },
+ },
+ webcam: {
+ enabled: webcamEnabled,
+ deviceId: webcamDeviceId,
+ width: WEBCAM_TARGET_WIDTH,
+ height: WEBCAM_TARGET_HEIGHT,
+ fps: WEBCAM_TARGET_FRAME_RATE,
+ },
+ };
+ const result = await window.electronAPI.startNativeWindowsRecording(request);
+ if (!result.success || !result.recordingId) {
+ throw new Error(result.error ?? "Native Windows capture failed.");
+ }
+
+ recordingId.current = result.recordingId;
+ nativeWindowsRecording.current = {
+ recordingId: result.recordingId,
+ finalizing: false,
+ };
+ accumulatedDurationMs.current = 0;
+ segmentStartedAt.current = result.recordingId;
+ allowAutoFinalize.current = true;
+ setRecording(true);
+ setPaused(false);
+ setElapsedSeconds(0);
+ return true;
+ } catch (error) {
+ console.error("Native Windows capture failed:", error);
+ throw error;
+ }
+ };
+
const startRecordCountdown = async () => {
if (countdownActive || recording) {
return;
@@ -573,6 +713,10 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
return;
}
+ if (await startNativeWindowsRecordingIfAvailable(selectedSource, countdownRunToken)) {
+ return;
+ }
+
let screenMediaStream: MediaStream;
// getDisplayMedia + setDisplayMediaRequestHandler (main.ts) supplies the
@@ -846,6 +990,19 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
const restartRecording = async () => {
if (restarting.current) return;
+ if (nativeWindowsRecording.current) {
+ const activeRecordingId = recordingId.current;
+ restarting.current = true;
+ discardRecordingId.current = activeRecordingId;
+ try {
+ await finalizeNativeWindowsRecording(true);
+ await startRecording();
+ } finally {
+ restarting.current = false;
+ }
+ return;
+ }
+
const activeScreenRecorder = screenRecorder.current;
if (!activeScreenRecorder || activeScreenRecorder.recorder.state === "inactive") return;
@@ -903,6 +1060,14 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
}, [getRecordingDurationMs, paused, recording]);
const cancelRecording = () => {
+ if (nativeWindowsRecording.current) {
+ const activeRecordingId = recordingId.current;
+ discardRecordingId.current = activeRecordingId;
+ allowAutoFinalize.current = false;
+ void finalizeNativeWindowsRecording(true);
+ return;
+ }
+
const activeScreenRecorder = screenRecorder.current;
if (
activeScreenRecorder?.recorder.state === "recording" ||
diff --git a/src/lib/cursor/nativeCursor.ts b/src/lib/cursor/nativeCursor.ts
index 3b62cb8..04ebccd 100644
--- a/src/lib/cursor/nativeCursor.ts
+++ b/src/lib/cursor/nativeCursor.ts
@@ -54,8 +54,8 @@ const PRETTY_NATIVE_CURSOR_ASSETS: Partial
Date: Tue, 5 May 2026 16:19:34 +0200
Subject: [PATCH 14/92] feat: add native Windows microphone capture
---
.../windows-native-recorder-roadmap.md | 4 +
electron/native/README.md | 6 +-
electron/native/wgc-capture/CMakeLists.txt | 2 +
.../wgc-capture/src/audio_sample_utils.cpp | 128 ++++++++++++++++++
.../wgc-capture/src/audio_sample_utils.h | 20 +++
electron/native/wgc-capture/src/main.cpp | 93 +++++++++++--
.../src/wasapi_loopback_capture.cpp | 33 ++++-
.../wgc-capture/src/wasapi_loopback_capture.h | 10 +-
package.json | 2 +
scripts/test-windows-wgc-helper.mjs | 12 +-
10 files changed, 290 insertions(+), 20 deletions(-)
create mode 100644 electron/native/wgc-capture/src/audio_sample_utils.cpp
create mode 100644 electron/native/wgc-capture/src/audio_sample_utils.h
diff --git a/docs/engineering/windows-native-recorder-roadmap.md b/docs/engineering/windows-native-recorder-roadmap.md
index c6a0a06..2fb6ab5 100644
--- a/docs/engineering/windows-native-recorder-roadmap.md
+++ b/docs/engineering/windows-native-recorder-roadmap.md
@@ -138,6 +138,8 @@ SSOT rules for this phase:
### 3. WASAPI Microphone
+Status: initial implementation in progress. The helper can open the default WASAPI capture endpoint, apply the OpenScreen microphone gain, encode mic-only audio, and mix mic into system-loopback packets when both endpoints expose the same runtime format. Browser `deviceId` to MMDevice id mapping, resampling between mismatched endpoint formats, and drift correction remain follow-up hardening work.
+
- Add microphone device enumeration and stable device-id mapping.
- Capture selected/default microphone through WASAPI.
- Apply OpenScreen's current mic gain policy.
@@ -192,6 +194,8 @@ Acceptance:
- `npm run test:wgc-webcam:win`: validates webcam output when a webcam is available, otherwise skips explicitly.
- Packaging check: confirms the helper is in `app.asar.unpacked`.
- Export check: exported MP4s generated from native recordings keep an AAC audio track when the source has audio.
+- `npm run test:wgc-mic:win`: validates default-microphone capture writes an AAC track when an input endpoint is available.
+- `npm run test:wgc-mixed-audio:win`: validates system loopback plus microphone writes one mixed AAC track when endpoint formats are compatible.
## Ship Criteria
diff --git a/electron/native/README.md b/electron/native/README.md
index 512517b..5df7290 100644
--- a/electron/native/README.md
+++ b/electron/native/README.md
@@ -32,6 +32,8 @@ Current V2 JSON shape:
"fps": 60,
"captureSystemAudio": false,
"captureMic": false,
+ "microphoneDeviceId": "default",
+ "microphoneGain": 1.4,
"webcamEnabled": false,
"outputs": {
"screenPath": "C:\\path\\recording-123.mp4",
@@ -40,11 +42,13 @@ Current V2 JSON shape:
}
```
-The current helper implementation supports display video capture and system audio loopback. Microphone, webcam, and window capture now fail explicitly in the helper rather than silently falling back to Electron capture on Windows. See `docs/engineering/windows-native-recorder-roadmap.md` for the phased implementation plan.
+The current helper implementation supports display video capture, system audio loopback, and initial default-microphone capture. Webcam and window capture now fail explicitly in the helper rather than silently falling back to Electron capture on Windows. See `docs/engineering/windows-native-recorder-roadmap.md` for the phased implementation plan.
Smoke-test the helper with:
```powershell
npm run test:wgc-helper:win
npm run test:wgc-audio:win
+npm run test:wgc-mic:win
+npm run test:wgc-mixed-audio:win
```
diff --git a/electron/native/wgc-capture/CMakeLists.txt b/electron/native/wgc-capture/CMakeLists.txt
index 76999f7..b21fd66 100644
--- a/electron/native/wgc-capture/CMakeLists.txt
+++ b/electron/native/wgc-capture/CMakeLists.txt
@@ -14,6 +14,8 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS OFF)
add_executable(wgc-capture
+ src/audio_sample_utils.cpp
+ src/audio_sample_utils.h
src/main.cpp
src/mf_encoder.cpp
src/mf_encoder.h
diff --git a/electron/native/wgc-capture/src/audio_sample_utils.cpp b/electron/native/wgc-capture/src/audio_sample_utils.cpp
new file mode 100644
index 0000000..6537d8e
--- /dev/null
+++ b/electron/native/wgc-capture/src/audio_sample_utils.cpp
@@ -0,0 +1,128 @@
+#include "audio_sample_utils.h"
+
+#include
+
+#include
+#include
+#include
+#include
+
+namespace {
+
+bool isFloatFormat(const AudioInputFormat& format) {
+ return format.subtype == MFAudioFormat_Float && format.bitsPerSample == 32;
+}
+
+bool isPcmFormat(const AudioInputFormat& format, UINT32 bitsPerSample) {
+ return format.subtype == MFAudioFormat_PCM && format.bitsPerSample == bitsPerSample;
+}
+
+template
+T clampTo(double value) {
+ const double minValue = static_cast(std::numeric_limits::min());
+ const double maxValue = static_cast(std::numeric_limits::max());
+ return static_cast(std::clamp(std::round(value), minValue, maxValue));
+}
+
+} // namespace
+
+bool sameAudioFormatForMixing(const AudioInputFormat& left, const AudioInputFormat& right) {
+ return left.subtype == right.subtype &&
+ left.sampleRate == right.sampleRate &&
+ left.channels == right.channels &&
+ left.bitsPerSample == right.bitsPerSample &&
+ left.blockAlign == right.blockAlign &&
+ left.avgBytesPerSec == right.avgBytesPerSec;
+}
+
+void copyAudioWithGain(
+ const BYTE* source,
+ DWORD byteCount,
+ const AudioInputFormat& format,
+ double gain,
+ std::vector& destination) {
+ destination.resize(byteCount);
+ if (!source || byteCount == 0) {
+ return;
+ }
+
+ if (std::abs(gain - 1.0) < 0.0001) {
+ std::memcpy(destination.data(), source, byteCount);
+ return;
+ }
+
+ if (isFloatFormat(format)) {
+ const auto* input = reinterpret_cast(source);
+ auto* output = reinterpret_cast(destination.data());
+ const size_t sampleCount = byteCount / sizeof(float);
+ for (size_t index = 0; index < sampleCount; index += 1) {
+ output[index] = static_cast(std::clamp(input[index] * gain, -1.0, 1.0));
+ }
+ return;
+ }
+
+ if (isPcmFormat(format, 16)) {
+ const auto* input = reinterpret_cast(source);
+ auto* output = reinterpret_cast(destination.data());
+ const size_t sampleCount = byteCount / sizeof(int16_t);
+ for (size_t index = 0; index < sampleCount; index += 1) {
+ output[index] = clampTo(static_cast(input[index]) * gain);
+ }
+ return;
+ }
+
+ if (isPcmFormat(format, 32)) {
+ const auto* input = reinterpret_cast(source);
+ auto* output = reinterpret_cast(destination.data());
+ const size_t sampleCount = byteCount / sizeof(int32_t);
+ for (size_t index = 0; index < sampleCount; index += 1) {
+ output[index] = clampTo(static_cast(input[index]) * gain);
+ }
+ return;
+ }
+
+ std::memcpy(destination.data(), source, byteCount);
+}
+
+void mixAudioInPlace(
+ std::vector& destination,
+ const BYTE* source,
+ DWORD byteCount,
+ const AudioInputFormat& format) {
+ if (!source || byteCount == 0 || destination.empty()) {
+ return;
+ }
+
+ const size_t mixByteCount = std::min(destination.size(), static_cast(byteCount));
+
+ if (isFloatFormat(format)) {
+ auto* output = reinterpret_cast(destination.data());
+ const auto* input = reinterpret_cast(source);
+ const size_t sampleCount = mixByteCount / sizeof(float);
+ for (size_t index = 0; index < sampleCount; index += 1) {
+ output[index] = static_cast(std::clamp(output[index] + input[index], -1.0f, 1.0f));
+ }
+ return;
+ }
+
+ if (isPcmFormat(format, 16)) {
+ auto* output = reinterpret_cast(destination.data());
+ const auto* input = reinterpret_cast(source);
+ const size_t sampleCount = mixByteCount / sizeof(int16_t);
+ for (size_t index = 0; index < sampleCount; index += 1) {
+ output[index] = clampTo(
+ static_cast(output[index]) + static_cast(input[index]));
+ }
+ return;
+ }
+
+ if (isPcmFormat(format, 32)) {
+ auto* output = reinterpret_cast(destination.data());
+ const auto* input = reinterpret_cast(source);
+ const size_t sampleCount = mixByteCount / sizeof(int32_t);
+ for (size_t index = 0; index < sampleCount; index += 1) {
+ output[index] = clampTo(
+ static_cast(output[index]) + static_cast(input[index]));
+ }
+ }
+}
diff --git a/electron/native/wgc-capture/src/audio_sample_utils.h b/electron/native/wgc-capture/src/audio_sample_utils.h
new file mode 100644
index 0000000..8022ae3
--- /dev/null
+++ b/electron/native/wgc-capture/src/audio_sample_utils.h
@@ -0,0 +1,20 @@
+#pragma once
+
+#include "mf_encoder.h"
+
+#include
+
+#include
+
+bool sameAudioFormatForMixing(const AudioInputFormat& left, const AudioInputFormat& right);
+void copyAudioWithGain(
+ const BYTE* source,
+ DWORD byteCount,
+ const AudioInputFormat& format,
+ double gain,
+ std::vector& destination);
+void mixAudioInPlace(
+ std::vector& destination,
+ const BYTE* source,
+ DWORD byteCount,
+ const AudioInputFormat& format);
diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp
index 39d5c62..603fda3 100644
--- a/electron/native/wgc-capture/src/main.cpp
+++ b/electron/native/wgc-capture/src/main.cpp
@@ -1,3 +1,4 @@
+#include "audio_sample_utils.h"
#include "mf_encoder.h"
#include "monitor_utils.h"
#include "wasapi_loopback_capture.h"
@@ -273,11 +274,6 @@ int main(int argc, char* argv[]) {
return 1;
}
- if (config.captureMic) {
- std::cerr << "ERROR: Microphone capture is not implemented in this helper yet" << std::endl;
- return 1;
- }
-
if (config.webcamEnabled) {
std::cerr << "ERROR: Native webcam capture is not implemented in this helper yet" << std::endl;
return 1;
@@ -309,16 +305,34 @@ int main(int argc, char* argv[]) {
const int bitrate = pixels >= 3840 * 2160 ? 45'000'000 : pixels >= 2560 * 1440 ? 28'000'000 : 18'000'000;
WasapiLoopbackCapture loopbackCapture;
+ WasapiLoopbackCapture microphoneCapture;
const AudioInputFormat* audioFormat = nullptr;
if (config.captureSystemAudio) {
- if (!loopbackCapture.initialize()) {
+ if (!loopbackCapture.initializeSystemLoopback()) {
std::cerr << "ERROR: Failed to initialize WASAPI loopback capture" << std::endl;
return 1;
}
audioFormat = &loopbackCapture.inputFormat();
- std::cout << "{\"event\":\"audio-format\",\"schemaVersion\":2,\"sampleRate\":"
- << audioFormat->sampleRate << ",\"channels\":" << audioFormat->channels
- << ",\"bitsPerSample\":" << audioFormat->bitsPerSample << "}" << std::endl;
+ }
+ if (config.captureMic) {
+ if (!microphoneCapture.initializeMicrophone(utf8ToWide(config.microphoneDeviceId))) {
+ std::cerr << "ERROR: Failed to initialize WASAPI microphone capture" << std::endl;
+ return 1;
+ }
+ if (!audioFormat) {
+ audioFormat = µphoneCapture.inputFormat();
+ } else if (!sameAudioFormatForMixing(*audioFormat, microphoneCapture.inputFormat())) {
+ std::cerr << "ERROR: System audio and microphone formats differ; native mixing is not supported yet"
+ << std::endl;
+ return 1;
+ }
+ }
+ if (audioFormat) {
+ std::cout << "{\"event\":\"audio-format\",\"schemaVersion\":2,\"sampleRate\":" << audioFormat->sampleRate
+ << ",\"channels\":" << audioFormat->channels
+ << ",\"bitsPerSample\":" << audioFormat->bitsPerSample
+ << ",\"system\":" << (config.captureSystemAudio ? "true" : "false")
+ << ",\"microphone\":" << (config.captureMic ? "true" : "false") << "}" << std::endl;
}
MFEncoder encoder;
@@ -358,24 +372,81 @@ int main(int argc, char* argv[]) {
}
});
+ std::mutex microphoneAudioMutex;
+ std::vector latestMicrophoneAudio;
+ std::vector mixedAudioBuffer;
+ std::vector microphoneGainBuffer;
+
+ if (config.captureMic) {
+ if (!microphoneCapture.start([&](const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns) {
+ if (stopRequested || !audioFormat) {
+ return;
+ }
+
+ copyAudioWithGain(
+ data,
+ byteCount,
+ microphoneCapture.inputFormat(),
+ config.microphoneGain,
+ microphoneGainBuffer);
+
+ if (config.captureSystemAudio) {
+ std::scoped_lock lock(microphoneAudioMutex);
+ latestMicrophoneAudio = microphoneGainBuffer;
+ return;
+ }
+
+ if (!encoder.writeAudio(
+ microphoneGainBuffer.data(),
+ static_cast(microphoneGainBuffer.size()),
+ timestampHns,
+ durationHns)) {
+ encodeFailed = true;
+ stopRequested = true;
+ cv.notify_all();
+ }
+ })) {
+ std::cerr << "ERROR: Failed to start WASAPI microphone capture" << std::endl;
+ return 1;
+ }
+ }
+
if (config.captureSystemAudio) {
if (!loopbackCapture.start([&](const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns) {
if (stopRequested) {
return;
}
- if (!encoder.writeAudio(data, byteCount, timestampHns, durationHns)) {
+ const BYTE* encodedData = data;
+ DWORD encodedByteCount = byteCount;
+ if (config.captureMic && audioFormat) {
+ mixedAudioBuffer.assign(data, data + byteCount);
+ {
+ std::scoped_lock lock(microphoneAudioMutex);
+ mixAudioInPlace(
+ mixedAudioBuffer,
+ latestMicrophoneAudio.data(),
+ static_cast(latestMicrophoneAudio.size()),
+ *audioFormat);
+ }
+ encodedData = mixedAudioBuffer.data();
+ encodedByteCount = static_cast(mixedAudioBuffer.size());
+ }
+
+ if (!encoder.writeAudio(encodedData, encodedByteCount, timestampHns, durationHns)) {
encodeFailed = true;
stopRequested = true;
cv.notify_all();
}
})) {
std::cerr << "ERROR: Failed to start WASAPI loopback capture" << std::endl;
+ microphoneCapture.stop();
return 1;
}
}
if (!session.start()) {
+ microphoneCapture.stop();
loopbackCapture.stop();
std::cerr << "ERROR: Failed to start WGC session" << std::endl;
return 1;
@@ -394,6 +465,7 @@ int main(int argc, char* argv[]) {
if (stdinThread.joinable()) {
stdinThread.detach();
}
+ microphoneCapture.stop();
loopbackCapture.stop();
std::cerr << "ERROR: Timed out waiting for first WGC frame" << std::endl;
return 1;
@@ -410,6 +482,7 @@ int main(int argc, char* argv[]) {
});
}
+ microphoneCapture.stop();
loopbackCapture.stop();
session.stop();
{
diff --git a/electron/native/wgc-capture/src/wasapi_loopback_capture.cpp b/electron/native/wgc-capture/src/wasapi_loopback_capture.cpp
index e4f254e..4e350a2 100644
--- a/electron/native/wgc-capture/src/wasapi_loopback_capture.cpp
+++ b/electron/native/wgc-capture/src/wasapi_loopback_capture.cpp
@@ -51,7 +51,15 @@ WasapiLoopbackCapture::~WasapiLoopbackCapture() {
}
}
-bool WasapiLoopbackCapture::initialize() {
+bool WasapiLoopbackCapture::initializeSystemLoopback() {
+ return initialize(WasapiCaptureEndpoint::SystemLoopback, {});
+}
+
+bool WasapiLoopbackCapture::initializeMicrophone(const std::wstring& deviceId) {
+ return initialize(WasapiCaptureEndpoint::Microphone, deviceId);
+}
+
+bool WasapiLoopbackCapture::initialize(WasapiCaptureEndpoint endpoint, const std::wstring& deviceId) {
HRESULT hr = CoCreateInstance(
__uuidof(MMDeviceEnumerator),
nullptr,
@@ -61,9 +69,22 @@ bool WasapiLoopbackCapture::initialize() {
return false;
}
- hr = deviceEnumerator_->GetDefaultAudioEndpoint(eRender, eConsole, &device_);
- if (!succeeded(hr, "GetDefaultAudioEndpoint(render)")) {
- return false;
+ if (endpoint == WasapiCaptureEndpoint::Microphone && !deviceId.empty() && deviceId != L"default") {
+ hr = deviceEnumerator_->GetDevice(deviceId.c_str(), &device_);
+ if (FAILED(hr)) {
+ std::wcerr << L"WARNING: Could not resolve microphone device id; using default capture endpoint"
+ << std::endl;
+ device_.Reset();
+ }
+ }
+
+ if (!device_) {
+ const EDataFlow flow =
+ endpoint == WasapiCaptureEndpoint::SystemLoopback ? eRender : eCapture;
+ hr = deviceEnumerator_->GetDefaultAudioEndpoint(flow, eConsole, &device_);
+ if (!succeeded(hr, "GetDefaultAudioEndpoint")) {
+ return false;
+ }
}
hr = device_->Activate(__uuidof(IAudioClient), CLSCTX_ALL, nullptr, &audioClient_);
@@ -81,9 +102,11 @@ bool WasapiLoopbackCapture::initialize() {
return false;
}
+ const DWORD streamFlags =
+ endpoint == WasapiCaptureEndpoint::SystemLoopback ? AUDCLNT_STREAMFLAGS_LOOPBACK : 0;
hr = audioClient_->Initialize(
AUDCLNT_SHAREMODE_SHARED,
- AUDCLNT_STREAMFLAGS_LOOPBACK,
+ streamFlags,
BufferDurationHns,
0,
mixFormat_,
diff --git a/electron/native/wgc-capture/src/wasapi_loopback_capture.h b/electron/native/wgc-capture/src/wasapi_loopback_capture.h
index e6fb7e8..8d2dbb9 100644
--- a/electron/native/wgc-capture/src/wasapi_loopback_capture.h
+++ b/electron/native/wgc-capture/src/wasapi_loopback_capture.h
@@ -10,9 +10,15 @@
#include
#include
#include