feat: add cursor overlay pipeline for high-fidelity cursor recording and playback

- Implement native bridge for Windows cursor capture via PowerShell/C#
- Add cursor-free capture using getDisplayMedia with setDisplayMediaRequestHandler
- Update video player and exporters to support native cursor telemetry
- Enable system audio capture on Windows via WASAPI loopback
- Add interpolation for smoother cursor movement in playback and export
- Improve cursor scaling and visibility handling in editor and playback

src/components/launch/LaunchWindow.tsx

@@ -259,6 +259,8 @@ export function LaunchWindow() {
 
 	const [selectedSource, setSelectedSource] = useState("Screen");
 	const [hasSelectedSource, setHasSelectedSource] = useState(false);
+	const [, setHudPointerDownCount] = useState(0);
+	const [, setRecordPointerDownCount] = useState(0);
 
 	useEffect(() => {
 		const checkSelectedSource = async () => {
@@ -541,6 +543,9 @@ export function LaunchWindow() {
 						onClick={toggleMicrophone}
 						disabled={recording}
 						title={microphoneEnabled ? t("audio.disableMicrophone") : t("audio.enableMicrophone")}
+						onPointerDown={() => {
+							setRecordPointerDownCount((count) => count + 1);
+						}}
 					>
 						{microphoneEnabled
 							? getIcon("micOn", "text-green-400")

src/components/video-editor/VideoEditor.tsx

@@ -1477,6 +1477,7 @@ export default function VideoEditor() {
 						videoPadding: padding,
 						cropRegion,
 						cursorRecordingData,
+						cursorScale: showCursor ? cursorSize : 0,
 						annotationRegions,
 						webcamLayoutPreset,
 						webcamMaskShape,
@@ -1619,6 +1620,7 @@ export default function VideoEditor() {
 						padding,
 						cropRegion,
 						cursorRecordingData,
+						cursorScale: showCursor ? cursorSize : 0,
 						annotationRegions,
 						webcamLayoutPreset,
 						webcamMaskShape,

src/components/video-editor/VideoPlayback.tsx

@@ -29,8 +29,9 @@ import { classifyWallpaper, DEFAULT_WALLPAPER, resolveImageWallpaperUrl } from "
 import { getCssClipPath } from "@/lib/webcamMaskShapes";
 import {
 	getNativeCursorDisplayMetrics,
+	hasNativeCursorRecordingData,
 	projectNativeCursorToStage,
-	resolveActiveNativeCursorFrame,
+	resolveInterpolatedNativeCursorFrame,
 } from "@/lib/cursor/nativeCursor";
 import type { CursorRecordingData } from "@/native/contracts";
 import {
@@ -635,6 +636,18 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
 			showCursorRef.current = showCursor;
 		}, [showCursor]);
 
+		useEffect(() => {
+			hasNativeCursorRecordingRef.current = hasNativeCursorRecording;
+		}, [hasNativeCursorRecording]);
+
+		useEffect(() => {
+			cursorRecordingDataRef.current = cursorRecordingData;
+		}, [cursorRecordingData]);
+
+		useEffect(() => {
+			cropRegionRef.current = cropRegion;
+		}, [cropRegion]);
+
 		useEffect(() => {
 			cursorSizeRef.current = cursorSize;
 		}, [cursorSize]);
@@ -1273,16 +1286,69 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
 				// Update cursor overlay
 				const cursorOverlay = cursorOverlayRef.current;
 				if (cursorOverlay) {
-					const timeMs = currentTimeRef.current;
+					const timeMs = currentTimeRef.current; // already in ms
 					cursorOverlay.update(
 						cursorTelemetryRef.current,
 						timeMs,
 						baseMaskRef.current,
-						showCursorRef.current,
+						showCursorRef.current && !hasNativeCursorRecordingRef.current,
 						!isPlayingRef.current || isSeekingRef.current,
 					);
 				}
 
+				// Update native cursor image position at ticker rate (60fps)
+				const nativeCursorImg = nativeCursorImgRef.current;
+				if (nativeCursorImg) {
+					const cameraContainerRc = cameraContainerRef.current;
+					const videoContainerRc = videoContainerRef.current;
+					if (
+						hasNativeCursorRecordingRef.current &&
+						showCursorRef.current &&
+						cameraContainerRc &&
+						videoContainerRc
+					) {
+						const timeMs = currentTimeRef.current; // already in ms
+						const frame = resolveInterpolatedNativeCursorFrame(
+							cursorRecordingDataRef.current,
+							timeMs,
+						);
+						if (frame) {
+							const projectedPoint = projectNativeCursorToStage({
+								cameraContainer: cameraContainerRc,
+								cropRegion: cropRegionRef.current ?? { x: 0, y: 0, width: 1, height: 1 },
+								maskRect: baseMaskRef.current,
+								videoContainerPosition: {
+									x: videoContainerRc.x,
+									y: videoContainerRc.y,
+								},
+								sample: frame.sample,
+							});
+							if (projectedPoint) {
+								const metrics = getNativeCursorDisplayMetrics(
+									frame.asset,
+									window.devicePixelRatio || 1,
+								);
+								const scale = Math.max(0, cursorSizeRef.current);
+								if (nativeCursorImg.dataset.cursorId !== frame.asset.id) {
+									nativeCursorImg.src = frame.asset.imageDataUrl;
+									nativeCursorImg.dataset.cursorId = frame.asset.id;
+								}
+								nativeCursorImg.style.left = `${projectedPoint.x - metrics.hotspotX * scale}px`;
+								nativeCursorImg.style.top = `${projectedPoint.y - metrics.hotspotY * scale}px`;
+								nativeCursorImg.style.width = `${metrics.width * scale}px`;
+								nativeCursorImg.style.height = `${metrics.height * scale}px`;
+								nativeCursorImg.style.display = "block";
+							} else {
+								nativeCursorImg.style.display = "none";
+							}
+						} else {
+							nativeCursorImg.style.display = "none";
+						}
+					} else {
+						nativeCursorImg.style.display = "none";
+					}
+				}
+
 				const composite3D = composite3DRef.current;
 				const outerWrapper = outerWrapperRef.current;
 				if (composite3D && outerWrapper) {
@@ -1571,17 +1637,14 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
 								className="absolute rounded-md border border-[#34B27B]/80 bg-[#34B27B]/20 shadow-[0_0_0_1px_rgba(52,178,123,0.35)]"
 								style={{ display: "none", pointerEvents: "none" }}
 							/>
-							{activeNativeCursor && nativeCursorStyle ? (
+							{hasNativeCursorRecording ? (
 								<img
+									ref={nativeCursorImgRef}
 									alt=""
 									aria-hidden="true"
-									src={activeNativeCursor.asset.imageDataUrl}
 									className="absolute select-none"
 									style={{
-										left: nativeCursorStyle.left,
-										top: nativeCursorStyle.top,
-										width: nativeCursorStyle.width,
-										height: nativeCursorStyle.height,
+										display: "none",
 										pointerEvents: "none",
 										userSelect: "none",
 									}}

src/hooks/useScreenRecorder.ts

@@ -25,7 +25,6 @@ const CODEC_ALIGNMENT = 2;
 
 const RECORDER_TIMESLICE_MS = 1000;
 const BITS_PER_MEGABIT = 1_000_000;
-const CHROME_MEDIA_SOURCE = "desktop";
 const RECORDING_FILE_PREFIX = "recording-";
 const VIDEO_FILE_EXTENSION = ".webm";
 const WEBCAM_FILE_SUFFIX = "-webcam";
@@ -576,42 +575,19 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
 
 			let screenMediaStream: MediaStream;
 
-			const videoConstraints = {
-				mandatory: {
-					chromeMediaSource: CHROME_MEDIA_SOURCE,
-					chromeMediaSourceId: selectedSource.id,
-					maxWidth: TARGET_WIDTH,
-					maxHeight: TARGET_HEIGHT,
-					maxFrameRate: TARGET_FRAME_RATE,
-					minFrameRate: MIN_FRAME_RATE,
-				},
-			};
-
-			if (systemAudioEnabled) {
-				try {
-					screenMediaStream = await navigator.mediaDevices.getUserMedia({
-						audio: {
-							mandatory: {
-								chromeMediaSource: CHROME_MEDIA_SOURCE,
-								chromeMediaSourceId: selectedSource.id,
-							},
-						},
-						video: videoConstraints,
-					} as unknown as MediaStreamConstraints);
-				} catch (audioErr) {
-					console.warn("System audio capture failed, falling back to video-only:", audioErr);
-					toast.error(t("recording.systemAudioUnavailable"));
-					screenMediaStream = await navigator.mediaDevices.getUserMedia({
-						audio: false,
-						video: videoConstraints,
-					} as unknown as MediaStreamConstraints);
-				}
-			} else {
-				screenMediaStream = await navigator.mediaDevices.getUserMedia({
-					audio: false,
-					video: videoConstraints,
-				} as unknown as MediaStreamConstraints);
-			}
+			// getDisplayMedia + setDisplayMediaRequestHandler (main.ts) supplies the
+			// pre-selected source and honors cursor:"never" to exclude the system cursor
+			// from every captured frame. System audio is provided via WASAPI loopback
+			// on Windows when the user has enabled it.
+			screenMediaStream = await navigator.mediaDevices.getDisplayMedia({
+				video: {
+					cursor: "never",
+					width: { max: TARGET_WIDTH },
+					height: { max: TARGET_HEIGHT },
+					frameRate: { ideal: TARGET_FRAME_RATE, min: MIN_FRAME_RATE },
+				} as MediaTrackConstraints,
+				audio: systemAudioEnabled,
+			} as DisplayMediaStreamOptions);
 			screenStream.current = screenMediaStream;
 
 			if (!isCountdownRunActive(countdownRunToken)) {

src/lib/cursor/nativeCursor.ts

@@ -14,7 +14,7 @@ export interface ActiveNativeCursorFrame {
 interface ProjectNativeCursorOptions {
 	cameraContainer: Container;
 	cropRegion: CropRegion;
-	maskRect: { width: number; height: number };
+	maskRect: { x: number; y: number; width: number; height: number };
 	videoContainerPosition: { x: number; y: number };
 	sample: CursorRecordingSample;
 }
@@ -23,6 +23,17 @@ function clamp(value: number, min: number, max: number) {
 	return Math.min(max, Math.max(min, value));
 }
 
+export function hasNativeCursorRecordingData(
+	recordingData: CursorRecordingData | null | undefined,
+): recordingData is CursorRecordingData {
+	return Boolean(
+		recordingData &&
+			recordingData.provider === "native" &&
+			recordingData.samples.length > 0 &&
+			recordingData.assets.length > 0,
+	);
+}
+
 function getCroppedCursorPosition(sample: CursorRecordingSample, cropRegion: CropRegion) {
 	if (cropRegion.width <= 0 || cropRegion.height <= 0) {
 		return null;
@@ -45,7 +56,7 @@ export function resolveActiveNativeCursorFrame(
 	recordingData: CursorRecordingData | null | undefined,
 	timeMs: number,
 ): ActiveNativeCursorFrame | null {
-	if (!recordingData || recordingData.provider !== "native" || recordingData.assets.length === 0) {
+	if (!hasNativeCursorRecordingData(recordingData)) {
 		return null;
 	}
 
@@ -70,6 +81,65 @@ export function resolveActiveNativeCursorFrame(
 	return null;
 }
 
+export function resolveInterpolatedNativeCursorFrame(
+	recordingData: CursorRecordingData | null | undefined,
+	timeMs: number,
+): ActiveNativeCursorFrame | null {
+	if (!hasNativeCursorRecordingData(recordingData)) {
+		return null;
+	}
+
+	const samples = recordingData.samples;
+	let activeIndex = -1;
+
+	for (let index = samples.length - 1; index >= 0; index -= 1) {
+		if (samples[index].timeMs <= timeMs) {
+			activeIndex = index;
+			break;
+		}
+	}
+
+	if (activeIndex < 0) {
+		return null;
+	}
+
+	const activeSample = samples[activeIndex];
+	if (activeSample.visible === false || !activeSample.assetId) {
+		return null;
+	}
+
+	const asset = recordingData.assets.find((candidate) => candidate.id === activeSample.assetId);
+	if (!asset) {
+		return null;
+	}
+
+	const nextSample = samples[activeIndex + 1];
+	if (
+		!nextSample ||
+		nextSample.timeMs <= activeSample.timeMs ||
+		nextSample.visible === false ||
+		nextSample.assetId !== activeSample.assetId ||
+		timeMs <= activeSample.timeMs
+	) {
+		return { asset, sample: activeSample };
+	}
+
+	const interpolation = clamp(
+		(timeMs - activeSample.timeMs) / (nextSample.timeMs - activeSample.timeMs),
+		0,
+		1,
+	);
+
+	return {
+		asset,
+		sample: {
+			...activeSample,
+			cx: activeSample.cx + (nextSample.cx - activeSample.cx) * interpolation,
+			cy: activeSample.cy + (nextSample.cy - activeSample.cy) * interpolation,
+		},
+	};
+}
+
 export function projectNativeCursorToStage({
 	cameraContainer,
 	cropRegion,
@@ -83,8 +153,8 @@ export function projectNativeCursorToStage({
 	}
 
 	const localPoint = new Point(
-		videoContainerPosition.x + croppedPosition.cx * maskRect.width,
-		videoContainerPosition.y + croppedPosition.cy * maskRect.height,
+		videoContainerPosition.x + maskRect.x + croppedPosition.cx * maskRect.width,
+		videoContainerPosition.y + maskRect.y + croppedPosition.cy * maskRect.height,
 	);
 
 	return cameraContainer.toGlobal(localPoint);

src/lib/exporter/frameRenderer.ts

@@ -59,7 +59,7 @@ import {
 import {
 	getNativeCursorDisplayMetrics,
 	projectNativeCursorToStage,
-	resolveActiveNativeCursorFrame,
+	resolveInterpolatedNativeCursorFrame,
 } from "@/lib/cursor/nativeCursor";
 import { BackgroundLoadError, classifyWallpaper, resolveImageWallpaperUrl } from "@/lib/wallpaper";
 import { drawCanvasClipPath } from "@/lib/webcamMaskShapes";
@@ -86,6 +86,7 @@ interface FrameRenderConfig {
 	padding?: number;
 	cropRegion: CropRegion;
 	cursorRecordingData?: CursorRecordingData | null;
+	cursorScale?: number;
 	videoWidth: number;
 	videoHeight: number;
 	webcamSize?: Size | null;
@@ -558,7 +559,11 @@ export class FrameRenderer {
 			return;
 		}
 
-		const activeNativeCursor = resolveActiveNativeCursorFrame(
+		if ((this.config.cursorScale ?? 1) <= 0) {
+			return;
+		}
+
+		const activeNativeCursor = resolveInterpolatedNativeCursorFrame(
 			this.config.cursorRecordingData,
 			timeMs,
 		);
@@ -582,13 +587,13 @@ export class FrameRenderer {
 
 		const image = await this.getCursorImage(activeNativeCursor.asset);
 		const metrics = getNativeCursorDisplayMetrics(activeNativeCursor.asset, 1);
-
+		const scale = Math.max(0, this.config.cursorScale ?? 1);
 		this.compositeCtx.drawImage(
 			image,
-			projectedPoint.x - metrics.hotspotX,
-			projectedPoint.y - metrics.hotspotY,
-			metrics.width,
-			metrics.height,
+			projectedPoint.x - metrics.hotspotX * scale,
+			projectedPoint.y - metrics.hotspotY * scale,
+			metrics.width * scale,
+			metrics.height * scale,
 		);
 	}
 

src/lib/exporter/gifExporter.ts

@@ -49,6 +49,7 @@ interface GifExporterConfig {
 	webcamSizePreset?: WebcamSizePreset;
 	webcamPosition?: { cx: number; cy: number } | null;
 	cursorRecordingData?: CursorRecordingData | null;
+	cursorScale?: number;
 	annotationRegions?: AnnotationRegion[];
 	previewWidth?: number;
 	previewHeight?: number;
@@ -154,6 +155,7 @@ export class GifExporter {
 				padding: this.config.padding,
 				cropRegion: this.config.cropRegion,
 				cursorRecordingData: this.config.cursorRecordingData,
+				cursorScale: this.config.cursorScale,
 				videoWidth: videoInfo.width,
 				videoHeight: videoInfo.height,
 				webcamSize: webcamInfo ? { width: webcamInfo.width, height: webcamInfo.height } : null,

src/lib/exporter/videoExporter.ts

@@ -40,6 +40,7 @@ interface VideoExporterConfig extends ExportConfig {
 	webcamSizePreset?: WebcamSizePreset;
 	webcamPosition?: { cx: number; cy: number } | null;
 	cursorRecordingData?: CursorRecordingData | null;
+	cursorScale?: number;
 	annotationRegions?: AnnotationRegion[];
 	previewWidth?: number;
 	previewHeight?: number;
@@ -149,6 +150,7 @@ export class VideoExporter {
 				padding: this.config.padding,
 				cropRegion: this.config.cropRegion,
 				cursorRecordingData: this.config.cursorRecordingData,
+				cursorScale: this.config.cursorScale,
 				videoWidth: videoInfo.width,
 				videoHeight: videoInfo.height,
 				webcamSize: webcamInfo ? { width: webcamInfo.width, height: webcamInfo.height } : null,