feat: add cursor overlay pipeline for high-fidelity cursor recording and playback

- Implement native bridge for Windows cursor capture via PowerShell/C#
- Add cursor-free capture using getDisplayMedia with setDisplayMediaRequestHandler
- Update video player and exporters to support native cursor telemetry
- Enable system audio capture on Windows via WASAPI loopback
- Add interpolation for smoother cursor movement in playback and export
- Improve cursor scaling and visibility handling in editor and playback
This commit is contained in:
Etienne Lescot
2026-03-26 11:16:41 +01:00
committed by EtienneLescot
parent 248ebabcf1
commit e9650225ba
14 changed files with 686 additions and 297 deletions
+5
View File
@@ -259,6 +259,8 @@ export function LaunchWindow() {
const [selectedSource, setSelectedSource] = useState("Screen");
const [hasSelectedSource, setHasSelectedSource] = useState(false);
const [, setHudPointerDownCount] = useState(0);
const [, setRecordPointerDownCount] = useState(0);
useEffect(() => {
const checkSelectedSource = async () => {
@@ -541,6 +543,9 @@ export function LaunchWindow() {
onClick={toggleMicrophone}
disabled={recording}
title={microphoneEnabled ? t("audio.disableMicrophone") : t("audio.enableMicrophone")}
onPointerDown={() => {
setRecordPointerDownCount((count) => count + 1);
}}
>
{microphoneEnabled
? getIcon("micOn", "text-green-400")
@@ -1477,6 +1477,7 @@ export default function VideoEditor() {
videoPadding: padding,
cropRegion,
cursorRecordingData,
cursorScale: showCursor ? cursorSize : 0,
annotationRegions,
webcamLayoutPreset,
webcamMaskShape,
@@ -1619,6 +1620,7 @@ export default function VideoEditor() {
padding,
cropRegion,
cursorRecordingData,
cursorScale: showCursor ? cursorSize : 0,
annotationRegions,
webcamLayoutPreset,
webcamMaskShape,
+72 -9
View File
@@ -29,8 +29,9 @@ import { classifyWallpaper, DEFAULT_WALLPAPER, resolveImageWallpaperUrl } from "
import { getCssClipPath } from "@/lib/webcamMaskShapes";
import {
getNativeCursorDisplayMetrics,
hasNativeCursorRecordingData,
projectNativeCursorToStage,
resolveActiveNativeCursorFrame,
resolveInterpolatedNativeCursorFrame,
} from "@/lib/cursor/nativeCursor";
import type { CursorRecordingData } from "@/native/contracts";
import {
@@ -635,6 +636,18 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
showCursorRef.current = showCursor;
}, [showCursor]);
useEffect(() => {
hasNativeCursorRecordingRef.current = hasNativeCursorRecording;
}, [hasNativeCursorRecording]);
useEffect(() => {
cursorRecordingDataRef.current = cursorRecordingData;
}, [cursorRecordingData]);
useEffect(() => {
cropRegionRef.current = cropRegion;
}, [cropRegion]);
useEffect(() => {
cursorSizeRef.current = cursorSize;
}, [cursorSize]);
@@ -1273,16 +1286,69 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
// Update cursor overlay
const cursorOverlay = cursorOverlayRef.current;
if (cursorOverlay) {
const timeMs = currentTimeRef.current;
const timeMs = currentTimeRef.current; // already in ms
cursorOverlay.update(
cursorTelemetryRef.current,
timeMs,
baseMaskRef.current,
showCursorRef.current,
showCursorRef.current && !hasNativeCursorRecordingRef.current,
!isPlayingRef.current || isSeekingRef.current,
);
}
// Update native cursor image position at ticker rate (60fps)
const nativeCursorImg = nativeCursorImgRef.current;
if (nativeCursorImg) {
const cameraContainerRc = cameraContainerRef.current;
const videoContainerRc = videoContainerRef.current;
if (
hasNativeCursorRecordingRef.current &&
showCursorRef.current &&
cameraContainerRc &&
videoContainerRc
) {
const timeMs = currentTimeRef.current; // already in ms
const frame = resolveInterpolatedNativeCursorFrame(
cursorRecordingDataRef.current,
timeMs,
);
if (frame) {
const projectedPoint = projectNativeCursorToStage({
cameraContainer: cameraContainerRc,
cropRegion: cropRegionRef.current ?? { x: 0, y: 0, width: 1, height: 1 },
maskRect: baseMaskRef.current,
videoContainerPosition: {
x: videoContainerRc.x,
y: videoContainerRc.y,
},
sample: frame.sample,
});
if (projectedPoint) {
const metrics = getNativeCursorDisplayMetrics(
frame.asset,
window.devicePixelRatio || 1,
);
const scale = Math.max(0, cursorSizeRef.current);
if (nativeCursorImg.dataset.cursorId !== frame.asset.id) {
nativeCursorImg.src = frame.asset.imageDataUrl;
nativeCursorImg.dataset.cursorId = frame.asset.id;
}
nativeCursorImg.style.left = `${projectedPoint.x - metrics.hotspotX * scale}px`;
nativeCursorImg.style.top = `${projectedPoint.y - metrics.hotspotY * scale}px`;
nativeCursorImg.style.width = `${metrics.width * scale}px`;
nativeCursorImg.style.height = `${metrics.height * scale}px`;
nativeCursorImg.style.display = "block";
} else {
nativeCursorImg.style.display = "none";
}
} else {
nativeCursorImg.style.display = "none";
}
} else {
nativeCursorImg.style.display = "none";
}
}
const composite3D = composite3DRef.current;
const outerWrapper = outerWrapperRef.current;
if (composite3D && outerWrapper) {
@@ -1571,17 +1637,14 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
className="absolute rounded-md border border-[#34B27B]/80 bg-[#34B27B]/20 shadow-[0_0_0_1px_rgba(52,178,123,0.35)]"
style={{ display: "none", pointerEvents: "none" }}
/>
{activeNativeCursor && nativeCursorStyle ? (
{hasNativeCursorRecording ? (
<img
ref={nativeCursorImgRef}
alt=""
aria-hidden="true"
src={activeNativeCursor.asset.imageDataUrl}
className="absolute select-none"
style={{
left: nativeCursorStyle.left,
top: nativeCursorStyle.top,
width: nativeCursorStyle.width,
height: nativeCursorStyle.height,
display: "none",
pointerEvents: "none",
userSelect: "none",
}}