feat: add cursor overlay pipeline for high-fidelity cursor recording and playback
- Implement native bridge for Windows cursor capture via PowerShell/C# - Add cursor-free capture using getDisplayMedia with setDisplayMediaRequestHandler - Update video player and exporters to support native cursor telemetry - Enable system audio capture on Windows via WASAPI loopback - Add interpolation for smoother cursor movement in playback and export - Improve cursor scaling and visibility handling in editor and playback
This commit is contained in:
committed by
EtienneLescot
parent
248ebabcf1
commit
e9650225ba
@@ -259,6 +259,8 @@ export function LaunchWindow() {
|
||||
|
||||
const [selectedSource, setSelectedSource] = useState("Screen");
|
||||
const [hasSelectedSource, setHasSelectedSource] = useState(false);
|
||||
const [, setHudPointerDownCount] = useState(0);
|
||||
const [, setRecordPointerDownCount] = useState(0);
|
||||
|
||||
useEffect(() => {
|
||||
const checkSelectedSource = async () => {
|
||||
@@ -541,6 +543,9 @@ export function LaunchWindow() {
|
||||
onClick={toggleMicrophone}
|
||||
disabled={recording}
|
||||
title={microphoneEnabled ? t("audio.disableMicrophone") : t("audio.enableMicrophone")}
|
||||
onPointerDown={() => {
|
||||
setRecordPointerDownCount((count) => count + 1);
|
||||
}}
|
||||
>
|
||||
{microphoneEnabled
|
||||
? getIcon("micOn", "text-green-400")
|
||||
|
||||
@@ -1477,6 +1477,7 @@ export default function VideoEditor() {
|
||||
videoPadding: padding,
|
||||
cropRegion,
|
||||
cursorRecordingData,
|
||||
cursorScale: showCursor ? cursorSize : 0,
|
||||
annotationRegions,
|
||||
webcamLayoutPreset,
|
||||
webcamMaskShape,
|
||||
@@ -1619,6 +1620,7 @@ export default function VideoEditor() {
|
||||
padding,
|
||||
cropRegion,
|
||||
cursorRecordingData,
|
||||
cursorScale: showCursor ? cursorSize : 0,
|
||||
annotationRegions,
|
||||
webcamLayoutPreset,
|
||||
webcamMaskShape,
|
||||
|
||||
@@ -29,8 +29,9 @@ import { classifyWallpaper, DEFAULT_WALLPAPER, resolveImageWallpaperUrl } from "
|
||||
import { getCssClipPath } from "@/lib/webcamMaskShapes";
|
||||
import {
|
||||
getNativeCursorDisplayMetrics,
|
||||
hasNativeCursorRecordingData,
|
||||
projectNativeCursorToStage,
|
||||
resolveActiveNativeCursorFrame,
|
||||
resolveInterpolatedNativeCursorFrame,
|
||||
} from "@/lib/cursor/nativeCursor";
|
||||
import type { CursorRecordingData } from "@/native/contracts";
|
||||
import {
|
||||
@@ -635,6 +636,18 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
showCursorRef.current = showCursor;
|
||||
}, [showCursor]);
|
||||
|
||||
useEffect(() => {
|
||||
hasNativeCursorRecordingRef.current = hasNativeCursorRecording;
|
||||
}, [hasNativeCursorRecording]);
|
||||
|
||||
useEffect(() => {
|
||||
cursorRecordingDataRef.current = cursorRecordingData;
|
||||
}, [cursorRecordingData]);
|
||||
|
||||
useEffect(() => {
|
||||
cropRegionRef.current = cropRegion;
|
||||
}, [cropRegion]);
|
||||
|
||||
useEffect(() => {
|
||||
cursorSizeRef.current = cursorSize;
|
||||
}, [cursorSize]);
|
||||
@@ -1273,16 +1286,69 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
// Update cursor overlay
|
||||
const cursorOverlay = cursorOverlayRef.current;
|
||||
if (cursorOverlay) {
|
||||
const timeMs = currentTimeRef.current;
|
||||
const timeMs = currentTimeRef.current; // already in ms
|
||||
cursorOverlay.update(
|
||||
cursorTelemetryRef.current,
|
||||
timeMs,
|
||||
baseMaskRef.current,
|
||||
showCursorRef.current,
|
||||
showCursorRef.current && !hasNativeCursorRecordingRef.current,
|
||||
!isPlayingRef.current || isSeekingRef.current,
|
||||
);
|
||||
}
|
||||
|
||||
// Update native cursor image position at ticker rate (60fps)
|
||||
const nativeCursorImg = nativeCursorImgRef.current;
|
||||
if (nativeCursorImg) {
|
||||
const cameraContainerRc = cameraContainerRef.current;
|
||||
const videoContainerRc = videoContainerRef.current;
|
||||
if (
|
||||
hasNativeCursorRecordingRef.current &&
|
||||
showCursorRef.current &&
|
||||
cameraContainerRc &&
|
||||
videoContainerRc
|
||||
) {
|
||||
const timeMs = currentTimeRef.current; // already in ms
|
||||
const frame = resolveInterpolatedNativeCursorFrame(
|
||||
cursorRecordingDataRef.current,
|
||||
timeMs,
|
||||
);
|
||||
if (frame) {
|
||||
const projectedPoint = projectNativeCursorToStage({
|
||||
cameraContainer: cameraContainerRc,
|
||||
cropRegion: cropRegionRef.current ?? { x: 0, y: 0, width: 1, height: 1 },
|
||||
maskRect: baseMaskRef.current,
|
||||
videoContainerPosition: {
|
||||
x: videoContainerRc.x,
|
||||
y: videoContainerRc.y,
|
||||
},
|
||||
sample: frame.sample,
|
||||
});
|
||||
if (projectedPoint) {
|
||||
const metrics = getNativeCursorDisplayMetrics(
|
||||
frame.asset,
|
||||
window.devicePixelRatio || 1,
|
||||
);
|
||||
const scale = Math.max(0, cursorSizeRef.current);
|
||||
if (nativeCursorImg.dataset.cursorId !== frame.asset.id) {
|
||||
nativeCursorImg.src = frame.asset.imageDataUrl;
|
||||
nativeCursorImg.dataset.cursorId = frame.asset.id;
|
||||
}
|
||||
nativeCursorImg.style.left = `${projectedPoint.x - metrics.hotspotX * scale}px`;
|
||||
nativeCursorImg.style.top = `${projectedPoint.y - metrics.hotspotY * scale}px`;
|
||||
nativeCursorImg.style.width = `${metrics.width * scale}px`;
|
||||
nativeCursorImg.style.height = `${metrics.height * scale}px`;
|
||||
nativeCursorImg.style.display = "block";
|
||||
} else {
|
||||
nativeCursorImg.style.display = "none";
|
||||
}
|
||||
} else {
|
||||
nativeCursorImg.style.display = "none";
|
||||
}
|
||||
} else {
|
||||
nativeCursorImg.style.display = "none";
|
||||
}
|
||||
}
|
||||
|
||||
const composite3D = composite3DRef.current;
|
||||
const outerWrapper = outerWrapperRef.current;
|
||||
if (composite3D && outerWrapper) {
|
||||
@@ -1571,17 +1637,14 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
className="absolute rounded-md border border-[#34B27B]/80 bg-[#34B27B]/20 shadow-[0_0_0_1px_rgba(52,178,123,0.35)]"
|
||||
style={{ display: "none", pointerEvents: "none" }}
|
||||
/>
|
||||
{activeNativeCursor && nativeCursorStyle ? (
|
||||
{hasNativeCursorRecording ? (
|
||||
<img
|
||||
ref={nativeCursorImgRef}
|
||||
alt=""
|
||||
aria-hidden="true"
|
||||
src={activeNativeCursor.asset.imageDataUrl}
|
||||
className="absolute select-none"
|
||||
style={{
|
||||
left: nativeCursorStyle.left,
|
||||
top: nativeCursorStyle.top,
|
||||
width: nativeCursorStyle.width,
|
||||
height: nativeCursorStyle.height,
|
||||
display: "none",
|
||||
pointerEvents: "none",
|
||||
userSelect: "none",
|
||||
}}
|
||||
|
||||
Reference in New Issue
Block a user