huanld/openscreen
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

feat: add windows native cursor capture and rendering

Browse Source
This commit is contained in:
Etienne Lescot
2026-03-16 10:41:46 +01:00
committed by EtienneLescot
parent 44f59bfa89
commit 248ebabcf1
12 changed files with 1035 additions and 720 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/components/video-editor/VideoEditor.tsx
+24 -40
View File
@@ -43,6 +43,7 @@ import {
getNativeAspectRatioValue,
isPortraitAspectRatio,
} from "@/utils/aspectRatioUtils";
import { nativeBridgeClient, useCursorRecordingData, useCursorTelemetry } from "@/native";
import { ExportDialog } from "./ExportDialog";
import PlaybackControls from "./PlaybackControls";
import {
@@ -61,7 +62,6 @@ import TimelineEditor from "./timeline/TimelineEditor";
import {
type AnnotationRegion,
type BlurData,
type CursorTelemetryPoint,
clampFocusToDepth,
DEFAULT_ANNOTATION_POSITION,
DEFAULT_ANNOTATION_SIZE,
@@ -133,8 +133,6 @@ export default function VideoEditor() {
currentTimeRef.current = currentTime;
const durationRef = useRef(duration);
durationRef.current = duration;
const [cursorTelemetry, setCursorTelemetry] = useState<CursorTelemetryPoint[]>([]);
const [cursorClickTimestamps, setCursorClickTimestamps] = useState<number[]>([]);
const [selectedZoomId, setSelectedZoomId] = useState<string | null>(null);
const [selectedTrimId, setSelectedTrimId] = useState<string | null>(null);
const [selectedSpeedId, setSelectedSpeedId] = useState<string | null>(null);
@@ -220,6 +218,13 @@ export default function VideoEditor() {
const project = candidate;
const sourcePath = project.videoPath;
const normalizedEditor = normalizeProjectEditor(project.editor);
const inferredDurationMs = Math.max(
0,
...normalizedEditor.zoomRegions.map((region) => region.endMs),
...normalizedEditor.trimRegions.map((region) => region.endMs),
...normalizedEditor.speedRegions.map((region) => region.endMs),
...normalizedEditor.annotationRegions.map((region) => region.endMs),
);
try {
videoPlaybackRef.current?.pause();
@@ -228,7 +233,7 @@ export default function VideoEditor() {
}
setIsPlaying(false);
setCurrentTime(0);
setDuration(0);
setDuration(inferredDurationMs > 0 ? inferredDurationMs / 1000 : 0);
setError(null);
setVideoSourcePath(sourcePath);
@@ -357,7 +362,7 @@ export default function VideoEditor() {
useEffect(() => {
async function loadInitialData() {
try {
const currentProjectResult = await window.electronAPI.loadCurrentProjectFile();
const currentProjectResult = await nativeBridgeClient.project.loadCurrentProjectFile();
if (currentProjectResult.success && currentProjectResult.project) {
const restored = await applyLoadedProject(
currentProjectResult.project,
@@ -394,7 +399,7 @@ export default function VideoEditor() {
return;
}
const result = await window.electronAPI.getCurrentVideoPath();
const result = await nativeBridgeClient.project.getCurrentVideoPath();
if (result.success && result.path) {
setVideoSourcePath(result.path);
setVideoPath(toFileUrl(result.path));
@@ -483,7 +488,7 @@ export default function VideoEditor() {
// Match the normalization path used by `currentProjectSnapshot` so the
// post-save baseline compares equal and `hasUnsavedChanges` clears.
const projectSnapshot = createProjectSnapshot(currentProjectMedia, editorState);
const result = await window.electronAPI.saveProjectFile(
const result = await nativeBridgeClient.project.saveProjectFile(
projectData,
fileNameBase,
forceSaveAs ? undefined : (currentProjectPath ?? undefined),
@@ -589,7 +594,7 @@ export default function VideoEditor() {
}, []);
const handleLoadProject = useCallback(async () => {
const result = await window.electronAPI.loadProjectFile();
const result = await nativeBridgeClient.project.loadProjectFile();
if (result.canceled) {
return;
@@ -622,40 +627,16 @@ export default function VideoEditor() {
}, [handleLoadProject, handleSaveProject, handleSaveProjectAs]);
useEffect(() => {
let mounted = true;
async function loadCursorTelemetry() {
const sourcePath = currentProjectMedia?.screenVideoPath ?? null;
if (!sourcePath) {
if (mounted) {
setCursorTelemetry([]);
setCursorClickTimestamps([]);
}
return;
}
try {
const result = await window.electronAPI.getCursorTelemetry(sourcePath);
if (mounted) {
setCursorTelemetry(result.success ? result.samples : []);
setCursorClickTimestamps(result.success ? (result.clicks ?? []) : []);
}
} catch (telemetryError) {
console.warn("Unable to load cursor telemetry:", telemetryError);
if (mounted) {
setCursorTelemetry([]);
setCursorClickTimestamps([]);
}
}
if (cursorTelemetryError) {
console.warn("Unable to load cursor telemetry:", cursorTelemetryError);
}
}, [cursorTelemetryError]);
loadCursorTelemetry();
return () => {
mounted = false;
};
}, [currentProjectMedia]);
useEffect(() => {
if (cursorRecordingDataError) {
console.warn("Unable to load cursor recording data:", cursorRecordingDataError);
}
}, [cursorRecordingDataError]);
function togglePlayPause() {
const playback = videoPlaybackRef.current;
@@ -1495,6 +1476,7 @@ export default function VideoEditor() {
padding,
videoPadding: padding,
cropRegion,
cursorRecordingData,
annotationRegions,
webcamLayoutPreset,
webcamMaskShape,
@@ -1636,6 +1618,7 @@ export default function VideoEditor() {
borderRadius,
padding,
cropRegion,
cursorRecordingData,
annotationRegions,
webcamLayoutPreset,
webcamMaskShape,
@@ -1715,6 +1698,7 @@ export default function VideoEditor() {
borderRadius,
padding,
cropRegion,
cursorRecordingData,
annotationRegions,
isPlaying,
aspectRatio,
src/components/video-editor/VideoPlayback.tsx
+77 -4
View File
@@ -27,6 +27,12 @@ import {
} from "@/lib/compositeLayout";
import { classifyWallpaper, DEFAULT_WALLPAPER, resolveImageWallpaperUrl } from "@/lib/wallpaper";
import { getCssClipPath } from "@/lib/webcamMaskShapes";
import {
getNativeCursorDisplayMetrics,
projectNativeCursorToStage,
resolveActiveNativeCursorFrame,
} from "@/lib/cursor/nativeCursor";
import type { CursorRecordingData } from "@/native/contracts";
import {
type AspectRatio,
formatAspectRatioForCSS,
@@ -123,6 +129,7 @@ interface VideoPlaybackProps {
trimRegions?: TrimRegion[];
speedRegions?: SpeedRegion[];
aspectRatio: AspectRatio;
cursorRecordingData?: CursorRecordingData | null;
annotationRegions?: AnnotationRegion[];
selectedAnnotationId?: string | null;
onSelectAnnotation?: (id: string | null) => void;
@@ -155,6 +162,22 @@ export interface VideoPlaybackRef {
pause: () => void;
}
function getResolvedVideoDuration(video: HTMLVideoElement): number | null {
if (Number.isFinite(video.duration) && video.duration > 0) {
return video.duration;
}
if (video.seekable.length > 0) {
const lastRangeIndex = video.seekable.length - 1;
const seekableEnd = video.seekable.end(lastRangeIndex);
if (Number.isFinite(seekableEnd) && seekableEnd > 0) {
return seekableEnd;
}
}
return null;
}
const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
(
{
@@ -188,6 +211,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
trimRegions = [],
speedRegions = [],
aspectRatio,
cursorRecordingData,
annotationRegions = [],
selectedAnnotationId,
onSelectAnnotation,
@@ -843,6 +867,8 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
useEffect(() => {
if (!videoPath) {
lastResolvedDurationRef.current = null;
isResolvingDurationRef.current = false;
setVideoReady(false);
return;
}
@@ -853,11 +879,18 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
video.currentTime = 0;
allowPlaybackRef.current = false;
lockedVideoDimensionsRef.current = null;
lastResolvedDurationRef.current = null;
isResolvingDurationRef.current = false;
if (durationResolutionTimeoutRef.current) {
clearTimeout(durationResolutionTimeoutRef.current);
durationResolutionTimeoutRef.current = null;
}
setVideoReady(false);
if (videoReadyRafRef.current) {
cancelAnimationFrame(videoReadyRafRef.current);
videoReadyRafRef.current = null;
}
video.load();
}, [videoPath]);
useEffect(() => {
@@ -1299,8 +1332,12 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
const handleLoadedMetadata = (e: React.SyntheticEvent<HTMLVideoElement, Event>) => {
const video = e.currentTarget;
onDurationChange(video.duration);
video.currentTime = 0;
const hasResolvedDuration = syncResolvedDuration(video);
if (!hasResolvedDuration) {
forceResolveDuration(video);
} else {
video.currentTime = 0;
}
video.pause();
allowPlaybackRef.current = false;
currentTimeRef.current = 0;
@@ -1313,6 +1350,9 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
const waitForRenderableFrame = () => {
const hasDimensions = video.videoWidth > 0 && video.videoHeight > 0;
const hasData = video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA;
if (!syncResolvedDuration(video)) {
forceResolveDuration(video);
}
if (hasDimensions && hasData) {
videoReadyRafRef.current = null;
setVideoReady(true);
@@ -1412,6 +1452,10 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
window.clearTimeout(scrubEndTimerRef.current);
scrubEndTimerRef.current = null;
}
if (durationResolutionTimeoutRef.current) {
clearTimeout(durationResolutionTimeoutRef.current);
durationResolutionTimeoutRef.current = null;
}
};
}, []);
@@ -1527,6 +1571,22 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
className="absolute rounded-md border border-[#34B27B]/80 bg-[#34B27B]/20 shadow-[0_0_0_1px_rgba(52,178,123,0.35)]"
style={{ display: "none", pointerEvents: "none" }}
/>
{activeNativeCursor && nativeCursorStyle ? (
<img
alt=""
aria-hidden="true"
src={activeNativeCursor.asset.imageDataUrl}
className="absolute select-none"
style={{
left: nativeCursorStyle.left,
top: nativeCursorStyle.top,
width: nativeCursorStyle.width,
height: nativeCursorStyle.height,
pointerEvents: "none",
userSelect: "none",
}}
/>
) : null}
{(() => {
const filteredAnnotations = (annotationRegions || []).filter((annotation) => {
if (
@@ -1672,11 +1732,24 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
ref={videoRef}
src={videoPath}
className="hidden"
preload="metadata"
preload="auto"
muted
playsInline
onLoadedMetadata={handleLoadedMetadata}
onDurationChange={(e) => {
onDurationChange(e.currentTarget.duration);
if (!syncResolvedDuration(e.currentTarget)) {
forceResolveDuration(e.currentTarget);
}
}}
onLoadedData={(e) => {
if (!syncResolvedDuration(e.currentTarget)) {
forceResolveDuration(e.currentTarget);
}
}}
onCanPlay={(e) => {
if (!syncResolvedDuration(e.currentTarget)) {
forceResolveDuration(e.currentTarget);
}
}}
onError={() => onError("Failed to load video")}
/>
src/components/video-editor/timeline/TimelineEditor.tsx
+10 -2
View File
@@ -51,6 +51,7 @@ const SUGGESTION_SPACING_MS = 1800;
interface TimelineEditorProps {
videoDuration: number;
hasVideoSource?: boolean;
currentTime: number;
onSeek?: (time: number) => void;
cursorTelemetry?: CursorTelemetryPoint[];
@@ -766,6 +767,7 @@ function Timeline({
export default function TimelineEditor({
videoDuration,
hasVideoSource = false,
currentTime,
onSeek,
cursorTelemetry = [],
@@ -1439,8 +1441,14 @@ export default function TimelineEditor({
<Plus className="w-6 h-6 text-slate-600" />
</div>
<div className="text-center">
<p className="text-sm font-medium text-slate-300">{t("emptyState.noVideo")}</p>
<p className="text-xs text-slate-500 mt-1">{t("emptyState.dragAndDrop")}</p>
<p className="text-sm font-medium text-slate-300">
{hasVideoSource ? "Loading Timeline" : "No Video Loaded"}
</p>
<p className="text-xs text-slate-500 mt-1">
{hasVideoSource
? "Video opened, waiting for duration metadata"
: "Drag and drop a video to start editing"}
</p>
</div>
</div>
);