cursor highlighting and clicks

This commit is contained in:
Siddharth
2026-05-02 23:03:14 -07:00
parent c8d4e867b2
commit 8d79a14e3b
15 changed files with 769 additions and 968 deletions
+5
View File
@@ -3,6 +3,11 @@
"$schema": "https://raw.githubusercontent.com/electron-userland/electron-builder/master/packages/app-builder-lib/scheme.json",
"appId": "com.siddharthvaddem.openscreen",
"asar": true,
// .node binaries can't be dlopen'd from inside an asar — must live unpacked.
"asarUnpack": [
"node_modules/uiohook-napi/**/*",
"**/*.node"
],
"productName": "Openscreen",
"npmRebuild": true,
"buildDependenciesFromSource": true,
+6
View File
@@ -37,6 +37,11 @@ interface Window {
status: string;
error?: string;
}>;
requestAccessibilityAccess: () => Promise<{
success: boolean;
granted: boolean;
error?: string;
}>;
assetBaseUrl: string;
storeRecordedVideo: (
videoData: ArrayBuffer,
@@ -68,6 +73,7 @@ interface Window {
getCursorTelemetry: (videoPath?: string) => Promise<{
success: boolean;
samples: CursorTelemetryPoint[];
clicks: number[];
message?: string;
error?: string;
}>;
+140 -5
View File
@@ -1,6 +1,10 @@
import fs from "node:fs/promises";
import { createRequire } from "node:module";
import path from "node:path";
import { fileURLToPath } from "node:url";
const nodeRequire = createRequire(import.meta.url);
import {
app,
BrowserWindow,
@@ -280,19 +284,24 @@ async function storeRecordedSessionFiles(payload: StoreRecordedSessionInput) {
const telemetryPath = `${screenVideoPath}.cursor.json`;
const pendingBatch = cursorTelemetryBuffer.takeNextBatch();
if (pendingBatch && pendingBatch.samples.length > 0) {
const pendingClicks = takeCursorClickTimestamps();
if ((pendingBatch && pendingBatch.samples.length > 0) || pendingClicks.length > 0) {
try {
await fs.writeFile(
telemetryPath,
JSON.stringify(
{ version: CURSOR_TELEMETRY_VERSION, samples: pendingBatch.samples },
{
version: CURSOR_TELEMETRY_VERSION,
samples: pendingBatch?.samples ?? [],
clicks: pendingClicks,
},
null,
2,
),
"utf-8",
);
} catch (err) {
cursorTelemetryBuffer.prependBatch(pendingBatch);
if (pendingBatch) cursorTelemetryBuffer.prependBatch(pendingBatch);
throw err;
}
}
@@ -321,15 +330,114 @@ const cursorTelemetryBuffer = createCursorTelemetryBuffer({
maxActiveSamples: MAX_CURSOR_SAMPLES,
});
// Mouse click timestamps (macOS only — uiohook-napi behind Accessibility).
const MAX_CURSOR_CLICKS = 60 * 60 * 60; // ~1 click/sec for an hour
let cursorClickTimestampsMs: number[] = [];
let uioHookInstance: {
start: () => void;
stop: () => void;
on: (...a: unknown[]) => void;
off?: (...a: unknown[]) => void;
removeListener?: (...a: unknown[]) => void;
} | null = null;
let uioHookMouseDownHandler: ((event: { time?: number }) => void) | null = null;
let uioHookFailureLogged = false;
function clamp(value: number, min: number, max: number) {
return Math.min(max, Math.max(min, value));
}
function loadUioHookForClicks(): typeof uioHookInstance {
try {
// Dynamic require + try/catch so a broken native binary doesn't crash startup.
const mod = nodeRequire("uiohook-napi");
const candidate = mod.uIOhook ?? mod.default?.uIOhook ?? mod.uiohook ?? mod.default;
if (candidate && typeof candidate.start === "function" && typeof candidate.on === "function") {
return candidate;
}
return null;
} catch (error) {
if (!uioHookFailureLogged) {
uioHookFailureLogged = true;
console.warn("[clickCapture] uiohook-napi unavailable:", error);
}
return null;
}
}
function startClickCapture() {
if (process.platform !== "darwin") return;
if (uioHookInstance) return;
// Passive check — the prompt fires from the renderer when the user toggles
// "Only on clicks" so it doesn't stack with the screen-recording prompt.
try {
if (!systemPreferences.isTrustedAccessibilityClient(false)) {
if (!uioHookFailureLogged) {
uioHookFailureLogged = true;
console.warn(
"[clickCapture] Accessibility permission not granted — click capture disabled.",
);
}
return;
}
} catch {
// fall through; uiohook will fail defensively below
}
const hook = loadUioHookForClicks();
if (!hook) return;
uioHookMouseDownHandler = (event) => {
const elapsed = Math.max(0, Date.now() - cursorCaptureStartTimeMs);
void event;
if (cursorClickTimestampsMs.length >= MAX_CURSOR_CLICKS) return;
cursorClickTimestampsMs.push(elapsed);
};
try {
hook.on("mousedown", uioHookMouseDownHandler);
hook.start();
uioHookInstance = hook;
} catch (error) {
if (!uioHookFailureLogged) {
uioHookFailureLogged = true;
console.warn("[clickCapture] failed to start uiohook:", error);
}
uioHookMouseDownHandler = null;
}
}
function stopClickCapture() {
if (!uioHookInstance) return;
try {
if (uioHookMouseDownHandler) {
if (typeof uioHookInstance.off === "function") {
uioHookInstance.off("mousedown", uioHookMouseDownHandler);
} else if (typeof uioHookInstance.removeListener === "function") {
uioHookInstance.removeListener("mousedown", uioHookMouseDownHandler);
}
}
uioHookInstance.stop();
} catch (error) {
console.warn("[clickCapture] failed to stop uiohook:", error);
}
uioHookInstance = null;
uioHookMouseDownHandler = null;
}
function takeCursorClickTimestamps(): number[] {
const out = cursorClickTimestampsMs;
cursorClickTimestampsMs = [];
return out;
}
function stopCursorCapture() {
if (cursorCaptureInterval) {
clearInterval(cursorCaptureInterval);
cursorCaptureInterval = null;
}
stopClickCapture();
}
function sampleCursorPoint() {
@@ -594,6 +702,22 @@ export function registerIpcHandlers(
}
});
// macOS Accessibility prompt for global click capture. First call shows the
// system dialog; the user has to toggle the app in System Settings (no
// programmatic grant exists for Accessibility).
ipcMain.handle("request-accessibility-access", () => {
if (process.platform !== "darwin") {
return { success: true, granted: true };
}
try {
const granted = systemPreferences.isTrustedAccessibilityClient(true);
return { success: true, granted };
} catch (error) {
console.error("Failed to request accessibility access:", error);
return { success: false, granted: false, error: String(error) };
}
});
ipcMain.handle("open-source-selector", () => {
const sourceSelectorWin = getSourceSelectorWindow();
if (sourceSelectorWin) {
@@ -723,6 +847,8 @@ export function registerIpcHandlers(
const id = typeof recordingId === "number" ? recordingId : Date.now();
cursorTelemetryBuffer.startSession(id);
cursorCaptureStartTimeMs = Date.now();
cursorClickTimestampsMs = [];
startClickCapture();
sampleCursorPoint();
cursorCaptureInterval = setInterval(sampleCursorPoint, CURSOR_SAMPLE_INTERVAL_MS);
} else {
@@ -787,11 +913,19 @@ export function registerIpcHandlers(
})
.sort((a: CursorTelemetryPoint, b: CursorTelemetryPoint) => a.timeMs - b.timeMs);
return { success: true, samples };
const rawClicks = Array.isArray(parsed?.clicks) ? parsed.clicks : [];
const clicks: number[] = rawClicks
.map((value: unknown) =>
typeof value === "number" && Number.isFinite(value) ? Math.max(0, value) : null,
)
.filter((v: number | null): v is number => v !== null)
.sort((a: number, b: number) => a - b);
return { success: true, samples, clicks };
} catch (error) {
const nodeError = error as NodeJS.ErrnoException;
if (nodeError.code === "ENOENT") {
return { success: true, samples: [] };
return { success: true, samples: [], clicks: [] };
}
console.error("Failed to load cursor telemetry:", error);
return {
@@ -799,6 +933,7 @@ export function registerIpcHandlers(
message: "Failed to load cursor telemetry",
error: String(error),
samples: [],
clicks: [],
};
}
});
+3
View File
@@ -40,6 +40,9 @@ contextBridge.exposeInMainWorld("electronAPI", {
requestCameraAccess: () => {
return ipcRenderer.invoke("request-camera-access");
},
requestAccessibilityAccess: () => {
return ipcRenderer.invoke("request-accessibility-access");
},
storeRecordedVideo: (videoData: ArrayBuffer, fileName: string) => {
return ipcRenderer.invoke("store-recorded-video", videoData, fileName);
+93 -961
View File
File diff suppressed because it is too large Load Diff
+5 -1
View File
@@ -29,7 +29,9 @@
"test:browser": "vitest --config vitest.browser.config.ts --run",
"test:browser:install": "playwright install --with-deps chromium-headless-shell",
"test:e2e": "playwright test",
"prepare": "husky"
"prepare": "husky",
"rebuild:native": "node ./node_modules/@electron/rebuild/lib/cli.js --force --only uiohook-napi",
"postinstall": "npm run rebuild:native"
},
"dependencies": {
"@fix-webm-duration/fix": "^1.0.1",
@@ -71,11 +73,13 @@
"sonner": "^2.0.7",
"tailwind-merge": "^3.5.0",
"tailwindcss-animate": "^1.0.7",
"uiohook-napi": "^1.5.5",
"uuid": "^13.0.0",
"web-demuxer": "^4.0.0"
},
"devDependencies": {
"@biomejs/biome": "^2.4.12",
"@electron/rebuild": "^4.0.4",
"@playwright/test": "^1.59.1",
"@testing-library/jest-dom": "^6.9.1",
"@testing-library/react": "^16.3.2",
@@ -1,5 +1,6 @@
import {
Bug,
ChevronDown,
Crop,
Download,
Film,
@@ -22,6 +23,7 @@ import {
AccordionTrigger,
} from "@/components/ui/accordion";
import { Button } from "@/components/ui/button";
import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
import {
Select,
SelectContent,
@@ -151,6 +153,12 @@ const GRADIENTS = [
];
interface SettingsPanelProps {
cursorHighlight?: import("./videoPlayback/cursorHighlight").CursorHighlightConfig;
onCursorHighlightChange?: (
next: import("./videoPlayback/cursorHighlight").CursorHighlightConfig,
) => void;
// macOS only — gates the "Only on clicks" toggle (needs uiohook).
cursorHighlightSupportsClicks?: boolean;
selected: string;
onWallpaperChange: (path: string) => void;
selectedZoomDepth?: ZoomDepth | null;
@@ -238,6 +246,9 @@ const ZOOM_DEPTH_OPTIONS: Array<{ depth: ZoomDepth; label: string }> = [
];
export function SettingsPanel({
cursorHighlight,
onCursorHighlightChange,
cursorHighlightSupportsClicks = false,
selected,
onWallpaperChange,
selectedZoomDepth,
@@ -991,6 +1002,181 @@ export function SettingsPanel({
</div>
</div>
{cursorHighlight && onCursorHighlightChange && (
<div className="p-2 rounded-lg bg-white/5 border border-white/5 mt-2 space-y-2">
<div className="flex items-center justify-between">
<div className="text-[10px] font-medium text-slate-300">Cursor highlight</div>
<button
type="button"
onClick={() =>
onCursorHighlightChange({
...cursorHighlight,
enabled: !cursorHighlight.enabled,
})
}
className={`text-[10px] px-2 py-0.5 rounded border transition-colors ${
cursorHighlight.enabled
? "bg-[#34B27B]/20 border-[#34B27B]/50 text-[#34B27B]"
: "bg-white/5 border-white/10 text-slate-400"
}`}
>
{cursorHighlight.enabled ? "On" : "Off"}
</button>
</div>
<div
className={`grid grid-cols-2 gap-1 ${cursorHighlight.enabled ? "" : "opacity-40 pointer-events-none"}`}
>
{(["dot", "ring"] as const).map((style) => (
<button
key={style}
type="button"
onClick={() => onCursorHighlightChange({ ...cursorHighlight, style })}
className={`text-[10px] px-2 py-1 rounded border capitalize transition-colors ${
cursorHighlight.style === style
? "bg-[#34B27B]/20 border-[#34B27B]/50 text-[#34B27B]"
: "bg-white/5 border-white/10 text-slate-300 hover:border-white/20"
}`}
>
{style}
</button>
))}
</div>
<div className={cursorHighlight.enabled ? "" : "opacity-40 pointer-events-none"}>
<div className="flex items-center justify-between mb-1">
<div className="text-[10px] text-slate-400">Size</div>
<span className="text-[10px] text-slate-500 font-mono">
{cursorHighlight.sizePx}px
</span>
</div>
<Slider
value={[cursorHighlight.sizePx]}
onValueChange={(values) =>
onCursorHighlightChange({ ...cursorHighlight, sizePx: values[0] })
}
min={10}
max={36}
step={1}
className="w-full [&_[role=slider]]:bg-[#34B27B] [&_[role=slider]]:border-[#34B27B] [&_[role=slider]]:h-3 [&_[role=slider]]:w-3"
/>
</div>
{cursorHighlightSupportsClicks && (
<div
className={`flex items-center justify-between ${cursorHighlight.enabled ? "" : "opacity-40 pointer-events-none"}`}
>
<div className="text-[10px] text-slate-400">Only on clicks</div>
<button
type="button"
onClick={async () => {
const turningOn = !cursorHighlight.onlyOnClicks;
if (turningOn) {
try {
const result = await window.electronAPI.requestAccessibilityAccess();
if (!result.granted) {
toast.message("Accessibility permission needed", {
description:
"Open System Settings → Privacy & Security → Accessibility, enable Openscreen, then restart the app.",
});
}
} catch (err) {
console.warn("Accessibility request failed:", err);
}
}
onCursorHighlightChange({
...cursorHighlight,
onlyOnClicks: turningOn,
});
}}
className={`text-[10px] px-2 py-0.5 rounded border transition-colors ${
cursorHighlight.onlyOnClicks
? "bg-[#34B27B]/20 border-[#34B27B]/50 text-[#34B27B]"
: "bg-white/5 border-white/10 text-slate-400"
}`}
>
{cursorHighlight.onlyOnClicks ? "On" : "Off"}
</button>
</div>
)}
<div className={cursorHighlight.enabled ? "" : "opacity-40 pointer-events-none"}>
<div className="text-[10px] text-slate-400 mb-1">Color</div>
<Popover>
<PopoverTrigger asChild>
<Button
variant="outline"
className="w-full h-8 justify-start gap-2 bg-white/5 border-white/10 hover:bg-white/10 px-2"
>
<div
className="w-4 h-4 rounded-full border border-white/20"
style={{ backgroundColor: cursorHighlight.color }}
/>
<span className="text-[10px] text-slate-300 truncate flex-1 text-left font-mono">
{cursorHighlight.color}
</span>
<ChevronDown className="h-3 w-3 opacity-50" />
</Button>
</PopoverTrigger>
<PopoverContent
side="top"
className="w-[260px] p-3 bg-[#1a1a1c] border border-white/10 rounded-xl shadow-xl"
>
<ColorPicker
selectedColor={cursorHighlight.color}
colorPalette={colorPalette}
translations={{
colorWheel: t("background.colorWheel"),
colorPalette: t("background.colorPalette"),
}}
onUpdateColor={(color) =>
onCursorHighlightChange({ ...cursorHighlight, color })
}
/>
</PopoverContent>
</Popover>
</div>
<div className={cursorHighlight.enabled ? "" : "opacity-40 pointer-events-none"}>
<div className="flex items-center justify-between mb-1">
<div className="text-[10px] text-slate-400">Offset X (window recordings)</div>
<span className="text-[10px] text-slate-500 font-mono">
{(cursorHighlight.offsetXNorm * 100).toFixed(1)}%
</span>
</div>
<Slider
value={[cursorHighlight.offsetXNorm]}
onValueChange={(values) =>
onCursorHighlightChange({
...cursorHighlight,
offsetXNorm: values[0],
})
}
min={-0.25}
max={0.25}
step={0.005}
className="w-full [&_[role=slider]]:bg-[#34B27B] [&_[role=slider]]:border-[#34B27B] [&_[role=slider]]:h-3 [&_[role=slider]]:w-3"
/>
</div>
<div className={cursorHighlight.enabled ? "" : "opacity-40 pointer-events-none"}>
<div className="flex items-center justify-between mb-1">
<div className="text-[10px] text-slate-400">Offset Y</div>
<span className="text-[10px] text-slate-500 font-mono">
{(cursorHighlight.offsetYNorm * 100).toFixed(1)}%
</span>
</div>
<Slider
value={[cursorHighlight.offsetYNorm]}
onValueChange={(values) =>
onCursorHighlightChange({
...cursorHighlight,
offsetYNorm: values[0],
})
}
min={-0.25}
max={0.25}
step={0.005}
className="w-full [&_[role=slider]]:bg-[#34B27B] [&_[role=slider]]:border-[#34B27B] [&_[role=slider]]:h-3 [&_[role=slider]]:w-3"
/>
</div>
</div>
)}
<Button
onClick={handleCropToggle}
variant="outline"
@@ -103,6 +103,7 @@ export default function VideoEditor() {
webcamMaskShape,
webcamSizePreset,
webcamPosition,
cursorHighlight,
} = editorState;
// ── Non-undoable state
@@ -121,6 +122,7 @@ export default function VideoEditor() {
const durationRef = useRef(duration);
durationRef.current = duration;
const [cursorTelemetry, setCursorTelemetry] = useState<CursorTelemetryPoint[]>([]);
const [cursorClickTimestamps, setCursorClickTimestamps] = useState<number[]>([]);
const [selectedZoomId, setSelectedZoomId] = useState<string | null>(null);
const [selectedTrimId, setSelectedTrimId] = useState<string | null>(null);
const [selectedSpeedId, setSelectedSpeedId] = useState<string | null>(null);
@@ -153,6 +155,12 @@ export default function VideoEditor() {
const nextSpeedIdRef = useRef(1);
const { shortcuts, isMac } = useShortcuts();
// Off-Mac doesn't have click telemetry, so force `onlyOnClicks` off for
// renderers while keeping the persisted value intact for round-tripping.
const effectiveCursorHighlight = useMemo(
() => (isMac ? cursorHighlight : { ...cursorHighlight, onlyOnClicks: false }),
[cursorHighlight, isMac],
);
const { locale, setLocale, t: rawT } = useI18n();
const t = useScopedT("editor");
const ts = useScopedT("settings");
@@ -452,6 +460,7 @@ export default function VideoEditor() {
gifFrameRate,
gifLoop,
gifSizePreset,
cursorHighlight,
};
const projectData = createProjectData(currentProjectMedia, editorState);
@@ -513,6 +522,7 @@ export default function VideoEditor() {
videoPath,
t,
webcamSizePreset,
cursorHighlight,
],
);
@@ -587,6 +597,7 @@ export default function VideoEditor() {
if (!sourcePath) {
if (mounted) {
setCursorTelemetry([]);
setCursorClickTimestamps([]);
}
return;
}
@@ -595,11 +606,13 @@ export default function VideoEditor() {
const result = await window.electronAPI.getCursorTelemetry(sourcePath);
if (mounted) {
setCursorTelemetry(result.success ? result.samples : []);
setCursorClickTimestamps(result.success ? (result.clicks ?? []) : []);
}
} catch (telemetryError) {
console.warn("Unable to load cursor telemetry:", telemetryError);
if (mounted) {
setCursorTelemetry([]);
setCursorClickTimestamps([]);
}
}
}
@@ -1394,6 +1407,8 @@ export default function VideoEditor() {
previewWidth,
previewHeight,
cursorTelemetry,
cursorClickTimestamps,
cursorHighlight: effectiveCursorHighlight,
onProgress: (progress: ExportProgress) => {
setExportProgress(progress);
},
@@ -1534,6 +1549,8 @@ export default function VideoEditor() {
previewWidth,
previewHeight,
cursorTelemetry,
cursorClickTimestamps,
cursorHighlight: effectiveCursorHighlight,
onProgress: (progress: ExportProgress) => {
setExportProgress(progress);
},
@@ -1617,6 +1634,8 @@ export default function VideoEditor() {
exportQuality,
handleExportSaved,
cursorTelemetry,
cursorClickTimestamps,
effectiveCursorHighlight,
t,
],
);
@@ -1874,6 +1893,8 @@ export default function VideoEditor() {
onBlurDataChange={handleBlurDataPreviewChange}
onBlurDataCommit={commitState}
cursorTelemetry={cursorTelemetry}
cursorHighlight={effectiveCursorHighlight}
cursorClickTimestamps={cursorClickTimestamps}
/>
</div>
</div>
@@ -1957,6 +1978,9 @@ export default function VideoEditor() {
{/* Right section: settings panel */}
<div className="flex-[3] min-w-[280px] max-w-[420px] h-full">
<SettingsPanel
cursorHighlight={cursorHighlight}
onCursorHighlightChange={(next) => pushState({ cursorHighlight: next })}
cursorHighlightSupportsClicks={isMac}
selected={wallpaper}
onWallpaperChange={(w) => pushState({ wallpaper: w })}
selectedZoomDepth={
+73 -1
View File
@@ -51,7 +51,17 @@ import {
ZOOM_SCALE_DEADZONE,
ZOOM_TRANSLATION_DEADZONE_PX,
} from "./videoPlayback/constants";
import { adaptiveSmoothFactor, smoothCursorFocus } from "./videoPlayback/cursorFollowUtils";
import {
adaptiveSmoothFactor,
interpolateCursorAt,
smoothCursorFocus,
} from "./videoPlayback/cursorFollowUtils";
import {
type CursorHighlightConfig,
clickEmphasisAlpha,
DEFAULT_CURSOR_HIGHLIGHT,
drawCursorHighlightGraphics,
} from "./videoPlayback/cursorHighlight";
import { clampFocusToStage as clampFocusToStageUtil } from "./videoPlayback/focusUtils";
import { layoutVideoContent as layoutVideoContentUtil } from "./videoPlayback/layoutUtils";
import { clamp01 } from "./videoPlayback/mathUtils";
@@ -110,6 +120,8 @@ interface VideoPlaybackProps {
onBlurDataChange?: (id: string, blurData: BlurData) => void;
onBlurDataCommit?: () => void;
cursorTelemetry?: import("./types").CursorTelemetryPoint[];
cursorHighlight?: CursorHighlightConfig;
cursorClickTimestamps?: number[];
}
export interface VideoPlaybackRef {
@@ -168,6 +180,8 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
onBlurDataChange,
onBlurDataCommit,
cursorTelemetry = [],
cursorHighlight = DEFAULT_CURSOR_HIGHLIGHT,
cursorClickTimestamps = [],
},
ref,
) => {
@@ -191,6 +205,9 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
const currentTimeRef = useRef(0);
const zoomRegionsRef = useRef<ZoomRegion[]>([]);
const cursorTelemetryRef = useRef<import("./types").CursorTelemetryPoint[]>([]);
const cursorHighlightRef = useRef<CursorHighlightConfig>(DEFAULT_CURSOR_HIGHLIGHT);
const cursorClickTimestampsRef = useRef<number[]>([]);
const cursorHighlightGraphicsRef = useRef<Graphics | null>(null);
const selectedZoomIdRef = useRef<string | null>(null);
const animationStateRef = useRef({
scale: 1,
@@ -515,6 +532,17 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
cursorTelemetryRef.current = cursorTelemetry;
}, [cursorTelemetry]);
useEffect(() => {
cursorHighlightRef.current = cursorHighlight;
if (cursorHighlightGraphicsRef.current) {
drawCursorHighlightGraphics(cursorHighlightGraphicsRef.current, cursorHighlight);
}
}, [cursorHighlight]);
useEffect(() => {
cursorClickTimestampsRef.current = cursorClickTimestamps;
}, [cursorClickTimestamps]);
useEffect(() => {
selectedZoomIdRef.current = selectedZoomId;
}, [selectedZoomId]);
@@ -738,6 +766,12 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
videoContainer.mask = maskGraphics;
maskGraphicsRef.current = maskGraphics;
const cursorHighlightGraphics = new Graphics();
cursorHighlightGraphics.visible = false;
videoContainer.addChild(cursorHighlightGraphics);
cursorHighlightGraphicsRef.current = cursorHighlightGraphics;
drawCursorHighlightGraphics(cursorHighlightGraphics, cursorHighlightRef.current);
animationStateRef.current = {
scale: 1,
focusX: DEFAULT_FOCUS.cx,
@@ -797,6 +831,11 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
videoContainer.removeChild(maskGraphics);
maskGraphics.destroy();
}
if (cursorHighlightGraphicsRef.current) {
videoContainer.removeChild(cursorHighlightGraphicsRef.current);
cursorHighlightGraphicsRef.current.destroy();
cursorHighlightGraphicsRef.current = null;
}
videoContainer.mask = null;
maskGraphicsRef.current = null;
if (blurFilterRef.current) {
@@ -1016,6 +1055,39 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
motionVector,
);
const cursorGraphics = cursorHighlightGraphicsRef.current;
const cursorConfig = cursorHighlightRef.current;
const lockedDims = lockedVideoDimensionsRef.current;
if (cursorGraphics) {
if (cursorConfig.enabled && lockedDims && cursorTelemetryRef.current.length > 0) {
const emphasisAlpha = clickEmphasisAlpha(
currentTimeRef.current,
cursorClickTimestampsRef.current,
cursorConfig,
);
const cursorPoint =
emphasisAlpha > 0
? interpolateCursorAt(cursorTelemetryRef.current, currentTimeRef.current)
: null;
if (cursorPoint) {
const baseScale = baseScaleRef.current;
const baseOffset = baseOffsetRef.current;
const cx = cursorPoint.cx + cursorConfig.offsetXNorm;
const cy = cursorPoint.cy + cursorConfig.offsetYNorm;
cursorGraphics.position.set(
baseOffset.x + cx * lockedDims.width * baseScale,
baseOffset.y + cy * lockedDims.height * baseScale,
);
cursorGraphics.alpha = emphasisAlpha;
cursorGraphics.visible = true;
} else {
cursorGraphics.visible = false;
}
} else {
cursorGraphics.visible = false;
}
}
const isMotionBlurActive = (motionBlurAmountRef.current || 0) > 0 && isPlayingRef.current;
if (isMotionBlurActive !== lastMotionBlurActive && videoContainerRef.current) {
@@ -80,6 +80,7 @@ export interface ProjectEditorState {
gifFrameRate: GifFrameRate;
gifLoop: boolean;
gifSizePreset: GifSizePreset;
cursorHighlight: import("./videoPlayback/cursorHighlight").CursorHighlightConfig;
}
export interface EditorProjectData {
@@ -494,6 +495,52 @@ export function normalizeProjectEditor(editor: Partial<ProjectEditorState>): Pro
editor.gifSizePreset === "original"
? editor.gifSizePreset
: "medium",
cursorHighlight: normalizeCursorHighlight(editor.cursorHighlight),
};
}
function normalizeCursorHighlight(
value: unknown,
): import("./videoPlayback/cursorHighlight").CursorHighlightConfig {
const fallback: import("./videoPlayback/cursorHighlight").CursorHighlightConfig = {
enabled: false,
style: "ring",
sizePx: 24,
color: "#FFD700",
opacity: 0.9,
onlyOnClicks: false,
clickEmphasisDurationMs: 350,
offsetXNorm: 0,
offsetYNorm: 0,
};
if (!value || typeof value !== "object") return fallback;
const v = value as Partial<import("./videoPlayback/cursorHighlight").CursorHighlightConfig>;
return {
enabled: typeof v.enabled === "boolean" ? v.enabled : fallback.enabled,
style: v.style === "dot" || v.style === "ring" ? v.style : fallback.style,
sizePx:
typeof v.sizePx === "number" && v.sizePx >= 10 && v.sizePx <= 36 ? v.sizePx : fallback.sizePx,
color:
typeof v.color === "string" && /^#([0-9a-fA-F]{3}|[0-9a-fA-F]{6})$/.test(v.color)
? v.color
: fallback.color,
opacity:
typeof v.opacity === "number" && v.opacity >= 0 && v.opacity <= 1
? v.opacity
: fallback.opacity,
onlyOnClicks: typeof v.onlyOnClicks === "boolean" ? v.onlyOnClicks : fallback.onlyOnClicks,
clickEmphasisDurationMs:
typeof v.clickEmphasisDurationMs === "number" && v.clickEmphasisDurationMs > 0
? v.clickEmphasisDurationMs
: fallback.clickEmphasisDurationMs,
offsetXNorm:
typeof v.offsetXNorm === "number" && Number.isFinite(v.offsetXNorm)
? Math.max(-1, Math.min(1, v.offsetXNorm))
: fallback.offsetXNorm,
offsetYNorm:
typeof v.offsetYNorm === "number" && Number.isFinite(v.offsetYNorm)
? Math.max(-1, Math.min(1, v.offsetYNorm))
: fallback.offsetYNorm,
};
}
@@ -0,0 +1,125 @@
import type { Graphics } from "pixi.js";
export type CursorHighlightStyle = "dot" | "ring";
export interface CursorHighlightConfig {
enabled: boolean;
style: CursorHighlightStyle;
sizePx: number;
color: string;
opacity: number;
// Show only on clicks (macOS — depends on click telemetry from uiohook).
onlyOnClicks: boolean;
clickEmphasisDurationMs: number;
// Per-recording manual nudge. Cursor telemetry is normalized to the display,
// but window recordings frame a subset of the display so the highlight
// lands offset. Users dial these in once to align with the actual cursor.
offsetXNorm: number;
offsetYNorm: number;
}
export const CURSOR_HIGHLIGHT_MIN_SIZE_PX = 10;
export const CURSOR_HIGHLIGHT_MAX_SIZE_PX = 36;
export const DEFAULT_CURSOR_HIGHLIGHT: CursorHighlightConfig = {
enabled: false,
style: "ring",
sizePx: 24,
color: "#FFD700",
opacity: 0.9,
onlyOnClicks: false,
clickEmphasisDurationMs: 350,
offsetXNorm: 0,
offsetYNorm: 0,
};
export const CURSOR_HIGHLIGHT_OFFSET_RANGE = 0.25; // ±25% of recorded surface
// Alpha multiplier for the highlight at `timeMs`. Returns 1 when not in
// click-only mode; in click-only mode fades 1→0 across each click's window.
export function clickEmphasisAlpha(
timeMs: number,
clickTimestampsMs: number[] | undefined,
config: CursorHighlightConfig,
): number {
if (!config.onlyOnClicks) return 1;
if (!clickTimestampsMs || clickTimestampsMs.length === 0) return 0;
const window = Math.max(1, config.clickEmphasisDurationMs);
for (let i = 0; i < clickTimestampsMs.length; i++) {
const dt = timeMs - clickTimestampsMs[i];
if (dt >= 0 && dt <= window) {
return 1 - dt / window;
}
}
return 0;
}
function parseHexColor(hex: string): number {
const cleaned = hex.replace("#", "");
if (cleaned.length === 3) {
const r = cleaned[0];
const g = cleaned[1];
const b = cleaned[2];
return Number.parseInt(`${r}${r}${g}${g}${b}${b}`, 16);
}
return Number.parseInt(cleaned.slice(0, 6), 16);
}
export function drawCursorHighlightGraphics(g: Graphics, config: CursorHighlightConfig): void {
g.clear();
if (!config.enabled) return;
const color = parseHexColor(config.color);
const radius = Math.max(1, config.sizePx / 2);
const alpha = Math.max(0, Math.min(1, config.opacity));
switch (config.style) {
case "dot": {
g.circle(0, 0, radius);
g.fill({ color, alpha });
break;
}
case "ring": {
g.circle(0, 0, radius);
g.stroke({ color, alpha, width: Math.max(2, radius * 0.18) });
break;
}
}
}
export function drawCursorHighlightCanvas(
ctx: CanvasRenderingContext2D,
cx: number,
cy: number,
config: CursorHighlightConfig,
pixelScale = 1,
): void {
if (!config.enabled) return;
const radius = Math.max(1, (config.sizePx / 2) * pixelScale);
const alpha = Math.max(0, Math.min(1, config.opacity));
const color = config.color;
ctx.save();
ctx.globalAlpha = alpha;
switch (config.style) {
case "dot": {
ctx.fillStyle = color;
ctx.beginPath();
ctx.arc(cx, cy, radius, 0, Math.PI * 2);
ctx.fill();
break;
}
case "ring": {
ctx.beginPath();
ctx.arc(cx, cy, radius, 0, Math.PI * 2);
ctx.strokeStyle = color;
ctx.lineWidth = Math.max(2, radius * 0.18);
ctx.stroke();
break;
}
}
ctx.restore();
}
+6
View File
@@ -17,6 +17,10 @@ import {
DEFAULT_WEBCAM_POSITION,
DEFAULT_WEBCAM_SIZE_PRESET,
} from "@/components/video-editor/types";
import {
type CursorHighlightConfig,
DEFAULT_CURSOR_HIGHLIGHT,
} from "@/components/video-editor/videoPlayback/cursorHighlight";
import { DEFAULT_WALLPAPER } from "@/lib/wallpaper";
import type { AspectRatio } from "@/utils/aspectRatioUtils";
@@ -39,6 +43,7 @@ export interface EditorState {
webcamMaskShape: WebcamMaskShape;
webcamSizePreset: WebcamSizePreset;
webcamPosition: WebcamPosition | null;
cursorHighlight: CursorHighlightConfig;
}
export const INITIAL_EDITOR_STATE: EditorState = {
@@ -58,6 +63,7 @@ export const INITIAL_EDITOR_STATE: EditorState = {
webcamMaskShape: DEFAULT_WEBCAM_MASK_SHAPE,
webcamSizePreset: DEFAULT_WEBCAM_SIZE_PRESET,
webcamPosition: DEFAULT_WEBCAM_POSITION,
cursorHighlight: DEFAULT_CURSOR_HIGHLIGHT,
};
type StateUpdate = Partial<EditorState> | ((prev: EditorState) => Partial<EditorState>);
+48
View File
@@ -28,8 +28,14 @@ import {
} from "@/components/video-editor/videoPlayback/constants";
import {
adaptiveSmoothFactor,
interpolateCursorAt,
smoothCursorFocus,
} from "@/components/video-editor/videoPlayback/cursorFollowUtils";
import {
type CursorHighlightConfig,
clickEmphasisAlpha,
drawCursorHighlightCanvas,
} from "@/components/video-editor/videoPlayback/cursorHighlight";
import { clampFocusToStage as clampFocusToStageUtil } from "@/components/video-editor/videoPlayback/focusUtils";
import { findDominantRegion } from "@/components/video-editor/videoPlayback/zoomRegionUtils";
import {
@@ -79,6 +85,8 @@ interface FrameRenderConfig {
previewWidth?: number;
previewHeight?: number;
cursorTelemetry?: import("@/components/video-editor/types").CursorTelemetryPoint[];
cursorHighlight?: CursorHighlightConfig;
cursorClickTimestamps?: number[];
platform: string;
}
@@ -387,6 +395,46 @@ export class FrameRenderer {
// Composite with shadows to final output canvas
this.compositeWithShadows(webcamFrame);
// Cursor highlight overlay (rendered above video, below annotations)
if (
this.config.cursorHighlight?.enabled &&
this.config.cursorTelemetry &&
this.config.cursorTelemetry.length > 0 &&
this.compositeCtx
) {
const emphasisAlpha = clickEmphasisAlpha(
timeMs,
this.config.cursorClickTimestamps,
this.config.cursorHighlight,
);
const cursorPoint =
emphasisAlpha > 0 ? interpolateCursorAt(this.config.cursorTelemetry, timeMs) : null;
if (cursorPoint) {
const cx = cursorPoint.cx + this.config.cursorHighlight.offsetXNorm;
const cy = cursorPoint.cy + this.config.cursorHighlight.offsetYNorm;
const stageX =
layoutCache.baseOffset.x + cx * this.config.videoWidth * layoutCache.baseScale;
const stageY =
layoutCache.baseOffset.y + cy * this.config.videoHeight * layoutCache.baseScale;
const appliedScale = this.animationState.appliedScale;
const canvasX = stageX * appliedScale + this.animationState.x;
const canvasY = stageY * appliedScale + this.animationState.y;
const previewW = this.config.previewWidth ?? this.config.width;
const previewH = this.config.previewHeight ?? this.config.height;
const cursorScale = (this.config.width / previewW + this.config.height / previewH) / 2;
drawCursorHighlightCanvas(
this.compositeCtx,
canvasX,
canvasY,
{
...this.config.cursorHighlight,
opacity: this.config.cursorHighlight.opacity * emphasisAlpha,
},
appliedScale * cursorScale,
);
}
}
// Render annotations on top if present
if (
this.config.annotationRegions &&
+4
View File
@@ -51,6 +51,8 @@ interface GifExporterConfig {
previewWidth?: number;
previewHeight?: number;
cursorTelemetry?: import("@/components/video-editor/types").CursorTelemetryPoint[];
cursorHighlight?: import("@/components/video-editor/videoPlayback/cursorHighlight").CursorHighlightConfig;
cursorClickTimestamps?: number[];
onProgress?: (progress: ExportProgress) => void;
}
@@ -161,6 +163,8 @@ export class GifExporter {
previewWidth: this.config.previewWidth,
previewHeight: this.config.previewHeight,
cursorTelemetry: this.config.cursorTelemetry,
cursorClickTimestamps: this.config.cursorClickTimestamps,
cursorHighlight: this.config.cursorHighlight,
platform,
});
await this.renderer.initialize();
+4
View File
@@ -42,6 +42,8 @@ interface VideoExporterConfig extends ExportConfig {
previewWidth?: number;
previewHeight?: number;
cursorTelemetry?: import("@/components/video-editor/types").CursorTelemetryPoint[];
cursorHighlight?: import("@/components/video-editor/videoPlayback/cursorHighlight").CursorHighlightConfig;
cursorClickTimestamps?: number[];
onProgress?: (progress: ExportProgress) => void;
}
@@ -156,6 +158,8 @@ export class VideoExporter {
previewWidth: this.config.previewWidth,
previewHeight: this.config.previewHeight,
cursorTelemetry: this.config.cursorTelemetry,
cursorClickTimestamps: this.config.cursorClickTimestamps,
cursorHighlight: this.config.cursorHighlight,
platform,
});
this.renderer = renderer;