cursor highlighting and clicks

This commit is contained in:
Siddharth
2026-05-02 23:03:14 -07:00
parent c8d4e867b2
commit 8d79a14e3b
15 changed files with 769 additions and 968 deletions
@@ -1,5 +1,6 @@
import {
Bug,
ChevronDown,
Crop,
Download,
Film,
@@ -22,6 +23,7 @@ import {
AccordionTrigger,
} from "@/components/ui/accordion";
import { Button } from "@/components/ui/button";
import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
import {
Select,
SelectContent,
@@ -151,6 +153,12 @@ const GRADIENTS = [
];
interface SettingsPanelProps {
cursorHighlight?: import("./videoPlayback/cursorHighlight").CursorHighlightConfig;
onCursorHighlightChange?: (
next: import("./videoPlayback/cursorHighlight").CursorHighlightConfig,
) => void;
// macOS only — gates the "Only on clicks" toggle (needs uiohook).
cursorHighlightSupportsClicks?: boolean;
selected: string;
onWallpaperChange: (path: string) => void;
selectedZoomDepth?: ZoomDepth | null;
@@ -238,6 +246,9 @@ const ZOOM_DEPTH_OPTIONS: Array<{ depth: ZoomDepth; label: string }> = [
];
export function SettingsPanel({
cursorHighlight,
onCursorHighlightChange,
cursorHighlightSupportsClicks = false,
selected,
onWallpaperChange,
selectedZoomDepth,
@@ -991,6 +1002,181 @@ export function SettingsPanel({
</div>
</div>
{cursorHighlight && onCursorHighlightChange && (
<div className="p-2 rounded-lg bg-white/5 border border-white/5 mt-2 space-y-2">
<div className="flex items-center justify-between">
<div className="text-[10px] font-medium text-slate-300">Cursor highlight</div>
<button
type="button"
onClick={() =>
onCursorHighlightChange({
...cursorHighlight,
enabled: !cursorHighlight.enabled,
})
}
className={`text-[10px] px-2 py-0.5 rounded border transition-colors ${
cursorHighlight.enabled
? "bg-[#34B27B]/20 border-[#34B27B]/50 text-[#34B27B]"
: "bg-white/5 border-white/10 text-slate-400"
}`}
>
{cursorHighlight.enabled ? "On" : "Off"}
</button>
</div>
<div
className={`grid grid-cols-2 gap-1 ${cursorHighlight.enabled ? "" : "opacity-40 pointer-events-none"}`}
>
{(["dot", "ring"] as const).map((style) => (
<button
key={style}
type="button"
onClick={() => onCursorHighlightChange({ ...cursorHighlight, style })}
className={`text-[10px] px-2 py-1 rounded border capitalize transition-colors ${
cursorHighlight.style === style
? "bg-[#34B27B]/20 border-[#34B27B]/50 text-[#34B27B]"
: "bg-white/5 border-white/10 text-slate-300 hover:border-white/20"
}`}
>
{style}
</button>
))}
</div>
<div className={cursorHighlight.enabled ? "" : "opacity-40 pointer-events-none"}>
<div className="flex items-center justify-between mb-1">
<div className="text-[10px] text-slate-400">Size</div>
<span className="text-[10px] text-slate-500 font-mono">
{cursorHighlight.sizePx}px
</span>
</div>
<Slider
value={[cursorHighlight.sizePx]}
onValueChange={(values) =>
onCursorHighlightChange({ ...cursorHighlight, sizePx: values[0] })
}
min={10}
max={36}
step={1}
className="w-full [&_[role=slider]]:bg-[#34B27B] [&_[role=slider]]:border-[#34B27B] [&_[role=slider]]:h-3 [&_[role=slider]]:w-3"
/>
</div>
{cursorHighlightSupportsClicks && (
<div
className={`flex items-center justify-between ${cursorHighlight.enabled ? "" : "opacity-40 pointer-events-none"}`}
>
<div className="text-[10px] text-slate-400">Only on clicks</div>
<button
type="button"
onClick={async () => {
const turningOn = !cursorHighlight.onlyOnClicks;
if (turningOn) {
try {
const result = await window.electronAPI.requestAccessibilityAccess();
if (!result.granted) {
toast.message("Accessibility permission needed", {
description:
"Open System Settings → Privacy & Security → Accessibility, enable Openscreen, then restart the app.",
});
}
} catch (err) {
console.warn("Accessibility request failed:", err);
}
}
onCursorHighlightChange({
...cursorHighlight,
onlyOnClicks: turningOn,
});
}}
className={`text-[10px] px-2 py-0.5 rounded border transition-colors ${
cursorHighlight.onlyOnClicks
? "bg-[#34B27B]/20 border-[#34B27B]/50 text-[#34B27B]"
: "bg-white/5 border-white/10 text-slate-400"
}`}
>
{cursorHighlight.onlyOnClicks ? "On" : "Off"}
</button>
</div>
)}
<div className={cursorHighlight.enabled ? "" : "opacity-40 pointer-events-none"}>
<div className="text-[10px] text-slate-400 mb-1">Color</div>
<Popover>
<PopoverTrigger asChild>
<Button
variant="outline"
className="w-full h-8 justify-start gap-2 bg-white/5 border-white/10 hover:bg-white/10 px-2"
>
<div
className="w-4 h-4 rounded-full border border-white/20"
style={{ backgroundColor: cursorHighlight.color }}
/>
<span className="text-[10px] text-slate-300 truncate flex-1 text-left font-mono">
{cursorHighlight.color}
</span>
<ChevronDown className="h-3 w-3 opacity-50" />
</Button>
</PopoverTrigger>
<PopoverContent
side="top"
className="w-[260px] p-3 bg-[#1a1a1c] border border-white/10 rounded-xl shadow-xl"
>
<ColorPicker
selectedColor={cursorHighlight.color}
colorPalette={colorPalette}
translations={{
colorWheel: t("background.colorWheel"),
colorPalette: t("background.colorPalette"),
}}
onUpdateColor={(color) =>
onCursorHighlightChange({ ...cursorHighlight, color })
}
/>
</PopoverContent>
</Popover>
</div>
<div className={cursorHighlight.enabled ? "" : "opacity-40 pointer-events-none"}>
<div className="flex items-center justify-between mb-1">
<div className="text-[10px] text-slate-400">Offset X (window recordings)</div>
<span className="text-[10px] text-slate-500 font-mono">
{(cursorHighlight.offsetXNorm * 100).toFixed(1)}%
</span>
</div>
<Slider
value={[cursorHighlight.offsetXNorm]}
onValueChange={(values) =>
onCursorHighlightChange({
...cursorHighlight,
offsetXNorm: values[0],
})
}
min={-0.25}
max={0.25}
step={0.005}
className="w-full [&_[role=slider]]:bg-[#34B27B] [&_[role=slider]]:border-[#34B27B] [&_[role=slider]]:h-3 [&_[role=slider]]:w-3"
/>
</div>
<div className={cursorHighlight.enabled ? "" : "opacity-40 pointer-events-none"}>
<div className="flex items-center justify-between mb-1">
<div className="text-[10px] text-slate-400">Offset Y</div>
<span className="text-[10px] text-slate-500 font-mono">
{(cursorHighlight.offsetYNorm * 100).toFixed(1)}%
</span>
</div>
<Slider
value={[cursorHighlight.offsetYNorm]}
onValueChange={(values) =>
onCursorHighlightChange({
...cursorHighlight,
offsetYNorm: values[0],
})
}
min={-0.25}
max={0.25}
step={0.005}
className="w-full [&_[role=slider]]:bg-[#34B27B] [&_[role=slider]]:border-[#34B27B] [&_[role=slider]]:h-3 [&_[role=slider]]:w-3"
/>
</div>
</div>
)}
<Button
onClick={handleCropToggle}
variant="outline"
@@ -103,6 +103,7 @@ export default function VideoEditor() {
webcamMaskShape,
webcamSizePreset,
webcamPosition,
cursorHighlight,
} = editorState;
// ── Non-undoable state
@@ -121,6 +122,7 @@ export default function VideoEditor() {
const durationRef = useRef(duration);
durationRef.current = duration;
const [cursorTelemetry, setCursorTelemetry] = useState<CursorTelemetryPoint[]>([]);
const [cursorClickTimestamps, setCursorClickTimestamps] = useState<number[]>([]);
const [selectedZoomId, setSelectedZoomId] = useState<string | null>(null);
const [selectedTrimId, setSelectedTrimId] = useState<string | null>(null);
const [selectedSpeedId, setSelectedSpeedId] = useState<string | null>(null);
@@ -153,6 +155,12 @@ export default function VideoEditor() {
const nextSpeedIdRef = useRef(1);
const { shortcuts, isMac } = useShortcuts();
// Off-Mac doesn't have click telemetry, so force `onlyOnClicks` off for
// renderers while keeping the persisted value intact for round-tripping.
const effectiveCursorHighlight = useMemo(
() => (isMac ? cursorHighlight : { ...cursorHighlight, onlyOnClicks: false }),
[cursorHighlight, isMac],
);
const { locale, setLocale, t: rawT } = useI18n();
const t = useScopedT("editor");
const ts = useScopedT("settings");
@@ -452,6 +460,7 @@ export default function VideoEditor() {
gifFrameRate,
gifLoop,
gifSizePreset,
cursorHighlight,
};
const projectData = createProjectData(currentProjectMedia, editorState);
@@ -513,6 +522,7 @@ export default function VideoEditor() {
videoPath,
t,
webcamSizePreset,
cursorHighlight,
],
);
@@ -587,6 +597,7 @@ export default function VideoEditor() {
if (!sourcePath) {
if (mounted) {
setCursorTelemetry([]);
setCursorClickTimestamps([]);
}
return;
}
@@ -595,11 +606,13 @@ export default function VideoEditor() {
const result = await window.electronAPI.getCursorTelemetry(sourcePath);
if (mounted) {
setCursorTelemetry(result.success ? result.samples : []);
setCursorClickTimestamps(result.success ? (result.clicks ?? []) : []);
}
} catch (telemetryError) {
console.warn("Unable to load cursor telemetry:", telemetryError);
if (mounted) {
setCursorTelemetry([]);
setCursorClickTimestamps([]);
}
}
}
@@ -1394,6 +1407,8 @@ export default function VideoEditor() {
previewWidth,
previewHeight,
cursorTelemetry,
cursorClickTimestamps,
cursorHighlight: effectiveCursorHighlight,
onProgress: (progress: ExportProgress) => {
setExportProgress(progress);
},
@@ -1534,6 +1549,8 @@ export default function VideoEditor() {
previewWidth,
previewHeight,
cursorTelemetry,
cursorClickTimestamps,
cursorHighlight: effectiveCursorHighlight,
onProgress: (progress: ExportProgress) => {
setExportProgress(progress);
},
@@ -1617,6 +1634,8 @@ export default function VideoEditor() {
exportQuality,
handleExportSaved,
cursorTelemetry,
cursorClickTimestamps,
effectiveCursorHighlight,
t,
],
);
@@ -1874,6 +1893,8 @@ export default function VideoEditor() {
onBlurDataChange={handleBlurDataPreviewChange}
onBlurDataCommit={commitState}
cursorTelemetry={cursorTelemetry}
cursorHighlight={effectiveCursorHighlight}
cursorClickTimestamps={cursorClickTimestamps}
/>
</div>
</div>
@@ -1957,6 +1978,9 @@ export default function VideoEditor() {
{/* Right section: settings panel */}
<div className="flex-[3] min-w-[280px] max-w-[420px] h-full">
<SettingsPanel
cursorHighlight={cursorHighlight}
onCursorHighlightChange={(next) => pushState({ cursorHighlight: next })}
cursorHighlightSupportsClicks={isMac}
selected={wallpaper}
onWallpaperChange={(w) => pushState({ wallpaper: w })}
selectedZoomDepth={
+73 -1
View File
@@ -51,7 +51,17 @@ import {
ZOOM_SCALE_DEADZONE,
ZOOM_TRANSLATION_DEADZONE_PX,
} from "./videoPlayback/constants";
import { adaptiveSmoothFactor, smoothCursorFocus } from "./videoPlayback/cursorFollowUtils";
import {
adaptiveSmoothFactor,
interpolateCursorAt,
smoothCursorFocus,
} from "./videoPlayback/cursorFollowUtils";
import {
type CursorHighlightConfig,
clickEmphasisAlpha,
DEFAULT_CURSOR_HIGHLIGHT,
drawCursorHighlightGraphics,
} from "./videoPlayback/cursorHighlight";
import { clampFocusToStage as clampFocusToStageUtil } from "./videoPlayback/focusUtils";
import { layoutVideoContent as layoutVideoContentUtil } from "./videoPlayback/layoutUtils";
import { clamp01 } from "./videoPlayback/mathUtils";
@@ -110,6 +120,8 @@ interface VideoPlaybackProps {
onBlurDataChange?: (id: string, blurData: BlurData) => void;
onBlurDataCommit?: () => void;
cursorTelemetry?: import("./types").CursorTelemetryPoint[];
cursorHighlight?: CursorHighlightConfig;
cursorClickTimestamps?: number[];
}
export interface VideoPlaybackRef {
@@ -168,6 +180,8 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
onBlurDataChange,
onBlurDataCommit,
cursorTelemetry = [],
cursorHighlight = DEFAULT_CURSOR_HIGHLIGHT,
cursorClickTimestamps = [],
},
ref,
) => {
@@ -191,6 +205,9 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
const currentTimeRef = useRef(0);
const zoomRegionsRef = useRef<ZoomRegion[]>([]);
const cursorTelemetryRef = useRef<import("./types").CursorTelemetryPoint[]>([]);
const cursorHighlightRef = useRef<CursorHighlightConfig>(DEFAULT_CURSOR_HIGHLIGHT);
const cursorClickTimestampsRef = useRef<number[]>([]);
const cursorHighlightGraphicsRef = useRef<Graphics | null>(null);
const selectedZoomIdRef = useRef<string | null>(null);
const animationStateRef = useRef({
scale: 1,
@@ -515,6 +532,17 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
cursorTelemetryRef.current = cursorTelemetry;
}, [cursorTelemetry]);
useEffect(() => {
cursorHighlightRef.current = cursorHighlight;
if (cursorHighlightGraphicsRef.current) {
drawCursorHighlightGraphics(cursorHighlightGraphicsRef.current, cursorHighlight);
}
}, [cursorHighlight]);
useEffect(() => {
cursorClickTimestampsRef.current = cursorClickTimestamps;
}, [cursorClickTimestamps]);
useEffect(() => {
selectedZoomIdRef.current = selectedZoomId;
}, [selectedZoomId]);
@@ -738,6 +766,12 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
videoContainer.mask = maskGraphics;
maskGraphicsRef.current = maskGraphics;
const cursorHighlightGraphics = new Graphics();
cursorHighlightGraphics.visible = false;
videoContainer.addChild(cursorHighlightGraphics);
cursorHighlightGraphicsRef.current = cursorHighlightGraphics;
drawCursorHighlightGraphics(cursorHighlightGraphics, cursorHighlightRef.current);
animationStateRef.current = {
scale: 1,
focusX: DEFAULT_FOCUS.cx,
@@ -797,6 +831,11 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
videoContainer.removeChild(maskGraphics);
maskGraphics.destroy();
}
if (cursorHighlightGraphicsRef.current) {
videoContainer.removeChild(cursorHighlightGraphicsRef.current);
cursorHighlightGraphicsRef.current.destroy();
cursorHighlightGraphicsRef.current = null;
}
videoContainer.mask = null;
maskGraphicsRef.current = null;
if (blurFilterRef.current) {
@@ -1016,6 +1055,39 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
motionVector,
);
const cursorGraphics = cursorHighlightGraphicsRef.current;
const cursorConfig = cursorHighlightRef.current;
const lockedDims = lockedVideoDimensionsRef.current;
if (cursorGraphics) {
if (cursorConfig.enabled && lockedDims && cursorTelemetryRef.current.length > 0) {
const emphasisAlpha = clickEmphasisAlpha(
currentTimeRef.current,
cursorClickTimestampsRef.current,
cursorConfig,
);
const cursorPoint =
emphasisAlpha > 0
? interpolateCursorAt(cursorTelemetryRef.current, currentTimeRef.current)
: null;
if (cursorPoint) {
const baseScale = baseScaleRef.current;
const baseOffset = baseOffsetRef.current;
const cx = cursorPoint.cx + cursorConfig.offsetXNorm;
const cy = cursorPoint.cy + cursorConfig.offsetYNorm;
cursorGraphics.position.set(
baseOffset.x + cx * lockedDims.width * baseScale,
baseOffset.y + cy * lockedDims.height * baseScale,
);
cursorGraphics.alpha = emphasisAlpha;
cursorGraphics.visible = true;
} else {
cursorGraphics.visible = false;
}
} else {
cursorGraphics.visible = false;
}
}
const isMotionBlurActive = (motionBlurAmountRef.current || 0) > 0 && isPlayingRef.current;
if (isMotionBlurActive !== lastMotionBlurActive && videoContainerRef.current) {
@@ -80,6 +80,7 @@ export interface ProjectEditorState {
gifFrameRate: GifFrameRate;
gifLoop: boolean;
gifSizePreset: GifSizePreset;
cursorHighlight: import("./videoPlayback/cursorHighlight").CursorHighlightConfig;
}
export interface EditorProjectData {
@@ -494,6 +495,52 @@ export function normalizeProjectEditor(editor: Partial<ProjectEditorState>): Pro
editor.gifSizePreset === "original"
? editor.gifSizePreset
: "medium",
cursorHighlight: normalizeCursorHighlight(editor.cursorHighlight),
};
}
function normalizeCursorHighlight(
value: unknown,
): import("./videoPlayback/cursorHighlight").CursorHighlightConfig {
const fallback: import("./videoPlayback/cursorHighlight").CursorHighlightConfig = {
enabled: false,
style: "ring",
sizePx: 24,
color: "#FFD700",
opacity: 0.9,
onlyOnClicks: false,
clickEmphasisDurationMs: 350,
offsetXNorm: 0,
offsetYNorm: 0,
};
if (!value || typeof value !== "object") return fallback;
const v = value as Partial<import("./videoPlayback/cursorHighlight").CursorHighlightConfig>;
return {
enabled: typeof v.enabled === "boolean" ? v.enabled : fallback.enabled,
style: v.style === "dot" || v.style === "ring" ? v.style : fallback.style,
sizePx:
typeof v.sizePx === "number" && v.sizePx >= 10 && v.sizePx <= 36 ? v.sizePx : fallback.sizePx,
color:
typeof v.color === "string" && /^#([0-9a-fA-F]{3}|[0-9a-fA-F]{6})$/.test(v.color)
? v.color
: fallback.color,
opacity:
typeof v.opacity === "number" && v.opacity >= 0 && v.opacity <= 1
? v.opacity
: fallback.opacity,
onlyOnClicks: typeof v.onlyOnClicks === "boolean" ? v.onlyOnClicks : fallback.onlyOnClicks,
clickEmphasisDurationMs:
typeof v.clickEmphasisDurationMs === "number" && v.clickEmphasisDurationMs > 0
? v.clickEmphasisDurationMs
: fallback.clickEmphasisDurationMs,
offsetXNorm:
typeof v.offsetXNorm === "number" && Number.isFinite(v.offsetXNorm)
? Math.max(-1, Math.min(1, v.offsetXNorm))
: fallback.offsetXNorm,
offsetYNorm:
typeof v.offsetYNorm === "number" && Number.isFinite(v.offsetYNorm)
? Math.max(-1, Math.min(1, v.offsetYNorm))
: fallback.offsetYNorm,
};
}
@@ -0,0 +1,125 @@
import type { Graphics } from "pixi.js";
export type CursorHighlightStyle = "dot" | "ring";
export interface CursorHighlightConfig {
enabled: boolean;
style: CursorHighlightStyle;
sizePx: number;
color: string;
opacity: number;
// Show only on clicks (macOS — depends on click telemetry from uiohook).
onlyOnClicks: boolean;
clickEmphasisDurationMs: number;
// Per-recording manual nudge. Cursor telemetry is normalized to the display,
// but window recordings frame a subset of the display so the highlight
// lands offset. Users dial these in once to align with the actual cursor.
offsetXNorm: number;
offsetYNorm: number;
}
export const CURSOR_HIGHLIGHT_MIN_SIZE_PX = 10;
export const CURSOR_HIGHLIGHT_MAX_SIZE_PX = 36;
export const DEFAULT_CURSOR_HIGHLIGHT: CursorHighlightConfig = {
enabled: false,
style: "ring",
sizePx: 24,
color: "#FFD700",
opacity: 0.9,
onlyOnClicks: false,
clickEmphasisDurationMs: 350,
offsetXNorm: 0,
offsetYNorm: 0,
};
export const CURSOR_HIGHLIGHT_OFFSET_RANGE = 0.25; // ±25% of recorded surface
// Alpha multiplier for the highlight at `timeMs`. Returns 1 when not in
// click-only mode; in click-only mode fades 1→0 across each click's window.
export function clickEmphasisAlpha(
timeMs: number,
clickTimestampsMs: number[] | undefined,
config: CursorHighlightConfig,
): number {
if (!config.onlyOnClicks) return 1;
if (!clickTimestampsMs || clickTimestampsMs.length === 0) return 0;
const window = Math.max(1, config.clickEmphasisDurationMs);
for (let i = 0; i < clickTimestampsMs.length; i++) {
const dt = timeMs - clickTimestampsMs[i];
if (dt >= 0 && dt <= window) {
return 1 - dt / window;
}
}
return 0;
}
function parseHexColor(hex: string): number {
const cleaned = hex.replace("#", "");
if (cleaned.length === 3) {
const r = cleaned[0];
const g = cleaned[1];
const b = cleaned[2];
return Number.parseInt(`${r}${r}${g}${g}${b}${b}`, 16);
}
return Number.parseInt(cleaned.slice(0, 6), 16);
}
export function drawCursorHighlightGraphics(g: Graphics, config: CursorHighlightConfig): void {
g.clear();
if (!config.enabled) return;
const color = parseHexColor(config.color);
const radius = Math.max(1, config.sizePx / 2);
const alpha = Math.max(0, Math.min(1, config.opacity));
switch (config.style) {
case "dot": {
g.circle(0, 0, radius);
g.fill({ color, alpha });
break;
}
case "ring": {
g.circle(0, 0, radius);
g.stroke({ color, alpha, width: Math.max(2, radius * 0.18) });
break;
}
}
}
export function drawCursorHighlightCanvas(
ctx: CanvasRenderingContext2D,
cx: number,
cy: number,
config: CursorHighlightConfig,
pixelScale = 1,
): void {
if (!config.enabled) return;
const radius = Math.max(1, (config.sizePx / 2) * pixelScale);
const alpha = Math.max(0, Math.min(1, config.opacity));
const color = config.color;
ctx.save();
ctx.globalAlpha = alpha;
switch (config.style) {
case "dot": {
ctx.fillStyle = color;
ctx.beginPath();
ctx.arc(cx, cy, radius, 0, Math.PI * 2);
ctx.fill();
break;
}
case "ring": {
ctx.beginPath();
ctx.arc(cx, cy, radius, 0, Math.PI * 2);
ctx.strokeStyle = color;
ctx.lineWidth = Math.max(2, radius * 0.18);
ctx.stroke();
break;
}
}
ctx.restore();
}