fix: improve macOS HUD interactions and audio preview

electron/electron-env.d.ts

@@ -197,6 +197,12 @@ interface Window {
 			message?: string;
 			error?: string;
 		}>;
+		preparePreviewAudioTrack: (filePath: string) => Promise<{
+			success: boolean;
+			path?: string | null;
+			message?: string;
+			error?: string;
+		}>;
 		clearCurrentVideoPath: () => Promise<{ success: boolean }>;
 		saveProjectFile: (
 			projectData: unknown,
@@ -237,6 +243,7 @@ interface Window {
 		hudOverlayHide: () => void;
 		hudOverlayClose: () => void;
 		setHudOverlayIgnoreMouseEvents: (ignore: boolean) => void;
+		moveHudOverlayBy: (deltaX: number, deltaY: number) => void;
 		showCountdownOverlay: (value: number, runId: number) => Promise<void>;
 		setCountdownOverlayValue: (value: number, runId: number) => Promise<void>;
 		hideCountdownOverlay: (runId: number) => Promise<void>;

electron/ipc/handlers.ts

@@ -46,6 +46,7 @@ const SHORTCUTS_FILE = path.join(app.getPath("userData"), "shortcuts.json");
 const RECORDING_FILE_PREFIX = "recording-";
 const RECORDING_SESSION_SUFFIX = ".session.json";
 const ALLOWED_IMPORT_VIDEO_EXTENSIONS = new Set([".webm", ".mp4", ".mov", ".avi", ".mkv"]);
+const PREVIEW_AUDIO_DIR = path.join(app.getPath("userData"), "preview-audio");
 
 /**
  * Paths explicitly approved by the user via file picker dialogs or project loads.
@@ -105,6 +106,102 @@ function hasAllowedImportVideoExtension(filePath: string): boolean {
 	return ALLOWED_IMPORT_VIDEO_EXTENSIONS.has(path.extname(filePath).toLowerCase());
 }
 
+function runProcess(
+	command: string,
+	args: string[],
+): Promise<{ code: number | null; stdout: string; stderr: string }> {
+	return new Promise((resolve, reject) => {
+		const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] });
+		let stdout = "";
+		let stderr = "";
+		child.stdout.on("data", (chunk) => {
+			stdout += chunk.toString();
+		});
+		child.stderr.on("data", (chunk) => {
+			stderr += chunk.toString();
+		});
+		child.on("error", reject);
+		child.on("close", (code) => resolve({ code, stdout, stderr }));
+	});
+}
+
+function parseAfinfoAudioTrackBitrates(output: string): number[] {
+	const bitrates: number[] = [];
+	const trackSections = output.split(/\n----\n/g).slice(1);
+	for (const section of trackSections) {
+		const match = section.match(/\bbit rate:\s*([0-9]+)\s*bits per second/i);
+		bitrates.push(match ? Number(match[1]) : 0);
+	}
+	return bitrates;
+}
+
+async function prepareSupplementalPreviewAudioTrack(videoPath: string) {
+	const normalizedPath = await approveReadableVideoPath(videoPath);
+	if (!normalizedPath) {
+		return {
+			success: false,
+			message: "File path is not approved or is not a supported video file",
+		};
+	}
+
+	if (process.platform !== "darwin" || path.extname(normalizedPath).toLowerCase() !== ".mp4") {
+		return { success: true, path: null };
+	}
+
+	const afinfo = await runProcess("/usr/bin/afinfo", [normalizedPath]);
+	if (afinfo.code !== 0) {
+		return { success: true, path: null };
+	}
+
+	const bitrates = parseAfinfoAudioTrackBitrates(`${afinfo.stdout}\n${afinfo.stderr}`);
+	if (bitrates.length <= 1) {
+		return { success: true, path: null };
+	}
+
+	let supplementalTrackIndex = 1;
+	for (let index = 2; index < bitrates.length; index += 1) {
+		if (bitrates[index] > bitrates[supplementalTrackIndex]) {
+			supplementalTrackIndex = index;
+		}
+	}
+
+	await fs.mkdir(PREVIEW_AUDIO_DIR, { recursive: true });
+	const sourceStat = await fs.stat(normalizedPath);
+	const parsedPath = path.parse(normalizedPath);
+	const outputPath = path.join(
+		PREVIEW_AUDIO_DIR,
+		`${parsedPath.name}.track-${supplementalTrackIndex}.${Math.round(sourceStat.mtimeMs)}.m4a`,
+	);
+
+	try {
+		const outputStat = await fs.stat(outputPath);
+		if (outputStat.mtimeMs >= sourceStat.mtimeMs) {
+			return { success: true, path: pathToFileURL(outputPath).toString() };
+		}
+	} catch {
+		// Generate below.
+	}
+
+	const conversion = await runProcess("/usr/bin/afconvert", [
+		"--read-track",
+		String(supplementalTrackIndex),
+		"-f",
+		"m4af",
+		"-d",
+		"aac",
+		normalizedPath,
+		outputPath,
+	]);
+	if (conversion.code !== 0) {
+		return {
+			success: false,
+			message: conversion.stderr || conversion.stdout || "Failed to prepare preview audio",
+		};
+	}
+
+	return { success: true, path: pathToFileURL(outputPath).toString() };
+}
+
 async function approveReadableVideoPath(
 	filePath?: string | null,
 	trustedDirs?: string[],
@@ -1273,6 +1370,16 @@ export function registerIpcHandlers(
 		createEditorWindow();
 	});
 
+	ipcMain.handle("switch-to-hud", () => {
+		_switchToHud?.();
+		return { success: true };
+	});
+
+	ipcMain.handle("start-new-recording", () => {
+		_switchToHud?.();
+		return { success: true };
+	});
+
 	ipcMain.handle("countdown-overlay-show", async (_, value: number, runId: number) => {
 		const overlayWindow = getCountdownOverlayWindow?.() ?? createCountdownOverlayWindow();
 		if (overlayWindow.isDestroyed()) {
@@ -2236,6 +2343,19 @@ export function registerIpcHandlers(
 		}
 	});
 
+	ipcMain.handle("prepare-preview-audio-track", async (_, filePath: string) => {
+		try {
+			return await prepareSupplementalPreviewAudioTrack(filePath);
+		} catch (error) {
+			console.error("Failed to prepare preview audio track:", error);
+			return {
+				success: false,
+				message: "Failed to prepare preview audio track",
+				error: String(error),
+			};
+		}
+	});
+
 	ipcMain.handle(
 		"save-project-file",
 		async (_, projectData: unknown, suggestedName?: string, existingProjectPath?: string) => {

electron/preload.ts

@@ -25,6 +25,9 @@ contextBridge.exposeInMainWorld("electronAPI", {
 	setHudOverlayIgnoreMouseEvents: (ignore: boolean) => {
 		ipcRenderer.send("hud-overlay-ignore-mouse-events", ignore);
 	},
+	moveHudOverlayBy: (deltaX: number, deltaY: number) => {
+		ipcRenderer.send("hud-overlay-move-by", deltaX, deltaY);
+	},
 	getSources: async (opts: Electron.SourcesOptions) => {
 		return await ipcRenderer.invoke("get-sources", opts);
 	},
@@ -142,6 +145,9 @@ contextBridge.exposeInMainWorld("electronAPI", {
 	readBinaryFile: (filePath: string) => {
 		return ipcRenderer.invoke("read-binary-file", filePath);
 	},
+	preparePreviewAudioTrack: (filePath: string) => {
+		return ipcRenderer.invoke("prepare-preview-audio-track", filePath);
+	},
 	clearCurrentVideoPath: () => {
 		return ipcRenderer.invoke("clear-current-video-path");
 	},

electron/windows.ts

@@ -30,6 +30,20 @@ ipcMain.on("hud-overlay-ignore-mouse-events", (_event, ignore: boolean) => {
 	}
 });
 
+ipcMain.on("hud-overlay-move-by", (_event, deltaX: number, deltaY: number) => {
+	if (
+		!hudOverlayWindow ||
+		hudOverlayWindow.isDestroyed() ||
+		!Number.isFinite(deltaX) ||
+		!Number.isFinite(deltaY)
+	) {
+		return;
+	}
+
+	const [x, y] = hudOverlayWindow.getPosition();
+	hudOverlayWindow.setPosition(Math.round(x + deltaX), Math.round(y + deltaY), false);
+});
+
 /**
  * Creates the always-on-top HUD overlay window centred at the bottom of the
  * primary display. The window is frameless, transparent, and follows the user

src/components/launch/LaunchWindow.tsx

@@ -1,5 +1,5 @@
 import { Check, ChevronDown, Languages } from "lucide-react";
-import { useEffect, useRef, useState } from "react";
+import { useCallback, useEffect, useRef, useState } from "react";
 import { createPortal } from "react-dom";
 import { BsPauseCircle, BsPlayCircle, BsRecordCircle } from "react-icons/bs";
 import { FaRegStopCircle } from "react-icons/fa";
@@ -282,6 +282,10 @@ export function LaunchWindow() {
 		return () => cancelAnimationFrame(id);
 	}, [isLanguageMenuOpen]);
 
+	const setHudMouseEventsEnabled = useCallback((enabled: boolean) => {
+		window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(!enabled);
+	}, []);
+
 	useEffect(() => {
 		window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(true);
 		return () => {
@@ -289,6 +293,12 @@ export function LaunchWindow() {
 		};
 	}, []);
 
+	useEffect(() => {
+		if (isLanguageMenuOpen) {
+			setHudMouseEventsEnabled(true);
+		}
+	}, [isLanguageMenuOpen, setHudMouseEventsEnabled]);
+
 	const [selectedSource, setSelectedSource] = useState("Screen");
 	const [hasSelectedSource, setHasSelectedSource] = useState(false);
 	const [, setRecordPointerDownCount] = useState(0);
@@ -358,6 +368,29 @@ export function LaunchWindow() {
 			setMicrophoneEnabled(!microphoneEnabled);
 		}
 	};
+	const dragLastPositionRef = useRef<{ x: number; y: number } | null>(null);
+	const handleHudDragPointerDown = (event: React.PointerEvent<HTMLDivElement>) => {
+		event.preventDefault();
+		event.stopPropagation();
+		setHudMouseEventsEnabled(true);
+		event.currentTarget.setPointerCapture(event.pointerId);
+		dragLastPositionRef.current = { x: event.screenX, y: event.screenY };
+	};
+	const handleHudDragPointerMove = (event: React.PointerEvent<HTMLDivElement>) => {
+		const lastPosition = dragLastPositionRef.current;
+		if (!lastPosition) return;
+		const deltaX = event.screenX - lastPosition.x;
+		const deltaY = event.screenY - lastPosition.y;
+		dragLastPositionRef.current = { x: event.screenX, y: event.screenY };
+		window.electronAPI?.moveHudOverlayBy?.(deltaX, deltaY);
+	};
+	const handleHudDragPointerEnd = (event: React.PointerEvent<HTMLDivElement>) => {
+		dragLastPositionRef.current = null;
+		if (event.currentTarget.hasPointerCapture(event.pointerId)) {
+			event.currentTarget.releasePointerCapture(event.pointerId);
+		}
+		setHudMouseEventsEnabled(true);
+	};
 
 	return (
 		// Root fills the HUD window only. Avoid w-screen/h-screen (100vw/100vh):
@@ -369,9 +402,13 @@ export function LaunchWindow() {
 			onPointerMove={(event) => {
 				const target = event.target as HTMLElement | null;
 				const shouldCapture = Boolean(target?.closest("[data-hud-interactive='true']"));
-				window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(!shouldCapture);
+				setHudMouseEventsEnabled(shouldCapture);
+			}}
+			onPointerLeave={() => {
+				if (!isLanguageMenuOpen) {
+					setHudMouseEventsEnabled(false);
+				}
 			}}
-			onPointerLeave={() => window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(true)}
 		>
 			{systemLocaleSuggestion && (
 				<div
@@ -549,9 +586,23 @@ export function LaunchWindow() {
 			<div
 				data-hud-interactive="true"
 				className={`fixed bottom-5 left-1/2 -translate-x-1/2 flex items-center gap-1.5 rounded-2xl border border-white/[0.10] bg-[#07080a]/90 px-2 py-1.5 shadow-[0_20px_60px_rgba(0,0,0,0.42),inset_0_1px_0_rgba(255,255,255,0.06)] backdrop-blur-2xl backdrop-saturate-[140%]`}
+				onPointerEnter={() => setHudMouseEventsEnabled(true)}
+				onPointerDown={() => setHudMouseEventsEnabled(true)}
+				onMouseEnter={() => setHudMouseEventsEnabled(true)}
+				onMouseLeave={() => {
+					if (!isLanguageMenuOpen) {
+						setHudMouseEventsEnabled(false);
+					}
+				}}
 			>
 				{/* Drag handle */}
-				<div className={`flex items-center px-1 ${styles.electronDrag}`}>
+				<div
+					className={`flex h-8 w-7 cursor-grab items-center justify-center active:cursor-grabbing ${styles.electronNoDrag}`}
+					onPointerDown={handleHudDragPointerDown}
+					onPointerMove={handleHudDragPointerMove}
+					onPointerUp={handleHudDragPointerEnd}
+					onPointerCancel={handleHudDragPointerEnd}
+				>
 					{getIcon("drag", "text-white/30")}
 				</div>
 
@@ -743,6 +794,7 @@ export function LaunchWindow() {
 						? createPortal(
 								<div
 									ref={languageMenuPanelRef}
+									data-hud-interactive="true"
 									role="menu"
 									className={`${styles.languageMenuPanel} ${styles.languageMenuScroll} ${styles.electronNoDrag}`}
 									style={
@@ -755,6 +807,12 @@ export function LaunchWindow() {
 										} as React.CSSProperties
 									}
 									onPointerDown={(event) => event.stopPropagation()}
+									onPointerEnter={() => setHudMouseEventsEnabled(true)}
+									onPointerMove={() => setHudMouseEventsEnabled(true)}
+									onWheel={(event) => {
+										setHudMouseEventsEnabled(true);
+										event.stopPropagation();
+									}}
 								>
 									{availableLocales.map((loc) => (
 										<button

src/components/video-editor/VideoPlayback.tsx

@@ -195,6 +195,26 @@ function getEndedVideoDuration(video: HTMLVideoElement): number | null {
 	return null;
 }
 
+type AudioTrackListLike = {
+	length: number;
+	[index: number]: { enabled: boolean };
+};
+
+type VideoElementWithAudioTracks = HTMLVideoElement & {
+	audioTracks?: AudioTrackListLike;
+};
+
+function enableAllPreviewAudioTracks(video: HTMLVideoElement) {
+	const audioTracks = (video as VideoElementWithAudioTracks).audioTracks;
+	if (!audioTracks || audioTracks.length <= 1) {
+		return;
+	}
+
+	for (let index = 0; index < audioTracks.length; index += 1) {
+		audioTracks[index].enabled = true;
+	}
+}
+
 const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
 	(
 		{
@@ -252,6 +272,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
 		ref,
 	) => {
 		const videoRef = useRef<HTMLVideoElement | null>(null);
+		const supplementalAudioRef = useRef<HTMLAudioElement | null>(null);
 		const webcamVideoRef = useRef<HTMLVideoElement | null>(null);
 		const containerRef = useRef<HTMLDivElement | null>(null);
 		const appRef = useRef<Application | null>(null);
@@ -261,6 +282,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
 		const timeUpdateAnimationRef = useRef<number | null>(null);
 		const [pixiReady, setPixiReady] = useState(false);
 		const [videoReady, setVideoReady] = useState(false);
+		const [supplementalAudioPath, setSupplementalAudioPath] = useState<string | null>(null);
 		const [overlaySize, setOverlaySize] = useState({ width: 800, height: 600 });
 		const [overlayElement, setOverlayElement] = useState<HTMLDivElement | null>(null);
 		const overlayRef = useRef<HTMLDivElement | null>(null);
@@ -582,10 +604,19 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
 				if (!vid) return;
 				try {
 					allowPlaybackRef.current = true;
+					enableAllPreviewAudioTracks(vid);
 					await vid.play().catch((err) => {
 						console.log("PLAY ERROR:", err);
 						throw err;
 					});
+					const supplementalAudio = supplementalAudioRef.current;
+					if (supplementalAudio) {
+						supplementalAudio.currentTime = vid.currentTime;
+						supplementalAudio.playbackRate = vid.playbackRate;
+						await supplementalAudio.play().catch(() => {
+							// The main video remains the source of truth for playback state.
+						});
+					}
 				} catch (error) {
 					allowPlaybackRef.current = false;
 					throw error;
@@ -598,6 +629,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
 					return;
 				}
 				video.pause();
+				supplementalAudioRef.current?.pause();
 			},
 		}));
 
@@ -1005,11 +1037,30 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
 				lastResolvedDurationRef.current = null;
 				isResolvingDurationRef.current = false;
 				setVideoReady(false);
+				setSupplementalAudioPath(null);
 				return;
 			}
 
+			let cancelled = false;
+			window.electronAPI
+				?.preparePreviewAudioTrack?.(videoPath)
+				.then((result) => {
+					if (!cancelled) {
+						setSupplementalAudioPath(result.success ? (result.path ?? null) : null);
+					}
+				})
+				.catch(() => {
+					if (!cancelled) {
+						setSupplementalAudioPath(null);
+					}
+				});
+
 			const video = videoRef.current;
-			if (!video) return;
+			if (!video) {
+				return () => {
+					cancelled = true;
+				};
+			}
 			video.pause();
 			video.currentTime = 0;
 			allowPlaybackRef.current = false;
@@ -1026,8 +1077,42 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
 				videoReadyRafRef.current = null;
 			}
 			video.load();
+
+			return () => {
+				cancelled = true;
+			};
 		}, [videoPath]);
 
+		useEffect(() => {
+			const video = videoRef.current;
+			const supplementalAudio = supplementalAudioRef.current;
+			if (!video || !supplementalAudio || !supplementalAudioPath) {
+				return;
+			}
+
+			const activeSpeedRegion =
+				speedRegions.find(
+					(region) => currentTime * 1000 >= region.startMs && currentTime * 1000 < region.endMs,
+				) ?? null;
+			supplementalAudio.playbackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1;
+
+			if (!isPlaying) {
+				supplementalAudio.pause();
+				if (Math.abs(supplementalAudio.currentTime - currentTime) > 0.05) {
+					supplementalAudio.currentTime = currentTime;
+				}
+				return;
+			}
+
+			if (Math.abs(supplementalAudio.currentTime - video.currentTime) > 0.15) {
+				supplementalAudio.currentTime = video.currentTime;
+			}
+
+			supplementalAudio.play().catch(() => {
+				// Keep video playback running even if supplemental preview audio is unavailable.
+			});
+		}, [currentTime, isPlaying, speedRegions, supplementalAudioPath]);
+
 		useEffect(() => {
 			if (!pixiReady || !videoReady) return;
 
@@ -1545,6 +1630,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
 
 		const handleLoadedMetadata = (e: React.SyntheticEvent<HTMLVideoElement, Event>) => {
 			const video = e.currentTarget;
+			enableAllPreviewAudioTracks(video);
 			const hasResolvedDuration = syncResolvedDuration(video);
 			if (!hasResolvedDuration) {
 				forceResolveDuration(video);
@@ -1928,22 +2014,28 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
 					playsInline
 					onLoadedMetadata={handleLoadedMetadata}
 					onDurationChange={(e) => {
+						enableAllPreviewAudioTracks(e.currentTarget);
 						if (!syncResolvedDuration(e.currentTarget)) {
 							forceResolveDuration(e.currentTarget);
 						}
 					}}
 					onLoadedData={(e) => {
+						enableAllPreviewAudioTracks(e.currentTarget);
 						if (!syncResolvedDuration(e.currentTarget)) {
 							forceResolveDuration(e.currentTarget);
 						}
 					}}
 					onCanPlay={(e) => {
+						enableAllPreviewAudioTracks(e.currentTarget);
 						if (!syncResolvedDuration(e.currentTarget)) {
 							forceResolveDuration(e.currentTarget);
 						}
 					}}
 					onError={() => onError("Failed to load video")}
 				/>
+				{supplementalAudioPath && (
+					<audio ref={supplementalAudioRef} src={supplementalAudioPath} preload="auto" />
+				)}
 			</div>
 		);
 	},