From df6da28ad24659bf153bedac8adcc2cbabe9144d Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Wed, 13 May 2026 14:48:50 +0200 Subject: [PATCH] fix: improve macOS HUD interactions and audio preview --- electron/electron-env.d.ts | 7 + electron/ipc/handlers.ts | 120 ++++++++++++++++++ electron/preload.ts | 6 + electron/windows.ts | 14 ++ src/components/launch/LaunchWindow.tsx | 66 +++++++++- src/components/video-editor/VideoPlayback.tsx | 94 +++++++++++++- 6 files changed, 302 insertions(+), 5 deletions(-) diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 3c2551e..abb688d 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -197,6 +197,12 @@ interface Window { message?: string; error?: string; }>; + preparePreviewAudioTrack: (filePath: string) => Promise<{ + success: boolean; + path?: string | null; + message?: string; + error?: string; + }>; clearCurrentVideoPath: () => Promise<{ success: boolean }>; saveProjectFile: ( projectData: unknown, @@ -237,6 +243,7 @@ interface Window { hudOverlayHide: () => void; hudOverlayClose: () => void; setHudOverlayIgnoreMouseEvents: (ignore: boolean) => void; + moveHudOverlayBy: (deltaX: number, deltaY: number) => void; showCountdownOverlay: (value: number, runId: number) => Promise; setCountdownOverlayValue: (value: number, runId: number) => Promise; hideCountdownOverlay: (runId: number) => Promise; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index e7522c7..1a5ba0b 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -46,6 +46,7 @@ const SHORTCUTS_FILE = path.join(app.getPath("userData"), "shortcuts.json"); const RECORDING_FILE_PREFIX = "recording-"; const RECORDING_SESSION_SUFFIX = ".session.json"; const ALLOWED_IMPORT_VIDEO_EXTENSIONS = new Set([".webm", ".mp4", ".mov", ".avi", ".mkv"]); +const PREVIEW_AUDIO_DIR = path.join(app.getPath("userData"), "preview-audio"); /** * Paths explicitly approved by the user via file picker dialogs or project loads. @@ -105,6 +106,102 @@ function hasAllowedImportVideoExtension(filePath: string): boolean { return ALLOWED_IMPORT_VIDEO_EXTENSIONS.has(path.extname(filePath).toLowerCase()); } +function runProcess( + command: string, + args: string[], +): Promise<{ code: number | null; stdout: string; stderr: string }> { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + child.on("error", reject); + child.on("close", (code) => resolve({ code, stdout, stderr })); + }); +} + +function parseAfinfoAudioTrackBitrates(output: string): number[] { + const bitrates: number[] = []; + const trackSections = output.split(/\n----\n/g).slice(1); + for (const section of trackSections) { + const match = section.match(/\bbit rate:\s*([0-9]+)\s*bits per second/i); + bitrates.push(match ? Number(match[1]) : 0); + } + return bitrates; +} + +async function prepareSupplementalPreviewAudioTrack(videoPath: string) { + const normalizedPath = await approveReadableVideoPath(videoPath); + if (!normalizedPath) { + return { + success: false, + message: "File path is not approved or is not a supported video file", + }; + } + + if (process.platform !== "darwin" || path.extname(normalizedPath).toLowerCase() !== ".mp4") { + return { success: true, path: null }; + } + + const afinfo = await runProcess("/usr/bin/afinfo", [normalizedPath]); + if (afinfo.code !== 0) { + return { success: true, path: null }; + } + + const bitrates = parseAfinfoAudioTrackBitrates(`${afinfo.stdout}\n${afinfo.stderr}`); + if (bitrates.length <= 1) { + return { success: true, path: null }; + } + + let supplementalTrackIndex = 1; + for (let index = 2; index < bitrates.length; index += 1) { + if (bitrates[index] > bitrates[supplementalTrackIndex]) { + supplementalTrackIndex = index; + } + } + + await fs.mkdir(PREVIEW_AUDIO_DIR, { recursive: true }); + const sourceStat = await fs.stat(normalizedPath); + const parsedPath = path.parse(normalizedPath); + const outputPath = path.join( + PREVIEW_AUDIO_DIR, + `${parsedPath.name}.track-${supplementalTrackIndex}.${Math.round(sourceStat.mtimeMs)}.m4a`, + ); + + try { + const outputStat = await fs.stat(outputPath); + if (outputStat.mtimeMs >= sourceStat.mtimeMs) { + return { success: true, path: pathToFileURL(outputPath).toString() }; + } + } catch { + // Generate below. + } + + const conversion = await runProcess("/usr/bin/afconvert", [ + "--read-track", + String(supplementalTrackIndex), + "-f", + "m4af", + "-d", + "aac", + normalizedPath, + outputPath, + ]); + if (conversion.code !== 0) { + return { + success: false, + message: conversion.stderr || conversion.stdout || "Failed to prepare preview audio", + }; + } + + return { success: true, path: pathToFileURL(outputPath).toString() }; +} + async function approveReadableVideoPath( filePath?: string | null, trustedDirs?: string[], @@ -1273,6 +1370,16 @@ export function registerIpcHandlers( createEditorWindow(); }); + ipcMain.handle("switch-to-hud", () => { + _switchToHud?.(); + return { success: true }; + }); + + ipcMain.handle("start-new-recording", () => { + _switchToHud?.(); + return { success: true }; + }); + ipcMain.handle("countdown-overlay-show", async (_, value: number, runId: number) => { const overlayWindow = getCountdownOverlayWindow?.() ?? createCountdownOverlayWindow(); if (overlayWindow.isDestroyed()) { @@ -2236,6 +2343,19 @@ export function registerIpcHandlers( } }); + ipcMain.handle("prepare-preview-audio-track", async (_, filePath: string) => { + try { + return await prepareSupplementalPreviewAudioTrack(filePath); + } catch (error) { + console.error("Failed to prepare preview audio track:", error); + return { + success: false, + message: "Failed to prepare preview audio track", + error: String(error), + }; + } + }); + ipcMain.handle( "save-project-file", async (_, projectData: unknown, suggestedName?: string, existingProjectPath?: string) => { diff --git a/electron/preload.ts b/electron/preload.ts index 393311e..361eb18 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -25,6 +25,9 @@ contextBridge.exposeInMainWorld("electronAPI", { setHudOverlayIgnoreMouseEvents: (ignore: boolean) => { ipcRenderer.send("hud-overlay-ignore-mouse-events", ignore); }, + moveHudOverlayBy: (deltaX: number, deltaY: number) => { + ipcRenderer.send("hud-overlay-move-by", deltaX, deltaY); + }, getSources: async (opts: Electron.SourcesOptions) => { return await ipcRenderer.invoke("get-sources", opts); }, @@ -142,6 +145,9 @@ contextBridge.exposeInMainWorld("electronAPI", { readBinaryFile: (filePath: string) => { return ipcRenderer.invoke("read-binary-file", filePath); }, + preparePreviewAudioTrack: (filePath: string) => { + return ipcRenderer.invoke("prepare-preview-audio-track", filePath); + }, clearCurrentVideoPath: () => { return ipcRenderer.invoke("clear-current-video-path"); }, diff --git a/electron/windows.ts b/electron/windows.ts index 4d4e752..3a7350e 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -30,6 +30,20 @@ ipcMain.on("hud-overlay-ignore-mouse-events", (_event, ignore: boolean) => { } }); +ipcMain.on("hud-overlay-move-by", (_event, deltaX: number, deltaY: number) => { + if ( + !hudOverlayWindow || + hudOverlayWindow.isDestroyed() || + !Number.isFinite(deltaX) || + !Number.isFinite(deltaY) + ) { + return; + } + + const [x, y] = hudOverlayWindow.getPosition(); + hudOverlayWindow.setPosition(Math.round(x + deltaX), Math.round(y + deltaY), false); +}); + /** * Creates the always-on-top HUD overlay window centred at the bottom of the * primary display. The window is frameless, transparent, and follows the user diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index f901297..5d394f2 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -1,5 +1,5 @@ import { Check, ChevronDown, Languages } from "lucide-react"; -import { useEffect, useRef, useState } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; import { createPortal } from "react-dom"; import { BsPauseCircle, BsPlayCircle, BsRecordCircle } from "react-icons/bs"; import { FaRegStopCircle } from "react-icons/fa"; @@ -282,6 +282,10 @@ export function LaunchWindow() { return () => cancelAnimationFrame(id); }, [isLanguageMenuOpen]); + const setHudMouseEventsEnabled = useCallback((enabled: boolean) => { + window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(!enabled); + }, []); + useEffect(() => { window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(true); return () => { @@ -289,6 +293,12 @@ export function LaunchWindow() { }; }, []); + useEffect(() => { + if (isLanguageMenuOpen) { + setHudMouseEventsEnabled(true); + } + }, [isLanguageMenuOpen, setHudMouseEventsEnabled]); + const [selectedSource, setSelectedSource] = useState("Screen"); const [hasSelectedSource, setHasSelectedSource] = useState(false); const [, setRecordPointerDownCount] = useState(0); @@ -358,6 +368,29 @@ export function LaunchWindow() { setMicrophoneEnabled(!microphoneEnabled); } }; + const dragLastPositionRef = useRef<{ x: number; y: number } | null>(null); + const handleHudDragPointerDown = (event: React.PointerEvent) => { + event.preventDefault(); + event.stopPropagation(); + setHudMouseEventsEnabled(true); + event.currentTarget.setPointerCapture(event.pointerId); + dragLastPositionRef.current = { x: event.screenX, y: event.screenY }; + }; + const handleHudDragPointerMove = (event: React.PointerEvent) => { + const lastPosition = dragLastPositionRef.current; + if (!lastPosition) return; + const deltaX = event.screenX - lastPosition.x; + const deltaY = event.screenY - lastPosition.y; + dragLastPositionRef.current = { x: event.screenX, y: event.screenY }; + window.electronAPI?.moveHudOverlayBy?.(deltaX, deltaY); + }; + const handleHudDragPointerEnd = (event: React.PointerEvent) => { + dragLastPositionRef.current = null; + if (event.currentTarget.hasPointerCapture(event.pointerId)) { + event.currentTarget.releasePointerCapture(event.pointerId); + } + setHudMouseEventsEnabled(true); + }; return ( // Root fills the HUD window only. Avoid w-screen/h-screen (100vw/100vh): @@ -369,9 +402,13 @@ export function LaunchWindow() { onPointerMove={(event) => { const target = event.target as HTMLElement | null; const shouldCapture = Boolean(target?.closest("[data-hud-interactive='true']")); - window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(!shouldCapture); + setHudMouseEventsEnabled(shouldCapture); + }} + onPointerLeave={() => { + if (!isLanguageMenuOpen) { + setHudMouseEventsEnabled(false); + } }} - onPointerLeave={() => window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(true)} > {systemLocaleSuggestion && (
setHudMouseEventsEnabled(true)} + onPointerDown={() => setHudMouseEventsEnabled(true)} + onMouseEnter={() => setHudMouseEventsEnabled(true)} + onMouseLeave={() => { + if (!isLanguageMenuOpen) { + setHudMouseEventsEnabled(false); + } + }} > {/* Drag handle */} -
+
{getIcon("drag", "text-white/30")}
@@ -743,6 +794,7 @@ export function LaunchWindow() { ? createPortal(
event.stopPropagation()} + onPointerEnter={() => setHudMouseEventsEnabled(true)} + onPointerMove={() => setHudMouseEventsEnabled(true)} + onWheel={(event) => { + setHudMouseEventsEnabled(true); + event.stopPropagation(); + }} > {availableLocales.map((loc) => (
); },