huanld/openscreen
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases 2 Wiki Activity

fix: improve macOS HUD interactions and audio preview

Browse Source
This commit is contained in:
Etienne Lescot
2026-05-13 14:48:50 +02:00
parent c1ba82fc71
commit df6da28ad2
6 changed files with 302 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
electron/electron-env.d.ts
Vendored
+7
View File
@@ -197,6 +197,12 @@ interface Window {
message?: string;
error?: string;
}>;
preparePreviewAudioTrack: (filePath: string) => Promise<{
success: boolean;
path?: string | null;
message?: string;
error?: string;
}>;
clearCurrentVideoPath: () => Promise<{ success: boolean }>;
saveProjectFile: (
projectData: unknown,
@@ -237,6 +243,7 @@ interface Window {
hudOverlayHide: () => void;
hudOverlayClose: () => void;
setHudOverlayIgnoreMouseEvents: (ignore: boolean) => void;
moveHudOverlayBy: (deltaX: number, deltaY: number) => void;
showCountdownOverlay: (value: number, runId: number) => Promise<void>;
setCountdownOverlayValue: (value: number, runId: number) => Promise<void>;
hideCountdownOverlay: (runId: number) => Promise<void>;
electron/ipc/handlers.ts
+120
View File
@@ -46,6 +46,7 @@ const SHORTCUTS_FILE = path.join(app.getPath("userData"), "shortcuts.json");
const RECORDING_FILE_PREFIX = "recording-";
const RECORDING_SESSION_SUFFIX = ".session.json";
const ALLOWED_IMPORT_VIDEO_EXTENSIONS = new Set([".webm", ".mp4", ".mov", ".avi", ".mkv"]);
const PREVIEW_AUDIO_DIR = path.join(app.getPath("userData"), "preview-audio");
/**
* Paths explicitly approved by the user via file picker dialogs or project loads.
@@ -105,6 +106,102 @@ function hasAllowedImportVideoExtension(filePath: string): boolean {
return ALLOWED_IMPORT_VIDEO_EXTENSIONS.has(path.extname(filePath).toLowerCase());
}
function runProcess(
command: string,
args: string[],
): Promise<{ code: number | null; stdout: string; stderr: string }> {
return new Promise((resolve, reject) => {
const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] });
let stdout = "";
let stderr = "";
child.stdout.on("data", (chunk) => {
stdout += chunk.toString();
});
child.stderr.on("data", (chunk) => {
stderr += chunk.toString();
});
child.on("error", reject);
child.on("close", (code) => resolve({ code, stdout, stderr }));
});
}
function parseAfinfoAudioTrackBitrates(output: string): number[] {
const bitrates: number[] = [];
const trackSections = output.split(/\n----\n/g).slice(1);
for (const section of trackSections) {
const match = section.match(/\bbit rate:\s*([0-9]+)\s*bits per second/i);
bitrates.push(match ? Number(match[1]) : 0);
}
return bitrates;
}
async function prepareSupplementalPreviewAudioTrack(videoPath: string) {
const normalizedPath = await approveReadableVideoPath(videoPath);
if (!normalizedPath) {
return {
success: false,
message: "File path is not approved or is not a supported video file",
};
}
if (process.platform !== "darwin" || path.extname(normalizedPath).toLowerCase() !== ".mp4") {
return { success: true, path: null };
}
const afinfo = await runProcess("/usr/bin/afinfo", [normalizedPath]);
if (afinfo.code !== 0) {
return { success: true, path: null };
}
const bitrates = parseAfinfoAudioTrackBitrates(`${afinfo.stdout}\n${afinfo.stderr}`);
if (bitrates.length <= 1) {
return { success: true, path: null };
}
let supplementalTrackIndex = 1;
for (let index = 2; index < bitrates.length; index += 1) {
if (bitrates[index] > bitrates[supplementalTrackIndex]) {
supplementalTrackIndex = index;
}
}
await fs.mkdir(PREVIEW_AUDIO_DIR, { recursive: true });
const sourceStat = await fs.stat(normalizedPath);
const parsedPath = path.parse(normalizedPath);
const outputPath = path.join(
PREVIEW_AUDIO_DIR,
`${parsedPath.name}.track-${supplementalTrackIndex}.${Math.round(sourceStat.mtimeMs)}.m4a`,
);
try {
const outputStat = await fs.stat(outputPath);
if (outputStat.mtimeMs >= sourceStat.mtimeMs) {
return { success: true, path: pathToFileURL(outputPath).toString() };
}
} catch {
// Generate below.
}
const conversion = await runProcess("/usr/bin/afconvert", [
"--read-track",
String(supplementalTrackIndex),
"-f",
"m4af",
"-d",
"aac",
normalizedPath,
outputPath,
]);
if (conversion.code !== 0) {
return {
success: false,
message: conversion.stderr || conversion.stdout || "Failed to prepare preview audio",
};
}
return { success: true, path: pathToFileURL(outputPath).toString() };
}
async function approveReadableVideoPath(
filePath?: string | null,
trustedDirs?: string[],
@@ -1273,6 +1370,16 @@ export function registerIpcHandlers(
createEditorWindow();
});
ipcMain.handle("switch-to-hud", () => {
_switchToHud?.();
return { success: true };
});
ipcMain.handle("start-new-recording", () => {
_switchToHud?.();
return { success: true };
});
ipcMain.handle("countdown-overlay-show", async (_, value: number, runId: number) => {
const overlayWindow = getCountdownOverlayWindow?.() ?? createCountdownOverlayWindow();
if (overlayWindow.isDestroyed()) {
@@ -2236,6 +2343,19 @@ export function registerIpcHandlers(
}
});
ipcMain.handle("prepare-preview-audio-track", async (_, filePath: string) => {
try {
return await prepareSupplementalPreviewAudioTrack(filePath);
} catch (error) {
console.error("Failed to prepare preview audio track:", error);
return {
success: false,
message: "Failed to prepare preview audio track",
error: String(error),
};
}
});
ipcMain.handle(
"save-project-file",
async (_, projectData: unknown, suggestedName?: string, existingProjectPath?: string) => {
electron/preload.ts
+6
View File
@@ -25,6 +25,9 @@ contextBridge.exposeInMainWorld("electronAPI", {
setHudOverlayIgnoreMouseEvents: (ignore: boolean) => {
ipcRenderer.send("hud-overlay-ignore-mouse-events", ignore);
},
moveHudOverlayBy: (deltaX: number, deltaY: number) => {
ipcRenderer.send("hud-overlay-move-by", deltaX, deltaY);
},
getSources: async (opts: Electron.SourcesOptions) => {
return await ipcRenderer.invoke("get-sources", opts);
},
@@ -142,6 +145,9 @@ contextBridge.exposeInMainWorld("electronAPI", {
readBinaryFile: (filePath: string) => {
return ipcRenderer.invoke("read-binary-file", filePath);
},
preparePreviewAudioTrack: (filePath: string) => {
return ipcRenderer.invoke("prepare-preview-audio-track", filePath);
},
clearCurrentVideoPath: () => {
return ipcRenderer.invoke("clear-current-video-path");
},
electron/windows.ts
+14
View File
@@ -30,6 +30,20 @@ ipcMain.on("hud-overlay-ignore-mouse-events", (_event, ignore: boolean) => {
}
});
ipcMain.on("hud-overlay-move-by", (_event, deltaX: number, deltaY: number) => {
if (
!hudOverlayWindow ||
hudOverlayWindow.isDestroyed() ||
!Number.isFinite(deltaX) ||
!Number.isFinite(deltaY)
) {
return;
}
const [x, y] = hudOverlayWindow.getPosition();
hudOverlayWindow.setPosition(Math.round(x + deltaX), Math.round(y + deltaY), false);
});
/**
* Creates the always-on-top HUD overlay window centred at the bottom of the
* primary display. The window is frameless, transparent, and follows the user
src/components/launch/LaunchWindow.tsx
+62 -4
View File
@@ -1,5 +1,5 @@
import { Check, ChevronDown, Languages } from "lucide-react";
import { useEffect, useRef, useState } from "react";
import { useCallback, useEffect, useRef, useState } from "react";
import { createPortal } from "react-dom";
import { BsPauseCircle, BsPlayCircle, BsRecordCircle } from "react-icons/bs";
import { FaRegStopCircle } from "react-icons/fa";
@@ -282,6 +282,10 @@ export function LaunchWindow() {
return () => cancelAnimationFrame(id);
}, [isLanguageMenuOpen]);
const setHudMouseEventsEnabled = useCallback((enabled: boolean) => {
window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(!enabled);
}, []);
useEffect(() => {
window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(true);
return () => {
@@ -289,6 +293,12 @@ export function LaunchWindow() {
};
}, []);
useEffect(() => {
if (isLanguageMenuOpen) {
setHudMouseEventsEnabled(true);
}
}, [isLanguageMenuOpen, setHudMouseEventsEnabled]);
const [selectedSource, setSelectedSource] = useState("Screen");
const [hasSelectedSource, setHasSelectedSource] = useState(false);
const [, setRecordPointerDownCount] = useState(0);
@@ -358,6 +368,29 @@ export function LaunchWindow() {
setMicrophoneEnabled(!microphoneEnabled);
}
};
const dragLastPositionRef = useRef<{ x: number; y: number } | null>(null);
const handleHudDragPointerDown = (event: React.PointerEvent<HTMLDivElement>) => {
event.preventDefault();
event.stopPropagation();
setHudMouseEventsEnabled(true);
event.currentTarget.setPointerCapture(event.pointerId);
dragLastPositionRef.current = { x: event.screenX, y: event.screenY };
};
const handleHudDragPointerMove = (event: React.PointerEvent<HTMLDivElement>) => {
const lastPosition = dragLastPositionRef.current;
if (!lastPosition) return;
const deltaX = event.screenX - lastPosition.x;
const deltaY = event.screenY - lastPosition.y;
dragLastPositionRef.current = { x: event.screenX, y: event.screenY };
window.electronAPI?.moveHudOverlayBy?.(deltaX, deltaY);
};
const handleHudDragPointerEnd = (event: React.PointerEvent<HTMLDivElement>) => {
dragLastPositionRef.current = null;
if (event.currentTarget.hasPointerCapture(event.pointerId)) {
event.currentTarget.releasePointerCapture(event.pointerId);
}
setHudMouseEventsEnabled(true);
};
return (
// Root fills the HUD window only. Avoid w-screen/h-screen (100vw/100vh):
@@ -369,9 +402,13 @@ export function LaunchWindow() {
onPointerMove={(event) => {
const target = event.target as HTMLElement | null;
const shouldCapture = Boolean(target?.closest("[data-hud-interactive='true']"));
window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(!shouldCapture);
setHudMouseEventsEnabled(shouldCapture);
}}
onPointerLeave={() => {
if (!isLanguageMenuOpen) {
setHudMouseEventsEnabled(false);
}
}}
onPointerLeave={() => window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(true)}
>
{systemLocaleSuggestion && (
<div
@@ -549,9 +586,23 @@ export function LaunchWindow() {
<div
data-hud-interactive="true"
className={`fixed bottom-5 left-1/2 -translate-x-1/2 flex items-center gap-1.5 rounded-2xl border border-white/[0.10] bg-[#07080a]/90 px-2 py-1.5 shadow-[0_20px_60px_rgba(0,0,0,0.42),inset_0_1px_0_rgba(255,255,255,0.06)] backdrop-blur-2xl backdrop-saturate-[140%]`}
onPointerEnter={() => setHudMouseEventsEnabled(true)}
onPointerDown={() => setHudMouseEventsEnabled(true)}
onMouseEnter={() => setHudMouseEventsEnabled(true)}
onMouseLeave={() => {
if (!isLanguageMenuOpen) {
setHudMouseEventsEnabled(false);
}
}}
>
{/* Drag handle */}
<div className={`flex items-center px-1 ${styles.electronDrag}`}>
<div
className={`flex h-8 w-7 cursor-grab items-center justify-center active:cursor-grabbing ${styles.electronNoDrag}`}
onPointerDown={handleHudDragPointerDown}
onPointerMove={handleHudDragPointerMove}
onPointerUp={handleHudDragPointerEnd}
onPointerCancel={handleHudDragPointerEnd}
>
{getIcon("drag", "text-white/30")}
</div>
@@ -743,6 +794,7 @@ export function LaunchWindow() {
? createPortal(
<div
ref={languageMenuPanelRef}
data-hud-interactive="true"
role="menu"
className={`${styles.languageMenuPanel} ${styles.languageMenuScroll} ${styles.electronNoDrag}`}
style={
@@ -755,6 +807,12 @@ export function LaunchWindow() {
} as React.CSSProperties
}
onPointerDown={(event) => event.stopPropagation()}
onPointerEnter={() => setHudMouseEventsEnabled(true)}
onPointerMove={() => setHudMouseEventsEnabled(true)}
onWheel={(event) => {
setHudMouseEventsEnabled(true);
event.stopPropagation();
}}
>
{availableLocales.map((loc) => (
<button
src/components/video-editor/VideoPlayback.tsx
+93 -1
View File
@@ -195,6 +195,26 @@ function getEndedVideoDuration(video: HTMLVideoElement): number | null {
return null;
}
type AudioTrackListLike = {
length: number;
[index: number]: { enabled: boolean };
};
type VideoElementWithAudioTracks = HTMLVideoElement & {
audioTracks?: AudioTrackListLike;
};
function enableAllPreviewAudioTracks(video: HTMLVideoElement) {
const audioTracks = (video as VideoElementWithAudioTracks).audioTracks;
if (!audioTracks || audioTracks.length <= 1) {
return;
}
for (let index = 0; index < audioTracks.length; index += 1) {
audioTracks[index].enabled = true;
}
}
const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
(
{
@@ -252,6 +272,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
ref,
) => {
const videoRef = useRef<HTMLVideoElement | null>(null);
const supplementalAudioRef = useRef<HTMLAudioElement | null>(null);
const webcamVideoRef = useRef<HTMLVideoElement | null>(null);
const containerRef = useRef<HTMLDivElement | null>(null);
const appRef = useRef<Application | null>(null);
@@ -261,6 +282,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
const timeUpdateAnimationRef = useRef<number | null>(null);
const [pixiReady, setPixiReady] = useState(false);
const [videoReady, setVideoReady] = useState(false);
const [supplementalAudioPath, setSupplementalAudioPath] = useState<string | null>(null);
const [overlaySize, setOverlaySize] = useState({ width: 800, height: 600 });
const [overlayElement, setOverlayElement] = useState<HTMLDivElement | null>(null);
const overlayRef = useRef<HTMLDivElement | null>(null);
@@ -582,10 +604,19 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
if (!vid) return;
try {
allowPlaybackRef.current = true;
enableAllPreviewAudioTracks(vid);
await vid.play().catch((err) => {
console.log("PLAY ERROR:", err);
throw err;
});
const supplementalAudio = supplementalAudioRef.current;
if (supplementalAudio) {
supplementalAudio.currentTime = vid.currentTime;
supplementalAudio.playbackRate = vid.playbackRate;
await supplementalAudio.play().catch(() => {
// The main video remains the source of truth for playback state.
});
}
} catch (error) {
allowPlaybackRef.current = false;
throw error;
@@ -598,6 +629,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
return;
}
video.pause();
supplementalAudioRef.current?.pause();
},
}));
@@ -1005,11 +1037,30 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
lastResolvedDurationRef.current = null;
isResolvingDurationRef.current = false;
setVideoReady(false);
setSupplementalAudioPath(null);
return;
}
let cancelled = false;
window.electronAPI
?.preparePreviewAudioTrack?.(videoPath)
.then((result) => {
if (!cancelled) {
setSupplementalAudioPath(result.success ? (result.path ?? null) : null);
}
})
.catch(() => {
if (!cancelled) {
setSupplementalAudioPath(null);
}
});
const video = videoRef.current;
if (!video) return;
if (!video) {
return () => {
cancelled = true;
};
}
video.pause();
video.currentTime = 0;
allowPlaybackRef.current = false;
@@ -1026,8 +1077,42 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
videoReadyRafRef.current = null;
}
video.load();
return () => {
cancelled = true;
};
}, [videoPath]);
useEffect(() => {
const video = videoRef.current;
const supplementalAudio = supplementalAudioRef.current;
if (!video || !supplementalAudio || !supplementalAudioPath) {
return;
}
const activeSpeedRegion =
speedRegions.find(
(region) => currentTime * 1000 >= region.startMs && currentTime * 1000 < region.endMs,
) ?? null;
supplementalAudio.playbackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1;
if (!isPlaying) {
supplementalAudio.pause();
if (Math.abs(supplementalAudio.currentTime - currentTime) > 0.05) {
supplementalAudio.currentTime = currentTime;
}
return;
}
if (Math.abs(supplementalAudio.currentTime - video.currentTime) > 0.15) {
supplementalAudio.currentTime = video.currentTime;
}
supplementalAudio.play().catch(() => {
// Keep video playback running even if supplemental preview audio is unavailable.
});
}, [currentTime, isPlaying, speedRegions, supplementalAudioPath]);
useEffect(() => {
if (!pixiReady || !videoReady) return;
@@ -1545,6 +1630,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
const handleLoadedMetadata = (e: React.SyntheticEvent<HTMLVideoElement, Event>) => {
const video = e.currentTarget;
enableAllPreviewAudioTracks(video);
const hasResolvedDuration = syncResolvedDuration(video);
if (!hasResolvedDuration) {
forceResolveDuration(video);
@@ -1928,22 +2014,28 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
playsInline
onLoadedMetadata={handleLoadedMetadata}
onDurationChange={(e) => {
enableAllPreviewAudioTracks(e.currentTarget);
if (!syncResolvedDuration(e.currentTarget)) {
forceResolveDuration(e.currentTarget);
}
}}
onLoadedData={(e) => {
enableAllPreviewAudioTracks(e.currentTarget);
if (!syncResolvedDuration(e.currentTarget)) {
forceResolveDuration(e.currentTarget);
}
}}
onCanPlay={(e) => {
enableAllPreviewAudioTracks(e.currentTarget);
if (!syncResolvedDuration(e.currentTarget)) {
forceResolveDuration(e.currentTarget);
}
}}
onError={() => onError("Failed to load video")}
/>
{supplementalAudioPath && (
<audio ref={supplementalAudioRef} src={supplementalAudioPath} preload="auto" />
)}
</div>
);
},