huanld/openscreen
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 2 Wiki Activity

fix: improve macOS HUD interactions and audio preview

Browse Source
This commit is contained in:
Etienne Lescot
2026-05-13 14:48:50 +02:00
parent c1ba82fc71
commit df6da28ad2
6 changed files with 302 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/components/launch/LaunchWindow.tsx
+62 -4
View File
@@ -1,5 +1,5 @@
import { Check, ChevronDown, Languages } from "lucide-react";
import { useEffect, useRef, useState } from "react";
import { useCallback, useEffect, useRef, useState } from "react";
import { createPortal } from "react-dom";
import { BsPauseCircle, BsPlayCircle, BsRecordCircle } from "react-icons/bs";
import { FaRegStopCircle } from "react-icons/fa";
@@ -282,6 +282,10 @@ export function LaunchWindow() {
return () => cancelAnimationFrame(id);
}, [isLanguageMenuOpen]);
const setHudMouseEventsEnabled = useCallback((enabled: boolean) => {
window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(!enabled);
}, []);
useEffect(() => {
window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(true);
return () => {
@@ -289,6 +293,12 @@ export function LaunchWindow() {
};
}, []);
useEffect(() => {
if (isLanguageMenuOpen) {
setHudMouseEventsEnabled(true);
}
}, [isLanguageMenuOpen, setHudMouseEventsEnabled]);
const [selectedSource, setSelectedSource] = useState("Screen");
const [hasSelectedSource, setHasSelectedSource] = useState(false);
const [, setRecordPointerDownCount] = useState(0);
@@ -358,6 +368,29 @@ export function LaunchWindow() {
setMicrophoneEnabled(!microphoneEnabled);
}
};
const dragLastPositionRef = useRef<{ x: number; y: number } | null>(null);
const handleHudDragPointerDown = (event: React.PointerEvent<HTMLDivElement>) => {
event.preventDefault();
event.stopPropagation();
setHudMouseEventsEnabled(true);
event.currentTarget.setPointerCapture(event.pointerId);
dragLastPositionRef.current = { x: event.screenX, y: event.screenY };
};
const handleHudDragPointerMove = (event: React.PointerEvent<HTMLDivElement>) => {
const lastPosition = dragLastPositionRef.current;
if (!lastPosition) return;
const deltaX = event.screenX - lastPosition.x;
const deltaY = event.screenY - lastPosition.y;
dragLastPositionRef.current = { x: event.screenX, y: event.screenY };
window.electronAPI?.moveHudOverlayBy?.(deltaX, deltaY);
};
const handleHudDragPointerEnd = (event: React.PointerEvent<HTMLDivElement>) => {
dragLastPositionRef.current = null;
if (event.currentTarget.hasPointerCapture(event.pointerId)) {
event.currentTarget.releasePointerCapture(event.pointerId);
}
setHudMouseEventsEnabled(true);
};
return (
// Root fills the HUD window only. Avoid w-screen/h-screen (100vw/100vh):
@@ -369,9 +402,13 @@ export function LaunchWindow() {
onPointerMove={(event) => {
const target = event.target as HTMLElement | null;
const shouldCapture = Boolean(target?.closest("[data-hud-interactive='true']"));
window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(!shouldCapture);
setHudMouseEventsEnabled(shouldCapture);
}}
onPointerLeave={() => {
if (!isLanguageMenuOpen) {
setHudMouseEventsEnabled(false);
}
}}
onPointerLeave={() => window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(true)}
>
{systemLocaleSuggestion && (
<div
@@ -549,9 +586,23 @@ export function LaunchWindow() {
<div
data-hud-interactive="true"
className={`fixed bottom-5 left-1/2 -translate-x-1/2 flex items-center gap-1.5 rounded-2xl border border-white/[0.10] bg-[#07080a]/90 px-2 py-1.5 shadow-[0_20px_60px_rgba(0,0,0,0.42),inset_0_1px_0_rgba(255,255,255,0.06)] backdrop-blur-2xl backdrop-saturate-[140%]`}
onPointerEnter={() => setHudMouseEventsEnabled(true)}
onPointerDown={() => setHudMouseEventsEnabled(true)}
onMouseEnter={() => setHudMouseEventsEnabled(true)}
onMouseLeave={() => {
if (!isLanguageMenuOpen) {
setHudMouseEventsEnabled(false);
}
}}
>
{/* Drag handle */}
<div className={`flex items-center px-1 ${styles.electronDrag}`}>
<div
className={`flex h-8 w-7 cursor-grab items-center justify-center active:cursor-grabbing ${styles.electronNoDrag}`}
onPointerDown={handleHudDragPointerDown}
onPointerMove={handleHudDragPointerMove}
onPointerUp={handleHudDragPointerEnd}
onPointerCancel={handleHudDragPointerEnd}
>
{getIcon("drag", "text-white/30")}
</div>
@@ -743,6 +794,7 @@ export function LaunchWindow() {
? createPortal(
<div
ref={languageMenuPanelRef}
data-hud-interactive="true"
role="menu"
className={`${styles.languageMenuPanel} ${styles.languageMenuScroll} ${styles.electronNoDrag}`}
style={
@@ -755,6 +807,12 @@ export function LaunchWindow() {
} as React.CSSProperties
}
onPointerDown={(event) => event.stopPropagation()}
onPointerEnter={() => setHudMouseEventsEnabled(true)}
onPointerMove={() => setHudMouseEventsEnabled(true)}
onWheel={(event) => {
setHudMouseEventsEnabled(true);
event.stopPropagation();
}}
>
{availableLocales.map((loc) => (
<button
src/components/video-editor/VideoPlayback.tsx
+93 -1
View File
@@ -195,6 +195,26 @@ function getEndedVideoDuration(video: HTMLVideoElement): number | null {
return null;
}
type AudioTrackListLike = {
length: number;
[index: number]: { enabled: boolean };
};
type VideoElementWithAudioTracks = HTMLVideoElement & {
audioTracks?: AudioTrackListLike;
};
function enableAllPreviewAudioTracks(video: HTMLVideoElement) {
const audioTracks = (video as VideoElementWithAudioTracks).audioTracks;
if (!audioTracks || audioTracks.length <= 1) {
return;
}
for (let index = 0; index < audioTracks.length; index += 1) {
audioTracks[index].enabled = true;
}
}
const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
(
{
@@ -252,6 +272,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
ref,
) => {
const videoRef = useRef<HTMLVideoElement | null>(null);
const supplementalAudioRef = useRef<HTMLAudioElement | null>(null);
const webcamVideoRef = useRef<HTMLVideoElement | null>(null);
const containerRef = useRef<HTMLDivElement | null>(null);
const appRef = useRef<Application | null>(null);
@@ -261,6 +282,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
const timeUpdateAnimationRef = useRef<number | null>(null);
const [pixiReady, setPixiReady] = useState(false);
const [videoReady, setVideoReady] = useState(false);
const [supplementalAudioPath, setSupplementalAudioPath] = useState<string | null>(null);
const [overlaySize, setOverlaySize] = useState({ width: 800, height: 600 });
const [overlayElement, setOverlayElement] = useState<HTMLDivElement | null>(null);
const overlayRef = useRef<HTMLDivElement | null>(null);
@@ -582,10 +604,19 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
if (!vid) return;
try {
allowPlaybackRef.current = true;
enableAllPreviewAudioTracks(vid);
await vid.play().catch((err) => {
console.log("PLAY ERROR:", err);
throw err;
});
const supplementalAudio = supplementalAudioRef.current;
if (supplementalAudio) {
supplementalAudio.currentTime = vid.currentTime;
supplementalAudio.playbackRate = vid.playbackRate;
await supplementalAudio.play().catch(() => {
// The main video remains the source of truth for playback state.
});
}
} catch (error) {
allowPlaybackRef.current = false;
throw error;
@@ -598,6 +629,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
return;
}
video.pause();
supplementalAudioRef.current?.pause();
},
}));
@@ -1005,11 +1037,30 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
lastResolvedDurationRef.current = null;
isResolvingDurationRef.current = false;
setVideoReady(false);
setSupplementalAudioPath(null);
return;
}
let cancelled = false;
window.electronAPI
?.preparePreviewAudioTrack?.(videoPath)
.then((result) => {
if (!cancelled) {
setSupplementalAudioPath(result.success ? (result.path ?? null) : null);
}
})
.catch(() => {
if (!cancelled) {
setSupplementalAudioPath(null);
}
});
const video = videoRef.current;
if (!video) return;
if (!video) {
return () => {
cancelled = true;
};
}
video.pause();
video.currentTime = 0;
allowPlaybackRef.current = false;
@@ -1026,8 +1077,42 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
videoReadyRafRef.current = null;
}
video.load();
return () => {
cancelled = true;
};
}, [videoPath]);
useEffect(() => {
const video = videoRef.current;
const supplementalAudio = supplementalAudioRef.current;
if (!video || !supplementalAudio || !supplementalAudioPath) {
return;
}
const activeSpeedRegion =
speedRegions.find(
(region) => currentTime * 1000 >= region.startMs && currentTime * 1000 < region.endMs,
) ?? null;
supplementalAudio.playbackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1;
if (!isPlaying) {
supplementalAudio.pause();
if (Math.abs(supplementalAudio.currentTime - currentTime) > 0.05) {
supplementalAudio.currentTime = currentTime;
}
return;
}
if (Math.abs(supplementalAudio.currentTime - video.currentTime) > 0.15) {
supplementalAudio.currentTime = video.currentTime;
}
supplementalAudio.play().catch(() => {
// Keep video playback running even if supplemental preview audio is unavailable.
});
}, [currentTime, isPlaying, speedRegions, supplementalAudioPath]);
useEffect(() => {
if (!pixiReady || !videoReady) return;
@@ -1545,6 +1630,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
const handleLoadedMetadata = (e: React.SyntheticEvent<HTMLVideoElement, Event>) => {
const video = e.currentTarget;
enableAllPreviewAudioTracks(video);
const hasResolvedDuration = syncResolvedDuration(video);
if (!hasResolvedDuration) {
forceResolveDuration(video);
@@ -1928,22 +2014,28 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
playsInline
onLoadedMetadata={handleLoadedMetadata}
onDurationChange={(e) => {
enableAllPreviewAudioTracks(e.currentTarget);
if (!syncResolvedDuration(e.currentTarget)) {
forceResolveDuration(e.currentTarget);
}
}}
onLoadedData={(e) => {
enableAllPreviewAudioTracks(e.currentTarget);
if (!syncResolvedDuration(e.currentTarget)) {
forceResolveDuration(e.currentTarget);
}
}}
onCanPlay={(e) => {
enableAllPreviewAudioTracks(e.currentTarget);
if (!syncResolvedDuration(e.currentTarget)) {
forceResolveDuration(e.currentTarget);
}
}}
onError={() => onError("Failed to load video")}
/>
{supplementalAudioPath && (
<audio ref={supplementalAudioRef} src={supplementalAudioPath} preload="auto" />
)}
</div>
);
},