From 64bc261c2024263ce04d10b54f646600880df133 Mon Sep 17 00:00:00 2001 From: Siddharth Date: Sat, 7 Mar 2026 15:56:11 -0800 Subject: [PATCH] audio recording and settings --- electron/electron-env.d.ts | 1 + electron/preload.ts | 3 + electron/windows.ts | 6 +- src/components/launch/LaunchWindow.tsx | 194 +++++++++++++++--------- src/components/ui/audio-level-meter.tsx | 37 +++++ src/hooks/useAudioLevelMeter.ts | 106 +++++++++++++ src/hooks/useMicrophoneDevices.ts | 80 ++++++++++ src/hooks/useScreenRecorder.ts | 96 ++++++++++-- src/lib/exporter/audioEncoder.ts | 173 +++++++++++++++++++++ src/lib/exporter/streamingDecoder.ts | 10 ++ src/lib/exporter/videoExporter.ts | 23 ++- src/vite-env.d.ts | 71 ++++----- 12 files changed, 682 insertions(+), 118 deletions(-) create mode 100644 src/components/ui/audio-level-meter.tsx create mode 100644 src/hooks/useAudioLevelMeter.ts create mode 100644 src/hooks/useMicrophoneDevices.ts create mode 100644 src/lib/exporter/audioEncoder.ts diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index cedf2f8..1f6885c 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -52,6 +52,7 @@ interface Window { saveShortcuts: (shortcuts: unknown) => Promise<{ success: boolean; error?: string }> hudOverlayHide: () => void; hudOverlayClose: () => void; + setMicrophoneExpanded: (expanded: boolean) => void; } } diff --git a/electron/preload.ts b/electron/preload.ts index ae3ebd7..2dda463 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -99,4 +99,7 @@ contextBridge.exposeInMainWorld('electronAPI', { saveShortcuts: (shortcuts: unknown) => { return ipcRenderer.invoke('save-shortcuts', shortcuts) }, + setMicrophoneExpanded: (expanded: boolean) => { + ipcRenderer.send('hud:setMicrophoneExpanded', expanded) + }, }) diff --git a/electron/windows.ts b/electron/windows.ts index f094fd6..7635626 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -23,7 +23,7 @@ export function createHudOverlayWindow(): BrowserWindow { const windowWidth = 500; - const windowHeight = 100; + const windowHeight = 155; const x = Math.floor(workArea.x + (workArea.width - windowWidth) / 2); const y = Math.floor(workArea.y + workArea.height - windowHeight - 5); @@ -33,8 +33,8 @@ export function createHudOverlayWindow(): BrowserWindow { height: windowHeight, minWidth: 500, maxWidth: 500, - minHeight: 100, - maxHeight: 100, + minHeight: 155, + maxHeight: 155, x: x, y: y, frame: false, diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index 40809a0..108cd14 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -1,20 +1,36 @@ import { useState, useEffect } from "react"; import styles from "./LaunchWindow.module.css"; import { useScreenRecorder } from "../../hooks/useScreenRecorder"; +import { useMicrophoneDevices } from "../../hooks/useMicrophoneDevices"; +import { useAudioLevelMeter } from "../../hooks/useAudioLevelMeter"; +import { AudioLevelMeter } from "../ui/audio-level-meter"; import { Button } from "../ui/button"; import { BsRecordCircle } from "react-icons/bs"; import { FaRegStopCircle } from "react-icons/fa"; -import { MdMonitor } from "react-icons/md"; +import { MdMonitor, MdMic, MdMicOff } from "react-icons/md"; import { RxDragHandleDots2 } from "react-icons/rx"; import { FaFolderMinus } from "react-icons/fa6"; import { FiMinus, FiX } from "react-icons/fi"; import { ContentClamp } from "../ui/content-clamp"; export function LaunchWindow() { - const { recording, toggleRecording } = useScreenRecorder(); + const { recording, toggleRecording, microphoneEnabled, setMicrophoneEnabled, microphoneDeviceId, setMicrophoneDeviceId } = useScreenRecorder(); const [recordingStart, setRecordingStart] = useState(null); const [elapsed, setElapsed] = useState(0); + const showMicControls = microphoneEnabled && !recording; + const { devices, selectedDeviceId, setSelectedDeviceId } = useMicrophoneDevices(microphoneEnabled); + const { level } = useAudioLevelMeter({ + enabled: showMicControls, + deviceId: microphoneDeviceId, + }); + + useEffect(() => { + if (selectedDeviceId && selectedDeviceId !== 'default') { + setMicrophoneDeviceId(selectedDeviceId); + } + }, [selectedDeviceId, setMicrophoneDeviceId]); + useEffect(() => { let timer: NodeJS.Timeout | null = null; if (recording) { @@ -93,10 +109,16 @@ export function LaunchWindow() { } }; + const toggleMicrophone = () => { + if (!recording) { + setMicrophoneEnabled(!microphoneEnabled); + } + }; + return ( -
+
-
+ {showMicControls && ( +
+ + +
+ )} - +
+
-
+ - - +
-
+ +
- + - {/* Separator before hide/close buttons */} -
- +
- + + +
+ + + +
); diff --git a/src/components/ui/audio-level-meter.tsx b/src/components/ui/audio-level-meter.tsx new file mode 100644 index 0000000..9124b90 --- /dev/null +++ b/src/components/ui/audio-level-meter.tsx @@ -0,0 +1,37 @@ +interface AudioLevelMeterProps { + level: number; // 0-100 + className?: string; +} + +const bars = [ + { threshold: 10, height: '30%' }, + { threshold: 25, height: '45%' }, + { threshold: 45, height: '60%' }, + { threshold: 65, height: '75%' }, + { threshold: 85, height: '90%' }, +]; + +function getBarColor(level: number, threshold: number) { + if (!level || level < threshold) return 'bg-slate-700'; + if (threshold > 80) return 'bg-red-500'; + if (threshold > 60) return 'bg-yellow-500'; + if (threshold > 40) return 'bg-green-500'; + return 'bg-emerald-500'; +} + +export function AudioLevelMeter({ level, className = "" }: AudioLevelMeterProps) { + return ( +
+ {bars.map((bar, index) => ( +
= bar.threshold ? bar.height : '15%', + opacity: level >= bar.threshold ? 1 : 0.4, + }} + /> + ))} +
+ ); +} diff --git a/src/hooks/useAudioLevelMeter.ts b/src/hooks/useAudioLevelMeter.ts new file mode 100644 index 0000000..ee2970b --- /dev/null +++ b/src/hooks/useAudioLevelMeter.ts @@ -0,0 +1,106 @@ +import { useState, useEffect, useRef } from 'react'; + +export interface AudioLevelMeterOptions { + enabled: boolean; + deviceId?: string; + smoothingFactor?: number; +} + +export function useAudioLevelMeter(options: AudioLevelMeterOptions) { + const [level, setLevel] = useState(0); + const audioContextRef = useRef(null); + const analyserRef = useRef(null); + const streamRef = useRef(null); + const animationFrameRef = useRef(null); + + useEffect(() => { + if (!options.enabled) { + cleanup(); + setLevel(0); + return; + } + + let mounted = true; + + const startMonitoring = async () => { + try { + const constraints: MediaStreamConstraints = { + audio: options.deviceId + ? { deviceId: { exact: options.deviceId } } + : true, + video: false, + }; + + const stream = await navigator.mediaDevices.getUserMedia(constraints); + + if (!mounted) { + stream.getTracks().forEach(track => track.stop()); + return; + } + + streamRef.current = stream; + + const audioContext = new AudioContext(); + audioContextRef.current = audioContext; + + const analyser = audioContext.createAnalyser(); + analyser.fftSize = 256; + analyser.smoothingTimeConstant = options.smoothingFactor ?? 0.8; + analyserRef.current = analyser; + + const source = audioContext.createMediaStreamSource(stream); + source.connect(analyser); + + const dataArray = new Uint8Array(analyser.frequencyBinCount); + + const updateLevel = () => { + if (!mounted || !analyserRef.current) return; + + analyser.getByteFrequencyData(dataArray); + + let sum = 0; + for (let i = 0; i < dataArray.length; i++) { + sum += dataArray[i] * dataArray[i]; + } + const rms = Math.sqrt(sum / dataArray.length); + const normalizedLevel = Math.min(100, (rms / 255) * 100 * 2); + + setLevel(normalizedLevel); + animationFrameRef.current = requestAnimationFrame(updateLevel); + }; + + updateLevel(); + } catch (err) { + console.error('Error starting audio level monitoring:', err); + if (mounted) { + setLevel(0); + } + } + }; + + startMonitoring(); + + return () => { + mounted = false; + cleanup(); + }; + }, [options.enabled, options.deviceId, options.smoothingFactor]); + + const cleanup = () => { + if (animationFrameRef.current) { + cancelAnimationFrame(animationFrameRef.current); + animationFrameRef.current = null; + } + if (streamRef.current) { + streamRef.current.getTracks().forEach(track => track.stop()); + streamRef.current = null; + } + if (audioContextRef.current) { + audioContextRef.current.close(); + audioContextRef.current = null; + } + analyserRef.current = null; + }; + + return { level }; +} diff --git a/src/hooks/useMicrophoneDevices.ts b/src/hooks/useMicrophoneDevices.ts new file mode 100644 index 0000000..1b31464 --- /dev/null +++ b/src/hooks/useMicrophoneDevices.ts @@ -0,0 +1,80 @@ +import { useState, useEffect } from 'react'; + +export interface MicrophoneDevice { + deviceId: string; + label: string; + groupId: string; +} + +export function useMicrophoneDevices(enabled: boolean = true) { + const [devices, setDevices] = useState([]); + const [selectedDeviceId, setSelectedDeviceId] = useState('default'); + const [isLoading, setIsLoading] = useState(false); + const [error, setError] = useState(null); + + useEffect(() => { + if (!enabled) { + return; + } + + let mounted = true; + + const loadDevices = async () => { + try { + setIsLoading(true); + setError(null); + + // Request permission first to get actual device labels + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + + const allDevices = await navigator.mediaDevices.enumerateDevices(); + const audioInputs = allDevices + .filter(device => device.kind === 'audioinput') + .map(device => ({ + deviceId: device.deviceId, + label: device.label || `Microphone ${device.deviceId.slice(0, 8)}`, + groupId: device.groupId, + })); + + // Stop the permission stream + stream.getTracks().forEach(track => track.stop()); + + if (mounted) { + setDevices(audioInputs); + if (selectedDeviceId === 'default' && audioInputs.length > 0) { + setSelectedDeviceId(audioInputs[0].deviceId); + } + setIsLoading(false); + } + } catch (err) { + if (mounted) { + const errorMessage = err instanceof Error ? err.message : 'Failed to enumerate audio devices'; + setError(errorMessage); + setIsLoading(false); + console.error('Error loading microphone devices:', err); + } + } + }; + + loadDevices(); + + const handleDeviceChange = () => { + loadDevices(); + }; + + navigator.mediaDevices.addEventListener('devicechange', handleDeviceChange); + + return () => { + mounted = false; + navigator.mediaDevices.removeEventListener('devicechange', handleDeviceChange); + }; + }, [enabled]); + + return { + devices, + selectedDeviceId, + setSelectedDeviceId, + isLoading, + error, + }; +} diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 5b25c16..82269ad 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -1,5 +1,6 @@ import { useState, useRef, useEffect } from "react"; import { fixWebmDuration } from "@fix-webm-duration/fix"; +import { toast } from "sonner"; // Target visually lossless 4K @ 60fps; fall back gracefully when hardware cannot keep up const TARGET_FRAME_RATE = 60; @@ -34,12 +35,20 @@ const VIDEO_FILE_EXTENSION = ".webm"; type UseScreenRecorderReturn = { recording: boolean; toggleRecording: () => void; + microphoneEnabled: boolean; + setMicrophoneEnabled: (enabled: boolean) => void; + microphoneDeviceId: string | undefined; + setMicrophoneDeviceId: (deviceId: string | undefined) => void; }; export function useScreenRecorder(): UseScreenRecorderReturn { const [recording, setRecording] = useState(false); + const [microphoneEnabled, setMicrophoneEnabled] = useState(false); + const [microphoneDeviceId, setMicrophoneDeviceId] = useState(undefined); const mediaRecorder = useRef(null); const stream = useRef(null); + const screenStream = useRef(null); + const microphoneStream = useRef(null); const chunks = useRef([]); const startTime = useRef(0); @@ -75,6 +84,14 @@ export function useScreenRecorder(): UseScreenRecorderReturn { if (stream.current) { stream.current.getTracks().forEach(track => track.stop()); } + if (screenStream.current) { + screenStream.current.getTracks().forEach(track => track.stop()); + screenStream.current = null; + } + if (microphoneStream.current) { + microphoneStream.current.getTracks().forEach(track => track.stop()); + microphoneStream.current = null; + } mediaRecorder.current.stop(); setRecording(false); @@ -93,7 +110,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { return () => { if (cleanup) cleanup(); - + if (mediaRecorder.current?.state === "recording") { mediaRecorder.current.stop(); } @@ -101,6 +118,14 @@ export function useScreenRecorder(): UseScreenRecorderReturn { stream.current.getTracks().forEach(track => track.stop()); stream.current = null; } + if (screenStream.current) { + screenStream.current.getTracks().forEach(track => track.stop()); + screenStream.current = null; + } + if (microphoneStream.current) { + microphoneStream.current.getTracks().forEach(track => track.stop()); + microphoneStream.current = null; + } }; }, []); @@ -112,7 +137,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { return; } - const mediaStream = await (navigator.mediaDevices as any).getUserMedia({ + const screenMediaStream = await (navigator.mediaDevices as any).getUserMedia({ audio: false, video: { mandatory: { @@ -125,19 +150,55 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }, }, }); - stream.current = mediaStream; - if (!stream.current) { - throw new Error("Media stream is not available."); + screenStream.current = screenMediaStream; + + // If microphone is enabled, request mic stream + if (microphoneEnabled) { + try { + microphoneStream.current = await navigator.mediaDevices.getUserMedia({ + audio: microphoneDeviceId + ? { + deviceId: { exact: microphoneDeviceId }, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + } + : { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + video: false, + }); + } catch (audioError) { + console.warn('Failed to get microphone access:', audioError); + toast.error('Microphone access denied. Recording will continue without audio.'); + setMicrophoneEnabled(false); + } + } + + // Combine streams + stream.current = new MediaStream(); + const videoTrack = screenMediaStream.getVideoTracks()[0]; + if (!videoTrack) { + throw new Error("Video track is not available."); + } + stream.current.addTrack(videoTrack); + + if (microphoneStream.current) { + const micAudioTrack = microphoneStream.current.getAudioTracks()[0]; + if (micAudioTrack) { + stream.current.addTrack(micAudioTrack); + } } - const videoTrack = stream.current.getVideoTracks()[0]; try { await videoTrack.applyConstraints({ frameRate: { ideal: TARGET_FRAME_RATE, max: TARGET_FRAME_RATE }, width: { ideal: TARGET_WIDTH, max: TARGET_WIDTH }, height: { ideal: TARGET_HEIGHT, max: TARGET_HEIGHT }, }); - } catch (error) { - console.warn("Unable to lock 4K/60fps constraints, using best available track settings.", error); + } catch (constraintError) { + console.warn("Unable to lock 4K/60fps constraints, using best available track settings.", constraintError); } let { width = DEFAULT_WIDTH, height = DEFAULT_HEIGHT, frameRate = TARGET_FRAME_RATE } = videoTrack.getSettings(); @@ -155,10 +216,13 @@ export function useScreenRecorder(): UseScreenRecorderReturn { )} Mbps` ); + const hasMicAudio = microphoneEnabled && microphoneStream.current !== null; + chunks.current = []; const recorder = new MediaRecorder(stream.current, { mimeType, videoBitsPerSecond, + ...(hasMicAudio ? { audioBitsPerSecond: 128_000 } : {}), }); mediaRecorder.current = recorder; recorder.ondataavailable = e => { @@ -200,11 +264,25 @@ export function useScreenRecorder(): UseScreenRecorderReturn { window.electronAPI?.setRecordingState(true); } catch (error) { console.error('Failed to start recording:', error); + const errorMsg = error instanceof Error ? error.message : 'Failed to start recording'; + if (errorMsg.includes('Permission denied') || errorMsg.includes('NotAllowedError')) { + toast.error('Recording permission denied. Please allow screen recording.'); + } else { + toast.error(errorMsg); + } setRecording(false); if (stream.current) { stream.current.getTracks().forEach(track => track.stop()); stream.current = null; } + if (screenStream.current) { + screenStream.current.getTracks().forEach(track => track.stop()); + screenStream.current = null; + } + if (microphoneStream.current) { + microphoneStream.current.getTracks().forEach(track => track.stop()); + microphoneStream.current = null; + } } }; @@ -212,5 +290,5 @@ export function useScreenRecorder(): UseScreenRecorderReturn { recording ? stopRecording.current() : startRecording(); }; - return { recording, toggleRecording }; + return { recording, toggleRecording, microphoneEnabled, setMicrophoneEnabled, microphoneDeviceId, setMicrophoneDeviceId }; } diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts new file mode 100644 index 0000000..b875acb --- /dev/null +++ b/src/lib/exporter/audioEncoder.ts @@ -0,0 +1,173 @@ +import type { WebDemuxer } from 'web-demuxer'; +import type { TrimRegion } from '@/components/video-editor/types'; +import type { VideoMuxer } from './muxer'; + +const AUDIO_BITRATE = 128_000; +const DECODE_BACKPRESSURE_LIMIT = 20; + +export class AudioProcessor { + private cancelled = false; + + async process( + demuxer: WebDemuxer, + muxer: VideoMuxer, + trimRegions?: TrimRegion[], + ): Promise { + let audioConfig: AudioDecoderConfig; + try { + audioConfig = await demuxer.getDecoderConfig('audio') as AudioDecoderConfig; + } catch { + console.warn('[AudioProcessor] No audio track found, skipping'); + return; + } + + const codecCheck = await AudioDecoder.isConfigSupported(audioConfig); + if (!codecCheck.supported) { + console.warn('[AudioProcessor] Audio codec not supported:', audioConfig.codec); + return; + } + + const sortedTrims = trimRegions + ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) + : []; + + // Phase 1: Decode audio from source, skipping trimmed regions + const decodedFrames: AudioData[] = []; + + const decoder = new AudioDecoder({ + output: (data: AudioData) => decodedFrames.push(data), + error: (e: DOMException) => console.error('[AudioProcessor] Decode error:', e), + }); + decoder.configure(audioConfig); + + const reader = (demuxer.read('audio') as ReadableStream).getReader(); + + while (!this.cancelled) { + const { done, value: chunk } = await reader.read(); + if (done || !chunk) break; + + const timestampMs = chunk.timestamp / 1000; + if (this.isInTrimRegion(timestampMs, sortedTrims)) continue; + + decoder.decode(chunk); + + while (decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT && !this.cancelled) { + await new Promise(resolve => setTimeout(resolve, 1)); + } + } + + if (decoder.state === 'configured') { + await decoder.flush(); + decoder.close(); + } + + if (this.cancelled || decodedFrames.length === 0) { + for (const f of decodedFrames) f.close(); + return; + } + + // Phase 2: Re-encode with timestamps adjusted for trim gaps + const encodedChunks: { chunk: EncodedAudioChunk; meta?: EncodedAudioChunkMetadata }[] = []; + + const encoder = new AudioEncoder({ + output: (chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) => { + encodedChunks.push({ chunk, meta }); + }, + error: (e: DOMException) => console.error('[AudioProcessor] Encode error:', e), + }); + + const sampleRate = audioConfig.sampleRate || 48000; + const channels = audioConfig.numberOfChannels || 2; + + const encodeConfig: AudioEncoderConfig = { + codec: 'opus', + sampleRate, + numberOfChannels: channels, + bitrate: AUDIO_BITRATE, + }; + + const encodeSupport = await AudioEncoder.isConfigSupported(encodeConfig); + if (!encodeSupport.supported) { + console.warn('[AudioProcessor] Opus encoding not supported, skipping audio'); + for (const f of decodedFrames) f.close(); + return; + } + + encoder.configure(encodeConfig); + + for (const audioData of decodedFrames) { + if (this.cancelled) { + audioData.close(); + continue; + } + + const timestampMs = audioData.timestamp / 1000; + const trimOffsetMs = this.computeTrimOffset(timestampMs, sortedTrims); + const adjustedTimestampUs = audioData.timestamp - trimOffsetMs * 1000; + + const adjusted = this.cloneWithTimestamp(audioData, Math.max(0, adjustedTimestampUs)); + audioData.close(); + + encoder.encode(adjusted); + adjusted.close(); + } + + if (encoder.state === 'configured') { + await encoder.flush(); + encoder.close(); + } + + // Phase 3: Flush encoded chunks to muxer + for (const { chunk, meta } of encodedChunks) { + if (this.cancelled) break; + await muxer.addAudioChunk(chunk, meta); + } + + console.log(`[AudioProcessor] Processed ${decodedFrames.length} audio frames, encoded ${encodedChunks.length} chunks`); + } + + private cloneWithTimestamp(src: AudioData, newTimestamp: number): AudioData { + const isPlanar = src.format?.includes('planar') ?? false; + const numPlanes = isPlanar ? src.numberOfChannels : 1; + + let totalSize = 0; + for (let p = 0; p < numPlanes; p++) { + totalSize += src.allocationSize({ planeIndex: p }); + } + + const buffer = new ArrayBuffer(totalSize); + let offset = 0; + for (let p = 0; p < numPlanes; p++) { + const planeSize = src.allocationSize({ planeIndex: p }); + src.copyTo(new Uint8Array(buffer, offset, planeSize), { planeIndex: p }); + offset += planeSize; + } + + return new AudioData({ + format: src.format!, + sampleRate: src.sampleRate, + numberOfFrames: src.numberOfFrames, + numberOfChannels: src.numberOfChannels, + timestamp: newTimestamp, + data: buffer, + }); + } + + private isInTrimRegion(timestampMs: number, trims: TrimRegion[]): boolean { + return trims.some(t => timestampMs >= t.startMs && timestampMs < t.endMs); + } + + private computeTrimOffset(timestampMs: number, trims: TrimRegion[]): number { + let offset = 0; + for (const trim of trims) { + if (trim.endMs <= timestampMs) { + offset += trim.endMs - trim.startMs; + } + } + return offset; + } + + cancel(): void { + this.cancelled = true; + } +} diff --git a/src/lib/exporter/streamingDecoder.ts b/src/lib/exporter/streamingDecoder.ts index d07e164..84e6ffb 100644 --- a/src/lib/exporter/streamingDecoder.ts +++ b/src/lib/exporter/streamingDecoder.ts @@ -7,6 +7,8 @@ export interface DecodedVideoInfo { duration: number; // seconds frameRate: number; codec: string; + hasAudio: boolean; + audioCodec?: string; } /** Caller must close the VideoFrame after use. */ @@ -53,12 +55,16 @@ export class StreamingVideoDecoder { } } + const audioStream = mediaInfo.streams.find(s => s.codec_type_string === 'audio'); + this.metadata = { width: videoStream?.width || 1920, height: videoStream?.height || 1080, duration: mediaInfo.duration, frameRate, codec: videoStream?.codec_string || 'unknown', + hasAudio: !!audioStream, + audioCodec: audioStream?.codec_string, }; return this.metadata; @@ -312,6 +318,10 @@ export class StreamingVideoDecoder { return result.filter(s => s.endSec - s.startSec > 0.0001); } + getDemuxer(): WebDemuxer | null { + return this.demuxer; + } + cancel(): void { this.cancelled = true; } diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index 07f7199..cd34f9b 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -2,6 +2,7 @@ import type { ExportConfig, ExportProgress, ExportResult } from './types'; import { StreamingVideoDecoder } from './streamingDecoder'; import { FrameRenderer } from './frameRenderer'; import { VideoMuxer } from './muxer'; +import { AudioProcessor } from './audioEncoder'; import type { ZoomRegion, CropRegion, TrimRegion, AnnotationRegion, SpeedRegion } from '@/components/video-editor/types'; interface VideoExporterConfig extends ExportConfig { @@ -30,6 +31,7 @@ export class VideoExporter { private renderer: FrameRenderer | null = null; private encoder: VideoEncoder | null = null; private muxer: VideoMuxer | null = null; + private audioProcessor: AudioProcessor | null = null; private cancelled = false; private encodeQueue = 0; // Increased queue size for better throughput with hardware encoding @@ -78,8 +80,9 @@ export class VideoExporter { // Initialize video encoder await this.initializeEncoder(); - // Initialize muxer - this.muxer = new VideoMuxer(this.config, false); + // Initialize muxer (with audio if source has an audio track) + const hasAudio = videoInfo.hasAudio; + this.muxer = new VideoMuxer(this.config, hasAudio); await this.muxer.initialize(); // Calculate effective duration and frame count (excluding trim regions) @@ -164,9 +167,19 @@ export class VideoExporter { await this.encoder.flush(); } - // Wait for all muxing operations to complete + // Wait for all video muxing operations to complete await Promise.all(this.muxingPromises); + // Process audio track if present + if (hasAudio && !this.cancelled) { + const demuxer = this.streamingDecoder!.getDemuxer(); + if (demuxer) { + console.log('[VideoExporter] Processing audio track...'); + this.audioProcessor = new AudioProcessor(); + await this.audioProcessor.process(demuxer, this.muxer!, this.config.trimRegions); + } + } + // Finalize muxer and get output blob const blob = await this.muxer!.finalize(); @@ -284,6 +297,9 @@ export class VideoExporter { if (this.streamingDecoder) { this.streamingDecoder.cancel(); } + if (this.audioProcessor) { + this.audioProcessor.cancel(); + } this.cleanup(); } @@ -317,6 +333,7 @@ export class VideoExporter { this.renderer = null; } + this.audioProcessor = null; this.muxer = null; this.encodeQueue = 0; this.muxingPromises = []; diff --git a/src/vite-env.d.ts b/src/vite-env.d.ts index 31bf196..08378ef 100644 --- a/src/vite-env.d.ts +++ b/src/vite-env.d.ts @@ -44,41 +44,42 @@ interface Window { }> onStopRecordingFromTray: (callback: () => void) => () => void openExternalUrl: (url: string) => Promise<{ success: boolean; error?: string }> - saveExportedVideo: (videoData: ArrayBuffer, fileName: string) => Promise<{ - success: boolean - path?: string - message?: string - canceled?: boolean - }> - openVideoFilePicker: () => Promise<{ success: boolean; path?: string; canceled?: boolean }> - setCurrentVideoPath: (path: string) => Promise<{ success: boolean }> - getCurrentVideoPath: () => Promise<{ success: boolean; path?: string }> - clearCurrentVideoPath: () => Promise<{ success: boolean }> - saveProjectFile: (projectData: unknown, suggestedName?: string, existingProjectPath?: string) => Promise<{ - success: boolean - path?: string - message?: string - canceled?: boolean - error?: string - }> - loadProjectFile: () => Promise<{ - success: boolean - path?: string - project?: unknown - message?: string - canceled?: boolean - error?: string - }> - loadCurrentProjectFile: () => Promise<{ - success: boolean - path?: string - project?: unknown - message?: string - canceled?: boolean - error?: string - }> - onMenuLoadProject: (callback: () => void) => () => void + saveExportedVideo: (videoData: ArrayBuffer, fileName: string) => Promise<{ + success: boolean + path?: string + message?: string + canceled?: boolean + }> + openVideoFilePicker: () => Promise<{ success: boolean; path?: string; canceled?: boolean }> + setCurrentVideoPath: (path: string) => Promise<{ success: boolean }> + getCurrentVideoPath: () => Promise<{ success: boolean; path?: string }> + clearCurrentVideoPath: () => Promise<{ success: boolean }> + saveProjectFile: (projectData: unknown, suggestedName?: string, existingProjectPath?: string) => Promise<{ + success: boolean + path?: string + message?: string + canceled?: boolean + error?: string + }> + loadProjectFile: () => Promise<{ + success: boolean + path?: string + project?: unknown + message?: string + canceled?: boolean + error?: string + }> + loadCurrentProjectFile: () => Promise<{ + success: boolean + path?: string + project?: unknown + message?: string + canceled?: boolean + error?: string + }> + onMenuLoadProject: (callback: () => void) => () => void onMenuSaveProject: (callback: () => void) => () => void onMenuSaveProjectAs: (callback: () => void) => () => void + setMicrophoneExpanded: (expanded: boolean) => void } -} +}