diff --git a/electron-builder.json5 b/electron-builder.json5 index c11dc56..741975b 100644 --- a/electron-builder.json5 +++ b/electron-builder.json5 @@ -6,8 +6,7 @@ // .node binaries can't be dlopen'd from inside an asar — must live unpacked. "asarUnpack": [ "node_modules/uiohook-napi/**/*", - "**/*.node", - "electron/native/bin/**" + "**/*.node" ], "productName": "Openscreen", "npmRebuild": true, @@ -19,10 +18,9 @@ "files": [ "dist", "dist-electron", - "electron/native/bin/**/*", "!*.png", - "!preview*.png", - "!*.md", + "!preview*.png", + "!*.md", "!README.md", "!CONTRIBUTING.md", "!LICENSE" @@ -67,12 +65,19 @@ "artifactName": "${productName}-Linux-${version}.${ext}", "category": "AudioVideo" }, - "win": { - "target": [ - "nsis" - ], - "icon": "icons/icons/win/icon.ico" - }, + "win": { + "target": [ + "nsis" + ], + "icon": "icons/icons/win/icon.ico", + "extraResources": [ + { + "from": "electron/native/bin", + "to": "electron/native/bin", + "filter": ["**/*"] + } + ] + }, "nsis": { "oneClick": false, "allowToChangeInstallationDirectory": true diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 3a53ba9..159ee2f 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -453,6 +453,14 @@ function resolveUnpackedAppPath(...segments: string[]) { return resolved; } +function resolvePackagedResourcePath(...segments: string[]) { + if (!app.isPackaged) { + return null; + } + + return path.join(process.resourcesPath, ...segments); +} + function getNativeWindowsCaptureHelperCandidates() { const envPath = process.env.OPENSCREEN_WGC_CAPTURE_EXE?.trim(); const archTag = process.arch === "arm64" ? "win32-arm64" : "win32-x64"; @@ -468,6 +476,7 @@ function getNativeWindowsCaptureHelperCandidates() { ), resolveUnpackedAppPath("electron", "native", "wgc-capture", "build", "wgc-capture.exe"), resolveUnpackedAppPath("electron", "native", "bin", archTag, "wgc-capture.exe"), + resolvePackagedResourcePath("electron", "native", "bin", archTag, "wgc-capture.exe"), ].filter((candidate): candidate is string => Boolean(candidate)); } diff --git a/electron/native-bridge/cursor/recording/factory.ts b/electron/native-bridge/cursor/recording/factory.ts index 52d6079..7835841 100644 --- a/electron/native-bridge/cursor/recording/factory.ts +++ b/electron/native-bridge/cursor/recording/factory.ts @@ -1,6 +1,6 @@ import type { Rectangle } from "electron"; +import type { CursorRecordingData } from "../../../../src/native/contracts"; import type { CursorRecordingSession } from "./session"; -import { TelemetryRecordingSession } from "./telemetryRecordingSession"; import { WindowsNativeRecordingSession } from "./windowsNativeRecordingSession"; interface CreateCursorRecordingSessionOptions { @@ -12,6 +12,21 @@ interface CreateCursorRecordingSessionOptions { startTimeMs?: number; } +class NoopCursorRecordingSession implements CursorRecordingSession { + async start(): Promise { + // Native cursor capture is currently Windows-only. + } + + async stop(): Promise { + return { + version: 2, + provider: "none", + assets: [], + samples: [], + }; + } +} + export function createCursorRecordingSession( options: CreateCursorRecordingSessionOptions, ): CursorRecordingSession { @@ -25,10 +40,5 @@ export function createCursorRecordingSession( }); } - return new TelemetryRecordingSession({ - getDisplayBounds: options.getDisplayBounds, - maxSamples: options.maxSamples, - sampleIntervalMs: options.sampleIntervalMs, - startTimeMs: options.startTimeMs, - }); + return new NoopCursorRecordingSession(); } diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 4971ff7..1c39e29 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -9,6 +9,7 @@ import { import { requestCameraAccess } from "@/lib/requestCameraAccess"; const TARGET_FRAME_RATE = 60; +const MIN_FRAME_RATE = 30; const TARGET_WIDTH = 3840; const TARGET_HEIGHT = 2160; const FOUR_K_PIXELS = TARGET_WIDTH * TARGET_HEIGHT; @@ -29,6 +30,7 @@ const CODEC_ALIGNMENT = 2; const RECORDER_TIMESLICE_MS = 1000; const BITS_PER_MEGABIT = 1_000_000; +const CHROME_MEDIA_SOURCE = "desktop"; const RECORDING_FILE_PREFIX = "recording-"; const VIDEO_FILE_EXTENSION = ".webm"; const WEBCAM_FILE_SUFFIX = "-webcam"; @@ -759,20 +761,60 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } let screenMediaStream: MediaStream; + const platform = await window.electronAPI.getPlatform(); - // getDisplayMedia + setDisplayMediaRequestHandler (main.ts) supplies the - // pre-selected source and honors cursor:"never" to exclude the system cursor - // from every captured frame. System audio is provided via WASAPI loopback - // on Windows when the user has enabled it. - screenMediaStream = await navigator.mediaDevices.getDisplayMedia({ - video: { - cursor: "never", - width: { max: TARGET_WIDTH }, - height: { max: TARGET_HEIGHT }, - frameRate: { ideal: TARGET_FRAME_RATE }, - } as MediaTrackConstraints, - audio: systemAudioEnabled, - } as DisplayMediaStreamOptions); + if (platform === "win32") { + // getDisplayMedia + setDisplayMediaRequestHandler (main.ts) supplies the + // pre-selected source and honors cursor:"never" to exclude the system cursor + // from every captured frame. System audio is provided via WASAPI loopback + // on Windows when the user has enabled it. + screenMediaStream = await navigator.mediaDevices.getDisplayMedia({ + video: { + cursor: "never", + width: { max: TARGET_WIDTH }, + height: { max: TARGET_HEIGHT }, + frameRate: { ideal: TARGET_FRAME_RATE }, + } as MediaTrackConstraints, + audio: systemAudioEnabled, + } as DisplayMediaStreamOptions); + } else { + const videoConstraints = { + mandatory: { + chromeMediaSource: CHROME_MEDIA_SOURCE, + chromeMediaSourceId: selectedSource.id, + maxWidth: TARGET_WIDTH, + maxHeight: TARGET_HEIGHT, + maxFrameRate: TARGET_FRAME_RATE, + minFrameRate: MIN_FRAME_RATE, + }, + }; + + if (systemAudioEnabled) { + try { + screenMediaStream = await navigator.mediaDevices.getUserMedia({ + audio: { + mandatory: { + chromeMediaSource: CHROME_MEDIA_SOURCE, + chromeMediaSourceId: selectedSource.id, + }, + }, + video: videoConstraints, + } as unknown as MediaStreamConstraints); + } catch (audioErr) { + console.warn("System audio capture failed, falling back to video-only:", audioErr); + toast.error(t("recording.systemAudioUnavailable")); + screenMediaStream = await navigator.mediaDevices.getUserMedia({ + audio: false, + video: videoConstraints, + } as unknown as MediaStreamConstraints); + } + } else { + screenMediaStream = await navigator.mediaDevices.getUserMedia({ + audio: false, + video: videoConstraints, + } as unknown as MediaStreamConstraints); + } + } screenStream.current = screenMediaStream; if (!isCountdownRunActive(countdownRunToken)) { diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 3237af2..870fdc3 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -1,16 +1,66 @@ import { WebDemuxer } from "web-demuxer"; import type { SpeedRegion, TrimRegion } from "@/components/video-editor/types"; -import type { VideoMuxer } from "./muxer"; +import type { ExportAudioMuxerCodec, VideoMuxer } from "./muxer"; const AUDIO_BITRATE = 128_000; -const EXPORT_AUDIO_CODEC = "mp4a.40.2"; const DECODE_BACKPRESSURE_LIMIT = 20; const MIN_SPEED_REGION_DELTA_MS = 0.0001; const SEEK_TIMEOUT_MS = 5_000; +export interface ExportAudioCodec { + encoderCodec: string; + muxerCodec: ExportAudioMuxerCodec; + label: string; +} + +const EXPORT_AUDIO_CODECS: ExportAudioCodec[] = [ + { encoderCodec: "mp4a.40.2", muxerCodec: "aac", label: "AAC" }, + { encoderCodec: "opus", muxerCodec: "opus", label: "Opus" }, +]; + export class AudioProcessor { private cancelled = false; + static async selectSupportedExportCodec( + sampleRate: number, + numberOfChannels: number, + ): Promise { + for (const codec of EXPORT_AUDIO_CODECS) { + const support = await AudioEncoder.isConfigSupported({ + codec: codec.encoderCodec, + sampleRate, + numberOfChannels, + bitrate: AUDIO_BITRATE, + }); + if (support.supported) { + return codec; + } + } + + return null; + } + + static async selectSupportedExportCodecForSource( + demuxer: WebDemuxer, + ): Promise { + let audioConfig: AudioDecoderConfig; + try { + audioConfig = await demuxer.getDecoderConfig("audio"); + } catch { + return null; + } + + const codecCheck = await AudioDecoder.isConfigSupported(audioConfig); + if (!codecCheck.supported) { + console.warn("[AudioProcessor] Audio codec not supported:", audioConfig.codec); + return null; + } + + const sampleRate = audioConfig.sampleRate || 48000; + const channels = audioConfig.numberOfChannels || 2; + return AudioProcessor.selectSupportedExportCodec(sampleRate, channels); + } + /** * Audio export has two modes: * 1) no speed regions -> fast WebCodecs trim-only pipeline @@ -23,6 +73,7 @@ export class AudioProcessor { trimRegions: TrimRegion[] | undefined, speedRegions: SpeedRegion[] | undefined, validatedDurationSec: number, + exportCodec: ExportAudioCodec, ): Promise { const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) : []; const sortedSpeedRegions = speedRegions @@ -40,7 +91,7 @@ export class AudioProcessor { validatedDurationSec, ); if (!this.cancelled && renderedAudioBlob.size > 0) { - await this.muxRenderedAudioBlob(renderedAudioBlob, muxer); + await this.muxRenderedAudioBlob(renderedAudioBlob, muxer, exportCodec); return; } return; @@ -50,7 +101,7 @@ export class AudioProcessor { // The +0.5s buffer mirrors streamingDecoder.decodeAll's read window so the trim-only // and speed-aware paths agree on how far to read past the validated duration boundary. const readEndSec = validatedDurationSec + 0.5; - await this.processTrimOnlyAudio(demuxer, muxer, sortedTrims, readEndSec); + await this.processTrimOnlyAudio(demuxer, muxer, sortedTrims, readEndSec, exportCodec); } // Legacy trim-only path. This is still used for projects without speed regions. @@ -59,6 +110,7 @@ export class AudioProcessor { muxer: VideoMuxer, sortedTrims: TrimRegion[], readEndSec?: number, + exportCodec?: ExportAudioCodec, ): Promise { let audioConfig: AudioDecoderConfig; try { @@ -137,9 +189,16 @@ export class AudioProcessor { const sampleRate = audioConfig.sampleRate || 48000; const channels = audioConfig.numberOfChannels || 2; + const selectedCodec = + exportCodec ?? (await AudioProcessor.selectSupportedExportCodec(sampleRate, channels)); + if (!selectedCodec) { + console.warn("[AudioProcessor] No supported audio export codec, skipping audio"); + for (const frame of decodedFrames) frame.close(); + return; + } const encodeConfig: AudioEncoderConfig = { - codec: EXPORT_AUDIO_CODEC, + codec: selectedCodec.encoderCodec, sampleRate, numberOfChannels: channels, bitrate: AUDIO_BITRATE, @@ -147,7 +206,9 @@ export class AudioProcessor { const encodeSupport = await AudioEncoder.isConfigSupported(encodeConfig); if (!encodeSupport.supported) { - console.warn("[AudioProcessor] AAC encoding not supported, skipping audio"); + console.warn( + `[AudioProcessor] ${selectedCodec.label} encoding not supported, skipping audio`, + ); for (const frame of decodedFrames) frame.close(); return; } @@ -389,7 +450,11 @@ export class AudioProcessor { } // Demuxes the rendered speed-adjusted blob and feeds encoded chunks into the MP4 muxer. - private async muxRenderedAudioBlob(blob: Blob, muxer: VideoMuxer): Promise { + private async muxRenderedAudioBlob( + blob: Blob, + muxer: VideoMuxer, + exportCodec: ExportAudioCodec, + ): Promise { if (this.cancelled) return; const file = new File([blob], "speed-audio.webm", { type: blob.type || "audio/webm" }); @@ -398,7 +463,7 @@ export class AudioProcessor { try { await demuxer.load(file); - await this.processTrimOnlyAudio(demuxer, muxer, []); + await this.processTrimOnlyAudio(demuxer, muxer, [], undefined, exportCodec); } finally { try { demuxer.destroy(); diff --git a/src/lib/exporter/muxer.ts b/src/lib/exporter/muxer.ts index a2cc76d..95d41ad 100644 --- a/src/lib/exporter/muxer.ts +++ b/src/lib/exporter/muxer.ts @@ -8,6 +8,8 @@ import { } from "mediabunny"; import type { ExportConfig } from "./types"; +export type ExportAudioMuxerCodec = "aac" | "opus"; + export class VideoMuxer { private output: Output | null = null; private videoSource: EncodedVideoPacketSource | null = null; @@ -15,10 +17,12 @@ export class VideoMuxer { private hasAudio: boolean; private target: BufferTarget | null = null; private config: ExportConfig; + private audioCodec: ExportAudioMuxerCodec; - constructor(config: ExportConfig, hasAudio = false) { + constructor(config: ExportConfig, hasAudio = false, audioCodec: ExportAudioMuxerCodec = "aac") { this.config = config; this.hasAudio = hasAudio; + this.audioCodec = audioCodec; } async initialize(): Promise { @@ -40,7 +44,7 @@ export class VideoMuxer { // Create audio source if needed if (this.hasAudio) { - this.audioSource = new EncodedAudioPacketSource("aac"); + this.audioSource = new EncodedAudioPacketSource(this.audioCodec); this.output.addAudioTrack(this.audioSource); } diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index 19cd5a0..8da602b 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -178,8 +178,17 @@ export class VideoExporter { await this.initializeEncoder(encoderPreference); - const hasAudio = videoInfo.hasAudio; - const muxer = new VideoMuxer(this.config, hasAudio); + const sourceDemuxer = streamingDecoder.getDemuxer(); + const audioExportCodec = + videoInfo.hasAudio && sourceDemuxer + ? await AudioProcessor.selectSupportedExportCodecForSource(sourceDemuxer) + : null; + if (videoInfo.hasAudio && !audioExportCodec) { + console.warn("[VideoExporter] No supported audio export codec, exporting video-only."); + } + + const hasAudio = Boolean(audioExportCodec); + const muxer = new VideoMuxer(this.config, hasAudio, audioExportCodec?.muxerCodec); this.muxer = muxer; await muxer.initialize(); @@ -361,7 +370,7 @@ export class VideoExporter { phase: "finalizing", }); - if (hasAudio && !this.cancelled) { + if (hasAudio && audioExportCodec && !this.cancelled) { const demuxer = streamingDecoder.getDemuxer(); if (demuxer) { console.log("[VideoExporter] Processing audio track..."); @@ -373,6 +382,7 @@ export class VideoExporter { this.config.trimRegions, this.config.speedRegions, videoInfo.duration, + audioExportCodec, ); } }