From 588a0a7be825028e674c9ad674669341320ceb7e Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Tue, 5 May 2026 16:19:34 +0200 Subject: [PATCH] feat: add native Windows microphone capture --- .../windows-native-recorder-roadmap.md | 4 + electron/native/README.md | 6 +- electron/native/wgc-capture/CMakeLists.txt | 2 + .../wgc-capture/src/audio_sample_utils.cpp | 128 ++++++++++++++++++ .../wgc-capture/src/audio_sample_utils.h | 20 +++ electron/native/wgc-capture/src/main.cpp | 93 +++++++++++-- .../src/wasapi_loopback_capture.cpp | 33 ++++- .../wgc-capture/src/wasapi_loopback_capture.h | 10 +- package.json | 2 + scripts/test-windows-wgc-helper.mjs | 12 +- 10 files changed, 290 insertions(+), 20 deletions(-) create mode 100644 electron/native/wgc-capture/src/audio_sample_utils.cpp create mode 100644 electron/native/wgc-capture/src/audio_sample_utils.h diff --git a/docs/engineering/windows-native-recorder-roadmap.md b/docs/engineering/windows-native-recorder-roadmap.md index c6a0a06..2fb6ab5 100644 --- a/docs/engineering/windows-native-recorder-roadmap.md +++ b/docs/engineering/windows-native-recorder-roadmap.md @@ -138,6 +138,8 @@ SSOT rules for this phase: ### 3. WASAPI Microphone +Status: initial implementation in progress. The helper can open the default WASAPI capture endpoint, apply the OpenScreen microphone gain, encode mic-only audio, and mix mic into system-loopback packets when both endpoints expose the same runtime format. Browser `deviceId` to MMDevice id mapping, resampling between mismatched endpoint formats, and drift correction remain follow-up hardening work. + - Add microphone device enumeration and stable device-id mapping. - Capture selected/default microphone through WASAPI. - Apply OpenScreen's current mic gain policy. @@ -192,6 +194,8 @@ Acceptance: - `npm run test:wgc-webcam:win`: validates webcam output when a webcam is available, otherwise skips explicitly. - Packaging check: confirms the helper is in `app.asar.unpacked`. - Export check: exported MP4s generated from native recordings keep an AAC audio track when the source has audio. +- `npm run test:wgc-mic:win`: validates default-microphone capture writes an AAC track when an input endpoint is available. +- `npm run test:wgc-mixed-audio:win`: validates system loopback plus microphone writes one mixed AAC track when endpoint formats are compatible. ## Ship Criteria diff --git a/electron/native/README.md b/electron/native/README.md index 512517b..5df7290 100644 --- a/electron/native/README.md +++ b/electron/native/README.md @@ -32,6 +32,8 @@ Current V2 JSON shape: "fps": 60, "captureSystemAudio": false, "captureMic": false, + "microphoneDeviceId": "default", + "microphoneGain": 1.4, "webcamEnabled": false, "outputs": { "screenPath": "C:\\path\\recording-123.mp4", @@ -40,11 +42,13 @@ Current V2 JSON shape: } ``` -The current helper implementation supports display video capture and system audio loopback. Microphone, webcam, and window capture now fail explicitly in the helper rather than silently falling back to Electron capture on Windows. See `docs/engineering/windows-native-recorder-roadmap.md` for the phased implementation plan. +The current helper implementation supports display video capture, system audio loopback, and initial default-microphone capture. Webcam and window capture now fail explicitly in the helper rather than silently falling back to Electron capture on Windows. See `docs/engineering/windows-native-recorder-roadmap.md` for the phased implementation plan. Smoke-test the helper with: ```powershell npm run test:wgc-helper:win npm run test:wgc-audio:win +npm run test:wgc-mic:win +npm run test:wgc-mixed-audio:win ``` diff --git a/electron/native/wgc-capture/CMakeLists.txt b/electron/native/wgc-capture/CMakeLists.txt index 76999f7..b21fd66 100644 --- a/electron/native/wgc-capture/CMakeLists.txt +++ b/electron/native/wgc-capture/CMakeLists.txt @@ -14,6 +14,8 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) add_executable(wgc-capture + src/audio_sample_utils.cpp + src/audio_sample_utils.h src/main.cpp src/mf_encoder.cpp src/mf_encoder.h diff --git a/electron/native/wgc-capture/src/audio_sample_utils.cpp b/electron/native/wgc-capture/src/audio_sample_utils.cpp new file mode 100644 index 0000000..6537d8e --- /dev/null +++ b/electron/native/wgc-capture/src/audio_sample_utils.cpp @@ -0,0 +1,128 @@ +#include "audio_sample_utils.h" + +#include + +#include +#include +#include +#include + +namespace { + +bool isFloatFormat(const AudioInputFormat& format) { + return format.subtype == MFAudioFormat_Float && format.bitsPerSample == 32; +} + +bool isPcmFormat(const AudioInputFormat& format, UINT32 bitsPerSample) { + return format.subtype == MFAudioFormat_PCM && format.bitsPerSample == bitsPerSample; +} + +template +T clampTo(double value) { + const double minValue = static_cast(std::numeric_limits::min()); + const double maxValue = static_cast(std::numeric_limits::max()); + return static_cast(std::clamp(std::round(value), minValue, maxValue)); +} + +} // namespace + +bool sameAudioFormatForMixing(const AudioInputFormat& left, const AudioInputFormat& right) { + return left.subtype == right.subtype && + left.sampleRate == right.sampleRate && + left.channels == right.channels && + left.bitsPerSample == right.bitsPerSample && + left.blockAlign == right.blockAlign && + left.avgBytesPerSec == right.avgBytesPerSec; +} + +void copyAudioWithGain( + const BYTE* source, + DWORD byteCount, + const AudioInputFormat& format, + double gain, + std::vector& destination) { + destination.resize(byteCount); + if (!source || byteCount == 0) { + return; + } + + if (std::abs(gain - 1.0) < 0.0001) { + std::memcpy(destination.data(), source, byteCount); + return; + } + + if (isFloatFormat(format)) { + const auto* input = reinterpret_cast(source); + auto* output = reinterpret_cast(destination.data()); + const size_t sampleCount = byteCount / sizeof(float); + for (size_t index = 0; index < sampleCount; index += 1) { + output[index] = static_cast(std::clamp(input[index] * gain, -1.0, 1.0)); + } + return; + } + + if (isPcmFormat(format, 16)) { + const auto* input = reinterpret_cast(source); + auto* output = reinterpret_cast(destination.data()); + const size_t sampleCount = byteCount / sizeof(int16_t); + for (size_t index = 0; index < sampleCount; index += 1) { + output[index] = clampTo(static_cast(input[index]) * gain); + } + return; + } + + if (isPcmFormat(format, 32)) { + const auto* input = reinterpret_cast(source); + auto* output = reinterpret_cast(destination.data()); + const size_t sampleCount = byteCount / sizeof(int32_t); + for (size_t index = 0; index < sampleCount; index += 1) { + output[index] = clampTo(static_cast(input[index]) * gain); + } + return; + } + + std::memcpy(destination.data(), source, byteCount); +} + +void mixAudioInPlace( + std::vector& destination, + const BYTE* source, + DWORD byteCount, + const AudioInputFormat& format) { + if (!source || byteCount == 0 || destination.empty()) { + return; + } + + const size_t mixByteCount = std::min(destination.size(), static_cast(byteCount)); + + if (isFloatFormat(format)) { + auto* output = reinterpret_cast(destination.data()); + const auto* input = reinterpret_cast(source); + const size_t sampleCount = mixByteCount / sizeof(float); + for (size_t index = 0; index < sampleCount; index += 1) { + output[index] = static_cast(std::clamp(output[index] + input[index], -1.0f, 1.0f)); + } + return; + } + + if (isPcmFormat(format, 16)) { + auto* output = reinterpret_cast(destination.data()); + const auto* input = reinterpret_cast(source); + const size_t sampleCount = mixByteCount / sizeof(int16_t); + for (size_t index = 0; index < sampleCount; index += 1) { + output[index] = clampTo( + static_cast(output[index]) + static_cast(input[index])); + } + return; + } + + if (isPcmFormat(format, 32)) { + auto* output = reinterpret_cast(destination.data()); + const auto* input = reinterpret_cast(source); + const size_t sampleCount = mixByteCount / sizeof(int32_t); + for (size_t index = 0; index < sampleCount; index += 1) { + output[index] = clampTo( + static_cast(output[index]) + static_cast(input[index])); + } + } +} diff --git a/electron/native/wgc-capture/src/audio_sample_utils.h b/electron/native/wgc-capture/src/audio_sample_utils.h new file mode 100644 index 0000000..8022ae3 --- /dev/null +++ b/electron/native/wgc-capture/src/audio_sample_utils.h @@ -0,0 +1,20 @@ +#pragma once + +#include "mf_encoder.h" + +#include + +#include + +bool sameAudioFormatForMixing(const AudioInputFormat& left, const AudioInputFormat& right); +void copyAudioWithGain( + const BYTE* source, + DWORD byteCount, + const AudioInputFormat& format, + double gain, + std::vector& destination); +void mixAudioInPlace( + std::vector& destination, + const BYTE* source, + DWORD byteCount, + const AudioInputFormat& format); diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp index 39d5c62..603fda3 100644 --- a/electron/native/wgc-capture/src/main.cpp +++ b/electron/native/wgc-capture/src/main.cpp @@ -1,3 +1,4 @@ +#include "audio_sample_utils.h" #include "mf_encoder.h" #include "monitor_utils.h" #include "wasapi_loopback_capture.h" @@ -273,11 +274,6 @@ int main(int argc, char* argv[]) { return 1; } - if (config.captureMic) { - std::cerr << "ERROR: Microphone capture is not implemented in this helper yet" << std::endl; - return 1; - } - if (config.webcamEnabled) { std::cerr << "ERROR: Native webcam capture is not implemented in this helper yet" << std::endl; return 1; @@ -309,16 +305,34 @@ int main(int argc, char* argv[]) { const int bitrate = pixels >= 3840 * 2160 ? 45'000'000 : pixels >= 2560 * 1440 ? 28'000'000 : 18'000'000; WasapiLoopbackCapture loopbackCapture; + WasapiLoopbackCapture microphoneCapture; const AudioInputFormat* audioFormat = nullptr; if (config.captureSystemAudio) { - if (!loopbackCapture.initialize()) { + if (!loopbackCapture.initializeSystemLoopback()) { std::cerr << "ERROR: Failed to initialize WASAPI loopback capture" << std::endl; return 1; } audioFormat = &loopbackCapture.inputFormat(); - std::cout << "{\"event\":\"audio-format\",\"schemaVersion\":2,\"sampleRate\":" - << audioFormat->sampleRate << ",\"channels\":" << audioFormat->channels - << ",\"bitsPerSample\":" << audioFormat->bitsPerSample << "}" << std::endl; + } + if (config.captureMic) { + if (!microphoneCapture.initializeMicrophone(utf8ToWide(config.microphoneDeviceId))) { + std::cerr << "ERROR: Failed to initialize WASAPI microphone capture" << std::endl; + return 1; + } + if (!audioFormat) { + audioFormat = µphoneCapture.inputFormat(); + } else if (!sameAudioFormatForMixing(*audioFormat, microphoneCapture.inputFormat())) { + std::cerr << "ERROR: System audio and microphone formats differ; native mixing is not supported yet" + << std::endl; + return 1; + } + } + if (audioFormat) { + std::cout << "{\"event\":\"audio-format\",\"schemaVersion\":2,\"sampleRate\":" << audioFormat->sampleRate + << ",\"channels\":" << audioFormat->channels + << ",\"bitsPerSample\":" << audioFormat->bitsPerSample + << ",\"system\":" << (config.captureSystemAudio ? "true" : "false") + << ",\"microphone\":" << (config.captureMic ? "true" : "false") << "}" << std::endl; } MFEncoder encoder; @@ -358,24 +372,81 @@ int main(int argc, char* argv[]) { } }); + std::mutex microphoneAudioMutex; + std::vector latestMicrophoneAudio; + std::vector mixedAudioBuffer; + std::vector microphoneGainBuffer; + + if (config.captureMic) { + if (!microphoneCapture.start([&](const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns) { + if (stopRequested || !audioFormat) { + return; + } + + copyAudioWithGain( + data, + byteCount, + microphoneCapture.inputFormat(), + config.microphoneGain, + microphoneGainBuffer); + + if (config.captureSystemAudio) { + std::scoped_lock lock(microphoneAudioMutex); + latestMicrophoneAudio = microphoneGainBuffer; + return; + } + + if (!encoder.writeAudio( + microphoneGainBuffer.data(), + static_cast(microphoneGainBuffer.size()), + timestampHns, + durationHns)) { + encodeFailed = true; + stopRequested = true; + cv.notify_all(); + } + })) { + std::cerr << "ERROR: Failed to start WASAPI microphone capture" << std::endl; + return 1; + } + } + if (config.captureSystemAudio) { if (!loopbackCapture.start([&](const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns) { if (stopRequested) { return; } - if (!encoder.writeAudio(data, byteCount, timestampHns, durationHns)) { + const BYTE* encodedData = data; + DWORD encodedByteCount = byteCount; + if (config.captureMic && audioFormat) { + mixedAudioBuffer.assign(data, data + byteCount); + { + std::scoped_lock lock(microphoneAudioMutex); + mixAudioInPlace( + mixedAudioBuffer, + latestMicrophoneAudio.data(), + static_cast(latestMicrophoneAudio.size()), + *audioFormat); + } + encodedData = mixedAudioBuffer.data(); + encodedByteCount = static_cast(mixedAudioBuffer.size()); + } + + if (!encoder.writeAudio(encodedData, encodedByteCount, timestampHns, durationHns)) { encodeFailed = true; stopRequested = true; cv.notify_all(); } })) { std::cerr << "ERROR: Failed to start WASAPI loopback capture" << std::endl; + microphoneCapture.stop(); return 1; } } if (!session.start()) { + microphoneCapture.stop(); loopbackCapture.stop(); std::cerr << "ERROR: Failed to start WGC session" << std::endl; return 1; @@ -394,6 +465,7 @@ int main(int argc, char* argv[]) { if (stdinThread.joinable()) { stdinThread.detach(); } + microphoneCapture.stop(); loopbackCapture.stop(); std::cerr << "ERROR: Timed out waiting for first WGC frame" << std::endl; return 1; @@ -410,6 +482,7 @@ int main(int argc, char* argv[]) { }); } + microphoneCapture.stop(); loopbackCapture.stop(); session.stop(); { diff --git a/electron/native/wgc-capture/src/wasapi_loopback_capture.cpp b/electron/native/wgc-capture/src/wasapi_loopback_capture.cpp index e4f254e..4e350a2 100644 --- a/electron/native/wgc-capture/src/wasapi_loopback_capture.cpp +++ b/electron/native/wgc-capture/src/wasapi_loopback_capture.cpp @@ -51,7 +51,15 @@ WasapiLoopbackCapture::~WasapiLoopbackCapture() { } } -bool WasapiLoopbackCapture::initialize() { +bool WasapiLoopbackCapture::initializeSystemLoopback() { + return initialize(WasapiCaptureEndpoint::SystemLoopback, {}); +} + +bool WasapiLoopbackCapture::initializeMicrophone(const std::wstring& deviceId) { + return initialize(WasapiCaptureEndpoint::Microphone, deviceId); +} + +bool WasapiLoopbackCapture::initialize(WasapiCaptureEndpoint endpoint, const std::wstring& deviceId) { HRESULT hr = CoCreateInstance( __uuidof(MMDeviceEnumerator), nullptr, @@ -61,9 +69,22 @@ bool WasapiLoopbackCapture::initialize() { return false; } - hr = deviceEnumerator_->GetDefaultAudioEndpoint(eRender, eConsole, &device_); - if (!succeeded(hr, "GetDefaultAudioEndpoint(render)")) { - return false; + if (endpoint == WasapiCaptureEndpoint::Microphone && !deviceId.empty() && deviceId != L"default") { + hr = deviceEnumerator_->GetDevice(deviceId.c_str(), &device_); + if (FAILED(hr)) { + std::wcerr << L"WARNING: Could not resolve microphone device id; using default capture endpoint" + << std::endl; + device_.Reset(); + } + } + + if (!device_) { + const EDataFlow flow = + endpoint == WasapiCaptureEndpoint::SystemLoopback ? eRender : eCapture; + hr = deviceEnumerator_->GetDefaultAudioEndpoint(flow, eConsole, &device_); + if (!succeeded(hr, "GetDefaultAudioEndpoint")) { + return false; + } } hr = device_->Activate(__uuidof(IAudioClient), CLSCTX_ALL, nullptr, &audioClient_); @@ -81,9 +102,11 @@ bool WasapiLoopbackCapture::initialize() { return false; } + const DWORD streamFlags = + endpoint == WasapiCaptureEndpoint::SystemLoopback ? AUDCLNT_STREAMFLAGS_LOOPBACK : 0; hr = audioClient_->Initialize( AUDCLNT_SHAREMODE_SHARED, - AUDCLNT_STREAMFLAGS_LOOPBACK, + streamFlags, BufferDurationHns, 0, mixFormat_, diff --git a/electron/native/wgc-capture/src/wasapi_loopback_capture.h b/electron/native/wgc-capture/src/wasapi_loopback_capture.h index e6fb7e8..8d2dbb9 100644 --- a/electron/native/wgc-capture/src/wasapi_loopback_capture.h +++ b/electron/native/wgc-capture/src/wasapi_loopback_capture.h @@ -10,9 +10,15 @@ #include #include #include +#include #include #include +enum class WasapiCaptureEndpoint { + SystemLoopback, + Microphone, +}; + class WasapiLoopbackCapture { public: using AudioCallback = std::function; @@ -23,13 +29,15 @@ public: WasapiLoopbackCapture(const WasapiLoopbackCapture&) = delete; WasapiLoopbackCapture& operator=(const WasapiLoopbackCapture&) = delete; - bool initialize(); + bool initializeSystemLoopback(); + bool initializeMicrophone(const std::wstring& deviceId); bool start(AudioCallback callback); void stop(); const AudioInputFormat& inputFormat() const; private: + bool initialize(WasapiCaptureEndpoint endpoint, const std::wstring& deviceId); void captureLoop(); bool resolveInputFormat(WAVEFORMATEX* mixFormat); diff --git a/package.json b/package.json index 4311f6e..9114207 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,8 @@ "test:cursor-native:win": "node scripts/test-windows-native-cursor.mjs", "test:wgc-helper:win": "node scripts/test-windows-wgc-helper.mjs", "test:wgc-audio:win": "node scripts/test-windows-wgc-helper.mjs --system-audio", + "test:wgc-mic:win": "node scripts/test-windows-wgc-helper.mjs --microphone", + "test:wgc-mixed-audio:win": "node scripts/test-windows-wgc-helper.mjs --system-audio --microphone", "capture:openscreen-preview": "node scripts/capture-openscreen-preview.mjs", "build-vite": "tsc && vite build", "test:browser": "vitest --config vitest.browser.config.ts --run", diff --git a/scripts/test-windows-wgc-helper.mjs b/scripts/test-windows-wgc-helper.mjs index 627c0ca..45dab7d 100644 --- a/scripts/test-windows-wgc-helper.mjs +++ b/scripts/test-windows-wgc-helper.mjs @@ -14,6 +14,10 @@ const DURATION_MS = Number(process.env.OPENSCREEN_WGC_TEST_DURATION_MS ?? 5000); const WITH_SYSTEM_AUDIO = process.env.OPENSCREEN_WGC_TEST_SYSTEM_AUDIO === "true" || process.argv.includes("--system-audio"); +const WITH_MICROPHONE = + process.env.OPENSCREEN_WGC_TEST_MICROPHONE === "true" || + process.argv.includes("--microphone") || + process.argv.includes("--mic"); function runHelper(config) { return new Promise((resolve, reject) => { @@ -101,7 +105,7 @@ if (!fs.existsSync(HELPER_PATH)) { const outputPath = path.join( os.tmpdir(), - `openscreen-wgc-helper-${WITH_SYSTEM_AUDIO ? "audio" : "video"}-${Date.now()}.mp4`, + `openscreen-wgc-helper-${WITH_SYSTEM_AUDIO || WITH_MICROPHONE ? "audio" : "video"}-${Date.now()}.mp4`, ); const config = { @@ -120,7 +124,9 @@ const config = { displayH: 1080, hasDisplayBounds: true, captureSystemAudio: WITH_SYSTEM_AUDIO, - captureMic: false, + captureMic: WITH_MICROPHONE, + microphoneDeviceId: "default", + microphoneGain: 1.4, webcamEnabled: false, outputs: { screenPath: outputPath }, }; @@ -139,7 +145,7 @@ const hasAudio = streams.some((stream) => stream.codec_type === "audio"); if (!hasVideo) { throw new Error(`WGC helper output has no video stream: ${outputPath}`); } -if (WITH_SYSTEM_AUDIO && !hasAudio) { +if ((WITH_SYSTEM_AUDIO || WITH_MICROPHONE) && !hasAudio) { throw new Error(`WGC helper output has no audio stream: ${outputPath}`); } const frameLuma = measureFirstFrameLuma(outputPath);