feat: add native Windows microphone capture

docs/engineering/windows-native-recorder-roadmap.md

@@ -138,6 +138,8 @@ SSOT rules for this phase:
 
 ### 3. WASAPI Microphone
 
+Status: initial implementation in progress. The helper can open the default WASAPI capture endpoint, apply the OpenScreen microphone gain, encode mic-only audio, and mix mic into system-loopback packets when both endpoints expose the same runtime format. Browser `deviceId` to MMDevice id mapping, resampling between mismatched endpoint formats, and drift correction remain follow-up hardening work.
+
 - Add microphone device enumeration and stable device-id mapping.
 - Capture selected/default microphone through WASAPI.
 - Apply OpenScreen's current mic gain policy.
@@ -192,6 +194,8 @@ Acceptance:
 - `npm run test:wgc-webcam:win`: validates webcam output when a webcam is available, otherwise skips explicitly.
 - Packaging check: confirms the helper is in `app.asar.unpacked`.
 - Export check: exported MP4s generated from native recordings keep an AAC audio track when the source has audio.
+- `npm run test:wgc-mic:win`: validates default-microphone capture writes an AAC track when an input endpoint is available.
+- `npm run test:wgc-mixed-audio:win`: validates system loopback plus microphone writes one mixed AAC track when endpoint formats are compatible.
 
 ## Ship Criteria
 

electron/native/README.md

@@ -32,6 +32,8 @@ Current V2 JSON shape:
   "fps": 60,
   "captureSystemAudio": false,
   "captureMic": false,
+  "microphoneDeviceId": "default",
+  "microphoneGain": 1.4,
   "webcamEnabled": false,
   "outputs": {
     "screenPath": "C:\\path\\recording-123.mp4",
@@ -40,11 +42,13 @@ Current V2 JSON shape:
 }
 ```
 
-The current helper implementation supports display video capture and system audio loopback. Microphone, webcam, and window capture now fail explicitly in the helper rather than silently falling back to Electron capture on Windows. See `docs/engineering/windows-native-recorder-roadmap.md` for the phased implementation plan.
+The current helper implementation supports display video capture, system audio loopback, and initial default-microphone capture. Webcam and window capture now fail explicitly in the helper rather than silently falling back to Electron capture on Windows. See `docs/engineering/windows-native-recorder-roadmap.md` for the phased implementation plan.
 
 Smoke-test the helper with:
 
 ```powershell
 npm run test:wgc-helper:win
 npm run test:wgc-audio:win
+npm run test:wgc-mic:win
+npm run test:wgc-mixed-audio:win
 ```

electron/native/wgc-capture/CMakeLists.txt

@@ -14,6 +14,8 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 
 add_executable(wgc-capture
+  src/audio_sample_utils.cpp
+  src/audio_sample_utils.h
   src/main.cpp
   src/mf_encoder.cpp
   src/mf_encoder.h

electron/native/wgc-capture/src/audio_sample_utils.cpp

@@ -0,0 +1,128 @@
+#include "audio_sample_utils.h"
+
+#include <mfapi.h>
+
+#include <algorithm>
+#include <cmath>
+#include <cstring>
+#include <limits>
+
+namespace {
+
+bool isFloatFormat(const AudioInputFormat& format) {
+    return format.subtype == MFAudioFormat_Float && format.bitsPerSample == 32;
+}
+
+bool isPcmFormat(const AudioInputFormat& format, UINT32 bitsPerSample) {
+    return format.subtype == MFAudioFormat_PCM && format.bitsPerSample == bitsPerSample;
+}
+
+template <typename T>
+T clampTo(double value) {
+    const double minValue = static_cast<double>(std::numeric_limits<T>::min());
+    const double maxValue = static_cast<double>(std::numeric_limits<T>::max());
+    return static_cast<T>(std::clamp(std::round(value), minValue, maxValue));
+}
+
+} // namespace
+
+bool sameAudioFormatForMixing(const AudioInputFormat& left, const AudioInputFormat& right) {
+    return left.subtype == right.subtype &&
+           left.sampleRate == right.sampleRate &&
+           left.channels == right.channels &&
+           left.bitsPerSample == right.bitsPerSample &&
+           left.blockAlign == right.blockAlign &&
+           left.avgBytesPerSec == right.avgBytesPerSec;
+}
+
+void copyAudioWithGain(
+    const BYTE* source,
+    DWORD byteCount,
+    const AudioInputFormat& format,
+    double gain,
+    std::vector<BYTE>& destination) {
+    destination.resize(byteCount);
+    if (!source || byteCount == 0) {
+        return;
+    }
+
+    if (std::abs(gain - 1.0) < 0.0001) {
+        std::memcpy(destination.data(), source, byteCount);
+        return;
+    }
+
+    if (isFloatFormat(format)) {
+        const auto* input = reinterpret_cast<const float*>(source);
+        auto* output = reinterpret_cast<float*>(destination.data());
+        const size_t sampleCount = byteCount / sizeof(float);
+        for (size_t index = 0; index < sampleCount; index += 1) {
+            output[index] = static_cast<float>(std::clamp(input[index] * gain, -1.0, 1.0));
+        }
+        return;
+    }
+
+    if (isPcmFormat(format, 16)) {
+        const auto* input = reinterpret_cast<const int16_t*>(source);
+        auto* output = reinterpret_cast<int16_t*>(destination.data());
+        const size_t sampleCount = byteCount / sizeof(int16_t);
+        for (size_t index = 0; index < sampleCount; index += 1) {
+            output[index] = clampTo<int16_t>(static_cast<double>(input[index]) * gain);
+        }
+        return;
+    }
+
+    if (isPcmFormat(format, 32)) {
+        const auto* input = reinterpret_cast<const int32_t*>(source);
+        auto* output = reinterpret_cast<int32_t*>(destination.data());
+        const size_t sampleCount = byteCount / sizeof(int32_t);
+        for (size_t index = 0; index < sampleCount; index += 1) {
+            output[index] = clampTo<int32_t>(static_cast<double>(input[index]) * gain);
+        }
+        return;
+    }
+
+    std::memcpy(destination.data(), source, byteCount);
+}
+
+void mixAudioInPlace(
+    std::vector<BYTE>& destination,
+    const BYTE* source,
+    DWORD byteCount,
+    const AudioInputFormat& format) {
+    if (!source || byteCount == 0 || destination.empty()) {
+        return;
+    }
+
+    const size_t mixByteCount = std::min(destination.size(), static_cast<size_t>(byteCount));
+
+    if (isFloatFormat(format)) {
+        auto* output = reinterpret_cast<float*>(destination.data());
+        const auto* input = reinterpret_cast<const float*>(source);
+        const size_t sampleCount = mixByteCount / sizeof(float);
+        for (size_t index = 0; index < sampleCount; index += 1) {
+            output[index] = static_cast<float>(std::clamp(output[index] + input[index], -1.0f, 1.0f));
+        }
+        return;
+    }
+
+    if (isPcmFormat(format, 16)) {
+        auto* output = reinterpret_cast<int16_t*>(destination.data());
+        const auto* input = reinterpret_cast<const int16_t*>(source);
+        const size_t sampleCount = mixByteCount / sizeof(int16_t);
+        for (size_t index = 0; index < sampleCount; index += 1) {
+            output[index] = clampTo<int16_t>(
+                static_cast<double>(output[index]) + static_cast<double>(input[index]));
+        }
+        return;
+    }
+
+    if (isPcmFormat(format, 32)) {
+        auto* output = reinterpret_cast<int32_t*>(destination.data());
+        const auto* input = reinterpret_cast<const int32_t*>(source);
+        const size_t sampleCount = mixByteCount / sizeof(int32_t);
+        for (size_t index = 0; index < sampleCount; index += 1) {
+            output[index] = clampTo<int32_t>(
+                static_cast<double>(output[index]) + static_cast<double>(input[index]));
+        }
+    }
+}

electron/native/wgc-capture/src/audio_sample_utils.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include "mf_encoder.h"
+
+#include <Windows.h>
+
+#include <vector>
+
+bool sameAudioFormatForMixing(const AudioInputFormat& left, const AudioInputFormat& right);
+void copyAudioWithGain(
+    const BYTE* source,
+    DWORD byteCount,
+    const AudioInputFormat& format,
+    double gain,
+    std::vector<BYTE>& destination);
+void mixAudioInPlace(
+    std::vector<BYTE>& destination,
+    const BYTE* source,
+    DWORD byteCount,
+    const AudioInputFormat& format);

electron/native/wgc-capture/src/main.cpp

@@ -1,3 +1,4 @@
+#include "audio_sample_utils.h"
 #include "mf_encoder.h"
 #include "monitor_utils.h"
 #include "wasapi_loopback_capture.h"
@@ -273,11 +274,6 @@ int main(int argc, char* argv[]) {
         return 1;
     }
 
-    if (config.captureMic) {
-        std::cerr << "ERROR: Microphone capture is not implemented in this helper yet" << std::endl;
-        return 1;
-    }
-
     if (config.webcamEnabled) {
         std::cerr << "ERROR: Native webcam capture is not implemented in this helper yet" << std::endl;
         return 1;
@@ -309,16 +305,34 @@ int main(int argc, char* argv[]) {
     const int bitrate = pixels >= 3840 * 2160 ? 45'000'000 : pixels >= 2560 * 1440 ? 28'000'000 : 18'000'000;
 
     WasapiLoopbackCapture loopbackCapture;
+    WasapiLoopbackCapture microphoneCapture;
     const AudioInputFormat* audioFormat = nullptr;
     if (config.captureSystemAudio) {
-        if (!loopbackCapture.initialize()) {
+        if (!loopbackCapture.initializeSystemLoopback()) {
             std::cerr << "ERROR: Failed to initialize WASAPI loopback capture" << std::endl;
             return 1;
         }
         audioFormat = &loopbackCapture.inputFormat();
-        std::cout << "{\"event\":\"audio-format\",\"schemaVersion\":2,\"sampleRate\":"
-                  << audioFormat->sampleRate << ",\"channels\":" << audioFormat->channels
-                  << ",\"bitsPerSample\":" << audioFormat->bitsPerSample << "}" << std::endl;
+    }
+    if (config.captureMic) {
+        if (!microphoneCapture.initializeMicrophone(utf8ToWide(config.microphoneDeviceId))) {
+            std::cerr << "ERROR: Failed to initialize WASAPI microphone capture" << std::endl;
+            return 1;
+        }
+        if (!audioFormat) {
+            audioFormat = &microphoneCapture.inputFormat();
+        } else if (!sameAudioFormatForMixing(*audioFormat, microphoneCapture.inputFormat())) {
+            std::cerr << "ERROR: System audio and microphone formats differ; native mixing is not supported yet"
+                      << std::endl;
+            return 1;
+        }
+    }
+    if (audioFormat) {
+        std::cout << "{\"event\":\"audio-format\",\"schemaVersion\":2,\"sampleRate\":" << audioFormat->sampleRate
+                  << ",\"channels\":" << audioFormat->channels
+                  << ",\"bitsPerSample\":" << audioFormat->bitsPerSample
+                  << ",\"system\":" << (config.captureSystemAudio ? "true" : "false")
+                  << ",\"microphone\":" << (config.captureMic ? "true" : "false") << "}" << std::endl;
     }
 
     MFEncoder encoder;
@@ -358,24 +372,81 @@ int main(int argc, char* argv[]) {
         }
     });
 
+    std::mutex microphoneAudioMutex;
+    std::vector<BYTE> latestMicrophoneAudio;
+    std::vector<BYTE> mixedAudioBuffer;
+    std::vector<BYTE> microphoneGainBuffer;
+
+    if (config.captureMic) {
+        if (!microphoneCapture.start([&](const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns) {
+                if (stopRequested || !audioFormat) {
+                    return;
+                }
+
+                copyAudioWithGain(
+                    data,
+                    byteCount,
+                    microphoneCapture.inputFormat(),
+                    config.microphoneGain,
+                    microphoneGainBuffer);
+
+                if (config.captureSystemAudio) {
+                    std::scoped_lock lock(microphoneAudioMutex);
+                    latestMicrophoneAudio = microphoneGainBuffer;
+                    return;
+                }
+
+                if (!encoder.writeAudio(
+                        microphoneGainBuffer.data(),
+                        static_cast<DWORD>(microphoneGainBuffer.size()),
+                        timestampHns,
+                        durationHns)) {
+                    encodeFailed = true;
+                    stopRequested = true;
+                    cv.notify_all();
+                }
+            })) {
+            std::cerr << "ERROR: Failed to start WASAPI microphone capture" << std::endl;
+            return 1;
+        }
+    }
+
     if (config.captureSystemAudio) {
         if (!loopbackCapture.start([&](const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns) {
                 if (stopRequested) {
                     return;
                 }
 
-                if (!encoder.writeAudio(data, byteCount, timestampHns, durationHns)) {
+                const BYTE* encodedData = data;
+                DWORD encodedByteCount = byteCount;
+                if (config.captureMic && audioFormat) {
+                    mixedAudioBuffer.assign(data, data + byteCount);
+                    {
+                        std::scoped_lock lock(microphoneAudioMutex);
+                        mixAudioInPlace(
+                            mixedAudioBuffer,
+                            latestMicrophoneAudio.data(),
+                            static_cast<DWORD>(latestMicrophoneAudio.size()),
+                            *audioFormat);
+                    }
+                    encodedData = mixedAudioBuffer.data();
+                    encodedByteCount = static_cast<DWORD>(mixedAudioBuffer.size());
+                }
+
+                if (!encoder.writeAudio(encodedData, encodedByteCount, timestampHns, durationHns)) {
                     encodeFailed = true;
                     stopRequested = true;
                     cv.notify_all();
                 }
             })) {
             std::cerr << "ERROR: Failed to start WASAPI loopback capture" << std::endl;
+            microphoneCapture.stop();
             return 1;
         }
     }
 
     if (!session.start()) {
+        microphoneCapture.stop();
         loopbackCapture.stop();
         std::cerr << "ERROR: Failed to start WGC session" << std::endl;
         return 1;
@@ -394,6 +465,7 @@ int main(int argc, char* argv[]) {
             if (stdinThread.joinable()) {
                 stdinThread.detach();
             }
+            microphoneCapture.stop();
             loopbackCapture.stop();
             std::cerr << "ERROR: Timed out waiting for first WGC frame" << std::endl;
             return 1;
@@ -410,6 +482,7 @@ int main(int argc, char* argv[]) {
         });
     }
 
+    microphoneCapture.stop();
     loopbackCapture.stop();
     session.stop();
     {

electron/native/wgc-capture/src/wasapi_loopback_capture.cpp

@@ -51,7 +51,15 @@ WasapiLoopbackCapture::~WasapiLoopbackCapture() {
     }
 }
 
-bool WasapiLoopbackCapture::initialize() {
+bool WasapiLoopbackCapture::initializeSystemLoopback() {
+    return initialize(WasapiCaptureEndpoint::SystemLoopback, {});
+}
+
+bool WasapiLoopbackCapture::initializeMicrophone(const std::wstring& deviceId) {
+    return initialize(WasapiCaptureEndpoint::Microphone, deviceId);
+}
+
+bool WasapiLoopbackCapture::initialize(WasapiCaptureEndpoint endpoint, const std::wstring& deviceId) {
     HRESULT hr = CoCreateInstance(
         __uuidof(MMDeviceEnumerator),
         nullptr,
@@ -61,9 +69,22 @@ bool WasapiLoopbackCapture::initialize() {
         return false;
     }
 
-    hr = deviceEnumerator_->GetDefaultAudioEndpoint(eRender, eConsole, &device_);
-    if (!succeeded(hr, "GetDefaultAudioEndpoint(render)")) {
-        return false;
+    if (endpoint == WasapiCaptureEndpoint::Microphone && !deviceId.empty() && deviceId != L"default") {
+        hr = deviceEnumerator_->GetDevice(deviceId.c_str(), &device_);
+        if (FAILED(hr)) {
+            std::wcerr << L"WARNING: Could not resolve microphone device id; using default capture endpoint"
+                       << std::endl;
+            device_.Reset();
+        }
+    }
+
+    if (!device_) {
+        const EDataFlow flow =
+            endpoint == WasapiCaptureEndpoint::SystemLoopback ? eRender : eCapture;
+        hr = deviceEnumerator_->GetDefaultAudioEndpoint(flow, eConsole, &device_);
+        if (!succeeded(hr, "GetDefaultAudioEndpoint")) {
+            return false;
+        }
     }
 
     hr = device_->Activate(__uuidof(IAudioClient), CLSCTX_ALL, nullptr, &audioClient_);
@@ -81,9 +102,11 @@ bool WasapiLoopbackCapture::initialize() {
         return false;
     }
 
+    const DWORD streamFlags =
+        endpoint == WasapiCaptureEndpoint::SystemLoopback ? AUDCLNT_STREAMFLAGS_LOOPBACK : 0;
     hr = audioClient_->Initialize(
         AUDCLNT_SHAREMODE_SHARED,
-        AUDCLNT_STREAMFLAGS_LOOPBACK,
+        streamFlags,
         BufferDurationHns,
         0,
         mixFormat_,

electron/native/wgc-capture/src/wasapi_loopback_capture.h

@@ -10,9 +10,15 @@
 #include <atomic>
 #include <cstdint>
 #include <functional>
+#include <string>
 #include <thread>
 #include <vector>
 
+enum class WasapiCaptureEndpoint {
+    SystemLoopback,
+    Microphone,
+};
+
 class WasapiLoopbackCapture {
 public:
     using AudioCallback = std::function<void(const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns)>;
@@ -23,13 +29,15 @@ public:
     WasapiLoopbackCapture(const WasapiLoopbackCapture&) = delete;
     WasapiLoopbackCapture& operator=(const WasapiLoopbackCapture&) = delete;
 
-    bool initialize();
+    bool initializeSystemLoopback();
+    bool initializeMicrophone(const std::wstring& deviceId);
     bool start(AudioCallback callback);
     void stop();
 
     const AudioInputFormat& inputFormat() const;
 
 private:
+    bool initialize(WasapiCaptureEndpoint endpoint, const std::wstring& deviceId);
     void captureLoop();
     bool resolveInputFormat(WAVEFORMATEX* mixFormat);
 

package.json

@@ -29,6 +29,8 @@
 		"test:cursor-native:win": "node scripts/test-windows-native-cursor.mjs",
 		"test:wgc-helper:win": "node scripts/test-windows-wgc-helper.mjs",
 		"test:wgc-audio:win": "node scripts/test-windows-wgc-helper.mjs --system-audio",
+		"test:wgc-mic:win": "node scripts/test-windows-wgc-helper.mjs --microphone",
+		"test:wgc-mixed-audio:win": "node scripts/test-windows-wgc-helper.mjs --system-audio --microphone",
 		"capture:openscreen-preview": "node scripts/capture-openscreen-preview.mjs",
 		"build-vite": "tsc && vite build",
 		"test:browser": "vitest --config vitest.browser.config.ts --run",

scripts/test-windows-wgc-helper.mjs

@@ -14,6 +14,10 @@ const DURATION_MS = Number(process.env.OPENSCREEN_WGC_TEST_DURATION_MS ?? 5000);
 const WITH_SYSTEM_AUDIO =
 	process.env.OPENSCREEN_WGC_TEST_SYSTEM_AUDIO === "true" ||
 	process.argv.includes("--system-audio");
+const WITH_MICROPHONE =
+	process.env.OPENSCREEN_WGC_TEST_MICROPHONE === "true" ||
+	process.argv.includes("--microphone") ||
+	process.argv.includes("--mic");
 
 function runHelper(config) {
 	return new Promise((resolve, reject) => {
@@ -101,7 +105,7 @@ if (!fs.existsSync(HELPER_PATH)) {
 
 const outputPath = path.join(
 	os.tmpdir(),
-	`openscreen-wgc-helper-${WITH_SYSTEM_AUDIO ? "audio" : "video"}-${Date.now()}.mp4`,
+	`openscreen-wgc-helper-${WITH_SYSTEM_AUDIO || WITH_MICROPHONE ? "audio" : "video"}-${Date.now()}.mp4`,
 );
 
 const config = {
@@ -120,7 +124,9 @@ const config = {
 	displayH: 1080,
 	hasDisplayBounds: true,
 	captureSystemAudio: WITH_SYSTEM_AUDIO,
-	captureMic: false,
+	captureMic: WITH_MICROPHONE,
+	microphoneDeviceId: "default",
+	microphoneGain: 1.4,
 	webcamEnabled: false,
 	outputs: { screenPath: outputPath },
 };
@@ -139,7 +145,7 @@ const hasAudio = streams.some((stream) => stream.codec_type === "audio");
 if (!hasVideo) {
 	throw new Error(`WGC helper output has no video stream: ${outputPath}`);
 }
-if (WITH_SYSTEM_AUDIO && !hasAudio) {
+if ((WITH_SYSTEM_AUDIO || WITH_MICROPHONE) && !hasAudio) {
 	throw new Error(`WGC helper output has no audio stream: ${outputPath}`);
 }
 const frameLuma = measureFirstFrameLuma(outputPath);