Merge pull request #597 from EtienneLescot/codex/fix-editable-cursor-native

Fix editable cursor mode for native Windows capture

docs/testing/windows-native-cursor.md

@@ -111,6 +111,7 @@ Smoke-test the helper directly:
 
 ```powershell
 npm run test:wgc-helper:win
+npm run test:wgc-helper:win -- --capture-cursor
 npm run test:wgc-window:win
 npm run test:wgc-audio:win
 npm run test:wgc-mic:win

electron/main.ts

@@ -13,7 +13,7 @@ import {
 	Tray,
 } from "electron";
 import { mainT, setMainLocale } from "./i18n";
-import { registerIpcHandlers } from "./ipc/handlers";
+import { getSelectedDesktopSource, registerIpcHandlers } from "./ipc/handlers";
 import {
 	createCountdownOverlayWindow,
 	createEditorWindow,
@@ -477,6 +477,22 @@ app.whenReady().then(async () => {
 		callback(allowed.includes(permission));
 	});
 
+	session.defaultSession.setDisplayMediaRequestHandler(
+		(request, callback) => {
+			const source = getSelectedDesktopSource();
+			if (!request.videoRequested || !source) {
+				callback({});
+				return;
+			}
+
+			callback({
+				video: source,
+				...(request.audioRequested && process.platform === "win32" ? { audio: "loopback" } : {}),
+			});
+		},
+		{ useSystemPicker: false },
+	);
+
 	// Request microphone and screen recording permissions from macOS
 	if (process.platform === "darwin") {
 		const micStatus = systemPreferences.getMediaAccessStatus("microphone");

electron/native/wgc-capture/src/wgc_session.cpp

@@ -140,11 +140,41 @@ bool WgcSession::createCaptureItem(HWND window) {
     return width_ > 0 && height_ > 0;
 }
 
-void WgcSession::applySessionOptions(bool captureCursor) {
+bool WgcSession::applySessionOptions(bool captureCursor) {
+    captureCursor_ = captureCursor;
+
     try {
-        session_.IsCursorCaptureEnabled(captureCursor);
+        auto session2 = session_.try_as<wgcap::IGraphicsCaptureSession2>();
+        if (!session2) {
+            if (!captureCursor) {
+                std::cerr << "ERROR: WGC cursor suppression is not supported by this Windows runtime"
+                          << std::endl;
+                return false;
+            }
+        } else {
+            session2.IsCursorCaptureEnabled(captureCursor);
+            const bool appliedCursorCapture = session2.IsCursorCaptureEnabled();
+            std::cout << "{\"event\":\"cursor-capture\",\"schemaVersion\":2,\"requested\":"
+                      << (captureCursor ? "true" : "false")
+                      << ",\"applied\":" << (appliedCursorCapture ? "true" : "false") << "}"
+                      << std::endl;
+
+            if (appliedCursorCapture != captureCursor) {
+                std::cerr << "ERROR: WGC cursor capture setting did not apply" << std::endl;
+                return false;
+            }
+        }
+    } catch (winrt::hresult_error const& error) {
+        std::cerr << "ERROR: Failed to configure WGC cursor capture (hr=0x" << std::hex
+                  << static_cast<uint32_t>(error.code()) << std::dec << ")" << std::endl;
+        if (!captureCursor) {
+            return false;
+        }
     } catch (...) {
-        // Older WGC builds can omit this property. They will keep the OS default.
+        std::cerr << "ERROR: Failed to configure WGC cursor capture" << std::endl;
+        if (!captureCursor) {
+            return false;
+        }
     }
 
     try {
@@ -152,6 +182,8 @@ void WgcSession::applySessionOptions(bool captureCursor) {
     } catch (...) {
         // IsBorderRequired is Windows 11-only. Ignore it on older builds.
     }
+
+    return true;
 }
 
 bool WgcSession::initialize(HMONITOR monitor, int fps, bool captureCursor) {
@@ -170,7 +202,9 @@ bool WgcSession::initialize(HMONITOR monitor, int fps, bool captureCursor) {
         item_.Size());
     session_ = framePool_.CreateCaptureSession(item_);
 
-    applySessionOptions(captureCursor);
+    if (!applySessionOptions(captureCursor)) {
+        return false;
+    }
 
     frameArrivedToken_ = framePool_.FrameArrived({this, &WgcSession::onFrameArrived});
     return true;
@@ -192,7 +226,9 @@ bool WgcSession::initialize(HWND window, int fps, bool captureCursor) {
         item_.Size());
     session_ = framePool_.CreateCaptureSession(item_);
 
-    applySessionOptions(captureCursor);
+    if (!applySessionOptions(captureCursor)) {
+        return false;
+    }
 
     frameArrivedToken_ = framePool_.FrameArrived({this, &WgcSession::onFrameArrived});
     return true;
@@ -207,6 +243,9 @@ bool WgcSession::start() {
     if (!session_) {
         return false;
     }
+    if (!applySessionOptions(captureCursor_)) {
+        return false;
+    }
     session_.StartCapture();
     started_ = true;
     return true;

electron/native/wgc-capture/src/wgc_session.h

@@ -37,7 +37,7 @@ private:
     bool createD3DDevice();
     bool createCaptureItem(HMONITOR monitor);
     bool createCaptureItem(HWND window);
-    void applySessionOptions(bool captureCursor);
+    bool applySessionOptions(bool captureCursor);
     void onFrameArrived(
         winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool const& sender,
         winrt::Windows::Foundation::IInspectable const&);
@@ -54,5 +54,6 @@ private:
     int width_ = 0;
     int height_ = 0;
     int fps_ = 60;
+    bool captureCursor_ = false;
     bool started_ = false;
 };

scripts/test-windows-wgc-helper.mjs

@@ -23,6 +23,9 @@ const WITH_WINDOW =
 	process.env.OPENSCREEN_WGC_TEST_WINDOW === "true" || process.argv.includes("--window");
 const WITH_WEBCAM =
 	process.env.OPENSCREEN_WGC_TEST_WEBCAM === "true" || process.argv.includes("--webcam");
+const CAPTURE_CURSOR =
+	process.env.OPENSCREEN_WGC_TEST_CAPTURE_CURSOR === "true" ||
+	process.argv.includes("--capture-cursor");
 
 function runHelper(config) {
 	return new Promise((resolve, reject) => {
@@ -247,6 +250,7 @@ const config = {
 	hasDisplayBounds: true,
 	captureSystemAudio: WITH_SYSTEM_AUDIO,
 	captureMic: WITH_MICROPHONE,
+	captureCursor: CAPTURE_CURSOR,
 	microphoneDeviceId: process.env.OPENSCREEN_WGC_TEST_MICROPHONE_DEVICE_ID ?? "default",
 	microphoneDeviceName: process.env.OPENSCREEN_WGC_TEST_MICROPHONE_DEVICE_NAME ?? "",
 	microphoneGain: 1.4,
@@ -297,6 +301,10 @@ const audioFormatLine = result.stdout
 	.split(/\r?\n/)
 	.find((line) => line.includes('"event":"audio-format"'));
 const audioFormat = audioFormatLine ? JSON.parse(audioFormatLine) : null;
+const cursorCaptureLine = result.stdout
+	.split(/\r?\n/)
+	.find((line) => line.includes('"event":"cursor-capture"'));
+const cursorCapture = cursorCaptureLine ? JSON.parse(cursorCaptureLine) : null;
 const nativeWebcamDiagnostics = result.stderr
 	.split(/\r?\n/)
 	.filter((line) => line.includes("Native webcam candidate"));
@@ -310,6 +318,15 @@ const nativeMicrophoneDiagnostics = result.stderr
 if (!hasVideo) {
 	throw new Error(`WGC helper output has no video stream: ${outputPath}`);
 }
+if (
+	(CAPTURE_CURSOR && !cursorCapture) ||
+	(cursorCapture &&
+		(cursorCapture.requested !== CAPTURE_CURSOR || cursorCapture.applied !== CAPTURE_CURSOR))
+) {
+	throw new Error(
+		`WGC helper did not apply requested cursor capture mode (${CAPTURE_CURSOR}): ${result.stdout}`,
+	);
+}
 if ((WITH_SYSTEM_AUDIO || WITH_MICROPHONE) && !hasAudio) {
 	throw new Error(`WGC helper output has no audio stream: ${outputPath}`);
 }
@@ -332,6 +349,7 @@ console.log(
 				codecName: stream.codec_name,
 				duration: stream.duration,
 			})),
+			cursorCapture,
 			selectedMicrophoneDeviceName: audioFormat?.microphoneDeviceName,
 			selectedWebcamDeviceName: webcamFormat?.deviceName,
 			nativeMicrophoneDiagnostics,