diff --git a/.gitignore b/.gitignore index 771c4bd..0861e39 100644 --- a/.gitignore +++ b/.gitignore @@ -1,45 +1,60 @@ -# Logs -logs -*.log -npm-debug.log* -yarn-debug.log* -yarn-error.log* -pnpm-debug.log* -lerna-debug.log* - -node_modules -dist -dist-electron -dist-ssr -*.local -.env - -# Editor directories and files -.vscode/* -.zed/ -!.vscode/extensions.json -.idea -.DS_Store -*.suo -*.ntvs* -*.njsproj -*.sln -*.sw? -release/** -*.kiro/ -.claude/ -# npx electron-builder --mac --win - -# Playwright -test-results -playwright-report/ - -# Vitest browser mode screenshots -__screenshots__/ - -# shell files -/shell.sh -# Nix -result -result-* -.direnv/ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-electron +dist-ssr +*.local +.env + +# Native helper build outputs +/electron/native/wgc-capture/build/ +/electron/native/screencapturekit/build/ +/electron/native/screencapturekit/.build/ +/electron/native/screencapturekit/.swiftpm/ +/electron/native/bin/ + +# Native macOS generated files +DerivedData/ +*.xcuserstate +xcuserdata/ + +# Editor directories and files +.vscode/* +.zed/ +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? +release/** +*.kiro/ +.claude/ +# npx electron-builder --mac --win + +# Playwright +test-results +playwright-report/ + +# Vitest browser mode screenshots +__screenshots__/ + +# shell files +/shell.sh +# Nix +result +result-* +.direnv/ + +#kilocode +.kilo/ diff --git a/docs/architecture/native-bridge.md b/docs/architecture/native-bridge.md new file mode 100644 index 0000000..ef320f7 --- /dev/null +++ b/docs/architecture/native-bridge.md @@ -0,0 +1,39 @@ +# Native Bridge Architecture + +## Goal + +Provide a single, resilient source of truth for platform-native capabilities while keeping Electron transport thin and renderer APIs unified. + +## Layers + +1. Native adapters +Platform-specific providers implement stable domain interfaces such as cursor telemetry or system asset discovery. + +2. Main-process services +Services orchestrate adapters, own runtime state, and expose domain-level operations. + +3. Unified IPC transport +Renderer code talks to a single `native-bridge:invoke` channel using versioned contracts. + +4. Renderer client +React code should consume `src/native/client.ts` rather than binding directly to ad hoc Electron APIs. + +## Principles + +- Single source of truth: runtime-native state lives in the Electron main process. +- Capability-first: renderer can query support before attempting native behavior. +- Versioned contracts: requests and responses are explicit and evolve predictably. +- Resilience: every response uses a consistent result envelope with stable error codes. + +## Current rollout + +This repository now contains the initial scaffold: + +- shared contracts in `src/native/contracts.ts` +- renderer SDK in `src/native/client.ts` +- main-process state store in `electron/native-bridge/store.ts` +- cursor telemetry adapter in `electron/native-bridge/cursor/telemetryCursorAdapter.ts` +- domain services in `electron/native-bridge/services/*` +- unified handler registration in `electron/ipc/nativeBridge.ts` + +The legacy `window.electronAPI` surface still exists for backward compatibility. New native-facing features should prefer the unified bridge client. \ No newline at end of file diff --git a/docs/engineering/macos-native-recorder-roadmap.md b/docs/engineering/macos-native-recorder-roadmap.md new file mode 100644 index 0000000..63afc42 --- /dev/null +++ b/docs/engineering/macos-native-recorder-roadmap.md @@ -0,0 +1,210 @@ +# macOS Native Recorder Roadmap + +OpenScreen's macOS recorder should follow the same architecture boundaries as the Windows native recorder: Electron owns session orchestration and persistence, while a platform-native helper owns capture, timing, encoding, and platform-specific permissions. + +This work is intentionally scoped as a macOS-only port. Windows native capture remains owned by the WGC helper, and Linux remains on the existing Electron path. + +## Goals + +- Capture displays and windows through ScreenCaptureKit. +- Exclude the real system cursor during capture when using the editable OpenScreen cursor overlay. +- Preserve the current high-quality cursor overlay path in preview and export. +- Capture macOS system audio through ScreenCaptureKit on supported macOS versions. +- Capture microphone audio through the same native timing domain where the OS supports it, or through an explicit companion path until it can be moved into the helper. +- Mix system audio and microphone audio into the primary MP4 without renderer-side track assembly. +- Capture webcam video natively and compose it into the helper-owned MP4 during the native-recording migration. +- Keep screen video, audio, webcam, and cursor aligned to one native timing origin. +- Package per-architecture helper binaries with macOS builds. + +## Non-Goals + +- Replacing the editor/export pipeline. +- Changing Windows native capture behavior. +- Adding Linux native capture. +- Shipping a silent fallback from native macOS capture to Electron capture when the user explicitly requested a native-only feature. + +## Architecture + +The renderer keeps the existing recording controls. On macOS, `useScreenRecorder` should eventually send a complete recording request to Electron instead of assembling display, audio, microphone, webcam, and cursor streams in the browser. + +Electron owns the native recording session: + +- resolves the selected display/window source; +- resolves output paths; +- starts cursor telemetry capture when editable cursor mode is selected; +- starts the ScreenCaptureKit helper process; +- sends pause/resume/stop/cancel commands; +- writes `RecordingSession` manifests; +- reports explicit errors when a macOS-native capability is unavailable. + +The helper owns macOS media capture: + +- ScreenCaptureKit display/window frames; +- ScreenCaptureKit system audio where supported; +- microphone capture or helper-owned companion audio capture; +- webcam capture and initial picture-in-picture composition; +- AVFoundation/VideoToolbox encoding and muxing; +- stream timestamp normalization. + +## Helper Contract V1 + +The helper receives a single JSON argument: + +```json +{ + "schemaVersion": 1, + "recordingId": 1234567890, + "source": { + "type": "display", + "sourceId": "screen:0:0", + "displayId": 1, + "windowId": null, + "bounds": { "x": 0, "y": 0, "width": 1920, "height": 1080 } + }, + "video": { + "fps": 60, + "width": 1920, + "height": 1080, + "bitrate": 18000000, + "hideSystemCursor": true + }, + "audio": { + "system": { "enabled": true }, + "microphone": { + "enabled": true, + "deviceId": "default", + "deviceName": "MacBook Pro Microphone", + "gain": 1.4 + } + }, + "webcam": { + "enabled": true, + "deviceId": "default", + "deviceName": "FaceTime HD Camera", + "width": 1280, + "height": 720, + "fps": 30 + }, + "cursor": { + "mode": "editable-overlay" + }, + "outputs": { + "screenPath": "/Users/me/Library/Application Support/openscreen/recordings/recording-123.mp4", + "manifestPath": "/Users/me/Library/Application Support/openscreen/recordings/recording-123.session.json" + } +} +``` + +The helper emits newline-delimited JSON events to stdout: + +```json +{ "event": "ready", "schemaVersion": 1 } +{ "event": "recording-started", "timestampMs": 1234567890 } +{ "event": "warning", "code": "microphone-unavailable", "message": "..." } +{ "event": "recording-stopped", "screenPath": "..." } +{ "event": "error", "code": "screen-permission-denied", "message": "..." } +``` + +## Implementation Phases + +Current PR status: macOS screen/window capture routes through the ScreenCaptureKit helper when it is available so editable-cursor recordings can hide the system cursor. The helper now writes ScreenCaptureKit system audio into the primary MP4 and attempts runtime-gated native microphone capture on macOS versions that expose ScreenCaptureKit microphone output. Webcam capture is currently an Electron-recorded sidecar attached to the same recording session; native AVFoundation webcam composition remains the target end state. + +### 1. Native Session Boundary + +- Add a structured macOS native recording request type. +- Add a macOS helper resolver and build script placeholders. +- Keep the helper contract process-based, matching the Windows helper boundary. +- Do not route production macOS recording through this helper until the helper is available and validated. + +Acceptance: + +- TypeScript build passes. +- The macOS helper path and request contract are documented and testable without affecting Windows/Linux behavior. + +### 2. ScreenCaptureKit Display Capture + +- Implement a Swift helper using ScreenCaptureKit. +- Select display captures by `displayId`. +- Encode H.264 MP4 through AVFoundation/VideoToolbox. +- Set `showsCursor = false` when editable cursor overlay mode is selected. + +Acceptance: + +- Display-only recording produces a valid MP4. +- The real cursor is not baked into editable-cursor recordings. + +### 3. ScreenCaptureKit Window Capture + +- Resolve Electron `window:*` selections to ScreenCaptureKit window ids. +- Capture `SCContentFilter(desktopIndependentWindow:)`. +- Handle closed/minimized/protected windows with explicit errors. +- Keep window selection and capture source resolution in Electron/main, not the renderer. + +Acceptance: + +- Capturing a normal app window works with cursor/audio/webcam disabled. +- Unsupported windows return clear native errors. + +### 4. System Audio + +- Enable ScreenCaptureKit system audio on supported macOS versions. +- Keep audio format and timing owned by the helper. +- Encode or mux AAC audio into the primary MP4. + +Acceptance: + +- System-audio-only recordings produce a valid AAC track. +- Unsupported macOS versions return an explicit capability error. + +### 5. Microphone + +- Resolve the selected microphone device from the renderer-provided browser `deviceId` and user-visible label. +- Capture microphone audio in the helper timing domain. +- Apply OpenScreen microphone gain policy. +- Mix system and microphone audio before final AAC output. + +Acceptance: + +- Mic-only and mic-plus-system recordings produce a valid, balanced AAC track. +- Device selection honors the selected microphone, not only the default device. + +### 6. Webcam Composition + +- Capture the selected camera natively through AVFoundation. +- Match browser device id first where possible, then user-visible label. +- Compose an initial picture-in-picture overlay into the primary MP4. +- Hide webcam output until the first usable frame to avoid black startup flashes. + +Acceptance: + +- Native display/window recordings can include webcam without returning to Electron capture. +- Selected camera is honored. + +### 7. Runtime Controls + +- Add pause/resume commands to the helper. +- Add cancel command that removes partial outputs. +- Keep restart as stop-discard-start until the helper exposes a native restart operation. + +Acceptance: + +- Pause/resume keeps output duration coherent. +- Cancel leaves no stale media/session files. + +### 8. Test Pipeline + +- `npm run build:native:mac`: builds Swift helper binaries on macOS. +- `npm run test:sck-helper:mac`: display-only helper smoke test. +- `npm run test:sck-window:mac`: window capture smoke test. +- `npm run test:sck-audio:mac`: system audio smoke test when supported. +- `npm run test:sck-mic:mac`: microphone smoke test. +- `npm run test:sck-webcam:mac`: webcam smoke test when a webcam is available. +- Packaging check: confirms helpers are available under `electron/native/bin/darwin-${arch}` in packaged builds. + +## SSOT Rules + +- `src/lib/nativeMacRecording.ts` is the renderer/main TypeScript request contract. +- This document is the feature-level contract and phase checklist. +- The Swift helper owns ScreenCaptureKit/AVFoundation media timing. +- Electron owns output paths, session manifests, and selected source/device resolution. +- Renderer code must use existing hooks/client APIs and should not bind directly to helper process details. diff --git a/docs/engineering/windows-native-recorder-roadmap.md b/docs/engineering/windows-native-recorder-roadmap.md new file mode 100644 index 0000000..146582a --- /dev/null +++ b/docs/engineering/windows-native-recorder-roadmap.md @@ -0,0 +1,248 @@ +# Windows Native Recorder Roadmap + +OpenScreen's Windows recorder should be owned by one native backend. Electron capture can remain available for non-Windows platforms and temporary developer diagnostics, but Windows production recording should not silently fall back to `getDisplayMedia` / `MediaRecorder`. + +## Goals + +- Capture displays and windows through Windows Graphics Capture (WGC). +- Render the native Windows cursor as OpenScreen's high-quality scalable cursor overlay. +- Capture system audio through WASAPI loopback. +- Capture microphone audio through WASAPI. +- Mix system audio and microphone audio into the primary screen recording. +- Capture webcam video natively and compose it into the Windows helper MP4 during the native-recording migration. +- Keep preview/export aligned because screen video, audio, webcam, and cursor share one native timing origin. +- Keep exported MP4s Windows-friendly: H.264 video plus AAC audio. Opus-in-MP4 is not an acceptable Windows export target. +- Package the native helper with the Windows app. + +## Non-Goals + +- Replacing the editor/export pipeline. +- Replacing the editor/export pipeline. A later pass can reintroduce a separate editable native `webcamVideoPath`; the current Windows-native milestone prioritizes a helper-owned multi-flux MP4 with deterministic screen/audio/mic/webcam sync. +- Adding a native fallback for macOS or Linux in this branch. + +## Target Architecture + +The renderer keeps the existing recording controls. On Windows, `useScreenRecorder` sends a complete recording request to Electron and does not assemble Windows `MediaStream` tracks with `MediaRecorder`. + +Electron owns the native recording session: + +- resolves the selected source; +- resolves output paths; +- starts cursor sampling; +- starts the helper process; +- sends pause/resume/stop/cancel commands; +- writes `RecordingSession` manifests; +- reports explicit errors when a Windows-native capability is unavailable. + +The helper owns Windows media capture: + +- WGC screen/window frames; +- WASAPI system loopback; +- WASAPI microphone input; +- Media Foundation webcam capture; +- DirectShow webcam fallback for virtual cameras not visible to Media Foundation; +- Media Foundation encoding/muxing; +- stream timestamp normalization. + +## Helper Contract V2 + +The helper receives a single JSON argument: + +```json +{ + "schemaVersion": 2, + "recordingId": 1234567890, + "source": { + "type": "display", + "sourceId": "screen:0:0", + "displayId": 123, + "windowHandle": null, + "bounds": { "x": 0, "y": 0, "width": 1920, "height": 1080 } + }, + "video": { + "fps": 60, + "width": 1920, + "height": 1080, + "bitrate": 18000000 + }, + "audio": { + "system": { "enabled": true }, + "microphone": { "enabled": true, "deviceId": "default", "gain": 1.4 } + }, + "webcam": { + "enabled": true, + "deviceId": "default", + "deviceName": "Camera (NVIDIA Broadcast)", + "width": 1280, + "height": 720, + "fps": 30, + "bitrate": 18000000 + }, + "outputs": { + "screenPath": "C:\\Users\\me\\recording-123.mp4", + "manifestPath": "C:\\Users\\me\\recording-123.session.json" + } +} +``` + +The helper emits newline-delimited JSON events to stdout: + +```json +{ "event": "ready", "schemaVersion": 2 } +{ "event": "recording-started", "timestampMs": 1234567890 } +{ "event": "warning", "code": "audio-device-unavailable", "message": "..." } +{ "event": "recording-stopped", "screenPath": "..." } +{ "event": "error", "code": "unsupported-window-source", "message": "..." } +``` + +During migration, Electron also accepts the current textual helper messages so existing display-only smoke tests keep working. + +## Implementation Phases + +### 1. Native Session Boundary + +- Add a structured Windows native recording request type. +- Pass source kind, audio flags, microphone device, webcam flags, and output paths into the helper. +- On Windows, do not silently fall back to Electron capture. If the helper is unavailable or a native feature is missing, show a clear error. +- Keep Electron fallback only for non-Windows and optional developer diagnostics. + +Acceptance: + +- Display-only recording still works. +- Enabling an unsupported native feature returns an explicit native error instead of recording through Electron. + +### 2. WASAPI System Audio + +Status: initial implementation landed. The helper captures the default render endpoint with WASAPI loopback, passes the runtime mix format into `MFEncoder`, and muxes AAC audio into the primary MP4. Long-run drift correction and explicit silence insertion remain follow-up hardening work. + +- Add `WasapiLoopbackCapture`. +- Capture the default render endpoint in shared loopback mode. +- Keep `WasapiLoopbackCapture` responsible only for device activation, packet capture, and packet timestamps. +- Keep `MFEncoder` responsible for all Media Foundation stream definitions and muxing. +- Feed the endpoint mix format into `MFEncoder` as the single source of truth for audio stream shape: sample rate, channel count, bits per sample, block alignment, average bytes/sec, and subtype (`PCM` or `Float`). +- Encode the primary screen MP4 with H.264 video and AAC audio through one `IMFSinkWriter`. +- Timestamp audio from the captured frame count in 100ns units. The first implementation uses the WASAPI packet timeline; later drift correction will add explicit silence or resampling if long recordings show measurable clock skew. +- Treat microphone mixing as a later phase. System loopback must land first without introducing renderer-side audio code. + +Acceptance: + +- Screen MP4 has an AAC audio track when system audio is enabled. +- A 5-minute recording has audio/video duration drift below one frame. + +SSOT rules for this phase: + +- `src/lib/nativeWindowsRecording.ts` is the renderer/main TypeScript request contract. +- `docs/engineering/windows-native-recorder-roadmap.md` is the feature-level contract and phase checklist. +- `WgcSession::captureWidth()/captureHeight()` is the encoded screen frame size until a dedicated native scaling stage exists. +- `WasapiLoopbackCapture::inputFormat()` is the runtime audio format source used by `MFEncoder`. +- The renderer passes both the browser webcam `deviceId` and selected display label as `deviceName`; `electron/native/wgc-capture/src/webcam_capture.*` is the only place that maps those values to Media Foundation devices. +- Electron resolves the selected label to a DirectShow filter CLSID once and passes it as `webcamDirectShowClsid`; the helper must not independently guess among DirectShow filters. +- No duplicated hard-coded audio format assumptions in `main.cpp`. + +### 3. WASAPI Microphone + +Status: initial implementation in progress. The helper can open the default WASAPI capture endpoint, apply the OpenScreen microphone gain, encode mic-only audio, and mix system loopback plus microphone through a single queued `AudioMixer` timeline when both endpoints expose the same runtime format. Audio endpoints are warmed before WGC starts, the mixer drops pre-roll and begins its paced timeline on the first encoded video frame, then cuts queued tail audio on stop so the MP4 does not drift past the video. Browser `deviceId` to MMDevice id mapping, resampling between mismatched endpoint formats, and drift correction remain follow-up hardening work. + +- Add microphone device enumeration and stable device-id mapping. +- Capture selected/default microphone through WASAPI. +- Apply OpenScreen's current mic gain policy. +- Mix microphone and system audio before AAC encoding. + +Acceptance: + +- Mic-only, system-only, and mixed audio recordings produce a valid AAC track. +- Device unplug/permission failure produces an explicit error or warning. + +### 4. Webcam Capture + +- Add Media Foundation webcam source reader. +- Select requested dimensions/fps or the nearest format accepted by Media Foundation. +- Convert webcam samples to BGRA and compose them into the primary helper MP4 as an initial bottom-right picture-in-picture overlay. +- Ignore black webcam warmup frames and keep the overlay hidden until the first visible frame is available, so virtual cameras do not flash a black picture-in-picture rectangle at recording start. +- Keep the helper process as the SSOT for screen/window, WASAPI system audio, microphone, webcam, and mux timing. +- Match the requested webcam through Media Foundation friendly names first, then browser device ids/symbolic links, so UI selection remains stable across Chromium and Windows native device namespaces. +- Use the Electron-resolved DirectShow CLSID when the selected virtual camera, for example NVIDIA Broadcast, is registered for DirectShow but absent from Media Foundation enumeration. +- Later: promote the same webcam capture source to a separate editable native `webcamVideoPath` if product requirements need post-recording layout edits. + +Acceptance: + +- Native display/window recordings can include webcam without returning to Electron capture. +- `npm run test:wgc-webcam:win` validates the helper path when a webcam is available and skips explicitly when no webcam device exists. +- Combined webcam + system audio + microphone produces one MP4 with H.264 video and AAC audio. + +### 5. Native Window Capture + +Status: initial implementation in progress. Electron parses the `window::...` desktop source id through the shared native Windows recording contract and passes `windowHandle` to the helper. The helper resolves the `HWND`, validates it with `IsWindow`, and creates the WGC item with `CreateForWindow(HWND)`. Resize/minimize/move hardening and protected-window diagnostics remain follow-up work. + +- Resolve Electron `window:*` selections to an `HWND`. +- Use WGC `CreateForWindow(HWND)`. +- Handle window close, minimize, resize, DPI scaling, and monitor moves. +- Return clear errors for unsupported protected windows. + +Acceptance: + +- Capturing a normal app window works with cursor/audio/mic/webcam. +- Window resize and movement do not corrupt the recording. + +### 6. Runtime Controls + +- Add pause/resume commands to the helper. +- Add cancel command that removes partial screen/webcam outputs. +- Keep restart as stop-discard-start from Electron until the helper supports a native restart event. + +Acceptance: + +- Pause/resume keeps preview duration coherent. +- Cancel leaves no stale media/session/cursor files. + +### 7. Test Pipeline + +- `npm run test:wgc-helper:win`: display-only helper smoke test. +- `npm run test:wgc-audio:win`: validates AAC track presence and duration. +- `npm run test:wgc-window:win`: captures a fixture window by HWND. +- `npm run test:wgc-webcam:win`: validates webcam output when a webcam is available, otherwise skips explicitly. +- Packaging check: confirms the helper is in `app.asar.unpacked`. +- Export check: exported MP4s generated from native recordings keep an AAC audio track when the source has audio. +- `npm run test:wgc-mic:win`: validates default-microphone capture writes an AAC track when an input endpoint is available. +- `npm run test:wgc-mixed-audio:win`: validates system loopback plus microphone writes one mixed AAC track when endpoint formats are compatible. + +## Backlog + +### Native Cursor Click Bounce Is Not Visibly Applied + +Status: open. Do not treat Windows native cursor `Click Bounce` as shipped. + +Problem: + +- The cursor settings UI exposes `Size`, `Smoothing`, `Motion Blur`, and `Click Bounce`. +- On Windows native cursor recordings, `Size`, `Smoothing`, and `Motion Blur` are visibly applied in preview/export. +- `Click Bounce` still has no visible effect in manual packaged-app testing, even after adding click-related sample metadata. + +What has already been tried: + +- Added `interactionType: "click" | "mouseup" | "move"` to native cursor samples. +- Added polling-based left-button state through `GetAsyncKeyState`. +- Added the `GetAsyncKeyState` low-bit path to catch quick clicks between samples. +- Added a PowerShell/C# `WH_MOUSE_LL` mouse hook experiment and launched the sampler through a temporary `.ps1` file to avoid Windows command-line length limits. +- Updated `npm run test:cursor-native:win` so the diagnostic can observe a synthetic short click and emit `clickSampleCount`. + +Current diagnosis: + +- The diagnostic can observe synthetic click events, but this has not translated into a visible `Click Bounce` effect in the real packaged app. +- The test currently proves that some click metadata can be recorded, not that the full OpenScreen record -> preview -> export path displays a bounce at the expected time. +- The current native implementation may be animating from metadata that is not present in the real recording session, may be using the wrong timestamp origin, or may be applying a scale change too subtle to notice on the DOM/native cursor path. + +Next investigation when resumed: + +- Inspect the actual `.cursor.json`/session sidecar generated by a packaged-app manual recording and confirm whether real clicks produce `interactionType: "click"` at the right `timeMs`. +- Add a targeted end-to-end fixture that records a known click, loads the generated project, and asserts the preview/export cursor scale changes across adjacent frames. +- Compare the native DOM cursor path against the older `PixiCursorOverlay` click visual state and decide whether native cursor bounce should be a scale-only animation, an additional click ring, or a short explicit keyframe animation independent of sample cadence. +- If event capture remains unreliable in the PowerShell sampler, move click events into a small native cursor helper instead of PowerShell/C# script injection. + +## Ship Criteria + +- Windows display capture works with cursor, system audio, microphone, and webcam. +- Windows window capture works with cursor, system audio, microphone, and webcam. +- Preview and export show no cursor position drift. +- Preview and export show no measurable audio/video/webcam drift. +- Windows production builds do not depend on Electron capture fallback. diff --git a/docs/testing/windows-native-cursor.md b/docs/testing/windows-native-cursor.md new file mode 100644 index 0000000..c8c5ac0 --- /dev/null +++ b/docs/testing/windows-native-cursor.md @@ -0,0 +1,130 @@ +# Windows native cursor test pipeline + +This branch includes two Windows-focused diagnostics for fast iteration on native cursor capture and rendering. They are intentionally local developer tools: they create short videos and JSON reports so cursor changes can be inspected without doing a full manual record/edit/export cycle. + +## Native sampler diagnostic + +```powershell +npm run test:cursor-native:win +``` + +This script does not launch OpenScreen. It: + +- starts a Windows `GetCursorInfo` sampler +- moves the real OS pointer with `SetCursorPos` +- captures native cursor handles, hotspots, assets, and standard `IDC_*` cursor types +- writes normalized `CursorRecordingData` +- generates an abstract preview video +- generates a real-screen preview video using screenshots of the current desktop + +The output directory is printed in the command result, for example: + +```text +C:\Users\\AppData\Local\Temp\openscreen-cursor-native-... +``` + +Useful files: + +- `report.json`: sample counts, asset counts, cursor handles, and generated artifact paths +- `cursor-recording-data.json`: sidecar-compatible cursor data +- `preview.webm`: abstract path/asset/hotspot preview +- `real-capture-preview.webm`: real desktop screenshot background with reconstructed cursor overlay +- `assets/*.png`: raw cursor bitmaps captured from Windows + +Environment overrides: + +```powershell +$env:CURSOR_TEST_DURATION_MS = "3000" +$env:CURSOR_TEST_SAMPLE_INTERVAL_MS = "16" +$env:CURSOR_TEST_SCREEN_FRAME_INTERVAL_MS = "80" +$env:CURSOR_TEST_OUTPUT_DIR = "C:\temp\openscreen-cursor-test" +npm run test:cursor-native:win +``` + +## OpenScreen preview capture + +```powershell +npm run capture:openscreen-preview +``` + +This script launches the real Electron app, injects a fixture video plus cursor sidecar data, opens the editor, captures frames from the actual OpenScreen preview UI, and encodes them into a WebM. + +By default it uses the latest `cursor-recording-data.json` generated by `npm run test:cursor-native:win`. To force a specific sidecar: + +```powershell +$env:CURSOR_RECORDING_DATA_PATH = "C:\path\to\cursor-recording-data.json" +npm run capture:openscreen-preview +``` + +Useful environment overrides: + +```powershell +$env:OPENSCREEN_PREVIEW_SKIP_BUILD = "true" +$env:OPENSCREEN_PREVIEW_FRAME_COUNT = "120" +$env:OPENSCREEN_PREVIEW_FPS = "30" +$env:OPENSCREEN_PREVIEW_OUTPUT_DIR = "C:\temp\openscreen-preview" +npm run capture:openscreen-preview +``` + +Useful files: + +- `openscreen-preview.webm`: video of the real OpenScreen editor preview +- `frames/*.png`: captured preview frames +- `report.json`: fixture paths, source sidecar, frame count, and output path + +## What these tests validate + +Together, the scripts make it quick to inspect: + +- whether Windows cursor samples are visible and continuous +- whether native hotspots stay anchored when scaling to `3x` +- whether standard Windows cursors are recognized via `IDC_*` +- whether high-quality SVG cursor replacements follow the native hotspot +- whether the real OpenScreen preview renders the same cursor behavior as the diagnostic pipeline + +They are not a full substitute for an end-to-end manual recording pass. Before shipping cursor changes, also test a real capture session and export from the packaged app. + +## Known Gap + +Windows native cursor `Click Bounce` is currently backlogged. `Size`, `Smoothing`, and `Motion Blur` can be validated through preview/export, but `Click Bounce` has not shown a visible effect in packaged-app manual testing. The current diagnostic can observe synthetic click metadata, but that is not enough to validate the real OpenScreen record -> preview -> export path. + +Track the open item in `docs/engineering/windows-native-recorder-roadmap.md` under `Native Cursor Click Bounce Is Not Visibly Applied`. + +## Native Windows capture backend + +The app now routes Windows recordings through an external WGC helper instead of Electron `getDisplayMedia`. This is meant to remove the coordinate and clock split that made the reconstructed cursor drift in the preview/export path. + +Current native availability rules: + +- Windows 10 build 19041 or newer +- a helper executable is available + +The helper currently implements display/window video capture, system audio loopback, default microphone capture, Media Foundation webcam capture, and DirectShow fallback for selected virtual cameras such as NVIDIA Broadcast. Webcam frames are composed into the primary MP4 as a bottom-right picture-in-picture overlay, and black webcam warmup frames are ignored until the first visible frame is available. + +Build OpenScreen's helper locally: + +```powershell +npm run build:native:win +``` + +Smoke-test the helper directly: + +```powershell +npm run test:wgc-helper:win +npm run test:wgc-helper:win -- --capture-cursor +npm run test:wgc-window:win +npm run test:wgc-audio:win +npm run test:wgc-mic:win +npm run test:wgc-mixed-audio:win +npm run test:wgc-webcam:win +``` + +For local diagnostics with another compatible helper, point OpenScreen at that executable: + +```powershell +$env:OPENSCREEN_WGC_CAPTURE_EXE = "C:\path\to\wgc-capture.exe" +npm run build-vite +npm run dev +``` + +The helper receives one JSON config argument, emits JSON lifecycle events, prints the legacy `Recording started` marker, accepts `stop` on stdin, and prints `Recording stopped. Output path: `. See `electron/native/README.md` for the exact contract and build output paths. diff --git a/electron-builder.json5 b/electron-builder.json5 index 372cdf7..8ad4a80 100644 --- a/electron-builder.json5 +++ b/electron-builder.json5 @@ -5,7 +5,6 @@ "asar": true, // .node binaries can't be dlopen'd from inside an asar — must live unpacked. "asarUnpack": [ - "node_modules/uiohook-napi/**/*", "**/*.node" ], "productName": "Openscreen", @@ -47,6 +46,13 @@ ], "icon": "icons/icons/mac/icon.icns", "artifactName": "${productName}-Mac-${arch}-${version}-Installer.${ext}", + "extraResources": [ + { + "from": "electron/native/bin", + "to": "electron/native/bin", + "filter": ["darwin-*/*"] + } + ], "extendInfo": { "NSAudioCaptureUsageDescription": "OpenScreen needs audio capture permission to record system audio.", "NSMicrophoneUsageDescription": "OpenScreen needs microphone access to record voice audio.", @@ -69,7 +75,14 @@ "target": [ "nsis" ], - "icon": "icons/icons/win/icon.ico" + "icon": "icons/icons/win/icon.ico", + "extraResources": [ + { + "from": "electron/native/bin", + "to": "electron/native/bin", + "filter": ["win32-*/*"] + } + ] }, "nsis": { "oneClick": false, diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 1d528cd..1b82992 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -24,11 +24,23 @@ declare namespace NodeJS { // Used in Renderer process, expose in `preload.ts` interface Window { electronAPI: { + invokeNativeBridge: ( + request: import("../src/native/contracts").NativeBridgeRequest, + ) => Promise>; getSources: (opts: Electron.SourcesOptions) => Promise; switchToEditor: () => Promise; switchToHud: () => Promise; startNewRecording: () => Promise<{ success: boolean; error?: string }>; - openSourceSelector: () => Promise; + openSourceSelector: () => Promise<{ + opened: boolean; + reason?: string; + access?: { + success: boolean; + granted: boolean; + status: string; + error?: string; + }; + }>; selectSource: (source: ProcessedDesktopSource) => Promise; getSelectedSource: () => Promise; requestCameraAccess: () => Promise<{ @@ -37,9 +49,16 @@ interface Window { status: string; error?: string; }>; - requestAccessibilityAccess: () => Promise<{ + requestScreenAccess: () => Promise<{ success: boolean; granted: boolean; + status: string; + error?: string; + }>; + requestNativeMacCursorAccess: () => Promise<{ + success: boolean; + granted: boolean; + status: string; error?: string; }>; assetBaseUrl: string; @@ -68,7 +87,75 @@ interface Window { message?: string; error?: string; }>; - setRecordingState: (recording: boolean, recordingId?: number) => Promise; + setRecordingState: ( + recording: boolean, + recordingId?: number, + cursorCaptureMode?: import("../src/lib/recordingSession").CursorCaptureMode, + ) => Promise; + isNativeWindowsCaptureAvailable: () => Promise<{ + success: boolean; + available: boolean; + helperPath?: string; + reason?: string; + error?: string; + }>; + isNativeMacCaptureAvailable: () => Promise<{ + success: boolean; + available: boolean; + helperPath?: string; + reason?: "unsupported-platform" | "missing-helper" | string; + error?: string; + }>; + startNativeWindowsRecording: ( + request: import("../src/lib/nativeWindowsRecording").NativeWindowsRecordingRequest, + ) => Promise; + stopNativeWindowsRecording: (discard?: boolean) => Promise<{ + success: boolean; + path?: string; + session?: import("../src/lib/recordingSession").RecordingSession; + message?: string; + discarded?: boolean; + error?: string; + }>; + pauseNativeWindowsRecording: () => Promise<{ + success: boolean; + error?: string; + }>; + resumeNativeWindowsRecording: () => Promise<{ + success: boolean; + error?: string; + }>; + startNativeMacRecording: ( + request: import("../src/lib/nativeMacRecording").NativeMacRecordingRequest, + ) => Promise; + pauseNativeMacRecording: () => Promise<{ + success: boolean; + error?: string; + }>; + resumeNativeMacRecording: () => Promise<{ + success: boolean; + error?: string; + }>; + stopNativeMacRecording: (discard?: boolean) => Promise<{ + success: boolean; + path?: string; + session?: import("../src/lib/recordingSession").RecordingSession; + message?: string; + discarded?: boolean; + error?: string; + }>; + attachNativeMacWebcamRecording: (payload: { + screenVideoPath: string; + recordingId: number; + webcam: import("../src/lib/recordingSession").RecordedVideoAssetInput; + cursorCaptureMode?: import("../src/lib/recordingSession").CursorCaptureMode; + }) => Promise<{ + success: boolean; + path?: string; + session?: import("../src/lib/recordingSession").RecordingSession; + message?: string; + error?: string; + }>; discardCursorTelemetry: (recordingId: number) => Promise; getCursorTelemetry: (videoPath?: string) => Promise<{ success: boolean; @@ -118,6 +205,12 @@ interface Window { message?: string; error?: string; }>; + preparePreviewAudioTrack: (filePath: string) => Promise<{ + success: boolean; + path?: string | null; + message?: string; + error?: string; + }>; clearCurrentVideoPath: () => Promise<{ success: boolean }>; saveProjectFile: ( projectData: unknown, @@ -158,6 +251,7 @@ interface Window { hudOverlayHide: () => void; hudOverlayClose: () => void; setHudOverlayIgnoreMouseEvents: (ignore: boolean) => void; + moveHudOverlayBy: (deltaX: number, deltaY: number) => void; showCountdownOverlay: (value: number, runId: number) => Promise; setCountdownOverlayValue: (value: number, runId: number) => Promise; hideCountdownOverlay: (runId: number) => Promise; diff --git a/electron/i18n.ts b/electron/i18n.ts index e16ac86..1492578 100644 --- a/electron/i18n.ts +++ b/electron/i18n.ts @@ -9,10 +9,14 @@ import commonEs from "../src/i18n/locales/es/common.json"; import dialogsEs from "../src/i18n/locales/es/dialogs.json"; import commonFr from "../src/i18n/locales/fr/common.json"; import dialogsFr from "../src/i18n/locales/fr/dialogs.json"; +import commonIt from "../src/i18n/locales/it/common.json"; +import dialogsIt from "../src/i18n/locales/it/dialogs.json"; import commonJa from "../src/i18n/locales/ja-JP/common.json"; import dialogsJa from "../src/i18n/locales/ja-JP/dialogs.json"; import commonKo from "../src/i18n/locales/ko-KR/common.json"; import dialogsKo from "../src/i18n/locales/ko-KR/dialogs.json"; +import commonRu from "../src/i18n/locales/ru/common.json"; +import dialogsRu from "../src/i18n/locales/ru/dialogs.json"; import commonTr from "../src/i18n/locales/tr/common.json"; import dialogsTr from "../src/i18n/locales/tr/dialogs.json"; import commonVi from "../src/i18n/locales/vi/common.json"; @@ -22,21 +26,35 @@ import dialogsZh from "../src/i18n/locales/zh-CN/dialogs.json"; import commonZhTw from "../src/i18n/locales/zh-TW/common.json"; import dialogsZhTw from "../src/i18n/locales/zh-TW/dialogs.json"; -type Locale = "en" | "zh-CN" | "zh-TW" | "es" | "fr" | "ja-JP" | "ko-KR" | "tr" | "ar" | "vi"; +type Locale = + | "en" + | "ar" + | "es" + | "fr" + | "it" + | "ja-JP" + | "ko-KR" + | "ru" + | "tr" + | "vi" + | "zh-CN" + | "zh-TW"; type Namespace = "common" | "dialogs"; type MessageMap = Record; const messages: Record> = { en: { common: commonEn, dialogs: dialogsEn }, - "zh-CN": { common: commonZh, dialogs: dialogsZh }, - "zh-TW": { common: commonZhTw, dialogs: dialogsZhTw }, + ar: { common: commonAr, dialogs: dialogsAr }, es: { common: commonEs, dialogs: dialogsEs }, fr: { common: commonFr, dialogs: dialogsFr }, + it: { common: commonIt, dialogs: dialogsIt }, "ja-JP": { common: commonJa, dialogs: dialogsJa }, "ko-KR": { common: commonKo, dialogs: dialogsKo }, + ru: { common: commonRu, dialogs: dialogsRu }, tr: { common: commonTr, dialogs: dialogsTr }, - ar: { common: commonAr, dialogs: dialogsAr }, vi: { common: commonVi, dialogs: dialogsVi }, + "zh-CN": { common: commonZh, dialogs: dialogsZh }, + "zh-TW": { common: commonZhTw, dialogs: dialogsZhTw }, }; let currentLocale: Locale = "en"; @@ -44,15 +62,17 @@ let currentLocale: Locale = "en"; export function setMainLocale(locale: string) { if ( locale === "en" || - locale === "zh-CN" || - locale === "zh-TW" || + locale === "ar" || locale === "es" || locale === "fr" || + locale === "it" || locale === "ja-JP" || locale === "ko-KR" || + locale === "ru" || locale === "tr" || - locale === "ar" || - locale === "vi" + locale === "vi" || + locale === "zh-CN" || + locale === "zh-TW" ) { currentLocale = locale; } diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 19178be..669f1dd 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -1,11 +1,11 @@ +import { type ChildProcessWithoutNullStreams, spawn } from "node:child_process"; +import { EventEmitter } from "node:events"; +import { constants as fsConstants } from "node:fs"; import fs from "node:fs/promises"; -import { createRequire } from "node:module"; import os from "node:os"; import path from "node:path"; -import { fileURLToPath } from "node:url"; - -const nodeRequire = createRequire(import.meta.url); - +import { fileURLToPath, pathToFileURL } from "node:url"; +import type { DesktopCapturerSource } from "electron"; import { app, BrowserWindow, @@ -16,24 +16,39 @@ import { shell, systemPreferences, } from "electron"; +import type { NativeMacRecordingRequest } from "../../src/lib/nativeMacRecording"; +import type { NativeWindowsRecordingRequest } from "../../src/lib/nativeWindowsRecording"; import { - type CursorTelemetryPoint, - createCursorTelemetryBuffer, -} from "../../src/lib/cursorTelemetryBuffer"; -import { + type CursorCaptureMode, + normalizeCursorCaptureMode, normalizeProjectMedia, normalizeRecordingSession, type ProjectMedia, + type RecordedVideoAssetInput, type RecordingSession, type StoreRecordedSessionInput, } from "../../src/lib/recordingSession"; +import type { + CursorRecordingData, + CursorRecordingSample, + NativeCursorAsset, + ProjectFileResult, + ProjectPathResult, +} from "../../src/native/contracts"; import { mainT } from "../i18n"; import { RECORDINGS_DIR } from "../main"; +import { createCursorRecordingSession } from "../native-bridge/cursor/recording/factory"; +import { requestMacCursorAccessibilityAccess } from "../native-bridge/cursor/recording/macNativeCursorRecordingSession"; +import type { CursorRecordingSession } from "../native-bridge/cursor/recording/session"; +import { registerNativeBridgeHandlers } from "./nativeBridge"; const PROJECT_FILE_EXTENSION = "openscreen"; const SHORTCUTS_FILE = path.join(app.getPath("userData"), "shortcuts.json"); +const RECORDING_FILE_PREFIX = "recording-"; const RECORDING_SESSION_SUFFIX = ".session.json"; const ALLOWED_IMPORT_VIDEO_EXTENSIONS = new Set([".webm", ".mp4", ".mov", ".avi", ".mkv"]); +const PREVIEW_AUDIO_DIR = path.join(app.getPath("userData"), "preview-audio"); +const nativeMacCaptureEvents = new EventEmitter(); /** * Paths explicitly approved by the user via file picker dialogs or project loads. @@ -61,6 +76,19 @@ function isPathAllowed(filePath: string): boolean { return getAllowedReadDirs().some((dir) => isPathWithinDir(resolved, dir)); } +function resolveApprovedVideoPath(videoPath?: string | null): string | null { + const normalizedPath = normalizeVideoSourcePath(videoPath); + if (!normalizedPath) { + return null; + } + + if (!hasAllowedImportVideoExtension(normalizedPath) || !isPathAllowed(normalizedPath)) { + return null; + } + + return normalizedPath; +} + /** * Helper function to build dialog options with a parent window only when it's valid. * This prevents passing stale or destroyed BrowserWindow references to dialog calls. @@ -80,6 +108,102 @@ function hasAllowedImportVideoExtension(filePath: string): boolean { return ALLOWED_IMPORT_VIDEO_EXTENSIONS.has(path.extname(filePath).toLowerCase()); } +function runProcess( + command: string, + args: string[], +): Promise<{ code: number | null; stdout: string; stderr: string }> { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + child.on("error", reject); + child.on("close", (code) => resolve({ code, stdout, stderr })); + }); +} + +function parseAfinfoAudioTrackBitrates(output: string): number[] { + const bitrates: number[] = []; + const trackSections = output.split(/\n----\n/g).slice(1); + for (const section of trackSections) { + const match = section.match(/\bbit rate:\s*([0-9]+)\s*bits per second/i); + bitrates.push(match ? Number(match[1]) : 0); + } + return bitrates; +} + +async function prepareSupplementalPreviewAudioTrack(videoPath: string) { + const normalizedPath = await approveReadableVideoPath(videoPath); + if (!normalizedPath) { + return { + success: false, + message: "File path is not approved or is not a supported video file", + }; + } + + if (process.platform !== "darwin" || path.extname(normalizedPath).toLowerCase() !== ".mp4") { + return { success: true, path: null }; + } + + const afinfo = await runProcess("/usr/bin/afinfo", [normalizedPath]); + if (afinfo.code !== 0) { + return { success: true, path: null }; + } + + const bitrates = parseAfinfoAudioTrackBitrates(`${afinfo.stdout}\n${afinfo.stderr}`); + if (bitrates.length <= 1) { + return { success: true, path: null }; + } + + let supplementalTrackIndex = 1; + for (let index = 2; index < bitrates.length; index += 1) { + if (bitrates[index] > bitrates[supplementalTrackIndex]) { + supplementalTrackIndex = index; + } + } + + await fs.mkdir(PREVIEW_AUDIO_DIR, { recursive: true }); + const sourceStat = await fs.stat(normalizedPath); + const parsedPath = path.parse(normalizedPath); + const outputPath = path.join( + PREVIEW_AUDIO_DIR, + `${parsedPath.name}.track-${supplementalTrackIndex}.${Math.round(sourceStat.mtimeMs)}.m4a`, + ); + + try { + const outputStat = await fs.stat(outputPath); + if (outputStat.mtimeMs >= sourceStat.mtimeMs) { + return { success: true, path: pathToFileURL(outputPath).toString() }; + } + } catch { + // Generate below. + } + + const conversion = await runProcess("/usr/bin/afconvert", [ + "--read-track", + String(supplementalTrackIndex), + "-f", + "m4af", + "-d", + "aac", + normalizedPath, + outputPath, + ]); + if (conversion.code !== 0) { + return { + success: false, + message: conversion.stderr || conversion.stdout || "Failed to prepare preview audio", + }; + } + + return { success: true, path: pathToFileURL(outputPath).toString() }; +} + async function approveReadableVideoPath( filePath?: string | null, trustedDirs?: string[], @@ -188,13 +312,33 @@ async function getApprovedProjectSession( type SelectedSource = { name: string; + id?: string; + display_id?: string; [key: string]: unknown; }; +type AttachNativeMacWebcamRecordingInput = { + screenVideoPath?: string; + recordingId?: number; + webcam?: RecordedVideoAssetInput; + cursorCaptureMode?: CursorCaptureMode; +}; + let selectedSource: SelectedSource | null = null; +let selectedDesktopSource: DesktopCapturerSource | null = null; +let lastEnumeratedSources = new Map(); let currentProjectPath: string | null = null; let currentRecordingSession: RecordingSession | null = null; +/** + * Returns the cached DesktopCapturerSource set when the user picked a source. + * Used by setDisplayMediaRequestHandler in main.ts for cursor-free capture. + */ +export function getSelectedDesktopSource(): DesktopCapturerSource | null { + return selectedDesktopSource; +} +let currentVideoPath: string | null = null; + function normalizePath(filePath: string) { return path.resolve(filePath); } @@ -227,254 +371,866 @@ function isTrustedProjectPath(filePath?: string | null) { return normalizePath(filePath) === normalizePath(currentProjectPath); } +const CURSOR_TELEMETRY_VERSION = 2; +const CURSOR_SAMPLE_INTERVAL_MS = 33; +const MAX_CURSOR_SAMPLES = 60 * 60 * 30; // 1 hour @ 30Hz + +let cursorRecordingSession: CursorRecordingSession | null = null; +let pendingCursorRecordingData: CursorRecordingData | null = null; +let nativeWindowsCaptureProcess: ChildProcessWithoutNullStreams | null = null; +let nativeWindowsCaptureOutput = ""; +let nativeWindowsCaptureTargetPath: string | null = null; +let nativeWindowsCaptureWebcamTargetPath: string | null = null; +let nativeWindowsCaptureRecordingId: number | null = null; +let nativeWindowsCursorOffsetMs = 0; +let nativeWindowsCursorCaptureMode: CursorCaptureMode = "editable-overlay"; +let nativeWindowsCursorRecordingStartMs = 0; +let nativeWindowsPauseStartedAtMs: number | null = null; +let nativeWindowsPauseRanges: Array<{ startMs: number; endMs: number }> = []; +let nativeWindowsIsPaused = false; +const NATIVE_WINDOWS_CAPTURE_STOP_TIMEOUT_MS = 15_000; +let nativeMacCaptureProcess: ChildProcessWithoutNullStreams | null = null; +let nativeMacCaptureOutput = ""; +let nativeMacCaptureTargetPath: string | null = null; +let nativeMacCaptureRecordingId: number | null = null; +let nativeMacCursorOffsetMs = 0; +let nativeMacCursorCaptureMode: CursorCaptureMode = "editable-overlay"; +let nativeMacCursorRecordingStartMs = 0; +let nativeMacPauseStartedAtMs: number | null = null; +let nativeMacPauseRanges: Array<{ startMs: number; endMs: number }> = []; +let nativeMacIsPaused = false; + +function normalizeCursorSample(sample: unknown): CursorRecordingSample | null { + if (!sample || typeof sample !== "object") { + return null; + } + + const point = sample as Partial; + const interactionType = + point.interactionType === "click" || + point.interactionType === "mouseup" || + point.interactionType === "move" + ? point.interactionType + : "move"; + return { + timeMs: + typeof point.timeMs === "number" && Number.isFinite(point.timeMs) + ? Math.max(0, point.timeMs) + : 0, + cx: typeof point.cx === "number" && Number.isFinite(point.cx) ? point.cx : 0.5, + cy: typeof point.cy === "number" && Number.isFinite(point.cy) ? point.cy : 0.5, + assetId: typeof point.assetId === "string" ? point.assetId : null, + visible: typeof point.visible === "boolean" ? point.visible : true, + cursorType: typeof point.cursorType === "string" ? point.cursorType : null, + interactionType, + }; +} + +function normalizeCursorAsset(asset: unknown): NativeCursorAsset | null { + if (!asset || typeof asset !== "object") { + return null; + } + + const candidate = asset as Partial; + if (typeof candidate.id !== "string" || typeof candidate.imageDataUrl !== "string") { + return null; + } + + return { + id: candidate.id, + platform: + candidate.platform === "win32" ? "win32" : process.platform === "darwin" ? "darwin" : "linux", + imageDataUrl: candidate.imageDataUrl, + width: + typeof candidate.width === "number" && Number.isFinite(candidate.width) + ? Math.max(1, Math.round(candidate.width)) + : 1, + height: + typeof candidate.height === "number" && Number.isFinite(candidate.height) + ? Math.max(1, Math.round(candidate.height)) + : 1, + hotspotX: + typeof candidate.hotspotX === "number" && Number.isFinite(candidate.hotspotX) + ? Math.max(0, Math.round(candidate.hotspotX)) + : 0, + hotspotY: + typeof candidate.hotspotY === "number" && Number.isFinite(candidate.hotspotY) + ? Math.max(0, Math.round(candidate.hotspotY)) + : 0, + scaleFactor: + typeof candidate.scaleFactor === "number" && Number.isFinite(candidate.scaleFactor) + ? Math.max(0.1, candidate.scaleFactor) + : undefined, + cursorType: typeof candidate.cursorType === "string" ? candidate.cursorType : null, + }; +} + +async function readCursorRecordingFile(targetVideoPath: string): Promise { + const telemetryPath = `${targetVideoPath}.cursor.json`; + try { + const content = await fs.readFile(telemetryPath, "utf-8"); + const parsed = JSON.parse(content); + const rawSamples = Array.isArray(parsed) + ? parsed + : Array.isArray(parsed?.samples) + ? parsed.samples + : []; + const rawAssets = Array.isArray(parsed?.assets) ? parsed.assets : []; + + const samples = rawSamples + .map((sample: unknown) => normalizeCursorSample(sample)) + .filter((sample: CursorRecordingSample | null): sample is CursorRecordingSample => + Boolean(sample), + ) + .sort((a: CursorRecordingSample, b: CursorRecordingSample) => a.timeMs - b.timeMs); + + const assets = rawAssets + .map((asset: unknown) => normalizeCursorAsset(asset)) + .filter((asset: NativeCursorAsset | null): asset is NativeCursorAsset => Boolean(asset)); + + return { + version: + typeof parsed?.version === "number" && Number.isFinite(parsed.version) ? parsed.version : 1, + provider: parsed?.provider === "native" ? "native" : "none", + samples, + assets, + }; + } catch (error) { + const nodeError = error as NodeJS.ErrnoException; + if (nodeError.code === "ENOENT") { + return { + version: CURSOR_TELEMETRY_VERSION, + provider: "none", + samples: [], + assets: [], + }; + } + + console.error("Failed to load cursor telemetry:", error); + throw error; + } +} + +async function readCursorTelemetryFile(targetVideoPath: string) { + try { + const recordingData = await readCursorRecordingFile(targetVideoPath); + return { + success: true, + samples: recordingData.samples.map((sample) => ({ + timeMs: sample.timeMs, + cx: sample.cx, + cy: sample.cy, + })), + }; + } catch (error) { + console.error("Failed to load cursor telemetry:", error); + return { + success: false, + message: "Failed to load cursor telemetry", + error: String(error), + samples: [], + }; + } +} + +function resolveAssetBasePath() { + try { + if (app.isPackaged) { + const assetPath = path.join(process.resourcesPath, "assets"); + return pathToFileURL(`${assetPath}${path.sep}`).toString(); + } + const assetPath = path.join(app.getAppPath(), "public", "assets"); + return pathToFileURL(`${assetPath}${path.sep}`).toString(); + } catch (err) { + console.error("Failed to resolve asset base path:", err); + return null; + } +} + +function getSelectedSourceBounds() { + const cursor = screen.getCursorScreenPoint(); + const sourceDisplayId = Number(selectedSource?.display_id); + const sourceDisplay = Number.isFinite(sourceDisplayId) + ? (screen.getAllDisplays().find((display) => display.id === sourceDisplayId) ?? null) + : null; + return (sourceDisplay ?? screen.getDisplayNearestPoint(cursor)).bounds; +} + +function getSelectedSourceId() { + return typeof selectedSource?.id === "string" ? selectedSource.id : null; +} + +function getSelectedDisplay() { + const sourceDisplayId = Number(selectedSource?.display_id); + if (!Number.isFinite(sourceDisplayId)) { + return null; + } + + return screen.getAllDisplays().find((display) => display.id === sourceDisplayId) ?? null; +} + +function resolveUnpackedAppPath(...segments: string[]) { + const resolved = path.join(app.getAppPath(), ...segments); + if (app.isPackaged) { + return resolved.replace(/\.asar([/\\])/, ".asar.unpacked$1"); + } + + return resolved; +} + +function resolvePackagedResourcePath(...segments: string[]) { + if (!app.isPackaged) { + return null; + } + + return path.join(process.resourcesPath, ...segments); +} + +function getNativeWindowsCaptureHelperCandidates() { + const envPath = process.env.OPENSCREEN_WGC_CAPTURE_EXE?.trim(); + const archTag = process.arch === "arm64" ? "win32-arm64" : "win32-x64"; + return [ + envPath, + resolveUnpackedAppPath( + "electron", + "native", + "wgc-capture", + "build", + "Release", + "wgc-capture.exe", + ), + resolveUnpackedAppPath("electron", "native", "wgc-capture", "build", "wgc-capture.exe"), + resolveUnpackedAppPath("electron", "native", "bin", archTag, "wgc-capture.exe"), + resolvePackagedResourcePath("electron", "native", "bin", archTag, "wgc-capture.exe"), + ].filter((candidate): candidate is string => Boolean(candidate)); +} + +async function findNativeWindowsCaptureHelperPath() { + if (process.platform !== "win32") { + return null; + } + + for (const candidate of getNativeWindowsCaptureHelperCandidates()) { + try { + await fs.access(candidate, fsConstants.X_OK); + return candidate; + } catch { + // Try the next configured helper location. + } + } + + return null; +} + +function getNativeMacCaptureHelperCandidates() { + const envPath = process.env.OPENSCREEN_SCK_CAPTURE_EXE?.trim(); + const archTag = process.arch === "arm64" ? "darwin-arm64" : "darwin-x64"; + const helperName = "openscreen-screencapturekit-helper"; + return [ + envPath, + resolveUnpackedAppPath("electron", "native", "screencapturekit", "build", helperName), + resolveUnpackedAppPath("electron", "native", "bin", archTag, helperName), + resolvePackagedResourcePath("electron", "native", "bin", archTag, helperName), + ].filter((candidate): candidate is string => Boolean(candidate)); +} + +async function findNativeMacCaptureHelperPath() { + if (process.platform !== "darwin") { + return null; + } + + for (const candidate of getNativeMacCaptureHelperCandidates()) { + try { + await fs.access(candidate, fsConstants.X_OK); + return candidate; + } catch { + // Try the next configured helper location. + } + } + + return null; +} + +function isWindowsGraphicsCaptureOsSupported() { + if (process.platform !== "win32") { + return false; + } + + const [, , build] = process.getSystemVersion().split(".").map(Number); + return Number.isFinite(build) && build >= 19041; +} + +function normalizeNativeDeviceName(value: string) { + return value + .toLowerCase() + .replace(/[^a-z0-9]+/g, " ") + .trim(); +} + +function scoreNativeDeviceName(candidateName: string, candidateId: string, requestedName?: string) { + const candidate = normalizeNativeDeviceName(candidateName); + const id = normalizeNativeDeviceName(candidateId); + const requested = normalizeNativeDeviceName(requestedName ?? ""); + if (!requested) { + return 0; + } + if (candidate === requested) { + return 1000; + } + if (candidate.includes(requested) || requested.includes(candidate)) { + return 900; + } + if (id.includes(requested) || requested.includes(id)) { + return 800; + } + + return requested + .split(/\s+/) + .filter((word) => word.length > 1 && !["camera", "webcam", "video", "input"].includes(word)) + .reduce((score, word) => { + if (candidate.includes(word)) return score + 100; + if (id.includes(word)) return score + 50; + return score; + }, 0); +} + +function queryDirectShowVideoInputRegistry() { + return new Promise((resolve) => { + const proc = spawn( + "reg.exe", + ["query", "HKCR\\CLSID\\{860BB310-5D01-11D0-BD3B-00A0C911CE86}\\Instance", "/s"], + { windowsHide: true }, + ); + let stdout = ""; + proc.stdout.on("data", (chunk: Buffer) => { + stdout += chunk.toString("utf16le").includes("\u0000") + ? chunk.toString("utf16le") + : chunk.toString(); + }); + proc.on("close", () => resolve(stdout)); + proc.on("error", () => resolve("")); + }); +} + +async function resolveDirectShowWebcamClsid(deviceName?: string) { + if (process.platform !== "win32" || !deviceName?.trim()) { + return null; + } + + const output = await queryDirectShowVideoInputRegistry(); + let current: { friendlyName?: string; clsid?: string } = {}; + const entries: Array<{ friendlyName?: string; clsid?: string }> = []; + for (const rawLine of output.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line) continue; + if (/^HKEY_/i.test(line)) { + if (current.friendlyName || current.clsid) entries.push(current); + current = {}; + continue; + } + const match = line.match(/^(\S+)\s+REG_SZ\s+(.+)$/); + if (!match) continue; + if (match[1] === "FriendlyName") current.friendlyName = match[2].trim(); + if (match[1] === "CLSID") current.clsid = match[2].trim(); + } + if (current.friendlyName || current.clsid) entries.push(current); + + let best: { clsid: string; friendlyName?: string; score: number } | null = null; + for (const entry of entries) { + if (!entry.clsid) continue; + const score = scoreNativeDeviceName(entry.friendlyName ?? "", entry.clsid, deviceName); + if (!best || score > best.score) { + best = { clsid: entry.clsid, friendlyName: entry.friendlyName, score }; + } + } + + if (!best || best.score <= 0) { + return null; + } + + console.info("[native-wgc] resolved DirectShow webcam filter", { + requestedName: deviceName, + filterName: best.friendlyName, + clsid: best.clsid, + score: best.score, + }); + return best.clsid; +} + +async function startCursorRecording(recordingId?: number) { + if (cursorRecordingSession) { + pendingCursorRecordingData = await cursorRecordingSession.stop(); + cursorRecordingSession = null; + } + + pendingCursorRecordingData = null; + cursorRecordingSession = createCursorRecordingSession({ + getDisplayBounds: getSelectedSourceBounds, + maxSamples: MAX_CURSOR_SAMPLES, + platform: process.platform, + sampleIntervalMs: CURSOR_SAMPLE_INTERVAL_MS, + sourceId: getSelectedSourceId(), + startTimeMs: + typeof recordingId === "number" && Number.isFinite(recordingId) ? recordingId : undefined, + }); + + try { + await cursorRecordingSession.start(); + } catch (error) { + console.error("Failed to start cursor recording session:", error); + cursorRecordingSession = null; + } +} + +async function stopCursorRecording() { + if (!cursorRecordingSession) { + return; + } + + try { + pendingCursorRecordingData = await cursorRecordingSession.stop(); + } catch (error) { + console.error("Failed to stop cursor recording session:", error); + pendingCursorRecordingData = null; + } finally { + cursorRecordingSession = null; + } +} + +async function writePendingCursorTelemetry(videoPath: string) { + const telemetryPath = `${videoPath}.cursor.json`; + if (pendingCursorRecordingData && pendingCursorRecordingData.samples.length > 0) { + await fs.writeFile(telemetryPath, JSON.stringify(pendingCursorRecordingData, null, 2), "utf-8"); + } + pendingCursorRecordingData = null; +} + +function shiftPendingCursorTelemetry(offsetMs: number) { + if (!pendingCursorRecordingData || !Number.isFinite(offsetMs) || offsetMs <= 0) { + return; + } + + pendingCursorRecordingData = { + ...pendingCursorRecordingData, + samples: pendingCursorRecordingData.samples + .map((sample) => ({ + ...sample, + timeMs: Math.max(0, sample.timeMs - offsetMs), + })) + .sort((a, b) => a.timeMs - b.timeMs), + }; +} + +function compactPendingCursorTelemetryPauseRanges( + ranges: Array<{ startMs: number; endMs: number }>, +) { + if (!pendingCursorRecordingData || ranges.length === 0) { + return; + } + + const normalizedRanges = ranges + .map((range) => ({ + startMs: Math.max(0, Math.min(range.startMs, range.endMs)), + endMs: Math.max(0, Math.max(range.startMs, range.endMs)), + })) + .filter((range) => Number.isFinite(range.startMs) && Number.isFinite(range.endMs)) + .filter((range) => range.endMs > range.startMs) + .sort((a, b) => a.startMs - b.startMs); + + if (normalizedRanges.length === 0) { + return; + } + + pendingCursorRecordingData = { + ...pendingCursorRecordingData, + samples: pendingCursorRecordingData.samples + .map((sample) => { + let pausedBeforeSampleMs = 0; + for (const range of normalizedRanges) { + if (sample.timeMs >= range.startMs && sample.timeMs <= range.endMs) { + return null; + } + if (sample.timeMs > range.endMs) { + pausedBeforeSampleMs += range.endMs - range.startMs; + } + } + + return { + ...sample, + timeMs: Math.max(0, sample.timeMs - pausedBeforeSampleMs), + }; + }) + .filter((sample): sample is CursorRecordingSample => Boolean(sample)) + .sort((a, b) => a.timeMs - b.timeMs), + }; +} + +function completeNativeMacCursorPauseRange(endMs = Date.now()) { + if (nativeMacPauseStartedAtMs === null || nativeMacCursorRecordingStartMs <= 0) { + return; + } + + nativeMacPauseRanges.push({ + startMs: Math.max(0, nativeMacPauseStartedAtMs - nativeMacCursorRecordingStartMs), + endMs: Math.max(0, endMs - nativeMacCursorRecordingStartMs), + }); + nativeMacPauseStartedAtMs = null; +} + +function completeNativeWindowsCursorPauseRange(endMs = Date.now()) { + if (nativeWindowsPauseStartedAtMs === null || nativeWindowsCursorRecordingStartMs <= 0) { + return; + } + + nativeWindowsPauseRanges.push({ + startMs: Math.max(0, nativeWindowsPauseStartedAtMs - nativeWindowsCursorRecordingStartMs), + endMs: Math.max(0, endMs - nativeWindowsCursorRecordingStartMs), + }); + nativeWindowsPauseStartedAtMs = null; +} + +function waitForNativeWindowsCaptureStart(proc: ChildProcessWithoutNullStreams) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + cleanup(); + reject(new Error("Timed out waiting for native Windows capture to start")); + }, 12000); + + const onOutput = (chunk: Buffer) => { + nativeWindowsCaptureOutput += chunk.toString(); + if (nativeWindowsCaptureOutput.includes("Recording started")) { + cleanup(); + resolve(); + } + }; + const onError = (error: Error) => { + cleanup(); + reject(error); + }; + const onExit = (code: number | null) => { + cleanup(); + reject( + new Error( + nativeWindowsCaptureOutput.trim() || + `Native Windows capture exited before recording started (code=${code ?? "unknown"})`, + ), + ); + }; + const cleanup = () => { + clearTimeout(timer); + proc.stdout.off("data", onOutput); + proc.stderr.off("data", onOutput); + proc.off("error", onError); + proc.off("exit", onExit); + }; + + proc.stdout.on("data", onOutput); + proc.stderr.on("data", onOutput); + proc.once("error", onError); + proc.once("exit", onExit); + }); +} + +function waitForNativeWindowsCaptureStop(proc: ChildProcessWithoutNullStreams) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + cleanup(); + if (!proc.killed) { + proc.kill(); + } + reject( + new Error( + `Timed out waiting for native Windows capture to stop. Output path: ${ + nativeWindowsCaptureTargetPath ?? "unknown" + }. Output: ${nativeWindowsCaptureOutput.trim()}`, + ), + ); + }, NATIVE_WINDOWS_CAPTURE_STOP_TIMEOUT_MS); + const onOutput = (chunk: Buffer) => { + nativeWindowsCaptureOutput += chunk.toString(); + }; + const onClose = (code: number | null) => { + cleanup(); + const match = nativeWindowsCaptureOutput.match(/Recording stopped\. Output path: (.+)/); + if (match?.[1]) { + resolve(match[1].trim()); + return; + } + if (code === 0 && nativeWindowsCaptureTargetPath) { + resolve(nativeWindowsCaptureTargetPath); + return; + } + reject( + new Error( + nativeWindowsCaptureOutput.trim() || + `Native Windows capture exited with code=${code ?? "unknown"}`, + ), + ); + }; + const onError = (error: Error) => { + cleanup(); + reject(error); + }; + const cleanup = () => { + clearTimeout(timer); + proc.stdout.off("data", onOutput); + proc.stderr.off("data", onOutput); + proc.off("close", onClose); + proc.off("error", onError); + }; + + proc.stdout.on("data", onOutput); + proc.stderr.on("data", onOutput); + proc.once("close", onClose); + proc.once("error", onError); + }); +} + +function readNativeWindowsWebcamFormat(output: string) { + const lines = output.split(/\r?\n/).filter((line) => line.includes('"event":"webcam-format"')); + const lastLine = lines.at(-1); + if (!lastLine) { + return null; + } + + try { + return JSON.parse(lastLine) as { + width?: number; + height?: number; + fps?: number; + deviceName?: string; + }; + } catch { + return null; + } +} + +function tryParseNativeHelperEvent(line: string) { + try { + const parsed = JSON.parse(line); + return parsed && typeof parsed === "object" ? parsed : null; + } catch { + return null; + } +} + +function inspectNativeMacCaptureOutput() { + for (const line of nativeMacCaptureOutput.split(/\r?\n/)) { + const event = tryParseNativeHelperEvent(line.trim()); + if (event) { + nativeMacCaptureEvents.emit("helper-event", event); + } + } +} + +function attachNativeMacCaptureOutputDrain(proc: ChildProcessWithoutNullStreams) { + let lineBuffer = ""; + const drain = (chunk: Buffer) => { + const text = chunk.toString(); + nativeMacCaptureOutput += text; + lineBuffer += text; + const lines = lineBuffer.split(/\r?\n/); + lineBuffer = lines.pop() ?? ""; + for (const line of lines) { + const event = tryParseNativeHelperEvent(line.trim()); + if (event) { + nativeMacCaptureEvents.emit("helper-event", event); + } + } + }; + const cleanup = () => { + proc.stdout.off("data", drain); + proc.stderr.off("data", drain); + proc.off("close", cleanup); + proc.off("error", cleanup); + }; + + proc.stdout.on("data", drain); + proc.stderr.on("data", drain); + proc.once("close", cleanup); + proc.once("error", cleanup); +} + +function waitForNativeMacCaptureStart(proc: ChildProcessWithoutNullStreams) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + cleanup(); + reject(new Error("Timed out waiting for native macOS capture to start")); + }, 10_000); + + const inspect = (event: Record) => { + if (event.event === "recording-started") { + cleanup(); + resolve(); + return; + } + if (event.event === "error") { + cleanup(); + reject(new Error(String(event.message ?? event.code ?? "Native macOS capture failed"))); + } + }; + + const onOutput = (event: Record) => inspect(event); + const onClose = (code: number | null) => { + cleanup(); + reject( + new Error( + nativeMacCaptureOutput.trim() || + `Native macOS capture exited before recording started (code=${code ?? "unknown"})`, + ), + ); + }; + const onError = (error: Error) => { + cleanup(); + reject(error); + }; + const cleanup = () => { + clearTimeout(timer); + nativeMacCaptureEvents.off("helper-event", onOutput); + proc.off("close", onClose); + proc.off("error", onError); + }; + + nativeMacCaptureEvents.on("helper-event", onOutput); + proc.once("close", onClose); + proc.once("error", onError); + inspectNativeMacCaptureOutput(); + }); +} + +function waitForNativeMacCaptureStop(proc: ChildProcessWithoutNullStreams) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + cleanup(); + reject( + new Error( + `Timed out waiting for native macOS capture to stop. Output path: ${ + nativeMacCaptureTargetPath ?? "unknown" + }. Output: ${nativeMacCaptureOutput.trim()}`, + ), + ); + }, 30_000); + + const inspect = (event: Record) => { + if (event.event === "recording-stopped") { + cleanup(); + resolve(String(event.screenPath ?? nativeMacCaptureTargetPath ?? "")); + return; + } + if (event.event === "error") { + cleanup(); + reject(new Error(String(event.message ?? event.code ?? "Native macOS capture failed"))); + } + }; + + const onOutput = (event: Record) => inspect(event); + const onClose = (code: number | null) => { + if (code === 0 && nativeMacCaptureTargetPath) { + cleanup(); + resolve(nativeMacCaptureTargetPath); + return; + } + cleanup(); + reject( + new Error( + nativeMacCaptureOutput.trim() || + `Native macOS capture exited with code=${code ?? "unknown"}`, + ), + ); + }; + const onError = (error: Error) => { + cleanup(); + reject(error); + }; + const cleanup = () => { + clearTimeout(timer); + nativeMacCaptureEvents.off("helper-event", onOutput); + proc.off("close", onClose); + proc.off("error", onError); + }; + + nativeMacCaptureEvents.on("helper-event", onOutput); + proc.once("close", onClose); + proc.once("error", onError); + inspectNativeMacCaptureOutput(); + }); +} + function setCurrentRecordingSessionState(session: RecordingSession | null) { currentRecordingSession = session; + currentVideoPath = session?.screenVideoPath ?? null; } function getSessionManifestPathForVideo(videoPath: string) { - const parsed = path.parse(videoPath); - const baseName = parsed.name.endsWith("-webcam") - ? parsed.name.slice(0, -"-webcam".length) - : parsed.name; - return path.join(parsed.dir, `${baseName}${RECORDING_SESSION_SUFFIX}`); + const parsedPath = path.parse(videoPath); + const baseName = parsedPath.name.endsWith("-webcam") + ? parsedPath.name.slice(0, -"-webcam".length) + : parsedPath.name; + return path.join(parsedPath.dir, `${baseName}${RECORDING_SESSION_SUFFIX}`); } async function loadRecordedSessionForVideoPath( videoPath: string, ): Promise { - const normalizedVideoPath = normalizeVideoSourcePath(videoPath); - if (!normalizedVideoPath) { - return null; - } - try { - const manifestPath = getSessionManifestPathForVideo(normalizedVideoPath); + const manifestPath = getSessionManifestPathForVideo(videoPath); + if (!isPathAllowed(manifestPath)) { + const parsedVideoPath = path.parse(videoPath); + if (!isPathWithinDir(path.resolve(manifestPath), parsedVideoPath.dir)) { + return null; + } + } + const content = await fs.readFile(manifestPath, "utf-8"); const session = normalizeRecordingSession(JSON.parse(content)); if (!session) { return null; } - const normalizedSession: RecordingSession = { - ...session, - screenVideoPath: normalizeVideoSourcePath(session.screenVideoPath) ?? session.screenVideoPath, - ...(session.webcamVideoPath - ? { - webcamVideoPath: - normalizeVideoSourcePath(session.webcamVideoPath) ?? session.webcamVideoPath, - } - : {}), - }; - - const targetPath = normalizePath(normalizedVideoPath); - const screenMatches = normalizePath(normalizedSession.screenVideoPath) === targetPath; - const webcamMatches = normalizedSession.webcamVideoPath - ? normalizePath(normalizedSession.webcamVideoPath) === targetPath - : false; - - return screenMatches || webcamMatches ? normalizedSession : null; - } catch { - return null; - } -} - -async function storeRecordedSessionFiles(payload: StoreRecordedSessionInput) { - const createdAt = - typeof payload.createdAt === "number" && Number.isFinite(payload.createdAt) - ? payload.createdAt - : Date.now(); - const screenVideoPath = resolveRecordingOutputPath(payload.screen.fileName); - await fs.writeFile(screenVideoPath, Buffer.from(payload.screen.videoData)); - - let webcamVideoPath: string | undefined; - if (payload.webcam) { - webcamVideoPath = resolveRecordingOutputPath(payload.webcam.fileName); - await fs.writeFile(webcamVideoPath, Buffer.from(payload.webcam.videoData)); - } - - const session: RecordingSession = webcamVideoPath - ? { screenVideoPath, webcamVideoPath, createdAt } - : { screenVideoPath, createdAt }; - setCurrentRecordingSessionState(session); - currentProjectPath = null; - - const telemetryPath = `${screenVideoPath}.cursor.json`; - const pendingBatch = cursorTelemetryBuffer.takeNextBatch(); - const pendingClicks = takeCursorClickTimestamps(); - if ((pendingBatch && pendingBatch.samples.length > 0) || pendingClicks.length > 0) { - try { - await fs.writeFile( - telemetryPath, - JSON.stringify( - { - version: CURSOR_TELEMETRY_VERSION, - samples: pendingBatch?.samples ?? [], - clicks: pendingClicks, - }, - null, - 2, - ), - "utf-8", - ); - } catch (err) { - if (pendingBatch) cursorTelemetryBuffer.prependBatch(pendingBatch); - throw err; + const normalizedVideoPath = normalizePath(videoPath); + const matchesScreen = normalizePath(session.screenVideoPath) === normalizedVideoPath; + const matchesWebcam = + typeof session.webcamVideoPath === "string" && + normalizePath(session.webcamVideoPath) === normalizedVideoPath; + if (!matchesScreen && !matchesWebcam) { + return null; } - } - const sessionManifestPath = path.join( - RECORDINGS_DIR, - `${path.parse(payload.screen.fileName).name}${RECORDING_SESSION_SUFFIX}`, - ); - await fs.writeFile(sessionManifestPath, JSON.stringify(session, null, 2), "utf-8"); - - return { - success: true, - path: screenVideoPath, - session, - message: "Recording session stored successfully", - }; -} - -const CURSOR_TELEMETRY_VERSION = 1; -const CURSOR_SAMPLE_INTERVAL_MS = 100; -const MAX_CURSOR_SAMPLES = 60 * 60 * 10; // 1 hour @ 10Hz - -let cursorCaptureInterval: NodeJS.Timeout | null = null; -let cursorCaptureStartTimeMs = 0; -const cursorTelemetryBuffer = createCursorTelemetryBuffer({ - maxActiveSamples: MAX_CURSOR_SAMPLES, -}); - -// Mouse click timestamps (macOS only — uiohook-napi behind Accessibility). -const MAX_CURSOR_CLICKS = 60 * 60 * 60; // ~1 click/sec for an hour -let cursorClickTimestampsMs: number[] = []; -let uioHookInstance: { - start: () => void; - stop: () => void; - on: (...a: unknown[]) => void; - off?: (...a: unknown[]) => void; - removeListener?: (...a: unknown[]) => void; -} | null = null; -let uioHookMouseDownHandler: ((event: { time?: number }) => void) | null = null; -let uioHookFailureLogged = false; - -function clamp(value: number, min: number, max: number) { - return Math.min(max, Math.max(min, value)); -} - -function loadUioHookForClicks(): typeof uioHookInstance { - try { - // Dynamic require + try/catch so a broken native binary doesn't crash startup. - const mod = nodeRequire("uiohook-napi"); - const candidate = mod.uIOhook ?? mod.default?.uIOhook ?? mod.uiohook ?? mod.default; - if (candidate && typeof candidate.start === "function" && typeof candidate.on === "function") { - return candidate; - } - return null; - } catch (error) { - if (!uioHookFailureLogged) { - uioHookFailureLogged = true; - console.warn("[clickCapture] uiohook-napi unavailable:", error); - } - return null; - } -} - -function startClickCapture() { - if (process.platform !== "darwin") return; - if (uioHookInstance) return; - - // Passive check — the prompt fires from the renderer when the user toggles - // "Only on clicks" so it doesn't stack with the screen-recording prompt. - try { - if (!systemPreferences.isTrustedAccessibilityClient(false)) { - if (!uioHookFailureLogged) { - uioHookFailureLogged = true; - console.warn( - "[clickCapture] Accessibility permission not granted — click capture disabled.", - ); + if (!isPathAllowed(session.screenVideoPath)) { + const approvedScreen = await approveReadableVideoPath(session.screenVideoPath, [ + path.dirname(manifestPath), + RECORDINGS_DIR, + ]); + if (!approvedScreen) { + return null; } - return; + session.screenVideoPath = approvedScreen; } - } catch { - // fall through; uiohook will fail defensively below - } - const hook = loadUioHookForClicks(); - if (!hook) return; - - uioHookMouseDownHandler = (event) => { - const elapsed = Math.max(0, Date.now() - cursorCaptureStartTimeMs); - void event; - if (cursorClickTimestampsMs.length >= MAX_CURSOR_CLICKS) return; - cursorClickTimestampsMs.push(elapsed); - }; - - try { - hook.on("mousedown", uioHookMouseDownHandler); - hook.start(); - uioHookInstance = hook; - } catch (error) { - if (!uioHookFailureLogged) { - uioHookFailureLogged = true; - console.warn("[clickCapture] failed to start uiohook:", error); - } - uioHookMouseDownHandler = null; - } -} - -function stopClickCapture() { - if (!uioHookInstance) return; - try { - if (uioHookMouseDownHandler) { - if (typeof uioHookInstance.off === "function") { - uioHookInstance.off("mousedown", uioHookMouseDownHandler); - } else if (typeof uioHookInstance.removeListener === "function") { - uioHookInstance.removeListener("mousedown", uioHookMouseDownHandler); + if (session.webcamVideoPath && !isPathAllowed(session.webcamVideoPath)) { + const approvedWebcam = await approveReadableVideoPath(session.webcamVideoPath, [ + path.dirname(manifestPath), + RECORDINGS_DIR, + ]); + if (!approvedWebcam) { + session.webcamVideoPath = undefined; + } else { + session.webcamVideoPath = approvedWebcam; } } - uioHookInstance.stop(); + + approveFilePath(session.screenVideoPath); + if (session.webcamVideoPath) { + approveFilePath(session.webcamVideoPath); + } + return session; } catch (error) { - console.warn("[clickCapture] failed to stop uiohook:", error); + const nodeError = error as NodeJS.ErrnoException; + if (nodeError.code !== "ENOENT") { + console.error("Failed to restore recording session manifest:", error); + } + return null; } - uioHookInstance = null; - uioHookMouseDownHandler = null; -} - -function takeCursorClickTimestamps(): number[] { - const out = cursorClickTimestampsMs; - cursorClickTimestampsMs = []; - return out; -} - -function stopCursorCapture() { - if (cursorCaptureInterval) { - clearInterval(cursorCaptureInterval); - cursorCaptureInterval = null; - } - stopClickCapture(); -} - -function sampleCursorPoint() { - const cursor = screen.getCursorScreenPoint(); - const sourceDisplayId = Number(selectedSource?.display_id); - const sourceDisplay = Number.isFinite(sourceDisplayId) - ? (screen.getAllDisplays().find((display) => display.id === sourceDisplayId) ?? null) - : null; - const display = sourceDisplay ?? screen.getDisplayNearestPoint(cursor); - const bounds = display.bounds; - const width = Math.max(1, bounds.width); - const height = Math.max(1, bounds.height); - - const cx = clamp((cursor.x - bounds.x) / width, 0, 1); - const cy = clamp((cursor.y - bounds.y) / height, 0, 1); - - cursorTelemetryBuffer.push({ - timeMs: Math.max(0, Date.now() - cursorCaptureStartTimeMs), - cx, - cy, - }); } export function registerIpcHandlers( @@ -483,198 +1239,79 @@ export function registerIpcHandlers( createCountdownOverlayWindow: () => BrowserWindow, getMainWindow: () => BrowserWindow | null, getSourceSelectorWindow: () => BrowserWindow | null, - getCountdownOverlayWindow: () => BrowserWindow | null, + getCountdownOverlayWindow?: () => BrowserWindow | null, onRecordingStateChange?: (recording: boolean, sourceName: string) => void, - switchToHud?: () => void, + _switchToHud?: () => void, ) { - const supportsWindowOpacity = process.platform !== "linux"; - const countdownOverlayState = { - visible: false, - value: null as number | null, - activeRunId: null as number | null, - hideCommitId: 0, - hideCommitTimer: null as ReturnType | null, - }; - const COUNTDOWN_OVERLAY_HIDE_DEBOUNCE_MS = 1200; - - const clearCountdownOverlayHideCommit = () => { - if (countdownOverlayState.hideCommitTimer) { - clearTimeout(countdownOverlayState.hideCommitTimer); - countdownOverlayState.hideCommitTimer = null; - } - }; - - const commitCountdownOverlayHide = (win: BrowserWindow, hideCommitId: number) => { - if (win.isDestroyed()) { - return; + async function requestScreenAccess() { + if (process.platform !== "darwin") { + return { success: true, granted: true, status: "granted" }; } - if (countdownOverlayState.visible || countdownOverlayState.hideCommitId !== hideCommitId) { - return; - } - - win.hide(); - if (supportsWindowOpacity) { - // Reset baseline opacity for the next show cycle. - win.setOpacity(1); - } - }; - - const flushCountdownOverlayState = (win: BrowserWindow) => { - if (win.isDestroyed()) { - return; - } - - clearCountdownOverlayHideCommit(); - win.webContents.send("countdown-overlay-value", countdownOverlayState.value); - if (!countdownOverlayState.visible) { - return; - } - - if (win.isVisible()) { - if (supportsWindowOpacity) { - win.setOpacity(1); - } - return; - } - - setTimeout(() => { - if (!win.isDestroyed() && countdownOverlayState.visible && !win.isVisible()) { - if (supportsWindowOpacity) { - win.setOpacity(0); - } - win.showInactive(); - - if (supportsWindowOpacity) { - setTimeout(() => { - if (!win.isDestroyed() && countdownOverlayState.visible && win.isVisible()) { - win.setOpacity(1); - } - }, 0); - } - } - }, 16); - }; - - ipcMain.handle("countdown-overlay-show", (_, value: number, runId: number) => { - countdownOverlayState.activeRunId = runId; - countdownOverlayState.visible = true; - countdownOverlayState.value = value; - - const win = getCountdownOverlayWindow() ?? createCountdownOverlayWindow(); - if (win.isDestroyed()) { - return; - } - - if (win.webContents.isLoading()) { - win.webContents.once("did-finish-load", () => { - if (!win.isDestroyed()) { - flushCountdownOverlayState(win); - } - }); - } else { - flushCountdownOverlayState(win); - } - }); - - ipcMain.handle("countdown-overlay-set-value", (_, value: number, runId: number) => { - if (countdownOverlayState.activeRunId !== runId || !countdownOverlayState.visible) { - return; - } - - countdownOverlayState.value = value; - - const win = getCountdownOverlayWindow(); - if (!win || win.isDestroyed()) { - return; - } - - if (win.webContents.isLoading()) { - return; - } - - win.webContents.send("countdown-overlay-value", value); - }); - - ipcMain.handle("countdown-overlay-hide", (_, runId: number) => { - if (countdownOverlayState.activeRunId !== runId) { - return; - } - - countdownOverlayState.visible = false; - countdownOverlayState.hideCommitId += 1; - const hideCommitId = countdownOverlayState.hideCommitId; - clearCountdownOverlayHideCommit(); - - const win = getCountdownOverlayWindow(); - if (!win || win.isDestroyed()) { - countdownOverlayState.value = null; - return; - } - - if (supportsWindowOpacity) { - // Hide visually immediately to avoid hide/show compositor flashes on rapid restart. - win.setOpacity(0); - } - - countdownOverlayState.value = null; - if (!win.webContents.isLoading()) { - win.webContents.send("countdown-overlay-value", countdownOverlayState.value); - } - - if (!supportsWindowOpacity) { - win.hide(); - return; - } - - countdownOverlayState.hideCommitTimer = setTimeout(() => { - countdownOverlayState.hideCommitTimer = null; - commitCountdownOverlayHide(win, hideCommitId); - }, COUNTDOWN_OVERLAY_HIDE_DEBOUNCE_MS); - }); - - ipcMain.handle("switch-to-hud", () => { - if (switchToHud) switchToHud(); - }); - ipcMain.handle("start-new-recording", () => { try { - setCurrentRecordingSessionState(null); - if (switchToHud) { - switchToHud(); + const status = systemPreferences.getMediaAccessStatus("screen"); + if (status === "granted") { + return { success: true, granted: true, status }; } - return { success: true }; + + // Screen recording has no askForMediaAccess equivalent. Trigger the + // TCC prompt without opening OpenScreen's source selector above it. + if (status === "not-determined") { + const mainWin = getMainWindow(); + if (mainWin && !mainWin.isDestroyed()) { + if (!mainWin.isVisible()) { + mainWin.show(); + } + mainWin.focus(); + } + app.focus({ steal: true }); + desktopCapturer + .getSources({ types: ["screen"], thumbnailSize: { width: 1, height: 1 } }) + .catch(() => { + // Permission probing failure is reported by the explicit status check below. + }); + return { success: true, granted: false, status: "not-determined" }; + } + + return { success: true, granted: false, status }; } catch (error) { - console.error("Failed to start new recording:", error); - return { success: false, error: String(error) }; + console.error("Failed to request screen access:", error); + return { success: false, granted: false, status: "unknown", error: String(error) }; } - }); + } ipcMain.handle("get-sources", async (_, opts) => { - const ownWindowSourceIds = new Set( - BrowserWindow.getAllWindows() - .map((win) => { - try { - return win.getMediaSourceId(); - } catch { - return null; - } - }) - .filter((id): id is string => Boolean(id)), - ); const sources = await desktopCapturer.getSources(opts); - return sources - .filter((source) => !ownWindowSourceIds.has(source.id)) - .map((source) => ({ - id: source.id, - name: source.name, - display_id: source.display_id, - thumbnail: source.thumbnail ? source.thumbnail.toDataURL() : null, - appIcon: source.appIcon ? source.appIcon.toDataURL() : null, - })); + lastEnumeratedSources = new Map(sources.map((source) => [source.id, source])); + return sources.map((source) => ({ + id: source.id, + name: source.name, + display_id: source.display_id, + thumbnail: source.thumbnail ? source.thumbnail.toDataURL() : null, + appIcon: source.appIcon ? source.appIcon.toDataURL() : null, + })); }); - ipcMain.handle("select-source", (_, source: SelectedSource) => { + ipcMain.handle("select-source", async (_, source: SelectedSource) => { selectedSource = source; + // Reuse the exact source object returned during enumeration to avoid + // Windows window-source id mismatches across separate getSources() calls. + selectedDesktopSource = + typeof source.id === "string" ? (lastEnumeratedSources.get(source.id) ?? null) : null; + + if (!selectedDesktopSource && typeof source.id === "string") { + try { + const sources = await desktopCapturer.getSources({ + types: ["screen", "window"], + thumbnailSize: { width: 0, height: 0 }, + fetchWindowIcons: true, + }); + lastEnumeratedSources = new Map(sources.map((candidate) => [candidate.id, candidate])); + selectedDesktopSource = lastEnumeratedSources.get(source.id) ?? null; + } catch { + selectedDesktopSource = null; + } + } const sourceSelectorWin = getSourceSelectorWindow(); if (sourceSelectorWin) { sourceSelectorWin.close(); @@ -719,56 +1356,51 @@ export function registerIpcHandlers( }); ipcMain.handle("request-screen-access", async () => { - if (process.platform !== "darwin") { - return { success: true, granted: true, status: "granted" }; - } - - try { - const status = systemPreferences.getMediaAccessStatus("screen"); - if (status === "granted") { - return { success: true, granted: true, status }; - } - - // Screen recording has no askForMediaAccess equivalent — the TCC prompt - // is triggered by desktopCapturer.getSources(). Fire it and return so - // the renderer can re-check status after the user responds. - if (status === "not-determined") { - desktopCapturer.getSources({ types: ["screen"] }).catch(() => { - // Ignore prompt trigger failures; the renderer will re-check status. - }); - return { success: true, granted: false, status: "not-determined" }; - } - - return { success: true, granted: false, status }; - } catch (error) { - console.error("Failed to request screen access:", error); - return { success: false, granted: false, status: "unknown", error: String(error) }; - } + return requestScreenAccess(); }); - // macOS Accessibility prompt for global click capture. First call shows the - // system dialog; the user has to toggle the app in System Settings (no - // programmatic grant exists for Accessibility). - ipcMain.handle("request-accessibility-access", () => { - if (process.platform !== "darwin") { - return { success: true, granted: true }; - } - try { - const granted = systemPreferences.isTrustedAccessibilityClient(true); - return { success: true, granted }; - } catch (error) { - console.error("Failed to request accessibility access:", error); - return { success: false, granted: false, error: String(error) }; - } + ipcMain.handle("request-native-mac-cursor-access", async () => { + return requestMacCursorAccessibilityAccess(); }); - ipcMain.handle("open-source-selector", () => { + ipcMain.handle("open-source-selector", async () => { + const access = await requestScreenAccess(); + if (!access.granted) { + if (process.platform === "darwin" && access.status !== "not-determined") { + const mainWin = getMainWindow(); + const messageOptions = { + type: "warning", + buttons: ["Open System Settings", "Cancel"], + defaultId: 0, + cancelId: 1, + message: "Screen Recording permission is required", + detail: + "Allow OpenScreen in macOS System Settings, then come back and choose a screen or window.", + } satisfies Electron.MessageBoxOptions; + const result = + mainWin && !mainWin.isDestroyed() + ? await dialog.showMessageBox(mainWin, messageOptions) + : await dialog.showMessageBox(messageOptions); + if (result.response === 0) { + await shell.openExternal( + "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture", + ); + } + } + return { + opened: false, + reason: "screen-access-required", + access, + }; + } + const sourceSelectorWin = getSourceSelectorWindow(); if (sourceSelectorWin) { sourceSelectorWin.focus(); - return; + return { opened: true }; } createSourceSelectorWindow(); + return { opened: true }; }); ipcMain.handle("switch-to-editor", () => { @@ -779,6 +1411,736 @@ export function registerIpcHandlers( createEditorWindow(); }); + ipcMain.handle("switch-to-hud", () => { + _switchToHud?.(); + return { success: true }; + }); + + ipcMain.handle("start-new-recording", () => { + _switchToHud?.(); + return { success: true }; + }); + + ipcMain.handle("countdown-overlay-show", async (_, value: number, runId: number) => { + const overlayWindow = getCountdownOverlayWindow?.() ?? createCountdownOverlayWindow(); + if (overlayWindow.isDestroyed()) { + return; + } + + if (!overlayWindow.isVisible()) { + overlayWindow.showInactive(); + } + + if (overlayWindow.webContents.isLoading()) { + await new Promise((resolve) => { + overlayWindow.webContents.once("did-finish-load", () => resolve()); + }); + } + + overlayWindow.webContents.send("countdown-overlay-value", value, runId); + }); + + ipcMain.handle("countdown-overlay-set-value", (_, value: number, runId: number) => { + const overlayWindow = getCountdownOverlayWindow?.(); + if (!overlayWindow || overlayWindow.isDestroyed()) { + return; + } + + overlayWindow.webContents.send("countdown-overlay-value", value, runId); + }); + + ipcMain.handle("countdown-overlay-hide", (_, runId: number) => { + const overlayWindow = getCountdownOverlayWindow?.(); + if (!overlayWindow || overlayWindow.isDestroyed()) { + return; + } + + overlayWindow.webContents.send("countdown-overlay-value", null, runId); + overlayWindow.hide(); + }); + + ipcMain.handle("is-native-windows-capture-available", async () => { + if (!isWindowsGraphicsCaptureOsSupported()) { + return { success: true, available: false, reason: "unsupported-os" }; + } + + const helperPath = await findNativeWindowsCaptureHelperPath(); + return helperPath + ? { success: true, available: true, helperPath } + : { success: true, available: false, reason: "missing-helper" }; + }); + + ipcMain.handle("is-native-mac-capture-available", async () => { + if (process.platform !== "darwin") { + return { success: true, available: false, reason: "unsupported-platform" }; + } + + const helperPath = await findNativeMacCaptureHelperPath(); + return helperPath + ? { success: true, available: true, helperPath } + : { success: true, available: false, reason: "missing-helper" }; + }); + + ipcMain.handle( + "start-native-windows-recording", + async (_, request: NativeWindowsRecordingRequest) => { + try { + if (!isWindowsGraphicsCaptureOsSupported()) { + return { + success: false, + error: "Windows Graphics Capture requires Windows 10 build 19041 or newer.", + }; + } + if (nativeWindowsCaptureProcess) { + return { success: false, error: "Native Windows capture is already running." }; + } + + const helperPath = await findNativeWindowsCaptureHelperPath(); + if (!helperPath) { + return { success: false, error: "Native Windows capture helper is not available." }; + } + + if (!request?.source?.sourceId) { + return { + success: false, + error: "Native Windows capture request is missing a source.", + }; + } + + const recordingId = + typeof request.recordingId === "number" && Number.isFinite(request.recordingId) + ? request.recordingId + : Date.now(); + const outputPath = path.join(RECORDINGS_DIR, `${RECORDING_FILE_PREFIX}${recordingId}.mp4`); + const webcamOutputPath = path.join( + RECORDINGS_DIR, + `${RECORDING_FILE_PREFIX}${recordingId}-webcam.mp4`, + ); + const sourceDisplay = + request.source.type === "display" && typeof request.source.displayId === "number" + ? (screen.getAllDisplays().find((display) => display.id === request.source.displayId) ?? + null) + : getSelectedDisplay(); + const bounds = sourceDisplay?.bounds ?? getSelectedSourceBounds(); + const displayId = + typeof request.source.displayId === "number" && Number.isFinite(request.source.displayId) + ? request.source.displayId + : Number(selectedSource?.display_id); + const webcamDirectShowClsid = request.webcam.enabled + ? await resolveDirectShowWebcamClsid(request.webcam.deviceName) + : null; + const cursorCaptureMode = + normalizeCursorCaptureMode(request.cursor?.mode) ?? "editable-overlay"; + const config = { + schemaVersion: 2, + recordingId, + outputPath, + sourceType: request.source.type, + sourceId: request.source.sourceId, + displayId: Number.isFinite(displayId) ? displayId : 0, + windowHandle: request.source.windowHandle ?? null, + fps: request.video.fps, + videoWidth: request.video.width, + videoHeight: request.video.height, + displayX: bounds.x, + displayY: bounds.y, + displayW: bounds.width, + displayH: bounds.height, + hasDisplayBounds: true, + captureSystemAudio: request.audio.system.enabled, + captureMic: request.audio.microphone.enabled, + microphoneDeviceId: request.audio.microphone.deviceId ?? null, + microphoneDeviceName: request.audio.microphone.deviceName ?? null, + microphoneGain: request.audio.microphone.gain, + webcamEnabled: request.webcam.enabled, + webcamDeviceId: request.webcam.deviceId ?? null, + webcamDeviceName: request.webcam.deviceName ?? null, + webcamDirectShowClsid, + webcamWidth: request.webcam.width, + webcamHeight: request.webcam.height, + webcamFps: request.webcam.fps, + captureCursor: cursorCaptureMode === "system", + cursorCaptureMode, + outputs: { + screenPath: outputPath, + webcamPath: webcamOutputPath, + }, + source: { + type: request.source.type, + sourceId: request.source.sourceId, + displayId: Number.isFinite(displayId) ? displayId : null, + windowHandle: request.source.windowHandle ?? null, + bounds, + }, + video: request.video, + audio: request.audio, + webcam: request.webcam, + cursor: { + mode: cursorCaptureMode, + }, + }; + + console.info("[native-wgc] starting Windows capture", { + helperPath, + source: request.source, + audio: request.audio, + webcam: request.webcam, + cursor: { mode: cursorCaptureMode }, + bounds, + sourceId: selectedSource?.id ?? null, + usedDisplayMatch: Boolean(sourceDisplay), + outputPath, + }); + + await fs.mkdir(RECORDINGS_DIR, { recursive: true }); + nativeWindowsCaptureOutput = ""; + nativeWindowsCaptureTargetPath = outputPath; + nativeWindowsCaptureWebcamTargetPath = request.webcam.enabled ? webcamOutputPath : null; + nativeWindowsCaptureRecordingId = recordingId; + nativeWindowsCursorOffsetMs = 0; + nativeWindowsCursorCaptureMode = cursorCaptureMode; + nativeWindowsCursorRecordingStartMs = 0; + nativeWindowsPauseStartedAtMs = null; + nativeWindowsPauseRanges = []; + nativeWindowsIsPaused = false; + + const cursorStartTimeMs = Date.now(); + if (cursorCaptureMode === "editable-overlay") { + nativeWindowsCursorRecordingStartMs = cursorStartTimeMs; + await startCursorRecording(cursorStartTimeMs); + console.info("[native-wgc] cursor sampler ready", { + cursorStartTimeMs, + warmupMs: Date.now() - cursorStartTimeMs, + }); + } else { + pendingCursorRecordingData = null; + } + + const proc = spawn(helperPath, [JSON.stringify(config)], { + cwd: RECORDINGS_DIR, + stdio: ["pipe", "pipe", "pipe"], + windowsHide: true, + }); + nativeWindowsCaptureProcess = proc; + + await waitForNativeWindowsCaptureStart(proc); + const captureStartedAtMs = Date.now(); + nativeWindowsCursorOffsetMs = + cursorCaptureMode === "editable-overlay" + ? Math.max(0, captureStartedAtMs - cursorStartTimeMs) + : 0; + const webcamFormat = readNativeWindowsWebcamFormat(nativeWindowsCaptureOutput); + console.info("[native-wgc] capture started", { + captureStartedAtMs, + cursorOffsetMs: nativeWindowsCursorOffsetMs, + webcamFormat, + }); + + const source = selectedSource || { name: "Screen" }; + if (onRecordingStateChange) { + onRecordingStateChange(true, source.name); + } + + return { + success: true, + recordingId, + path: outputPath, + helperPath, + }; + } catch (error) { + console.error("Failed to start native Windows recording:", error); + nativeWindowsCaptureProcess?.kill(); + nativeWindowsCaptureProcess = null; + nativeWindowsCaptureTargetPath = null; + nativeWindowsCaptureWebcamTargetPath = null; + nativeWindowsCaptureRecordingId = null; + nativeWindowsCursorOffsetMs = 0; + nativeWindowsCursorCaptureMode = "editable-overlay"; + nativeWindowsCursorRecordingStartMs = 0; + nativeWindowsPauseStartedAtMs = null; + nativeWindowsPauseRanges = []; + nativeWindowsIsPaused = false; + await stopCursorRecording(); + return { success: false, error: String(error) }; + } + }, + ); + + ipcMain.handle("start-native-mac-recording", async (_, request: NativeMacRecordingRequest) => { + try { + if (process.platform !== "darwin") { + return { success: false, error: "Native macOS capture requires macOS." }; + } + if (nativeMacCaptureProcess) { + return { success: false, error: "Native macOS capture is already running." }; + } + + const helperPath = await findNativeMacCaptureHelperPath(); + if (!helperPath) { + return { success: false, error: "Native macOS capture helper is not available." }; + } + + if (!request?.source?.sourceId) { + return { success: false, error: "Native macOS capture request is missing a source." }; + } + + const recordingId = + typeof request.recordingId === "number" && Number.isFinite(request.recordingId) + ? request.recordingId + : Date.now(); + const outputPath = path.join(RECORDINGS_DIR, `${RECORDING_FILE_PREFIX}${recordingId}.mp4`); + const cursorCaptureMode = + normalizeCursorCaptureMode(request.cursor?.mode) ?? "editable-overlay"; + try { + await desktopCapturer.getSources({ + types: ["screen"], + thumbnailSize: { width: 1, height: 1 }, + }); + } catch { + // The helper reports the final ScreenCaptureKit permission status. + } + if (request.audio?.microphone?.enabled) { + const micStatus = systemPreferences.getMediaAccessStatus("microphone"); + if (micStatus !== "granted") { + await systemPreferences.askForMediaAccess("microphone"); + } + } + const sourceDisplay = + request.source.type === "display" && typeof request.source.displayId === "number" + ? (screen.getAllDisplays().find((display) => display.id === request.source.displayId) ?? + null) + : getSelectedDisplay(); + const bounds = request.source.bounds ?? sourceDisplay?.bounds ?? getSelectedSourceBounds(); + const config: NativeMacRecordingRequest = { + ...request, + schemaVersion: 1, + recordingId, + source: { + ...request.source, + bounds, + }, + video: { + ...request.video, + hideSystemCursor: cursorCaptureMode === "editable-overlay", + }, + webcam: { + ...request.webcam, + enabled: false, + }, + cursor: { + mode: cursorCaptureMode, + }, + outputs: { + screenPath: outputPath, + manifestPath: path.join( + RECORDINGS_DIR, + `${RECORDING_FILE_PREFIX}${recordingId}${RECORDING_SESSION_SUFFIX}`, + ), + }, + }; + + console.info("[native-sck] starting macOS capture", { + helperPath, + source: config.source, + audio: config.audio, + webcam: config.webcam, + cursor: config.cursor, + outputPath, + }); + + await fs.mkdir(RECORDINGS_DIR, { recursive: true }); + nativeMacCaptureOutput = ""; + nativeMacCaptureTargetPath = outputPath; + nativeMacCaptureRecordingId = recordingId; + nativeMacCursorOffsetMs = 0; + nativeMacCursorCaptureMode = cursorCaptureMode; + nativeMacCursorRecordingStartMs = 0; + nativeMacPauseStartedAtMs = null; + nativeMacPauseRanges = []; + nativeMacIsPaused = false; + + const cursorStartTimeMs = Date.now(); + if (cursorCaptureMode === "editable-overlay") { + nativeMacCursorRecordingStartMs = cursorStartTimeMs; + await startCursorRecording(cursorStartTimeMs); + } else { + pendingCursorRecordingData = null; + } + + const proc = spawn(helperPath, [JSON.stringify(config)], { + cwd: RECORDINGS_DIR, + stdio: ["pipe", "pipe", "pipe"], + }); + nativeMacCaptureProcess = proc; + attachNativeMacCaptureOutputDrain(proc); + + await waitForNativeMacCaptureStart(proc); + const captureStartedAtMs = Date.now(); + nativeMacCursorOffsetMs = + cursorCaptureMode === "editable-overlay" + ? Math.max(0, captureStartedAtMs - cursorStartTimeMs) + : 0; + + const source = selectedSource || { name: "Screen" }; + if (onRecordingStateChange) { + onRecordingStateChange(true, source.name); + } + + return { + success: true, + recordingId, + path: outputPath, + helperPath, + }; + } catch (error) { + console.error("Failed to start native macOS recording:", error); + nativeMacCaptureProcess?.kill(); + nativeMacCaptureProcess = null; + nativeMacCaptureTargetPath = null; + nativeMacCaptureRecordingId = null; + nativeMacCursorOffsetMs = 0; + nativeMacCursorCaptureMode = "editable-overlay"; + nativeMacCursorRecordingStartMs = 0; + nativeMacPauseStartedAtMs = null; + nativeMacPauseRanges = []; + nativeMacIsPaused = false; + await stopCursorRecording(); + return { success: false, error: error instanceof Error ? error.message : String(error) }; + } + }); + + ipcMain.handle("pause-native-mac-recording", async () => { + if (process.platform !== "darwin") { + return { success: false, error: "Native macOS capture requires macOS." }; + } + + const proc = nativeMacCaptureProcess; + if (!proc) { + return { success: false, error: "Native macOS capture is not running." }; + } + if (nativeMacIsPaused) { + return { success: true }; + } + if (!proc.stdin.writable) { + return { success: false, error: "Native macOS capture command channel is closed." }; + } + + try { + proc.stdin.write("pause\n"); + nativeMacIsPaused = true; + nativeMacPauseStartedAtMs = Date.now(); + return { success: true }; + } catch (error) { + return { success: false, error: error instanceof Error ? error.message : String(error) }; + } + }); + + ipcMain.handle("resume-native-mac-recording", async () => { + if (process.platform !== "darwin") { + return { success: false, error: "Native macOS capture requires macOS." }; + } + + const proc = nativeMacCaptureProcess; + if (!proc) { + return { success: false, error: "Native macOS capture is not running." }; + } + if (!nativeMacIsPaused) { + return { success: true }; + } + if (!proc.stdin.writable) { + return { success: false, error: "Native macOS capture command channel is closed." }; + } + + try { + proc.stdin.write("resume\n"); + completeNativeMacCursorPauseRange(); + nativeMacIsPaused = false; + return { success: true }; + } catch (error) { + return { success: false, error: error instanceof Error ? error.message : String(error) }; + } + }); + + ipcMain.handle("pause-native-windows-recording", async () => { + const proc = nativeWindowsCaptureProcess; + if (!proc) { + return { success: false, error: "Native Windows capture is not running." }; + } + if (nativeWindowsIsPaused) { + return { success: true }; + } + if (!proc.stdin.writable) { + return { success: false, error: "Native Windows capture command channel is closed." }; + } + + try { + proc.stdin.write("pause\n"); + nativeWindowsIsPaused = true; + nativeWindowsPauseStartedAtMs = Date.now(); + return { success: true }; + } catch (error) { + return { success: false, error: error instanceof Error ? error.message : String(error) }; + } + }); + + ipcMain.handle("resume-native-windows-recording", async () => { + const proc = nativeWindowsCaptureProcess; + if (!proc) { + return { success: false, error: "Native Windows capture is not running." }; + } + if (!nativeWindowsIsPaused) { + return { success: true }; + } + if (!proc.stdin.writable) { + return { success: false, error: "Native Windows capture command channel is closed." }; + } + + try { + proc.stdin.write("resume\n"); + completeNativeWindowsCursorPauseRange(); + nativeWindowsIsPaused = false; + return { success: true }; + } catch (error) { + return { success: false, error: error instanceof Error ? error.message : String(error) }; + } + }); + + ipcMain.handle("stop-native-windows-recording", async (_, discard?: boolean) => { + const proc = nativeWindowsCaptureProcess; + const preferredPath = nativeWindowsCaptureTargetPath; + const preferredWebcamPath = nativeWindowsCaptureWebcamTargetPath; + const recordingId = nativeWindowsCaptureRecordingId ?? Date.now(); + const cursorCaptureMode = nativeWindowsCursorCaptureMode; + + if (!proc) { + return { success: false, error: "Native Windows capture is not running." }; + } + + try { + completeNativeWindowsCursorPauseRange(); + const stoppedPathPromise = waitForNativeWindowsCaptureStop(proc); + proc.stdin.write("stop\n"); + const stoppedPath = await stoppedPathPromise; + const screenVideoPath = stoppedPath || preferredPath; + if (!screenVideoPath) { + throw new Error("Native Windows capture did not return an output path."); + } + + if (cursorCaptureMode === "editable-overlay") { + await stopCursorRecording(); + } else { + pendingCursorRecordingData = null; + } + if (discard) { + pendingCursorRecordingData = null; + await Promise.all([ + fs.rm(screenVideoPath, { force: true }), + preferredWebcamPath ? fs.rm(preferredWebcamPath, { force: true }) : Promise.resolve(), + fs.rm(`${screenVideoPath}.cursor.json`, { force: true }), + ]); + return { success: true, discarded: true }; + } + + if (cursorCaptureMode === "editable-overlay") { + compactPendingCursorTelemetryPauseRanges(nativeWindowsPauseRanges); + shiftPendingCursorTelemetry(nativeWindowsCursorOffsetMs); + await writePendingCursorTelemetry(screenVideoPath); + } + let webcamVideoPath: string | undefined; + if (preferredWebcamPath) { + try { + await fs.access(preferredWebcamPath, fsConstants.R_OK); + webcamVideoPath = preferredWebcamPath; + } catch { + webcamVideoPath = undefined; + } + } + const session: RecordingSession = webcamVideoPath + ? { screenVideoPath, webcamVideoPath, createdAt: recordingId, cursorCaptureMode } + : { screenVideoPath, createdAt: recordingId, cursorCaptureMode }; + setCurrentRecordingSessionState(session); + currentProjectPath = null; + + const sessionManifestPath = path.join( + RECORDINGS_DIR, + `${path.parse(screenVideoPath).name}${RECORDING_SESSION_SUFFIX}`, + ); + await fs.writeFile(sessionManifestPath, JSON.stringify(session, null, 2), "utf-8"); + + return { + success: true, + path: screenVideoPath, + session, + message: "Native Windows recording session stored successfully", + }; + } catch (error) { + console.error("Failed to stop native Windows recording:", error); + await stopCursorRecording(); + return { success: false, error: String(error) }; + } finally { + nativeWindowsCaptureProcess = null; + nativeWindowsCaptureTargetPath = null; + nativeWindowsCaptureWebcamTargetPath = null; + nativeWindowsCaptureRecordingId = null; + nativeWindowsCursorOffsetMs = 0; + nativeWindowsCursorCaptureMode = "editable-overlay"; + nativeWindowsCursorRecordingStartMs = 0; + nativeWindowsPauseStartedAtMs = null; + nativeWindowsPauseRanges = []; + nativeWindowsIsPaused = false; + const source = selectedSource || { name: "Screen" }; + if (onRecordingStateChange) { + onRecordingStateChange(false, source.name); + } + } + }); + + ipcMain.handle("stop-native-mac-recording", async (_, discard?: boolean) => { + if (process.platform !== "darwin") { + return { success: false, error: "Native macOS capture requires macOS." }; + } + + const proc = nativeMacCaptureProcess; + const preferredPath = nativeMacCaptureTargetPath; + const recordingId = nativeMacCaptureRecordingId ?? Date.now(); + const cursorCaptureMode = nativeMacCursorCaptureMode; + + if (!proc) { + return { success: false, error: "Native macOS capture is not running." }; + } + + try { + completeNativeMacCursorPauseRange(); + const stoppedPathPromise = waitForNativeMacCaptureStop(proc); + proc.stdin.write("stop\n"); + const stoppedPath = await stoppedPathPromise; + const screenVideoPath = stoppedPath || preferredPath; + if (!screenVideoPath) { + throw new Error("Native macOS capture did not return an output path."); + } + + if (cursorCaptureMode === "editable-overlay") { + await stopCursorRecording(); + } else { + pendingCursorRecordingData = null; + } + if (discard) { + pendingCursorRecordingData = null; + await Promise.all([ + fs.rm(screenVideoPath, { force: true }), + fs.rm(`${screenVideoPath}.cursor.json`, { force: true }), + ]); + return { success: true, discarded: true }; + } + + if (cursorCaptureMode === "editable-overlay") { + compactPendingCursorTelemetryPauseRanges(nativeMacPauseRanges); + shiftPendingCursorTelemetry(nativeMacCursorOffsetMs); + await writePendingCursorTelemetry(screenVideoPath); + } + + const session: RecordingSession = { + screenVideoPath, + createdAt: recordingId, + cursorCaptureMode, + }; + setCurrentRecordingSessionState(session); + currentProjectPath = null; + + const sessionManifestPath = path.join( + RECORDINGS_DIR, + `${path.parse(screenVideoPath).name}${RECORDING_SESSION_SUFFIX}`, + ); + await fs.writeFile(sessionManifestPath, JSON.stringify(session, null, 2), "utf-8"); + + return { + success: true, + path: screenVideoPath, + session, + message: "Native macOS recording session stored successfully", + }; + } catch (error) { + console.error("Failed to stop native macOS recording:", error); + await stopCursorRecording(); + return { success: false, error: error instanceof Error ? error.message : String(error) }; + } finally { + nativeMacCaptureProcess = null; + nativeMacCaptureTargetPath = null; + nativeMacCaptureRecordingId = null; + nativeMacCursorOffsetMs = 0; + nativeMacCursorCaptureMode = "editable-overlay"; + nativeMacCursorRecordingStartMs = 0; + nativeMacPauseStartedAtMs = null; + nativeMacPauseRanges = []; + nativeMacIsPaused = false; + const source = selectedSource || { name: "Screen" }; + if (onRecordingStateChange) { + onRecordingStateChange(false, source.name); + } + } + }); + + ipcMain.handle( + "attach-native-mac-webcam-recording", + async (_, payload: AttachNativeMacWebcamRecordingInput) => { + try { + if (process.platform !== "darwin") { + return { success: false, error: "Native macOS webcam attachment requires macOS." }; + } + + const screenVideoPath = normalizeVideoSourcePath(payload.screenVideoPath); + if (!screenVideoPath || !isPathWithinDir(screenVideoPath, RECORDINGS_DIR)) { + return { + success: false, + error: "Native macOS webcam attachment requires a recording output path.", + }; + } + + await fs.access(screenVideoPath, fsConstants.R_OK); + + if (!payload.webcam?.fileName || !payload.webcam.videoData) { + return { success: false, error: "Native macOS webcam attachment is missing video data." }; + } + + const webcamVideoPath = resolveRecordingOutputPath(payload.webcam.fileName); + await fs.writeFile(webcamVideoPath, Buffer.from(payload.webcam.videoData)); + + const createdAt = + typeof payload.recordingId === "number" && Number.isFinite(payload.recordingId) + ? payload.recordingId + : Date.now(); + const cursorCaptureMode = normalizeCursorCaptureMode(payload.cursorCaptureMode); + const session: RecordingSession = { + screenVideoPath, + webcamVideoPath, + createdAt, + ...(cursorCaptureMode ? { cursorCaptureMode } : {}), + }; + setCurrentRecordingSessionState(session); + currentProjectPath = null; + + const sessionManifestPath = path.join( + RECORDINGS_DIR, + `${path.parse(screenVideoPath).name}${RECORDING_SESSION_SUFFIX}`, + ); + await fs.writeFile(sessionManifestPath, JSON.stringify(session, null, 2), "utf-8"); + + return { + success: true, + path: screenVideoPath, + session, + message: "Native macOS webcam recording attached successfully", + }; + } catch (error) { + console.error("Failed to attach native macOS webcam recording:", error); + return { + success: false, + error: error instanceof Error ? error.message : String(error), + }; + } + }, + ); + ipcMain.handle("store-recorded-session", async (_, payload: StoreRecordedSessionInput) => { try { return await storeRecordedSessionFiles(payload); @@ -792,6 +2154,48 @@ export function registerIpcHandlers( } }); + async function storeRecordedSessionFiles(payload: StoreRecordedSessionInput) { + const createdAt = + typeof payload.createdAt === "number" && Number.isFinite(payload.createdAt) + ? payload.createdAt + : Date.now(); + const cursorCaptureMode = normalizeCursorCaptureMode(payload.cursorCaptureMode); + const screenVideoPath = resolveRecordingOutputPath(payload.screen.fileName); + await fs.writeFile(screenVideoPath, Buffer.from(payload.screen.videoData)); + + let webcamVideoPath: string | undefined; + if (payload.webcam) { + webcamVideoPath = resolveRecordingOutputPath(payload.webcam.fileName); + await fs.writeFile(webcamVideoPath, Buffer.from(payload.webcam.videoData)); + } + + const session: RecordingSession = webcamVideoPath + ? { + screenVideoPath, + webcamVideoPath, + createdAt, + ...(cursorCaptureMode ? { cursorCaptureMode } : {}), + } + : { screenVideoPath, createdAt, ...(cursorCaptureMode ? { cursorCaptureMode } : {}) }; + setCurrentRecordingSessionState(session); + currentProjectPath = null; + + await writePendingCursorTelemetry(screenVideoPath); + + const sessionManifestPath = path.join( + RECORDINGS_DIR, + `${path.parse(payload.screen.fileName).name}${RECORDING_SESSION_SUFFIX}`, + ); + await fs.writeFile(sessionManifestPath, JSON.stringify(session, null, 2), "utf-8"); + + return { + success: true, + path: screenVideoPath, + session, + message: "Recording session stored successfully", + }; + } + ipcMain.handle("store-recorded-video", async (_, videoData: ArrayBuffer, fileName: string) => { try { return await storeRecordedSessionFiles({ @@ -823,24 +2227,7 @@ export function registerIpcHandlers( return { success: false, message: "No recorded video found" }; } - // Sort by most recently modified to reliably get the latest recording. - // Lexicographic sort is unreliable (e.g. recording-9.webm > recording-10.webm). - let latestVideo: string | null = null; - let latestMtimeMs = 0; - for (const file of videoFiles) { - try { - const stat = await fs.stat(path.join(RECORDINGS_DIR, file)); - if (stat.mtimeMs > latestMtimeMs) { - latestMtimeMs = stat.mtimeMs; - latestVideo = file; - } - } catch { - // Skip inaccessible files. - } - } - if (!latestVideo) { - return { success: false, message: "No recorded video found" }; - } + const latestVideo = videoFiles.sort().reverse()[0]; const videoPath = path.join(RECORDINGS_DIR, latestVideo); return { success: true, path: videoPath }; @@ -850,153 +2237,38 @@ export function registerIpcHandlers( } }); - ipcMain.handle("read-binary-file", async (_, inputPath: string) => { - try { - const normalizedPath = normalizeVideoSourcePath(inputPath); - if (!normalizedPath) { - return { success: false, message: "Invalid file path" }; + ipcMain.handle( + "set-recording-state", + async (_, recording: boolean, recordingId?: number, cursorCaptureMode?: CursorCaptureMode) => { + const normalizedCursorCaptureMode = + normalizeCursorCaptureMode(cursorCaptureMode) ?? "editable-overlay"; + if (recording && normalizedCursorCaptureMode === "editable-overlay") { + await startCursorRecording(recordingId); + } else { + await stopCursorRecording(); } - if (!isPathAllowed(normalizedPath)) { - console.warn( - "[read-binary-file] Rejected path outside allowed directories:", - normalizedPath, - ); - return { success: false, message: "Access denied: path outside allowed directories" }; + const source = selectedSource || { name: "Screen" }; + if (onRecordingStateChange) { + onRecordingStateChange(recording, source.name); } - - const data = await fs.readFile(normalizedPath); - return { - success: true, - data: data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength), - path: normalizedPath, - }; - } catch (error) { - console.error("Failed to read binary file:", error); - return { - success: false, - message: "Failed to read binary file", - error: String(error), - }; - } - }); - - ipcMain.handle("set-recording-state", (_, recording: boolean, recordingId?: number) => { - if (recording) { - stopCursorCapture(); - // The renderer is the source of truth for the recording id (it - // uses the same id as the saved fileName). Fall back to a - // timestamp only if the renderer didn't supply one, so the - // buffer always has a stable key per session. - const id = typeof recordingId === "number" ? recordingId : Date.now(); - cursorTelemetryBuffer.startSession(id); - cursorCaptureStartTimeMs = Date.now(); - cursorClickTimestampsMs = []; - startClickCapture(); - sampleCursorPoint(); - cursorCaptureInterval = setInterval(sampleCursorPoint, CURSOR_SAMPLE_INTERVAL_MS); - } else { - stopCursorCapture(); - cursorTelemetryBuffer.endSession(); - } - - const source = selectedSource || { name: "Screen" }; - if (onRecordingStateChange) { - onRecordingStateChange(recording, source.name); - } - }); - - ipcMain.handle("discard-cursor-telemetry", (_, recordingId: number) => { - cursorTelemetryBuffer.discardBatch(recordingId); - }); + }, + ); ipcMain.handle("get-cursor-telemetry", async (_, videoPath?: string) => { - const targetVideoPath = normalizeVideoSourcePath( + const targetVideoPath = resolveApprovedVideoPath( videoPath ?? currentRecordingSession?.screenVideoPath, ); if (!targetVideoPath) { return { success: true, samples: [] }; } - if (!isPathAllowed(targetVideoPath)) { - console.warn( - "[get-cursor-telemetry] Rejected path outside allowed directories:", - targetVideoPath, - ); - return { success: true, samples: [] }; - } - - const telemetryPath = `${targetVideoPath}.cursor.json`; - try { - const content = await fs.readFile(telemetryPath, "utf-8"); - const parsed = JSON.parse(content); - const rawSamples = Array.isArray(parsed) - ? parsed - : Array.isArray(parsed?.samples) - ? parsed.samples - : []; - - const samples: CursorTelemetryPoint[] = rawSamples - .filter((sample: unknown) => Boolean(sample && typeof sample === "object")) - .map((sample: unknown) => { - const point = sample as Partial; - return { - timeMs: - typeof point.timeMs === "number" && Number.isFinite(point.timeMs) - ? Math.max(0, point.timeMs) - : 0, - cx: - typeof point.cx === "number" && Number.isFinite(point.cx) - ? clamp(point.cx, 0, 1) - : 0.5, - cy: - typeof point.cy === "number" && Number.isFinite(point.cy) - ? clamp(point.cy, 0, 1) - : 0.5, - }; - }) - .sort((a: CursorTelemetryPoint, b: CursorTelemetryPoint) => a.timeMs - b.timeMs); - - const rawClicks = Array.isArray(parsed?.clicks) ? parsed.clicks : []; - const clicks: number[] = rawClicks - .map((value: unknown) => - typeof value === "number" && Number.isFinite(value) ? Math.max(0, value) : null, - ) - .filter((v: number | null): v is number => v !== null) - .sort((a: number, b: number) => a - b); - - return { success: true, samples, clicks }; - } catch (error) { - const nodeError = error as NodeJS.ErrnoException; - if (nodeError.code === "ENOENT") { - return { success: true, samples: [], clicks: [] }; - } - console.error("Failed to load cursor telemetry:", error); - return { - success: false, - message: "Failed to load cursor telemetry", - error: String(error), - samples: [], - clicks: [], - }; - } + return readCursorTelemetryFile(targetVideoPath); }); ipcMain.handle("open-external-url", async (_, url: string) => { try { - const ALLOWED_SCHEMES = ["http:", "https:", "mailto:"]; - let parsed: URL; - try { - parsed = new URL(url); - } catch { - return { success: false, error: "Invalid URL" }; - } - - if (!ALLOWED_SCHEMES.includes(parsed.protocol)) { - return { success: false, error: `Unsupported URL scheme: ${parsed.protocol}` }; - } - - await shell.openExternal(parsed.toString()); + await shell.openExternal(url); return { success: true }; } catch (error) { console.error("Failed to open URL:", error); @@ -1004,15 +2276,10 @@ export function registerIpcHandlers( } }); - /** - * Handles saving an exported video file. - * Shows a save dialog, normalizes the file path for the current OS, - * ensures the directory exists, and writes the video data. - * @param _ - Unused event parameter. - * @param videoData - The exported video as an ArrayBuffer. - * @param fileName - Suggested filename for the save dialog. - * @returns Object with success status, optional file path, and error details. - */ + // Return base path for assets so renderer can resolve file:// paths in production + ipcMain.handle("get-asset-base-path", () => { + return resolveAssetBasePath(); + }); ipcMain.handle("pick-export-save-path", async (_, fileName: string, exportFolder?: string) => { try { @@ -1096,6 +2363,7 @@ export function registerIpcHandlers( }; } }); + ipcMain.handle("open-video-file-picker", async () => { try { const dialogOptions = buildDialogOptions( @@ -1119,17 +2387,18 @@ export function registerIpcHandlers( return { success: false, canceled: true }; } - const approvedPath = await approveReadableVideoPath(result.filePaths[0]); - if (!approvedPath) { + const normalizedPath = await approveReadableVideoPath(result.filePaths[0]); + if (!normalizedPath) { return { success: false, - message: "Selected file is not a supported video", + message: "Selected file is not a supported readable video file", }; } + currentProjectPath = null; return { success: true, - path: approvedPath, + path: normalizedPath, }; } catch (error) { console.error("Failed to open file picker:", error); @@ -1165,78 +2434,129 @@ export function registerIpcHandlers( } }); + ipcMain.handle("read-binary-file", async (_, filePath: string) => { + try { + const normalizedPath = await approveReadableVideoPath(filePath); + if (!normalizedPath) { + return { + success: false, + message: "File path is not approved or is not a supported video file", + }; + } + + const data = await fs.readFile(normalizedPath); + return { + success: true, + data: data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength), + path: normalizedPath, + }; + } catch (error) { + console.error("Failed to read binary file:", error); + return { + success: false, + message: "Failed to read binary file", + error: String(error), + }; + } + }); + + ipcMain.handle("prepare-preview-audio-track", async (_, filePath: string) => { + try { + return await prepareSupplementalPreviewAudioTrack(filePath); + } catch (error) { + console.error("Failed to prepare preview audio track:", error); + return { + success: false, + message: "Failed to prepare preview audio track", + error: String(error), + }; + } + }); + ipcMain.handle( "save-project-file", async (_, projectData: unknown, suggestedName?: string, existingProjectPath?: string) => { - try { - const trustedExistingProjectPath = isTrustedProjectPath(existingProjectPath) - ? existingProjectPath - : null; - - if (trustedExistingProjectPath) { - await fs.writeFile( - trustedExistingProjectPath, - JSON.stringify(projectData, null, 2), - "utf-8", - ); - currentProjectPath = trustedExistingProjectPath; - return { - success: true, - path: trustedExistingProjectPath, - message: "Project saved successfully", - }; - } - - const safeName = (suggestedName || `project-${Date.now()}`).replace(/[^a-zA-Z0-9-_]/g, "_"); - const defaultName = safeName.endsWith(`.${PROJECT_FILE_EXTENSION}`) - ? safeName - : `${safeName}.${PROJECT_FILE_EXTENSION}`; - - const dialogOptions = buildDialogOptions( - { - title: mainT("dialogs", "fileDialogs.saveProject"), - defaultPath: path.join(RECORDINGS_DIR, defaultName), - filters: [ - { - name: mainT("dialogs", "fileDialogs.openscreenProject"), - extensions: [PROJECT_FILE_EXTENSION], - }, - { name: "JSON", extensions: ["json"] }, - ], - properties: ["createDirectory", "showOverwriteConfirmation"], - }, - getMainWindow(), - ); - const result = await dialog.showSaveDialog(dialogOptions); - - if (result.canceled || !result.filePath) { - return { - success: false, - canceled: true, - message: "Save project canceled", - }; - } - - await fs.writeFile(result.filePath, JSON.stringify(projectData, null, 2), "utf-8"); - currentProjectPath = result.filePath; - - return { - success: true, - path: result.filePath, - message: "Project saved successfully", - }; - } catch (error) { - console.error("Failed to save project file:", error); - return { - success: false, - message: "Failed to save project file", - error: String(error), - }; - } + return saveProjectFile(projectData, suggestedName, existingProjectPath); }, ); + async function saveProjectFile( + projectData: unknown, + suggestedName?: string, + existingProjectPath?: string, + ): Promise { + try { + const trustedExistingProjectPath = isTrustedProjectPath(existingProjectPath) + ? existingProjectPath + : null; + + if (trustedExistingProjectPath) { + await fs.writeFile( + trustedExistingProjectPath, + JSON.stringify(projectData, null, 2), + "utf-8", + ); + currentProjectPath = trustedExistingProjectPath; + return { + success: true, + path: trustedExistingProjectPath, + message: "Project saved successfully", + }; + } + + const safeName = (suggestedName || `project-${Date.now()}`).replace(/[^a-zA-Z0-9-_]/g, "_"); + const defaultName = safeName.endsWith(`.${PROJECT_FILE_EXTENSION}`) + ? safeName + : `${safeName}.${PROJECT_FILE_EXTENSION}`; + + const dialogOptions = buildDialogOptions( + { + title: mainT("dialogs", "fileDialogs.saveProject"), + defaultPath: path.join(RECORDINGS_DIR, defaultName), + filters: [ + { + name: mainT("dialogs", "fileDialogs.openscreenProject"), + extensions: [PROJECT_FILE_EXTENSION], + }, + { name: "JSON", extensions: ["json"] }, + ], + properties: ["createDirectory", "showOverwriteConfirmation"], + }, + getMainWindow(), + ); + const result = await dialog.showSaveDialog(dialogOptions); + + if (result.canceled || !result.filePath) { + return { + success: false, + canceled: true, + message: "Save project canceled", + }; + } + + await fs.writeFile(result.filePath, JSON.stringify(projectData, null, 2), "utf-8"); + currentProjectPath = result.filePath; + + return { + success: true, + path: result.filePath, + message: "Project saved successfully", + }; + } catch (error) { + console.error("Failed to save project file:", error); + return { + success: false, + message: "Failed to save project file", + error: String(error), + }; + } + } + ipcMain.handle("load-project-file", async () => { + return loadProjectFile(); + }); + + async function loadProjectFile(): Promise { try { const dialogOptions = buildDialogOptions( { @@ -1263,9 +2583,8 @@ export function registerIpcHandlers( const filePath = result.filePaths[0]; const content = await fs.readFile(filePath, "utf-8"); const project = JSON.parse(content); - const session = await getApprovedProjectSession(project, filePath); currentProjectPath = filePath; - setCurrentRecordingSessionState(session); + setCurrentRecordingSessionState(await getApprovedProjectSession(project, filePath)); return { success: true, @@ -1280,9 +2599,13 @@ export function registerIpcHandlers( error: String(error), }; } - }); + } ipcMain.handle("load-current-project-file", async () => { + return loadCurrentProjectFile(); + }); + + async function loadCurrentProjectFile(): Promise { try { if (!currentProjectPath) { return { success: false, message: "No active project" }; @@ -1290,8 +2613,7 @@ export function registerIpcHandlers( const content = await fs.readFile(currentProjectPath, "utf-8"); const project = JSON.parse(content); - const session = await getApprovedProjectSession(project, currentProjectPath); - setCurrentRecordingSessionState(session); + setCurrentRecordingSessionState(await getApprovedProjectSession(project, currentProjectPath)); return { success: true, path: currentProjectPath, @@ -1305,12 +2627,18 @@ export function registerIpcHandlers( error: String(error), }; } + } + + ipcMain.handle("set-current-video-path", async (_, path: string) => { + return setCurrentVideoPath(path); }); + ipcMain.handle("set-current-recording-session", (_, session: RecordingSession | null) => { - const normalized = normalizeRecordingSession(session); - setCurrentRecordingSessionState(normalized); + const normalizedSession = normalizeRecordingSession(session); + setCurrentRecordingSessionState(normalizedSession); + currentVideoPath = normalizedSession?.screenVideoPath ?? null; currentProjectPath = null; - return { success: true, session: normalized ?? undefined }; + return { success: true, session: currentRecordingSession }; }); ipcMain.handle("get-current-recording-session", () => { @@ -1319,19 +2647,17 @@ export function registerIpcHandlers( : { success: false }; }); - ipcMain.handle("set-current-video-path", async (_, path: string) => { + async function setCurrentVideoPath(path: string): Promise { const normalizedPath = normalizeVideoSourcePath(path); if (!normalizedPath || !isPathAllowed(normalizedPath)) { - return { success: false, message: "Video path has not been approved" }; + return { + success: false, + message: "Video path has not been approved", + }; } const restoredSession = await loadRecordedSessionForVideoPath(normalizedPath); if (restoredSession) { - // Approve all media paths from the restored session so they can be read later - approveFilePath(restoredSession.screenVideoPath); - if (restoredSession.webcamVideoPath) { - approveFilePath(restoredSession.webcamVideoPath); - } setCurrentRecordingSessionState(restoredSession); } else { setCurrentRecordingSessionState({ @@ -1340,20 +2666,26 @@ export function registerIpcHandlers( }); } currentProjectPath = null; - return { success: true }; - }); + return { success: true, path: currentVideoPath ?? normalizedPath }; + } ipcMain.handle("get-current-video-path", () => { - return currentRecordingSession?.screenVideoPath - ? { success: true, path: currentRecordingSession.screenVideoPath } - : { success: false }; + return getCurrentVideoPathResult(); }); + function getCurrentVideoPathResult(): ProjectPathResult { + return currentVideoPath ? { success: true, path: currentVideoPath } : { success: false }; + } + ipcMain.handle("clear-current-video-path", () => { - setCurrentRecordingSessionState(null); - return { success: true }; + return clearCurrentVideoPath(); }); + function clearCurrentVideoPath(): ProjectPathResult { + currentVideoPath = null; + return { success: true }; + } + ipcMain.handle("get-platform", () => { return process.platform; }); @@ -1417,4 +2749,21 @@ export function registerIpcHandlers( } }, ); + + registerNativeBridgeHandlers({ + getPlatform: () => process.platform, + getCurrentProjectPath: () => currentProjectPath, + getCurrentVideoPath: () => currentVideoPath, + saveProjectFile, + loadProjectFile, + loadCurrentProjectFile, + setCurrentVideoPath, + getCurrentVideoPathResult, + clearCurrentVideoPath, + resolveAssetBasePath, + resolveVideoPath: (videoPath?: string | null) => + normalizeVideoSourcePath(videoPath ?? currentVideoPath), + loadCursorRecordingData: readCursorRecordingFile, + loadCursorTelemetry: readCursorTelemetryFile, + }); } diff --git a/electron/ipc/nativeBridge.ts b/electron/ipc/nativeBridge.ts new file mode 100644 index 0000000..7f7b24b --- /dev/null +++ b/electron/ipc/nativeBridge.ts @@ -0,0 +1,229 @@ +import { ipcMain } from "electron"; +import { + NATIVE_BRIDGE_CHANNEL, + NATIVE_BRIDGE_VERSION, + type NativeBridgeErrorCode, + type NativeBridgeRequest, + type NativeBridgeResponse, + type NativePlatform, + type ProjectFileResult, + type ProjectPathResult, +} from "../../src/native/contracts"; +import type { CursorTelemetryLoadResult } from "../native-bridge/cursor/adapter"; +import { TelemetryCursorAdapter } from "../native-bridge/cursor/telemetryCursorAdapter"; +import { CursorService } from "../native-bridge/services/cursorService"; +import { ProjectService } from "../native-bridge/services/projectService"; +import { SystemService } from "../native-bridge/services/systemService"; +import { NativeBridgeStateStore } from "../native-bridge/store"; + +export interface NativeBridgeContext { + getPlatform: () => NodeJS.Platform; + getCurrentProjectPath: () => string | null; + getCurrentVideoPath: () => string | null; + saveProjectFile: ( + projectData: unknown, + suggestedName?: string, + existingProjectPath?: string, + ) => Promise; + loadProjectFile: () => Promise; + loadCurrentProjectFile: () => Promise; + setCurrentVideoPath: (path: string) => ProjectPathResult | Promise; + getCurrentVideoPathResult: () => ProjectPathResult; + clearCurrentVideoPath: () => ProjectPathResult; + resolveAssetBasePath: () => string | null; + resolveVideoPath: (videoPath?: string | null) => string | null; + loadCursorRecordingData: ( + videoPath: string, + ) => Promise; + loadCursorTelemetry: (videoPath: string) => Promise; +} + +function normalizePlatform(platform: NodeJS.Platform): NativePlatform { + if (platform === "darwin" || platform === "win32") { + return platform; + } + + return "linux"; +} + +function createMeta(requestId?: string) { + return { + version: NATIVE_BRIDGE_VERSION, + requestId: requestId || `native-${Date.now()}`, + timestampMs: Date.now(), + } as const; +} + +function createSuccessResponse(requestId: string | undefined, data: TData) { + return { + ok: true, + data, + meta: createMeta(requestId), + } satisfies NativeBridgeResponse; +} + +function createErrorResponse( + requestId: string | undefined, + code: NativeBridgeErrorCode, + message: string, + retryable = false, +) { + return { + ok: false, + error: { + code, + message, + retryable, + }, + meta: createMeta(requestId), + } satisfies NativeBridgeResponse; +} + +function isBridgeRequest(value: unknown): value is NativeBridgeRequest { + if (!value || typeof value !== "object") { + return false; + } + + const candidate = value as Partial; + return typeof candidate.domain === "string" && typeof candidate.action === "string"; +} + +export function registerNativeBridgeHandlers(context: NativeBridgeContext) { + ipcMain.removeHandler(NATIVE_BRIDGE_CHANNEL); + + const platform = normalizePlatform(context.getPlatform()); + const store = new NativeBridgeStateStore(platform); + const projectService = new ProjectService({ + store, + getCurrentProjectPath: context.getCurrentProjectPath, + getCurrentVideoPath: context.getCurrentVideoPath, + saveProjectFile: context.saveProjectFile, + loadProjectFile: context.loadProjectFile, + loadCurrentProjectFile: context.loadCurrentProjectFile, + setCurrentVideoPath: context.setCurrentVideoPath, + getCurrentVideoPathResult: context.getCurrentVideoPathResult, + clearCurrentVideoPath: context.clearCurrentVideoPath, + }); + const cursorService = new CursorService({ + store, + adapter: new TelemetryCursorAdapter({ + loadRecordingData: context.loadCursorRecordingData, + resolveVideoPath: context.resolveVideoPath, + loadTelemetry: context.loadCursorTelemetry, + }), + }); + const systemService = new SystemService({ + store, + getPlatform: () => platform, + getAssetBasePath: context.resolveAssetBasePath, + getCursorCapabilities: () => cursorService.getCapabilities(), + }); + + ipcMain.handle(NATIVE_BRIDGE_CHANNEL, async (_, request: unknown) => { + if (!isBridgeRequest(request)) { + return createErrorResponse(undefined, "INVALID_REQUEST", "Invalid native bridge request."); + } + + const requestId = request.requestId; + const domain = request.domain as string; + + try { + switch (request.domain) { + case "system": { + const action = request.action as string; + switch (request.action) { + case "getPlatform": + return createSuccessResponse(requestId, systemService.getPlatform()); + case "getAssetBasePath": + return createSuccessResponse(requestId, systemService.getAssetBasePath()); + case "getCapabilities": + return createSuccessResponse(requestId, await systemService.getCapabilities()); + default: + return createErrorResponse( + requestId, + "UNSUPPORTED_ACTION", + `Unsupported system action: ${action}`, + ); + } + } + + case "project": { + const action = request.action as string; + switch (request.action) { + case "getCurrentContext": + return createSuccessResponse(requestId, projectService.getCurrentContext()); + case "saveProjectFile": + return createSuccessResponse( + requestId, + await projectService.saveProjectFile( + request.payload.projectData, + request.payload.suggestedName, + request.payload.existingProjectPath, + ), + ); + case "loadProjectFile": + return createSuccessResponse(requestId, await projectService.loadProjectFile()); + case "loadCurrentProjectFile": + return createSuccessResponse( + requestId, + await projectService.loadCurrentProjectFile(), + ); + case "setCurrentVideoPath": + return createSuccessResponse( + requestId, + await projectService.setCurrentVideoPath(request.payload.path), + ); + case "getCurrentVideoPath": + return createSuccessResponse(requestId, projectService.getCurrentVideoPath()); + case "clearCurrentVideoPath": + return createSuccessResponse(requestId, projectService.clearCurrentVideoPath()); + default: + return createErrorResponse( + requestId, + "UNSUPPORTED_ACTION", + `Unsupported project action: ${action}`, + ); + } + } + + case "cursor": { + const action = request.action as string; + switch (request.action) { + case "getCapabilities": + return createSuccessResponse(requestId, await cursorService.getCapabilities()); + case "getTelemetry": + return createSuccessResponse( + requestId, + await cursorService.getTelemetry(request.payload?.videoPath), + ); + case "getRecordingData": + return createSuccessResponse( + requestId, + await cursorService.getRecordingData(request.payload?.videoPath), + ); + default: + return createErrorResponse( + requestId, + "UNSUPPORTED_ACTION", + `Unsupported cursor action: ${action}`, + ); + } + } + + default: + return createErrorResponse( + requestId, + "UNSUPPORTED_ACTION", + `Unsupported bridge domain: ${domain}`, + ); + } + } catch (error) { + return createErrorResponse( + requestId, + "INTERNAL_ERROR", + error instanceof Error ? error.message : "Unknown native bridge error.", + true, + ); + } + }); +} diff --git a/electron/main.ts b/electron/main.ts index 3fe8599..3e2258f 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -4,7 +4,6 @@ import { fileURLToPath } from "node:url"; import { app, BrowserWindow, - desktopCapturer, ipcMain, Menu, nativeImage, @@ -13,7 +12,7 @@ import { Tray, } from "electron"; import { mainT, setMainLocale } from "./i18n"; -import { registerIpcHandlers } from "./ipc/handlers"; +import { getSelectedDesktopSource, registerIpcHandlers } from "./ipc/handlers"; import { createCountdownOverlayWindow, createEditorWindow, @@ -477,23 +476,30 @@ app.whenReady().then(async () => { callback(allowed.includes(permission)); }); - // Request microphone and screen recording permissions from macOS + session.defaultSession.setDisplayMediaRequestHandler( + (request, callback) => { + const source = getSelectedDesktopSource(); + if (!request.videoRequested || !source) { + callback({}); + return; + } + + callback({ + video: source, + ...(request.audioRequested && process.platform === "win32" ? { audio: "loopback" } : {}), + }); + }, + { useSystemPicker: false }, + ); + + // Request microphone permission from macOS. Screen Recording is requested + // lazily from the source-picker action so the system prompt is not hidden + // behind OpenScreen's source selector window. if (process.platform === "darwin") { const micStatus = systemPreferences.getMediaAccessStatus("microphone"); if (micStatus !== "granted") { await systemPreferences.askForMediaAccess("microphone"); } - - // Screen recording has no askForMediaAccess equivalent — the TCC prompt is - // triggered by the first desktopCapturer.getSources() call. Firing it here - // at startup settles the permission state early and prevents repeated prompts - // driven by later getSources() calls (fixes repeated permission dialog). - const screenStatus = systemPreferences.getMediaAccessStatus("screen"); - if (screenStatus === "not-determined") { - desktopCapturer.getSources({ types: ["screen"] }).catch(() => { - // Ignore prompt trigger failures; later capture attempts report status. - }); - } } // Listen for HUD overlay quit event (macOS only) diff --git a/electron/native-bridge/cursor/adapter.ts b/electron/native-bridge/cursor/adapter.ts new file mode 100644 index 0000000..cdb88e2 --- /dev/null +++ b/electron/native-bridge/cursor/adapter.ts @@ -0,0 +1,20 @@ +import type { + CursorCapabilities, + CursorProviderKind, + CursorRecordingData, + CursorTelemetryPoint, +} from "../../../src/native/contracts"; + +export interface CursorTelemetryLoadResult { + success: boolean; + samples: CursorTelemetryPoint[]; + message?: string; + error?: string; +} + +export interface CursorNativeAdapter { + readonly kind: CursorProviderKind; + getCapabilities(): Promise; + getRecordingData(videoPath?: string | null): Promise; + getTelemetry(videoPath?: string | null): Promise; +} diff --git a/electron/native-bridge/cursor/recording/factory.ts b/electron/native-bridge/cursor/recording/factory.ts new file mode 100644 index 0000000..0ba3077 --- /dev/null +++ b/electron/native-bridge/cursor/recording/factory.ts @@ -0,0 +1,46 @@ +import type { Rectangle } from "electron"; +import { MacNativeCursorRecordingSession } from "./macNativeCursorRecordingSession"; +import type { CursorRecordingSession } from "./session"; +import { TelemetryRecordingSession } from "./telemetryRecordingSession"; +import { WindowsNativeRecordingSession } from "./windowsNativeRecordingSession"; + +interface CreateCursorRecordingSessionOptions { + getDisplayBounds: () => Rectangle | null; + maxSamples: number; + platform: NodeJS.Platform; + sampleIntervalMs: number; + sourceId?: string | null; + startTimeMs?: number; +} + +export function createCursorRecordingSession( + options: CreateCursorRecordingSessionOptions, +): CursorRecordingSession { + if (options.platform === "win32") { + return new WindowsNativeRecordingSession({ + getDisplayBounds: options.getDisplayBounds, + maxSamples: options.maxSamples, + sampleIntervalMs: options.sampleIntervalMs, + sourceId: options.sourceId, + startTimeMs: options.startTimeMs, + }); + } + + if (options.platform === "darwin") { + return new MacNativeCursorRecordingSession({ + getDisplayBounds: options.getDisplayBounds, + maxSamples: options.maxSamples, + sampleIntervalMs: options.sampleIntervalMs, + startTimeMs: options.startTimeMs, + }); + } + + // Linux: capture cursor positions via Electron's `screen` API on an interval. + // No cursor sprites/assets and no clicks — just position telemetry. + return new TelemetryRecordingSession({ + getDisplayBounds: options.getDisplayBounds, + maxSamples: options.maxSamples, + sampleIntervalMs: options.sampleIntervalMs, + startTimeMs: options.startTimeMs, + }); +} diff --git a/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts b/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts new file mode 100644 index 0000000..95ed10c --- /dev/null +++ b/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts @@ -0,0 +1,411 @@ +import { type ChildProcessByStdio, spawn } from "node:child_process"; +import { accessSync, constants as fsConstants } from "node:fs"; +import path from "node:path"; +import type { Readable } from "node:stream"; +import { type Rectangle, screen, systemPreferences } from "electron"; +import type { + CursorRecordingData, + CursorRecordingSample, + NativeCursorType, +} from "../../../../src/native/contracts"; +import type { CursorRecordingSession } from "./session"; + +interface MacNativeCursorRecordingSessionOptions { + getDisplayBounds: () => Rectangle | null; + maxSamples: number; + sampleIntervalMs: number; + startTimeMs?: number; +} + +type MacCursorEvent = + | { + type: "ready"; + timestampMs: number; + accessibilityTrusted?: boolean; + mouseTapReady?: boolean; + } + | { + type: "sample"; + timestampMs: number; + cursorType?: NativeCursorType | null; + leftButtonDown?: boolean; + leftButtonPressed?: boolean; + leftButtonReleased?: boolean; + }; + +const HELPER_NAME = "openscreen-macos-cursor-helper"; +const READY_TIMEOUT_MS = 5_000; + +function helperCandidates() { + const envPath = process.env.OPENSCREEN_MAC_CURSOR_HELPER_EXE?.trim(); + const appRoot = process.env.APP_ROOT ? path.resolve(process.env.APP_ROOT) : process.cwd(); + const archTag = process.arch === "arm64" ? "darwin-arm64" : "darwin-x64"; + const resourceRoot = + typeof process.resourcesPath === "string" + ? process.resourcesPath + : path.join(appRoot, "resources"); + + return [ + envPath, + path.join(appRoot, "electron", "native", "screencapturekit", "build", HELPER_NAME), + path.join(appRoot, "electron", "native", "bin", archTag, HELPER_NAME), + path.join(resourceRoot, "electron", "native", "bin", archTag, HELPER_NAME), + ].filter((candidate): candidate is string => Boolean(candidate)); +} + +export function findMacCursorHelperPath() { + for (const candidate of helperCandidates()) { + try { + accessSync(candidate, fsConstants.X_OK); + return candidate; + } catch { + // Try the next helper location. + } + } + + return null; +} + +export async function requestMacCursorAccessibilityAccess() { + if (process.platform !== "darwin") { + return { success: true, granted: true, status: "granted" }; + } + + try { + systemPreferences.isTrustedAccessibilityClient(true); + } catch { + // Continue with helper probing; it can trigger the same macOS prompt. + } + + const helperPath = findMacCursorHelperPath(); + if (!helperPath) { + return { success: true, granted: false, status: "missing-helper" }; + } + + return new Promise<{ success: boolean; granted: boolean; status: string; error?: string }>( + (resolve) => { + const child = spawn(helperPath, [JSON.stringify({ sampleIntervalMs: 250 })], { + stdio: ["ignore", "pipe", "pipe"], + }); + let settled = false; + let lineBuffer = ""; + const finish = (result: { + success: boolean; + granted: boolean; + status: string; + error?: string; + }) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + if (!child.killed) { + child.kill("SIGTERM"); + } + resolve(result); + }; + const timer = setTimeout(() => { + finish({ + success: false, + granted: false, + status: "timeout", + error: "Timed out waiting for macOS cursor helper", + }); + }, READY_TIMEOUT_MS); + + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + lineBuffer += chunk; + const lines = lineBuffer.split(/\r?\n/); + lineBuffer = lines.pop() ?? ""; + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed) { + continue; + } + try { + const event = JSON.parse(trimmed) as MacCursorEvent; + if (event.type === "ready") { + finish({ + success: true, + granted: event.accessibilityTrusted === true, + status: event.accessibilityTrusted === true ? "granted" : "not-determined", + }); + return; + } + } catch { + // Ignore non-JSON helper output. + } + } + }); + + child.once("error", (error) => { + finish({ + success: false, + granted: false, + status: "error", + error: error.message, + }); + }); + child.once("exit", (code, signal) => { + finish({ + success: false, + granted: false, + status: "exited", + error: `macOS cursor helper exited before ready (code=${code}, signal=${signal})`, + }); + }); + }, + ); +} + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)); +} + +function normalizeCursorType(value: unknown): NativeCursorType | null { + return value === "arrow" || value === "pointer" || value === "text" ? value : null; +} + +export class MacNativeCursorRecordingSession implements CursorRecordingSession { + private samples: CursorRecordingSample[] = []; + private process: ChildProcessByStdio | null = null; + private lineBuffer = ""; + private startTimeMs = 0; + private fallbackInterval: NodeJS.Timeout | null = null; + private readyResolve: (() => void) | null = null; + private readyReject: ((error: Error) => void) | null = null; + private readyTimer: NodeJS.Timeout | null = null; + private previousLeftButtonDown = false; + private consecutiveOutsideSamples = 0; + // Only hide after this many consecutive out-of-bounds samples (≈100ms at 33ms interval). + // Fast swipes that briefly exit the display are clipped by clip-path instead of disappearing. + private static readonly OUTSIDE_HIDE_THRESHOLD = 3; + + constructor(private readonly options: MacNativeCursorRecordingSessionOptions) {} + + async start(): Promise { + this.samples = []; + this.lineBuffer = ""; + this.startTimeMs = this.options.startTimeMs ?? Date.now(); + this.previousLeftButtonDown = false; + this.consecutiveOutsideSamples = 0; + + try { + systemPreferences.isTrustedAccessibilityClient(true); + } catch { + // Link cursor detection degrades to arrow when Accessibility is unavailable. + } + + const helperPath = findMacCursorHelperPath(); + if (!helperPath) { + this.startPositionOnlyFallback(); + return; + } + + const child = spawn( + helperPath, + [ + JSON.stringify({ + sampleIntervalMs: this.options.sampleIntervalMs, + }), + ], + { + stdio: ["ignore", "pipe", "pipe"], + }, + ); + this.process = child; + + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => this.handleStdoutChunk(chunk)); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + const message = chunk.trim(); + if (message) { + console.error("[cursor-macos]", message); + } + }); + child.once("exit", (code, signal) => { + this.rejectReady( + new Error(`macOS cursor helper exited before ready (code=${code}, signal=${signal})`), + ); + this.process = null; + }); + child.once("error", (error) => { + this.rejectReady(error); + this.process = null; + }); + + try { + await this.waitUntilReady(); + } catch (error) { + this.killHelperProcess(child); + this.process = null; + console.warn("[cursor-macos] falling back to position-only cursor telemetry:", error); + this.startPositionOnlyFallback(); + } + } + + async stop(): Promise { + const child = this.process; + this.process = null; + this.clearReadyState(); + + if (this.fallbackInterval) { + clearInterval(this.fallbackInterval); + this.fallbackInterval = null; + } + + if (child) { + this.killHelperProcess(child); + } + + return { + version: 2, + provider: "none", + samples: this.samples, + assets: [], + }; + } + + private startPositionOnlyFallback() { + this.captureSample(Date.now(), null, false, false, false); + this.fallbackInterval = setInterval(() => { + this.captureSample(Date.now(), null, false, false, false); + }, this.options.sampleIntervalMs); + } + + private handleStdoutChunk(chunk: string) { + this.lineBuffer += chunk; + const lines = this.lineBuffer.split(/\r?\n/); + this.lineBuffer = lines.pop() ?? ""; + + for (const line of lines) { + const trimmedLine = line.trim(); + if (!trimmedLine) { + continue; + } + + try { + this.handleEvent(JSON.parse(trimmedLine) as MacCursorEvent); + } catch (error) { + console.error("Failed to parse macOS cursor helper output:", error, trimmedLine); + } + } + } + + private handleEvent(payload: MacCursorEvent) { + if (payload.type === "ready") { + if (payload.accessibilityTrusted === false) { + console.warn( + "[cursor-macos] Accessibility is not trusted; cursor shape detection will be arrow-only.", + ); + } + this.resolveReady(); + return; + } + + if (payload.type === "sample") { + this.captureSample( + payload.timestampMs, + normalizeCursorType(payload.cursorType), + payload.leftButtonDown === true, + payload.leftButtonPressed === true, + payload.leftButtonReleased === true, + ); + } + } + + private captureSample( + timestampMs: number, + cursorType: NativeCursorType | null, + leftButtonDown: boolean, + leftButtonPressed: boolean, + leftButtonReleased: boolean, + ) { + const cursor = screen.getCursorScreenPoint(); + const bounds = this.options.getDisplayBounds() ?? screen.getDisplayNearestPoint(cursor).bounds; + const width = Math.max(1, bounds.width); + const height = Math.max(1, bounds.height); + const normalizedX = (cursor.x - bounds.x) / width; + const normalizedY = (cursor.y - bounds.y) / height; + const isOutsideDisplay = + normalizedX < 0 || normalizedX > 1 || normalizedY < 0 || normalizedY > 1; + // Fast swipes that briefly exit the display ( this.options.maxSamples) { + this.samples.shift(); + } + } + + private waitUntilReady() { + return new Promise((resolve, reject) => { + this.readyResolve = resolve; + this.readyReject = reject; + this.readyTimer = setTimeout(() => { + this.rejectReady(new Error("Timed out waiting for macOS cursor helper")); + }, READY_TIMEOUT_MS); + }); + } + + private resolveReady() { + const resolve = this.readyResolve; + this.clearReadyState(); + resolve?.(); + } + + private rejectReady(error: Error) { + const reject = this.readyReject; + this.clearReadyState(); + reject?.(error); + } + + private clearReadyState() { + if (this.readyTimer) { + clearTimeout(this.readyTimer); + this.readyTimer = null; + } + this.readyResolve = null; + this.readyReject = null; + } + + private killHelperProcess(child: ChildProcessByStdio) { + if (child.killed) { + return; + } + + child.kill("SIGTERM"); + setTimeout(() => { + if (!child.killed) { + child.kill("SIGKILL"); + } + }, 500).unref(); + } +} diff --git a/electron/native-bridge/cursor/recording/session.ts b/electron/native-bridge/cursor/recording/session.ts new file mode 100644 index 0000000..9cebe9f --- /dev/null +++ b/electron/native-bridge/cursor/recording/session.ts @@ -0,0 +1,6 @@ +import type { CursorRecordingData } from "../../../../src/native/contracts"; + +export interface CursorRecordingSession { + start(): Promise; + stop(): Promise; +} diff --git a/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts b/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts new file mode 100644 index 0000000..e719d8e --- /dev/null +++ b/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts @@ -0,0 +1,63 @@ +import { type Rectangle, screen } from "electron"; +import type { CursorRecordingData, CursorRecordingSample } from "../../../../src/native/contracts"; +import type { CursorRecordingSession } from "./session"; + +interface TelemetryRecordingSessionOptions { + getDisplayBounds: () => Rectangle | null; + maxSamples: number; + sampleIntervalMs: number; + startTimeMs?: number; +} + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)); +} + +export class TelemetryRecordingSession implements CursorRecordingSession { + private samples: CursorRecordingSample[] = []; + private interval: NodeJS.Timeout | null = null; + private startTimeMs = 0; + + constructor(private readonly options: TelemetryRecordingSessionOptions) {} + + async start(): Promise { + this.samples = []; + this.startTimeMs = this.options.startTimeMs ?? Date.now(); + this.captureSample(); + this.interval = setInterval(() => { + this.captureSample(); + }, this.options.sampleIntervalMs); + } + + async stop(): Promise { + if (this.interval) { + clearInterval(this.interval); + this.interval = null; + } + + return { + version: 2, + provider: "none", + samples: this.samples, + assets: [], + }; + } + + private captureSample() { + const cursor = screen.getCursorScreenPoint(); + const display = this.options.getDisplayBounds() ?? screen.getDisplayNearestPoint(cursor).bounds; + const width = Math.max(1, display.width); + const height = Math.max(1, display.height); + + this.samples.push({ + timeMs: Math.max(0, Date.now() - this.startTimeMs), + cx: clamp((cursor.x - display.x) / width, 0, 1), + cy: clamp((cursor.y - display.y) / height, 0, 1), + visible: true, + }); + + if (this.samples.length > this.options.maxSamples) { + this.samples.shift(); + } + } +} diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts new file mode 100644 index 0000000..5c318f0 --- /dev/null +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts @@ -0,0 +1,326 @@ +import { type ChildProcessByStdio, spawn } from "node:child_process"; +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import type { Readable } from "node:stream"; +import { app, screen } from "electron"; +import { parseWindowHandleFromSourceId } from "../../../../src/lib/nativeWindowsRecording"; +import type { + CursorRecordingData, + CursorRecordingSample, + NativeCursorAsset, +} from "../../../../src/native/contracts"; +import type { CursorRecordingSession } from "./session"; +import type { + WindowsCursorEvent, + WindowsNativeRecordingSessionOptions, +} from "./windowsNativeRecordingSession.types"; + +function getCursorSamplerCandidates(): string[] { + const envPath = process.env.OPENSCREEN_CURSOR_SAMPLER_EXE?.trim(); + const archTag = process.arch === "arm64" ? "win32-arm64" : "win32-x64"; + const resolve = (...segs: string[]) => { + const p = join(app.getAppPath(), ...segs); + return app.isPackaged ? p.replace(/\.asar([/\\])/, ".asar.unpacked$1") : p; + }; + return [ + envPath, + resolve("electron", "native", "wgc-capture", "build", "cursor-sampler.exe"), + resolve("electron", "native", "bin", archTag, "cursor-sampler.exe"), + ].filter((c): c is string => Boolean(c)); +} + +function findCursorSamplerPath(): string | null { + for (const candidate of getCursorSamplerCandidates()) { + if (existsSync(candidate)) return candidate; + } + return null; +} + +const READY_TIMEOUT_MS = 5_000; + +interface NormalizedSample { + sample: CursorRecordingSample; + withinBounds: boolean; +} + +export class WindowsNativeRecordingSession implements CursorRecordingSession { + private assets = new Map(); + private samples: CursorRecordingSample[] = []; + private process: ChildProcessByStdio | null = null; + private lineBuffer = ""; + private startTimeMs = 0; + private readyResolve: (() => void) | null = null; + private readyReject: ((error: Error) => void) | null = null; + private readyTimer: NodeJS.Timeout | null = null; + private sampleCount = 0; + private outOfBoundsSampleCount = 0; + private previousLeftButtonDown = false; + + constructor(private readonly options: WindowsNativeRecordingSessionOptions) {} + + async start(): Promise { + this.assets.clear(); + this.samples = []; + this.lineBuffer = ""; + this.startTimeMs = this.options.startTimeMs ?? Date.now(); + this.sampleCount = 0; + this.outOfBoundsSampleCount = 0; + this.previousLeftButtonDown = false; + + const helperPath = findCursorSamplerPath(); + if (!helperPath) { + throw new Error("Windows cursor sampler helper is not available."); + } + + const windowHandle = parseWindowHandleFromSourceId(this.options.sourceId); + const args = [String(this.options.sampleIntervalMs)]; + if (windowHandle) args.push(windowHandle); + + const child = spawn(helperPath, args, { + stdio: ["ignore", "pipe", "pipe"], + windowsHide: true, + }); + + this.process = child; + this.logDiagnostic("spawn", { + pid: child.pid ?? null, + sampleIntervalMs: this.options.sampleIntervalMs, + sourceId: this.options.sourceId ?? null, + windowHandle, + }); + + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + this.handleStdoutChunk(chunk); + }); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + const message = chunk.trim(); + if (message) { + this.logDiagnostic("stderr", { message }); + } + console.error("[cursor-native]", message); + }); + child.once("exit", (code, signal) => { + this.logDiagnostic("exit", { + code, + signal, + sampleCount: this.sampleCount, + assetCount: this.assets.size, + outOfBoundsSampleCount: this.outOfBoundsSampleCount, + }); + this.rejectReady( + new Error(`Windows cursor helper exited before ready (code=${code}, signal=${signal})`), + ); + }); + child.once("error", (error) => { + this.logDiagnostic("process-error", { message: error.message }); + this.rejectReady(error); + }); + + try { + await this.waitUntilReady(); + } catch (error) { + this.terminateHelperProcess(); + throw error; + } + } + + async stop(): Promise { + const child = this.process; + this.process = null; + this.clearReadyState(); + + this.killHelperProcess(child); + + this.logDiagnostic("stop", { + sampleCount: this.sampleCount, + assetCount: this.assets.size, + outOfBoundsSampleCount: this.outOfBoundsSampleCount, + }); + + return { + version: 2, + provider: this.assets.size > 0 ? "native" : "none", + samples: this.samples, + assets: [...this.assets.values()], + }; + } + + private handleStdoutChunk(chunk: string) { + this.lineBuffer += chunk; + const lines = this.lineBuffer.split(/\r?\n/); + this.lineBuffer = lines.pop() ?? ""; + + for (const line of lines) { + const trimmedLine = line.trim(); + if (!trimmedLine) { + continue; + } + + try { + const payload = JSON.parse(trimmedLine) as WindowsCursorEvent; + this.handleEvent(payload); + } catch (error) { + console.error("Failed to parse Windows cursor helper output:", error, trimmedLine); + } + } + } + + private handleEvent(payload: WindowsCursorEvent) { + if (payload.type === "error") { + this.logDiagnostic("helper-error", { message: payload.message }); + console.error("Windows cursor helper error:", payload.message); + this.failHelper(new Error(payload.message)); + return; + } + + if (payload.type === "ready") { + this.logDiagnostic("ready", { timestampMs: payload.timestampMs }); + this.resolveReady(); + return; + } + + if (payload.asset?.id && !this.assets.has(payload.asset.id)) { + const assetDisplay = screen.getDisplayNearestPoint({ x: payload.x, y: payload.y }); + this.assets.set(payload.asset.id, { + id: payload.asset.id, + platform: "win32", + imageDataUrl: payload.asset.imageDataUrl, + width: payload.asset.width, + height: payload.asset.height, + hotspotX: payload.asset.hotspotX, + hotspotY: payload.asset.hotspotY, + scaleFactor: assetDisplay.scaleFactor, + cursorType: payload.asset.cursorType ?? payload.cursorType ?? null, + }); + this.logDiagnostic("asset", { + id: payload.asset.id, + width: payload.asset.width, + height: payload.asset.height, + hotspotX: payload.asset.hotspotX, + hotspotY: payload.asset.hotspotY, + scaleFactor: assetDisplay.scaleFactor, + }); + } + + const normalized = this.normalizeSample(payload); + this.sampleCount += 1; + if (!normalized.withinBounds) { + this.outOfBoundsSampleCount += 1; + } + + this.samples.push(normalized.sample); + + if (this.samples.length > this.options.maxSamples) { + this.samples.shift(); + } + } + + private normalizeSample( + payload: Extract, + ): NormalizedSample { + const bounds = + payload.bounds ?? this.options.getDisplayBounds() ?? screen.getPrimaryDisplay().bounds; + const width = Math.max(1, bounds.width); + const height = Math.max(1, bounds.height); + const normalizedX = (payload.x - bounds.x) / width; + const normalizedY = (payload.y - bounds.y) / height; + const withinBounds = + normalizedX >= 0 && normalizedX <= 1 && normalizedY >= 0 && normalizedY <= 1; + const leftButtonDown = payload.leftButtonDown === true; + const leftButtonPressed = payload.leftButtonPressed === true; + const leftButtonReleased = payload.leftButtonReleased === true; + const interactionType = + leftButtonPressed || (leftButtonDown && !this.previousLeftButtonDown) + ? "click" + : leftButtonReleased || (!leftButtonDown && this.previousLeftButtonDown) + ? "mouseup" + : "move"; + this.previousLeftButtonDown = leftButtonDown; + + if (this.sampleCount === 0 || (!withinBounds && this.outOfBoundsSampleCount === 0)) { + this.logDiagnostic("sample", { + rawX: payload.x, + rawY: payload.y, + normalizedX, + normalizedY, + visible: payload.visible, + withinBounds, + bounds, + handle: payload.handle, + }); + } + + return { + withinBounds, + sample: { + timeMs: Math.max(0, payload.timestampMs - this.startTimeMs), + cx: normalizedX, + cy: normalizedY, + assetId: payload.handle, + visible: payload.visible && withinBounds, + cursorType: payload.cursorType ?? payload.asset?.cursorType ?? null, + interactionType, + }, + }; + } + + private waitUntilReady() { + return new Promise((resolve, reject) => { + this.readyResolve = resolve; + this.readyReject = reject; + this.readyTimer = setTimeout(() => { + this.rejectReady(new Error("Timed out waiting for Windows cursor helper readiness")); + }, READY_TIMEOUT_MS); + }); + } + + private resolveReady() { + const resolve = this.readyResolve; + this.clearReadyState(); + resolve?.(); + } + + private rejectReady(error: Error) { + const reject = this.readyReject; + this.clearReadyState(); + reject?.(error); + } + + private failHelper(error: Error) { + this.rejectReady(error); + this.terminateHelperProcess(); + } + + private terminateHelperProcess() { + const child = this.process; + this.process = null; + this.killHelperProcess(child); + } + + private killHelperProcess(child: ChildProcessByStdio | null) { + if (child && !child.killed) { + child.kill(); + } + } + + private clearReadyState() { + if (this.readyTimer) { + clearTimeout(this.readyTimer); + this.readyTimer = null; + } + this.readyResolve = null; + this.readyReject = null; + } + + private logDiagnostic(event: string, data: Record) { + console.info( + "[cursor-native][win32]", + JSON.stringify({ + event, + ...data, + }), + ); + } +} diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts new file mode 100644 index 0000000..f3b69da --- /dev/null +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts @@ -0,0 +1,56 @@ +import type { Rectangle } from "electron"; +import type { NativeCursorType } from "../../../../src/native/contracts"; + +export interface WindowsCursorSampleEvent { + type: "sample"; + timestampMs: number; + x: number; + y: number; + visible: boolean; + handle: string | null; + cursorType?: NativeCursorType | null; + leftButtonDown?: boolean; + leftButtonPressed?: boolean; + leftButtonReleased?: boolean; + bounds?: { + x: number; + y: number; + width: number; + height: number; + } | null; + asset: WindowsCursorAssetPayload | null; +} + +export interface WindowsCursorReadyEvent { + type: "ready"; + timestampMs: number; +} + +export interface WindowsCursorErrorEvent { + type: "error"; + timestampMs: number; + message: string; +} + +export interface WindowsCursorAssetPayload { + id: string; + imageDataUrl: string; + width: number; + height: number; + hotspotX: number; + hotspotY: number; + cursorType?: NativeCursorType | null; +} + +export type WindowsCursorEvent = + | WindowsCursorSampleEvent + | WindowsCursorReadyEvent + | WindowsCursorErrorEvent; + +export interface WindowsNativeRecordingSessionOptions { + getDisplayBounds: () => Rectangle | null; + maxSamples: number; + sampleIntervalMs: number; + sourceId?: string | null; + startTimeMs?: number; +} diff --git a/electron/native-bridge/cursor/telemetryCursorAdapter.ts b/electron/native-bridge/cursor/telemetryCursorAdapter.ts new file mode 100644 index 0000000..073b183 --- /dev/null +++ b/electron/native-bridge/cursor/telemetryCursorAdapter.ts @@ -0,0 +1,49 @@ +import type { CursorCapabilities, CursorRecordingData } from "../../../src/native/contracts"; +import type { CursorNativeAdapter, CursorTelemetryLoadResult } from "./adapter"; + +interface TelemetryCursorAdapterOptions { + loadRecordingData: (videoPath: string) => Promise; + resolveVideoPath: (videoPath?: string | null) => string | null; + loadTelemetry: (videoPath: string) => Promise; +} + +export class TelemetryCursorAdapter implements CursorNativeAdapter { + readonly kind = "none" as const; + + constructor(private readonly options: TelemetryCursorAdapterOptions) {} + + async getCapabilities(): Promise { + return { + telemetry: true, + systemAssets: false, + provider: this.kind, + }; + } + + async getRecordingData(videoPath?: string | null): Promise { + const resolvedVideoPath = this.options.resolveVideoPath(videoPath); + if (!resolvedVideoPath) { + return { + version: 2, + provider: this.kind, + samples: [], + assets: [], + }; + } + + return this.options.loadRecordingData(resolvedVideoPath); + } + + async getTelemetry(videoPath?: string | null) { + const resolvedVideoPath = this.options.resolveVideoPath(videoPath); + if (!resolvedVideoPath) { + return { + success: false, + message: "No video path is available for cursor telemetry", + samples: [], + } satisfies CursorTelemetryLoadResult; + } + + return this.options.loadTelemetry(resolvedVideoPath); + } +} diff --git a/electron/native-bridge/services/cursorService.ts b/electron/native-bridge/services/cursorService.ts new file mode 100644 index 0000000..e3e9a25 --- /dev/null +++ b/electron/native-bridge/services/cursorService.ts @@ -0,0 +1,46 @@ +import type { + CursorCapabilities, + CursorRecordingData, + CursorTelemetryPoint, +} from "../../../src/native/contracts"; +import type { CursorNativeAdapter } from "../cursor/adapter"; +import type { NativeBridgeStateStore } from "../store"; + +interface CursorServiceOptions { + store: NativeBridgeStateStore; + adapter: CursorNativeAdapter; +} + +export class CursorService { + constructor(private readonly options: CursorServiceOptions) {} + + async getCapabilities(): Promise { + const capabilities = await this.options.adapter.getCapabilities(); + this.options.store.setCursorCapabilities(capabilities); + return capabilities; + } + + async getTelemetry(videoPath?: string | null): Promise { + const result = await this.options.adapter.getTelemetry(videoPath); + if (!result.success) { + throw new Error(result.message || result.error || "Failed to load cursor telemetry"); + } + + const resolvedVideoPath = videoPath ?? this.options.store.getState().project.currentVideoPath; + if (resolvedVideoPath) { + this.options.store.markCursorTelemetryLoaded(resolvedVideoPath, result.samples.length); + } + + return result.samples; + } + + async getRecordingData(videoPath?: string | null): Promise { + const data = await this.options.adapter.getRecordingData(videoPath); + const resolvedVideoPath = videoPath ?? this.options.store.getState().project.currentVideoPath; + if (resolvedVideoPath) { + this.options.store.markCursorTelemetryLoaded(resolvedVideoPath, data.samples.length); + } + + return data; + } +} diff --git a/electron/native-bridge/services/projectService.ts b/electron/native-bridge/services/projectService.ts new file mode 100644 index 0000000..965b4fb --- /dev/null +++ b/electron/native-bridge/services/projectService.ts @@ -0,0 +1,80 @@ +import type { + ProjectContext, + ProjectFileResult, + ProjectPathResult, +} from "../../../src/native/contracts"; +import type { NativeBridgeStateStore } from "../store"; + +interface ProjectServiceOptions { + store: NativeBridgeStateStore; + getCurrentProjectPath: () => string | null; + getCurrentVideoPath: () => string | null; + saveProjectFile: ( + projectData: unknown, + suggestedName?: string, + existingProjectPath?: string, + ) => Promise; + loadProjectFile: () => Promise; + loadCurrentProjectFile: () => Promise; + setCurrentVideoPath: (path: string) => ProjectPathResult | Promise; + getCurrentVideoPathResult: () => ProjectPathResult; + clearCurrentVideoPath: () => ProjectPathResult; +} + +export class ProjectService { + constructor(private readonly options: ProjectServiceOptions) {} + + getCurrentContext(): ProjectContext { + const context = { + currentProjectPath: this.options.getCurrentProjectPath(), + currentVideoPath: this.options.getCurrentVideoPath(), + }; + + this.options.store.setProjectContext(context); + return context; + } + + async saveProjectFile( + projectData: unknown, + suggestedName?: string, + existingProjectPath?: string, + ) { + const result = await this.options.saveProjectFile( + projectData, + suggestedName, + existingProjectPath, + ); + this.getCurrentContext(); + return result; + } + + async loadProjectFile() { + const result = await this.options.loadProjectFile(); + this.getCurrentContext(); + return result; + } + + async loadCurrentProjectFile() { + const result = await this.options.loadCurrentProjectFile(); + this.getCurrentContext(); + return result; + } + + async setCurrentVideoPath(path: string) { + const result = await this.options.setCurrentVideoPath(path); + this.getCurrentContext(); + return result; + } + + getCurrentVideoPath() { + const result = this.options.getCurrentVideoPathResult(); + this.getCurrentContext(); + return result; + } + + clearCurrentVideoPath() { + const result = this.options.clearCurrentVideoPath(); + this.getCurrentContext(); + return result; + } +} diff --git a/electron/native-bridge/services/systemService.ts b/electron/native-bridge/services/systemService.ts new file mode 100644 index 0000000..50eff28 --- /dev/null +++ b/electron/native-bridge/services/systemService.ts @@ -0,0 +1,43 @@ +import type { + CursorCapabilities, + NativePlatform, + SystemCapabilities, +} from "../../../src/native/contracts"; +import { NATIVE_BRIDGE_VERSION } from "../../../src/native/contracts"; +import type { NativeBridgeStateStore } from "../store"; + +interface SystemServiceOptions { + store: NativeBridgeStateStore; + getPlatform: () => NativePlatform; + getAssetBasePath: () => string | null; + getCursorCapabilities: () => Promise; +} + +export class SystemService { + constructor(private readonly options: SystemServiceOptions) {} + + getPlatform() { + return this.options.getPlatform(); + } + + getAssetBasePath() { + return this.options.getAssetBasePath(); + } + + async getCapabilities(): Promise { + const platform = this.getPlatform(); + const cursorCapabilities = await this.options.getCursorCapabilities(); + + const capabilities: SystemCapabilities = { + bridgeVersion: NATIVE_BRIDGE_VERSION, + platform, + cursor: cursorCapabilities, + project: { + currentContext: true, + }, + }; + + this.options.store.setSystemCapabilities(capabilities); + return capabilities; + } +} diff --git a/electron/native-bridge/store.ts b/electron/native-bridge/store.ts new file mode 100644 index 0000000..dcdbed1 --- /dev/null +++ b/electron/native-bridge/store.ts @@ -0,0 +1,88 @@ +import type { + CursorCapabilities, + NativePlatform, + ProjectContext, + SystemCapabilities, +} from "../../src/native/contracts"; + +export interface NativeBridgeState { + system: { + platform: NativePlatform; + capabilities: SystemCapabilities | null; + }; + project: ProjectContext; + cursor: { + capabilities: CursorCapabilities | null; + lastTelemetryLoad: { + videoPath: string; + sampleCount: number; + loadedAt: number; + } | null; + }; +} + +export class NativeBridgeStateStore { + private state: NativeBridgeState; + + constructor(platform: NativePlatform) { + this.state = { + system: { + platform, + capabilities: null, + }, + project: { + currentProjectPath: null, + currentVideoPath: null, + }, + cursor: { + capabilities: null, + lastTelemetryLoad: null, + }, + }; + } + + getState() { + return this.state; + } + + setProjectContext(project: ProjectContext) { + this.state = { + ...this.state, + project, + }; + } + + setSystemCapabilities(capabilities: SystemCapabilities) { + this.state = { + ...this.state, + system: { + ...this.state.system, + capabilities, + }, + }; + } + + setCursorCapabilities(capabilities: CursorCapabilities) { + this.state = { + ...this.state, + cursor: { + ...this.state.cursor, + capabilities, + }, + }; + } + + markCursorTelemetryLoaded(videoPath: string, sampleCount: number) { + this.state = { + ...this.state, + cursor: { + ...this.state.cursor, + lastTelemetryLoad: { + videoPath, + sampleCount, + loadedAt: Date.now(), + }, + }, + }; + } +} diff --git a/electron/native/README.md b/electron/native/README.md new file mode 100644 index 0000000..59930ba --- /dev/null +++ b/electron/native/README.md @@ -0,0 +1,111 @@ +# Native capture helpers + +## macOS + +macOS native recording will use a ScreenCaptureKit helper with the same process boundary as the Windows WGC helper: + +1. Electron resolves the selected source, output paths, and user-selected devices. +2. The helper receives one structured JSON request. +3. The helper owns ScreenCaptureKit/AVFoundation capture, timing, encoding, and muxing. +4. Electron persists the resulting media/session manifest and reports helper errors explicitly. + +Helper locations: + +1. `OPENSCREEN_SCK_CAPTURE_EXE`, for local development and diagnostics. +2. `electron/native/screencapturekit/build/openscreen-screencapturekit-helper`, for locally built Swift output. +3. `electron/native/bin/darwin-arm64/openscreen-screencapturekit-helper` or `electron/native/bin/darwin-x64/openscreen-screencapturekit-helper`, for packaged prebuilt helpers. + +The macOS cursor-shape helper is resolved from `OPENSCREEN_MAC_CURSOR_HELPER_EXE` first, then the matching `openscreen-macos-cursor-helper` binary in the same local build and packaged `electron/native/bin/darwin-${arch}` directories. + +Build the macOS helper with: + +```bash +npm run build:native:mac +``` + +On non-macOS hosts this command exits successfully and does not affect Windows/Linux development. On macOS it builds the Swift package at `electron/native/screencapturekit`, writes the development binaries to `electron/native/screencapturekit/build`, and copies redistributable binaries to `electron/native/bin/darwin-${arch}`. + +The current helper implementation supports display/window ScreenCaptureKit video capture, cursor exclusion through `SCStreamConfiguration.showsCursor`, H.264 encoding, MP4 muxing, and ScreenCaptureKit system audio. It also attempts native ScreenCaptureKit microphone capture when the running macOS version exposes that capability. Webcam recording currently stays as an Electron sidecar and is attached to the same recording session after the native screen capture stops. + +Electron exposes `is-native-mac-capture-available` for capability probing. It resolves the same helper locations listed above and reports `missing-helper` until a Swift helper binary is present. When available, macOS recording routes screen/window capture through the native helper so editable cursor recordings do not bake the system cursor into the video. Cursor positions are sampled in Electron; when the cursor helper is available and Accessibility is granted, samples are also tagged with link/text cursor hints such as `pointer`. + +See `docs/engineering/macos-native-recorder-roadmap.md` for the contract, rollout phases, and SSOT rules. + +## Windows + +Windows native recording is resolved from one of these locations: + +1. `OPENSCREEN_WGC_CAPTURE_EXE`, for local development and diagnostics. +2. `electron/native/wgc-capture/build/wgc-capture.exe`, for a locally built Ninja helper. +3. `electron/native/wgc-capture/build/Release/wgc-capture.exe`, for a locally built multi-config helper. +4. `electron/native/bin/win32-x64/wgc-capture.exe` or `electron/native/bin/win32-arm64/wgc-capture.exe`, for packaged prebuilt helpers. + +Build the Windows helper with: + +```powershell +npm run build:native:win +``` + +The build writes the CMake output to `electron/native/wgc-capture/build/wgc-capture.exe` and copies the redistributable binary to `electron/native/bin/win32-x64/wgc-capture.exe`. + +The helper contract is process-based: the app starts the process with one JSON argument and sends commands on stdin. `stop\n` finalizes the recording. During migration the helper prints both newline-delimited JSON events and the legacy text messages `Recording started` / `Recording stopped. Output path: `. + +Current V2 JSON shape: + +```json +{ + "schemaVersion": 2, + "recordingId": 123, + "sourceType": "display", + "sourceId": "screen:0:0", + "displayId": 1, + "windowHandle": null, + "outputPath": "C:\\path\\recording-123.mp4", + "videoWidth": 1920, + "videoHeight": 1080, + "fps": 60, + "captureSystemAudio": false, + "captureMic": false, + "microphoneDeviceId": "default", + "microphoneDeviceName": "Microphone (NVIDIA Broadcast)", + "microphoneGain": 1.4, + "webcamEnabled": true, + "webcamDeviceId": "default", + "webcamDeviceName": "Camera (NVIDIA Broadcast)", + "webcamWidth": 1280, + "webcamHeight": 720, + "webcamFps": 30, + "outputs": { + "screenPath": "C:\\path\\recording-123.mp4" + } +} +``` + +The current helper implementation supports display/window video capture, system audio loopback, selected-microphone capture, Media Foundation webcam capture, and a DirectShow webcam fallback for virtual cameras that are not exposed through Media Foundation. Webcam frames are currently composed into the primary MP4 as a bottom-right picture-in-picture overlay. Browser `deviceId` values do not always map to Media Foundation symbolic links or WASAPI endpoint IDs, so the renderer passes both browser IDs and user-visible device names. For microphones, the helper tries the requested WASAPI endpoint ID first, then resolves an active capture endpoint by `microphoneDeviceName`, then falls back to the default endpoint. For webcams, Electron resolves a matching DirectShow filter CLSID for the selected label; the helper uses Media Foundation first, then that exact DirectShow filter when the requested camera is absent from Media Foundation. + +Smoke-test the helper with: + +```powershell +npm run test:wgc-helper:win +npm run test:wgc-window:win +npm run test:wgc-audio:win +npm run test:wgc-mic:win +npm run test:wgc-mixed-audio:win +npm run test:wgc-webcam:win +``` + +To validate a specific native webcam manually: + +```powershell +$env:OPENSCREEN_WGC_TEST_WEBCAM_DEVICE_NAME = "NVIDIA Broadcast" +npm run test:wgc-webcam:win +Remove-Item Env:OPENSCREEN_WGC_TEST_WEBCAM_DEVICE_NAME +``` + +To validate a specific native microphone manually: + +```powershell +$env:OPENSCREEN_WGC_TEST_MICROPHONE_DEVICE_NAME = "Microphone (NVIDIA Broadcast)" +npm run test:wgc-mic:win +Remove-Item Env:OPENSCREEN_WGC_TEST_MICROPHONE_DEVICE_NAME +``` diff --git a/electron/native/screencapturekit/Package.swift b/electron/native/screencapturekit/Package.swift new file mode 100644 index 0000000..ec3b1d9 --- /dev/null +++ b/electron/native/screencapturekit/Package.swift @@ -0,0 +1,30 @@ +// swift-tools-version: 5.9 + +import PackageDescription + +let package = Package( + name: "OpenScreenScreenCaptureKitHelper", + platforms: [ + .macOS(.v13) + ], + products: [ + .executable( + name: "openscreen-screencapturekit-helper", + targets: ["OpenScreenScreenCaptureKitHelper"] + ), + .executable( + name: "openscreen-macos-cursor-helper", + targets: ["OpenScreenMacOSCursorHelper"] + ) + ], + targets: [ + .executableTarget( + name: "OpenScreenScreenCaptureKitHelper", + path: "Sources/OpenScreenScreenCaptureKitHelper" + ), + .executableTarget( + name: "OpenScreenMacOSCursorHelper", + path: "Sources/OpenScreenMacOSCursorHelper" + ) + ] +) diff --git a/electron/native/screencapturekit/Sources/OpenScreenMacOSCursorHelper/main.swift b/electron/native/screencapturekit/Sources/OpenScreenMacOSCursorHelper/main.swift new file mode 100644 index 0000000..672e86f --- /dev/null +++ b/electron/native/screencapturekit/Sources/OpenScreenMacOSCursorHelper/main.swift @@ -0,0 +1,268 @@ +import AppKit +import ApplicationServices +import Foundation + +struct CursorHelperRequest: Decodable { + let sampleIntervalMs: Int? +} + +final class MouseButtonTracker { + private let lock = NSLock() + private var leftDownCount = 0 + private var leftUpCount = 0 + private var eventTap: CFMachPort? + private var runLoopSource: CFRunLoopSource? + + struct Events { + let leftDownCount: Int + let leftUpCount: Int + } + + func start() -> Bool { + let mask = + (1 << CGEventType.leftMouseDown.rawValue) | + (1 << CGEventType.leftMouseUp.rawValue) + guard let tap = CGEvent.tapCreate( + tap: .cgSessionEventTap, + place: .headInsertEventTap, + options: .listenOnly, + eventsOfInterest: CGEventMask(mask), + callback: { _, type, event, userInfo in + if let userInfo { + let tracker = Unmanaged.fromOpaque(userInfo).takeUnretainedValue() + tracker.record(type) + } + return Unmanaged.passUnretained(event) + }, + userInfo: UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque()) + ) else { + return false + } + + guard let source = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, tap, 0) else { + return false + } + + eventTap = tap + runLoopSource = source + CFRunLoopAddSource(CFRunLoopGetCurrent(), source, .commonModes) + CGEvent.tapEnable(tap: tap, enable: true) + return true + } + + func pump() { + CFRunLoopRunInMode(.defaultMode, 0.001, false) + } + + func consume() -> Events { + lock.lock() + defer { lock.unlock() } + let events = Events(leftDownCount: leftDownCount, leftUpCount: leftUpCount) + leftDownCount = 0 + leftUpCount = 0 + return events + } + + private func record(_ type: CGEventType) { + lock.lock() + defer { lock.unlock() } + if type == .tapDisabledByTimeout || type == .tapDisabledByUserInput { + reenableTap() + return + } + if type == .leftMouseDown { + leftDownCount += 1 + } else if type == .leftMouseUp { + leftUpCount += 1 + } + } + + private func reenableTap() { + if let eventTap { + CGEvent.tapEnable(tap: eventTap, enable: true) + } + } +} + +func emit(_ fields: [String: Any?]) { + let compacted = fields.compactMapValues { $0 } + if let data = try? JSONSerialization.data(withJSONObject: compacted, options: []), + let line = String(data: data, encoding: .utf8) + { + print(line) + fflush(stdout) + } +} + +func stringAttribute(_ element: AXUIElement, _ attribute: String) -> String? { + var value: CFTypeRef? + let result = AXUIElementCopyAttributeValue(element, attribute as CFString, &value) + guard result == .success else { + return nil + } + + return value as? String +} + +func parentElement(_ element: AXUIElement) -> AXUIElement? { + var value: CFTypeRef? + let result = AXUIElementCopyAttributeValue(element, kAXParentAttribute as CFString, &value) + guard result == .success else { + return nil + } + + guard CFGetTypeID(value) == AXUIElementGetTypeID() else { + return nil + } + + return (value as! AXUIElement) +} + +func roleDescription(_ element: AXUIElement) -> String? { + var value: CFTypeRef? + let result = AXUIElementCopyAttributeValue(element, kAXRoleDescriptionAttribute as CFString, &value) + guard result == .success else { + return nil + } + + return value as? String +} + +func actionNames(_ element: AXUIElement) -> [String] { + var value: CFArray? + let result = AXUIElementCopyActionNames(element, &value) + guard result == .success, let value else { + return [] + } + + return (value as NSArray).compactMap { $0 as? String } +} +func isTextInputRole(_ role: String?) -> Bool { + role == "AXTextField" || + role == "AXTextArea" || + role == "AXTextView" || + role == "AXComboBox" +} + +func isPointerRole(_ role: String?, _ subrole: String?, _ description: String?) -> Bool { + if role == "AXLink" || + subrole?.localizedCaseInsensitiveContains("link") == true || + description?.contains("link") == true + { + return true + } + + return role == "AXButton" || + role == "AXMenuButton" || + role == "AXPopUpButton" || + role == "AXCheckBox" || + role == "AXRadioButton" || + role == "AXSwitch" || + role == "AXDisclosureTriangle" || + role == "AXTab" || + role == "AXMenuItem" +} + +func cursorTypeForElement(_ element: AXUIElement) -> String? { + var current: AXUIElement? = element + + for _ in 0..<5 { + guard let element = current else { + break + } + + let role = stringAttribute(element, kAXRoleAttribute) + let subrole = stringAttribute(element, kAXSubroleAttribute) + let description = roleDescription(element)?.lowercased() + + if isTextInputRole(role) { + return "text" + } + + if isPointerRole(role, subrole, description) { + return "pointer" + } + + current = parentElement(element) + } + + return nil +} + +func accessibilityPointForMouse() -> CGPoint { + let mouse = NSEvent.mouseLocation + let primaryHeight = NSScreen.screens.first?.frame.height ?? NSScreen.main?.frame.height ?? 0 + return CGPoint(x: mouse.x, y: primaryHeight - mouse.y) +} + +func currentCursorType() -> String? { + guard AXIsProcessTrusted() else { + return nil + } + + let point = accessibilityPointForMouse() + let systemWide = AXUIElementCreateSystemWide() + var element: AXUIElement? + let result = AXUIElementCopyElementAtPosition( + systemWide, + Float(point.x), + Float(point.y), + &element + ) + + guard result == .success, let element else { + return "arrow" + } + + return cursorTypeForElement(element) ?? "arrow" +} + +func timestampMs() -> Int { + Int(Date().timeIntervalSince1970 * 1000) +} + +func leftButtonDown() -> Bool { + CGEventSource.buttonState(.hidSystemState, button: .left) +} + +func requestAccessibilityTrust() -> Bool { + let options = [ + kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true + ] as CFDictionary + return AXIsProcessTrustedWithOptions(options) +} + +let request: CursorHelperRequest +if CommandLine.arguments.count >= 2, + let data = CommandLine.arguments[1].data(using: .utf8), + let decoded = try? JSONDecoder().decode(CursorHelperRequest.self, from: data) +{ + request = decoded +} else { + request = CursorHelperRequest(sampleIntervalMs: nil) +} + +let intervalMs = max(8, request.sampleIntervalMs ?? 33) +let accessibilityTrusted = requestAccessibilityTrust() +let mouseTracker = MouseButtonTracker() +let mouseTapReady = mouseTracker.start() +emit([ + "type": "ready", + "timestampMs": timestampMs(), + "accessibilityTrusted": accessibilityTrusted, + "mouseTapReady": mouseTapReady, +]) + +while true { + mouseTracker.pump() + let mouseEvents = mouseTracker.consume() + emit([ + "type": "sample", + "timestampMs": timestampMs(), + "cursorType": currentCursorType(), + "leftButtonDown": leftButtonDown(), + "leftButtonPressed": mouseEvents.leftDownCount > 0, + "leftButtonReleased": mouseEvents.leftUpCount > 0, + ]) + Thread.sleep(forTimeInterval: Double(intervalMs) / 1000.0) +} diff --git a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/main.swift b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/main.swift new file mode 100644 index 0000000..14860b0 --- /dev/null +++ b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/main.swift @@ -0,0 +1,673 @@ +import AVFoundation +import CoreGraphics +import CoreMedia +import Foundation +import ScreenCaptureKit + +struct Rectangle: Decodable { + let x: Double + let y: Double + let width: Double + let height: Double +} + +struct RecordingRequest: Decodable { + struct Source: Decodable { + let type: String + let sourceId: String + let displayId: UInt32? + let windowId: UInt32? + let bounds: Rectangle? + } + + struct Video: Decodable { + let fps: Int + let width: Int + let height: Int + let bitrate: Int? + let hideSystemCursor: Bool + } + + struct Audio: Decodable { + struct SystemAudio: Decodable { + let enabled: Bool + } + + struct Microphone: Decodable { + let enabled: Bool + let deviceId: String? + let deviceName: String? + let gain: Double + } + + let system: SystemAudio + let microphone: Microphone + } + + struct Webcam: Decodable { + let enabled: Bool + let deviceId: String? + let deviceName: String? + let width: Int + let height: Int + let fps: Int + } + + struct Cursor: Decodable { + let mode: String + } + + struct Outputs: Decodable { + let screenPath: String + let manifestPath: String? + } + + let schemaVersion: Int? + let recordingId: Int? + let source: Source + let video: Video + let audio: Audio + let webcam: Webcam + let cursor: Cursor + let outputs: Outputs +} + +enum HelperError: Error, CustomStringConvertible { + case invalidArguments + case unsupportedMacOS + case unsupportedFeature(String) + case sourceNotFound(String) + case invalidSourceType(String) + case permissionDenied(String) + case writerSetupFailed(String) + + var description: String { + switch self { + case .invalidArguments: + return "Expected one JSON recording request argument." + case .unsupportedMacOS: + return "ScreenCaptureKit recording requires macOS 13 or newer." + case .unsupportedFeature(let message): + return message + case .sourceNotFound(let message): + return message + case .invalidSourceType(let sourceType): + return "Unsupported source type: \(sourceType)." + case .permissionDenied(let message): + return message + case .writerSetupFailed(let message): + return message + } + } +} + +func emit(_ fields: [String: Any]) { + if let data = try? JSONSerialization.data(withJSONObject: fields, options: []), + let line = String(data: data, encoding: .utf8) + { + print(line) + fflush(stdout) + } +} + +func emitError(code: String, message: String) { + emit([ + "event": "error", + "code": code, + "message": message, + ]) +} + +@available(macOS 13.0, *) +final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { + private struct CaptureTarget { + let filter: SCContentFilter + let width: Int + let height: Int + } + + private let request: RecordingRequest + private let sampleQueue = DispatchQueue(label: "app.openscreen.sck-helper.samples") + private let stateQueue = DispatchQueue(label: "app.openscreen.sck-helper.state") + private var stream: SCStream? + private var writer: AVAssetWriter? + private var videoInput: AVAssetWriterInput? + private var systemAudioInput: AVAssetWriterInput? + private var microphoneAudioInput: AVAssetWriterInput? + private var didStartWriting = false + private var didEmitRecordingStarted = false + private var isStopping = false + private var isPaused = false + private var pauseStartedAt: CMTime? + private var totalPausedDuration = CMTime.zero + private var nativeMicrophoneEnabled = false + private var outputWidth = 1920 + private var outputHeight = 1080 + private let microphoneOutputTypeRawValue = 2 + private let hostClock = CMClockGetHostTimeClock() + + init(request: RecordingRequest) { + self.request = request + } + + func start() async throws { + try ensureRequestedPermissions() + + let content = try await SCShareableContent.excludingDesktopWindows( + false, + onScreenWindowsOnly: true + ) + let target = try makeCaptureTarget(from: content) + outputWidth = target.width + outputHeight = target.height + let configuration = makeStreamConfiguration() + let stream = SCStream(filter: target.filter, configuration: configuration, delegate: self) + + try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: sampleQueue) + if request.audio.system.enabled { + try stream.addStreamOutput(self, type: .audio, sampleHandlerQueue: sampleQueue) + } + if nativeMicrophoneEnabled { + guard let microphoneOutputType = SCStreamOutputType(rawValue: microphoneOutputTypeRawValue) else { + throw HelperError.unsupportedFeature( + "Native microphone capture requires a macOS version with ScreenCaptureKit microphone output." + ) + } + try stream.addStreamOutput(self, type: microphoneOutputType, sampleHandlerQueue: sampleQueue) + } + try setupWriter() + + self.stream = stream + emit(["event": "ready", "schemaVersion": 1]) + try await stream.startCapture() + } + + func stop() async { + let shouldStop = stateQueue.sync { + if isStopping { + return false + } + isStopping = true + return true + } + if !shouldStop { + return + } + + do { + try await stream?.stopCapture() + } catch { + emit([ + "event": "warning", + "code": "stop-capture-failed", + "message": "\(error)", + ]) + } + + await finishWriter() + } + + func pause() { + let didPause = stateQueue.sync { + if isStopping || isPaused { + return false + } + + isPaused = true + pauseStartedAt = CMClockGetTime(hostClock) + return true + } + + if didPause { + emit([ + "event": "recording-paused", + "timestampMs": Int(Date().timeIntervalSince1970 * 1000), + ]) + } + } + + func resume() { + let didResume = stateQueue.sync { + if isStopping || !isPaused { + return false + } + + if let pauseStartedAt { + let now = CMClockGetTime(hostClock) + totalPausedDuration = CMTimeAdd( + totalPausedDuration, + CMTimeSubtract(now, pauseStartedAt) + ) + } + isPaused = false + pauseStartedAt = nil + return true + } + + if didResume { + emit([ + "event": "recording-resumed", + "timestampMs": Int(Date().timeIntervalSince1970 * 1000), + ]) + } + } + + func stream(_ stream: SCStream, didStopWithError error: Error) { + emitError(code: "capture-stopped-with-error", message: "\(error)") + Task { + await stop() + } + } + + func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) { + guard CMSampleBufferDataIsReady(sampleBuffer) else { + return + } + let pauseState = currentPauseState() + if pauseState.paused { + return + } + guard let sampleBuffer = retimedSampleBuffer(sampleBuffer, subtracting: pauseState.offset) else { + return + } + + if type == .audio { + appendAudioSampleBuffer(sampleBuffer, to: systemAudioInput) + return + } + + if type.rawValue == microphoneOutputTypeRawValue { + appendAudioSampleBuffer(sampleBuffer, to: microphoneAudioInput) + return + } + + guard type == .screen else { + return + } + guard isCompleteFrame(sampleBuffer) else { + return + } + guard let videoInput, let writer else { + return + } + let presentationTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) + if !didStartWriting { + writer.startWriting() + writer.startSession(atSourceTime: presentationTime) + didStartWriting = true + } + + if videoInput.isReadyForMoreMediaData { + if videoInput.append(sampleBuffer), !didEmitRecordingStarted { + didEmitRecordingStarted = true + emit([ + "event": "recording-started", + "timestampMs": Int(Date().timeIntervalSince1970 * 1000), + "width": outputWidth, + "height": outputHeight, + ]) + } + } + } + + private func ensureRequestedPermissions() throws { + if !CGPreflightScreenCaptureAccess() { + let granted = CGRequestScreenCaptureAccess() + if !granted { + throw HelperError.permissionDenied("Screen recording permission is required for ScreenCaptureKit capture.") + } + } + + if request.audio.microphone.enabled { + switch AVCaptureDevice.authorizationStatus(for: .audio) { + case .authorized: + break + case .notDetermined: + let semaphore = DispatchSemaphore(value: 0) + AVCaptureDevice.requestAccess(for: .audio) { _ in + semaphore.signal() + } + let waitResult = semaphore.wait(timeout: .now() + 30) + if waitResult == .timedOut || AVCaptureDevice.authorizationStatus(for: .audio) != .authorized { + throw HelperError.permissionDenied("Microphone permission is required for native microphone capture.") + } + default: + throw HelperError.permissionDenied("Microphone permission is required for native microphone capture.") + } + } + } + + private func makeCaptureTarget(from content: SCShareableContent) throws -> CaptureTarget { + switch request.source.type { + case "display": + guard let displayId = request.source.displayId else { + throw HelperError.sourceNotFound("Display capture requires source.displayId.") + } + guard let display = content.displays.first(where: { $0.displayID == displayId }) else { + throw HelperError.sourceNotFound("No ScreenCaptureKit display found for id \(displayId).") + } + let width = Int(CGDisplayPixelsWide(display.displayID)) + let height = Int(CGDisplayPixelsHigh(display.displayID)) + return CaptureTarget( + filter: SCContentFilter(display: display, excludingWindows: []), + width: clampCaptureDimension(width, fallback: request.video.width), + height: clampCaptureDimension(height, fallback: request.video.height) + ) + case "window": + guard let windowId = request.source.windowId else { + throw HelperError.sourceNotFound("Window capture requires source.windowId.") + } + guard let window = content.windows.first(where: { $0.windowID == windowId }) else { + throw HelperError.sourceNotFound("No ScreenCaptureKit window found for id \(windowId).") + } + let candidateDisplay = content.displays.first { + $0.frame.intersects(window.frame) || $0.frame.contains(CGPoint(x: window.frame.midX, y: window.frame.midY)) + } + let scaleFactor = Self.scaleFactor(for: candidateDisplay?.displayID ?? CGMainDisplayID()) + let width = Int(window.frame.width) * scaleFactor + let height = Int(window.frame.height) * scaleFactor + return CaptureTarget( + filter: SCContentFilter(desktopIndependentWindow: window), + width: clampCaptureDimension(width, fallback: request.video.width), + height: clampCaptureDimension(height, fallback: request.video.height) + ) + default: + throw HelperError.invalidSourceType(request.source.type) + } + } + + private func makeStreamConfiguration() -> SCStreamConfiguration { + let configuration = SCStreamConfiguration() + configuration.width = outputWidth + configuration.height = outputHeight + configuration.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(max(1, request.video.fps))) + configuration.queueDepth = 6 + configuration.showsCursor = !request.video.hideSystemCursor + configuration.pixelFormat = kCVPixelFormatType_32BGRA + configuration.sampleRate = 48_000 + configuration.channelCount = 2 + configuration.excludesCurrentProcessAudio = true + configuration.capturesAudio = request.audio.system.enabled + + if request.audio.microphone.enabled { + guard supportsNativeMicrophoneCapture(streamConfig: configuration) else { + nativeMicrophoneEnabled = false + emit([ + "event": "warning", + "code": "microphone-unavailable", + "message": "Native microphone capture requires ScreenCaptureKit microphone support on this macOS version.", + ]) + return configuration + } + nativeMicrophoneEnabled = true + configuration.capturesAudio = true + configuration.setValue(true, forKey: "captureMicrophone") + if let deviceId = resolveMicrophoneCaptureDeviceID() { + configuration.setValue(deviceId, forKey: "microphoneCaptureDeviceID") + } + } else { + nativeMicrophoneEnabled = false + } + + return configuration + } + + private func setupWriter() throws { + let outputUrl = URL(fileURLWithPath: request.outputs.screenPath) + try? FileManager.default.removeItem(at: outputUrl) + try FileManager.default.createDirectory( + at: outputUrl.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + + let writer = try AVAssetWriter(outputURL: outputUrl, fileType: .mp4) + let settings: [String: Any] = [ + AVVideoCodecKey: AVVideoCodecType.h264, + AVVideoWidthKey: outputWidth, + AVVideoHeightKey: outputHeight, + AVVideoCompressionPropertiesKey: [ + AVVideoAverageBitRateKey: request.video.bitrate ?? 18_000_000, + AVVideoExpectedSourceFrameRateKey: request.video.fps, + ], + ] + let input = AVAssetWriterInput(mediaType: .video, outputSettings: settings) + input.expectsMediaDataInRealTime = true + + guard writer.canAdd(input) else { + throw HelperError.writerSetupFailed("Unable to add H.264 video input to AVAssetWriter.") + } + + writer.add(input) + self.writer = writer + self.videoInput = input + + if request.audio.system.enabled { + systemAudioInput = try addAudioInput(to: writer, bitRate: 192_000) + } + if nativeMicrophoneEnabled { + microphoneAudioInput = try addAudioInput(to: writer, bitRate: 128_000) + } + } + + private func finishWriter() async { + guard let writer else { + return + } + + videoInput?.markAsFinished() + systemAudioInput?.markAsFinished() + microphoneAudioInput?.markAsFinished() + + await withCheckedContinuation { continuation in + writer.finishWriting { + continuation.resume() + } + } + + if writer.status == .completed { + emit([ + "event": "recording-stopped", + "screenPath": request.outputs.screenPath, + ]) + } else { + emitError( + code: "writer-failed", + message: writer.error.map { "\($0)" } ?? "AVAssetWriter failed with status \(writer.status.rawValue)." + ) + } + } + + private func addAudioInput(to writer: AVAssetWriter, bitRate: Int) throws -> AVAssetWriterInput { + let settings: [String: Any] = [ + AVFormatIDKey: kAudioFormatMPEG4AAC, + AVSampleRateKey: 48_000, + AVNumberOfChannelsKey: 2, + AVEncoderBitRateKey: bitRate, + ] + let input = AVAssetWriterInput(mediaType: .audio, outputSettings: settings) + input.expectsMediaDataInRealTime = true + + guard writer.canAdd(input) else { + throw HelperError.writerSetupFailed("Unable to add AAC audio input to AVAssetWriter.") + } + + writer.add(input) + return input + } + + private func appendAudioSampleBuffer(_ sampleBuffer: CMSampleBuffer, to input: AVAssetWriterInput?) { + guard didStartWriting else { + return + } + guard let input, input.isReadyForMoreMediaData else { + return + } + + input.append(sampleBuffer) + } + + private func currentPauseState() -> (paused: Bool, offset: CMTime) { + stateQueue.sync { + (isPaused, totalPausedDuration) + } + } + + private func retimedSampleBuffer(_ sampleBuffer: CMSampleBuffer, subtracting offset: CMTime) -> CMSampleBuffer? { + if !offset.isValid || offset == .zero { + return sampleBuffer + } + + let sampleCount = CMSampleBufferGetNumSamples(sampleBuffer) + if sampleCount <= 0 { + return sampleBuffer + } + + var timing = Array(repeating: CMSampleTimingInfo(), count: sampleCount) + let timingStatus = CMSampleBufferGetSampleTimingInfoArray( + sampleBuffer, + entryCount: sampleCount, + arrayToFill: &timing, + entriesNeededOut: nil + ) + if timingStatus != noErr { + emit([ + "event": "warning", + "code": "sample-retime-failed", + "message": "Unable to read sample timing info: \(timingStatus).", + ]) + return sampleBuffer + } + + for index in timing.indices { + if timing[index].presentationTimeStamp.isValid { + timing[index].presentationTimeStamp = CMTimeSubtract( + timing[index].presentationTimeStamp, + offset + ) + } + if timing[index].decodeTimeStamp.isValid { + timing[index].decodeTimeStamp = CMTimeSubtract(timing[index].decodeTimeStamp, offset) + } + } + + var retimedBuffer: CMSampleBuffer? + let copyStatus = CMSampleBufferCreateCopyWithNewTiming( + allocator: kCFAllocatorDefault, + sampleBuffer: sampleBuffer, + sampleTimingEntryCount: sampleCount, + sampleTimingArray: &timing, + sampleBufferOut: &retimedBuffer + ) + if copyStatus != noErr { + emit([ + "event": "warning", + "code": "sample-retime-failed", + "message": "Unable to copy sample timing info: \(copyStatus).", + ]) + return sampleBuffer + } + + return retimedBuffer + } + + private func isCompleteFrame(_ sampleBuffer: CMSampleBuffer) -> Bool { + guard let attachments = CMSampleBufferGetSampleAttachmentsArray( + sampleBuffer, + createIfNecessary: false + ) as? [[SCStreamFrameInfo: Any]], + let attachment = attachments.first, + let statusRawValue = attachment[SCStreamFrameInfo.status] as? Int, + let status = SCFrameStatus(rawValue: statusRawValue) + else { + return true + } + + return status == .complete + } + + private func clampCaptureDimension(_ value: Int, fallback: Int) -> Int { + let requested = max(2, fallback) + let candidate = value > 0 ? value : requested + let clamped = min(candidate, requested) + return max(2, clamped - (clamped % 2)) + } + + private static func scaleFactor(for displayId: CGDirectDisplayID) -> Int { + guard let mode = CGDisplayCopyDisplayMode(displayId) else { + return 1 + } + + return max(1, mode.pixelWidth / max(1, mode.width)) + } + + private func supportsNativeMicrophoneCapture(streamConfig: SCStreamConfiguration) -> Bool { + streamConfig.responds(to: Selector(("setCaptureMicrophone:"))) && + streamConfig.responds(to: Selector(("setMicrophoneCaptureDeviceID:"))) && + SCStreamOutputType(rawValue: microphoneOutputTypeRawValue) != nil + } + + private func resolveMicrophoneCaptureDeviceID() -> String? { + let devices = AVCaptureDevice.devices(for: .audio) + + if let deviceName = request.audio.microphone.deviceName?.trimmingCharacters(in: .whitespacesAndNewlines), + !deviceName.isEmpty, + let device = devices.first(where: { $0.localizedName == deviceName }) + { + return device.uniqueID + } + + if let deviceId = request.audio.microphone.deviceId?.trimmingCharacters(in: .whitespacesAndNewlines), + !deviceId.isEmpty, + devices.contains(where: { $0.uniqueID == deviceId }) + { + return deviceId + } + + return nil + } +} + +@main +struct OpenScreenScreenCaptureKitHelper { + static func main() async { + do { + guard CommandLine.arguments.count == 2 else { + throw HelperError.invalidArguments + } + + guard #available(macOS 13.0, *) else { + throw HelperError.unsupportedMacOS + } + + let requestData = Data(CommandLine.arguments[1].utf8) + let decoder = JSONDecoder() + let request = try decoder.decode(RecordingRequest.self, from: requestData) + let recorder = ScreenCaptureRecorder(request: request) + let stopTask = Task.detached { + while let line = readLine() { + let command = line.trimmingCharacters(in: .whitespacesAndNewlines) + switch command { + case "pause": + recorder.pause() + case "resume": + recorder.resume() + case "stop": + await recorder.stop() + exit(0) + default: + break + } + } + } + + try await recorder.start() + await stopTask.value + } catch let error as HelperError { + emitError(code: "helper-error", message: error.description) + exit(1) + } catch { + emitError(code: "helper-error", message: "\(error)") + exit(1) + } + } +} diff --git a/electron/native/wgc-capture/CMakeLists.txt b/electron/native/wgc-capture/CMakeLists.txt new file mode 100644 index 0000000..32c5d6e --- /dev/null +++ b/electron/native/wgc-capture/CMakeLists.txt @@ -0,0 +1,67 @@ +cmake_minimum_required(VERSION 3.20) + +# The local Windows SDK image used by some contributors can miss gdi32.lib, +# while CMake's default MSVC console template links it unconditionally. This +# helper does not use GDI, so keep the standard library set minimal and explicit. +set(CMAKE_CXX_STANDARD_LIBRARIES + "kernel32.lib user32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib comdlg32.lib advapi32.lib" + CACHE STRING "" FORCE) + +project(openscreen-wgc-capture LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +add_executable(wgc-capture + src/audio_sample_utils.cpp + src/audio_sample_utils.h + src/dshow_webcam_capture.cpp + src/dshow_webcam_capture.h + src/main.cpp + src/mf_encoder.cpp + src/mf_encoder.h + src/monitor_utils.cpp + src/monitor_utils.h + src/wasapi_loopback_capture.cpp + src/wasapi_loopback_capture.h + src/webcam_capture.cpp + src/webcam_capture.h + src/wgc_session.cpp + src/wgc_session.h +) + +target_compile_definitions(wgc-capture PRIVATE + NOMINMAX + WIN32_LEAN_AND_MEAN + _WIN32_WINNT=0x0A00 +) + +target_compile_options(wgc-capture PRIVATE /EHsc /W4 /utf-8) + +target_link_libraries(wgc-capture PRIVATE + d3d11 + dxgi + mf + mfplat + mfreadwrite + mfuuid + runtimeobject + windowsapp +) + +add_executable(cursor-sampler + src/cursor-sampler.cpp +) + +target_compile_definitions(cursor-sampler PRIVATE + NOMINMAX + _WIN32_WINNT=0x0A00 +) + +target_compile_options(cursor-sampler PRIVATE /EHsc /W4 /utf-8) + +target_link_libraries(cursor-sampler PRIVATE + gdi32 + gdiplus +) diff --git a/electron/native/wgc-capture/src/audio_sample_utils.cpp b/electron/native/wgc-capture/src/audio_sample_utils.cpp new file mode 100644 index 0000000..6b50325 --- /dev/null +++ b/electron/native/wgc-capture/src/audio_sample_utils.cpp @@ -0,0 +1,439 @@ +#include "audio_sample_utils.h" + +#include + +#include +#include +#include +#include +#include + +namespace { + +bool isFloatFormat(const AudioInputFormat& format) { + return format.subtype == MFAudioFormat_Float && format.bitsPerSample == 32; +} + +bool isPcmFormat(const AudioInputFormat& format, UINT32 bitsPerSample) { + return format.subtype == MFAudioFormat_PCM && format.bitsPerSample == bitsPerSample; +} + +template +T clampTo(double value) { + const double minValue = static_cast(std::numeric_limits::min()); + const double maxValue = static_cast(std::numeric_limits::max()); + return static_cast(std::clamp(std::round(value), minValue, maxValue)); +} + +size_t bytesPerSample(const AudioInputFormat& format) { + return format.bitsPerSample / 8; +} + +double readSampleAsDouble(const BYTE* source, const AudioInputFormat& format, size_t frameIndex, UINT32 channelIndex) { + if (!source || format.blockAlign == 0 || channelIndex >= format.channels) { + return 0.0; + } + + const size_t offset = frameIndex * format.blockAlign + channelIndex * bytesPerSample(format); + if (isFloatFormat(format)) { + return static_cast(*reinterpret_cast(source + offset)); + } + if (isPcmFormat(format, 16)) { + return static_cast(*reinterpret_cast(source + offset)) / 32768.0; + } + if (isPcmFormat(format, 32)) { + return static_cast(*reinterpret_cast(source + offset)) / 2147483648.0; + } + return 0.0; +} + +void writeSampleFromDouble(BYTE* destination, const AudioInputFormat& format, size_t frameIndex, UINT32 channelIndex, double value) { + if (!destination || format.blockAlign == 0 || channelIndex >= format.channels) { + return; + } + + const double clamped = std::clamp(value, -1.0, 1.0); + const size_t offset = frameIndex * format.blockAlign + channelIndex * bytesPerSample(format); + if (isFloatFormat(format)) { + *reinterpret_cast(destination + offset) = static_cast(clamped); + return; + } + if (isPcmFormat(format, 16)) { + *reinterpret_cast(destination + offset) = clampTo(clamped * 32767.0); + return; + } + if (isPcmFormat(format, 32)) { + *reinterpret_cast(destination + offset) = clampTo(clamped * 2147483647.0); + } +} + +double readMappedChannel(const BYTE* source, const AudioInputFormat& format, size_t frameIndex, UINT32 targetChannel, UINT32 targetChannels) { + if (format.channels == 0) { + return 0.0; + } + if (format.channels == targetChannels && targetChannel < format.channels) { + return readSampleAsDouble(source, format, frameIndex, targetChannel); + } + if (format.channels == 1) { + return readSampleAsDouble(source, format, frameIndex, 0); + } + if (targetChannels == 1) { + double sum = 0.0; + for (UINT32 channel = 0; channel < format.channels; ++channel) { + sum += readSampleAsDouble(source, format, frameIndex, channel); + } + return sum / static_cast(format.channels); + } + return readSampleAsDouble(source, format, frameIndex, std::min(targetChannel, format.channels - 1)); +} + +} // namespace + +constexpr int64_t HnsPerSecond = 10'000'000; + +bool sameAudioFormatForMixing(const AudioInputFormat& left, const AudioInputFormat& right) { + return left.subtype == right.subtype && + left.sampleRate == right.sampleRate && + left.channels == right.channels && + left.bitsPerSample == right.bitsPerSample && + left.blockAlign == right.blockAlign && + left.avgBytesPerSec == right.avgBytesPerSec; +} + +AudioInputFormat makeAacCompatibleAudioFormat(const AudioInputFormat& source) { + AudioInputFormat format{}; + format.subtype = MFAudioFormat_PCM; + format.sampleRate = source.sampleRate > 0 ? source.sampleRate : 48000; + format.channels = 2; + format.bitsPerSample = 16; + format.blockAlign = format.channels * (format.bitsPerSample / 8); + format.avgBytesPerSec = format.sampleRate * format.blockAlign; + return format; +} + +void copyAudioWithGain( + const BYTE* source, + DWORD byteCount, + const AudioInputFormat& format, + double gain, + std::vector& destination) { + destination.resize(byteCount); + if (!source || byteCount == 0) { + std::fill(destination.begin(), destination.end(), static_cast(0)); + return; + } + + if (std::abs(gain - 1.0) < 0.0001) { + std::memcpy(destination.data(), source, byteCount); + return; + } + + if (isFloatFormat(format)) { + const auto* input = reinterpret_cast(source); + auto* output = reinterpret_cast(destination.data()); + const size_t sampleCount = byteCount / sizeof(float); + for (size_t index = 0; index < sampleCount; index += 1) { + output[index] = static_cast(std::clamp(input[index] * gain, -1.0, 1.0)); + } + return; + } + + if (isPcmFormat(format, 16)) { + const auto* input = reinterpret_cast(source); + auto* output = reinterpret_cast(destination.data()); + const size_t sampleCount = byteCount / sizeof(int16_t); + for (size_t index = 0; index < sampleCount; index += 1) { + output[index] = clampTo(static_cast(input[index]) * gain); + } + return; + } + + if (isPcmFormat(format, 32)) { + const auto* input = reinterpret_cast(source); + auto* output = reinterpret_cast(destination.data()); + const size_t sampleCount = byteCount / sizeof(int32_t); + for (size_t index = 0; index < sampleCount; index += 1) { + output[index] = clampTo(static_cast(input[index]) * gain); + } + return; + } + + std::memcpy(destination.data(), source, byteCount); +} + +void convertAudioWithGain( + const BYTE* source, + DWORD byteCount, + const AudioInputFormat& sourceFormat, + const AudioInputFormat& targetFormat, + double gain, + std::vector& destination) { + if (!source || byteCount == 0 || sourceFormat.blockAlign == 0 || targetFormat.blockAlign == 0 || + sourceFormat.sampleRate == 0 || targetFormat.sampleRate == 0 || sourceFormat.channels == 0 || + targetFormat.channels == 0) { + destination.clear(); + return; + } + + if (sameAudioFormatForMixing(sourceFormat, targetFormat)) { + copyAudioWithGain(source, byteCount, targetFormat, gain, destination); + return; + } + + const size_t sourceFrames = byteCount / sourceFormat.blockAlign; + if (sourceFrames == 0) { + destination.clear(); + return; + } + + const double rateRatio = static_cast(targetFormat.sampleRate) / + static_cast(sourceFormat.sampleRate); + const size_t targetFrames = std::max(1, static_cast(std::llround(sourceFrames * rateRatio))); + destination.assign(targetFrames * targetFormat.blockAlign, 0); + + for (size_t targetFrame = 0; targetFrame < targetFrames; ++targetFrame) { + const double sourcePosition = static_cast(targetFrame) / rateRatio; + const size_t sourceFrame = std::min( + sourceFrames - 1, + static_cast(std::llround(sourcePosition))); + for (UINT32 channel = 0; channel < targetFormat.channels; ++channel) { + const double sample = readMappedChannel( + source, + sourceFormat, + sourceFrame, + channel, + targetFormat.channels); + writeSampleFromDouble(destination.data(), targetFormat, targetFrame, channel, sample * gain); + } + } +} + +void mixAudioInPlace( + std::vector& destination, + const BYTE* source, + DWORD byteCount, + const AudioInputFormat& format) { + if (!source || byteCount == 0 || destination.empty()) { + return; + } + + const size_t mixByteCount = std::min(destination.size(), static_cast(byteCount)); + + if (isFloatFormat(format)) { + auto* output = reinterpret_cast(destination.data()); + const auto* input = reinterpret_cast(source); + const size_t sampleCount = mixByteCount / sizeof(float); + for (size_t index = 0; index < sampleCount; index += 1) { + output[index] = static_cast(std::clamp(output[index] + input[index], -1.0f, 1.0f)); + } + return; + } + + if (isPcmFormat(format, 16)) { + auto* output = reinterpret_cast(destination.data()); + const auto* input = reinterpret_cast(source); + const size_t sampleCount = mixByteCount / sizeof(int16_t); + for (size_t index = 0; index < sampleCount; index += 1) { + output[index] = clampTo( + static_cast(output[index]) + static_cast(input[index])); + } + return; + } + + if (isPcmFormat(format, 32)) { + auto* output = reinterpret_cast(destination.data()); + const auto* input = reinterpret_cast(source); + const size_t sampleCount = mixByteCount / sizeof(int32_t); + for (size_t index = 0; index < sampleCount; index += 1) { + output[index] = clampTo( + static_cast(output[index]) + static_cast(input[index])); + } + } +} + +AudioMixer::AudioMixer( + const AudioInputFormat& format, + const AudioInputFormat& systemFormat, + const AudioInputFormat& microphoneFormat, + bool includeSystem, + bool includeMicrophone, + double microphoneGain, + OutputCallback output) + : format_(format), + systemFormat_(systemFormat), + microphoneFormat_(microphoneFormat), + includeSystem_(includeSystem), + includeMicrophone_(includeMicrophone), + microphoneGain_(microphoneGain), + output_(std::move(output)) {} + +AudioMixer::~AudioMixer() { + stop(); +} + +bool AudioMixer::start() { + if (!output_ || format_.sampleRate == 0 || format_.blockAlign == 0) { + return false; + } + + stopRequested_ = false; + emittedFrames_ = 0; + timelineStarted_ = false; + paused_ = false; + thread_ = std::thread([this] { + mixLoop(); + }); + return true; +} + +void AudioMixer::beginTimeline() { + { + std::scoped_lock lock(mutex_); + systemQueue_.clear(); + microphoneQueue_.clear(); + emittedFrames_ = 0; + timelineStarted_ = true; + } + cv_.notify_all(); +} + +void AudioMixer::setPaused(bool paused) { + { + std::scoped_lock lock(mutex_); + paused_ = paused; + if (paused_) { + systemQueue_.clear(); + microphoneQueue_.clear(); + } + } + cv_.notify_all(); +} + +void AudioMixer::stop() { + stopRequested_ = true; + cv_.notify_all(); + if (thread_.joinable()) { + thread_.join(); + } +} + +void AudioMixer::pushSystem(const BYTE* data, DWORD byteCount) { + if (!includeSystem_ || stopRequested_) { + return; + } + + { + std::scoped_lock lock(mutex_); + if (paused_) { + return; + } + append(systemQueue_, data, byteCount, systemFormat_, 1.0); + } + cv_.notify_all(); +} + +void AudioMixer::pushMicrophone(const BYTE* data, DWORD byteCount) { + if (!includeMicrophone_ || stopRequested_) { + return; + } + + { + std::scoped_lock lock(mutex_); + if (paused_) { + return; + } + append(microphoneQueue_, data, byteCount, microphoneFormat_, microphoneGain_); + } + cv_.notify_all(); +} + +void AudioMixer::append( + std::vector& queue, + const BYTE* data, + DWORD byteCount, + const AudioInputFormat& sourceFormat, + double gain) { + if (!data || byteCount == 0) { + return; + } + + convertAudioWithGain(data, byteCount, sourceFormat, format_, gain, gainBuffer_); + queue.insert(queue.end(), gainBuffer_.begin(), gainBuffer_.end()); +} + +bool AudioMixer::pop(std::vector& queue, std::vector& chunk, size_t byteCount) { + if (queue.empty()) { + chunk.assign(byteCount, 0); + return false; + } + + chunk.assign(byteCount, 0); + const size_t copiedBytes = std::min(byteCount, queue.size()); + std::memcpy(chunk.data(), queue.data(), copiedBytes); + queue.erase(queue.begin(), queue.begin() + static_cast(copiedBytes)); + return copiedBytes > 0; +} + +void AudioMixer::mixLoop() { + const uint32_t chunkFrames = std::max(1, format_.sampleRate / 100); + const size_t chunkBytes = static_cast(chunkFrames) * format_.blockAlign; + std::vector mixedChunk; + std::vector sourceChunk; + std::chrono::steady_clock::time_point audioClockStart; + bool audioClockStarted = false; + + while (true) { + { + std::unique_lock lock(mutex_); + cv_.wait_for(lock, std::chrono::milliseconds(20), [&] { + const bool hasSystem = !includeSystem_ || systemQueue_.size() >= chunkBytes; + const bool hasMicrophone = !includeMicrophone_ || microphoneQueue_.size() >= chunkBytes; + const bool hasAnySource = !systemQueue_.empty() || !microphoneQueue_.empty(); + return stopRequested_.load() || + (timelineStarted_ && !paused_ && (hasSystem || hasMicrophone) && hasAnySource); + }); + + if (stopRequested_) { + break; + } + if (!timelineStarted_ || paused_) { + continue; + } + + const bool hasAnyQueuedAudio = !systemQueue_.empty() || !microphoneQueue_.empty(); + if (!hasAnyQueuedAudio) { + continue; + } + + mixedChunk.assign(chunkBytes, 0); + if (includeSystem_) { + pop(systemQueue_, sourceChunk, chunkBytes); + mixAudioInPlace(mixedChunk, sourceChunk.data(), static_cast(sourceChunk.size()), format_); + } + if (includeMicrophone_) { + pop(microphoneQueue_, sourceChunk, chunkBytes); + mixAudioInPlace(mixedChunk, sourceChunk.data(), static_cast(sourceChunk.size()), format_); + } + } + + if (!audioClockStarted) { + audioClockStart = std::chrono::steady_clock::now(); + audioClockStarted = true; + } + + const int64_t timestampHns = + static_cast((emittedFrames_ * HnsPerSecond) / format_.sampleRate); + const int64_t durationHns = + static_cast((static_cast(chunkFrames) * HnsPerSecond) / format_.sampleRate); + if (!output_(mixedChunk.data(), static_cast(mixedChunk.size()), timestampHns, durationHns)) { + stopRequested_ = true; + break; + } + emittedFrames_ += chunkFrames; + + const auto nextDeadline = audioClockStart + + std::chrono::duration_cast( + std::chrono::duration(static_cast(emittedFrames_) / format_.sampleRate)); + std::this_thread::sleep_until(nextDeadline); + } +} diff --git a/electron/native/wgc-capture/src/audio_sample_utils.h b/electron/native/wgc-capture/src/audio_sample_utils.h new file mode 100644 index 0000000..0bdbc08 --- /dev/null +++ b/electron/native/wgc-capture/src/audio_sample_utils.h @@ -0,0 +1,87 @@ +#pragma once + +#include "mf_encoder.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +bool sameAudioFormatForMixing(const AudioInputFormat& left, const AudioInputFormat& right); +AudioInputFormat makeAacCompatibleAudioFormat(const AudioInputFormat& source); +void copyAudioWithGain( + const BYTE* source, + DWORD byteCount, + const AudioInputFormat& format, + double gain, + std::vector& destination); +void convertAudioWithGain( + const BYTE* source, + DWORD byteCount, + const AudioInputFormat& sourceFormat, + const AudioInputFormat& targetFormat, + double gain, + std::vector& destination); +void mixAudioInPlace( + std::vector& destination, + const BYTE* source, + DWORD byteCount, + const AudioInputFormat& format); + +class AudioMixer { +public: + using OutputCallback = std::function; + + AudioMixer( + const AudioInputFormat& format, + const AudioInputFormat& systemFormat, + const AudioInputFormat& microphoneFormat, + bool includeSystem, + bool includeMicrophone, + double microphoneGain, + OutputCallback output); + ~AudioMixer(); + + AudioMixer(const AudioMixer&) = delete; + AudioMixer& operator=(const AudioMixer&) = delete; + + bool start(); + void beginTimeline(); + void setPaused(bool paused); + void stop(); + void pushSystem(const BYTE* data, DWORD byteCount); + void pushMicrophone(const BYTE* data, DWORD byteCount); + +private: + void append( + std::vector& queue, + const BYTE* data, + DWORD byteCount, + const AudioInputFormat& sourceFormat, + double gain); + bool pop(std::vector& queue, std::vector& chunk, size_t byteCount); + void mixLoop(); + + AudioInputFormat format_{}; + AudioInputFormat systemFormat_{}; + AudioInputFormat microphoneFormat_{}; + bool includeSystem_ = false; + bool includeMicrophone_ = false; + double microphoneGain_ = 1.0; + OutputCallback output_; + std::mutex mutex_; + std::condition_variable cv_; + std::vector systemQueue_; + std::vector microphoneQueue_; + std::vector gainBuffer_; + std::thread thread_; + std::atomic stopRequested_ = false; + bool timelineStarted_ = false; + bool paused_ = false; + uint64_t emittedFrames_ = 0; +}; diff --git a/electron/native/wgc-capture/src/cursor-sampler.cpp b/electron/native/wgc-capture/src/cursor-sampler.cpp new file mode 100644 index 0000000..21558c7 --- /dev/null +++ b/electron/native/wgc-capture/src/cursor-sampler.cpp @@ -0,0 +1,482 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// ───────────────────────────────────────────────────────────────────────────── +// Global mouse-hook state +// ───────────────────────────────────────────────────────────────────────────── +static HHOOK g_mouseHook = nullptr; +static DWORD g_mainThreadId = 0; +static std::atomic g_leftDownCount{0}; +static std::atomic g_leftUpCount{0}; +static std::atomic g_stop{false}; +static std::mutex g_stdoutMtx; + +static LRESULT CALLBACK LowLevelMouseProc(int nCode, WPARAM wParam, LPARAM lParam) { + if (nCode >= 0) { + if (wParam == WM_LBUTTONDOWN) g_leftDownCount.fetch_add(1, std::memory_order_relaxed); + else if (wParam == WM_LBUTTONUP) g_leftUpCount.fetch_add(1, std::memory_order_relaxed); + } + return CallNextHookEx(g_mouseHook, nCode, wParam, lParam); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Utilities +// ───────────────────────────────────────────────────────────────────────────── +static int64_t nowMs() { + return static_cast( + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count()); +} + +static void writeJsonLine(const std::string& json) { + std::lock_guard lock(g_stdoutMtx); + std::cout << json << '\n'; + std::cout.flush(); +} + +static std::string jsonEscape(const std::string& s) { + std::string r; + r.reserve(s.size()); + for (unsigned char c : s) { + switch (c) { + case '"': r += "\\\""; break; + case '\\': r += "\\\\"; break; + case '\n': r += "\\n"; break; + case '\r': r += "\\r"; break; + case '\t': r += "\\t"; break; + default: r.push_back(static_cast(c)); break; + } + } + return r; +} + +static const char kBase64Chars[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +static std::string base64Encode(const uint8_t* data, size_t len) { + std::string out; + out.reserve(((len + 2) / 3) * 4); + for (size_t i = 0; i < len; i += 3) { + const uint32_t b = + (static_cast(data[i]) << 16) | + (i + 1 < len ? static_cast(data[i + 1]) << 8 : 0u) | + (i + 2 < len ? static_cast(data[i + 2]) : 0u); + out.push_back(kBase64Chars[(b >> 18) & 0x3F]); + out.push_back(kBase64Chars[(b >> 12) & 0x3F]); + out.push_back(i + 1 < len ? kBase64Chars[(b >> 6) & 0x3F] : '='); + out.push_back(i + 2 < len ? kBase64Chars[(b ) & 0x3F] : '='); + } + return out; +} + +// ───────────────────────────────────────────────────────────────────────────── +// GDI+ PNG encoder CLSID +// ───────────────────────────────────────────────────────────────────────────── +static bool getPngClsid(CLSID& out) { + UINT num = 0, sz = 0; + if (Gdiplus::GetImageEncodersSize(&num, &sz) != Gdiplus::Ok || sz == 0) return false; + std::vector buf(sz); + auto* enc = reinterpret_cast(buf.data()); + if (Gdiplus::GetImageEncoders(num, sz, enc) != Gdiplus::Ok) return false; + for (UINT i = 0; i < num; ++i) { + if (std::wstring(enc[i].MimeType) == L"image/png") { + out = enc[i].Clsid; + return true; + } + } + return false; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Standard cursor-type lookup +// ───────────────────────────────────────────────────────────────────────────── +static const char* standardCursorType(HCURSOR hc) { + if (!hc) return nullptr; + static const struct { WORD id; const char* name; } kMap[] = { + {32512, "arrow"}, + {32513, "text"}, + {32514, "wait"}, + {32515, "crosshair"}, + {32516, "up-arrow"}, + {32642, "resize-nwse"}, + {32643, "resize-nesw"}, + {32644, "resize-ew"}, + {32645, "resize-ns"}, + {32646, "move"}, + {32648, "not-allowed"}, + {32649, "pointer"}, + {32650, "app-starting"}, + {32651, "help"}, + }; + static constexpr int N = static_cast(sizeof(kMap) / sizeof(kMap[0])); + static HCURSOR g_handles[N] = {}; + static bool g_init = false; + if (!g_init) { + for (int i = 0; i < N; ++i) + g_handles[i] = LoadCursor(nullptr, MAKEINTRESOURCE(kMap[i].id)); + g_init = true; + } + for (int i = 0; i < N; ++i) + if (g_handles[i] && g_handles[i] == hc) return kMap[i].name; + return nullptr; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Custom cursor-type detection (replicates the PowerShell heuristic) +// ───────────────────────────────────────────────────────────────────────────── +static const char* detectCustomCursorType( + const uint32_t* pixels, int w, int h, int hotX, int hotY) +{ + if (w < 24 || h < 24 || w > 64 || h > 64) return nullptr; + if (hotX < w * 0.25 || hotX > w * 0.75) return nullptr; + if (hotY < h * 0.15 || hotY > h * 0.55) return nullptr; + + int opaque = 0, topHalf = 0; + int left = w, top = h, right = -1, bottom = -1; + + for (int y = 0; y < h; ++y) { + for (int x = 0; x < w; ++x) { + const uint8_t a = static_cast(pixels[y * w + x] >> 24); + if (a <= 32) continue; + ++opaque; + if (y < h / 2) ++topHalf; + if (x < left) left = x; + if (x > right) right = x; + if (y < top) top = y; + if (y > bottom) bottom = y; + } + } + + if (opaque < 90 || right < left || bottom < top) return nullptr; + + const int ow = right - left + 1; + const int oh = bottom - top + 1; + if (ow < w * 0.35 || ow > w * 0.9) return nullptr; + if (oh < h * 0.45 || oh > static_cast(h)) return nullptr; + if (top > h * 0.45 || bottom < h * 0.65) return nullptr; + + return topHalf > opaque * 0.55 ? "closed-hand" : "open-hand"; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Build asset JSON for the given cursor (returns empty string on failure) +// +// Renders the cursor via GDI DrawIconEx onto a 32-bpp transparent DIB section +// and then encodes to PNG — matching the PowerShell approach of +// Graphics.Clear(Transparent) + Graphics.DrawIcon(). This correctly preserves +// per-pixel alpha for 32-bit cursors, unlike Gdiplus::Bitmap::FromHICON which +// can produce incorrect alpha for cursor handles. +// ───────────────────────────────────────────────────────────────────────────── +static std::string buildAssetJson( + HCURSOR hCursor, + const std::string& handleStr, + const CLSID& pngClsid, + const char** outCustomType) +{ + *outCustomType = nullptr; + + // Get hotspot and cursor dimensions from the icon info. + // For color cursors hbmColor gives the size; for monochrome cursors the + // mask bitmap is twice the cursor height (AND mask stacked on XOR mask). + ICONINFO ii{}; + if (!GetIconInfo(hCursor, &ii)) return {}; + const int hotX = static_cast(ii.xHotspot); + const int hotY = static_cast(ii.yHotspot); + + int w = 0, h = 0; + if (ii.hbmColor) { + BITMAP bm{}; + if (GetObject(ii.hbmColor, sizeof(bm), &bm)) { w = bm.bmWidth; h = bm.bmHeight; } + } + if (ii.hbmMask && (w == 0 || h == 0)) { + BITMAP bm{}; + if (GetObject(ii.hbmMask, sizeof(bm), &bm)) { + w = bm.bmWidth; + h = ii.hbmColor ? bm.bmHeight : bm.bmHeight / 2; + } + } + if (ii.hbmMask) DeleteObject(ii.hbmMask); + if (ii.hbmColor) DeleteObject(ii.hbmColor); + if (w <= 0 || h <= 0) return {}; + + // Copy the cursor handle so DrawIconEx cannot affect the live system cursor. + const HICON hCopy = CopyIcon(hCursor); + if (!hCopy) return {}; + + // Allocate a 32-bpp top-down DIB section and clear it to transparent black, + // then draw the cursor with DI_NORMAL. For 32-bit alpha cursors Windows + // writes correct per-pixel alpha into the high byte of each BGRA pixel. + const int stride = w * 4; + BITMAPINFOHEADER bih{}; + bih.biSize = sizeof(bih); + bih.biWidth = w; + bih.biHeight = -h; // negative = top-down scanline order + bih.biPlanes = 1; + bih.biBitCount = 32; + bih.biCompression = BI_RGB; + + void* pBits = nullptr; + HDC hDC = CreateCompatibleDC(nullptr); + HBITMAP hBmp = hDC ? CreateDIBSection(hDC, + reinterpret_cast(&bih), + DIB_RGB_COLORS, &pBits, nullptr, 0) + : nullptr; + + if (!hBmp || !pBits) { + if (hBmp) DeleteObject(hBmp); + if (hDC) DeleteDC(hDC); + DestroyIcon(hCopy); + return {}; + } + + HGDIOBJ hOld = SelectObject(hDC, hBmp); + std::memset(pBits, 0, static_cast(stride * h)); // transparent black + DrawIconEx(hDC, 0, 0, hCopy, w, h, 0, nullptr, DI_NORMAL); + GdiFlush(); + SelectObject(hDC, hOld); + DeleteDC(hDC); + DestroyIcon(hCopy); + + // GDI's 32-bit DIB stores pixels as BGRA in memory. GDI+'s + // PixelFormat32bppARGB interprets each 32-bit word as 0xAARRGGBB which is + // identical to BGRA on little-endian, so the alpha byte is always >> 24. + { + const auto* px = static_cast(pBits); + *outCustomType = detectCustomCursorType(px, w, h, hotX, hotY); + } + + // Wrap the DIB pixels in a GDI+ Bitmap (zero-copy) and save to PNG. + // Keep hBmp alive until after gBmp is destroyed so pBits remains valid. + std::vector pngData; + { + Gdiplus::Bitmap gBmp(w, h, stride, PixelFormat32bppARGB, + static_cast(pBits)); + if (gBmp.GetLastStatus() == Gdiplus::Ok) { + IStream* pStream = nullptr; + if (SUCCEEDED(CreateStreamOnHGlobal(nullptr, TRUE, &pStream))) { + if (gBmp.Save(pStream, &pngClsid) == Gdiplus::Ok) { + ULARGE_INTEGER sz{}; + LARGE_INTEGER zero{}; + pStream->Seek(zero, STREAM_SEEK_END, &sz); + pStream->Seek(zero, STREAM_SEEK_SET, nullptr); + pngData.resize(static_cast(sz.QuadPart)); + ULONG n = 0; + pStream->Read(pngData.data(), static_cast(pngData.size()), &n); + pngData.resize(n); + } + pStream->Release(); + } + } + } // gBmp destroyed here; pBits (owned by hBmp) still valid + DeleteObject(hBmp); + + if (pngData.empty()) return {}; + + const std::string dataUrl = + "data:image/png;base64," + base64Encode(pngData.data(), pngData.size()); + + std::string json; + json.reserve(dataUrl.size() + 128); + json = "{\"id\":\"" + handleStr + "\""; + json += ",\"imageDataUrl\":\"" + jsonEscape(dataUrl) + "\""; + json += ",\"width\":" + std::to_string(w); + json += ",\"height\":" + std::to_string(h); + json += ",\"hotspotX\":" + std::to_string(hotX); + json += ",\"hotspotY\":" + std::to_string(hotY); + if (*outCustomType) { + json += ",\"cursorType\":\""; + json += *outCustomType; + json += "\""; + } else { + json += ",\"cursorType\":null"; + } + json += "}"; + return json; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Sampling loop (background thread) +// ───────────────────────────────────────────────────────────────────────────── +static void runSamplingLoop(int intervalMs, HWND targetWindow, const CLSID& pngClsid) { + HCURSOR lastCursor = nullptr; + + while (!g_stop.load(std::memory_order_relaxed)) { + const int downCount = g_leftDownCount.exchange(0, std::memory_order_relaxed); + const int upCount = g_leftUpCount.exchange(0, std::memory_order_relaxed); + + CURSORINFO ci{}; + ci.cbSize = sizeof(ci); + if (!GetCursorInfo(&ci)) { + char buf[160]; + std::snprintf(buf, sizeof(buf), + "{\"type\":\"error\",\"timestampMs\":%" PRId64 ",\"message\":\"GetCursorInfo failed\"}", + nowMs()); + writeJsonLine(buf); + std::this_thread::sleep_for(std::chrono::milliseconds(intervalMs)); + continue; + } + + const bool visible = (ci.flags & CURSOR_SHOWING) != 0; + const HCURSOR hc = ci.hCursor; + + // Handle string ("0xHEX" or empty for null cursor) + char handleBuf[32] = {}; + if (hc) + std::snprintf(handleBuf, sizeof(handleBuf), + "0x%" PRIX64, static_cast(reinterpret_cast(hc))); + const std::string handleStr = hc ? handleBuf : ""; + + // Standard cursor type + const char* cursorType = standardCursorType(hc); + + // Mouse button state + const SHORT ks = GetAsyncKeyState(VK_LBUTTON); + const bool leftDown = (ks & 0x8000) != 0; + const bool leftPressed = downCount > 0 || (ks & 0x0001) != 0; + const bool leftReleased = upCount > 0; + + // Asset — only when the cursor handle changes + std::string assetJson; + if (visible && hc && hc != lastCursor) { + const char* customType = nullptr; + assetJson = buildAssetJson(hc, handleStr, pngClsid, &customType); + if (!assetJson.empty() && !cursorType && customType) + cursorType = customType; + lastCursor = hc; + } + + // Window bounds + std::string boundsJson = "null"; + if (targetWindow && IsWindow(targetWindow)) { + RECT r{}; + if (GetWindowRect(targetWindow, &r)) { + const int bw = r.right - r.left; + const int bh = r.bottom - r.top; + if (bw > 0 && bh > 0) { + char buf[128]; + std::snprintf(buf, sizeof(buf), + "{\"x\":%ld,\"y\":%ld,\"width\":%d,\"height\":%d}", + r.left, r.top, bw, bh); + boundsJson = buf; + } + } + } + + // Emit sample JSON + std::string out; + out.reserve(256); + out += "{\"type\":\"sample\""; + out += ",\"timestampMs\":"; out += std::to_string(nowMs()); + out += ",\"x\":"; out += std::to_string(ci.ptScreenPos.x); + out += ",\"y\":"; out += std::to_string(ci.ptScreenPos.y); + out += ",\"visible\":"; out += visible ? "true" : "false"; + out += ",\"handle\":"; out += hc ? ("\"" + handleStr + "\"") : "null"; + out += ",\"cursorType\":"; out += cursorType ? ("\"" + std::string(cursorType) + "\"") : "null"; + out += ",\"leftButtonDown\":"; out += leftDown ? "true" : "false"; + out += ",\"leftButtonPressed\":"; out += leftPressed ? "true" : "false"; + out += ",\"leftButtonReleased\":"; out += leftReleased ? "true" : "false"; + out += ",\"bounds\":"; out += boundsJson; + out += ",\"asset\":"; out += assetJson.empty() ? "null" : assetJson; + out += "}"; + + writeJsonLine(out); + + // Exit if stdout pipe is broken (parent process died) + if (std::cout.fail()) { + PostThreadMessage(g_mainThreadId, WM_QUIT, 0, 0); + break; + } + + std::this_thread::sleep_for(std::chrono::milliseconds(intervalMs)); + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// main +// ───────────────────────────────────────────────────────────────────────────── +int main(int argc, char* argv[]) { + if (argc < 2) { + std::cerr << "Usage: cursor-sampler [windowHandle]" << std::endl; + return 1; + } + + const int intervalMs = std::max(1, std::atoi(argv[1])); + + HWND targetWindow = nullptr; + if (argc >= 3) { + const std::string arg = argv[2]; + if (!arg.empty() && arg != "null") { + try { + const int base = (arg.rfind("0x", 0) == 0 || arg.rfind("0X", 0) == 0) ? 16 : 10; + const uint64_t v = std::stoull(arg, nullptr, base); + if (v) targetWindow = reinterpret_cast(static_cast(v)); + } catch (...) {} + } + } + + // Initialize GDI+ + Gdiplus::GdiplusStartupInput gdipInput{}; + ULONG_PTR gdipToken = 0; + if (Gdiplus::GdiplusStartup(&gdipToken, &gdipInput, nullptr) != Gdiplus::Ok) { + std::cerr << "GDI+ init failed" << std::endl; + return 1; + } + + CLSID pngClsid{}; + if (!getPngClsid(pngClsid)) { + std::cerr << "PNG encoder not found" << std::endl; + Gdiplus::GdiplusShutdown(gdipToken); + return 1; + } + + // Install global low-level mouse hook on this thread + g_mouseHook = SetWindowsHookEx(WH_MOUSE_LL, LowLevelMouseProc, GetModuleHandle(nullptr), 0); + if (!g_mouseHook) { + std::cerr << "SetWindowsHookEx failed" << std::endl; + Gdiplus::GdiplusShutdown(gdipToken); + return 1; + } + + // Prime GetAsyncKeyState so the first poll doesn't return stale "since-last-call" bits + GetAsyncKeyState(VK_LBUTTON); + + // Signal readiness + g_mainThreadId = GetCurrentThreadId(); + { + char buf[80]; + std::snprintf(buf, sizeof(buf), + "{\"type\":\"ready\",\"timestampMs\":%" PRId64 "}", nowMs()); + writeJsonLine(buf); + } + + // Start sampling on a background thread + std::thread sampler(runSamplingLoop, intervalMs, targetWindow, std::cref(pngClsid)); + + // Run the message pump on the main thread — required for WH_MOUSE_LL callbacks + MSG msg; + while (GetMessage(&msg, nullptr, 0, 0) > 0) { + TranslateMessage(&msg); + DispatchMessage(&msg); + } + + g_stop.store(true, std::memory_order_relaxed); + if (sampler.joinable()) sampler.join(); + UnhookWindowsHookEx(g_mouseHook); + Gdiplus::GdiplusShutdown(gdipToken); + return 0; +} diff --git a/electron/native/wgc-capture/src/dshow_webcam_capture.cpp b/electron/native/wgc-capture/src/dshow_webcam_capture.cpp new file mode 100644 index 0000000..7e3f8b7 --- /dev/null +++ b/electron/native/wgc-capture/src/dshow_webcam_capture.cpp @@ -0,0 +1,427 @@ +#include "dshow_webcam_capture.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +const CLSID CLSID_SampleGrabberLocal = {0xC1F400A0, 0x3F08, 0x11D3, {0x9F, 0x0B, 0x00, 0x60, 0x08, 0x03, 0x9E, 0x37}}; +const CLSID CLSID_NullRendererLocal = {0xC1F400A4, 0x3F08, 0x11D3, {0x9F, 0x0B, 0x00, 0x60, 0x08, 0x03, 0x9E, 0x37}}; + +MIDL_INTERFACE("6B652FFF-11FE-4FCE-92AD-0266B5D7C78F") +ISampleGrabber : public IUnknown { +public: + virtual HRESULT STDMETHODCALLTYPE SetOneShot(BOOL oneShot) = 0; + virtual HRESULT STDMETHODCALLTYPE SetMediaType(const AM_MEDIA_TYPE* type) = 0; + virtual HRESULT STDMETHODCALLTYPE GetConnectedMediaType(AM_MEDIA_TYPE* type) = 0; + virtual HRESULT STDMETHODCALLTYPE SetBufferSamples(BOOL bufferThem) = 0; + virtual HRESULT STDMETHODCALLTYPE GetCurrentBuffer(long* bufferSize, long* buffer) = 0; + virtual HRESULT STDMETHODCALLTYPE GetCurrentSample(IMediaSample** sample) = 0; + virtual HRESULT STDMETHODCALLTYPE SetCallback(IUnknown* callback, long whichMethodToCallback) = 0; +}; + +bool succeeded(HRESULT hr, const char* label) { + if (SUCCEEDED(hr)) { + return true; + } + + std::cerr << "ERROR: " << label << " failed (hr=0x" << std::hex << hr << std::dec << ")" + << std::endl; + return false; +} + +std::string guidToString(const GUID& guid) { + if (guid == MEDIASUBTYPE_RGB32) { + return "RGB32"; + } + if (guid == MEDIASUBTYPE_YUY2) { + return "YUY2"; + } + if (guid == MEDIASUBTYPE_NV12) { + return "NV12"; + } + + std::ostringstream stream; + stream << std::hex << std::setfill('0') + << '{' << std::setw(8) << guid.Data1 + << '-' << std::setw(4) << guid.Data2 + << '-' << std::setw(4) << guid.Data3 + << '-'; + for (int index = 0; index < 2; index += 1) { + stream << std::setw(2) << static_cast(guid.Data4[index]); + } + stream << '-'; + for (int index = 2; index < 8; index += 1) { + stream << std::setw(2) << static_cast(guid.Data4[index]); + } + stream << '}'; + return stream.str(); +} + +void freeMediaType(AM_MEDIA_TYPE& type) { + if (type.cbFormat != 0) { + CoTaskMemFree(type.pbFormat); + type.cbFormat = 0; + type.pbFormat = nullptr; + } + if (type.pUnk) { + type.pUnk->Release(); + type.pUnk = nullptr; + } +} + +BYTE clampToByte(int value) { + return static_cast(std::clamp(value, 0, 255)); +} + +std::array yuvToBgr(int y, int u, int v) { + const int c = y - 16; + const int d = u - 128; + const int e = v - 128; + const int blue = (298 * c + 516 * d + 128) >> 8; + const int green = (298 * c - 100 * d - 208 * e + 128) >> 8; + const int red = (298 * c + 409 * e + 128) >> 8; + return {clampToByte(blue), clampToByte(green), clampToByte(red)}; +} + +} // namespace + +struct DirectShowWebcamCapture::Impl { + Microsoft::WRL::ComPtr graph; + Microsoft::WRL::ComPtr captureGraph; + Microsoft::WRL::ComPtr captureFilter; + Microsoft::WRL::ComPtr sampleGrabberFilter; + Microsoft::WRL::ComPtr sampleGrabber; + Microsoft::WRL::ComPtr nullRenderer; + Microsoft::WRL::ComPtr mediaControl; + bool comInitialized = false; + bool running = false; +}; + +DirectShowWebcamCapture::~DirectShowWebcamCapture() { + stop(); + delete impl_; +} + +bool DirectShowWebcamCapture::initialize( + const std::wstring& deviceId, + const std::wstring& deviceName, + const std::wstring& directShowClsid, + int requestedWidth, + int requestedHeight, + int requestedFps) { + (void)deviceId; + stop(); + delete impl_; + impl_ = nullptr; + impl_ = new Impl(); + fps_ = std::clamp(requestedFps > 0 ? requestedFps : 30, 1, 60); + + HRESULT hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); + if (SUCCEEDED(hr)) { + impl_->comInitialized = true; + } else if (hr != RPC_E_CHANGED_MODE) { + return succeeded(hr, "CoInitializeEx(DirectShow webcam)"); + } + + if (directShowClsid.empty()) { + std::cerr << "ERROR: DirectShow webcam fallback requires a resolved filter CLSID" << std::endl; + return false; + } + + CLSID selectedClsid{}; + if (FAILED(CLSIDFromString(directShowClsid.c_str(), &selectedClsid))) { + std::cerr << "ERROR: DirectShow webcam fallback received an invalid filter CLSID" << std::endl; + return false; + } + selectedDeviceName_ = deviceName.empty() ? directShowClsid : deviceName; + + if (!succeeded(CoCreateInstance(selectedClsid, nullptr, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&impl_->captureFilter)), + "CoCreateInstance(DirectShow webcam filter)")) { + return false; + } + if (!succeeded(CoCreateInstance(CLSID_FilterGraph, nullptr, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&impl_->graph)), + "CoCreateInstance(FilterGraph)")) { + return false; + } + if (!succeeded(CoCreateInstance(CLSID_CaptureGraphBuilder2, nullptr, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&impl_->captureGraph)), + "CoCreateInstance(CaptureGraphBuilder2)")) { + return false; + } + if (!succeeded(impl_->captureGraph->SetFiltergraph(impl_->graph.Get()), "SetFiltergraph(DirectShow webcam)")) { + return false; + } + if (!succeeded(impl_->graph->AddFilter(impl_->captureFilter.Get(), L"OpenScreen Webcam Source"), + "AddFilter(DirectShow webcam source)")) { + return false; + } + + if (!succeeded(CoCreateInstance(CLSID_SampleGrabberLocal, nullptr, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&impl_->sampleGrabberFilter)), + "CoCreateInstance(SampleGrabber)")) { + return false; + } + if (!succeeded(impl_->sampleGrabberFilter.As(&impl_->sampleGrabber), "QueryInterface(ISampleGrabber)")) { + return false; + } + + AM_MEDIA_TYPE requestedType{}; + requestedType.majortype = MEDIATYPE_Video; + requestedType.formattype = FORMAT_VideoInfo; + if (!succeeded(impl_->sampleGrabber->SetMediaType(&requestedType), "SetMediaType(DirectShow video)")) { + return false; + } + + if (!succeeded(impl_->graph->AddFilter(impl_->sampleGrabberFilter.Get(), L"OpenScreen Webcam Sample Grabber"), + "AddFilter(SampleGrabber)")) { + return false; + } + if (!succeeded(CoCreateInstance(CLSID_NullRendererLocal, nullptr, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&impl_->nullRenderer)), + "CoCreateInstance(NullRenderer)")) { + return false; + } + if (!succeeded(impl_->graph->AddFilter(impl_->nullRenderer.Get(), L"OpenScreen Webcam Null Renderer"), + "AddFilter(NullRenderer)")) { + return false; + } + + if (!succeeded(impl_->captureGraph->RenderStream( + &PIN_CATEGORY_CAPTURE, + &MEDIATYPE_Video, + impl_->captureFilter.Get(), + impl_->sampleGrabberFilter.Get(), + impl_->nullRenderer.Get()), + "RenderStream(DirectShow webcam)")) { + return false; + } + + AM_MEDIA_TYPE connectedType{}; + if (!succeeded(impl_->sampleGrabber->GetConnectedMediaType(&connectedType), "GetConnectedMediaType(DirectShow webcam)")) { + return false; + } + if (connectedType.subtype == MEDIASUBTYPE_YUY2) { + pixelFormat_ = PixelFormat::Yuy2; + } else if (connectedType.subtype == MEDIASUBTYPE_NV12) { + pixelFormat_ = PixelFormat::Nv12; + } else if (connectedType.subtype == MEDIASUBTYPE_RGB32) { + pixelFormat_ = PixelFormat::Bgra; + } else { + std::cerr << "ERROR: Unsupported DirectShow webcam media subtype " + << guidToString(connectedType.subtype) << std::endl; + freeMediaType(connectedType); + return false; + } + if (connectedType.formattype == FORMAT_VideoInfo && connectedType.pbFormat) { + const auto* videoInfo = reinterpret_cast(connectedType.pbFormat); + width_ = std::abs(videoInfo->bmiHeader.biWidth); + height_ = std::abs(videoInfo->bmiHeader.biHeight); + const int bitsPerPixel = videoInfo->bmiHeader.biBitCount > 0 ? videoInfo->bmiHeader.biBitCount : 16; + if (pixelFormat_ == PixelFormat::Nv12) { + sourceStride_ = ((width_ + 3) / 4) * 4; + } else { + sourceStride_ = ((width_ * bitsPerPixel + 31) / 32) * 4; + } + sourceTopDown_ = pixelFormat_ != PixelFormat::Bgra || videoInfo->bmiHeader.biHeight < 0; + } + std::cerr << "INFO: DirectShow webcam connected subtype " << guidToString(connectedType.subtype) + << " " << width_ << "x" << height_ << " stride=" << sourceStride_ << std::endl; + freeMediaType(connectedType); + if (width_ <= 0 || height_ <= 0) { + width_ = requestedWidth > 0 ? requestedWidth : 1280; + height_ = requestedHeight > 0 ? requestedHeight : 720; + } + if (sourceStride_ <= 0) { + sourceStride_ = pixelFormat_ == PixelFormat::Bgra ? width_ * 4 : ((width_ + 3) / 4) * 4; + } + + impl_->sampleGrabber->SetBufferSamples(TRUE); + impl_->sampleGrabber->SetOneShot(FALSE); + if (!succeeded(impl_->graph.As(&impl_->mediaControl), "QueryInterface(IMediaControl)")) { + return false; + } + + return true; +} + +bool DirectShowWebcamCapture::start() { + if (!impl_ || !impl_->mediaControl || impl_->running) { + return false; + } + HRESULT hr = impl_->mediaControl->Run(); + if (!succeeded(hr, "Run(DirectShow webcam)")) { + return false; + } + stopRequested_ = false; + try { + thread_ = std::thread(&DirectShowWebcamCapture::captureLoop, this); + } catch (const std::exception& error) { + stopRequested_ = true; + impl_->mediaControl->Stop(); + std::cerr << "ERROR: Failed to start DirectShow webcam capture thread: " << error.what() << std::endl; + return false; + } catch (...) { + stopRequested_ = true; + impl_->mediaControl->Stop(); + std::cerr << "ERROR: Failed to start DirectShow webcam capture thread" << std::endl; + return false; + } + impl_->running = true; + return true; +} + +void DirectShowWebcamCapture::stop() { + stopRequested_ = true; + if (thread_.joinable()) { + thread_.join(); + } + if (!impl_) { + return; + } + if (impl_->mediaControl && impl_->running) { + impl_->mediaControl->Stop(); + } + impl_->running = false; + impl_->mediaControl.Reset(); + impl_->nullRenderer.Reset(); + impl_->sampleGrabber.Reset(); + impl_->sampleGrabberFilter.Reset(); + impl_->captureFilter.Reset(); + impl_->captureGraph.Reset(); + impl_->graph.Reset(); + if (impl_->comInitialized) { + CoUninitialize(); + impl_->comInitialized = false; + } +} + +void DirectShowWebcamCapture::captureLoop() { + const HRESULT coinitHr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); + while (!stopRequested_ && impl_ && impl_->sampleGrabber) { + long bufferSize = 0; + HRESULT hr = impl_->sampleGrabber->GetCurrentBuffer(&bufferSize, nullptr); + if (SUCCEEDED(hr) && bufferSize > 0) { + std::vector buffer(static_cast(bufferSize)); + hr = impl_->sampleGrabber->GetCurrentBuffer(&bufferSize, reinterpret_cast(buffer.data())); + if (SUCCEEDED(hr)) { + storeFrame(buffer.data(), bufferSize); + } + } + std::this_thread::sleep_for(std::chrono::milliseconds(1000 / std::max(1, fps_))); + } + if (SUCCEEDED(coinitHr)) { + CoUninitialize(); + } +} + +void DirectShowWebcamCapture::storeFrame(const BYTE* buffer, long length) { + const int destinationStride = width_ * 4; + const int sourceStride = sourceStride_ > 0 ? sourceStride_ : destinationStride; + const int expectedLength = pixelFormat_ == PixelFormat::Nv12 + ? sourceStride * height_ + sourceStride * ((height_ + 1) / 2) + : sourceStride * height_; + if (!buffer || length < expectedLength || width_ <= 0 || height_ <= 0) { + return; + } + + std::vector frame(static_cast(destinationStride * height_)); + for (int y = 0; y < height_; y += 1) { + const int sourceY = sourceTopDown_ ? y : height_ - 1 - y; + const BYTE* source = buffer + sourceY * sourceStride; + BYTE* destination = frame.data() + y * destinationStride; + if (pixelFormat_ == PixelFormat::Bgra) { + std::copy(source, source + destinationStride, destination); + for (int x = 0; x < width_; x += 1) { + destination[x * 4 + 3] = 255; + } + continue; + } + + if (pixelFormat_ == PixelFormat::Nv12) { + const BYTE* yPlane = buffer + sourceY * sourceStride; + const BYTE* uvPlane = buffer + sourceStride * height_ + (sourceY / 2) * sourceStride; + for (int x = 0; x < width_; x += 1) { + const int uvX = (x / 2) * 2; + const auto color = yuvToBgr(yPlane[x], uvPlane[uvX], uvPlane[uvX + 1]); + BYTE* pixel = destination + x * 4; + pixel[0] = color[0]; + pixel[1] = color[1]; + pixel[2] = color[2]; + pixel[3] = 255; + } + continue; + } + + for (int x = 0; x + 1 < width_; x += 2) { + const BYTE y0 = source[x * 2]; + const BYTE u = source[x * 2 + 1]; + const BYTE y1 = source[x * 2 + 2]; + const BYTE v = source[x * 2 + 3]; + const auto first = yuvToBgr(y0, u, v); + const auto second = yuvToBgr(y1, u, v); + BYTE* firstPixel = destination + x * 4; + BYTE* secondPixel = firstPixel + 4; + firstPixel[0] = first[0]; + firstPixel[1] = first[1]; + firstPixel[2] = first[2]; + firstPixel[3] = 255; + secondPixel[0] = second[0]; + secondPixel[1] = second[1]; + secondPixel[2] = second[2]; + secondPixel[3] = 255; + } + if (width_ % 2 == 1) { + const int x = width_ - 1; + const int previousPairStart = ((x - 1) / 2) * 4; + const BYTE y = source[x * 2]; + const BYTE u = source[previousPairStart + 1]; + const BYTE v = source[previousPairStart + 3]; + const auto color = yuvToBgr(y, u, v); + BYTE* pixel = destination + x * 4; + pixel[0] = color[0]; + pixel[1] = color[1]; + pixel[2] = color[2]; + pixel[3] = 255; + } + } + + std::scoped_lock lock(frameMutex_); + latestFrame_ = std::move(frame); + latestFrameSequence_ += 1; +} + +bool DirectShowWebcamCapture::copyLatestFrame(WebcamFrameSnapshot& destination) { + std::scoped_lock lock(frameMutex_); + if (latestFrame_.empty() || width_ <= 0 || height_ <= 0) { + return false; + } + + destination.data = latestFrame_; + destination.width = width_; + destination.height = height_; + destination.sequence = latestFrameSequence_; + return true; +} + +int DirectShowWebcamCapture::width() const { + return width_; +} + +int DirectShowWebcamCapture::height() const { + return height_; +} + +int DirectShowWebcamCapture::fps() const { + return fps_; +} + +const std::wstring& DirectShowWebcamCapture::selectedDeviceName() const { + return selectedDeviceName_; +} diff --git a/electron/native/wgc-capture/src/dshow_webcam_capture.h b/electron/native/wgc-capture/src/dshow_webcam_capture.h new file mode 100644 index 0000000..3debcbe --- /dev/null +++ b/electron/native/wgc-capture/src/dshow_webcam_capture.h @@ -0,0 +1,67 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +struct WebcamFrameSnapshot { + std::vector data; + int width = 0; + int height = 0; + uint64_t sequence = 0; +}; + +class DirectShowWebcamCapture { +public: + DirectShowWebcamCapture() = default; + ~DirectShowWebcamCapture(); + + DirectShowWebcamCapture(const DirectShowWebcamCapture&) = delete; + DirectShowWebcamCapture& operator=(const DirectShowWebcamCapture&) = delete; + + bool initialize( + const std::wstring& deviceId, + const std::wstring& deviceName, + const std::wstring& directShowClsid, + int requestedWidth, + int requestedHeight, + int requestedFps); + bool start(); + void stop(); + bool copyLatestFrame(WebcamFrameSnapshot& destination); + + int width() const; + int height() const; + int fps() const; + const std::wstring& selectedDeviceName() const; + void storeFrame(const BYTE* buffer, long length); + +private: + enum class PixelFormat { + Bgra, + Nv12, + Yuy2, + }; + + struct Impl; + void captureLoop(); + + Impl* impl_ = nullptr; + std::thread thread_; + std::atomic stopRequested_ = false; + std::mutex frameMutex_; + std::vector latestFrame_; + uint64_t latestFrameSequence_ = 0; + int width_ = 0; + int height_ = 0; + int fps_ = 30; + int sourceStride_ = 0; + bool sourceTopDown_ = false; + PixelFormat pixelFormat_ = PixelFormat::Bgra; + std::wstring selectedDeviceName_; +}; diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp new file mode 100644 index 0000000..bb741d3 --- /dev/null +++ b/electron/native/wgc-capture/src/main.cpp @@ -0,0 +1,859 @@ +#include "audio_sample_utils.h" +#include "mf_encoder.h" +#include "monitor_utils.h" +#include "wasapi_loopback_capture.h" +#include "webcam_capture.h" +#include "wgc_session.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +struct CaptureConfig { + int schemaVersion = 1; + int64_t displayId = 0; + int64_t recordingId = 0; + std::string sourceType = "display"; + std::string sourceId; + std::string windowHandle; + std::string outputPath; + std::string webcamOutputPath; + int fps = 60; + int width = 0; + int height = 0; + MonitorBounds bounds{}; + bool hasDisplayBounds = false; + bool captureSystemAudio = false; + bool captureMic = false; + bool captureCursor = false; + bool webcamEnabled = false; + std::string microphoneDeviceId; + std::string microphoneDeviceName; + double microphoneGain = 1.0; + std::string webcamDeviceId; + std::string webcamDeviceName; + std::string webcamDirectShowClsid; + int webcamWidth = 0; + int webcamHeight = 0; + int webcamFps = 0; +}; + +struct CaptureControl { + std::atomic stopRequested = false; + std::atomic paused = false; + std::mutex mutex; + std::condition_variable cv; + std::chrono::steady_clock::time_point pauseStartedAt; + std::chrono::steady_clock::duration totalPausedDuration{}; + + int64_t pausedDurationHns() { + std::scoped_lock lock(mutex); + auto total = totalPausedDuration; + if (paused.load()) { + total += std::chrono::steady_clock::now() - pauseStartedAt; + } + return std::chrono::duration_cast(total).count() / 100; + } + + void setPaused(bool nextPaused) { + std::scoped_lock lock(mutex); + if (nextPaused == paused.load()) { + return; + } + if (nextPaused) { + pauseStartedAt = std::chrono::steady_clock::now(); + } else { + totalPausedDuration += std::chrono::steady_clock::now() - pauseStartedAt; + } + paused = nextPaused; + } +}; + +std::wstring utf8ToWide(const std::string& value) { + if (value.empty()) { + return {}; + } + + const int size = MultiByteToWideChar(CP_UTF8, 0, value.data(), static_cast(value.size()), nullptr, 0); + std::wstring result(static_cast(size), L'\0'); + MultiByteToWideChar(CP_UTF8, 0, value.data(), static_cast(value.size()), result.data(), size); + return result; +} + +std::string wideToUtf8(const std::wstring& value) { + if (value.empty()) { + return {}; + } + + const int size = WideCharToMultiByte(CP_UTF8, 0, value.data(), static_cast(value.size()), nullptr, 0, nullptr, nullptr); + std::string result(static_cast(size), '\0'); + WideCharToMultiByte(CP_UTF8, 0, value.data(), static_cast(value.size()), result.data(), size, nullptr, nullptr); + return result; +} + +std::string jsonEscape(const std::string& value) { + std::string result; + result.reserve(value.size()); + for (const char c : value) { + switch (c) { + case '\\': + result += "\\\\"; + break; + case '"': + result += "\\\""; + break; + case '\n': + result += "\\n"; + break; + case '\r': + result += "\\r"; + break; + case '\t': + result += "\\t"; + break; + default: + result.push_back(c); + break; + } + } + return result; +} + +bool hasVisibleBgraContent(const std::vector& frame) { + if (frame.size() < 4) { + return false; + } + + uint64_t lumaTotal = 0; + BYTE maxLuma = 0; + const size_t pixelCount = frame.size() / 4; + const size_t step = std::max(1, pixelCount / 4096); + size_t sampledPixels = 0; + for (size_t pixel = 0; pixel < pixelCount; pixel += step) { + const size_t offset = pixel * 4; + const BYTE b = frame[offset + 0]; + const BYTE g = frame[offset + 1]; + const BYTE r = frame[offset + 2]; + const BYTE luma = static_cast((static_cast(r) * 54 + static_cast(g) * 183 + static_cast(b) * 19) >> 8); + lumaTotal += luma; + maxLuma = std::max(maxLuma, luma); + sampledPixels += 1; + } + + const uint64_t averageLuma = sampledPixels > 0 ? lumaTotal / sampledPixels : 0; + return maxLuma > 24 || averageLuma > 4; +} + +bool findBool(const std::string& json, const std::string& key, bool fallback) { + auto pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) { + return fallback; + } + pos = json.find(':', pos); + if (pos == std::string::npos) { + return fallback; + } + pos += 1; + while (pos < json.size() && std::isspace(static_cast(json[pos]))) { + pos += 1; + } + if (json.compare(pos, 4, "true") == 0) { + return true; + } + if (json.compare(pos, 5, "false") == 0) { + return false; + } + return fallback; +} + +int64_t findInt64(const std::string& json, const std::string& key, int64_t fallback) { + auto pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) { + return fallback; + } + pos = json.find(':', pos); + if (pos == std::string::npos) { + return fallback; + } + pos += 1; + while (pos < json.size() && std::isspace(static_cast(json[pos]))) { + pos += 1; + } + try { + return std::stoll(json.substr(pos)); + } catch (...) { + return fallback; + } +} + +int findInt(const std::string& json, const std::string& key, int fallback) { + return static_cast(findInt64(json, key, fallback)); +} + +double findDouble(const std::string& json, const std::string& key, double fallback) { + auto pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) { + return fallback; + } + pos = json.find(':', pos); + if (pos == std::string::npos) { + return fallback; + } + pos += 1; + while (pos < json.size() && std::isspace(static_cast(json[pos]))) { + pos += 1; + } + try { + return std::stod(json.substr(pos)); + } catch (...) { + return fallback; + } +} + +std::string findString(const std::string& json, const std::string& key) { + auto pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) { + return {}; + } + pos = json.find(':', pos); + if (pos == std::string::npos) { + return {}; + } + pos += 1; + while (pos < json.size() && std::isspace(static_cast(json[pos]))) { + pos += 1; + } + if (pos >= json.size() || json[pos] != '"') { + return {}; + } + pos += 1; + + std::string result; + while (pos < json.size()) { + const char c = json[pos++]; + if (c == '"') { + break; + } + if (c == '\\' && pos < json.size()) { + const char escaped = json[pos++]; + switch (escaped) { + case '\\': + case '"': + case '/': + result.push_back(escaped); + break; + case 'n': + result.push_back('\n'); + break; + case 'r': + result.push_back('\r'); + break; + case 't': + result.push_back('\t'); + break; + default: + result.push_back(escaped); + break; + } + continue; + } + result.push_back(c); + } + return result; +} + +std::string parseWindowHandleFromSourceId(const std::string& sourceId) { + constexpr char prefix[] = "window:"; + if (sourceId.rfind(prefix, 0) != 0) { + return {}; + } + + const size_t start = sizeof(prefix) - 1; + const size_t end = sourceId.find(':', start); + const std::string handle = sourceId.substr(start, end == std::string::npos ? std::string::npos : end - start); + return handle.empty() ? std::string{} : handle; +} + +HWND parseWindowHandle(const std::string& value) { + if (value.empty()) { + return nullptr; + } + + try { + size_t parsed = 0; + const int base = value.rfind("0x", 0) == 0 || value.rfind("0X", 0) == 0 ? 16 : 10; + const uint64_t handleValue = std::stoull(value, &parsed, base); + if (parsed != value.size() || handleValue == 0) { + return nullptr; + } + return reinterpret_cast(static_cast(handleValue)); + } catch (...) { + return nullptr; + } +} + +bool parseConfig(const std::string& json, CaptureConfig& config) { + config.schemaVersion = findInt(json, "schemaVersion", 1); + config.outputPath = findString(json, "screenPath"); + if (config.outputPath.empty()) { + config.outputPath = findString(json, "outputPath"); + } + if (config.outputPath.empty()) { + return false; + } + + config.recordingId = findInt64(json, "recordingId", 0); + config.sourceType = findString(json, "sourceType"); + if (config.sourceType.empty()) { + config.sourceType = "display"; + } + config.sourceId = findString(json, "sourceId"); + config.windowHandle = findString(json, "windowHandle"); + if (config.windowHandle.empty()) { + config.windowHandle = parseWindowHandleFromSourceId(config.sourceId); + } + config.displayId = findInt64(json, "displayId", 0); + config.fps = std::clamp(findInt(json, "fps", 60), 1, 120); + config.width = findInt(json, "videoWidth", findInt(json, "width", 0)); + config.height = findInt(json, "videoHeight", findInt(json, "height", 0)); + config.bounds.x = findInt(json, "displayX", 0); + config.bounds.y = findInt(json, "displayY", 0); + config.bounds.width = findInt(json, "displayW", 0); + config.bounds.height = findInt(json, "displayH", 0); + config.hasDisplayBounds = findBool(json, "hasDisplayBounds", false); + config.captureSystemAudio = findBool(json, "captureSystemAudio", false); + config.captureMic = findBool(json, "captureMic", false); + config.captureCursor = findBool(json, "captureCursor", false); + config.webcamEnabled = findBool(json, "webcamEnabled", false); + config.microphoneDeviceId = findString(json, "microphoneDeviceId"); + config.microphoneDeviceName = findString(json, "microphoneDeviceName"); + config.microphoneGain = findDouble(json, "microphoneGain", 1.0); + config.webcamDeviceId = findString(json, "webcamDeviceId"); + config.webcamDeviceName = findString(json, "webcamDeviceName"); + config.webcamDirectShowClsid = findString(json, "webcamDirectShowClsid"); + config.webcamOutputPath = findString(json, "webcamPath"); + config.webcamWidth = findInt(json, "webcamWidth", 0); + config.webcamHeight = findInt(json, "webcamHeight", 0); + config.webcamFps = findInt(json, "webcamFps", 0); + return true; +} + +void readCaptureCommands(CaptureControl& control, const std::function& onPauseChanged) { + std::string line; + while (std::getline(std::cin, line)) { + if (line == "stop" || line == "q" || line == "quit") { + control.stopRequested = true; + control.cv.notify_all(); + return; + } + if (line == "pause") { + control.setPaused(true); + onPauseChanged(true); + std::cout << "{\"event\":\"recording-paused\",\"schemaVersion\":2}" << std::endl; + control.cv.notify_all(); + continue; + } + if (line == "resume") { + control.setPaused(false); + onPauseChanged(false); + std::cout << "{\"event\":\"recording-resumed\",\"schemaVersion\":2}" << std::endl; + control.cv.notify_all(); + continue; + } + } + control.stopRequested = true; + control.cv.notify_all(); +} + +} // namespace + +int main(int argc, char* argv[]) { + if (argc < 2) { + std::cerr << "ERROR: Missing JSON config argument" << std::endl; + return 1; + } + + winrt::init_apartment(winrt::apartment_type::multi_threaded); + + CaptureConfig config; + if (!parseConfig(argv[1], config)) { + std::cerr << "ERROR: Failed to parse config JSON" << std::endl; + return 1; + } + + std::cout << "{\"event\":\"ready\",\"schemaVersion\":2}" << std::endl; + + WgcSession session; + if (config.sourceType == "display") { + HMONITOR monitor = findMonitorForCapture( + config.displayId, + config.hasDisplayBounds ? &config.bounds : nullptr); + if (!monitor) { + std::cerr << "ERROR: Could not resolve monitor" << std::endl; + return 1; + } + if (!session.initialize(monitor, config.fps, config.captureCursor)) { + std::cerr << "ERROR: Failed to initialize WGC display session" << std::endl; + return 1; + } + } else if (config.sourceType == "window") { + HWND window = parseWindowHandle(config.windowHandle); + if (!window || !IsWindow(window)) { + std::cerr << "ERROR: Native window capture requires a valid HWND" << std::endl; + return 1; + } + if (!session.initialize(window, config.fps, config.captureCursor)) { + std::cerr << "ERROR: Failed to initialize WGC window session" << std::endl; + return 1; + } + } else { + std::cerr << "ERROR: Unsupported native capture source type: " << config.sourceType << std::endl; + return 1; + } + + // WGC owns the captured texture size. Encoding must use that exact size + // until a dedicated GPU scaling pass is introduced; CopyResource requires + // matching resource dimensions. + int width = session.captureWidth(); + int height = session.captureHeight(); + width = (std::max(2, width) / 2) * 2; + height = (std::max(2, height) / 2) * 2; + + const int pixels = width * height; + const int bitrate = pixels >= 3840 * 2160 ? 45'000'000 : pixels >= 2560 * 1440 ? 28'000'000 : 18'000'000; + + WebcamCapture webcamCapture; + bool webcamActive = false; + bool writeSeparateWebcam = false; + if (config.webcamEnabled) { + if (!webcamCapture.initialize( + utf8ToWide(config.webcamDeviceId), + utf8ToWide(config.webcamDeviceName), + utf8ToWide(config.webcamDirectShowClsid), + config.webcamWidth, + config.webcamHeight, + config.webcamFps > 0 ? config.webcamFps : config.fps)) { + std::cerr << "ERROR: Failed to initialize native webcam capture" << std::endl; + return 1; + } + std::cout << "{\"event\":\"webcam-format\",\"schemaVersion\":2,\"width\":" << webcamCapture.width() + << ",\"height\":" << webcamCapture.height() + << ",\"fps\":" << webcamCapture.fps() + << ",\"deviceName\":\"" << jsonEscape(wideToUtf8(webcamCapture.selectedDeviceName())) + << "\"}" << std::endl; + writeSeparateWebcam = !config.webcamOutputPath.empty(); + } + + WasapiLoopbackCapture loopbackCapture; + WasapiLoopbackCapture microphoneCapture; + const AudioInputFormat* audioFormat = nullptr; + AudioInputFormat encoderAudioFormat{}; + AudioInputFormat systemAudioFormat{}; + AudioInputFormat microphoneAudioFormat{}; + if (config.captureSystemAudio) { + if (!loopbackCapture.initializeSystemLoopback()) { + std::cerr << "ERROR: Failed to initialize WASAPI loopback capture" << std::endl; + return 1; + } + systemAudioFormat = loopbackCapture.inputFormat(); + audioFormat = &loopbackCapture.inputFormat(); + } + if (config.captureMic) { + if (!microphoneCapture.initializeMicrophone( + utf8ToWide(config.microphoneDeviceId), + utf8ToWide(config.microphoneDeviceName))) { + std::cerr << "ERROR: Failed to initialize WASAPI microphone capture" << std::endl; + return 1; + } + microphoneAudioFormat = microphoneCapture.inputFormat(); + if (!audioFormat) { + audioFormat = µphoneCapture.inputFormat(); + } + } + if (audioFormat) { + std::cout << "{\"event\":\"audio-format\",\"schemaVersion\":2,\"sampleRate\":" << audioFormat->sampleRate + << ",\"channels\":" << audioFormat->channels + << ",\"bitsPerSample\":" << audioFormat->bitsPerSample + << ",\"system\":" << (config.captureSystemAudio ? "true" : "false") + << ",\"microphone\":" << (config.captureMic ? "true" : "false"); + if (config.captureMic) { + std::cout << ",\"microphoneDeviceName\":\"" + << jsonEscape(wideToUtf8(microphoneCapture.selectedDeviceName())) << "\""; + } + std::cout << "}" << std::endl; + encoderAudioFormat = makeAacCompatibleAudioFormat(*audioFormat); + std::cout << "{\"event\":\"encoder-audio-format\",\"schemaVersion\":2,\"sampleRate\":" + << encoderAudioFormat.sampleRate + << ",\"channels\":" << encoderAudioFormat.channels + << ",\"bitsPerSample\":" << encoderAudioFormat.bitsPerSample + << "}" << std::endl; + } + + MFEncoder encoder; + if (!encoder.initialize( + utf8ToWide(config.outputPath), + width, + height, + config.fps, + bitrate, + session.device(), + session.context(), + audioFormat ? &encoderAudioFormat : nullptr)) { + std::cerr << "ERROR: Failed to initialize Media Foundation encoder" << std::endl; + return 1; + } + + MFEncoder webcamEncoder; + if (writeSeparateWebcam) { + const int webcamPixels = std::max(1, webcamCapture.width()) * std::max(1, webcamCapture.height()); + const int webcamBitrate = webcamPixels >= 1280 * 720 ? 8'000'000 : 4'000'000; + if (!webcamEncoder.initialize( + utf8ToWide(config.webcamOutputPath), + webcamCapture.width(), + webcamCapture.height(), + webcamCapture.fps(), + webcamBitrate, + session.device(), + session.context(), + nullptr)) { + std::cerr << "ERROR: Failed to initialize native webcam encoder" << std::endl; + return 1; + } + } + + std::mutex mutex; + CaptureControl control; + std::atomic firstFrameWritten = false; + std::atomic encodeFailed = false; + Microsoft::WRL::ComPtr latestFrameTexture; + int64_t latestFrameTimestampHns = 0; + int64_t firstFrameTimestampHns = -1; + std::vector latestWebcamFrame; + int latestWebcamWidth = 0; + int latestWebcamHeight = 0; + uint64_t latestWebcamSequence = 0; + bool hasVisibleWebcamFrame = false; + + session.setFrameCallback([&](ID3D11Texture2D* texture, int64_t timestampHns) { + if (control.stopRequested || control.paused) { + return; + } + + std::scoped_lock lock(mutex); + if (!latestFrameTexture) { + D3D11_TEXTURE2D_DESC desc{}; + texture->GetDesc(&desc); + desc.BindFlags = 0; + desc.CPUAccessFlags = 0; + desc.MiscFlags = 0; + if (FAILED(session.device()->CreateTexture2D(&desc, nullptr, &latestFrameTexture))) { + encodeFailed = true; + control.stopRequested = true; + control.cv.notify_all(); + return; + } + } + + session.context()->CopyResource(latestFrameTexture.Get(), texture); + latestFrameTimestampHns = timestampHns; + if (!firstFrameWritten.exchange(true)) { + control.cv.notify_all(); + } + }); + + auto writeVideoFrames = [&]() { + const auto frameDuration = std::chrono::duration_cast( + std::chrono::duration(1.0 / config.fps)); + uint64_t frameIndex = 0; + uint64_t lastWrittenWebcamSequence = 0; + uint64_t webcamOutputFrameIndex = 0; + int64_t lastEncodedVideoTimestampHns = -1; + + while (!control.stopRequested && !encodeFailed) { + { + std::unique_lock lock(mutex); + control.cv.wait(lock, [&] { + return control.stopRequested.load() || + encodeFailed.load() || + (!control.paused.load() && latestFrameTexture); + }); + if (control.stopRequested || encodeFailed) { + break; + } + if (webcamActive) { + WebcamFrameSnapshot candidateWebcamFrame; + if (webcamCapture.copyLatestFrame(candidateWebcamFrame) && + candidateWebcamFrame.sequence != latestWebcamSequence && + hasVisibleBgraContent(candidateWebcamFrame.data)) { + latestWebcamFrame = std::move(candidateWebcamFrame.data); + latestWebcamWidth = candidateWebcamFrame.width; + latestWebcamHeight = candidateWebcamFrame.height; + latestWebcamSequence = candidateWebcamFrame.sequence; + hasVisibleWebcamFrame = true; + } + } + const BgraFrameView webcamFrame{ + hasVisibleWebcamFrame && !latestWebcamFrame.empty() ? latestWebcamFrame.data() : nullptr, + latestWebcamWidth, + latestWebcamHeight, + }; + const int64_t syntheticTimestampHns = + static_cast((frameIndex * 10'000'000ULL) / config.fps); + const int64_t sourceTimestampHns = + latestFrameTimestampHns > 0 ? latestFrameTimestampHns : syntheticTimestampHns; + if (firstFrameTimestampHns < 0) { + firstFrameTimestampHns = sourceTimestampHns; + } + int64_t frameTimestampHns = + std::max( + 0, + sourceTimestampHns - firstFrameTimestampHns - control.pausedDurationHns()); + if (lastEncodedVideoTimestampHns >= 0 && + frameTimestampHns <= lastEncodedVideoTimestampHns) { + frameTimestampHns = + lastEncodedVideoTimestampHns + static_cast(10'000'000ULL / config.fps); + } + if (writeSeparateWebcam && webcamFrame.data && + latestWebcamSequence != lastWrittenWebcamSequence) { + const int64_t webcamTimestampHns = static_cast( + (webcamOutputFrameIndex * 10'000'000ULL) / std::max(1, webcamCapture.fps())); + if (!webcamEncoder.writeBgraFrame(webcamFrame, webcamTimestampHns)) { + encodeFailed = true; + stopRequested = true; + cv.notify_all(); + return; + } + lastWrittenWebcamSequence = latestWebcamSequence; + webcamOutputFrameIndex += 1; + } + if (latestFrameTexture && !encoder.writeFrame( + latestFrameTexture.Get(), + frameTimestampHns, + !writeSeparateWebcam && webcamFrame.data ? &webcamFrame : nullptr)) { + encodeFailed = true; + control.stopRequested = true; + control.cv.notify_all(); + return; + } + if (latestFrameTexture) { + lastEncodedVideoTimestampHns = frameTimestampHns; + } + } + + frameIndex += 1; + std::this_thread::sleep_for(frameDuration); + } + }; + + std::thread videoWriterThread; + + auto stopVideoWriter = [&]() { + if (videoWriterThread.joinable()) { + videoWriterThread.join(); + } + }; + + auto startVideoWriter = [&]() { + videoWriterThread = std::thread(writeVideoFrames); + }; + + std::unique_ptr audioMixer; + auto startAudioCaptures = [&]() -> bool { + if (!audioFormat) { + return true; + } + + audioMixer = std::make_unique( + encoderAudioFormat, + config.captureSystemAudio ? systemAudioFormat : encoderAudioFormat, + config.captureMic ? microphoneAudioFormat : encoderAudioFormat, + config.captureSystemAudio, + config.captureMic, + config.microphoneGain, + [&](const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns) { + if (!encoder.writeAudio(data, byteCount, timestampHns, durationHns)) { + encodeFailed = true; + control.stopRequested = true; + control.cv.notify_all(); + return false; + } + return true; + }); + + if (!audioMixer->start()) { + std::cerr << "ERROR: Failed to start native audio mixer" << std::endl; + return false; + } + + if (config.captureMic) { + if (!microphoneCapture.start([&](const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns) { + (void)timestampHns; + (void)durationHns; + if (control.stopRequested || !audioMixer) { + return; + } + + audioMixer->pushMicrophone(data, byteCount); + })) { + std::cerr << "ERROR: Failed to start WASAPI microphone capture" << std::endl; + audioMixer->stop(); + return false; + } + } + + if (config.captureSystemAudio) { + if (!loopbackCapture.start([&](const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns) { + (void)timestampHns; + (void)durationHns; + if (control.stopRequested || !audioMixer) { + return; + } + + audioMixer->pushSystem(data, byteCount); + })) { + std::cerr << "ERROR: Failed to start WASAPI loopback capture" << std::endl; + microphoneCapture.stop(); + audioMixer->stop(); + return false; + } + } + + return true; + }; + + if (!startAudioCaptures()) { + return 1; + } + if (config.webcamEnabled) { + if (!webcamCapture.start()) { + microphoneCapture.stop(); + loopbackCapture.stop(); + if (audioMixer) { + audioMixer->stop(); + } + std::cerr << "ERROR: Failed to start native webcam capture" << std::endl; + return 1; + } + webcamActive = true; + const auto webcamDeadline = std::chrono::steady_clock::now() + std::chrono::seconds(3); + while (std::chrono::steady_clock::now() < webcamDeadline && !hasVisibleWebcamFrame) { + WebcamFrameSnapshot candidateWebcamFrame; + if (webcamCapture.copyLatestFrame(candidateWebcamFrame) && + hasVisibleBgraContent(candidateWebcamFrame.data)) { + latestWebcamFrame = std::move(candidateWebcamFrame.data); + latestWebcamWidth = candidateWebcamFrame.width; + latestWebcamHeight = candidateWebcamFrame.height; + latestWebcamSequence = candidateWebcamFrame.sequence; + hasVisibleWebcamFrame = true; + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } + if (!hasVisibleWebcamFrame) { + std::cerr << "WARNING: Native webcam started but no visible frame was available before screen capture" + << std::endl; + } + } + + if (!session.start()) { + webcamCapture.stop(); + microphoneCapture.stop(); + loopbackCapture.stop(); + if (audioMixer) { + audioMixer->stop(); + } + std::cerr << "ERROR: Failed to start WGC session" << std::endl; + return 1; + } + + std::thread stdinThread(readCaptureCommands, std::ref(control), [&](bool isPaused) { + if (audioMixer) { + audioMixer->setPaused(isPaused); + } + }); + + { + std::unique_lock lock(mutex); + const bool started = control.cv.wait_for(lock, std::chrono::seconds(10), [&] { + return firstFrameWritten.load() || control.stopRequested.load(); + }); + if (!started || !firstFrameWritten) { + control.stopRequested = true; + control.cv.notify_all(); + if (stdinThread.joinable()) { + stdinThread.detach(); + } + microphoneCapture.stop(); + loopbackCapture.stop(); + webcamCapture.stop(); + if (audioMixer) { + audioMixer->stop(); + } + session.stop(); + std::cerr << "ERROR: Timed out waiting for first WGC frame" << std::endl; + return 1; + } + } + + if (audioMixer) { + audioMixer->beginTimeline(); + } + startVideoWriter(); + + std::cout << "{\"event\":\"recording-started\",\"schemaVersion\":2}" << std::endl; + std::cout << "Recording started" << std::endl; + + { + std::unique_lock lock(mutex); + control.cv.wait(lock, [&] { + return control.stopRequested.load(); + }); + } + + microphoneCapture.stop(); + loopbackCapture.stop(); + webcamCapture.stop(); + if (audioMixer) { + audioMixer->stop(); + } + stopVideoWriter(); + session.stop(); + { + std::scoped_lock lock(mutex); + encoder.finalize(); + if (writeSeparateWebcam) { + webcamEncoder.finalize(); + } + } + + if (stdinThread.joinable()) { + stdinThread.detach(); + } + + if (encodeFailed) { + std::cerr << "ERROR: Failed to encode WGC frame" << std::endl; + return 1; + } + + std::cout << "{\"event\":\"recording-stopped\",\"schemaVersion\":2,\"screenPath\":\"" + << jsonEscape(config.outputPath) << "\""; + if (writeSeparateWebcam) { + std::cout << ",\"webcamPath\":\"" << jsonEscape(config.webcamOutputPath) << "\""; + } + std::cout << "}" << std::endl; + std::cout << "Recording stopped. Output path: " << config.outputPath << std::endl; + return 0; +} diff --git a/electron/native/wgc-capture/src/mf_encoder.cpp b/electron/native/wgc-capture/src/mf_encoder.cpp new file mode 100644 index 0000000..18bc4cc --- /dev/null +++ b/electron/native/wgc-capture/src/mf_encoder.cpp @@ -0,0 +1,450 @@ +#include "mf_encoder.h" + +#include "audio_sample_utils.h" + +#include +#include +#include + +#include +#include +#include + +namespace { + +bool succeeded(HRESULT hr, const char* label) { + if (SUCCEEDED(hr)) { + return true; + } + + std::cerr << "ERROR: " << label << " failed (hr=0x" << std::hex << hr << std::dec << ")" + << std::endl; + return false; +} + +void setFrameSize(IMFMediaType* type, UINT32 width, UINT32 height) { + MFSetAttributeSize(type, MF_MT_FRAME_SIZE, width, height); +} + +void setFrameRate(IMFMediaType* type, UINT32 fps) { + MFSetAttributeRatio(type, MF_MT_FRAME_RATE, fps, 1); +} + +void setPixelAspectRatio(IMFMediaType* type) { + MFSetAttributeRatio(type, MF_MT_PIXEL_ASPECT_RATIO, 1, 1); +} + +void setAudioFormat(IMFMediaType* type, UINT32 channels, UINT32 sampleRate, UINT32 bitsPerSample) { + type->SetUINT32(MF_MT_AUDIO_NUM_CHANNELS, channels); + type->SetUINT32(MF_MT_AUDIO_SAMPLES_PER_SECOND, sampleRate); + type->SetUINT32(MF_MT_AUDIO_BITS_PER_SAMPLE, bitsPerSample); +} + +void compositeWebcam(BYTE* destination, int width, int height, const BgraFrameView& webcamFrame) { + if (!webcamFrame.data || webcamFrame.width <= 0 || webcamFrame.height <= 0 || width <= 0 || height <= 0) { + return; + } + + const int margin = std::max(16, std::min(width, height) / 60); + const int maxOverlayWidth = std::max(2, width / 4); + int overlayWidth = maxOverlayWidth; + int overlayHeight = static_cast( + (static_cast(overlayWidth) * webcamFrame.height) / std::max(1, webcamFrame.width)); + const int maxOverlayHeight = std::max(2, height / 3); + if (overlayHeight > maxOverlayHeight) { + overlayHeight = maxOverlayHeight; + overlayWidth = static_cast( + (static_cast(overlayHeight) * webcamFrame.width) / std::max(1, webcamFrame.height)); + } + + overlayWidth = std::max(2, std::min(overlayWidth, width - margin * 2)); + overlayHeight = std::max(2, std::min(overlayHeight, height - margin * 2)); + const int originX = std::max(0, width - overlayWidth - margin); + const int originY = std::max(0, height - overlayHeight - margin); + + for (int y = 0; y < overlayHeight; y += 1) { + const int sourceY = static_cast((static_cast(y) * webcamFrame.height) / overlayHeight); + BYTE* destinationRow = destination + ((originY + y) * width + originX) * 4; + for (int x = 0; x < overlayWidth; x += 1) { + const int sourceX = static_cast((static_cast(x) * webcamFrame.width) / overlayWidth); + const BYTE* source = webcamFrame.data + (sourceY * webcamFrame.width + sourceX) * 4; + BYTE* target = destinationRow + x * 4; + target[0] = source[0]; + target[1] = source[1]; + target[2] = source[2]; + target[3] = 255; + } + } +} + +} // namespace + +MFEncoder::~MFEncoder() { + finalize(); +} + +bool MFEncoder::initialize( + const std::wstring& outputPath, + int width, + int height, + int fps, + int bitrate, + ID3D11Device* device, + ID3D11DeviceContext* context, + const AudioInputFormat* audioFormat) { + width_ = (std::max(2, width) / 2) * 2; + height_ = (std::max(2, height) / 2) * 2; + fps_ = std::max(1, fps); + device_ = device; + context_ = context; + + if (!succeeded(MFStartup(MF_VERSION), "MFStartup")) { + return false; + } + + Microsoft::WRL::ComPtr outputType; + if (!succeeded(MFCreateMediaType(&outputType), "MFCreateMediaType(output)")) { + return false; + } + outputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + outputType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264); + outputType->SetUINT32(MF_MT_AVG_BITRATE, static_cast(std::max(1, bitrate))); + outputType->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); + setFrameSize(outputType.Get(), static_cast(width_), static_cast(height_)); + setFrameRate(outputType.Get(), static_cast(fps_)); + setPixelAspectRatio(outputType.Get()); + + if (!succeeded(MFCreateSinkWriterFromURL(outputPath.c_str(), nullptr, nullptr, &sinkWriter_), + "MFCreateSinkWriterFromURL")) { + return false; + } + if (!succeeded(sinkWriter_->AddStream(outputType.Get(), &videoStreamIndex_), "AddStream")) { + return false; + } + + if (audioFormat && !configureAudioStream(*audioFormat)) { + return false; + } + + Microsoft::WRL::ComPtr inputType; + if (!succeeded(MFCreateMediaType(&inputType), "MFCreateMediaType(input)")) { + return false; + } + inputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + inputType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_RGB32); + inputType->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); + inputType->SetUINT32(MF_MT_DEFAULT_STRIDE, static_cast(width_ * 4)); + setFrameSize(inputType.Get(), static_cast(width_), static_cast(height_)); + setFrameRate(inputType.Get(), static_cast(fps_)); + setPixelAspectRatio(inputType.Get()); + + if (!succeeded(sinkWriter_->SetInputMediaType(videoStreamIndex_, inputType.Get(), nullptr), + "SetInputMediaType")) { + return false; + } + if (!succeeded(sinkWriter_->BeginWriting(), "BeginWriting")) { + return false; + } + + return true; +} + +bool MFEncoder::configureAudioStream(const AudioInputFormat& audioFormat) { + if (!sinkWriter_) { + return false; + } + if (audioFormat.sampleRate == 0 || audioFormat.channels == 0 || audioFormat.blockAlign == 0) { + std::cerr << "ERROR: Invalid audio input format" << std::endl; + return false; + } + + const AudioInputFormat encoderFormat = makeAacCompatibleAudioFormat(audioFormat); + const UINT32 aacBytesPerSecond = 24'000; + + Microsoft::WRL::ComPtr outputType; + if (!succeeded(MFCreateMediaType(&outputType), "MFCreateMediaType(audio output)")) { + return false; + } + outputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Audio); + outputType->SetGUID(MF_MT_SUBTYPE, MFAudioFormat_AAC); + setAudioFormat(outputType.Get(), encoderFormat.channels, encoderFormat.sampleRate, 16); + outputType->SetUINT32(MF_MT_AUDIO_AVG_BYTES_PER_SECOND, aacBytesPerSecond); + outputType->SetUINT32(MF_MT_AAC_PAYLOAD_TYPE, 0); + + if (!succeeded(sinkWriter_->AddStream(outputType.Get(), &audioStreamIndex_), "AddStream(audio)")) { + return false; + } + + Microsoft::WRL::ComPtr inputType; + if (!succeeded(MFCreateMediaType(&inputType), "MFCreateMediaType(audio input)")) { + return false; + } + inputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Audio); + inputType->SetGUID(MF_MT_SUBTYPE, encoderFormat.subtype); + setAudioFormat(inputType.Get(), encoderFormat.channels, encoderFormat.sampleRate, encoderFormat.bitsPerSample); + inputType->SetUINT32(MF_MT_AUDIO_BLOCK_ALIGNMENT, encoderFormat.blockAlign); + inputType->SetUINT32(MF_MT_AUDIO_AVG_BYTES_PER_SECOND, encoderFormat.avgBytesPerSec); + inputType->SetUINT32(MF_MT_ALL_SAMPLES_INDEPENDENT, TRUE); + + if (!succeeded(sinkWriter_->SetInputMediaType(audioStreamIndex_, inputType.Get(), nullptr), + "SetInputMediaType(audio)")) { + return false; + } + + hasAudioStream_ = true; + return true; +} + +bool MFEncoder::ensureStagingTexture(ID3D11Texture2D* texture) { + if (stagingTexture_) { + return true; + } + + D3D11_TEXTURE2D_DESC desc{}; + texture->GetDesc(&desc); + desc.Width = static_cast(width_); + desc.Height = static_cast(height_); + desc.MipLevels = 1; + desc.ArraySize = 1; + desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + desc.SampleDesc.Count = 1; + desc.SampleDesc.Quality = 0; + desc.Usage = D3D11_USAGE_STAGING; + desc.BindFlags = 0; + desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + desc.MiscFlags = 0; + + return succeeded(device_->CreateTexture2D(&desc, nullptr, &stagingTexture_), + "CreateTexture2D(staging)"); +} + +bool MFEncoder::copyFrameToBuffer( + ID3D11Texture2D* texture, + BYTE* destination, + DWORD destinationSize, + const BgraFrameView* webcamFrame) { + if (!ensureStagingTexture(texture)) { + return false; + } + + context_->CopyResource(stagingTexture_.Get(), texture); + + D3D11_MAPPED_SUBRESOURCE mapped{}; + if (!succeeded(context_->Map(stagingTexture_.Get(), 0, D3D11_MAP_READ, 0, &mapped), "Map")) { + return false; + } + + const DWORD rowBytes = static_cast(width_ * 4); + const DWORD requiredBytes = rowBytes * static_cast(height_); + if (destinationSize < requiredBytes) { + context_->Unmap(stagingTexture_.Get(), 0); + std::cerr << "ERROR: Media Foundation buffer is too small" << std::endl; + return false; + } + + auto* source = static_cast(mapped.pData); + for (int y = 0; y < height_; y += 1) { + std::memcpy(destination + rowBytes * y, source + mapped.RowPitch * y, rowBytes); + } + if (webcamFrame) { + compositeWebcam(destination, width_, height_, *webcamFrame); + } + + context_->Unmap(stagingTexture_.Get(), 0); + return true; +} + +bool MFEncoder::copyBgraFrameToBuffer(const BgraFrameView& frame, BYTE* destination, DWORD destinationSize) { + if (!frame.data || frame.width <= 0 || frame.height <= 0) { + return false; + } + + const DWORD rowBytes = static_cast(width_ * 4); + const DWORD requiredBytes = rowBytes * static_cast(height_); + if (destinationSize < requiredBytes) { + std::cerr << "ERROR: Media Foundation webcam buffer is too small" << std::endl; + return false; + } + + if (frame.width == width_ && frame.height == height_) { + for (DWORD i = 0; i < requiredBytes; i += 4) { + destination[i] = frame.data[i]; + destination[i + 1] = frame.data[i + 1]; + destination[i + 2] = frame.data[i + 2]; + destination[i + 3] = 255; + } + return true; + } + + for (int y = 0; y < height_; y += 1) { + const int sourceY = static_cast((static_cast(y) * frame.height) / height_); + BYTE* destinationRow = destination + rowBytes * y; + for (int x = 0; x < width_; x += 1) { + const int sourceX = static_cast((static_cast(x) * frame.width) / width_); + const BYTE* source = frame.data + (sourceY * frame.width + sourceX) * 4; + BYTE* target = destinationRow + x * 4; + target[0] = source[0]; + target[1] = source[1]; + target[2] = source[2]; + target[3] = 255; + } + } + + return true; +} + +bool MFEncoder::writeFrame(ID3D11Texture2D* texture, int64_t timestampHns, const BgraFrameView* webcamFrame) { + std::scoped_lock writerLock(writerMutex_); + if (!sinkWriter_ || finalized_) { + return false; + } + + if (firstTimestampHns_ < 0) { + firstTimestampHns_ = timestampHns; + } + + int64_t sampleTime = timestampHns - firstTimestampHns_; + if (sampleTime <= lastTimestampHns_) { + sampleTime = lastTimestampHns_ + (10'000'000LL / fps_); + } + const int64_t sampleDuration = 10'000'000LL / fps_; + lastTimestampHns_ = sampleTime; + + Microsoft::WRL::ComPtr buffer; + const DWORD frameBytes = static_cast(width_ * height_ * 4); + if (!succeeded(MFCreateMemoryBuffer(frameBytes, &buffer), "MFCreateMemoryBuffer")) { + return false; + } + + BYTE* data = nullptr; + DWORD maxLength = 0; + DWORD currentLength = 0; + if (!succeeded(buffer->Lock(&data, &maxLength, ¤tLength), "IMFMediaBuffer::Lock")) { + return false; + } + + const bool copied = copyFrameToBuffer(texture, data, maxLength, webcamFrame); + buffer->Unlock(); + if (!copied) { + return false; + } + buffer->SetCurrentLength(frameBytes); + + Microsoft::WRL::ComPtr sample; + if (!succeeded(MFCreateSample(&sample), "MFCreateSample")) { + return false; + } + sample->AddBuffer(buffer.Get()); + sample->SetSampleTime(sampleTime); + sample->SetSampleDuration(sampleDuration); + + return succeeded(sinkWriter_->WriteSample(videoStreamIndex_, sample.Get()), "WriteSample"); +} + +bool MFEncoder::writeBgraFrame(const BgraFrameView& frame, int64_t timestampHns) { + std::scoped_lock writerLock(writerMutex_); + if (!sinkWriter_ || finalized_) { + return false; + } + + if (firstTimestampHns_ < 0) { + firstTimestampHns_ = timestampHns; + } + + int64_t sampleTime = timestampHns - firstTimestampHns_; + if (sampleTime <= lastTimestampHns_) { + sampleTime = lastTimestampHns_ + (10'000'000LL / fps_); + } + const int64_t sampleDuration = 10'000'000LL / fps_; + lastTimestampHns_ = sampleTime; + + Microsoft::WRL::ComPtr buffer; + const DWORD frameBytes = static_cast(width_ * height_ * 4); + if (!succeeded(MFCreateMemoryBuffer(frameBytes, &buffer), "MFCreateMemoryBuffer(webcam)")) { + return false; + } + + BYTE* data = nullptr; + DWORD maxLength = 0; + DWORD currentLength = 0; + if (!succeeded(buffer->Lock(&data, &maxLength, ¤tLength), "IMFMediaBuffer::Lock(webcam)")) { + return false; + } + + const bool copied = copyBgraFrameToBuffer(frame, data, maxLength); + buffer->Unlock(); + if (!copied) { + return false; + } + buffer->SetCurrentLength(frameBytes); + + Microsoft::WRL::ComPtr sample; + if (!succeeded(MFCreateSample(&sample), "MFCreateSample(webcam)")) { + return false; + } + sample->AddBuffer(buffer.Get()); + sample->SetSampleTime(sampleTime); + sample->SetSampleDuration(sampleDuration); + + return succeeded(sinkWriter_->WriteSample(videoStreamIndex_, sample.Get()), "WriteSample(webcam)"); +} + +bool MFEncoder::writeAudio(const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns) { + std::scoped_lock writerLock(writerMutex_); + if (!sinkWriter_ || finalized_ || !hasAudioStream_) { + return false; + } + if (!data || byteCount == 0 || durationHns <= 0) { + return true; + } + + Microsoft::WRL::ComPtr buffer; + if (!succeeded(MFCreateMemoryBuffer(byteCount, &buffer), "MFCreateMemoryBuffer(audio)")) { + return false; + } + + BYTE* destination = nullptr; + DWORD maxLength = 0; + DWORD currentLength = 0; + if (!succeeded(buffer->Lock(&destination, &maxLength, ¤tLength), + "IMFMediaBuffer::Lock(audio)")) { + return false; + } + if (maxLength < byteCount) { + buffer->Unlock(); + std::cerr << "ERROR: Media Foundation audio buffer is too small" << std::endl; + return false; + } + std::memcpy(destination, data, byteCount); + buffer->Unlock(); + buffer->SetCurrentLength(byteCount); + + Microsoft::WRL::ComPtr sample; + if (!succeeded(MFCreateSample(&sample), "MFCreateSample(audio)")) { + return false; + } + sample->AddBuffer(buffer.Get()); + sample->SetSampleTime(std::max(0, timestampHns)); + sample->SetSampleDuration(durationHns); + + return succeeded(sinkWriter_->WriteSample(audioStreamIndex_, sample.Get()), "WriteSample(audio)"); +} + +bool MFEncoder::finalize() { + std::scoped_lock writerLock(writerMutex_); + if (finalized_) { + return true; + } + + finalized_ = true; + bool ok = true; + if (sinkWriter_) { + ok = succeeded(sinkWriter_->Finalize(), "SinkWriter::Finalize"); + sinkWriter_.Reset(); + } + stagingTexture_.Reset(); + context_.Reset(); + device_.Reset(); + MFShutdown(); + return ok; +} diff --git a/electron/native/wgc-capture/src/mf_encoder.h b/electron/native/wgc-capture/src/mf_encoder.h new file mode 100644 index 0000000..e7821e9 --- /dev/null +++ b/electron/native/wgc-capture/src/mf_encoder.h @@ -0,0 +1,75 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +struct BgraFrameView { + const BYTE* data = nullptr; + int width = 0; + int height = 0; +}; + +struct AudioInputFormat { + GUID subtype = MFAudioFormat_PCM; + UINT32 sampleRate = 0; + UINT32 channels = 0; + UINT32 bitsPerSample = 0; + UINT32 blockAlign = 0; + UINT32 avgBytesPerSec = 0; +}; + +class MFEncoder { +public: + MFEncoder() = default; + ~MFEncoder(); + + MFEncoder(const MFEncoder&) = delete; + MFEncoder& operator=(const MFEncoder&) = delete; + + bool initialize( + const std::wstring& outputPath, + int width, + int height, + int fps, + int bitrate, + ID3D11Device* device, + ID3D11DeviceContext* context, + const AudioInputFormat* audioFormat = nullptr); + bool writeFrame(ID3D11Texture2D* texture, int64_t timestampHns, const BgraFrameView* webcamFrame = nullptr); + bool writeBgraFrame(const BgraFrameView& frame, int64_t timestampHns); + bool writeAudio(const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns); + bool finalize(); + +private: + bool ensureStagingTexture(ID3D11Texture2D* texture); + bool copyFrameToBuffer( + ID3D11Texture2D* texture, + BYTE* destination, + DWORD destinationSize, + const BgraFrameView* webcamFrame); + bool copyBgraFrameToBuffer(const BgraFrameView& frame, BYTE* destination, DWORD destinationSize); + bool configureAudioStream(const AudioInputFormat& audioFormat); + + Microsoft::WRL::ComPtr sinkWriter_; + Microsoft::WRL::ComPtr device_; + Microsoft::WRL::ComPtr context_; + Microsoft::WRL::ComPtr stagingTexture_; + std::mutex writerMutex_; + DWORD videoStreamIndex_ = 0; + DWORD audioStreamIndex_ = 0; + bool hasAudioStream_ = false; + int width_ = 0; + int height_ = 0; + int fps_ = 60; + int64_t firstTimestampHns_ = -1; + int64_t lastTimestampHns_ = -1; + bool finalized_ = false; +}; diff --git a/electron/native/wgc-capture/src/monitor_utils.cpp b/electron/native/wgc-capture/src/monitor_utils.cpp new file mode 100644 index 0000000..f83e77d --- /dev/null +++ b/electron/native/wgc-capture/src/monitor_utils.cpp @@ -0,0 +1,88 @@ +#include "monitor_utils.h" + +#include +#include +#include + +namespace { + +struct MonitorCandidate { + HMONITOR monitor = nullptr; + RECT rect{}; +}; + +std::vector enumerateMonitors() { + std::vector monitors; + EnumDisplayMonitors( + nullptr, + nullptr, + [](HMONITOR monitor, HDC, LPRECT rect, LPARAM userData) -> BOOL { + auto* result = reinterpret_cast*>(userData); + result->push_back({monitor, *rect}); + return TRUE; + }, + reinterpret_cast(&monitors)); + return monitors; +} + +bool rectMatchesBounds(const RECT& rect, const MonitorBounds& bounds) { + return rect.left == bounds.x && + rect.top == bounds.y && + (rect.right - rect.left) == bounds.width && + (rect.bottom - rect.top) == bounds.height; +} + +int64_t overlapArea(const RECT& rect, const MonitorBounds& bounds) { + const LONG left = std::max(rect.left, bounds.x); + const LONG top = std::max(rect.top, bounds.y); + const LONG right = std::min(rect.right, bounds.x + bounds.width); + const LONG bottom = std::min(rect.bottom, bounds.y + bounds.height); + if (right <= left || bottom <= top) { + return 0; + } + return static_cast(right - left) * static_cast(bottom - top); +} + +} // namespace + +HMONITOR findMonitorForCapture(int64_t displayId, const MonitorBounds* bounds) { + const auto monitors = enumerateMonitors(); + if (monitors.empty()) { + return MonitorFromPoint({0, 0}, MONITOR_DEFAULTTOPRIMARY); + } + + // Electron's display_id is not stable across all Windows capture backends. + // Bounds are the most reliable contract because they come from Electron's + // selected display and match the WGC monitor coordinate space. + if (bounds && bounds->width > 0 && bounds->height > 0) { + for (const auto& candidate : monitors) { + if (rectMatchesBounds(candidate.rect, *bounds)) { + return candidate.monitor; + } + } + + HMONITOR bestMonitor = nullptr; + int64_t bestArea = 0; + for (const auto& candidate : monitors) { + const int64_t area = overlapArea(candidate.rect, *bounds); + if (area > bestArea) { + bestArea = area; + bestMonitor = candidate.monitor; + } + } + if (bestMonitor) { + return bestMonitor; + } + } + + // Best-effort fallback for helpers invoked without bounds. Some callers pass + // zero-based ids while Win32 monitor handles are pointer values, so only use + // this when it exactly matches the HMONITOR value. + for (const auto& candidate : monitors) { + if (reinterpret_cast(candidate.monitor) == displayId) { + return candidate.monitor; + } + } + + return MonitorFromPoint({0, 0}, MONITOR_DEFAULTTOPRIMARY); +} diff --git a/electron/native/wgc-capture/src/monitor_utils.h b/electron/native/wgc-capture/src/monitor_utils.h new file mode 100644 index 0000000..11d5d83 --- /dev/null +++ b/electron/native/wgc-capture/src/monitor_utils.h @@ -0,0 +1,14 @@ +#pragma once + +#include + +#include + +struct MonitorBounds { + int x = 0; + int y = 0; + int width = 0; + int height = 0; +}; + +HMONITOR findMonitorForCapture(int64_t displayId, const MonitorBounds* bounds); diff --git a/electron/native/wgc-capture/src/wasapi_loopback_capture.cpp b/electron/native/wgc-capture/src/wasapi_loopback_capture.cpp new file mode 100644 index 0000000..0256b04 --- /dev/null +++ b/electron/native/wgc-capture/src/wasapi_loopback_capture.cpp @@ -0,0 +1,411 @@ +#include "wasapi_loopback_capture.h" + +#include +#include +#include + +#include +#include +#include +#include + +namespace { + +constexpr REFERENCE_TIME BufferDurationHns = 10'000'000; +constexpr int64_t HnsPerSecond = 10'000'000; + +bool succeeded(HRESULT hr, const char* label) { + if (SUCCEEDED(hr)) { + return true; + } + + std::cerr << "ERROR: " << label << " failed (hr=0x" << std::hex << hr << std::dec << ")" + << std::endl; + return false; +} + +GUID audioSubtypeFromFormat(WAVEFORMATEX* format) { + if (format->wFormatTag == WAVE_FORMAT_IEEE_FLOAT) { + return MFAudioFormat_Float; + } + if (format->wFormatTag == WAVE_FORMAT_PCM) { + return MFAudioFormat_PCM; + } + if (format->wFormatTag == WAVE_FORMAT_EXTENSIBLE && + format->cbSize >= sizeof(WAVEFORMATEXTENSIBLE) - sizeof(WAVEFORMATEX)) { + auto* extensible = reinterpret_cast(format); + if (extensible->SubFormat == KSDATAFORMAT_SUBTYPE_IEEE_FLOAT) { + return MFAudioFormat_Float; + } + if (extensible->SubFormat == KSDATAFORMAT_SUBTYPE_PCM) { + return MFAudioFormat_PCM; + } + } + return GUID_NULL; +} + +std::wstring normalizeDeviceName(const std::wstring& value) { + std::wstring result; + result.reserve(value.size()); + bool lastWasSpace = true; + + for (const wchar_t c : value) { + if (std::iswalnum(c)) { + result.push_back(static_cast(std::towlower(c))); + lastWasSpace = false; + } else if (!lastWasSpace) { + result.push_back(L' '); + lastWasSpace = true; + } + } + + if (!result.empty() && result.back() == L' ') { + result.pop_back(); + } + return result; +} + +int scoreDeviceName(const std::wstring& candidateName, const std::wstring& candidateId, const std::wstring& requestedName) { + const std::wstring candidate = normalizeDeviceName(candidateName); + const std::wstring id = normalizeDeviceName(candidateId); + const std::wstring requested = normalizeDeviceName(requestedName); + if (requested.empty()) { + return 0; + } + if (candidate == requested) { + return 1000; + } + if (!candidate.empty() && (candidate.find(requested) != std::wstring::npos || requested.find(candidate) != std::wstring::npos)) { + return 900; + } + if (!id.empty() && (id.find(requested) != std::wstring::npos || requested.find(id) != std::wstring::npos)) { + return 800; + } + + int score = 0; + size_t pos = 0; + while (pos < requested.size()) { + const size_t end = requested.find(L' ', pos); + const std::wstring word = requested.substr(pos, end == std::wstring::npos ? std::wstring::npos : end - pos); + if (word.size() > 1 && word != L"microphone" && word != L"mic" && word != L"audio" && word != L"input") { + if (candidate.find(word) != std::wstring::npos) { + score += 100; + } else if (id.find(word) != std::wstring::npos) { + score += 50; + } + } + if (end == std::wstring::npos) { + break; + } + pos = end + 1; + } + return score; +} + +std::wstring getDeviceFriendlyName(IMMDevice* device) { + if (!device) { + return {}; + } + + Microsoft::WRL::ComPtr properties; + HRESULT hr = device->OpenPropertyStore(STGM_READ, &properties); + if (FAILED(hr) || !properties) { + return {}; + } + + PROPVARIANT value; + PropVariantInit(&value); + hr = properties->GetValue(PKEY_Device_FriendlyName, &value); + std::wstring name; + if (SUCCEEDED(hr) && value.vt == VT_LPWSTR && value.pwszVal) { + name = value.pwszVal; + } + PropVariantClear(&value); + return name; +} + +} // namespace + +WasapiLoopbackCapture::~WasapiLoopbackCapture() { + stop(); + if (mixFormat_) { + CoTaskMemFree(mixFormat_); + mixFormat_ = nullptr; + } +} + +bool WasapiLoopbackCapture::initializeSystemLoopback() { + return initialize(WasapiCaptureEndpoint::SystemLoopback, {}, {}); +} + +bool WasapiLoopbackCapture::initializeMicrophone(const std::wstring& deviceId, const std::wstring& deviceName) { + return initialize(WasapiCaptureEndpoint::Microphone, deviceId, deviceName); +} + +bool WasapiLoopbackCapture::initialize(WasapiCaptureEndpoint endpoint, const std::wstring& deviceId, const std::wstring& deviceName) { + HRESULT hr = CoCreateInstance( + __uuidof(MMDeviceEnumerator), + nullptr, + CLSCTX_ALL, + IID_PPV_ARGS(&deviceEnumerator_)); + if (!succeeded(hr, "CoCreateInstance(MMDeviceEnumerator)")) { + return false; + } + + if (endpoint == WasapiCaptureEndpoint::Microphone && !deviceId.empty() && deviceId != L"default") { + hr = deviceEnumerator_->GetDevice(deviceId.c_str(), &device_); + if (FAILED(hr)) { + std::wcerr << L"WARNING: Could not resolve microphone device id directly" + << std::endl; + device_.Reset(); + } + } + + if (endpoint == WasapiCaptureEndpoint::Microphone && !device_ && !deviceName.empty()) { + if (!resolveMicrophoneByName(deviceName)) { + std::wcerr << L"WARNING: Could not resolve microphone by name; using default capture endpoint" + << std::endl; + } + } + + if (!device_) { + const EDataFlow flow = + endpoint == WasapiCaptureEndpoint::SystemLoopback ? eRender : eCapture; + hr = deviceEnumerator_->GetDefaultAudioEndpoint(flow, eConsole, &device_); + if (!succeeded(hr, "GetDefaultAudioEndpoint")) { + return false; + } + } + + selectedDeviceName_ = getDeviceFriendlyName(device_.Get()); + + hr = device_->Activate(__uuidof(IAudioClient), CLSCTX_ALL, nullptr, &audioClient_); + if (!succeeded(hr, "IMMDevice::Activate(IAudioClient)")) { + return false; + } + + hr = audioClient_->GetMixFormat(&mixFormat_); + if (!succeeded(hr, "IAudioClient::GetMixFormat") || !mixFormat_) { + return false; + } + + if (!resolveInputFormat(mixFormat_)) { + std::cerr << "ERROR: Unsupported WASAPI loopback mix format" << std::endl; + return false; + } + + const DWORD streamFlags = + endpoint == WasapiCaptureEndpoint::SystemLoopback ? AUDCLNT_STREAMFLAGS_LOOPBACK : 0; + hr = audioClient_->Initialize( + AUDCLNT_SHAREMODE_SHARED, + streamFlags, + BufferDurationHns, + 0, + mixFormat_, + nullptr); + if (!succeeded(hr, "IAudioClient::Initialize(loopback)")) { + return false; + } + + hr = audioClient_->GetService(IID_PPV_ARGS(&captureClient_)); + if (!succeeded(hr, "IAudioClient::GetService(IAudioCaptureClient)")) { + return false; + } + + return true; +} + +bool WasapiLoopbackCapture::resolveMicrophoneByName(const std::wstring& deviceName) { + if (!deviceEnumerator_ || deviceName.empty()) { + return false; + } + + Microsoft::WRL::ComPtr devices; + HRESULT hr = deviceEnumerator_->EnumAudioEndpoints(eCapture, DEVICE_STATE_ACTIVE, &devices); + if (!succeeded(hr, "IMMDeviceEnumerator::EnumAudioEndpoints(eCapture)")) { + return false; + } + + UINT count = 0; + hr = devices->GetCount(&count); + if (!succeeded(hr, "IMMDeviceCollection::GetCount")) { + return false; + } + + Microsoft::WRL::ComPtr bestDevice; + std::wstring bestId; + std::wstring bestName; + int bestScore = 0; + for (UINT i = 0; i < count; ++i) { + Microsoft::WRL::ComPtr candidate; + hr = devices->Item(i, &candidate); + if (FAILED(hr) || !candidate) { + continue; + } + + LPWSTR rawId = nullptr; + std::wstring candidateId; + if (SUCCEEDED(candidate->GetId(&rawId)) && rawId) { + candidateId = rawId; + CoTaskMemFree(rawId); + } + + const std::wstring candidateName = getDeviceFriendlyName(candidate.Get()); + const int score = scoreDeviceName(candidateName, candidateId, deviceName); + std::wcerr << L"Native microphone candidate: " << candidateName << L" score=" << score << std::endl; + if (score > bestScore) { + bestScore = score; + bestDevice = candidate; + bestId = candidateId; + bestName = candidateName; + } + } + + if (!bestDevice || bestScore <= 0) { + return false; + } + + device_ = bestDevice; + std::wcerr << L"Selected native microphone endpoint: " << bestName << L" id=" << bestId << std::endl; + return true; +} + +bool WasapiLoopbackCapture::resolveInputFormat(WAVEFORMATEX* mixFormat) { + const GUID subtype = audioSubtypeFromFormat(mixFormat); + if (subtype == GUID_NULL) { + return false; + } + + inputFormat_.subtype = subtype; + inputFormat_.sampleRate = mixFormat->nSamplesPerSec; + inputFormat_.channels = mixFormat->nChannels; + inputFormat_.bitsPerSample = mixFormat->wBitsPerSample; + inputFormat_.blockAlign = mixFormat->nBlockAlign; + inputFormat_.avgBytesPerSec = mixFormat->nAvgBytesPerSec; + return inputFormat_.sampleRate > 0 && inputFormat_.channels > 0 && inputFormat_.blockAlign > 0; +} + +bool WasapiLoopbackCapture::start(AudioCallback callback) { + if (!audioClient_ || !captureClient_ || !callback) { + return false; + } + + callback_ = std::move(callback); + stopRequested_ = false; + writtenFrames_ = 0; + lastDevicePositionEnd_ = 0; + hasLastDevicePosition_ = false; + + HRESULT hr = audioClient_->Start(); + if (!succeeded(hr, "IAudioClient::Start")) { + return false; + } + + thread_ = std::thread([this] { + captureLoop(); + }); + return true; +} + +void WasapiLoopbackCapture::stop() { + stopRequested_ = true; + if (thread_.joinable()) { + thread_.join(); + } + if (audioClient_) { + audioClient_->Stop(); + } +} + +const AudioInputFormat& WasapiLoopbackCapture::inputFormat() const { + return inputFormat_; +} + +const std::wstring& WasapiLoopbackCapture::selectedDeviceName() const { + return selectedDeviceName_; +} + +void WasapiLoopbackCapture::captureLoop() { + auto emitSilenceFrames = [&](uint64_t frames, int64_t timestampHns) { + constexpr uint64_t MaxSilenceChunkFrames = 4800; + uint64_t remainingFrames = frames; + int64_t currentTimestampHns = timestampHns; + while (remainingFrames > 0 && !stopRequested_) { + const uint64_t chunkFrames = std::min(remainingFrames, MaxSilenceChunkFrames); + const DWORD chunkBytes = static_cast(chunkFrames * inputFormat_.blockAlign); + const int64_t chunkDurationHns = + static_cast((chunkFrames * HnsPerSecond) / inputFormat_.sampleRate); + silenceBuffer_.assign(chunkBytes, 0); + callback_(silenceBuffer_.data(), chunkBytes, currentTimestampHns, chunkDurationHns); + remainingFrames -= chunkFrames; + currentTimestampHns += chunkDurationHns; + } + }; + + while (!stopRequested_) { + UINT32 packetFrames = 0; + HRESULT hr = captureClient_->GetNextPacketSize(&packetFrames); + if (FAILED(hr)) { + std::cerr << "ERROR: IAudioCaptureClient::GetNextPacketSize failed (hr=0x" << std::hex + << hr << std::dec << ")" << std::endl; + break; + } + + while (packetFrames > 0 && !stopRequested_) { + BYTE* data = nullptr; + UINT32 framesAvailable = 0; + DWORD flags = 0; + UINT64 devicePosition = 0; + UINT64 qpcPosition = 0; + + hr = captureClient_->GetBuffer(&data, &framesAvailable, &flags, &devicePosition, &qpcPosition); + if (FAILED(hr)) { + std::cerr << "ERROR: IAudioCaptureClient::GetBuffer failed (hr=0x" << std::hex + << hr << std::dec << ")" << std::endl; + break; + } + + (void)qpcPosition; + if (hasLastDevicePosition_ && devicePosition > lastDevicePositionEnd_) { + const uint64_t gapFrames = devicePosition - lastDevicePositionEnd_; + if ((flags & AUDCLNT_BUFFERFLAGS_DATA_DISCONTINUITY) != 0 || gapFrames > framesAvailable) { + const int64_t gapTimestampHns = + static_cast((lastDevicePositionEnd_ * HnsPerSecond) / inputFormat_.sampleRate); + emitSilenceFrames(gapFrames, gapTimestampHns); + } + } + + const DWORD byteCount = framesAvailable * inputFormat_.blockAlign; + const int64_t timestampHns = + static_cast((devicePosition * HnsPerSecond) / inputFormat_.sampleRate); + const int64_t durationHns = + static_cast((static_cast(framesAvailable) * HnsPerSecond) / + inputFormat_.sampleRate); + + if (byteCount > 0) { + if ((flags & AUDCLNT_BUFFERFLAGS_SILENT) != 0 || !data) { + silenceBuffer_.assign(byteCount, 0); + callback_(silenceBuffer_.data(), byteCount, timestampHns, durationHns); + } else { + callback_(data, byteCount, timestampHns, durationHns); + } + } + + writtenFrames_ += framesAvailable; + lastDevicePositionEnd_ = devicePosition + framesAvailable; + hasLastDevicePosition_ = true; + captureClient_->ReleaseBuffer(framesAvailable); + + hr = captureClient_->GetNextPacketSize(&packetFrames); + if (FAILED(hr)) { + std::cerr << "ERROR: IAudioCaptureClient::GetNextPacketSize failed (hr=0x" + << std::hex << hr << std::dec << ")" << std::endl; + packetFrames = 0; + break; + } + } + + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + +} diff --git a/electron/native/wgc-capture/src/wasapi_loopback_capture.h b/electron/native/wgc-capture/src/wasapi_loopback_capture.h new file mode 100644 index 0000000..5c5f2b7 --- /dev/null +++ b/electron/native/wgc-capture/src/wasapi_loopback_capture.h @@ -0,0 +1,60 @@ +#pragma once + +#include "mf_encoder.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +enum class WasapiCaptureEndpoint { + SystemLoopback, + Microphone, +}; + +class WasapiLoopbackCapture { +public: + using AudioCallback = std::function; + + WasapiLoopbackCapture() = default; + ~WasapiLoopbackCapture(); + + WasapiLoopbackCapture(const WasapiLoopbackCapture&) = delete; + WasapiLoopbackCapture& operator=(const WasapiLoopbackCapture&) = delete; + + bool initializeSystemLoopback(); + bool initializeMicrophone(const std::wstring& deviceId, const std::wstring& deviceName); + bool start(AudioCallback callback); + void stop(); + + const AudioInputFormat& inputFormat() const; + const std::wstring& selectedDeviceName() const; + +private: + bool initialize(WasapiCaptureEndpoint endpoint, const std::wstring& deviceId, const std::wstring& deviceName); + bool resolveMicrophoneByName(const std::wstring& deviceName); + void captureLoop(); + bool resolveInputFormat(WAVEFORMATEX* mixFormat); + + Microsoft::WRL::ComPtr deviceEnumerator_; + Microsoft::WRL::ComPtr device_; + Microsoft::WRL::ComPtr audioClient_; + Microsoft::WRL::ComPtr captureClient_; + WAVEFORMATEX* mixFormat_ = nullptr; + AudioInputFormat inputFormat_{}; + std::wstring selectedDeviceName_; + AudioCallback callback_; + std::thread thread_; + std::atomic stopRequested_ = false; + std::vector silenceBuffer_; + uint64_t writtenFrames_ = 0; + uint64_t lastDevicePositionEnd_ = 0; + bool hasLastDevicePosition_ = false; +}; diff --git a/electron/native/wgc-capture/src/webcam_capture.cpp b/electron/native/wgc-capture/src/webcam_capture.cpp new file mode 100644 index 0000000..783b854 --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_capture.cpp @@ -0,0 +1,419 @@ +#include "webcam_capture.h" + +#include +#include +#include + +#include +#include +#include +#include + +namespace { + +bool succeeded(HRESULT hr, const char* label) { + if (SUCCEEDED(hr)) { + return true; + } + + std::cerr << "ERROR: " << label << " failed (hr=0x" << std::hex << hr << std::dec << ")" + << std::endl; + return false; +} + +std::wstring readAllocatedString(IMFActivate* activate, REFGUID key) { + WCHAR* value = nullptr; + UINT32 length = 0; + if (FAILED(activate->GetAllocatedString(key, &value, &length)) || !value) { + return {}; + } + + std::wstring result(value, value + length); + CoTaskMemFree(value); + return result; +} + +bool containsInsensitive(const std::wstring& haystack, const std::wstring& needle) { + if (haystack.empty() || needle.empty()) { + return false; + } + + std::wstring lowerHaystack = haystack; + std::wstring lowerNeedle = needle; + std::transform(lowerHaystack.begin(), lowerHaystack.end(), lowerHaystack.begin(), ::towlower); + std::transform(lowerNeedle.begin(), lowerNeedle.end(), lowerNeedle.begin(), ::towlower); + return lowerHaystack.find(lowerNeedle) != std::wstring::npos || + lowerNeedle.find(lowerHaystack) != std::wstring::npos; +} + +std::wstring normalizeDeviceName(const std::wstring& value) { + std::wstring normalized; + normalized.reserve(value.size()); + bool lastWasSpace = true; + for (const wchar_t ch : value) { + if (std::iswalnum(ch)) { + normalized.push_back(static_cast(std::towlower(ch))); + lastWasSpace = false; + continue; + } + if (!lastWasSpace) { + normalized.push_back(L' '); + lastWasSpace = true; + } + } + while (!normalized.empty() && normalized.back() == L' ') { + normalized.pop_back(); + } + return normalized; +} + +std::vector splitWords(const std::wstring& value) { + std::vector words; + size_t start = 0; + while (start < value.size()) { + const size_t end = value.find(L' ', start); + const auto word = value.substr(start, end == std::wstring::npos ? std::wstring::npos : end - start); + if (word.size() > 1 && word != L"camera" && word != L"webcam" && word != L"video" && word != L"input") { + words.push_back(word); + } + if (end == std::wstring::npos) { + break; + } + start = end + 1; + } + return words; +} + +int deviceMatchScore( + const std::wstring& candidateName, + const std::wstring& candidateLink, + const std::wstring& requestedName, + const std::wstring& requestedId) { + int score = 0; + const auto normalizedName = normalizeDeviceName(candidateName); + const auto normalizedLink = normalizeDeviceName(candidateLink); + const auto normalizedRequestedName = normalizeDeviceName(requestedName); + const auto normalizedRequestedId = normalizeDeviceName(requestedId); + + if (!normalizedRequestedName.empty()) { + if (normalizedName == normalizedRequestedName) { + score = std::max(score, 1000); + } + if (containsInsensitive(normalizedName, normalizedRequestedName)) { + score = std::max(score, 900); + } + if (containsInsensitive(normalizedLink, normalizedRequestedName)) { + score = std::max(score, 800); + } + + int wordScore = 0; + for (const auto& word : splitWords(normalizedRequestedName)) { + if (normalizedName.find(word) != std::wstring::npos) { + wordScore += 100; + } else if (normalizedLink.find(word) != std::wstring::npos) { + wordScore += 50; + } + } + score = std::max(score, wordScore); + } + + if (!normalizedRequestedId.empty()) { + if (containsInsensitive(normalizedLink, normalizedRequestedId)) { + score = std::max(score, 700); + } + if (containsInsensitive(normalizedName, normalizedRequestedId)) { + score = std::max(score, 600); + } + } + + return score; +} + +} // namespace + +WebcamCapture::~WebcamCapture() { + stop(); +} + +bool WebcamCapture::initialize( + const std::wstring& deviceId, + const std::wstring& deviceName, + const std::wstring& directShowClsid, + int requestedWidth, + int requestedHeight, + int requestedFps) { + fps_ = std::clamp(requestedFps > 0 ? requestedFps : 30, 1, 60); + usingDirectShow_ = false; + selectedMatchScore_ = 0; + if (!succeeded(MFStartup(MF_VERSION), "MFStartup(webcam)")) { + if (directShowCapture_.initialize(deviceId, deviceName, directShowClsid, requestedWidth, requestedHeight, fps_)) { + usingDirectShow_ = true; + return true; + } + return false; + } + mfStarted_ = true; + if (!selectDevice(deviceId, deviceName)) { + if (mfStarted_) { + MFShutdown(); + mfStarted_ = false; + } + if (directShowCapture_.initialize(deviceId, deviceName, directShowClsid, requestedWidth, requestedHeight, fps_)) { + usingDirectShow_ = true; + return true; + } + return false; + } + + if ((!deviceId.empty() || !deviceName.empty()) && selectedMatchScore_ <= 0) { + if (mediaSource_) { + mediaSource_->Shutdown(); + } + sourceReader_.Reset(); + mediaSource_.Reset(); + if (mfStarted_) { + MFShutdown(); + mfStarted_ = false; + } + if (directShowCapture_.initialize(deviceId, deviceName, directShowClsid, requestedWidth, requestedHeight, fps_)) { + usingDirectShow_ = true; + return true; + } + std::cerr << "ERROR: Requested webcam device was not found by native Windows webcam providers" + << std::endl; + return false; + } + + return configureReader(requestedWidth, requestedHeight, fps_); +} + +bool WebcamCapture::selectDevice(const std::wstring& deviceId, const std::wstring& deviceName) { + Microsoft::WRL::ComPtr attributes; + if (!succeeded(MFCreateAttributes(&attributes, 1), "MFCreateAttributes(webcam enumeration)")) { + return false; + } + if (!succeeded(attributes->SetGUID( + MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE, + MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID), + "SetGUID(webcam source type)")) { + return false; + } + + IMFActivate** devices = nullptr; + UINT32 deviceCount = 0; + HRESULT hr = MFEnumDeviceSources(attributes.Get(), &devices, &deviceCount); + if (!succeeded(hr, "MFEnumDeviceSources") || deviceCount == 0) { + if (devices) { + CoTaskMemFree(devices); + } + std::cerr << "ERROR: No native Windows webcam devices were found" << std::endl; + return false; + } + + UINT32 selectedIndex = 0; + int bestScore = 0; + for (UINT32 index = 0; index < deviceCount; index += 1) { + const std::wstring name = readAllocatedString(devices[index], MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME); + const std::wstring symbolicLink = readAllocatedString(devices[index], MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_SYMBOLIC_LINK); + const int score = deviceMatchScore(name, symbolicLink, deviceName, deviceId); + std::wcerr << L"INFO: Native webcam candidate [" << index << L"] name=\"" << name << L"\" score=" << score << std::endl; + if (score > bestScore) { + selectedIndex = index; + bestScore = score; + } + } + + if ((!deviceId.empty() || !deviceName.empty()) && bestScore <= 0) { + std::cerr << "WARNING: Requested webcam device was not found by Media Foundation; trying DirectShow" + << std::endl; + } + + selectedMatchScore_ = bestScore; + selectedDeviceName_ = readAllocatedString(devices[selectedIndex], MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME); + hr = devices[selectedIndex]->ActivateObject(IID_PPV_ARGS(&mediaSource_)); + + for (UINT32 index = 0; index < deviceCount; index += 1) { + devices[index]->Release(); + } + CoTaskMemFree(devices); + + return succeeded(hr, "ActivateObject(webcam)"); +} + +bool WebcamCapture::configureReader(int requestedWidth, int requestedHeight, int requestedFps) { + Microsoft::WRL::ComPtr attributes; + if (!succeeded(MFCreateAttributes(&attributes, 2), "MFCreateAttributes(webcam reader)")) { + return false; + } + attributes->SetUINT32(MF_SOURCE_READER_ENABLE_VIDEO_PROCESSING, TRUE); + attributes->SetUINT32(MF_READWRITE_DISABLE_CONVERTERS, FALSE); + + if (!succeeded(MFCreateSourceReaderFromMediaSource(mediaSource_.Get(), attributes.Get(), &sourceReader_), + "MFCreateSourceReaderFromMediaSource(webcam)")) { + return false; + } + + Microsoft::WRL::ComPtr mediaType; + if (!succeeded(MFCreateMediaType(&mediaType), "MFCreateMediaType(webcam output)")) { + return false; + } + mediaType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + mediaType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_RGB32); + if (requestedWidth > 0 && requestedHeight > 0) { + MFSetAttributeSize(mediaType.Get(), MF_MT_FRAME_SIZE, static_cast(requestedWidth), static_cast(requestedHeight)); + } + MFSetAttributeRatio(mediaType.Get(), MF_MT_FRAME_RATE, static_cast(std::max(1, requestedFps)), 1); + + if (!succeeded(sourceReader_->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, mediaType.Get()), + "SetCurrentMediaType(webcam RGB32)")) { + return false; + } + sourceReader_->SetStreamSelection(MF_SOURCE_READER_ALL_STREAMS, FALSE); + sourceReader_->SetStreamSelection(MF_SOURCE_READER_FIRST_VIDEO_STREAM, TRUE); + + Microsoft::WRL::ComPtr currentType; + if (!succeeded(sourceReader_->GetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, ¤tType), + "GetCurrentMediaType(webcam)")) { + return false; + } + + UINT32 width = 0; + UINT32 height = 0; + if (FAILED(MFGetAttributeSize(currentType.Get(), MF_MT_FRAME_SIZE, &width, &height)) || width == 0 || height == 0) { + width = static_cast(requestedWidth > 0 ? requestedWidth : 1280); + height = static_cast(requestedHeight > 0 ? requestedHeight : 720); + } + width_ = static_cast(width); + height_ = static_cast(height); + return true; +} + +bool WebcamCapture::start() { + if (usingDirectShow_) { + return directShowCapture_.start(); + } + if (!sourceReader_ || thread_.joinable()) { + return false; + } + + stopRequested_ = false; + thread_ = std::thread(&WebcamCapture::captureLoop, this); + return true; +} + +void WebcamCapture::stop() { + directShowCapture_.stop(); + stopRequested_ = true; + if (thread_.joinable()) { + thread_.join(); + } + if (mediaSource_) { + mediaSource_->Shutdown(); + } + sourceReader_.Reset(); + mediaSource_.Reset(); + if (mfStarted_) { + MFShutdown(); + mfStarted_ = false; + } +} + +void WebcamCapture::captureLoop() { + CoInitializeEx(nullptr, COINIT_MULTITHREADED); + + while (!stopRequested_) { + DWORD streamIndex = 0; + DWORD flags = 0; + LONGLONG timestamp = 0; + Microsoft::WRL::ComPtr sample; + HRESULT hr = sourceReader_->ReadSample( + MF_SOURCE_READER_FIRST_VIDEO_STREAM, + 0, + &streamIndex, + &flags, + ×tamp, + &sample); + (void)streamIndex; + (void)timestamp; + + if (FAILED(hr)) { + std::cerr << "WARNING: Failed to read webcam sample (hr=0x" << std::hex << hr << std::dec << ")" + << std::endl; + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + continue; + } + if ((flags & MF_SOURCE_READERF_ENDOFSTREAM) != 0) { + break; + } + if (!sample) { + continue; + } + + Microsoft::WRL::ComPtr buffer; + if (FAILED(sample->ConvertToContiguousBuffer(&buffer)) || !buffer) { + continue; + } + + BYTE* data = nullptr; + DWORD maxLength = 0; + DWORD currentLength = 0; + if (FAILED(buffer->Lock(&data, &maxLength, ¤tLength)) || !data) { + continue; + } + + const DWORD expectedLength = static_cast(std::max(0, width_) * std::max(0, height_) * 4); + if (currentLength >= expectedLength && expectedLength > 0) { + std::scoped_lock lock(frameMutex_); + latestFrame_.assign(data, data + expectedLength); + latestFrameSequence_ += 1; + } + + buffer->Unlock(); + } + + CoUninitialize(); +} + +bool WebcamCapture::copyLatestFrame(WebcamFrameSnapshot& destination) { + if (usingDirectShow_) { + return directShowCapture_.copyLatestFrame(destination); + } + std::scoped_lock lock(frameMutex_); + if (latestFrame_.empty() || width_ <= 0 || height_ <= 0) { + return false; + } + + destination.data = latestFrame_; + destination.width = width_; + destination.height = height_; + destination.sequence = latestFrameSequence_; + return true; +} + +int WebcamCapture::width() const { + if (usingDirectShow_) { + return directShowCapture_.width(); + } + return width_; +} + +int WebcamCapture::height() const { + if (usingDirectShow_) { + return directShowCapture_.height(); + } + return height_; +} + +int WebcamCapture::fps() const { + if (usingDirectShow_) { + return directShowCapture_.fps(); + } + return fps_; +} + +const std::wstring& WebcamCapture::selectedDeviceName() const { + if (usingDirectShow_) { + return directShowCapture_.selectedDeviceName(); + } + return selectedDeviceName_; +} diff --git a/electron/native/wgc-capture/src/webcam_capture.h b/electron/native/wgc-capture/src/webcam_capture.h new file mode 100644 index 0000000..5b61aa6 --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_capture.h @@ -0,0 +1,61 @@ +#pragma once + +#include "dshow_webcam_capture.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +class WebcamCapture { +public: + WebcamCapture() = default; + ~WebcamCapture(); + + WebcamCapture(const WebcamCapture&) = delete; + WebcamCapture& operator=(const WebcamCapture&) = delete; + + bool initialize( + const std::wstring& deviceId, + const std::wstring& deviceName, + const std::wstring& directShowClsid, + int requestedWidth, + int requestedHeight, + int requestedFps); + bool start(); + void stop(); + bool copyLatestFrame(WebcamFrameSnapshot& destination); + + int width() const; + int height() const; + int fps() const; + const std::wstring& selectedDeviceName() const; + +private: + bool selectDevice(const std::wstring& deviceId, const std::wstring& deviceName); + bool configureReader(int requestedWidth, int requestedHeight, int requestedFps); + void captureLoop(); + + Microsoft::WRL::ComPtr mediaSource_; + Microsoft::WRL::ComPtr sourceReader_; + DirectShowWebcamCapture directShowCapture_; + std::thread thread_; + std::atomic stopRequested_ = false; + std::mutex frameMutex_; + std::vector latestFrame_; + uint64_t latestFrameSequence_ = 0; + int width_ = 0; + int height_ = 0; + int fps_ = 30; + bool mfStarted_ = false; + bool usingDirectShow_ = false; + int selectedMatchScore_ = 0; + std::wstring selectedDeviceName_; +}; diff --git a/electron/native/wgc-capture/src/wgc_session.cpp b/electron/native/wgc-capture/src/wgc_session.cpp new file mode 100644 index 0000000..e20096c --- /dev/null +++ b/electron/native/wgc-capture/src/wgc_session.cpp @@ -0,0 +1,315 @@ +#include "wgc_session.h" + +#include +#include +#include +#include + +#include + +namespace wf = winrt::Windows::Foundation; +namespace wgcap = winrt::Windows::Graphics::Capture; +namespace wgdx = winrt::Windows::Graphics::DirectX; +namespace wgd3d = winrt::Windows::Graphics::DirectX::Direct3D11; + +extern "C" HRESULT __stdcall CreateDirect3D11DeviceFromDXGIDevice( + ::IDXGIDevice* dxgiDevice, + ::IInspectable** graphicsDevice); + +namespace { + +bool succeeded(HRESULT hr, const char* label) { + if (SUCCEEDED(hr)) { + return true; + } + + std::cerr << "ERROR: " << label << " failed (hr=0x" << std::hex << hr << std::dec << ")" + << std::endl; + return false; +} + +int64_t timeSpanToHns(wf::TimeSpan const& value) { + return value.count(); +} + +} // namespace + +WgcSession::~WgcSession() { + stop(); +} + +bool WgcSession::createD3DDevice() { + UINT flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT; +#if defined(_DEBUG) + flags |= D3D11_CREATE_DEVICE_DEBUG; +#endif + + D3D_FEATURE_LEVEL featureLevels[] = { + D3D_FEATURE_LEVEL_11_1, + D3D_FEATURE_LEVEL_11_0, + D3D_FEATURE_LEVEL_10_1, + D3D_FEATURE_LEVEL_10_0, + }; + D3D_FEATURE_LEVEL featureLevel{}; + + HRESULT hr = D3D11CreateDevice( + nullptr, + D3D_DRIVER_TYPE_HARDWARE, + nullptr, + flags, + featureLevels, + ARRAYSIZE(featureLevels), + D3D11_SDK_VERSION, + &d3dDevice_, + &featureLevel, + &d3dContext_); + +#if defined(_DEBUG) + if (FAILED(hr)) { + flags &= ~D3D11_CREATE_DEVICE_DEBUG; + hr = D3D11CreateDevice( + nullptr, + D3D_DRIVER_TYPE_HARDWARE, + nullptr, + flags, + featureLevels, + ARRAYSIZE(featureLevels), + D3D11_SDK_VERSION, + &d3dDevice_, + &featureLevel, + &d3dContext_); + } +#endif + + if (!succeeded(hr, "D3D11CreateDevice")) { + return false; + } + + Microsoft::WRL::ComPtr dxgiDevice; + if (!succeeded(d3dDevice_.As(&dxgiDevice), "Query IDXGIDevice")) { + return false; + } + + winrt::com_ptr<::IInspectable> inspectableDevice; + if (!succeeded(CreateDirect3D11DeviceFromDXGIDevice(dxgiDevice.Get(), inspectableDevice.put()), + "CreateDirect3D11DeviceFromDXGIDevice")) { + return false; + } + + winrtDevice_ = inspectableDevice.as(); + return true; +} + +bool WgcSession::createCaptureItem(HMONITOR monitor) { + auto factory = winrt::get_activation_factory(); + auto interop = factory.as(); + + wgcap::GraphicsCaptureItem item{nullptr}; + HRESULT hr = interop->CreateForMonitor( + monitor, + winrt::guid_of(), + reinterpret_cast(winrt::put_abi(item))); + if (!succeeded(hr, "CreateForMonitor")) { + return false; + } + + item_ = item; + const auto size = item_.Size(); + width_ = static_cast(size.Width); + height_ = static_cast(size.Height); + return width_ > 0 && height_ > 0; +} + +bool WgcSession::createCaptureItem(HWND window) { + auto factory = winrt::get_activation_factory(); + auto interop = factory.as(); + + wgcap::GraphicsCaptureItem item{nullptr}; + HRESULT hr = interop->CreateForWindow( + window, + winrt::guid_of(), + reinterpret_cast(winrt::put_abi(item))); + if (!succeeded(hr, "CreateForWindow")) { + return false; + } + + item_ = item; + const auto size = item_.Size(); + width_ = static_cast(size.Width); + height_ = static_cast(size.Height); + return width_ > 0 && height_ > 0; +} + +bool WgcSession::applySessionOptions(bool captureCursor) { + captureCursor_ = captureCursor; + + try { + auto session2 = session_.try_as(); + if (!session2) { + if (!captureCursor) { + std::cerr << "ERROR: WGC cursor suppression is not supported by this Windows runtime" + << std::endl; + return false; + } + } else { + session2.IsCursorCaptureEnabled(captureCursor); + const bool appliedCursorCapture = session2.IsCursorCaptureEnabled(); + std::cout << "{\"event\":\"cursor-capture\",\"schemaVersion\":2,\"requested\":" + << (captureCursor ? "true" : "false") + << ",\"applied\":" << (appliedCursorCapture ? "true" : "false") << "}" + << std::endl; + + if (appliedCursorCapture != captureCursor) { + std::cerr << "ERROR: WGC cursor capture setting did not apply" << std::endl; + return false; + } + } + } catch (winrt::hresult_error const& error) { + std::cerr << "ERROR: Failed to configure WGC cursor capture (hr=0x" << std::hex + << static_cast(error.code()) << std::dec << ")" << std::endl; + if (!captureCursor) { + return false; + } + } catch (...) { + std::cerr << "ERROR: Failed to configure WGC cursor capture" << std::endl; + if (!captureCursor) { + return false; + } + } + + try { + session_.IsBorderRequired(false); + } catch (...) { + // IsBorderRequired is Windows 11-only. Ignore it on older builds. + } + + return true; +} + +bool WgcSession::initialize(HMONITOR monitor, int fps, bool captureCursor) { + fps_ = fps > 0 ? fps : 60; + if (!createD3DDevice()) { + return false; + } + if (!createCaptureItem(monitor)) { + return false; + } + + framePool_ = wgcap::Direct3D11CaptureFramePool::CreateFreeThreaded( + winrtDevice_, + wgdx::DirectXPixelFormat::B8G8R8A8UIntNormalized, + 2, + item_.Size()); + session_ = framePool_.CreateCaptureSession(item_); + + if (!applySessionOptions(captureCursor)) { + return false; + } + + frameArrivedToken_ = framePool_.FrameArrived({this, &WgcSession::onFrameArrived}); + return true; +} + +bool WgcSession::initialize(HWND window, int fps, bool captureCursor) { + fps_ = fps > 0 ? fps : 60; + if (!createD3DDevice()) { + return false; + } + if (!createCaptureItem(window)) { + return false; + } + + framePool_ = wgcap::Direct3D11CaptureFramePool::CreateFreeThreaded( + winrtDevice_, + wgdx::DirectXPixelFormat::B8G8R8A8UIntNormalized, + 2, + item_.Size()); + session_ = framePool_.CreateCaptureSession(item_); + + if (!applySessionOptions(captureCursor)) { + return false; + } + + frameArrivedToken_ = framePool_.FrameArrived({this, &WgcSession::onFrameArrived}); + return true; +} + +void WgcSession::setFrameCallback(FrameCallback callback) { + std::scoped_lock lock(callbackMutex_); + frameCallback_ = std::move(callback); +} + +bool WgcSession::start() { + if (!session_) { + return false; + } + if (!applySessionOptions(captureCursor_)) { + return false; + } + session_.StartCapture(); + started_ = true; + return true; +} + +void WgcSession::stop() { + if (framePool_) { + framePool_.FrameArrived(frameArrivedToken_); + } + if (session_) { + session_.Close(); + session_ = nullptr; + } + if (framePool_) { + framePool_.Close(); + framePool_ = nullptr; + } + item_ = nullptr; + winrtDevice_ = nullptr; + d3dContext_.Reset(); + d3dDevice_.Reset(); + started_ = false; +} + +void WgcSession::onFrameArrived( + wgcap::Direct3D11CaptureFramePool const& sender, + wf::IInspectable const&) { + auto frame = sender.TryGetNextFrame(); + if (!frame) { + return; + } + + auto surface = frame.Surface(); + auto access = surface.as<::Windows::Graphics::DirectX::Direct3D11::IDirect3DDxgiInterfaceAccess>(); + Microsoft::WRL::ComPtr texture; + HRESULT hr = access->GetInterface(__uuidof(ID3D11Texture2D), reinterpret_cast(texture.GetAddressOf())); + if (FAILED(hr) || !texture) { + return; + } + + FrameCallback callback; + { + std::scoped_lock lock(callbackMutex_); + callback = frameCallback_; + } + + if (callback) { + callback(texture.Get(), timeSpanToHns(frame.SystemRelativeTime())); + } + frame.Close(); +} + +int WgcSession::captureWidth() const { + return width_; +} + +int WgcSession::captureHeight() const { + return height_; +} + +ID3D11Device* WgcSession::device() const { + return d3dDevice_.Get(); +} + +ID3D11DeviceContext* WgcSession::context() const { + return d3dContext_.Get(); +} diff --git a/electron/native/wgc-capture/src/wgc_session.h b/electron/native/wgc-capture/src/wgc_session.h new file mode 100644 index 0000000..43de21a --- /dev/null +++ b/electron/native/wgc-capture/src/wgc_session.h @@ -0,0 +1,59 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +class WgcSession { +public: + using FrameCallback = std::function; + + WgcSession() = default; + ~WgcSession(); + + WgcSession(const WgcSession&) = delete; + WgcSession& operator=(const WgcSession&) = delete; + + bool initialize(HMONITOR monitor, int fps, bool captureCursor); + bool initialize(HWND window, int fps, bool captureCursor); + void setFrameCallback(FrameCallback callback); + bool start(); + void stop(); + + int captureWidth() const; + int captureHeight() const; + ID3D11Device* device() const; + ID3D11DeviceContext* context() const; + +private: + bool createD3DDevice(); + bool createCaptureItem(HMONITOR monitor); + bool createCaptureItem(HWND window); + bool applySessionOptions(bool captureCursor); + void onFrameArrived( + winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool const& sender, + winrt::Windows::Foundation::IInspectable const&); + + Microsoft::WRL::ComPtr d3dDevice_; + Microsoft::WRL::ComPtr d3dContext_; + winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice winrtDevice_{nullptr}; + winrt::Windows::Graphics::Capture::GraphicsCaptureItem item_{nullptr}; + winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool framePool_{nullptr}; + winrt::Windows::Graphics::Capture::GraphicsCaptureSession session_{nullptr}; + winrt::event_token frameArrivedToken_{}; + FrameCallback frameCallback_; + std::mutex callbackMutex_; + int width_ = 0; + int height_ = 0; + int fps_ = 60; + bool captureCursor_ = false; + bool started_ = false; +}; diff --git a/electron/preload.ts b/electron/preload.ts index 5980b4c..933ce9d 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -1,5 +1,8 @@ import { contextBridge, ipcRenderer } from "electron"; +import type { NativeMacRecordingRequest } from "../src/lib/nativeMacRecording"; +import type { NativeWindowsRecordingRequest } from "../src/lib/nativeWindowsRecording"; import type { RecordingSession, StoreRecordedSessionInput } from "../src/lib/recordingSession"; +import { NATIVE_BRIDGE_CHANNEL, type NativeBridgeRequest } from "../src/native/contracts"; // Asset base URL is passed from the main process via webPreferences.additionalArguments // (see windows.ts). Sandboxed preloads cannot import node:path / node:url, so we @@ -10,6 +13,9 @@ const assetBaseUrl = assetBaseUrlArg ? assetBaseUrlArg.slice(ASSET_BASE_URL_ARG_ contextBridge.exposeInMainWorld("electronAPI", { assetBaseUrl, + invokeNativeBridge: (request: NativeBridgeRequest) => { + return ipcRenderer.invoke(NATIVE_BRIDGE_CHANNEL, request) as Promise; + }, hudOverlayHide: () => { ipcRenderer.send("hud-overlay-hide"); }, @@ -19,6 +25,9 @@ contextBridge.exposeInMainWorld("electronAPI", { setHudOverlayIgnoreMouseEvents: (ignore: boolean) => { ipcRenderer.send("hud-overlay-ignore-mouse-events", ignore); }, + moveHudOverlayBy: (deltaX: number, deltaY: number) => { + ipcRenderer.send("hud-overlay-move-by", deltaX, deltaY); + }, getSources: async (opts: Electron.SourcesOptions) => { return await ipcRenderer.invoke("get-sources", opts); }, @@ -43,10 +52,12 @@ contextBridge.exposeInMainWorld("electronAPI", { requestCameraAccess: () => { return ipcRenderer.invoke("request-camera-access"); }, - requestAccessibilityAccess: () => { - return ipcRenderer.invoke("request-accessibility-access"); + requestScreenAccess: () => { + return ipcRenderer.invoke("request-screen-access"); + }, + requestNativeMacCursorAccess: () => { + return ipcRenderer.invoke("request-native-mac-cursor-access"); }, - storeRecordedVideo: (videoData: ArrayBuffer, fileName: string) => { return ipcRenderer.invoke("store-recorded-video", videoData, fileName); }, @@ -57,8 +68,50 @@ contextBridge.exposeInMainWorld("electronAPI", { getRecordedVideoPath: () => { return ipcRenderer.invoke("get-recorded-video-path"); }, - setRecordingState: (recording: boolean, recordingId?: number) => { - return ipcRenderer.invoke("set-recording-state", recording, recordingId); + setRecordingState: ( + recording: boolean, + recordingId?: number, + cursorCaptureMode?: import("../src/lib/recordingSession").CursorCaptureMode, + ) => { + return ipcRenderer.invoke("set-recording-state", recording, recordingId, cursorCaptureMode); + }, + isNativeWindowsCaptureAvailable: () => { + return ipcRenderer.invoke("is-native-windows-capture-available"); + }, + isNativeMacCaptureAvailable: () => { + return ipcRenderer.invoke("is-native-mac-capture-available"); + }, + startNativeWindowsRecording: (request: NativeWindowsRecordingRequest) => { + return ipcRenderer.invoke("start-native-windows-recording", request); + }, + stopNativeWindowsRecording: (discard?: boolean) => { + return ipcRenderer.invoke("stop-native-windows-recording", discard); + }, + pauseNativeWindowsRecording: () => { + return ipcRenderer.invoke("pause-native-windows-recording"); + }, + resumeNativeWindowsRecording: () => { + return ipcRenderer.invoke("resume-native-windows-recording"); + }, + startNativeMacRecording: (request: NativeMacRecordingRequest) => { + return ipcRenderer.invoke("start-native-mac-recording", request); + }, + pauseNativeMacRecording: () => { + return ipcRenderer.invoke("pause-native-mac-recording"); + }, + resumeNativeMacRecording: () => { + return ipcRenderer.invoke("resume-native-mac-recording"); + }, + stopNativeMacRecording: (discard?: boolean) => { + return ipcRenderer.invoke("stop-native-mac-recording", discard); + }, + attachNativeMacWebcamRecording: (payload: { + screenVideoPath: string; + recordingId: number; + webcam: { fileName: string; videoData: ArrayBuffer }; + cursorCaptureMode?: import("../src/lib/recordingSession").CursorCaptureMode; + }) => { + return ipcRenderer.invoke("attach-native-mac-webcam-recording", payload); }, getCursorTelemetry: (videoPath?: string) => { return ipcRenderer.invoke("get-cursor-telemetry", videoPath); @@ -98,6 +151,9 @@ contextBridge.exposeInMainWorld("electronAPI", { readBinaryFile: (filePath: string) => { return ipcRenderer.invoke("read-binary-file", filePath); }, + preparePreviewAudioTrack: (filePath: string) => { + return ipcRenderer.invoke("prepare-preview-audio-track", filePath); + }, clearCurrentVideoPath: () => { return ipcRenderer.invoke("clear-current-video-path"); }, diff --git a/electron/windows.ts b/electron/windows.ts index 4d4e752..3a7350e 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -30,6 +30,20 @@ ipcMain.on("hud-overlay-ignore-mouse-events", (_event, ignore: boolean) => { } }); +ipcMain.on("hud-overlay-move-by", (_event, deltaX: number, deltaY: number) => { + if ( + !hudOverlayWindow || + hudOverlayWindow.isDestroyed() || + !Number.isFinite(deltaX) || + !Number.isFinite(deltaY) + ) { + return; + } + + const [x, y] = hudOverlayWindow.getPosition(); + hudOverlayWindow.setPosition(Math.round(x + deltaX), Math.round(y + deltaY), false); +}); + /** * Creates the always-on-top HUD overlay window centred at the bottom of the * primary display. The window is frameless, transparent, and follows the user diff --git a/package-lock.json b/package-lock.json index afe2091..50ecc9d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,7 +7,6 @@ "": { "name": "openscreen", "version": "1.4.0", - "hasInstallScript": true, "dependencies": { "@fix-webm-duration/fix": "^1.0.1", "@pixi/filter-drop-shadow": "^5.2.0", @@ -48,7 +47,6 @@ "sonner": "^2.0.7", "tailwind-merge": "^3.5.0", "tailwindcss-animate": "^1.0.7", - "uiohook-napi": "^1.5.5", "uuid": "^13.0.0", "web-demuxer": "^4.0.0" }, @@ -188,7 +186,6 @@ "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -397,7 +394,6 @@ "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.2.tgz", "integrity": "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g==", "license": "MIT", - "peer": true, "engines": { "node": ">=6.9.0" } @@ -721,7 +717,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": ">=20.19.0" }, @@ -770,7 +765,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": ">=20.19.0" } @@ -1202,6 +1196,7 @@ "dev": true, "license": "BSD-2-Clause", "optional": true, + "peer": true, "dependencies": { "cross-dirname": "^0.1.0", "debug": "^4.3.4", @@ -1223,6 +1218,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -1239,6 +1235,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "dependencies": { "universalify": "^2.0.0" }, @@ -1253,6 +1250,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "engines": { "node": ">= 10.0.0" } @@ -1962,6 +1960,7 @@ "resolved": "https://registry.npmjs.org/@pixi/color/-/color-7.4.3.tgz", "integrity": "sha512-a6R+bXKeXMDcRmjYQoBIK+v2EYqxSX49wcjAY579EYM/WrFKS98nSees6lqVUcLKrcQh2DT9srJHX7XMny3voQ==", "license": "MIT", + "peer": true, "dependencies": { "@pixi/colord": "^2.9.6" } @@ -1976,7 +1975,8 @@ "version": "7.4.3", "resolved": "https://registry.npmjs.org/@pixi/constants/-/constants-7.4.3.tgz", "integrity": "sha512-QGmwJUNQy/vVEHzL6VGQvnwawLZ1wceZMI8HwJAT4/I2uAzbBeFDdmCS8WsTpSWLZjF/DszDc1D8BFp4pVJ5UQ==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@pixi/core": { "version": "7.4.3", @@ -2003,7 +2003,8 @@ "version": "7.4.3", "resolved": "https://registry.npmjs.org/@pixi/extensions/-/extensions-7.4.3.tgz", "integrity": "sha512-FhoiYkHQEDYHUE7wXhqfsTRz6KxLXjuMbSiAwnLb9uG1vAgp6q6qd6HEsf4X30YaZbLFY8a4KY6hFZWjF+4Fdw==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@pixi/filter-drop-shadow": { "version": "5.2.0", @@ -2030,19 +2031,22 @@ "version": "7.4.3", "resolved": "https://registry.npmjs.org/@pixi/math/-/math-7.4.3.tgz", "integrity": "sha512-/uJOVhR2DOZ+zgdI6Bs/CwcXT4bNRKsS+TqX3ekRIxPCwaLra+Qdm7aDxT5cTToDzdxbKL5+rwiLu3Y1egILDw==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@pixi/runner": { "version": "7.4.3", "resolved": "https://registry.npmjs.org/@pixi/runner/-/runner-7.4.3.tgz", "integrity": "sha512-TJyfp7y23u5vvRAyYhVSa7ytq0PdKSvPLXu4G3meoFh1oxTLHH6g/RIzLuxUAThPG2z7ftthuW3qWq6dRV+dhw==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@pixi/settings": { "version": "7.4.3", "resolved": "https://registry.npmjs.org/@pixi/settings/-/settings-7.4.3.tgz", "integrity": "sha512-SmGK8smc0PxRB9nr0UJioEtE9hl4gvj9OedCvZx3bxBwA3omA5BmP3CyhQfN8XJ29+o2OUL01r3zAPVol4l4lA==", "license": "MIT", + "peer": true, "dependencies": { "@pixi/constants": "7.4.3", "@types/css-font-loading-module": "^0.0.12", @@ -2054,6 +2058,7 @@ "resolved": "https://registry.npmjs.org/@pixi/ticker/-/ticker-7.4.3.tgz", "integrity": "sha512-tHsAD0iOUb6QSGGw+c8cyRBvxsq/NlfzIFBZLEHhWZ+Bx4a0MmXup6I/yJDGmyPCYE+ctCcAfY13wKAzdiVFgQ==", "license": "MIT", + "peer": true, "dependencies": { "@pixi/extensions": "7.4.3", "@pixi/settings": "7.4.3", @@ -2065,6 +2070,7 @@ "resolved": "https://registry.npmjs.org/@pixi/utils/-/utils-7.4.3.tgz", "integrity": "sha512-NO3Y9HAn2UKS1YdxffqsPp+kDpVm8XWvkZcS/E+rBzY9VTLnNOI7cawSRm+dacdET3a8Jad3aDKEDZ0HmAqAFA==", "license": "MIT", + "peer": true, "dependencies": { "@pixi/color": "7.4.3", "@pixi/constants": "7.4.3", @@ -3643,7 +3649,8 @@ "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz", "integrity": "sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@types/babel__core": { "version": "7.20.5", @@ -3718,7 +3725,8 @@ "version": "0.0.12", "resolved": "https://registry.npmjs.org/@types/css-font-loading-module/-/css-font-loading-module-0.0.12.tgz", "integrity": "sha512-x2tZZYkSxXqWvTDgveSynfjq/T2HyiZHXb00j/+gy19yp70PHCizM48XFdjBCWH7eHBD0R5i/pw9yMBP/BH5uA==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@types/debug": { "version": "4.1.13", @@ -3756,7 +3764,8 @@ "version": "2.1.4", "resolved": "https://registry.npmjs.org/@types/earcut/-/earcut-2.1.4.tgz", "integrity": "sha512-qp3m9PPz4gULB9MhjGID7wpo3gJ4bTGXm7ltNDsmOvsPduTeHp8wSW9YckBj3mljeOh4F0m2z/0JKAALRKbmLQ==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@types/estree": { "version": "1.0.8", @@ -3855,7 +3864,6 @@ "integrity": "sha512-z9VXpC7MWrhfWipitjNdgCauoMLRdIILQsAEV+ZesIzBq/oUlxk0m3ApZuMFCXdnS4U7KrI+l3WRUEGQ8K1QKw==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "@types/prop-types": "*", "csstype": "^3.2.2" @@ -3867,7 +3875,6 @@ "integrity": "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==", "devOptional": true, "license": "MIT", - "peer": true, "peerDependencies": { "@types/react": "^18.0.0" } @@ -4149,7 +4156,6 @@ "integrity": "sha512-CWy0lBQJq97nionyJJdnaU4961IXTl43a7UCu5nHy51IoKxAt6PVIJLo+76rVl7KOOgcWHNkG4kbJu/pW7knvA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@vitest/browser": "4.1.5", "@vitest/mocker": "4.1.5", @@ -4342,7 +4348,6 @@ "integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -4868,7 +4873,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.10.12", "caniuse-lite": "^1.0.30001782", @@ -5050,6 +5054,7 @@ "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", "license": "MIT", + "peer": true, "dependencies": { "call-bind-apply-helpers": "^1.0.2", "get-intrinsic": "^1.3.0" @@ -5400,7 +5405,8 @@ "integrity": "sha512-+R08/oI0nl3vfPcqftZRpytksBXDzOUveBq/NBVx0sUp1axwzPQrKinNx5yd5sxPu8j1wIy8AfnVQ+5eFdha6Q==", "dev": true, "license": "MIT", - "optional": true + "optional": true, + "peer": true }, "node_modules/cross-spawn": { "version": "7.0.6", @@ -5690,7 +5696,6 @@ "integrity": "sha512-glMJgnTreo8CFINujtAhCgN96QAqApDMZ8Vl1r8f0QT8QprvC1UCltV4CcWj20YoIyLZx6IUskaJZ0NV8fokcg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "app-builder-lib": "26.8.1", "builder-util": "26.8.1", @@ -5783,7 +5788,8 @@ "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz", "integrity": "sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/dotenv": { "version": "16.6.1", @@ -5832,7 +5838,8 @@ "version": "2.2.4", "resolved": "https://registry.npmjs.org/earcut/-/earcut-2.2.4.tgz", "integrity": "sha512-/pjZsA1b4RPHbeWZQn66SWS8nZZWLQQ23oE3Eam7aroEFGEvwKAsJfZ9ytiEMycfzXWpca4FA9QIOehf7PocBQ==", - "license": "ISC" + "license": "ISC", + "peer": true }, "node_modules/ejs": { "version": "3.1.10", @@ -6015,6 +6022,7 @@ "dev": true, "hasInstallScript": true, "license": "MIT", + "peer": true, "dependencies": { "@electron/asar": "^3.2.1", "debug": "^4.1.1", @@ -6035,6 +6043,7 @@ "integrity": "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "graceful-fs": "^4.1.2", "jsonfile": "^4.0.0", @@ -6277,7 +6286,8 @@ "version": "4.0.7", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/expect-type": { "version": "1.3.0", @@ -7689,6 +7699,7 @@ "integrity": "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==", "dev": true, "license": "MIT", + "peer": true, "bin": { "lz-string": "bin/bin.js" } @@ -7908,6 +7919,7 @@ "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "minimist": "^1.2.6" }, @@ -8093,17 +8105,6 @@ "node": "^20.17.0 || >=22.9.0" } }, - "node_modules/node-gyp-build": { - "version": "4.8.4", - "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", - "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==", - "license": "MIT", - "bin": { - "node-gyp-build": "bin.js", - "node-gyp-build-optional": "optional.js", - "node-gyp-build-test": "build-test.js" - } - }, "node_modules/node-gyp/node_modules/isexe": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-4.0.0.tgz", @@ -8221,6 +8222,7 @@ "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", "license": "MIT", + "peer": true, "engines": { "node": ">= 0.4" }, @@ -8429,7 +8431,6 @@ "resolved": "https://registry.npmjs.org/pixi.js/-/pixi.js-8.18.1.tgz", "integrity": "sha512-6LUPWYgulZhp/w4kam2XHXB0QedISZIqrJbRdHLLQ3csn5a38uzKxAp6B5j6s89QFYaIJbg95kvgTRcbgpO1ow==", "license": "MIT", - "peer": true, "workspaces": [ "examples", "playground" @@ -8475,7 +8476,6 @@ "integrity": "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "playwright-core": "1.59.1" }, @@ -8546,7 +8546,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -8691,6 +8690,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "dependencies": { "commander": "^9.4.0" }, @@ -8708,6 +8708,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "engines": { "node": "^12.20.0 || >=14" } @@ -8718,6 +8719,7 @@ "integrity": "sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "ansi-regex": "^5.0.1", "ansi-styles": "^5.0.0", @@ -8733,6 +8735,7 @@ "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=10" }, @@ -8846,6 +8849,7 @@ "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.1.tgz", "integrity": "sha512-6YHEFRL9mfgcAvql/XhwTvf5jKcOiiupt2FiJxHkiX1z4j7WL8J/jRHYLluORvc1XxB5rV20KoeK00gVJamspg==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "side-channel": "^1.1.0" }, @@ -8904,7 +8908,6 @@ "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", "license": "MIT", - "peer": true, "dependencies": { "loose-envify": "^1.1.0" }, @@ -8917,7 +8920,6 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", "license": "MIT", - "peer": true, "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" @@ -8954,7 +8956,8 @@ "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/react-refresh": { "version": "0.18.0", @@ -9275,6 +9278,7 @@ "deprecated": "Rimraf versions prior to v4 are no longer supported", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "glob": "^7.1.3" }, @@ -9481,6 +9485,7 @@ "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", "license": "MIT", + "peer": true, "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3", @@ -9500,6 +9505,7 @@ "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", "license": "MIT", + "peer": true, "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.4" @@ -9516,6 +9522,7 @@ "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", "license": "MIT", + "peer": true, "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", @@ -9534,6 +9541,7 @@ "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", "license": "MIT", + "peer": true, "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", @@ -9874,7 +9882,6 @@ "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.4.19.tgz", "integrity": "sha512-3ofp+LL8E+pK/JuPLPggVAIaEuhvIz4qNcf3nA1Xn2o/7fb7s/TYpHhwGDv1ZU3PkBluUVaF8PyCHcm48cKLWQ==", "license": "MIT", - "peer": true, "dependencies": { "@alloc/quick-lru": "^5.2.0", "arg": "^5.0.2", @@ -9958,6 +9965,7 @@ "integrity": "sha512-yYrrsWnrXMcdsnu/7YMYAofM1ktpL5By7vZhf15CrXijWWrEYZks5AXBudalfSWJLlnen/QUJUB5aoB0kqZUGA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "mkdirp": "^0.5.1", "rimraf": "~2.6.2" @@ -10272,19 +10280,6 @@ "node": ">=14.17" } }, - "node_modules/uiohook-napi": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/uiohook-napi/-/uiohook-napi-1.5.5.tgz", - "integrity": "sha512-oSlTdnECw2GBfsJPTbBQBeE4v/EXP0EZmX6BJq5nzH/JgFaBE8JpFwEA/kLhiEP7HxQw28FViWiYgdIZzWuuJQ==", - "hasInstallScript": true, - "license": "MIT", - "dependencies": { - "node-gyp-build": "^4.8.4" - }, - "engines": { - "node": ">= 16" - } - }, "node_modules/undici": { "version": "7.25.0", "resolved": "https://registry.npmjs.org/undici/-/undici-7.25.0.tgz", @@ -10358,6 +10353,7 @@ "resolved": "https://registry.npmjs.org/url/-/url-0.11.4.tgz", "integrity": "sha512-oCwdVC7mTuWiPyjLUz/COz5TLk6wgp0RCsN+wHZ2Ekneac9w8uuV0njcbbie2ME+Vs+d6duwmYuR3HgQXs1fOg==", "license": "MIT", + "peer": true, "dependencies": { "punycode": "^1.4.1", "qs": "^6.12.3" @@ -10370,7 +10366,8 @@ "version": "1.4.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", "integrity": "sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/use-callback-ref": { "version": "1.3.3", @@ -10463,7 +10460,6 @@ "integrity": "sha512-Bby3NOsna2jsjfLVOHKes8sGwgl4TT0E6vvpYgnAYDIF/tie7MRaFthmKuHx1NSXjiTueXH3do80FMQgvEktRg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.27.0", "fdir": "^6.5.0", @@ -10553,8 +10549,7 @@ "resolved": "https://registry.npmjs.org/vite-plugin-electron-renderer/-/vite-plugin-electron-renderer-0.14.6.tgz", "integrity": "sha512-oqkWFa7kQIkvHXG7+Mnl1RTroA4sP0yesKatmAy0gjZC4VwUqlvF9IvOpHd1fpLWsqYX/eZlVxlhULNtaQ78Jw==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/vite/node_modules/fsevents": { "version": "2.3.3", @@ -10577,7 +10572,6 @@ "integrity": "sha512-9Xx1v3/ih3m9hN+SbfkUyy0JAs72ap3r7joc87XL6jwF0jGg6mFBvQ1SrwaX+h8BlkX6Hz9shdd1uo6AF+ZGpg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@vitest/expect": "4.1.5", "@vitest/mocker": "4.1.5", diff --git a/package.json b/package.json index 2ccb0b3..fd0c4cf 100644 --- a/package.json +++ b/package.json @@ -20,18 +20,29 @@ "format": "biome format --write .", "i18n:check": "node scripts/i18n-check.mjs", "preview": "vite preview", - "build:mac": "tsc && vite build && electron-builder --mac", - "build:win": "tsc && vite build && electron-builder --win --config.npmRebuild=false", + "build:native:mac": "node scripts/build-macos-screencapturekit-helper.mjs", + "build:mac": "npm run build:native:mac && tsc && vite build && electron-builder --mac", + "build:native:win": "node scripts/build-windows-wgc-helper.mjs", + "build:win": "npm run build:native:win && tsc && vite build && electron-builder --win --config.npmRebuild=false", "build:linux": "tsc && vite build && electron-builder --linux AppImage deb pacman --config.npmRebuild=false", "test": "vitest --run", "test:watch": "vitest", + "test:cursor-native:win": "node scripts/test-windows-native-cursor.mjs", + "test:wgc-helper:win": "node scripts/test-windows-wgc-helper.mjs", + "test:wgc-window:win": "node scripts/test-windows-wgc-helper.mjs --window", + "test:wgc-audio:win": "node scripts/test-windows-wgc-helper.mjs --system-audio", + "test:wgc-mic:win": "node scripts/test-windows-wgc-helper.mjs --microphone", + "test:wgc-mixed-audio:win": "node scripts/test-windows-wgc-helper.mjs --system-audio --microphone", + "test:wgc-webcam:win": "node scripts/test-windows-wgc-helper.mjs --webcam", + "test:wgc-full:win": "node scripts/test-windows-wgc-helper.mjs --webcam --system-audio --microphone", + "capture:openscreen-preview": "node scripts/capture-openscreen-preview.mjs", + "inspect:cursor-click-bounce": "node scripts/inspect-native-cursor-click-bounce.mjs", "build-vite": "tsc && vite build", "test:browser": "vitest --config vitest.browser.config.ts --run", "test:browser:install": "playwright install --with-deps chromium-headless-shell", "test:e2e": "playwright test", - "prepare": "husky", - "rebuild:native": "node ./scripts/rebuild-native.mjs", - "postinstall": "npm run rebuild:native" + "test:e2e:windows-native-checklist": "playwright test tests/e2e/windows-native-checklist.spec.ts", + "prepare": "husky" }, "dependencies": { "@fix-webm-duration/fix": "^1.0.1", @@ -73,7 +84,6 @@ "sonner": "^2.0.7", "tailwind-merge": "^3.5.0", "tailwindcss-animate": "^1.0.7", - "uiohook-napi": "^1.5.5", "uuid": "^13.0.0", "web-demuxer": "^4.0.0" }, diff --git a/scripts/build-macos-screencapturekit-helper.mjs b/scripts/build-macos-screencapturekit-helper.mjs new file mode 100644 index 0000000..8e7c973 --- /dev/null +++ b/scripts/build-macos-screencapturekit-helper.mjs @@ -0,0 +1,91 @@ +#!/usr/bin/env node + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; + +if (process.platform !== "darwin") { + console.log("Skipping macOS ScreenCaptureKit helper build: host platform is not macOS."); + process.exit(0); +} + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const root = path.resolve(__dirname, ".."); +const helperName = "openscreen-screencapturekit-helper"; +const cursorHelperName = "openscreen-macos-cursor-helper"; +const packageDir = path.join(root, "electron", "native", "screencapturekit"); +const buildDir = path.join(packageDir, "build"); +const swiftBuildDir = path.join(buildDir, "swiftpm"); +const builtHelperPath = path.join(swiftBuildDir, "release", helperName); +const localHelperPath = path.join(buildDir, helperName); +const builtCursorHelperPath = path.join(swiftBuildDir, "release", cursorHelperName); +const localCursorHelperPath = path.join(buildDir, cursorHelperName); +const archTag = process.arch === "arm64" ? "darwin-arm64" : "darwin-x64"; +const distributableDir = path.join(root, "electron", "native", "bin", archTag); +const distributablePath = path.join(distributableDir, helperName); +const distributableCursorHelperPath = path.join(distributableDir, cursorHelperName); + +const xcodebuildVersion = spawnSync("xcodebuild", ["-version"], { + cwd: root, + encoding: "utf8", +}); + +if (xcodebuildVersion.status !== 0) { + const message = `${xcodebuildVersion.stderr ?? ""}${xcodebuildVersion.stdout ?? ""}`.trim(); + console.error( + [ + "Unable to build the macOS ScreenCaptureKit helper because full Xcode is not active.", + "", + message, + "", + "Install Xcode from the App Store or Apple Developer downloads, then run:", + " sudo xcode-select --switch /Applications/Xcode.app/Contents/Developer", + " sudo xcodebuild -license accept", + "", + "Command Line Tools alone may not include the Swift SDK/platform metadata required by SwiftPM.", + ].join("\n"), + ); + process.exit(1); +} + +const result = spawnSync( + "swift", + ["build", "-c", "release", "--package-path", packageDir, "--build-path", swiftBuildDir], + { + cwd: root, + stdio: "inherit", + }, +); + +if (result.error) { + console.error(`Failed to start Swift build: ${result.error.message}`); + process.exit(1); +} + +if (result.status !== 0) { + process.exit(result.status ?? 1); +} + +fs.mkdirSync(buildDir, { recursive: true }); +fs.mkdirSync(distributableDir, { recursive: true }); +for (const artifactPath of [builtHelperPath, builtCursorHelperPath]) { + if (!fs.existsSync(artifactPath)) { + console.error(`Swift build completed but expected artifact was not found: ${artifactPath}`); + process.exit(1); + } +} +fs.copyFileSync(builtHelperPath, localHelperPath); +fs.copyFileSync(builtHelperPath, distributablePath); +fs.copyFileSync(builtCursorHelperPath, localCursorHelperPath); +fs.copyFileSync(builtCursorHelperPath, distributableCursorHelperPath); +fs.chmodSync(localHelperPath, 0o755); +fs.chmodSync(distributablePath, 0o755); +fs.chmodSync(localCursorHelperPath, 0o755); +fs.chmodSync(distributableCursorHelperPath, 0o755); + +console.log(`Built macOS ScreenCaptureKit helper: ${localHelperPath}`); +console.log(`Copied redistributable helper: ${distributablePath}`); +console.log(`Built macOS cursor helper: ${localCursorHelperPath}`); +console.log(`Copied redistributable cursor helper: ${distributableCursorHelperPath}`); diff --git a/scripts/build-windows-wgc-helper.mjs b/scripts/build-windows-wgc-helper.mjs new file mode 100644 index 0000000..29df4d8 --- /dev/null +++ b/scripts/build-windows-wgc-helper.mjs @@ -0,0 +1,139 @@ +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.join(__dirname, ".."); +const SOURCE_DIR = path.join(ROOT, "electron", "native", "wgc-capture"); +const BUILD_DIR = path.join(SOURCE_DIR, "build"); +const COMPAT_LIB_DIR = path.join(BUILD_DIR, "compat-libs"); +const BIN_DIR = path.join(ROOT, "electron", "native", "bin", "win32-x64"); +const CMAKE = process.env.CMAKE_EXE ?? "cmake"; + +function findVcVarsAll() { + const explicit = process.env.VCVARSALL; + if (explicit && fs.existsSync(explicit)) { + return explicit; + } + + const roots = [ + process.env.VSINSTALLDIR, + "C:\\Program Files\\Microsoft Visual Studio\\2022\\Community", + "C:\\Program Files\\Microsoft Visual Studio\\2022\\Professional", + "C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise", + "C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\BuildTools", + "C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\Community", + ]; + + for (const root of roots.filter(Boolean)) { + const candidate = path.join(root, "VC", "Auxiliary", "Build", "vcvarsall.bat"); + if (fs.existsSync(candidate)) { + return candidate; + } + } + + return null; +} + +function findWindowsSdkUmLibDir() { + const sdkLibRoot = "C:\\Program Files (x86)\\Windows Kits\\10\\Lib"; + if (!fs.existsSync(sdkLibRoot)) { + return null; + } + + return fs + .readdirSync(sdkLibRoot, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => path.join(sdkLibRoot, entry.name, "um", "x64")) + .filter((candidate) => fs.existsSync(path.join(candidate, "kernel32.lib"))) + .sort() + .at(-1); +} + +function run(command, args, options = {}) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd: ROOT, + stdio: "inherit", + windowsHide: true, + ...options, + }); + child.once("error", reject); + child.once("exit", (code) => { + if (code === 0) { + resolve(); + } else { + reject(new Error(`${command} ${args.join(" ")} failed with code ${code}`)); + } + }); + }); +} + +async function runInVsEnv(command) { + const vcvarsAll = findVcVarsAll(); + if (!vcvarsAll) { + throw new Error( + "Could not find Visual Studio vcvarsall.bat. Install Visual Studio Build Tools with C++.", + ); + } + + const sdkUmLibDir = findWindowsSdkUmLibDir(); + + const cmdPath = path.join(os.tmpdir(), `openscreen-build-wgc-${process.pid}-${Date.now()}.cmd`); + fs.writeFileSync( + cmdPath, + [ + "@echo off", + `call "${vcvarsAll}" x64`, + "if errorlevel 1 exit /b %errorlevel%", + `if not exist "${COMPAT_LIB_DIR}" mkdir "${COMPAT_LIB_DIR}"`, + `for %%L in (gdi32.lib gdiplus.lib winspool.lib shell32.lib oleaut32.lib uuid.lib comdlg32.lib advapi32.lib) do if not exist "%WindowsSdkDir%Lib\\%WindowsSDKLibVersion%um\\x64\\%%L" copy /Y "%WindowsSdkDir%Lib\\%WindowsSDKLibVersion%um\\x64\\kernel32.Lib" "${COMPAT_LIB_DIR}\\%%L" >nul`, + "if errorlevel 1 exit /b %errorlevel%", + `set "LIB=${sdkUmLibDir ? `${sdkUmLibDir};` : ""}%LIB%;${COMPAT_LIB_DIR}"`, + command, + "exit /b %errorlevel%", + "", + ].join("\r\n"), + ); + try { + await run("cmd.exe", ["/d", "/c", cmdPath]); + } finally { + fs.rmSync(cmdPath, { force: true }); + } +} + +if (process.platform !== "win32") { + console.log("Skipping WGC helper build: Windows-only."); + process.exit(0); +} + +fs.mkdirSync(BUILD_DIR, { recursive: true }); + +await runInVsEnv( + `"${CMAKE}" -S "${SOURCE_DIR}" -B "${BUILD_DIR}" -G Ninja -DCMAKE_BUILD_TYPE=Release`, +); +await runInVsEnv(`"${CMAKE}" --build "${BUILD_DIR}" --config Release`); + +const outputPath = path.join(BUILD_DIR, "wgc-capture.exe"); +if (!fs.existsSync(outputPath)) { + throw new Error(`WGC helper build completed but ${outputPath} was not found.`); +} + +const cursorSamplerOutputPath = path.join(BUILD_DIR, "cursor-sampler.exe"); +if (!fs.existsSync(cursorSamplerOutputPath)) { + throw new Error(`WGC helper build completed but ${cursorSamplerOutputPath} was not found.`); +} + +fs.mkdirSync(BIN_DIR, { recursive: true }); +const distributablePath = path.join(BIN_DIR, "wgc-capture.exe"); +fs.copyFileSync(outputPath, distributablePath); + +const cursorSamplerDistributablePath = path.join(BIN_DIR, "cursor-sampler.exe"); +fs.copyFileSync(cursorSamplerOutputPath, cursorSamplerDistributablePath); + +console.log(`Built ${outputPath}`); +console.log(`Copied ${distributablePath}`); +console.log(`Built ${cursorSamplerOutputPath}`); +console.log(`Copied ${cursorSamplerDistributablePath}`); diff --git a/scripts/capture-openscreen-preview.mjs b/scripts/capture-openscreen-preview.mjs new file mode 100644 index 0000000..25f86db --- /dev/null +++ b/scripts/capture-openscreen-preview.mjs @@ -0,0 +1,258 @@ +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { chromium, _electron as electron } from "@playwright/test"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.join(__dirname, ".."); +const MAIN_JS = path.join(ROOT, "dist-electron", "main.js"); +const TEST_VIDEO = path.join(ROOT, "tests", "fixtures", "sample.webm"); +const OUTPUT_DIR = + process.env.OPENSCREEN_PREVIEW_OUTPUT_DIR ?? + path.join(os.tmpdir(), `openscreen-real-preview-${Date.now()}`); +const FRAME_COUNT = Number(process.env.OPENSCREEN_PREVIEW_FRAME_COUNT ?? 90); +const FPS = Number(process.env.OPENSCREEN_PREVIEW_FPS ?? 30); + +function findLatestCursorRecordingData() { + const explicit = process.env.CURSOR_RECORDING_DATA_PATH; + if (explicit) { + if (!fs.existsSync(explicit)) { + throw new Error(`CURSOR_RECORDING_DATA_PATH does not exist: ${explicit}`); + } + return explicit; + } + + const tempDir = os.tmpdir(); + const candidates = fs + .readdirSync(tempDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && entry.name.startsWith("openscreen-cursor-native-")) + .map((entry) => path.join(tempDir, entry.name, "cursor-recording-data.json")) + .filter((candidate) => fs.existsSync(candidate)) + .map((candidate) => ({ path: candidate, mtimeMs: fs.statSync(candidate).mtimeMs })) + .sort((a, b) => b.mtimeMs - a.mtimeMs); + + if (!candidates[0]) { + throw new Error( + "No cursor-recording-data.json found. Run npm run test:cursor-native:win first.", + ); + } + + return candidates[0].path; +} + +function findPlaywrightChromiumExecutable(defaultPath) { + if (fs.existsSync(defaultPath)) { + return defaultPath; + } + + const baseDir = path.join(process.env.LOCALAPPDATA ?? "", "ms-playwright"); + if (!baseDir || !fs.existsSync(baseDir)) { + return defaultPath; + } + + const candidates = fs + .readdirSync(baseDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && entry.name.startsWith("chromium-")) + .map((entry) => path.join(baseDir, entry.name, "chrome-win64", "chrome.exe")) + .filter((candidate) => fs.existsSync(candidate)) + .sort() + .reverse(); + + return candidates[0] ?? defaultPath; +} + +function ensureBuildExists() { + if (!fs.existsSync(MAIN_JS)) { + throw new Error(`Missing ${MAIN_JS}. Run npm run build-vite first.`); + } + if (!fs.existsSync(path.join(ROOT, "dist", "index.html"))) { + throw new Error(`Missing renderer build. Run npm run build-vite first.`); + } +} + +function runNpmBuildViteIfRequested() { + if (process.env.OPENSCREEN_PREVIEW_SKIP_BUILD === "true") { + ensureBuildExists(); + return Promise.resolve(); + } + + return new Promise((resolve, reject) => { + const child = spawn("cmd.exe", ["/d", "/s", "/c", "npm run build-vite"], { + cwd: ROOT, + stdio: "inherit", + }); + child.once("error", reject); + child.once("exit", (code) => { + if (code === 0) resolve(); + else reject(new Error(`npm run build-vite failed with code ${code}`)); + }); + }); +} + +async function encodeFramesToWebm(framePaths, outputPath) { + const frameData = framePaths.map((framePath) => ({ + src: `data:image/png;base64,${fs.readFileSync(framePath).toString("base64")}`, + })); + const html = ` + + + + + +`; + + const browser = await chromium.launch({ + executablePath: findPlaywrightChromiumExecutable(chromium.executablePath()), + headless: true, + }); + try { + const page = await browser.newPage(); + await page.setContent(html); + const base64 = await page.evaluate(() => window.__encode()); + fs.writeFileSync(outputPath, Buffer.from(base64, "base64")); + } finally { + await browser.close(); + } +} + +fs.mkdirSync(OUTPUT_DIR, { recursive: true }); +const cursorRecordingDataPath = findLatestCursorRecordingData(); +const fixtureVideoPath = path.join(OUTPUT_DIR, "openscreen-preview-fixture.webm"); +const outputVideoPath = path.join(OUTPUT_DIR, "openscreen-preview.webm"); +fs.copyFileSync(TEST_VIDEO, fixtureVideoPath); +fs.copyFileSync(cursorRecordingDataPath, `${fixtureVideoPath}.cursor.json`); + +await runNpmBuildViteIfRequested(); + +const app = await electron.launch({ + args: [MAIN_JS, "--no-sandbox", "--enable-unsafe-swiftshader"], + env: { + ...process.env, + HEADLESS: "false", + }, +}); + +app.process().stdout?.on("data", (data) => process.stdout.write(`[electron] ${data}`)); +app.process().stderr?.on("data", (data) => process.stderr.write(`[electron] ${data}`)); + +const framesDir = path.join(OUTPUT_DIR, "frames"); +fs.mkdirSync(framesDir, { recursive: true }); + +try { + const hudWindow = await app.firstWindow({ timeout: 60_000 }); + await hudWindow.waitForLoadState("domcontentloaded"); + await hudWindow.evaluate(async () => { + for (let attempt = 0; attempt < 100; attempt += 1) { + try { + await window.electronAPI.getCurrentRecordingSession(); + await window.electronAPI.getCurrentVideoPath(); + return; + } catch { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + } + throw new Error("Timed out waiting for OpenScreen IPC handlers."); + }); + + try { + await hudWindow.evaluate(async (videoPath) => { + await window.electronAPI.setCurrentVideoPath(videoPath); + await window.electronAPI.switchToEditor(); + }, fixtureVideoPath); + } catch { + // switchToEditor closes the HUD page before the evaluate promise can always resolve. + } + + const editorWindow = await app.waitForEvent("window", { + predicate: (window) => window.url().includes("windowType=editor"), + timeout: 30_000, + }); + await editorWindow.waitForLoadState("domcontentloaded"); + await editorWindow.waitForSelector("video", { state: "attached", timeout: 30_000 }); + await editorWindow.waitForSelector("canvas", { state: "attached", timeout: 30_000 }); + + await editorWindow.setViewportSize({ width: 1280, height: 800 }); + await editorWindow.evaluate(async () => { + await document.fonts.ready; + for (const video of [...document.querySelectorAll("video")]) { + video.muted = true; + video.currentTime = 0; + video.dispatchEvent(new Event("timeupdate")); + } + }); + await editorWindow.waitForTimeout(1000); + + const framePaths = []; + for (let index = 0; index < FRAME_COUNT; index += 1) { + const timeSec = index / FPS; + await editorWindow.evaluate((time) => { + for (const video of [...document.querySelectorAll("video")]) { + video.currentTime = Math.min(time, Math.max(0, video.duration || time)); + video.dispatchEvent(new Event("timeupdate")); + } + }, timeSec); + await editorWindow.waitForTimeout(40); + const framePath = path.join(framesDir, `frame-${String(index).padStart(4, "0")}.png`); + await editorWindow.screenshot({ path: framePath }); + framePaths.push(framePath); + } + + await encodeFramesToWebm(framePaths, outputVideoPath); + + const report = { + outputDir: OUTPUT_DIR, + sourceCursorRecordingDataPath: cursorRecordingDataPath, + fixtureVideoPath, + outputVideoPath, + frameCount: framePaths.length, + fps: FPS, + }; + fs.writeFileSync(path.join(OUTPUT_DIR, "report.json"), JSON.stringify(report, null, 2)); + console.log(JSON.stringify(report, null, 2)); +} finally { + await app.close(); +} diff --git a/scripts/inspect-native-cursor-click-bounce.mjs b/scripts/inspect-native-cursor-click-bounce.mjs new file mode 100644 index 0000000..870ee8d --- /dev/null +++ b/scripts/inspect-native-cursor-click-bounce.mjs @@ -0,0 +1,110 @@ +#!/usr/bin/env node +import fs from "node:fs"; +import path from "node:path"; + +const CLICK_ANIMATION_MS = 260; + +function usage() { + console.error( + "Usage: node scripts/inspect-native-cursor-click-bounce.mjs [--bounce=5]", + ); + process.exit(1); +} + +function getCursorJsonPath(inputPath) { + if (!inputPath) { + usage(); + } + + const resolved = path.resolve(inputPath); + if (resolved.endsWith(".cursor.json")) { + return resolved; + } + return `${resolved}.cursor.json`; +} + +function getBounceValue() { + const arg = process.argv.find((value) => value.startsWith("--bounce=")); + const parsed = Number(arg?.slice("--bounce=".length) ?? 5); + return Number.isFinite(parsed) ? Math.min(5, Math.max(0, parsed)) : 5; +} + +function clickBounceProgress(samples, timeMs) { + for (let index = samples.length - 1; index >= 0; index -= 1) { + const sample = samples[index]; + if (sample.timeMs > timeMs) { + continue; + } + + const ageMs = timeMs - sample.timeMs; + if (ageMs > CLICK_ANIMATION_MS) { + return 0; + } + + if (sample.interactionType === "click") { + return 1 - ageMs / CLICK_ANIMATION_MS; + } + } + + return 0; +} + +function clickBounceScale(clickBounce, progress) { + if (progress <= 0 || clickBounce <= 0) { + return 1; + } + + const intensity = Math.min(5, Math.max(0, clickBounce)) / 5; + const elapsed = 1 - Math.min(1, Math.max(0, progress)); + if (elapsed < 0.38) { + const pressProgress = Math.sin((elapsed / 0.38) * Math.PI); + return 1 - pressProgress * intensity * 0.24; + } + + const reboundProgress = Math.sin(((elapsed - 0.38) / 0.62) * Math.PI); + return 1 + reboundProgress * intensity * 0.16; +} + +const cursorJsonPath = getCursorJsonPath(process.argv[2]); +const clickBounce = getBounceValue(); +const parsed = JSON.parse(fs.readFileSync(cursorJsonPath, "utf8")); +const samples = (Array.isArray(parsed) ? parsed : (parsed.samples ?? [])).sort( + (a, b) => (a.timeMs ?? 0) - (b.timeMs ?? 0), +); +const clicks = samples.filter((sample) => sample.interactionType === "click"); + +const windows = clicks.slice(0, 8).map((click) => { + const times = [0, 33, 66, 100, 133, 166, 200, 233, 260].map( + (offsetMs) => click.timeMs + offsetMs, + ); + return { + clickTimeMs: click.timeMs, + cursorType: click.cursorType ?? null, + assetId: click.assetId ?? null, + scales: times.map((timeMs) => ({ + timeMs, + progress: Number(clickBounceProgress(samples, timeMs).toFixed(3)), + scale: Number(clickBounceScale(clickBounce, clickBounceProgress(samples, timeMs)).toFixed(3)), + })), + }; +}); + +const report = { + cursorJsonPath, + provider: parsed.provider ?? (Array.isArray(parsed) ? "legacy-array" : null), + sampleCount: samples.length, + assetCount: Array.isArray(parsed.assets) ? parsed.assets.length : 0, + clickCount: clicks.length, + interactionCounts: samples.reduce((counts, sample) => { + const key = sample.interactionType ?? "missing"; + counts[key] = (counts[key] ?? 0) + 1; + return counts; + }, {}), + clickBounce, + windows, +}; + +console.log(JSON.stringify(report, null, 2)); +if (clicks.length === 0) { + process.exitCode = 2; +} diff --git a/scripts/rebuild-native.mjs b/scripts/rebuild-native.mjs deleted file mode 100644 index e028602..0000000 --- a/scripts/rebuild-native.mjs +++ /dev/null @@ -1,21 +0,0 @@ -import { spawnSync } from "node:child_process"; -import process from "node:process"; - -// uiohook-napi click capture is macOS-only at runtime (gated in -// electron/ipc/handlers.ts). Skip the rebuild on other platforms so CI runners -// without X11 dev headers don't fail npm install. The library's prebuilt -// .node binaries are still bundled and loadable; we just don't need a fresh -// build against Electron's ABI on platforms where we don't load it. -if (process.platform !== "darwin") { - console.log( - `[rebuild:native] Skipping uiohook-napi rebuild on ${process.platform} (macOS-only).`, - ); - process.exit(0); -} - -const result = spawnSync( - process.execPath, - ["./node_modules/@electron/rebuild/lib/cli.js", "--force", "--only", "uiohook-napi"], - { stdio: "inherit" }, -); -process.exit(result.status ?? 0); diff --git a/scripts/test-windows-native-cursor.mjs b/scripts/test-windows-native-cursor.mjs new file mode 100644 index 0000000..44cabbe --- /dev/null +++ b/scripts/test-windows-native-cursor.mjs @@ -0,0 +1,1342 @@ +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +function readPositiveIntEnv(name, fallback) { + const raw = process.env[name]; + if (raw === undefined) { + return fallback; + } + + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) { + console.warn(`[cursor-native-test] ignoring invalid ${name}=${raw}; using ${fallback}`); + return fallback; + } + + return Math.floor(parsed); +} + +const SAMPLE_INTERVAL_MS = readPositiveIntEnv("CURSOR_TEST_SAMPLE_INTERVAL_MS", 25); +const DURATION_MS = readPositiveIntEnv("CURSOR_TEST_DURATION_MS", 1800); +const SCREEN_FRAME_INTERVAL_MS = readPositiveIntEnv("CURSOR_TEST_SCREEN_FRAME_INTERVAL_MS", 100); +const READY_TIMEOUT_MS = readPositiveIntEnv("CURSOR_TEST_READY_TIMEOUT_MS", 5000); +const OUTPUT_DIR = + process.env.CURSOR_TEST_OUTPUT_DIR ?? + path.join(os.tmpdir(), `openscreen-cursor-native-${Date.now()}`); + +if (process.platform !== "win32") { + console.error("This diagnostic is Windows-only."); + process.exit(1); +} + +function encodePowerShell(script) { + return Buffer.from(script, "utf16le").toString("base64"); +} + +function quotePowerShellString(value) { + return `'${String(value).replaceAll("'", "''")}'`; +} + +function runPowerShell(script) { + return new Promise((resolve, reject) => { + const child = spawn( + "powershell.exe", + [ + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-EncodedCommand", + encodePowerShell(script), + ], + { stdio: ["ignore", "pipe", "pipe"], windowsHide: true }, + ); + + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.once("error", reject); + child.once("exit", (code, signal) => { + if (code === 0) { + resolve(stdout); + return; + } + + reject( + new Error(`PowerShell command failed (code=${code}, signal=${signal}): ${stderr.trim()}`), + ); + }); + }); +} + +function spawnPowerShell(script, { onStdout, onStderr } = {}) { + const scriptPath = path.join( + os.tmpdir(), + `openscreen-powershell-${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2)}.ps1`, + ); + fs.writeFileSync(scriptPath, script, "utf8"); + const child = spawn( + "powershell.exe", + ["-NoLogo", "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-File", scriptPath], + { stdio: ["ignore", "pipe", "pipe"], windowsHide: true }, + ); + + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => onStdout?.(chunk)); + child.stderr.on("data", (chunk) => onStderr?.(chunk)); + + const done = new Promise((resolve, reject) => { + const cleanup = () => { + fs.rmSync(scriptPath, { force: true }); + }; + child.once("error", (error) => { + cleanup(); + reject(error); + }); + child.once("exit", (code, signal) => { + cleanup(); + if (code === 0 || child.killed) { + resolve({ code, signal }); + return; + } + + reject(new Error(`PowerShell process failed (code=${code}, signal=${signal})`)); + }); + }); + + return { child, done }; +} + +function buildSamplerScript() { + return String.raw` +$ErrorActionPreference = 'Stop' +Add-Type -AssemblyName System.Drawing +Add-Type -AssemblyName System.Windows.Forms + +$source = @" +using System; +using System.Diagnostics; +using System.Runtime.InteropServices; + +public static class OpenScreenCursorDiagnosticInterop { + private const int WH_MOUSE_LL = 14; + private const int WM_LBUTTONDOWN = 0x0201; + private const int WM_LBUTTONUP = 0x0202; + private static readonly object MouseSync = new object(); + private static int LeftDownCount = 0; + private static int LeftUpCount = 0; + private static IntPtr MouseHook = IntPtr.Zero; + private static LowLevelMouseProc MouseProcDelegate = MouseHookCallback; + + public delegate IntPtr LowLevelMouseProc(int nCode, IntPtr wParam, IntPtr lParam); + + public struct MouseButtonEvents { + public int LeftDownCount; + public int LeftUpCount; + } + + [StructLayout(LayoutKind.Sequential)] + public struct POINT { + public int X; + public int Y; + } + + [StructLayout(LayoutKind.Sequential)] + public struct CURSORINFO { + public int cbSize; + public int flags; + public IntPtr hCursor; + public POINT ptScreenPos; + } + + [StructLayout(LayoutKind.Sequential)] + public struct ICONINFO { + [MarshalAs(UnmanagedType.Bool)] + public bool fIcon; + public int xHotspot; + public int yHotspot; + public IntPtr hbmMask; + public IntPtr hbmColor; + } + + public static bool InstallMouseHook() { + if (MouseHook != IntPtr.Zero) { + return true; + } + + using (Process process = Process.GetCurrentProcess()) + using (ProcessModule module = process.MainModule) { + MouseHook = SetWindowsHookEx(WH_MOUSE_LL, MouseProcDelegate, GetModuleHandle(module.ModuleName), 0); + } + + return MouseHook != IntPtr.Zero; + } + + public static MouseButtonEvents ConsumeMouseButtonEvents() { + lock (MouseSync) { + MouseButtonEvents events = new MouseButtonEvents { + LeftDownCount = LeftDownCount, + LeftUpCount = LeftUpCount + }; + LeftDownCount = 0; + LeftUpCount = 0; + return events; + } + } + + private static IntPtr MouseHookCallback(int nCode, IntPtr wParam, IntPtr lParam) { + if (nCode >= 0) { + int message = wParam.ToInt32(); + if (message == WM_LBUTTONDOWN || message == WM_LBUTTONUP) { + lock (MouseSync) { + if (message == WM_LBUTTONDOWN) { + LeftDownCount += 1; + } else { + LeftUpCount += 1; + } + } + } + } + + return CallNextHookEx(MouseHook, nCode, wParam, lParam); + } + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool GetCursorInfo(ref CURSORINFO pci); + + [DllImport("user32.dll")] + public static extern short GetAsyncKeyState(int vKey); + + [DllImport("user32.dll", SetLastError = true)] + public static extern IntPtr LoadCursor(IntPtr hInstance, IntPtr lpCursorName); + + [DllImport("user32.dll", SetLastError = true)] + public static extern IntPtr CopyIcon(IntPtr hIcon); + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool DestroyIcon(IntPtr hIcon); + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool GetIconInfo(IntPtr hIcon, out ICONINFO piconinfo); + + [DllImport("gdi32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool DeleteObject(IntPtr hObject); + + [DllImport("user32.dll", SetLastError = true)] + private static extern IntPtr SetWindowsHookEx(int idHook, LowLevelMouseProc lpfn, IntPtr hMod, uint dwThreadId); + + [DllImport("user32.dll", SetLastError = true)] + private static extern IntPtr CallNextHookEx(IntPtr hhk, int nCode, IntPtr wParam, IntPtr lParam); + + [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)] + private static extern IntPtr GetModuleHandle(string lpModuleName); +} +"@ + +Add-Type -TypeDefinition $source + +$standardCursors = @{ + arrow = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32512)) + text = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32513)) + wait = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32514)) + crosshair = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32515)) + 'up-arrow' = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32516)) + 'resize-nwse' = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32642)) + 'resize-nesw' = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32643)) + 'resize-ew' = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32644)) + 'resize-ns' = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32645)) + move = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32646)) + 'not-allowed' = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32648)) + pointer = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32649)) + 'app-starting' = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32650)) + help = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32651)) +} + +function Get-StandardCursorType($cursorHandle) { + if ($cursorHandle -eq [IntPtr]::Zero) { + return $null + } + + foreach ($entry in $standardCursors.GetEnumerator()) { + if ($entry.Value -eq $cursorHandle) { + return $entry.Key + } + } + + return $null +} + +function Write-JsonLine($payload) { + [Console]::Out.WriteLine(($payload | ConvertTo-Json -Compress -Depth 6)) +} + +function Get-CustomCursorType($bitmap, $hotspotX, $hotspotY) { + if ($bitmap.Width -lt 24 -or $bitmap.Height -lt 24 -or $bitmap.Width -gt 64 -or $bitmap.Height -gt 64) { + return $null + } + + if ($hotspotX -lt ($bitmap.Width * 0.25) -or $hotspotX -gt ($bitmap.Width * 0.75) -or + $hotspotY -lt ($bitmap.Height * 0.15) -or $hotspotY -gt ($bitmap.Height * 0.55)) { + return $null + } + + $opaquePixels = 0 + $topHalfOpaquePixels = 0 + $left = $bitmap.Width + $top = $bitmap.Height + $right = -1 + $bottom = -1 + + for ($y = 0; $y -lt $bitmap.Height; $y++) { + for ($x = 0; $x -lt $bitmap.Width; $x++) { + if ($bitmap.GetPixel($x, $y).A -le 32) { + continue + } + + $opaquePixels += 1 + if ($y -lt ($bitmap.Height / 2)) { + $topHalfOpaquePixels += 1 + } + if ($x -lt $left) { $left = $x } + if ($x -gt $right) { $right = $x } + if ($y -lt $top) { $top = $y } + if ($y -gt $bottom) { $bottom = $y } + } + } + + if ($opaquePixels -lt 90 -or $right -lt $left -or $bottom -lt $top) { + return $null + } + + $opaqueWidth = $right - $left + 1 + $opaqueHeight = $bottom - $top + 1 + if ($opaqueWidth -lt ($bitmap.Width * 0.35) -or $opaqueWidth -gt ($bitmap.Width * 0.9) -or + $opaqueHeight -lt ($bitmap.Height * 0.45) -or $opaqueHeight -gt $bitmap.Height) { + return $null + } + + if ($top -gt ($bitmap.Height * 0.45) -or $bottom -lt ($bitmap.Height * 0.65)) { + return $null + } + + if ($topHalfOpaquePixels -gt ($opaquePixels * 0.55)) { + return 'closed-hand' + } + + return 'open-hand' +} + +function Get-CursorAsset($cursorHandle, $cursorId) { + $copiedHandle = [OpenScreenCursorDiagnosticInterop]::CopyIcon($cursorHandle) + if ($copiedHandle -eq [IntPtr]::Zero) { + return $null + } + + $iconInfo = New-Object OpenScreenCursorDiagnosticInterop+ICONINFO + $hasIconInfo = [OpenScreenCursorDiagnosticInterop]::GetIconInfo($copiedHandle, [ref]$iconInfo) + + try { + $icon = [System.Drawing.Icon]::FromHandle($copiedHandle) + $bitmap = New-Object System.Drawing.Bitmap $icon.Width, $icon.Height, ([System.Drawing.Imaging.PixelFormat]::Format32bppArgb) + $graphics = [System.Drawing.Graphics]::FromImage($bitmap) + $memoryStream = New-Object System.IO.MemoryStream + + try { + $graphics.Clear([System.Drawing.Color]::Transparent) + $graphics.DrawIcon($icon, 0, 0) + $hotspotX = if ($hasIconInfo) { $iconInfo.xHotspot } else { 0 } + $hotspotY = if ($hasIconInfo) { $iconInfo.yHotspot } else { 0 } + $customCursorType = Get-CustomCursorType -bitmap $bitmap -hotspotX $hotspotX -hotspotY $hotspotY + $bitmap.Save($memoryStream, [System.Drawing.Imaging.ImageFormat]::Png) + $base64 = [System.Convert]::ToBase64String($memoryStream.ToArray()) + + return @{ + id = $cursorId + imageDataUrl = "data:image/png;base64,$base64" + width = $bitmap.Width + height = $bitmap.Height + hotspotX = $hotspotX + hotspotY = $hotspotY + cursorType = $customCursorType + } + } + finally { + $memoryStream.Dispose() + $graphics.Dispose() + $bitmap.Dispose() + $icon.Dispose() + } + } + finally { + if ($hasIconInfo) { + if ($iconInfo.hbmMask -ne [IntPtr]::Zero) { + [OpenScreenCursorDiagnosticInterop]::DeleteObject($iconInfo.hbmMask) | Out-Null + } + if ($iconInfo.hbmColor -ne [IntPtr]::Zero) { + [OpenScreenCursorDiagnosticInterop]::DeleteObject($iconInfo.hbmColor) | Out-Null + } + } + [OpenScreenCursorDiagnosticInterop]::DestroyIcon($copiedHandle) | Out-Null + } +} + +[OpenScreenCursorDiagnosticInterop]::InstallMouseHook() | Out-Null +[OpenScreenCursorDiagnosticInterop]::GetAsyncKeyState(0x01) | Out-Null +Write-JsonLine @{ type = 'ready'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() } + +$lastCursorId = $null +$screenBounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds +while ($true) { + [System.Windows.Forms.Application]::DoEvents() + $mouseEvents = [OpenScreenCursorDiagnosticInterop]::ConsumeMouseButtonEvents() + $cursorInfo = New-Object OpenScreenCursorDiagnosticInterop+CURSORINFO + $cursorInfo.cbSize = [Runtime.InteropServices.Marshal]::SizeOf([type][OpenScreenCursorDiagnosticInterop+CURSORINFO]) + + if (-not [OpenScreenCursorDiagnosticInterop]::GetCursorInfo([ref]$cursorInfo)) { + Write-JsonLine @{ type = 'error'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds(); message = 'GetCursorInfo failed' } + Start-Sleep -Milliseconds ${SAMPLE_INTERVAL_MS} + continue + } + + $visible = ($cursorInfo.flags -band 1) -ne 0 + $cursorId = if ($cursorInfo.hCursor -eq [IntPtr]::Zero) { $null } else { ('0x{0:X}' -f $cursorInfo.hCursor.ToInt64()) } + $cursorType = Get-StandardCursorType $cursorInfo.hCursor + $leftButtonState = [OpenScreenCursorDiagnosticInterop]::GetAsyncKeyState(0x01) + $leftButtonDown = ($leftButtonState -band 0x8000) -ne 0 + $leftButtonPressed = ($mouseEvents.LeftDownCount -gt 0) -or (($leftButtonState -band 0x0001) -ne 0) + $leftButtonReleased = $mouseEvents.LeftUpCount -gt 0 + $asset = $null + + if ($visible -and $cursorId -and $cursorId -ne $lastCursorId) { + $asset = Get-CursorAsset -cursorHandle $cursorInfo.hCursor -cursorId $cursorId + if ($asset -and $cursorType) { + $asset.cursorType = $cursorType + } elseif ($asset -and $asset.cursorType) { + $cursorType = $asset.cursorType + } + $lastCursorId = $cursorId + } + + Write-JsonLine @{ + type = 'sample' + timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + x = $cursorInfo.ptScreenPos.X + y = $cursorInfo.ptScreenPos.Y + visible = $visible + handle = $cursorId + cursorType = $cursorType + leftButtonDown = $leftButtonDown + leftButtonPressed = $leftButtonPressed + leftButtonReleased = $leftButtonReleased + bounds = @{ + x = $screenBounds.Left + y = $screenBounds.Top + width = $screenBounds.Width + height = $screenBounds.Height + } + asset = $asset + } + + Start-Sleep -Milliseconds ${SAMPLE_INTERVAL_MS} +} +`; +} + +function buildMousePathScript(durationMs) { + const stepMs = 120; + const steps = Math.max(8, Math.floor(durationMs / stepMs)); + + return String.raw` +$ErrorActionPreference = 'Stop' +Add-Type -AssemblyName System.Windows.Forms + +$source = @" +using System.Runtime.InteropServices; +using System; + +public static class OpenScreenMouseDiagnosticInterop { + [DllImport("user32.dll")] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool SetCursorPos(int X, int Y); + + [DllImport("user32.dll")] + public static extern void mouse_event(uint dwFlags, uint dx, uint dy, uint dwData, UIntPtr dwExtraInfo); +} +"@ + +Add-Type -TypeDefinition $source + +$bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds +$points = @() +for ($i = 0; $i -lt ${steps}; $i++) { + $t = if (${steps} -le 1) { 0 } else { $i / (${steps} - 1) } + $x = [int]($bounds.Left + 80 + (($bounds.Width - 160) * $t)) + $wave = [Math]::Sin($t * [Math]::PI * 2) + $y = [int]($bounds.Top + ($bounds.Height / 2) + ($wave * [Math]::Min(180, $bounds.Height / 4))) + $points += @{ x = $x; y = $y } +} + +for ($i = 0; $i -lt $points.Count; $i++) { + $point = $points[$i] + [OpenScreenMouseDiagnosticInterop]::SetCursorPos($point.x, $point.y) | Out-Null + if ($i -eq [int]([Math]::Floor($points.Count / 2))) { + [OpenScreenMouseDiagnosticInterop]::mouse_event(0x0002, 0, 0, 0, [UIntPtr]::Zero) + Start-Sleep -Milliseconds 12 + [OpenScreenMouseDiagnosticInterop]::mouse_event(0x0004, 0, 0, 0, [UIntPtr]::Zero) + } + Start-Sleep -Milliseconds ${stepMs} +} +`; +} + +function buildScreenRecorderScript(outputDir, durationMs) { + const framesDir = path.join(outputDir, "screen-frames"); + + return String.raw` +$ErrorActionPreference = 'Stop' +Add-Type -AssemblyName System.Drawing +Add-Type -AssemblyName System.Windows.Forms + +$framesDir = ${quotePowerShellString(framesDir)} +New-Item -ItemType Directory -Force -Path $framesDir | Out-Null + +$bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds +$targetWidth = 960 +$targetHeight = [int]([Math]::Round($targetWidth * ($bounds.Height / $bounds.Width))) +$frames = New-Object System.Collections.Generic.List[object] +$stopwatch = [System.Diagnostics.Stopwatch]::StartNew() +$index = 0 + +while ($stopwatch.ElapsedMilliseconds -le ${durationMs + 700}) { + $sourceBitmap = New-Object System.Drawing.Bitmap $bounds.Width, $bounds.Height, ([System.Drawing.Imaging.PixelFormat]::Format32bppArgb) + $graphics = [System.Drawing.Graphics]::FromImage($sourceBitmap) + $scaledBitmap = New-Object System.Drawing.Bitmap $targetWidth, $targetHeight, ([System.Drawing.Imaging.PixelFormat]::Format32bppArgb) + $scaledGraphics = [System.Drawing.Graphics]::FromImage($scaledBitmap) + $timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + $fileName = ('frame_{0:D4}.png' -f $index) + $path = Join-Path $framesDir $fileName + + try { + $graphics.CopyFromScreen($bounds.Left, $bounds.Top, 0, 0, $bounds.Size) + $scaledGraphics.InterpolationMode = [System.Drawing.Drawing2D.InterpolationMode]::HighQualityBicubic + $scaledGraphics.DrawImage($sourceBitmap, 0, 0, $targetWidth, $targetHeight) + $scaledBitmap.Save($path, [System.Drawing.Imaging.ImageFormat]::Png) + $frames.Add(@{ + index = $index + timestampMs = $timestampMs + path = $path + width = $targetWidth + height = $targetHeight + bounds = @{ + x = $bounds.Left + y = $bounds.Top + width = $bounds.Width + height = $bounds.Height + } + }) | Out-Null + } + finally { + $scaledGraphics.Dispose() + $scaledBitmap.Dispose() + $graphics.Dispose() + $sourceBitmap.Dispose() + } + + $index += 1 + Start-Sleep -Milliseconds ${SCREEN_FRAME_INTERVAL_MS} +} + +($frames | ConvertTo-Json -Depth 6) | Set-Content -Path (Join-Path $framesDir 'frames.json') -Encoding UTF8 +`; +} + +function createReadyWaiter() { + let settled = false; + let resolveReady = null; + const promise = new Promise((resolve, reject) => { + const timer = setTimeout(() => { + if (settled) { + return; + } + settled = true; + reject(new Error("Timed out waiting for cursor sampler readiness.")); + }, READY_TIMEOUT_MS); + + resolveReady = () => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + resolve(); + }; + }); + + return { + promise, + resolve: () => resolveReady?.(), + }; +} + +function writeAssets(assets, outputDir) { + const assetDir = path.join(outputDir, "assets"); + fs.mkdirSync(assetDir, { recursive: true }); + + for (const asset of assets.values()) { + const base64 = asset.imageDataUrl?.replace(/^data:image\/png;base64,/, ""); + if (!base64) { + continue; + } + + const safeId = String(asset.id).replace(/[^a-zA-Z0-9_-]/g, "_"); + fs.writeFileSync(path.join(assetDir, `${safeId}.png`), Buffer.from(base64, "base64")); + } +} + +function toRecordingData(samples, assets) { + const firstTimestampMs = samples[0]?.timestampMs ?? Date.now(); + let previousLeftButtonDown = false; + const normalizedSamples = samples.flatMap((sample) => { + const bounds = sample.bounds; + if (!bounds || bounds.width <= 0 || bounds.height <= 0) { + return []; + } + + const leftButtonDown = sample.leftButtonDown === true; + const leftButtonPressed = sample.leftButtonPressed === true; + const leftButtonReleased = sample.leftButtonReleased === true; + const interactionType = + leftButtonPressed || (leftButtonDown && !previousLeftButtonDown) + ? "click" + : leftButtonReleased || (!leftButtonDown && previousLeftButtonDown) + ? "mouseup" + : "move"; + previousLeftButtonDown = leftButtonDown; + + return [ + { + timeMs: Math.max(0, sample.timestampMs - firstTimestampMs), + cx: (sample.x - bounds.x) / bounds.width, + cy: (sample.y - bounds.y) / bounds.height, + assetId: sample.handle, + visible: Boolean(sample.visible), + cursorType: sample.cursorType ?? null, + interactionType, + }, + ]; + }); + + return { + version: 2, + provider: assets.size > 0 ? "native" : "none", + samples: normalizedSamples, + assets: [...assets.values()].map((asset) => ({ + id: asset.id, + platform: "win32", + imageDataUrl: asset.imageDataUrl, + width: asset.width, + height: asset.height, + hotspotX: asset.hotspotX, + hotspotY: asset.hotspotY, + scaleFactor: 1, + cursorType: asset.cursorType ?? null, + })), + }; +} + +function escapeScriptJson(value) { + return JSON.stringify(value).replace(/ + + + + +OpenScreen native cursor diagnostic + + + +
+

OpenScreen native cursor diagnostic

+
+
${report.sampleCount}samples
+
${report.assetCount}assets
+
${report.uniquePositionCount}positions
+
${report.errorCount}errors
+
+

The red cross is the captured native hotspot. Native bitmaps are drawn at 1x, 2x, and 3x. The last cursor is a crisp vector 3x replacement anchored on the same hotspot.

+ +
+
+ + + +`; +} + +function readScreenFrames(outputDir, recordingStartTimestampMs) { + const framesJsonPath = path.join(outputDir, "screen-frames", "frames.json"); + if (!fs.existsSync(framesJsonPath)) { + return []; + } + + const rawFrames = JSON.parse(fs.readFileSync(framesJsonPath, "utf8").replace(/^\uFEFF/, "")); + const frames = Array.isArray(rawFrames) ? rawFrames : [rawFrames]; + + return frames + .filter((frame) => frame?.path && fs.existsSync(frame.path)) + .map((frame) => ({ + ...frame, + timeMs: Math.max(0, frame.timestampMs - recordingStartTimestampMs), + imageDataUrl: `data:image/png;base64,${fs.readFileSync(frame.path).toString("base64")}`, + })); +} + +function buildRealCaptureHtml(report, recordingData, screenFrames) { + return ` + + + + +OpenScreen native cursor real capture diagnostic + + + +
+

Real screen capture + reconstructed native cursor

+

Background frames are real Windows screenshots. Native bitmaps are reconstructed at 1x, 2x, and 3x; the last cursor is a crisp vector 3x replacement. The red cross marks the recorded hotspot.

+ +
+ + + + +`; +} + +function findPlaywrightChromiumExecutable(defaultPath) { + if (fs.existsSync(defaultPath)) { + return defaultPath; + } + + const baseDir = path.join(process.env.LOCALAPPDATA ?? "", "ms-playwright"); + if (!baseDir || !fs.existsSync(baseDir)) { + return defaultPath; + } + + const candidates = fs + .readdirSync(baseDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && entry.name.startsWith("chromium-")) + .map((entry) => ({ + executablePath: path.join(baseDir, entry.name, "chrome-win64", "chrome.exe"), + revision: Number.parseInt(entry.name.slice("chromium-".length), 10), + })) + .filter( + (candidate) => Number.isFinite(candidate.revision) && fs.existsSync(candidate.executablePath), + ) + .sort((a, b) => b.revision - a.revision) + .map((candidate) => candidate.executablePath); + + return candidates[0] ?? defaultPath; +} + +async function writePreviewVideo(reportPath, outputPath) { + const { chromium } = await import("playwright"); + const browser = await chromium.launch({ + executablePath: findPlaywrightChromiumExecutable(chromium.executablePath()), + headless: true, + }); + try { + const page = await browser.newPage({ viewport: { width: 1180, height: 760 } }); + await page.goto(`file:///${reportPath.replaceAll("\\", "/")}`); + const base64 = await page.evaluate(() => window.__exportWebm()); + fs.writeFileSync(outputPath, Buffer.from(base64, "base64")); + } finally { + await browser.close(); + } +} + +function assertReport(report) { + const failures = []; + if (report.sampleCount < Math.floor(DURATION_MS / SAMPLE_INTERVAL_MS / 3)) { + failures.push(`Too few samples: ${report.sampleCount}.`); + } + if (report.visibleSampleCount === 0) { + failures.push("No visible cursor samples were captured."); + } + if (report.assetCount === 0) { + failures.push("No cursor asset PNG was captured."); + } + if (report.uniquePositionCount < 4) { + failures.push(`Cursor movement was not observed enough times: ${report.uniquePositionCount}.`); + } + if (report.errorCount > 0) { + failures.push(`Sampler reported ${report.errorCount} error event(s).`); + } + if (report.leftButtonPressedSampleCount === 0 || report.clickSampleCount === 0) { + failures.push("Left button click interaction was not observed."); + } + + if (failures.length > 0) { + throw new Error(failures.join(" ")); + } +} + +fs.mkdirSync(OUTPUT_DIR, { recursive: true }); + +const events = []; +const assets = new Map(); +let lineBuffer = ""; +let stoppingSampler = false; +const readyWaiter = createReadyWaiter(); +const sampler = spawnPowerShell(buildSamplerScript(), { + onStdout: (chunk) => { + lineBuffer += chunk; + const lines = lineBuffer.split(/\r?\n/); + lineBuffer = lines.pop() ?? ""; + + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed) { + continue; + } + + let event; + try { + event = JSON.parse(trimmed); + } catch { + process.stderr.write(`[cursor-native-test] dropping non-JSON line: ${trimmed}\n`); + continue; + } + events.push(event); + if (event.type === "ready") { + readyWaiter.resolve(); + } + if (event.asset?.id && !assets.has(event.asset.id)) { + assets.set(event.asset.id, event.asset); + } + } + }, + onStderr: (chunk) => { + if (!stoppingSampler && !chunk.startsWith("#< CLIXML")) { + process.stderr.write(`[cursor-native-test] ${chunk}`); + } + }, +}); +let screenRecorder = null; + +try { + await readyWaiter.promise; + screenRecorder = spawnPowerShell(buildScreenRecorderScript(OUTPUT_DIR, DURATION_MS), { + onStderr: (chunk) => { + if (!chunk.startsWith("#< CLIXML") && !chunk.startsWith(" setTimeout(resolve, 150)); + await runPowerShell(buildMousePathScript(DURATION_MS)); + await new Promise((resolve) => setTimeout(resolve, Math.max(250, SAMPLE_INTERVAL_MS * 3))); + await screenRecorder.done; +} finally { + if (!sampler.child.killed) { + stoppingSampler = true; + sampler.child.kill(); + } + if (screenRecorder && !screenRecorder.child.killed) { + screenRecorder.child.kill(); + } +} + +const samples = events.filter((event) => event.type === "sample"); +const errors = events.filter((event) => event.type === "error"); +const recordingStartTimestampMs = samples[0]?.timestampMs ?? Date.now(); +const uniquePositions = new Set(samples.map((sample) => `${sample.x},${sample.y}`)); +let previousLeftButtonDown = false; +let clickSampleCount = 0; +for (const sample of samples) { + const leftButtonDown = sample.leftButtonDown === true; + const leftButtonPressed = sample.leftButtonPressed === true; + if (leftButtonPressed || (leftButtonDown && !previousLeftButtonDown)) { + clickSampleCount += 1; + } + previousLeftButtonDown = leftButtonDown; +} +const report = { + outputDir: OUTPUT_DIR, + sampleIntervalMs: SAMPLE_INTERVAL_MS, + durationMs: DURATION_MS, + eventCount: events.length, + sampleCount: samples.length, + visibleSampleCount: samples.filter((sample) => sample.visible).length, + assetCount: assets.size, + uniqueCursorHandleCount: new Set(samples.map((sample) => sample.handle).filter(Boolean)).size, + uniquePositionCount: uniquePositions.size, + leftButtonDownSampleCount: samples.filter((sample) => sample.leftButtonDown === true).length, + leftButtonPressedSampleCount: samples.filter((sample) => sample.leftButtonPressed === true) + .length, + clickSampleCount, + errorCount: errors.length, + firstSample: samples[0] ?? null, + lastSample: samples.at(-1) ?? null, + assets: [...assets.values()].map((asset) => ({ + id: asset.id, + width: asset.width, + height: asset.height, + hotspotX: asset.hotspotX, + hotspotY: asset.hotspotY, + cursorType: asset.cursorType ?? null, + })), +}; +const recordingData = toRecordingData(samples, assets); +const screenFrames = readScreenFrames(OUTPUT_DIR, recordingStartTimestampMs); +const reportHtmlPath = path.join(OUTPUT_DIR, "report.html"); +const previewVideoPath = path.join(OUTPUT_DIR, "preview.webm"); +const realCaptureHtmlPath = path.join(OUTPUT_DIR, "real-capture-report.html"); +const realCaptureVideoPath = path.join(OUTPUT_DIR, "real-capture-preview.webm"); + +writeAssets(assets, OUTPUT_DIR); +fs.writeFileSync(path.join(OUTPUT_DIR, "events.json"), JSON.stringify(events, null, 2)); +fs.writeFileSync( + path.join(OUTPUT_DIR, "cursor-recording-data.json"), + JSON.stringify(recordingData, null, 2), +); +fs.writeFileSync(path.join(OUTPUT_DIR, "report.json"), JSON.stringify(report, null, 2)); +fs.writeFileSync(reportHtmlPath, buildVisualReportHtml(report, recordingData)); +if (screenFrames.length > 0) { + fs.writeFileSync(realCaptureHtmlPath, buildRealCaptureHtml(report, recordingData, screenFrames)); + report.screenFrameCount = screenFrames.length; +} + +try { + await writePreviewVideo(reportHtmlPath, previewVideoPath); + report.previewVideoPath = previewVideoPath; +} catch (error) { + report.previewVideoError = error instanceof Error ? error.message : String(error); +} + +if (screenFrames.length > 0) { + try { + await writePreviewVideo(realCaptureHtmlPath, realCaptureVideoPath); + report.realCaptureVideoPath = realCaptureVideoPath; + } catch (error) { + report.realCaptureVideoError = error instanceof Error ? error.message : String(error); + } +} + +fs.writeFileSync(path.join(OUTPUT_DIR, "report.json"), JSON.stringify(report, null, 2)); + +assertReport(report); + +console.log(JSON.stringify(report, null, 2)); diff --git a/scripts/test-windows-wgc-helper.mjs b/scripts/test-windows-wgc-helper.mjs new file mode 100644 index 0000000..5dd2dcc --- /dev/null +++ b/scripts/test-windows-wgc-helper.mjs @@ -0,0 +1,387 @@ +import { spawn, spawnSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.join(__dirname, ".."); +const HELPER_PATH = + process.env.OPENSCREEN_WGC_CAPTURE_EXE ?? + path.join(ROOT, "electron", "native", "bin", "win32-x64", "wgc-capture.exe"); + +const DURATION_MS = Number(process.env.OPENSCREEN_WGC_TEST_DURATION_MS ?? 5000); +const WITH_SYSTEM_AUDIO = + process.env.OPENSCREEN_WGC_TEST_SYSTEM_AUDIO === "true" || + process.argv.includes("--system-audio"); +const WITH_MICROPHONE = + process.env.OPENSCREEN_WGC_TEST_MICROPHONE === "true" || + process.argv.includes("--microphone") || + process.argv.includes("--mic"); +const WITH_WINDOW = + process.env.OPENSCREEN_WGC_TEST_WINDOW === "true" || process.argv.includes("--window"); +const WITH_WEBCAM = + process.env.OPENSCREEN_WGC_TEST_WEBCAM === "true" || process.argv.includes("--webcam"); +const CAPTURE_CURSOR = + process.env.OPENSCREEN_WGC_TEST_CAPTURE_CURSOR === "true" || + process.argv.includes("--capture-cursor"); + +function runHelper(config) { + return new Promise((resolve, reject) => { + const child = spawn(HELPER_PATH, [JSON.stringify(config)], { + stdio: ["pipe", "pipe", "pipe"], + windowsHide: true, + }); + + let stdout = ""; + let stderr = ""; + let stopTimer = null; + const scheduleStop = () => { + if (stopTimer) { + return; + } + stopTimer = setTimeout(() => { + child.stdin.write("stop\n"); + }, DURATION_MS); + }; + const fallbackTimer = setTimeout(scheduleStop, 15_000); + + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + if (stdout.includes('"recording-started"') || stdout.includes("Recording started")) { + scheduleStop(); + } + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + child.once("error", reject); + child.once("exit", (code) => { + clearTimeout(fallbackTimer); + if (stopTimer) { + clearTimeout(stopTimer); + } + resolve({ code, stdout, stderr }); + }); + }); +} + +function startFixtureWindow() { + return new Promise((resolve, reject) => { + const child = spawn("mspaint.exe", [], { + stdio: ["ignore", "ignore", "ignore"], + windowsHide: false, + }); + + const poll = setInterval(() => { + const lookup = spawnSync( + "powershell", + [ + "-NoProfile", + "-Command", + `(Get-Process -Id ${child.pid} -ErrorAction SilentlyContinue).MainWindowHandle`, + ], + { encoding: "utf8", windowsHide: true }, + ); + const handle = lookup.stdout + .trim() + .split(/\r?\n/) + .find((line) => /^\d+$/.test(line.trim())); + if (handle && handle !== "0") { + clearInterval(poll); + clearTimeout(timer); + resolve({ child, sourceId: `window:${handle.trim()}:0` }); + } + }, 250); + + const timer = setTimeout(() => { + clearInterval(poll); + child.kill(); + reject(new Error("Timed out waiting for fixture window handle")); + }, 10_000); + child.once("error", (error) => { + clearInterval(poll); + clearTimeout(timer); + reject(error); + }); + }); +} + +function normalizeDeviceName(value) { + return value + .toLowerCase() + .replace(/[^a-z0-9]+/g, " ") + .trim(); +} + +function scoreDeviceName(candidateName, candidateId, requestedName) { + const candidate = normalizeDeviceName(candidateName ?? ""); + const id = normalizeDeviceName(candidateId ?? ""); + const requested = normalizeDeviceName(requestedName ?? ""); + if (!requested) return 0; + if (candidate === requested) return 1000; + if (candidate.includes(requested) || requested.includes(candidate)) return 900; + if (id.includes(requested) || requested.includes(id)) return 800; + return requested + .split(/\s+/) + .filter((word) => word.length > 1 && !["camera", "webcam", "video", "input"].includes(word)) + .reduce((score, word) => { + if (candidate.includes(word)) return score + 100; + if (id.includes(word)) return score + 50; + return score; + }, 0); +} + +function resolveDirectShowWebcamClsid(requestedName) { + if (!requestedName) return ""; + const query = spawnSync( + "reg.exe", + ["query", "HKCR\\CLSID\\{860BB310-5D01-11D0-BD3B-00A0C911CE86}\\Instance", "/s"], + { encoding: "utf8", windowsHide: true }, + ); + if (query.status !== 0) return ""; + const entries = []; + let current = {}; + for (const rawLine of query.stdout.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line) continue; + if (/^HKEY_/i.test(line)) { + if (current.friendlyName || current.clsid) entries.push(current); + current = {}; + continue; + } + const match = line.match(/^(\S+)\s+REG_SZ\s+(.+)$/); + if (!match) continue; + if (match[1] === "FriendlyName") current.friendlyName = match[2].trim(); + if (match[1] === "CLSID") current.clsid = match[2].trim(); + } + if (current.friendlyName || current.clsid) entries.push(current); + + let best = null; + for (const entry of entries) { + if (!entry.clsid) continue; + const score = scoreDeviceName(entry.friendlyName, entry.clsid, requestedName); + if (!best || score > best.score) { + best = { ...entry, score }; + } + } + return best && best.score > 0 ? best.clsid : ""; +} + +function probeStreams(outputPath) { + const ffprobe = spawnSync( + "ffprobe", + ["-v", "error", "-show_streams", "-of", "json", outputPath], + { encoding: "utf8", windowsHide: true }, + ); + if (ffprobe.status !== 0) { + throw new Error(`ffprobe failed: ${ffprobe.stderr || ffprobe.stdout}`); + } + return JSON.parse(ffprobe.stdout).streams ?? []; +} + +function measureFirstFrameLuma(outputPath) { + const ffmpeg = spawnSync( + "ffmpeg", + [ + "-v", + "error", + "-i", + outputPath, + "-frames:v", + "1", + "-f", + "rawvideo", + "-pix_fmt", + "gray", + "pipe:1", + ], + { windowsHide: true, maxBuffer: 64 * 1024 * 1024 }, + ); + if (ffmpeg.status !== 0) { + throw new Error(`ffmpeg frame extraction failed: ${ffmpeg.stderr?.toString() ?? ""}`); + } + const data = ffmpeg.stdout; + if (!data || data.length === 0) { + throw new Error(`ffmpeg did not return frame data for ${outputPath}`); + } + let sum = 0; + let max = 0; + for (const value of data) { + sum += value; + if (value > max) { + max = value; + } + } + return { average: sum / data.length, max }; +} + +if (process.platform !== "win32") { + console.log("Skipping WGC helper smoke test: Windows-only."); + process.exit(0); +} + +if (!fs.existsSync(HELPER_PATH)) { + throw new Error(`WGC helper not found at ${HELPER_PATH}. Run npm run build:native:win first.`); +} + +const outputPath = path.join( + os.tmpdir(), + `openscreen-wgc-helper-${WITH_WEBCAM ? "webcam" : WITH_WINDOW ? "window" : WITH_SYSTEM_AUDIO || WITH_MICROPHONE ? "audio" : "video"}-${process.pid}-${Date.now()}-${randomUUID()}.mp4`, +); +const webcamOutputPath = WITH_WEBCAM ? outputPath.replace(/\.mp4$/i, "-webcam.mp4") : null; + +const fixtureWindow = WITH_WINDOW ? await startFixtureWindow() : null; + +const config = { + schemaVersion: 2, + recordingId: Date.now(), + outputPath, + sourceType: fixtureWindow ? "window" : "display", + sourceId: fixtureWindow ? fixtureWindow.sourceId : "screen:0:0", + displayId: 0, + fps: 30, + videoWidth: 1280, + videoHeight: 720, + displayX: 0, + displayY: 0, + displayW: 1920, + displayH: 1080, + hasDisplayBounds: true, + captureSystemAudio: WITH_SYSTEM_AUDIO, + captureMic: WITH_MICROPHONE, + captureCursor: CAPTURE_CURSOR, + microphoneDeviceId: process.env.OPENSCREEN_WGC_TEST_MICROPHONE_DEVICE_ID ?? "default", + microphoneDeviceName: process.env.OPENSCREEN_WGC_TEST_MICROPHONE_DEVICE_NAME ?? "", + microphoneGain: 1.4, + webcamEnabled: WITH_WEBCAM, + webcamDeviceId: process.env.OPENSCREEN_WGC_TEST_WEBCAM_DEVICE_ID ?? "", + webcamDeviceName: process.env.OPENSCREEN_WGC_TEST_WEBCAM_DEVICE_NAME ?? "", + webcamDirectShowClsid: resolveDirectShowWebcamClsid( + process.env.OPENSCREEN_WGC_TEST_WEBCAM_DEVICE_NAME ?? "", + ), + webcamWidth: 640, + webcamHeight: 360, + webcamFps: 30, + outputs: { + screenPath: outputPath, + ...(webcamOutputPath ? { webcamPath: webcamOutputPath } : {}), + }, +}; + +let result; +try { + result = await runHelper(config); +} finally { + if (fixtureWindow) { + fixtureWindow.child.kill(); + } +} +if (result.code !== 0) { + if ( + WITH_WEBCAM && + /No native Windows webcam devices were found|Failed to initialize native webcam/.test( + result.stderr, + ) + ) { + console.log("Skipping WGC webcam smoke test: no native Windows webcam device is available."); + process.exit(0); + } + throw new Error(`WGC helper exited with ${result.code}\n${result.stdout}\n${result.stderr}`); +} +if (!fs.existsSync(outputPath) || fs.statSync(outputPath).size === 0) { + throw new Error(`WGC helper did not produce a video at ${outputPath}`); +} +if (WITH_WEBCAM && (!fs.existsSync(webcamOutputPath) || fs.statSync(webcamOutputPath).size === 0)) { + throw new Error(`WGC helper did not produce a webcam video at ${webcamOutputPath}`); +} + +const streams = probeStreams(outputPath); +const webcamStreams = + webcamOutputPath && fs.existsSync(webcamOutputPath) ? probeStreams(webcamOutputPath) : []; +const hasVideo = streams.some((stream) => stream.codec_type === "video"); +const hasAudio = streams.some((stream) => stream.codec_type === "audio"); +const webcamFormatLine = result.stdout + .split(/\r?\n/) + .find((line) => line.includes('"event":"webcam-format"')); +const webcamFormat = webcamFormatLine ? JSON.parse(webcamFormatLine) : null; +const audioFormatLine = result.stdout + .split(/\r?\n/) + .find((line) => line.includes('"event":"audio-format"')); +const audioFormat = audioFormatLine ? JSON.parse(audioFormatLine) : null; +const cursorCaptureLine = result.stdout + .split(/\r?\n/) + .find((line) => line.includes('"event":"cursor-capture"')); +const cursorCapture = cursorCaptureLine ? JSON.parse(cursorCaptureLine) : null; +const nativeWebcamDiagnostics = result.stderr + .split(/\r?\n/) + .filter((line) => line.includes("Native webcam candidate")); +const nativeMicrophoneDiagnostics = result.stderr + .split(/\r?\n/) + .filter( + (line) => + line.includes("Native microphone candidate") || + line.includes("Selected native microphone endpoint"), + ); +if (!hasVideo) { + throw new Error(`WGC helper output has no video stream: ${outputPath}`); +} +if (WITH_WEBCAM && !webcamStreams.some((stream) => stream.codec_type === "video")) { + throw new Error(`WGC helper webcam output has no video stream: ${webcamOutputPath}`); +} +if ( + (CAPTURE_CURSOR && !cursorCapture) || + (cursorCapture && + (cursorCapture.requested !== CAPTURE_CURSOR || cursorCapture.applied !== CAPTURE_CURSOR)) +) { + throw new Error( + `WGC helper did not apply requested cursor capture mode (${CAPTURE_CURSOR}): ${result.stdout}`, + ); +} +if ((WITH_SYSTEM_AUDIO || WITH_MICROPHONE) && !hasAudio) { + throw new Error(`WGC helper output has no audio stream: ${outputPath}`); +} +const frameLuma = measureFirstFrameLuma(outputPath); +if (frameLuma.average < 1 && frameLuma.max < 5) { + throw new Error( + `WGC helper output first frame is black: ${outputPath}\n${result.stdout}\n${result.stderr}`, + ); +} + +console.log( + JSON.stringify( + { + success: true, + outputPath, + webcamOutputPath, + bytes: fs.statSync(outputPath).size, + webcamBytes: + webcamOutputPath && fs.existsSync(webcamOutputPath) + ? fs.statSync(webcamOutputPath).size + : undefined, + streams: streams.map((stream) => ({ + index: stream.index, + codecType: stream.codec_type, + codecName: stream.codec_name, + duration: stream.duration, + })), + webcamStreams: webcamStreams.map((stream) => ({ + index: stream.index, + codecType: stream.codec_type, + codecName: stream.codec_name, + width: stream.width, + height: stream.height, + duration: stream.duration, + })), + cursorCapture, + selectedMicrophoneDeviceName: audioFormat?.microphoneDeviceName, + selectedWebcamDeviceName: webcamFormat?.deviceName, + nativeMicrophoneDiagnostics, + nativeWebcamDiagnostics, + firstFrameLuma: frameLuma, + }, + null, + 2, + ), +); diff --git a/src/assets/cursors/Cursor=App-Starting.svg b/src/assets/cursors/Cursor=App-Starting.svg new file mode 100644 index 0000000..7a10d40 --- /dev/null +++ b/src/assets/cursors/Cursor=App-Starting.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/src/assets/cursors/Cursor=Beachball.svg b/src/assets/cursors/Cursor=Beachball.svg new file mode 100644 index 0000000..30bdbe5 --- /dev/null +++ b/src/assets/cursors/Cursor=Beachball.svg @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/assets/cursors/Cursor=Cross.svg b/src/assets/cursors/Cursor=Cross.svg new file mode 100644 index 0000000..b404553 --- /dev/null +++ b/src/assets/cursors/Cursor=Cross.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Default.svg b/src/assets/cursors/Cursor=Default.svg new file mode 100644 index 0000000..f76f31f --- /dev/null +++ b/src/assets/cursors/Cursor=Default.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Hand-(Grabbing).svg b/src/assets/cursors/Cursor=Hand-(Grabbing).svg new file mode 100644 index 0000000..0827867 --- /dev/null +++ b/src/assets/cursors/Cursor=Hand-(Grabbing).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Hand-(Open).svg b/src/assets/cursors/Cursor=Hand-(Open).svg new file mode 100644 index 0000000..4ceafb0 --- /dev/null +++ b/src/assets/cursors/Cursor=Hand-(Open).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Hand-(Pointing).svg b/src/assets/cursors/Cursor=Hand-(Pointing).svg new file mode 100644 index 0000000..19a70a6 --- /dev/null +++ b/src/assets/cursors/Cursor=Hand-(Pointing).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Help.svg b/src/assets/cursors/Cursor=Help.svg new file mode 100644 index 0000000..d187c52 --- /dev/null +++ b/src/assets/cursors/Cursor=Help.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/src/assets/cursors/Cursor=Menu.svg b/src/assets/cursors/Cursor=Menu.svg new file mode 100644 index 0000000..3489257 --- /dev/null +++ b/src/assets/cursors/Cursor=Menu.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/src/assets/cursors/Cursor=Move.svg b/src/assets/cursors/Cursor=Move.svg new file mode 100644 index 0000000..50e56b7 --- /dev/null +++ b/src/assets/cursors/Cursor=Move.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Not-Allowed.svg b/src/assets/cursors/Cursor=Not-Allowed.svg new file mode 100644 index 0000000..8b2c3f8 --- /dev/null +++ b/src/assets/cursors/Cursor=Not-Allowed.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/src/assets/cursors/Cursor=Resize-(Down).svg b/src/assets/cursors/Cursor=Resize-(Down).svg new file mode 100644 index 0000000..fba3672 --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-(Down).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-(Left).svg b/src/assets/cursors/Cursor=Resize-(Left).svg new file mode 100644 index 0000000..6e21fb7 --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-(Left).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-(Left-Right).svg b/src/assets/cursors/Cursor=Resize-(Left-Right).svg new file mode 100644 index 0000000..7021d22 --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-(Left-Right).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-(Right).svg b/src/assets/cursors/Cursor=Resize-(Right).svg new file mode 100644 index 0000000..1ce801c --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-(Right).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-(Up).svg b/src/assets/cursors/Cursor=Resize-(Up).svg new file mode 100644 index 0000000..9c4ac0f --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-(Up).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-(Up-Down).svg b/src/assets/cursors/Cursor=Resize-(Up-Down).svg new file mode 100644 index 0000000..b01a40e --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-(Up-Down).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-North-East-South-West.svg b/src/assets/cursors/Cursor=Resize-North-East-South-West.svg new file mode 100644 index 0000000..1185c1f --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-North-East-South-West.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-North-South.svg b/src/assets/cursors/Cursor=Resize-North-South.svg new file mode 100644 index 0000000..57eaa05 --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-North-South.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-North-West-South-East.svg b/src/assets/cursors/Cursor=Resize-North-West-South-East.svg new file mode 100644 index 0000000..f00fc87 --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-North-West-South-East.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-West-East.svg b/src/assets/cursors/Cursor=Resize-West-East.svg new file mode 100644 index 0000000..ef1929f --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-West-East.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Text-Cursor.svg b/src/assets/cursors/Cursor=Text-Cursor.svg new file mode 100644 index 0000000..1bfd080 --- /dev/null +++ b/src/assets/cursors/Cursor=Text-Cursor.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Up-Arrow.svg b/src/assets/cursors/Cursor=Up-Arrow.svg new file mode 100644 index 0000000..b742e70 --- /dev/null +++ b/src/assets/cursors/Cursor=Up-Arrow.svg @@ -0,0 +1,4 @@ + + + + diff --git a/src/assets/cursors/Cursor=Wait.svg b/src/assets/cursors/Cursor=Wait.svg new file mode 100644 index 0000000..2b56934 --- /dev/null +++ b/src/assets/cursors/Cursor=Wait.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/src/assets/cursors/Cursor=Zoom-In.svg b/src/assets/cursors/Cursor=Zoom-In.svg new file mode 100644 index 0000000..8ec9b3c --- /dev/null +++ b/src/assets/cursors/Cursor=Zoom-In.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/src/assets/cursors/Cursor=Zoom-Out.svg b/src/assets/cursors/Cursor=Zoom-Out.svg new file mode 100644 index 0000000..810878b --- /dev/null +++ b/src/assets/cursors/Cursor=Zoom-Out.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index 6a14fc0..570ec28 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -1,5 +1,5 @@ import { Check, ChevronDown, Languages } from "lucide-react"; -import { useEffect, useRef, useState } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; import { createPortal } from "react-dom"; import { BsPauseCircle, BsPlayCircle, BsRecordCircle } from "react-icons/bs"; import { FaRegStopCircle } from "react-icons/fa"; @@ -10,6 +10,7 @@ import { MdMic, MdMicOff, MdMonitor, + MdMouse, MdRestartAlt, MdVideocam, MdVideocamOff, @@ -20,6 +21,7 @@ import { import { RxDragHandleDots2 } from "react-icons/rx"; import { useI18n, useScopedT } from "@/contexts/I18nContext"; import { getAvailableLocales, getLocaleName } from "@/i18n/loader"; +import { nativeBridgeClient } from "@/native"; import { useAudioLevelMeter } from "../../hooks/useAudioLevelMeter"; import { useCameraDevices } from "../../hooks/useCameraDevices"; import { useMicrophoneDevices } from "../../hooks/useMicrophoneDevices"; @@ -42,6 +44,7 @@ const ICON_CONFIG = { micOff: { icon: MdMicOff, size: ICON_SIZE }, webcamOn: { icon: MdVideocam, size: ICON_SIZE }, webcamOff: { icon: MdVideocamOff, size: ICON_SIZE }, + cursor: { icon: MdMouse, size: ICON_SIZE }, pause: { icon: BsPauseCircle, size: ICON_SIZE }, resume: { icon: BsPlayCircle, size: ICON_SIZE }, stop: { icon: FaRegStopCircle, size: ICON_SIZE }, @@ -95,18 +98,23 @@ export function LaunchWindow() { elapsedSeconds, toggleRecording, togglePaused, + canPauseRecording, restartRecording, cancelRecording, microphoneEnabled, setMicrophoneEnabled, microphoneDeviceId, setMicrophoneDeviceId, + setMicrophoneDeviceName, systemAudioEnabled, setSystemAudioEnabled, webcamEnabled, setWebcamEnabled, webcamDeviceId, setWebcamDeviceId, + setWebcamDeviceName, + cursorCaptureMode, + setCursorCaptureMode, } = useScreenRecorder(); const showMicControls = microphoneEnabled && !recording; @@ -120,6 +128,7 @@ export function LaunchWindow() { const [isWebcamFocused, setIsWebcamFocused] = useState(false); const webcamExpanded = isWebcamHovered || isWebcamFocused; const [isLanguageMenuOpen, setIsLanguageMenuOpen] = useState(false); + const [supportsCursorModeToggle, setSupportsCursorModeToggle] = useState(false); const languageTriggerRef = useRef(null); const languageMenuPanelRef = useRef(null); const [languageMenuStyle, setLanguageMenuStyle] = useState<{ @@ -148,14 +157,16 @@ export function LaunchWindow() { const selectedMicLabel = micDevices.find((d) => d.deviceId === (microphoneDeviceId || selectedMicId))?.label || t("audio.defaultMicrophone"); + const selectedCameraDevice = cameraDevices.find( + (d) => d.deviceId === (webcamDeviceId || selectedCameraId), + ); const selectedCameraLabel = isCameraDevicesLoading ? t("webcam.searching") : cameraDevicesError ? t("webcam.unavailable") : cameraDevices.length === 0 ? t("webcam.noneFound") - : cameraDevices.find((d) => d.deviceId === (webcamDeviceId || selectedCameraId))?.label || - t("webcam.defaultCamera"); + : selectedCameraDevice?.label || t("webcam.defaultCamera"); const { level } = useAudioLevelMeter({ enabled: showMicControls, @@ -165,14 +176,36 @@ export function LaunchWindow() { useEffect(() => { if (selectedMicId && selectedMicId !== "default") { setMicrophoneDeviceId(selectedMicId); + setMicrophoneDeviceName(micDevices.find((d) => d.deviceId === selectedMicId)?.label); } - }, [selectedMicId, setMicrophoneDeviceId]); + }, [selectedMicId, micDevices, setMicrophoneDeviceId, setMicrophoneDeviceName]); useEffect(() => { if (selectedCameraId) { setWebcamDeviceId(selectedCameraId); + setWebcamDeviceName(cameraDevices.find((d) => d.deviceId === selectedCameraId)?.label); } - }, [selectedCameraId, setWebcamDeviceId]); + }, [selectedCameraId, cameraDevices, setWebcamDeviceId, setWebcamDeviceName]); + + useEffect(() => { + let cancelled = false; + nativeBridgeClient.system + .getPlatform() + .then((platform) => { + if (!cancelled) { + setSupportsCursorModeToggle(platform === "win32" || platform === "darwin"); + } + }) + .catch(() => { + if (!cancelled) { + setSupportsCursorModeToggle(false); + } + }); + + return () => { + cancelled = true; + }; + }, []); useEffect(() => { if (!import.meta.env.DEV) { @@ -249,15 +282,29 @@ export function LaunchWindow() { return () => cancelAnimationFrame(id); }, [isLanguageMenuOpen]); + const hudMouseEventsEnabledRef = useRef(undefined); + const setHudMouseEventsEnabled = useCallback((enabled: boolean) => { + if (hudMouseEventsEnabledRef.current === enabled) { + return; + } + hudMouseEventsEnabledRef.current = enabled; + window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(!enabled); + }, []); + useEffect(() => { - window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(true); + setHudMouseEventsEnabled(false); return () => { window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(false); }; - }, []); + }, [setHudMouseEventsEnabled]); + + useEffect(() => { + setHudMouseEventsEnabled(isLanguageMenuOpen); + }, [isLanguageMenuOpen, setHudMouseEventsEnabled]); const [selectedSource, setSelectedSource] = useState("Screen"); const [hasSelectedSource, setHasSelectedSource] = useState(false); + const [, setRecordPointerDownCount] = useState(0); useEffect(() => { const checkSelectedSource = async () => { @@ -293,13 +340,17 @@ export function LaunchWindow() { } if (result.success && result.path) { - await window.electronAPI.setCurrentVideoPath(result.path); + const setVideoPathResult = await nativeBridgeClient.project.setCurrentVideoPath(result.path); + if (!setVideoPathResult.success) { + console.error("Failed to set current video path:", setVideoPathResult); + return; + } await window.electronAPI.switchToEditor(); } }; const openProjectFile = async () => { - const result = await window.electronAPI.loadProjectFile(); + const result = await nativeBridgeClient.project.loadProjectFile(); if (result.canceled || !result.success) return; await window.electronAPI.switchToEditor(); }; @@ -320,6 +371,29 @@ export function LaunchWindow() { setMicrophoneEnabled(!microphoneEnabled); } }; + const dragLastPositionRef = useRef<{ x: number; y: number } | null>(null); + const handleHudDragPointerDown = (event: React.PointerEvent) => { + event.preventDefault(); + event.stopPropagation(); + setHudMouseEventsEnabled(true); + event.currentTarget.setPointerCapture(event.pointerId); + dragLastPositionRef.current = { x: event.screenX, y: event.screenY }; + }; + const handleHudDragPointerMove = (event: React.PointerEvent) => { + const lastPosition = dragLastPositionRef.current; + if (!lastPosition) return; + const deltaX = event.screenX - lastPosition.x; + const deltaY = event.screenY - lastPosition.y; + dragLastPositionRef.current = { x: event.screenX, y: event.screenY }; + window.electronAPI?.moveHudOverlayBy?.(deltaX, deltaY); + }; + const handleHudDragPointerEnd = (event: React.PointerEvent) => { + dragLastPositionRef.current = null; + if (event.currentTarget.hasPointerCapture(event.pointerId)) { + event.currentTarget.releasePointerCapture(event.pointerId); + } + setHudMouseEventsEnabled(false); + }; return ( // Root fills the HUD window only. Avoid w-screen/h-screen (100vw/100vh): @@ -330,13 +404,19 @@ export function LaunchWindow() { className={`h-full w-full min-w-0 max-w-full overflow-x-hidden overflow-y-hidden bg-transparent ${styles.electronDrag}`} onPointerMove={(event) => { const target = event.target as HTMLElement | null; - const shouldCapture = Boolean(target?.closest("[data-hud-interactive='true']")); - window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(!shouldCapture); + const shouldCapture = + isLanguageMenuOpen || Boolean(target?.closest("[data-hud-interactive='true']")); + setHudMouseEventsEnabled(shouldCapture); + }} + onPointerLeave={() => { + if (!isLanguageMenuOpen) { + setHudMouseEventsEnabled(false); + } }} - onPointerLeave={() => window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(true)} > {systemLocaleSuggestion && (
@@ -396,8 +476,10 @@ export function LaunchWindow() { { + const device = cameraDevices.find( + (item) => item.deviceId === e.target.value, + ); setSelectedCameraId(e.target.value); setWebcamDeviceId(e.target.value); + setWebcamDeviceName(device?.label); }} className="w-full appearance-none bg-white/5 text-white text-[11px] rounded-lg pl-2 pr-6 py-1 border border-white/10 outline-none hover:bg-white/10 transition-colors cursor-pointer" > @@ -480,8 +566,10 @@ export function LaunchWindow() { setIsFocused(true)} @@ -222,12 +232,6 @@ const GRADIENTS = [ ]; interface SettingsPanelProps { - cursorHighlight?: import("./videoPlayback/cursorHighlight").CursorHighlightConfig; - onCursorHighlightChange?: ( - next: import("./videoPlayback/cursorHighlight").CursorHighlightConfig, - ) => void; - // macOS only — gates the "Only on clicks" toggle (needs uiohook). - cursorHighlightSupportsClicks?: boolean; selected: string; onWallpaperChange: (path: string) => void; selectedZoomDepth?: ZoomDepth | null; @@ -310,6 +314,20 @@ interface SettingsPanelProps { onWebcamSizePresetChange?: (size: WebcamSizePreset) => void; onWebcamSizePresetCommit?: () => void; onSaveDiagnostic?: () => Promise; + showCursor?: boolean; + onShowCursorChange?: (show: boolean) => void; + cursorSize?: number; + onCursorSizeChange?: (size: number) => void; + cursorSmoothing?: number; + onCursorSmoothingChange?: (smoothing: number) => void; + cursorMotionBlur?: number; + onCursorMotionBlurChange?: (blur: number) => void; + cursorClickBounce?: number; + onCursorClickBounceChange?: (bounce: number) => void; + cursorClipToBounds?: boolean; + onCursorClipToBoundsChange?: (clip: boolean) => void; + hasCursorData?: boolean; + showCursorSettings?: boolean; } export default SettingsPanel; @@ -325,10 +343,24 @@ const ZOOM_DEPTH_OPTIONS: Array<{ depth: ZoomDepth; label: string }> = [ type SettingsPanelMode = "background" | "effects" | "layout" | "cursor" | "export"; +const MP4_EXPORT_SHORT_SIDES = { + medium: 720, + good: 1080, +} as const; + +function formatSourceDimensions(videoElement?: HTMLVideoElement | null, cropRegion?: CropRegion) { + const width = videoElement?.videoWidth ?? 0; + const height = videoElement?.videoHeight ?? 0; + + if (width <= 0 || height <= 0) { + return null; + } + + const dimensions = calculateEffectiveSourceDimensions(width, height, cropRegion); + return { ...dimensions, shortSide: Math.min(dimensions.width, dimensions.height) }; +} + export function SettingsPanel({ - cursorHighlight, - onCursorHighlightChange, - cursorHighlightSupportsClicks = false, selected, onWallpaperChange, selectedZoomDepth, @@ -359,24 +391,24 @@ export function SettingsPanel({ borderRadius = 0, onBorderRadiusChange, onBorderRadiusCommit, - padding = 50, + padding = DEFAULT_EDITOR_LAYOUT_SETTINGS.padding, onPaddingChange, onPaddingCommit, cropRegion, onCropChange, aspectRatio, videoElement, - exportQuality = "good", + exportQuality = DEFAULT_EXPORT_SETTINGS.quality, onExportQualityChange, - exportFormat = "mp4", + exportFormat = DEFAULT_EXPORT_SETTINGS.format, onExportFormatChange, - gifFrameRate = 15, + gifFrameRate = DEFAULT_GIF_SETTINGS.frameRate, onGifFrameRateChange, - gifLoop = true, + gifLoop = DEFAULT_GIF_SETTINGS.loop, onGifLoopChange, - gifSizePreset = "medium", + gifSizePreset = DEFAULT_GIF_SETTINGS.sizePreset, onGifSizePresetChange, - gifOutputDimensions = { width: 1280, height: 720 }, + gifOutputDimensions = DEFAULT_GIF_SETTINGS.outputDimensions, onExport, unsavedExport, onSaveUnsavedExport, @@ -398,17 +430,32 @@ export function SettingsPanel({ onSpeedChange, onSpeedDelete, hasWebcam = false, - webcamLayoutPreset = "picture-in-picture", + webcamLayoutPreset = DEFAULT_WEBCAM_SETTINGS.layoutPreset, onWebcamLayoutPresetChange, - webcamMaskShape = "rectangle", + webcamMaskShape = DEFAULT_WEBCAM_SETTINGS.maskShape, onWebcamMaskShapeChange, - webcamSizePreset = DEFAULT_WEBCAM_SIZE_PRESET, + webcamSizePreset = DEFAULT_WEBCAM_SETTINGS.sizePreset, onWebcamSizePresetChange, onWebcamSizePresetCommit, onSaveDiagnostic, + showCursor = DEFAULT_CURSOR_SETTINGS.show, + onShowCursorChange, + cursorSize = DEFAULT_CURSOR_SETTINGS.size, + onCursorSizeChange, + cursorSmoothing = DEFAULT_CURSOR_SETTINGS.smoothing, + onCursorSmoothingChange, + cursorMotionBlur = DEFAULT_CURSOR_SETTINGS.motionBlur, + onCursorMotionBlurChange, + cursorClickBounce = DEFAULT_CURSOR_SETTINGS.clickBounce, + onCursorClickBounceChange, + cursorClipToBounds = DEFAULT_CURSOR_SETTINGS.clipToBounds, + onCursorClipToBoundsChange, + hasCursorData = false, + showCursorSettings = true, }: SettingsPanelProps) { const t = useScopedT("settings"); const [activePanelMode, setActivePanelMode] = useState("background"); + const sourceDimensions = formatSourceDimensions(videoElement, cropRegion); // Resolved URLs are for DOM rendering only (backgroundImage). The canonical // `/wallpapers/wallpaperN.jpg` form in WALLPAPER_PATHS is what gets persisted // on click — never the machine-specific file:// URL. @@ -436,14 +483,12 @@ export function SettingsPanel({ const [selectedColor, setSelectedColor] = useState("#ADADAD"); const [gradient, setGradient] = useState(GRADIENTS[0]); - const [showCropModal, setShowCropModal] = useState(false); - const cropSnapshotRef = useRef(null); const [cropAspectLocked, setCropAspectLocked] = useState(false); const [cropAspectRatio, setCropAspectRatio] = useState(""); const isPortraitCanvas = isPortraitAspectRatio(aspectRatio); - const videoWidth = videoElement?.videoWidth || 1920; - const videoHeight = videoElement?.videoHeight || 1080; + const videoWidth = videoElement?.videoWidth || DEFAULT_SOURCE_DIMENSIONS.width; + const videoHeight = videoElement?.videoHeight || DEFAULT_SOURCE_DIMENSIONS.height; const handleCropNumericChange = useCallback( (field: "x" | "y" | "width" | "height", pixelValue: number) => { @@ -538,10 +583,13 @@ export function SettingsPanel({ }, [cropRegion, videoWidth, videoHeight], ); + const [showCropDropdown, setShowCropDropdown] = useState(false); + const handleCropToggle = () => setShowCropDropdown((open) => !open); const zoomEnabled = Boolean(selectedZoomDepth); const trimEnabled = Boolean(selectedTrimId); const hasTimelineSelection = Boolean(selectedZoomId || selectedTrimId || selectedSpeedId); + const hasCursorPanel = showCursorSettings && hasCursorData; const panelModes: Array<{ id: SettingsPanelMode; label: string; @@ -551,7 +599,15 @@ export function SettingsPanel({ { id: "background", label: t("background.title"), icon: Palette }, { id: "effects", label: t("effects.title"), icon: SlidersHorizontal }, { id: "layout", label: t("layout.title"), icon: LayoutPanelTop, disabled: !hasWebcam }, - { id: "cursor", label: t("effects.cursorHighlight.title"), icon: MousePointerClick }, + ...(hasCursorPanel + ? [ + { + id: "cursor" as const, + label: t("effects.title"), + icon: MousePointerClick, + }, + ] + : []), ]; const exportPanelMode = { id: "export" as const, @@ -624,20 +680,6 @@ export function SettingsPanel({ } }; - const handleCropToggle = () => { - if (!showCropModal && cropRegion) { - cropSnapshotRef.current = { ...cropRegion }; - } - setShowCropModal(!showCropModal); - }; - - const handleCropCancel = () => { - if (cropSnapshotRef.current && onCropChange) { - onCropChange(cropSnapshotRef.current); - } - setShowCropModal(false); - }; - // Find selected annotation const selectedAnnotation = selectedAnnotationId ? annotationRegions.find((a) => a.id === selectedAnnotationId) @@ -657,7 +699,7 @@ export function SettingsPanel({ className="flex-1 flex items-center justify-center gap-1.5 text-[10px] text-slate-500 hover:text-slate-300 py-1.5 transition-colors" > - {t("links.reportBug")} + {t("support.reportBug")} {onSaveDiagnostic && ( )}
); @@ -773,6 +815,7 @@ export function SettingsPanel({
@@ -1373,218 +1412,107 @@ export function SettingsPanel({ )} - {activePanelMode === "cursor" && cursorHighlight && onCursorHighlightChange && ( -
+ {activePanelMode === "cursor" && showCursorSettings && hasCursorData && ( +
- {t("effects.cursorHighlight.title")} + {t("cursor.show")}
- -
-
- {(["dot", "ring"] as const).map((style) => ( - - ))} -
-
-
-
- {t("effects.cursorHighlight.size")} -
- - {cursorHighlight.sizePx}px - -
- - onCursorHighlightChange({ - ...cursorHighlight, - sizePx: values[0], - }) - } - min={10} - max={36} - step={1} - className="w-full [&_[role=slider]]:bg-[#34B27B] [&_[role=slider]]:border-[#34B27B] [&_[role=slider]]:h-3 [&_[role=slider]]:w-3" +
- {cursorHighlightSupportsClicks && ( -
-
- {t("effects.cursorHighlight.onlyOnClicks")} -
- -
- )} -
-
- {t("effects.cursorHighlight.color")} -
- - - - - - - onCursorHighlightChange({ - ...cursorHighlight, - color, - }) - } + {showCursor && ( + <> +
+
+ {t("cursor.clipToBounds")} +
+ - - -
-
-
-
- {t("effects.cursorHighlight.offsetX")}
- - {(cursorHighlight.offsetXNorm * 100).toFixed(1)}% - -
- - onCursorHighlightChange({ - ...cursorHighlight, - offsetXNorm: values[0], - }) - } - min={-0.25} - max={0.25} - step={0.005} - className="w-full [&_[role=slider]]:bg-[#34B27B] [&_[role=slider]]:border-[#34B27B] [&_[role=slider]]:h-3 [&_[role=slider]]:w-3" - /> -
-
-
-
- {t("effects.cursorHighlight.offsetY")} +
+
+
+
+ {t("cursor.size")} +
+ + {cursorSize.toFixed(1)} + +
+ onCursorSizeChange?.(values[0])} + min={0.5} + max={10} + step={0.1} + className="w-full [&_[role=slider]]:bg-[#34B27B] [&_[role=slider]]:border-[#34B27B] [&_[role=slider]]:h-3 [&_[role=slider]]:w-3" + /> +
+
+
+
+ {t("cursor.smoothing")} +
+ + {Math.round(cursorSmoothing * 100)}% + +
+ onCursorSmoothingChange?.(values[0])} + min={0} + max={1} + step={0.01} + className="w-full [&_[role=slider]]:bg-[#34B27B] [&_[role=slider]]:border-[#34B27B] [&_[role=slider]]:h-3 [&_[role=slider]]:w-3" + /> +
+
+
+
+ {t("cursor.motionBlur")} +
+ + {Math.round(cursorMotionBlur * 100)}% + +
+ onCursorMotionBlurChange?.(values[0])} + min={0} + max={1} + step={0.01} + className="w-full [&_[role=slider]]:bg-[#34B27B] [&_[role=slider]]:border-[#34B27B] [&_[role=slider]]:h-3 [&_[role=slider]]:w-3" + /> +
+
+
+
+ {t("cursor.clickBounce")} +
+ + {cursorClickBounce.toFixed(1)} + +
+ onCursorClickBounceChange?.(values[0])} + min={0} + max={5} + step={0.1} + className="w-full [&_[role=slider]]:bg-[#34B27B] [&_[role=slider]]:border-[#34B27B] [&_[role=slider]]:h-3 [&_[role=slider]]:w-3" + /> +
- - {(cursorHighlight.offsetYNorm * 100).toFixed(1)}% - -
- - onCursorHighlightChange({ - ...cursorHighlight, - offsetYNorm: values[0], - }) - } - min={-0.25} - max={0.25} - step={0.005} - className="w-full [&_[role=slider]]:bg-[#34B27B] [&_[role=slider]]:border-[#34B27B] [&_[role=slider]]:h-3 [&_[role=slider]]:w-3" - /> -
+ + )}
)} @@ -1744,11 +1672,11 @@ export function SettingsPanel({
- {showCropModal && cropRegion && onCropChange && ( + {showCropDropdown && cropRegion && onCropChange && ( <>
setShowCropDropdown(false)} />
@@ -1759,7 +1687,7 @@ export function SettingsPanel({
{exportFormat === "mp4" && ( -
- - - +
+ {sourceDimensions && ( +
+ {t("exportQuality.title")} + + Source {sourceDimensions.width}x{sourceDimensions.height} + +
+ )} +
+ + + +
)} diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index e1f6a60..b44d9b6 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -16,7 +16,10 @@ import { useShortcuts } from "@/contexts/ShortcutsContext"; import { INITIAL_EDITOR_STATE, useEditorHistory } from "@/hooks/useEditorHistory"; import { type Locale } from "@/i18n/config"; import { getAvailableLocales, getLocaleName } from "@/i18n/loader"; +import { hasNativeCursorRecordingData } from "@/lib/cursor/nativeCursor"; import { + calculateEffectiveSourceDimensions, + calculateMp4ExportSettings, calculateOutputDimensions, type ExportFormat, type ExportProgress, @@ -29,7 +32,7 @@ import { VideoExporter, } from "@/lib/exporter"; import { computeFrameStepTime } from "@/lib/frameStep"; -import type { ProjectMedia } from "@/lib/recordingSession"; +import type { CursorCaptureMode, ProjectMedia } from "@/lib/recordingSession"; import { matchesShortcut } from "@/lib/shortcuts"; import { getExportFolder, @@ -38,12 +41,20 @@ import { saveUserPreferences, } from "@/lib/userPreferences"; import { BackgroundLoadError } from "@/lib/wallpaper"; +import { nativeBridgeClient, useCursorRecordingData, useCursorTelemetry } from "@/native"; +import type { NativePlatform } from "@/native/contracts"; import { getAspectRatioValue, getNativeAspectRatioValue, isPortraitAspectRatio, } from "@/utils/aspectRatioUtils"; import { ExportDialog } from "./ExportDialog"; +import { + DEFAULT_CURSOR_SETTINGS, + DEFAULT_EXPORT_SETTINGS, + DEFAULT_GIF_SETTINGS, + DEFAULT_SOURCE_DIMENSIONS, +} from "./editorDefaults"; import PlaybackControls from "./PlaybackControls"; import { createProjectData, @@ -61,7 +72,6 @@ import TimelineEditor from "./timeline/TimelineEditor"; import { type AnnotationRegion, type BlurData, - type CursorTelemetryPoint, clampFocusToDepth, DEFAULT_ANNOTATION_POSITION, DEFAULT_ANNOTATION_SIZE, @@ -84,6 +94,62 @@ import { import { UnsavedChangesDialog } from "./UnsavedChangesDialog"; import VideoPlayback, { VideoPlaybackRef } from "./VideoPlayback"; +function isClickInteractionType(interactionType: string | null | undefined) { + return ( + interactionType === "click" || + interactionType === "double-click" || + interactionType === "right-click" || + interactionType === "middle-click" + ); +} + +interface ExportDiagnostics { + formatLabel: "GIF" | "Video"; + reason?: string; + sourcePath?: string | null; + width?: number; + height?: number; + frameRate?: number; + codec?: string; + bitrate?: number; +} + +function getFileNameForDiagnostics(filePath?: string | null) { + if (!filePath) return "unknown"; + + try { + const url = new URL(filePath); + if (url.protocol === "file:") { + return decodeURIComponent(url.pathname).split(/[\\/]/).pop() || filePath; + } + } catch { + // Treat non-URL values as filesystem paths. + } + + return filePath.split(/[\\/]/).pop() || filePath; +} + +function buildExportDiagnosticMessage(diagnostics: ExportDiagnostics) { + const details = [ + diagnostics.reason ? `Reason: ${diagnostics.reason}` : null, + `Source: ${getFileNameForDiagnostics(diagnostics.sourcePath)}`, + diagnostics.width && diagnostics.height + ? `Output: ${diagnostics.width}x${diagnostics.height}${ + diagnostics.frameRate ? ` @ ${diagnostics.frameRate} fps` : "" + }` + : null, + diagnostics.codec ? `Codec: ${diagnostics.codec}` : null, + diagnostics.bitrate ? `Bitrate: ${Math.round(diagnostics.bitrate / 1_000_000)} Mbps` : null, + `VideoEncoder: ${"VideoEncoder" in window ? "available" : "unavailable"}`, + ].filter(Boolean); + + return `${diagnostics.formatLabel} export failed\n${details.join("\n")}`; +} + +function buildSaveDiagnosticMessage(formatLabel: "GIF" | "Video", reason?: string) { + return `${formatLabel} export save failed${reason ? `\nReason: ${reason}` : ""}`; +} + export default function VideoEditor() { const { state: editorState, @@ -111,7 +177,6 @@ export default function VideoEditor() { webcamMaskShape, webcamSizePreset, webcamPosition, - cursorHighlight, } = editorState; // ── Non-undoable state @@ -129,8 +194,6 @@ export default function VideoEditor() { currentTimeRef.current = currentTime; const durationRef = useRef(duration); durationRef.current = duration; - const [cursorTelemetry, setCursorTelemetry] = useState([]); - const [cursorClickTimestamps, setCursorClickTimestamps] = useState([]); const [selectedZoomId, setSelectedZoomId] = useState(null); const [selectedTrimId, setSelectedTrimId] = useState(null); const [selectedSpeedId, setSelectedSpeedId] = useState(null); @@ -141,11 +204,15 @@ export default function VideoEditor() { const [exportError, setExportError] = useState(null); const [showExportDialog, setShowExportDialog] = useState(false); const [showNewRecordingDialog, setShowNewRecordingDialog] = useState(false); - const [exportQuality, setExportQuality] = useState("good"); - const [exportFormat, setExportFormat] = useState("mp4"); - const [gifFrameRate, setGifFrameRate] = useState(15); - const [gifLoop, setGifLoop] = useState(true); - const [gifSizePreset, setGifSizePreset] = useState("medium"); + const [exportQuality, setExportQuality] = useState( + DEFAULT_EXPORT_SETTINGS.quality, + ); + const [exportFormat, setExportFormat] = useState(DEFAULT_EXPORT_SETTINGS.format); + const [gifFrameRate, setGifFrameRate] = useState(DEFAULT_GIF_SETTINGS.frameRate); + const [gifLoop, setGifLoop] = useState(DEFAULT_GIF_SETTINGS.loop); + const [gifSizePreset, setGifSizePreset] = useState( + DEFAULT_GIF_SETTINGS.sizePreset, + ); const [exportedFilePath, setExportedFilePath] = useState(null); const [lastSavedSnapshot, setLastSavedSnapshot] = useState(null); const [unsavedExport, setUnsavedExport] = useState<{ @@ -155,8 +222,39 @@ export default function VideoEditor() { } | null>(null); const [isFullscreen, setIsFullscreen] = useState(false); const [showCloseConfirmDialog, setShowCloseConfirmDialog] = useState(false); + const playerContainerRef = useRef(null); + const cursorTelemetrySourcePath = videoSourcePath ?? (videoPath ? fromFileUrl(videoPath) : null); + const { samples: cursorTelemetry, error: cursorTelemetryError } = + useCursorTelemetry(cursorTelemetrySourcePath); + const { data: cursorRecordingData, error: cursorRecordingDataError } = + useCursorRecordingData(cursorTelemetrySourcePath); + const cursorClickTimestamps = useMemo(() => { + const recordingClicks = + cursorRecordingData?.samples + .filter((sample) => isClickInteractionType(sample.interactionType)) + .map((sample) => sample.timeMs) ?? []; + if (recordingClicks.length > 0) { + return recordingClicks; + } + + return cursorTelemetry + .filter((sample) => isClickInteractionType(sample.interactionType)) + .map((sample) => sample.timeMs); + }, [cursorRecordingData, cursorTelemetry]); + + // Cursor & motion blur visual settings (non-undoable preferences) + const [showCursor, setShowCursor] = useState(DEFAULT_CURSOR_SETTINGS.show); + const [cursorSize, setCursorSize] = useState(DEFAULT_CURSOR_SETTINGS.size); + const [cursorSmoothing, setCursorSmoothing] = useState(DEFAULT_CURSOR_SETTINGS.smoothing); + const [cursorMotionBlur, setCursorMotionBlur] = useState(DEFAULT_CURSOR_SETTINGS.motionBlur); + const [cursorClickBounce, setCursorClickBounce] = useState(DEFAULT_CURSOR_SETTINGS.clickBounce); + const [cursorClipToBounds, setCursorClipToBounds] = useState( + DEFAULT_CURSOR_SETTINGS.clipToBounds, + ); + const [nativePlatform, setNativePlatform] = useState(null); + const [recordingCursorCaptureMode, setRecordingCursorCaptureMode] = + useState(null); - const playerContainerRef = useRef(null); const videoPlaybackRef = useRef(null); const nextZoomIdRef = useRef(1); @@ -164,12 +262,15 @@ export default function VideoEditor() { const nextSpeedIdRef = useRef(1); const { shortcuts, isMac } = useShortcuts(); - // Off-Mac doesn't have click telemetry, so force `onlyOnClicks` off for - // renderers while keeping the persisted value intact for round-tripping. - const effectiveCursorHighlight = useMemo( - () => (isMac ? cursorHighlight : { ...cursorHighlight, onlyOnClicks: false }), - [cursorHighlight, isMac], - ); + // Native Windows recordings include captured cursor assets. Native macOS + // recordings hide the system cursor in ScreenCaptureKit and use telemetry + // samples with OpenScreen's default arrow asset for the editable overlay. + const hasEditableCursorRecording = + recordingCursorCaptureMode === "editable-overlay" && + (nativePlatform === "win32" || nativePlatform === "darwin") && + hasNativeCursorRecordingData(cursorRecordingData); + const effectiveShowCursor = showCursor && hasEditableCursorRecording; + const showCursorSettings = hasEditableCursorRecording; const { locale, setLocale, t: rawT } = useI18n(); const t = useScopedT("editor"); const ts = useScopedT("settings"); @@ -196,10 +297,18 @@ export default function VideoEditor() { const webcamSourcePath = webcamVideoSourcePath ?? (webcamVideoPath ? fromFileUrl(webcamVideoPath) : null); - return webcamSourcePath - ? { screenVideoPath, webcamVideoPath: webcamSourcePath } - : { screenVideoPath }; - }, [videoPath, videoSourcePath, webcamVideoPath, webcamVideoSourcePath]); + return { + screenVideoPath, + ...(webcamSourcePath ? { webcamVideoPath: webcamSourcePath } : {}), + ...(recordingCursorCaptureMode ? { cursorCaptureMode: recordingCursorCaptureMode } : {}), + }; + }, [ + videoPath, + videoSourcePath, + webcamVideoPath, + webcamVideoSourcePath, + recordingCursorCaptureMode, + ]); const applyLoadedProject = useCallback( async (candidate: unknown, path?: string | null) => { @@ -208,13 +317,21 @@ export default function VideoEditor() { } const project = candidate; - const media = resolveProjectMedia(project); - if (!media) { + const projectMedia = resolveProjectMedia(project); + if (!projectMedia) { return false; } - const sourcePath = fromFileUrl(media.screenVideoPath); - const webcamSourcePath = media.webcamVideoPath ? fromFileUrl(media.webcamVideoPath) : null; + const sourcePath = projectMedia.screenVideoPath; + const webcamSourcePath = projectMedia.webcamVideoPath ?? null; + const projectCursorCaptureMode = projectMedia.cursorCaptureMode ?? null; const normalizedEditor = normalizeProjectEditor(project.editor); + const inferredDurationMs = Math.max( + 0, + ...normalizedEditor.zoomRegions.map((region) => region.endMs), + ...normalizedEditor.trimRegions.map((region) => region.endMs), + ...normalizedEditor.speedRegions.map((region) => region.endMs), + ...normalizedEditor.annotationRegions.map((region) => region.endMs), + ); try { videoPlaybackRef.current?.pause(); @@ -223,13 +340,14 @@ export default function VideoEditor() { } setIsPlaying(false); setCurrentTime(0); - setDuration(0); + setDuration(inferredDurationMs > 0 ? inferredDurationMs / 1000 : 0); setError(null); setVideoSourcePath(sourcePath); setVideoPath(toFileUrl(sourcePath)); setWebcamVideoSourcePath(webcamSourcePath); setWebcamVideoPath(webcamSourcePath ? toFileUrl(webcamSourcePath) : null); + setRecordingCursorCaptureMode(projectCursorCaptureMode); setCurrentProjectPath(path ?? null); pushState({ @@ -286,9 +404,11 @@ export default function VideoEditor() { setLastSavedSnapshot( createProjectSnapshot( - webcamSourcePath - ? { screenVideoPath: sourcePath, webcamVideoPath: webcamSourcePath } - : { screenVideoPath: sourcePath }, + { + screenVideoPath: sourcePath, + ...(webcamSourcePath ? { webcamVideoPath: webcamSourcePath } : {}), + ...(projectCursorCaptureMode ? { cursorCaptureMode: projectCursorCaptureMode } : {}), + }, normalizedEditor, ), ); @@ -352,7 +472,7 @@ export default function VideoEditor() { useEffect(() => { async function loadInitialData() { try { - const currentProjectResult = await window.electronAPI.loadCurrentProjectFile(); + const currentProjectResult = await nativeBridgeClient.project.loadCurrentProjectFile(); if (currentProjectResult.success && currentProjectResult.project) { const restored = await applyLoadedProject( currentProjectResult.project, @@ -374,31 +494,31 @@ export default function VideoEditor() { setVideoPath(toFileUrl(sourcePath)); setWebcamVideoSourcePath(webcamSourcePath); setWebcamVideoPath(webcamSourcePath ? toFileUrl(webcamSourcePath) : null); + setRecordingCursorCaptureMode(session.cursorCaptureMode ?? null); setCurrentProjectPath(null); setLastSavedSnapshot( createProjectSnapshot( - webcamSourcePath - ? { - screenVideoPath: sourcePath, - webcamVideoPath: webcamSourcePath, - } - : { screenVideoPath: sourcePath }, + { + screenVideoPath: sourcePath, + ...(webcamSourcePath ? { webcamVideoPath: webcamSourcePath } : {}), + ...(session.cursorCaptureMode + ? { cursorCaptureMode: session.cursorCaptureMode } + : {}), + }, INITIAL_EDITOR_STATE, ), ); return; } - const result = await window.electronAPI.getCurrentVideoPath(); + const result = await nativeBridgeClient.project.getCurrentVideoPath(); if (result.success && result.path) { - const sourcePath = fromFileUrl(result.path); - setVideoSourcePath(sourcePath); - setVideoPath(toFileUrl(sourcePath)); - setWebcamVideoSourcePath(null); - setWebcamVideoPath(null); + setVideoSourcePath(result.path); + setVideoPath(toFileUrl(result.path)); + setRecordingCursorCaptureMode(null); setCurrentProjectPath(null); setLastSavedSnapshot( - createProjectSnapshot({ screenVideoPath: sourcePath }, INITIAL_EDITOR_STATE), + createProjectSnapshot({ screenVideoPath: result.path }, INITIAL_EDITOR_STATE), ); } else { setError("No video to load. Please record or select a video."); @@ -469,7 +589,6 @@ export default function VideoEditor() { gifFrameRate, gifLoop, gifSizePreset, - cursorHighlight, }; const projectData = createProjectData(currentProjectMedia, editorState); @@ -481,7 +600,7 @@ export default function VideoEditor() { // Match the normalization path used by `currentProjectSnapshot` so the // post-save baseline compares equal and `hasUnsavedChanges` clears. const projectSnapshot = createProjectSnapshot(currentProjectMedia, editorState); - const result = await window.electronAPI.saveProjectFile( + const result = await nativeBridgeClient.project.saveProjectFile( projectData, fileNameBase, forceSaveAs ? undefined : (currentProjectPath ?? undefined), @@ -531,7 +650,6 @@ export default function VideoEditor() { videoPath, t, webcamSizePreset, - cursorHighlight, ], ); @@ -587,7 +705,7 @@ export default function VideoEditor() { }, []); const handleLoadProject = useCallback(async () => { - const result = await window.electronAPI.loadProjectFile(); + const result = await nativeBridgeClient.project.loadProjectFile(); if (result.canceled) { return; @@ -620,40 +738,37 @@ export default function VideoEditor() { }, [handleLoadProject, handleSaveProject, handleSaveProjectAs]); useEffect(() => { - let mounted = true; - - async function loadCursorTelemetry() { - const sourcePath = currentProjectMedia?.screenVideoPath ?? null; - - if (!sourcePath) { - if (mounted) { - setCursorTelemetry([]); - setCursorClickTimestamps([]); + let canceled = false; + nativeBridgeClient.system + .getPlatform() + .then((platform) => { + if (!canceled) { + setNativePlatform(platform); } - return; - } - - try { - const result = await window.electronAPI.getCursorTelemetry(sourcePath); - if (mounted) { - setCursorTelemetry(result.success ? result.samples : []); - setCursorClickTimestamps(result.success ? (result.clicks ?? []) : []); + }) + .catch((error) => { + console.warn("Unable to resolve native platform for cursor settings:", error); + if (!canceled) { + setNativePlatform(null); } - } catch (telemetryError) { - console.warn("Unable to load cursor telemetry:", telemetryError); - if (mounted) { - setCursorTelemetry([]); - setCursorClickTimestamps([]); - } - } - } - - loadCursorTelemetry(); + }); return () => { - mounted = false; + canceled = true; }; - }, [currentProjectMedia]); + }, []); + + useEffect(() => { + if (cursorTelemetryError) { + console.warn("Unable to load cursor telemetry:", cursorTelemetryError); + } + }, [cursorTelemetryError]); + + useEffect(() => { + if (cursorRecordingDataError) { + console.warn("Unable to load cursor recording data:", cursorRecordingDataError); + } + }, [cursorRecordingDataError]); function togglePlayPause() { const playback = videoPlaybackRef.current; @@ -756,11 +871,10 @@ export default function VideoEditor() { customScale: ZOOM_DEPTH_SCALES[DEFAULT_ZOOM_DEPTH], focus: clampFocusToDepth(focus, DEFAULT_ZOOM_DEPTH), }; + // Bulk suggest must not steal selection — keeping a zoom selected hides + // the export panel (SettingsPanel gates it on !hasTimelineSelection), + // trapping users who just want to export after auto-zoom. pushState((prev) => ({ zoomRegions: [...prev.zoomRegions, newRegion] })); - setSelectedZoomId(id); - setSelectedTrimId(null); - setSelectedAnnotationId(null); - setSelectedBlurId(null); }, [pushState], ); @@ -1411,11 +1525,21 @@ export default function VideoEditor() { setUnsavedExport(null); handleExportSaved(unsavedExport.format === "gif" ? "GIF" : "Video", saveResult.path); } else { - toast.error(saveResult.message || "Failed to save export"); + toast.error( + buildSaveDiagnosticMessage( + unsavedExport.format === "gif" ? "GIF" : "Video", + saveResult.message || "Failed to save export", + ), + ); } } catch (error) { console.error("Error saving unsaved export:", error); - toast.error("Failed to save exported video"); + toast.error( + buildSaveDiagnosticMessage( + unsavedExport.format === "gif" ? "GIF" : "Video", + error instanceof Error ? error.message : "Failed to save exported video", + ), + ); } }, [unsavedExport, handleExportSaved]); @@ -1458,8 +1582,13 @@ export default function VideoEditor() { videoPlaybackRef.current?.pause(); } - const sourceWidth = video.videoWidth || 1920; - const sourceHeight = video.videoHeight || 1080; + const sourceWidth = video.videoWidth || DEFAULT_SOURCE_DIMENSIONS.width; + const sourceHeight = video.videoHeight || DEFAULT_SOURCE_DIMENSIONS.height; + const effectiveSourceDimensions = calculateEffectiveSourceDimensions( + sourceWidth, + sourceHeight, + cropRegion, + ); const aspectRatioValue = aspectRatio === "native" ? getNativeAspectRatioValue(sourceWidth, sourceHeight, cropRegion) @@ -1468,8 +1597,8 @@ export default function VideoEditor() { // Get preview CONTAINER dimensions for scaling const playbackRef = videoPlaybackRef.current; const containerElement = playbackRef?.containerRef?.current; - const previewWidth = containerElement?.clientWidth || 1920; - const previewHeight = containerElement?.clientHeight || 1080; + const previewWidth = containerElement?.clientWidth || DEFAULT_SOURCE_DIMENSIONS.width; + const previewHeight = containerElement?.clientHeight || DEFAULT_SOURCE_DIMENSIONS.height; if (settings.format === "gif" && settings.gifConfig) { // GIF Export @@ -1493,6 +1622,12 @@ export default function VideoEditor() { padding, videoPadding: padding, cropRegion, + cursorRecordingData, + cursorScale: effectiveShowCursor ? cursorSize : 0, + cursorSmoothing, + cursorMotionBlur, + cursorClickBounce, + cursorClipToBounds, annotationRegions, webcamLayoutPreset, webcamMaskShape, @@ -1502,7 +1637,6 @@ export default function VideoEditor() { previewHeight, cursorTelemetry, cursorClickTimestamps, - cursorHighlight: effectiveCursorHighlight, onProgress: (progress: ExportProgress) => { setExportProgress(progress); }, @@ -1527,93 +1661,38 @@ export default function VideoEditor() { handleExportSaved("GIF", saveResult.path); } else { setUnsavedExport({ arrayBuffer, fileName: targetFileName, format: "gif" }); - setExportError(saveResult.message || "Failed to save GIF"); - toast.error(saveResult.message || "Failed to save GIF"); + const message = buildSaveDiagnosticMessage( + "GIF", + saveResult.message || "Failed to save GIF", + ); + setExportError(message); + toast.error(message); } } else { - setExportError(result.error || "GIF export failed"); - toast.error(result.error || "GIF export failed"); + const message = buildExportDiagnosticMessage({ + formatLabel: "GIF", + reason: result.error || "GIF export failed", + sourcePath: videoSourcePath ?? videoPath, + width: settings.gifConfig.width, + height: settings.gifConfig.height, + frameRate: settings.gifConfig.frameRate, + }); + setExportError(message); + toast.error(message); } } else { // MP4 Export const quality = settings.quality || exportQuality; - let exportWidth: number; - let exportHeight: number; - let bitrate: number; - - if (quality === "source") { - exportWidth = sourceWidth; - exportHeight = sourceHeight; - - // Use the source's longer dimension as the long axis of the export so - // a landscape recording can still fill a portrait target (and vice versa). - const sourceLongDim = Math.max(sourceWidth, sourceHeight); - - if (aspectRatioValue === 1) { - const baseDimension = Math.floor(Math.min(sourceWidth, sourceHeight) / 2) * 2; - exportWidth = baseDimension; - exportHeight = baseDimension; - } else if (aspectRatioValue > 1) { - const baseWidth = Math.floor(sourceLongDim / 2) * 2; - let found = false; - for (let w = baseWidth; w >= 100 && !found; w -= 2) { - const h = Math.round(w / aspectRatioValue); - if (h % 2 === 0 && Math.abs(w / h - aspectRatioValue) < 0.0001) { - exportWidth = w; - exportHeight = h; - found = true; - } - } - if (!found) { - exportWidth = baseWidth; - exportHeight = Math.floor(baseWidth / aspectRatioValue / 2) * 2; - } - } else { - const baseHeight = Math.floor(sourceLongDim / 2) * 2; - let found = false; - for (let h = baseHeight; h >= 100 && !found; h -= 2) { - const w = Math.round(h * aspectRatioValue); - if (w % 2 === 0 && Math.abs(w / h - aspectRatioValue) < 0.0001) { - exportWidth = w; - exportHeight = h; - found = true; - } - } - if (!found) { - exportHeight = baseHeight; - exportWidth = Math.floor((baseHeight * aspectRatioValue) / 2) * 2; - } - } - - const totalPixels = exportWidth * exportHeight; - bitrate = 30_000_000; - if (totalPixels > 1920 * 1080 && totalPixels <= 2560 * 1440) { - bitrate = 50_000_000; - } else if (totalPixels > 2560 * 1440) { - bitrate = 80_000_000; - } - } else { - // Quality presets target the SHORT side; the long side derives from the - // aspect ratio. This keeps 1080p portrait at 1080×1920 instead of 607×1080. - const targetShortDim = quality === "medium" ? 720 : 1080; - - if (aspectRatioValue >= 1) { - exportHeight = Math.floor(targetShortDim / 2) * 2; - exportWidth = Math.floor((exportHeight * aspectRatioValue) / 2) * 2; - } else { - exportWidth = Math.floor(targetShortDim / 2) * 2; - exportHeight = Math.floor(exportWidth / aspectRatioValue / 2) * 2; - } - - const totalPixels = exportWidth * exportHeight; - if (totalPixels <= 1280 * 720) { - bitrate = 10_000_000; - } else if (totalPixels <= 1920 * 1080) { - bitrate = 20_000_000; - } else { - bitrate = 30_000_000; - } - } + const { + width: exportWidth, + height: exportHeight, + bitrate, + } = calculateMp4ExportSettings({ + quality, + sourceWidth: effectiveSourceDimensions.width, + sourceHeight: effectiveSourceDimensions.height, + aspectRatioValue, + }); const exporter = new VideoExporter({ videoUrl: videoPath, @@ -1634,6 +1713,12 @@ export default function VideoEditor() { borderRadius, padding, cropRegion, + cursorRecordingData, + cursorScale: effectiveShowCursor ? cursorSize : 0, + cursorSmoothing, + cursorMotionBlur, + cursorClickBounce, + cursorClipToBounds, annotationRegions, webcamLayoutPreset, webcamMaskShape, @@ -1643,7 +1728,6 @@ export default function VideoEditor() { previewHeight, cursorTelemetry, cursorClickTimestamps, - cursorHighlight: effectiveCursorHighlight, onProgress: (progress: ExportProgress) => { setExportProgress(progress); }, @@ -1668,12 +1752,26 @@ export default function VideoEditor() { handleExportSaved("Video", saveResult.path); } else { setUnsavedExport({ arrayBuffer, fileName: targetFileName, format: "mp4" }); - setExportError(saveResult.message || "Failed to save video"); - toast.error(saveResult.message || "Failed to save video"); + const message = buildSaveDiagnosticMessage( + "Video", + saveResult.message || "Failed to save video", + ); + setExportError(message); + toast.error(message); } } else { - setExportError(result.error || "Export failed"); - toast.error(result.error || "Export failed"); + const message = buildExportDiagnosticMessage({ + formatLabel: "Video", + reason: result.error || "Export failed", + sourcePath: videoSourcePath ?? videoPath, + width: exportWidth, + height: exportHeight, + frameRate: 60, + codec: "avc1.640033", + bitrate, + }); + setExportError(message); + toast.error(message); } } @@ -1688,8 +1786,13 @@ export default function VideoEditor() { toast.error(message); } else { const errorMessage = error instanceof Error ? error.message : "Unknown error"; - setExportError(errorMessage); - toast.error(t("errors.exportFailedWithError", { error: errorMessage })); + const message = buildExportDiagnosticMessage({ + formatLabel: settings.format === "gif" ? "GIF" : "Video", + reason: errorMessage, + sourcePath: videoSourcePath ?? videoPath, + }); + setExportError(message); + toast.error(t("errors.exportFailedWithError", { error: message })); } } finally { setIsExporting(false); @@ -1702,6 +1805,7 @@ export default function VideoEditor() { }, [ videoPath, + videoSourcePath, webcamVideoPath, wallpaper, zoomRegions, @@ -1713,6 +1817,7 @@ export default function VideoEditor() { borderRadius, padding, cropRegion, + cursorRecordingData, annotationRegions, isPlaying, aspectRatio, @@ -1724,7 +1829,12 @@ export default function VideoEditor() { handleExportSaved, cursorTelemetry, cursorClickTimestamps, - effectiveCursorHighlight, + effectiveShowCursor, + cursorSize, + cursorSmoothing, + cursorMotionBlur, + cursorClickBounce, + cursorClipToBounds, t, ], ); @@ -1742,15 +1852,20 @@ export default function VideoEditor() { } // Build export settings from current state - const sourceWidth = video.videoWidth || 1920; - const sourceHeight = video.videoHeight || 1080; + const sourceWidth = video.videoWidth || DEFAULT_SOURCE_DIMENSIONS.width; + const sourceHeight = video.videoHeight || DEFAULT_SOURCE_DIMENSIONS.height; + const effectiveSourceDimensions = calculateEffectiveSourceDimensions( + sourceWidth, + sourceHeight, + cropRegion, + ); const aspectRatioValue = aspectRatio === "native" ? getNativeAspectRatioValue(sourceWidth, sourceHeight, cropRegion) : getAspectRatioValue(aspectRatio); const gifDimensions = calculateOutputDimensions( - sourceWidth, - sourceHeight, + effectiveSourceDimensions.width, + effectiveSourceDimensions.height, gifSizePreset, GIF_SIZE_PRESETS, aspectRatioValue, @@ -1942,8 +2057,10 @@ export default function VideoEditor() { aspectRatio: aspectRatio === "native" ? getNativeAspectRatioValue( - videoPlaybackRef.current?.video?.videoWidth || 1920, - videoPlaybackRef.current?.video?.videoHeight || 1080, + videoPlaybackRef.current?.video?.videoWidth || + DEFAULT_SOURCE_DIMENSIONS.width, + videoPlaybackRef.current?.video?.videoHeight || + DEFAULT_SOURCE_DIMENSIONS.height, cropRegion, ) : getAspectRatioValue(aspectRatio), @@ -1980,6 +2097,7 @@ export default function VideoEditor() { borderRadius={borderRadius} padding={padding} cropRegion={cropRegion} + cursorRecordingData={cursorRecordingData} trimRegions={trimRegions} speedRegions={speedRegions} annotationRegions={annotationOnlyRegions} @@ -1995,8 +2113,13 @@ export default function VideoEditor() { onBlurDataChange={handleBlurDataPreviewChange} onBlurDataCommit={commitState} cursorTelemetry={cursorTelemetry} - cursorHighlight={effectiveCursorHighlight} cursorClickTimestamps={cursorClickTimestamps} + showCursor={effectiveShowCursor} + cursorSize={cursorSize} + cursorSmoothing={cursorSmoothing} + cursorMotionBlur={cursorMotionBlur} + cursorClickBounce={cursorClickBounce} + cursorClipToBounds={cursorClipToBounds} />
@@ -2019,9 +2142,6 @@ export default function VideoEditor() {
pushState({ cursorHighlight: next })} - cursorHighlightSupportsClicks={isMac} selected={wallpaper} onWallpaperChange={(w) => pushState({ wallpaper: w })} selectedZoomDepth={ @@ -2105,14 +2225,28 @@ export default function VideoEditor() { gifSizePreset={gifSizePreset} onGifSizePresetChange={setGifSizePreset} gifOutputDimensions={calculateOutputDimensions( - videoPlaybackRef.current?.video?.videoWidth || 1920, - videoPlaybackRef.current?.video?.videoHeight || 1080, + calculateEffectiveSourceDimensions( + videoPlaybackRef.current?.video?.videoWidth || + DEFAULT_SOURCE_DIMENSIONS.width, + videoPlaybackRef.current?.video?.videoHeight || + DEFAULT_SOURCE_DIMENSIONS.height, + cropRegion, + ).width, + calculateEffectiveSourceDimensions( + videoPlaybackRef.current?.video?.videoWidth || + DEFAULT_SOURCE_DIMENSIONS.width, + videoPlaybackRef.current?.video?.videoHeight || + DEFAULT_SOURCE_DIMENSIONS.height, + cropRegion, + ).height, gifSizePreset, GIF_SIZE_PRESETS, aspectRatio === "native" ? getNativeAspectRatioValue( - videoPlaybackRef.current?.video?.videoWidth || 1920, - videoPlaybackRef.current?.video?.videoHeight || 1080, + videoPlaybackRef.current?.video?.videoWidth || + DEFAULT_SOURCE_DIMENSIONS.width, + videoPlaybackRef.current?.video?.videoHeight || + DEFAULT_SOURCE_DIMENSIONS.height, cropRegion, ) : getAspectRatioValue(aspectRatio), @@ -2142,6 +2276,22 @@ export default function VideoEditor() { unsavedExport={unsavedExport} onSaveUnsavedExport={handleSaveUnsavedExport} onSaveDiagnostic={handleSaveDiagnostic} + showCursor={showCursor} + onShowCursorChange={setShowCursor} + cursorSize={cursorSize} + onCursorSizeChange={setCursorSize} + cursorSmoothing={cursorSmoothing} + onCursorSmoothingChange={setCursorSmoothing} + cursorMotionBlur={cursorMotionBlur} + onCursorMotionBlurChange={setCursorMotionBlur} + cursorClickBounce={cursorClickBounce} + onCursorClickBounceChange={setCursorClickBounce} + cursorClipToBounds={cursorClipToBounds} + onCursorClipToBoundsChange={setCursorClipToBounds} + hasCursorData={ + cursorTelemetry.length > 0 || hasNativeCursorRecordingData(cursorRecordingData) + } + showCursorSettings={showCursorSettings} />
diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx index 90a3dbd..5b25ced 100644 --- a/src/components/video-editor/VideoPlayback.tsx +++ b/src/components/video-editor/VideoPlayback.tsx @@ -25,25 +25,48 @@ import { type WebcamLayoutPreset, type WebcamSizePreset, } from "@/lib/compositeLayout"; +import { + createNativeCursorMotionBlurState, + createNativeCursorSmoothingState, + getNativeCursorClickBounceProgress, + getNativeCursorClickBounceScale, + getNativeCursorMotionBlurPx, + hasNativeCursorRecordingData, + projectNativeCursorToLocal, + projectNativeCursorToStage, + resetNativeCursorMotionBlurState, + resetNativeCursorSmoothingState, + resolveInterpolatedNativeCursorFrame, + resolveNativeCursorRenderAsset, + smoothNativeCursorSample, +} from "@/lib/cursor/nativeCursor"; import { classifyWallpaper, DEFAULT_WALLPAPER, resolveImageWallpaperUrl } from "@/lib/wallpaper"; import { getCssClipPath } from "@/lib/webcamMaskShapes"; +import type { CursorRecordingData } from "@/native/contracts"; import { type AspectRatio, formatAspectRatioForCSS, getNativeAspectRatioValue, } from "@/utils/aspectRatioUtils"; import { AnnotationOverlay } from "./AnnotationOverlay"; +import { + DEFAULT_CURSOR_SETTINGS, + DEFAULT_EDITOR_LAYOUT_SETTINGS, + DEFAULT_SOURCE_DIMENSIONS, +} from "./editorDefaults"; import { type AnnotationRegion, type BlurData, + type CursorTelemetryPoint, computeRotation3DContainScale, DEFAULT_ROTATION_3D, - getZoomScale, isRotation3DIdentity, lerpRotation3D, rotation3DPerspective, type SpeedRegion, type TrimRegion, + ZOOM_DEPTH_SCALES, + type ZoomDepth, type ZoomFocus, type ZoomRegion, } from "./types"; @@ -55,18 +78,13 @@ import { ZOOM_SCALE_DEADZONE, ZOOM_TRANSLATION_DEADZONE_PX, } from "./videoPlayback/constants"; +import { adaptiveSmoothFactor, smoothCursorFocus } from "./videoPlayback/cursorFollowUtils"; import { - adaptiveSmoothFactor, - interpolateCursorAt, - smoothCursorFocus, -} from "./videoPlayback/cursorFollowUtils"; -import { - type CursorHighlightConfig, - clickEmphasisAlpha, - DEFAULT_CURSOR_HIGHLIGHT, - drawCursorHighlightGraphics, -} from "./videoPlayback/cursorHighlight"; -import { clampFocusToScale } from "./videoPlayback/focusUtils"; + DEFAULT_CURSOR_CONFIG, + PixiCursorOverlay, + preloadCursorAssets, +} from "./videoPlayback/cursorRenderer"; +import { clampFocusToStage as clampFocusToStageUtil } from "./videoPlayback/focusUtils"; import { layoutVideoContent as layoutVideoContentUtil } from "./videoPlayback/layoutUtils"; import { clamp01 } from "./videoPlayback/mathUtils"; import { updateOverlayIndicator } from "./videoPlayback/overlayUtils"; @@ -80,13 +98,6 @@ import { type MotionBlurState, } from "./videoPlayback/zoomTransform"; -type BlurPreviewCanvasSource = { - clientHeight?: number; - clientWidth?: number; - height: number; - width: number; -}; - interface VideoPlaybackProps { videoPath: string; webcamVideoPath?: string; @@ -118,6 +129,7 @@ interface VideoPlaybackProps { trimRegions?: TrimRegion[]; speedRegions?: SpeedRegion[]; aspectRatio: AspectRatio; + cursorRecordingData?: CursorRecordingData | null; annotationRegions?: AnnotationRegion[]; selectedAnnotationId?: string | null; onSelectAnnotation?: (id: string | null) => void; @@ -130,9 +142,14 @@ interface VideoPlaybackProps { onBlurSizeChange?: (id: string, size: { width: number; height: number }) => void; onBlurDataChange?: (id: string, blurData: BlurData) => void; onBlurDataCommit?: () => void; - cursorTelemetry?: import("./types").CursorTelemetryPoint[]; - cursorHighlight?: CursorHighlightConfig; + cursorTelemetry?: CursorTelemetryPoint[]; cursorClickTimestamps?: number[]; + showCursor?: boolean; + cursorSize?: number; + cursorSmoothing?: number; + cursorMotionBlur?: number; + cursorClickBounce?: number; + cursorClipToBounds?: boolean; } export interface VideoPlaybackRef { @@ -145,6 +162,61 @@ export interface VideoPlaybackRef { pause: () => void; } +function getResolvedVideoDuration(video: HTMLVideoElement): number | null { + if (Number.isFinite(video.duration) && video.duration > 0) { + return video.duration; + } + + if (video.seekable.length > 0) { + const lastRangeIndex = video.seekable.length - 1; + const seekableEnd = video.seekable.end(lastRangeIndex); + if (Number.isFinite(seekableEnd) && seekableEnd > 0) { + return seekableEnd; + } + } + + return null; +} + +function getEndedVideoDuration(video: HTMLVideoElement): number | null { + const currentTime = video.currentTime; + if (!Number.isFinite(currentTime) || currentTime <= 0) { + return null; + } + + if (video.ended) { + return currentTime; + } + + const resolvedDuration = getResolvedVideoDuration(video); + const durationEpsilonSeconds = 0.05; + if (resolvedDuration && currentTime >= resolvedDuration - durationEpsilonSeconds) { + return resolvedDuration; + } + + return null; +} + +type AudioTrackListLike = { + length: number; + [index: number]: { enabled: boolean }; +}; + +type VideoElementWithAudioTracks = HTMLVideoElement & { + audioTracks?: AudioTrackListLike; +}; + +function enableAllPreviewAudioTracks(video: HTMLVideoElement) { + const audioTracks = (video as VideoElementWithAudioTracks).audioTracks; + if (!audioTracks || audioTracks.length <= 1) { + return; + } + + for (let index = 0; index < audioTracks.length; index += 1) { + audioTracks[index].enabled = true; + } +} + const VideoPlayback = forwardRef( ( { @@ -173,11 +245,12 @@ const VideoPlayback = forwardRef( showBlur, motionBlurAmount = 0, borderRadius = 0, - padding = 50, + padding = DEFAULT_EDITOR_LAYOUT_SETTINGS.padding, cropRegion, trimRegions = [], speedRegions = [], aspectRatio, + cursorRecordingData, annotationRegions = [], selectedAnnotationId, onSelectAnnotation, @@ -191,12 +264,18 @@ const VideoPlayback = forwardRef( onBlurDataChange, onBlurDataCommit, cursorTelemetry = [], - cursorHighlight = DEFAULT_CURSOR_HIGHLIGHT, cursorClickTimestamps = [], + showCursor = false, + cursorSize = DEFAULT_CURSOR_SETTINGS.size, + cursorSmoothing = DEFAULT_CURSOR_SETTINGS.smoothing, + cursorMotionBlur = DEFAULT_CURSOR_SETTINGS.motionBlur, + cursorClickBounce = DEFAULT_CURSOR_SETTINGS.clickBounce, + cursorClipToBounds = DEFAULT_CURSOR_SETTINGS.clipToBounds, }, ref, ) => { const videoRef = useRef(null); + const supplementalAudioRef = useRef(null); const webcamVideoRef = useRef(null); const containerRef = useRef(null); const appRef = useRef(null); @@ -206,6 +285,7 @@ const VideoPlayback = forwardRef( const timeUpdateAnimationRef = useRef(null); const [pixiReady, setPixiReady] = useState(false); const [videoReady, setVideoReady] = useState(false); + const [supplementalAudioPath, setSupplementalAudioPath] = useState(null); const [overlaySize, setOverlaySize] = useState({ width: 800, height: 600 }); const [overlayElement, setOverlayElement] = useState(null); const overlayRef = useRef(null); @@ -217,10 +297,8 @@ const VideoPlayback = forwardRef( const [webcamDimensions, setWebcamDimensions] = useState(null); const currentTimeRef = useRef(0); const zoomRegionsRef = useRef([]); - const cursorTelemetryRef = useRef([]); - const cursorHighlightRef = useRef(DEFAULT_CURSOR_HIGHLIGHT); + const cursorTelemetryRef = useRef([]); const cursorClickTimestampsRef = useRef([]); - const cursorHighlightGraphicsRef = useRef(null); const selectedZoomIdRef = useRef(null); const animationStateRef = useRef({ scale: 1, @@ -257,18 +335,146 @@ const VideoPlayback = forwardRef( const trimRegionsRef = useRef([]); const speedRegionsRef = useRef([]); const motionBlurAmountRef = useRef(motionBlurAmount); + const cursorOverlayRef = useRef(null); + const showCursorRef = useRef(showCursor); + const cursorSizeRef = useRef(cursorSize); + const cursorSmoothingRef = useRef(cursorSmoothing); + const cursorMotionBlurRef = useRef(cursorMotionBlur); + const cursorClickBounceRef = useRef(cursorClickBounce); + const cursorClipToBoundsRef = useRef(cursorClipToBounds); const motionBlurStateRef = useRef(createMotionBlurState()); const onTimeUpdateRef = useRef(onTimeUpdate); const onPlayStateChangeRef = useRef(onPlayStateChange); const videoReadyRafRef = useRef(null); const smoothedAutoFocusRef = useRef(null); const prevTargetProgressRef = useRef(0); - const blurPreviewSnapshotRef = useRef<{ - bucket: number; - canvas: BlurPreviewCanvasSource | null; - height: number; - width: number; - }>({ bucket: -1, canvas: null, height: 0, width: 0 }); + const durationResolutionTimeoutRef = useRef(null); + const lastResolvedDurationRef = useRef(null); + const isResolvingDurationRef = useRef(false); + const hasNativeCursorRecordingRef = useRef(false); + const cursorRecordingDataRef = useRef(cursorRecordingData); + const cropRegionRef = useRef(cropRegion); + const nativeCursorSpriteRef = useRef(null); + const nativeCursorTextureIdRef = useRef(null); + const nativeCursorImageRef = useRef(null); + const nativeCursorImageIdRef = useRef(null); + const nativeCursorSmoothingStateRef = useRef(createNativeCursorSmoothingState()); + const nativeCursorMotionBlurStateRef = useRef(createNativeCursorMotionBlurState()); + const nativeCursorClipRef = useRef(null); + const borderRadiusRef = useRef(0); + + const hasNativeCursorRecording = useMemo( + () => hasNativeCursorRecordingData(cursorRecordingData), + [cursorRecordingData], + ); + + const syncResolvedDuration = useCallback( + (video: HTMLVideoElement) => { + const resolvedDuration = getResolvedVideoDuration(video); + if (!resolvedDuration) { + return false; + } + + const normalizedDuration = Math.round(resolvedDuration * 1000) / 1000; + if (lastResolvedDurationRef.current !== normalizedDuration) { + lastResolvedDurationRef.current = normalizedDuration; + onDurationChange(normalizedDuration); + } + + return true; + }, + [onDurationChange], + ); + + const forceResolveDuration = useCallback( + (video: HTMLVideoElement) => { + if (isResolvingDurationRef.current) { + return; + } + + if (video.readyState < HTMLMediaElement.HAVE_METADATA) { + return; + } + + isResolvingDurationRef.current = true; + const previousMuted = video.muted; + + const finalize = () => { + video.removeEventListener("durationchange", handleProgress); + video.removeEventListener("timeupdate", handleProgress); + video.removeEventListener("loadeddata", handleProgress); + video.removeEventListener("ended", handleProgress); + if (durationResolutionTimeoutRef.current) { + clearTimeout(durationResolutionTimeoutRef.current); + durationResolutionTimeoutRef.current = null; + } + video.muted = previousMuted; + isResolvingDurationRef.current = false; + }; + + const resolveCurrentDuration = () => { + if (syncResolvedDuration(video)) { + return true; + } + + const endedDuration = getEndedVideoDuration(video); + if (endedDuration) { + lastResolvedDurationRef.current = null; + onDurationChange(Math.round(endedDuration * 1000) / 1000); + return true; + } + + return false; + }; + + const handleProgress = () => { + if (!resolveCurrentDuration()) { + return; + } + + try { + video.pause(); + video.currentTime = 0; + } catch { + // no-op + } + currentTimeRef.current = 0; + finalize(); + }; + + video.addEventListener("durationchange", handleProgress); + video.addEventListener("timeupdate", handleProgress); + video.addEventListener("loadeddata", handleProgress); + video.addEventListener("ended", handleProgress); + durationResolutionTimeoutRef.current = window.setTimeout(() => { + handleProgress(); + finalize(); + }, 1500); + video.muted = true; + + const playAttempt = video.play(); + if (playAttempt && typeof playAttempt.catch === "function") { + playAttempt.catch(() => { + try { + video.currentTime = Math.max(video.currentTime, 0.1); + } catch { + finalize(); + } + }); + } + + try { + video.currentTime = Math.max(video.currentTime, 0.1); + } catch { + finalize(); + } + }, + [onDurationChange, syncResolvedDuration], + ); + + const clampFocusToStage = useCallback((focus: ZoomFocus, depth: ZoomDepth) => { + return clampFocusToStageUtil(focus, depth, stageSizeRef.current); + }, []); const updateOverlayForRegion = useCallback( (region: ZoomRegion | null, focusOverride?: ZoomFocus) => { @@ -353,6 +559,7 @@ const VideoPlayback = forwardRef( baseScaleRef.current = result.baseScale; baseOffsetRef.current = result.baseOffset; baseMaskRef.current = result.maskRect; + borderRadiusRef.current = result.maskBorderRadius; cropBoundsRef.current = result.cropBounds; setWebcamLayout(result.webcamRect); @@ -404,10 +611,19 @@ const VideoPlayback = forwardRef( if (!vid) return; try { allowPlaybackRef.current = true; + enableAllPreviewAudioTracks(vid); await vid.play().catch((err) => { console.log("PLAY ERROR:", err); throw err; }); + const supplementalAudio = supplementalAudioRef.current; + if (supplementalAudio) { + supplementalAudio.currentTime = vid.currentTime; + supplementalAudio.playbackRate = vid.playbackRate; + await supplementalAudio.play().catch(() => { + // The main video remains the source of truth for playback state. + }); + } } catch (error) { allowPlaybackRef.current = false; throw error; @@ -420,6 +636,7 @@ const VideoPlayback = forwardRef( return; } video.pause(); + supplementalAudioRef.current?.pause(); }, })); @@ -450,7 +667,7 @@ const VideoPlayback = forwardRef( cx: clamp01(localX / stageWidth), cy: clamp01(localY / stageHeight), }; - const clampedFocus = clampFocusToScale(unclampedFocus, getZoomScale(region)); + const clampedFocus = clampFocusToStage(unclampedFocus, region.depth); onZoomFocusChange(region.id, clampedFocus); updateOverlayForRegion({ ...region, focus: clampedFocus }, clampedFocus); @@ -550,13 +767,6 @@ const VideoPlayback = forwardRef( cursorTelemetryRef.current = cursorTelemetry; }, [cursorTelemetry]); - useEffect(() => { - cursorHighlightRef.current = cursorHighlight; - if (cursorHighlightGraphicsRef.current) { - drawCursorHighlightGraphics(cursorHighlightGraphicsRef.current, cursorHighlight); - } - }, [cursorHighlight]); - useEffect(() => { cursorClickTimestampsRef.current = cursorClickTimestamps; }, [cursorClickTimestamps]); @@ -581,6 +791,59 @@ const VideoPlayback = forwardRef( motionBlurAmountRef.current = motionBlurAmount; }, [motionBlurAmount]); + useEffect(() => { + cursorTelemetryRef.current = cursorTelemetry; + }, [cursorTelemetry]); + + useEffect(() => { + showCursorRef.current = showCursor; + }, [showCursor]); + + useEffect(() => { + hasNativeCursorRecordingRef.current = hasNativeCursorRecording; + }, [hasNativeCursorRecording]); + + useEffect(() => { + cursorRecordingDataRef.current = cursorRecordingData; + resetNativeCursorSmoothingState(nativeCursorSmoothingStateRef.current); + resetNativeCursorMotionBlurState(nativeCursorMotionBlurStateRef.current); + }, [cursorRecordingData]); + + useEffect(() => { + cropRegionRef.current = cropRegion; + }, [cropRegion]); + + useEffect(() => { + cursorSizeRef.current = cursorSize; + }, [cursorSize]); + + useEffect(() => { + cursorSmoothingRef.current = cursorSmoothing; + }, [cursorSmoothing]); + + useEffect(() => { + cursorMotionBlurRef.current = cursorMotionBlur; + }, [cursorMotionBlur]); + + useEffect(() => { + cursorClickBounceRef.current = cursorClickBounce; + }, [cursorClickBounce]); + + useEffect(() => { + cursorClipToBoundsRef.current = cursorClipToBounds; + }, [cursorClipToBounds]); + + // Sync cursor overlay config when settings change + useEffect(() => { + const overlay = cursorOverlayRef.current; + if (!overlay) return; + overlay.setDotRadius(DEFAULT_CURSOR_CONFIG.dotRadius * cursorSize); + overlay.setSmoothingFactor(cursorSmoothing); + overlay.setMotionBlur(cursorMotionBlur); + overlay.setClickBounce(cursorClickBounce); + overlay.reset(); + }, [cursorSize, cursorSmoothing, cursorMotionBlur, cursorClickBounce]); + useEffect(() => { onTimeUpdateRef.current = onTimeUpdate; }, [onTimeUpdate]); @@ -700,6 +963,13 @@ const VideoPlayback = forwardRef( let app: Application | null = null; (async () => { + let cursorOverlayEnabled = true; + try { + await preloadCursorAssets(); + } catch { + cursorOverlayEnabled = false; + } + app = new Application(); await app.init({ @@ -735,12 +1005,30 @@ const VideoPlayback = forwardRef( videoContainerRef.current = videoContainer; cameraContainer.addChild(videoContainer); + // Cursor overlay - rendered above the masked video + if (cursorOverlayEnabled) { + const cursorOverlay = new PixiCursorOverlay({ + dotRadius: DEFAULT_CURSOR_CONFIG.dotRadius * cursorSizeRef.current, + smoothingFactor: cursorSmoothingRef.current, + motionBlur: cursorMotionBlurRef.current, + clickBounce: cursorClickBounceRef.current, + }); + cursorOverlayRef.current = cursorOverlay; + } + setPixiReady(true); })(); return () => { mounted = false; setPixiReady(false); + if (cursorOverlayRef.current) { + cursorOverlayRef.current.destroy(); + cursorOverlayRef.current = null; + } + nativeCursorSpriteRef.current = null; + nativeCursorTextureIdRef.current = null; + nativeCursorImageIdRef.current = null; if (app && app.renderer) { app.destroy(true, { children: true, @@ -757,23 +1045,85 @@ const VideoPlayback = forwardRef( useEffect(() => { if (!videoPath) { + lastResolvedDurationRef.current = null; + isResolvingDurationRef.current = false; setVideoReady(false); + setSupplementalAudioPath(null); return; } + let cancelled = false; + window.electronAPI + ?.preparePreviewAudioTrack?.(videoPath) + .then((result) => { + if (!cancelled) { + setSupplementalAudioPath(result.success ? (result.path ?? null) : null); + } + }) + .catch(() => { + if (!cancelled) { + setSupplementalAudioPath(null); + } + }); + const video = videoRef.current; - if (!video) return; + if (!video) { + return () => { + cancelled = true; + }; + } video.pause(); video.currentTime = 0; allowPlaybackRef.current = false; lockedVideoDimensionsRef.current = null; + lastResolvedDurationRef.current = null; + isResolvingDurationRef.current = false; + if (durationResolutionTimeoutRef.current) { + clearTimeout(durationResolutionTimeoutRef.current); + durationResolutionTimeoutRef.current = null; + } setVideoReady(false); if (videoReadyRafRef.current) { cancelAnimationFrame(videoReadyRafRef.current); videoReadyRafRef.current = null; } + video.load(); + + return () => { + cancelled = true; + }; }, [videoPath]); + useEffect(() => { + const video = videoRef.current; + const supplementalAudio = supplementalAudioRef.current; + if (!video || !supplementalAudio || !supplementalAudioPath) { + return; + } + + const activeSpeedRegion = + speedRegions.find( + (region) => currentTime * 1000 >= region.startMs && currentTime * 1000 < region.endMs, + ) ?? null; + supplementalAudio.playbackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1; + + if (!isPlaying) { + supplementalAudio.pause(); + if (Math.abs(supplementalAudio.currentTime - currentTime) > 0.05) { + supplementalAudio.currentTime = currentTime; + } + return; + } + + if (Math.abs(supplementalAudio.currentTime - video.currentTime) > 0.15) { + supplementalAudio.currentTime = video.currentTime; + } + + supplementalAudio.play().catch(() => { + // Keep video playback running even if supplemental preview audio is unavailable. + }); + }, [currentTime, isPlaying, speedRegions, supplementalAudioPath]); + useEffect(() => { if (!pixiReady || !videoReady) return; @@ -801,12 +1151,15 @@ const VideoPlayback = forwardRef( videoContainer.addChild(maskGraphics); videoContainer.mask = maskGraphics; maskGraphicsRef.current = maskGraphics; + const nativeCursorSprite = new Sprite(Texture.EMPTY); + nativeCursorSprite.visible = false; + nativeCursorSprite.eventMode = "none"; + nativeCursorSpriteRef.current = nativeCursorSprite; + if (cursorOverlayRef.current) { + videoContainer.addChild(cursorOverlayRef.current.container); + } - const cursorHighlightGraphics = new Graphics(); - cursorHighlightGraphics.visible = false; - videoContainer.addChild(cursorHighlightGraphics); - cursorHighlightGraphicsRef.current = cursorHighlightGraphics; - drawCursorHighlightGraphics(cursorHighlightGraphics, cursorHighlightRef.current); + videoContainer.addChild(nativeCursorSprite); animationStateRef.current = { scale: 1, @@ -870,10 +1223,11 @@ const VideoPlayback = forwardRef( videoContainer.removeChild(maskGraphics); maskGraphics.destroy(); } - if (cursorHighlightGraphicsRef.current) { - videoContainer.removeChild(cursorHighlightGraphicsRef.current); - cursorHighlightGraphicsRef.current.destroy(); - cursorHighlightGraphicsRef.current = null; + if (nativeCursorSpriteRef.current) { + videoContainer.removeChild(nativeCursorSpriteRef.current); + nativeCursorSpriteRef.current.destroy(); + nativeCursorSpriteRef.current = null; + nativeCursorTextureIdRef.current = null; } videoContainer.mask = null; maskGraphicsRef.current = null; @@ -959,7 +1313,7 @@ const VideoPlayback = forwardRef( const shouldShowUnzoomedView = hasSelectedZoom && !isPlayingRef.current; if (region && strength > 0 && !shouldShowUnzoomedView) { - const zoomScale = blendedScale ?? getZoomScale(region); + const zoomScale = blendedScale ?? ZOOM_DEPTH_SCALES[region.depth]; const regionFocus = region.focus; targetScaleFactor = zoomScale; @@ -1096,39 +1450,6 @@ const VideoPlayback = forwardRef( motionVector, ); - const cursorGraphics = cursorHighlightGraphicsRef.current; - const cursorConfig = cursorHighlightRef.current; - const lockedDims = lockedVideoDimensionsRef.current; - if (cursorGraphics) { - if (cursorConfig.enabled && lockedDims && cursorTelemetryRef.current.length > 0) { - const emphasisAlpha = clickEmphasisAlpha( - currentTimeRef.current, - cursorClickTimestampsRef.current, - cursorConfig, - ); - const cursorPoint = - emphasisAlpha > 0 - ? interpolateCursorAt(cursorTelemetryRef.current, currentTimeRef.current) - : null; - if (cursorPoint) { - const baseScale = baseScaleRef.current; - const baseOffset = baseOffsetRef.current; - const cx = cursorPoint.cx + cursorConfig.offsetXNorm; - const cy = cursorPoint.cy + cursorConfig.offsetYNorm; - cursorGraphics.position.set( - baseOffset.x + cx * lockedDims.width * baseScale, - baseOffset.y + cy * lockedDims.height * baseScale, - ); - cursorGraphics.alpha = emphasisAlpha; - cursorGraphics.visible = true; - } else { - cursorGraphics.visible = false; - } - } else { - cursorGraphics.visible = false; - } - } - const isMotionBlurActive = (motionBlurAmountRef.current || 0) > 0 && isPlayingRef.current && !isScrubbingRef.current; @@ -1147,6 +1468,159 @@ const VideoPlayback = forwardRef( } } + // Update cursor overlay + const cursorOverlay = cursorOverlayRef.current; + if (cursorOverlay) { + const timeMs = currentTimeRef.current; // already in ms + cursorOverlay.update( + cursorTelemetryRef.current, + timeMs, + baseMaskRef.current, + showCursorRef.current && !hasNativeCursorRecordingRef.current, + !isPlayingRef.current || isSeekingRef.current, + ); + } + + // Keep the native cursor preview in the same transformed coordinate space as PIXI. + const nativeCursorSprite = nativeCursorSpriteRef.current; + const nativeCursorImage = nativeCursorImageRef.current; + const hideNativeCursorPreview = () => { + if (nativeCursorSprite) { + nativeCursorSprite.visible = false; + } + if (nativeCursorImage) { + nativeCursorImage.style.display = "none"; + nativeCursorImage.style.filter = "none"; + } + if (nativeCursorClipRef.current) { + nativeCursorClipRef.current.style.clipPath = ""; + } + resetNativeCursorSmoothingState(nativeCursorSmoothingStateRef.current); + resetNativeCursorMotionBlurState(nativeCursorMotionBlurStateRef.current); + }; + if (nativeCursorImage) { + if (hasNativeCursorRecordingRef.current && showCursorRef.current) { + const timeMs = currentTimeRef.current; // already in ms + const frame = resolveInterpolatedNativeCursorFrame( + cursorRecordingDataRef.current, + timeMs, + ); + if (frame) { + const displaySample = smoothNativeCursorSample({ + forceSnap: !isPlayingRef.current || isSeekingRef.current, + sample: frame.sample, + smoothing: cursorSmoothingRef.current, + state: nativeCursorSmoothingStateRef.current, + timeMs, + }); + const cameraContainer = cameraContainerRef.current; + const videoContainer = videoContainerRef.current; + const cropRegionValue = cropRegionRef.current ?? { x: 0, y: 0, width: 1, height: 1 }; + const projectedLocalPoint = projectNativeCursorToLocal({ + cropRegion: cropRegionValue, + maskRect: baseMaskRef.current, + sample: displaySample, + }); + const projectedStagePoint = + cameraContainer && videoContainer + ? projectNativeCursorToStage({ + cameraContainer, + cropRegion: cropRegionValue, + maskRect: baseMaskRef.current, + videoContainerPosition: { + x: videoContainer.x, + y: videoContainer.y, + }, + sample: displaySample, + }) + : null; + if (projectedLocalPoint && projectedStagePoint) { + // Pass deviceScaleFactor=1 — asset.scaleFactor already encodes DPR. + // Size is normalized below so preview matches export proportionally. + const renderAsset = resolveNativeCursorRenderAsset(frame.asset, 1, displaySample); + const bounceProgress = getNativeCursorClickBounceProgress( + cursorRecordingDataRef.current, + timeMs, + ); + const scale = + Math.max(0, cursorSizeRef.current) * + getNativeCursorClickBounceScale(cursorClickBounceRef.current, bounceProgress); + // Normalize cursor size to the displayed video width so the cursor + // appears at the same fraction of the video in both preview and export. + const crop = cropRegionRef.current ?? { x: 0, y: 0, width: 1, height: 1 }; + const croppedVideoWidth = (videoRef.current?.videoWidth ?? 0) * crop.width; + const sizeNorm = + croppedVideoWidth > 0 ? baseMaskRef.current.width / croppedVideoWidth : 1; + const transformedScale = scale * Math.abs(cameraContainer?.scale.x || 1) * sizeNorm; + const blurPx = + !isPlayingRef.current || isSeekingRef.current + ? 0 + : getNativeCursorMotionBlurPx({ + motionBlur: cursorMotionBlurRef.current, + point: projectedStagePoint, + state: nativeCursorMotionBlurStateRef.current, + timeMs, + }); + if (nativeCursorImageIdRef.current !== renderAsset.id) { + nativeCursorImage.src = renderAsset.imageDataUrl; + nativeCursorImageIdRef.current = renderAsset.id; + } + nativeCursorImage.style.display = "block"; + // Update clip-path on nativeCursorClipRef to the camera-aware video boundary. + // clip-path works correctly here because nativeCursorClipRef is outside preserve-3d. + // When cursorClipToBounds is off, allow the cursor to overflow the canvas. + if (nativeCursorClipRef.current) { + if (!cursorClipToBoundsRef.current) { + nativeCursorClipRef.current.style.clipPath = "none"; + } else { + const mask = baseMaskRef.current; + const stage = stageSizeRef.current; + const br = borderRadiusRef.current; + const s = cameraContainer ? Math.abs(cameraContainer.scale.x) : 1; + const camX = cameraContainer ? cameraContainer.position.x : 0; + const camY = cameraContainer ? cameraContainer.position.y : 0; + const clipLeft = camX + s * mask.x; + const clipTop = camY + s * mask.y; + const clipRight = camX + s * (mask.x + mask.width); + const clipBottom = camY + s * (mask.y + mask.height); + nativeCursorClipRef.current.style.clipPath = `inset(${clipTop}px ${stage.width - clipRight}px ${stage.height - clipBottom}px ${clipLeft}px round ${br * s}px)`; + } + } + nativeCursorImage.style.width = `${renderAsset.width * transformedScale}px`; + nativeCursorImage.style.height = `${renderAsset.height * transformedScale}px`; + nativeCursorImage.style.filter = + blurPx > 0 ? `blur(${blurPx.toFixed(2)}px)` : "none"; + // translate3d is relative to nativeCursorClipRef (absolute inset-0 = stage origin). + // projectedStagePoint.x is the stage-space cursor position — no offset needed. + nativeCursorImage.style.transform = `translate3d(${ + projectedStagePoint.x - renderAsset.hotspotX * transformedScale + }px, ${projectedStagePoint.y - renderAsset.hotspotY * transformedScale}px, 0)`; + if (nativeCursorSprite) { + nativeCursorSprite.visible = false; + if (nativeCursorTextureIdRef.current !== renderAsset.id) { + nativeCursorSprite.texture = Texture.from(renderAsset.imageDataUrl); + nativeCursorTextureIdRef.current = renderAsset.id; + } + nativeCursorSprite.position.set( + projectedLocalPoint.x - renderAsset.hotspotX * scale, + projectedLocalPoint.y - renderAsset.hotspotY * scale, + ); + nativeCursorSprite.width = renderAsset.width * scale; + nativeCursorSprite.height = renderAsset.height * scale; + } + } else { + hideNativeCursorPreview(); + } + } else { + hideNativeCursorPreview(); + } + } else { + hideNativeCursorPreview(); + } + } else { + hideNativeCursorPreview(); + } + const composite3D = composite3DRef.current; const outerWrapper = outerWrapperRef.current; if (composite3D && outerWrapper) { @@ -1161,6 +1635,9 @@ const VideoPlayback = forwardRef( composite3D.style.willChange = "auto"; lastTransformIsIdentity = true; } + if (nativeCursorClipRef.current) { + nativeCursorClipRef.current.style.transform = ""; + } if (lastPerspectiveValue !== 0) { outerWrapper.style.perspective = ""; lastPerspectiveValue = 0; @@ -1177,6 +1654,9 @@ const VideoPlayback = forwardRef( ); composite3D.style.transform = `scale(${containScale}) rotateX(${effectiveRotation.rotationX}deg) rotateY(${effectiveRotation.rotationY}deg) rotateZ(${effectiveRotation.rotationZ}deg)`; composite3D.style.willChange = "transform"; + if (nativeCursorClipRef.current) { + nativeCursorClipRef.current.style.transform = composite3D.style.transform; + } lastTransformIsIdentity = false; if (persp !== lastPerspectiveValue) { outerWrapper.style.perspective = `${persp}px`; @@ -1196,8 +1676,13 @@ const VideoPlayback = forwardRef( const handleLoadedMetadata = (e: React.SyntheticEvent) => { const video = e.currentTarget; - onDurationChange(video.duration); - video.currentTime = 0; + enableAllPreviewAudioTracks(video); + const hasResolvedDuration = syncResolvedDuration(video); + if (!hasResolvedDuration) { + forceResolveDuration(video); + } else { + video.currentTime = 0; + } video.pause(); allowPlaybackRef.current = false; currentTimeRef.current = 0; @@ -1210,6 +1695,9 @@ const VideoPlayback = forwardRef( const waitForRenderableFrame = () => { const hasDimensions = video.videoWidth > 0 && video.videoHeight > 0; const hasData = video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA; + if (!syncResolvedDuration(video)) { + forceResolveDuration(video); + } if (hasDimensions && hasData) { videoReadyRafRef.current = null; setVideoReady(true); @@ -1309,6 +1797,10 @@ const VideoPlayback = forwardRef( window.clearTimeout(scrubEndTimerRef.current); scrubEndTimerRef.current = null; } + if (durationResolutionTimeoutRef.current) { + clearTimeout(durationResolutionTimeoutRef.current); + durationResolutionTimeoutRef.current = null; + } }; }, []); @@ -1333,8 +1825,8 @@ const VideoPlayback = forwardRef( aspectRatio, aspectRatio === "native" ? getNativeAspectRatioValue( - lockedVideoDimensionsRef.current?.width || 1920, - lockedVideoDimensionsRef.current?.height || 1080, + lockedVideoDimensionsRef.current?.width || DEFAULT_SOURCE_DIMENSIONS.width, + lockedVideoDimensionsRef.current?.height || DEFAULT_SOURCE_DIMENSIONS.height, cropRegion, ) : undefined, @@ -1461,32 +1953,15 @@ const VideoPlayback = forwardRef( region: blurRegion, })), ].sort((a, b) => a.region.zIndex - b.region.zIndex); - const previewSnapshotBucket = Math.floor(currentTime * 10); const previewSnapshotCanvas = filteredBlurRegions.length > 0 ? (() => { - const cached = blurPreviewSnapshotRef.current; - if ( - cached.bucket === previewSnapshotBucket && - cached.width === overlaySize.width && - cached.height === overlaySize.height - ) { - return cached.canvas; - } - const app = appRef.current; - if (!app?.renderer?.extract) return cached.canvas; + if (!app?.renderer?.extract) return null; try { - const canvas = app.renderer.extract.canvas(app.stage); - blurPreviewSnapshotRef.current = { - bucket: previewSnapshotBucket, - canvas, - height: overlaySize.height, - width: overlaySize.width, - }; - return canvas; + return app.renderer.extract.canvas(app.stage); } catch { - return cached.canvas; + return null; } })() : null; @@ -1558,25 +2033,64 @@ const VideoPlayback = forwardRef( : item.region.id === selectedAnnotationId } previewSourceCanvas={previewSnapshotCanvas} - previewFrameVersion={previewSnapshotBucket} + previewFrameVersion={Math.round(currentTime * 1000)} /> )); })()}
)} + {/* Clip the native cursor overlay to the exact video canvas boundary. + Placed OUTSIDE composite3DRef (preserve-3d) so clip-path works + correctly even during 3D zoom rotation regions. + clip-path is set dynamically to the camera-aware video bounds. */} +
+ +