diff --git a/docs/engineering/macos-native-recorder-roadmap.md b/docs/engineering/macos-native-recorder-roadmap.md new file mode 100644 index 0000000..469401b --- /dev/null +++ b/docs/engineering/macos-native-recorder-roadmap.md @@ -0,0 +1,208 @@ +# macOS Native Recorder Roadmap + +OpenScreen's macOS recorder should follow the same architecture boundaries as the Windows native recorder: Electron owns session orchestration and persistence, while a platform-native helper owns capture, timing, encoding, and platform-specific permissions. + +This work is intentionally scoped as a macOS-only port. Windows native capture remains owned by the WGC helper, and Linux remains on the existing Electron path. + +## Goals + +- Capture displays and windows through ScreenCaptureKit. +- Exclude the real system cursor during capture when using the editable OpenScreen cursor overlay. +- Preserve the current high-quality cursor overlay path in preview and export. +- Capture macOS system audio through ScreenCaptureKit on supported macOS versions. +- Capture microphone audio through the same native timing domain where the OS supports it, or through an explicit companion path until it can be moved into the helper. +- Mix system audio and microphone audio into the primary MP4 without renderer-side track assembly. +- Capture webcam video natively and compose it into the helper-owned MP4 during the native-recording migration. +- Keep screen video, audio, webcam, and cursor aligned to one native timing origin. +- Package per-architecture helper binaries with macOS builds. + +## Non-Goals + +- Replacing the editor/export pipeline. +- Changing Windows native capture behavior. +- Adding Linux native capture. +- Shipping a silent fallback from native macOS capture to Electron capture when the user explicitly requested a native-only feature. + +## Architecture + +The renderer keeps the existing recording controls. On macOS, `useScreenRecorder` should eventually send a complete recording request to Electron instead of assembling display, audio, microphone, webcam, and cursor streams in the browser. + +Electron owns the native recording session: + +- resolves the selected display/window source; +- resolves output paths; +- starts cursor telemetry capture when editable cursor mode is selected; +- starts the ScreenCaptureKit helper process; +- sends pause/resume/stop/cancel commands; +- writes `RecordingSession` manifests; +- reports explicit errors when a macOS-native capability is unavailable. + +The helper owns macOS media capture: + +- ScreenCaptureKit display/window frames; +- ScreenCaptureKit system audio where supported; +- microphone capture or helper-owned companion audio capture; +- webcam capture and initial picture-in-picture composition; +- AVFoundation/VideoToolbox encoding and muxing; +- stream timestamp normalization. + +## Helper Contract V1 + +The helper receives a single JSON argument: + +```json +{ + "schemaVersion": 1, + "recordingId": 1234567890, + "source": { + "type": "display", + "sourceId": "screen:0:0", + "displayId": 1, + "windowId": null, + "bounds": { "x": 0, "y": 0, "width": 1920, "height": 1080 } + }, + "video": { + "fps": 60, + "width": 1920, + "height": 1080, + "bitrate": 18000000, + "hideSystemCursor": true + }, + "audio": { + "system": { "enabled": true }, + "microphone": { + "enabled": true, + "deviceId": "default", + "deviceName": "MacBook Pro Microphone", + "gain": 1.4 + } + }, + "webcam": { + "enabled": true, + "deviceId": "default", + "deviceName": "FaceTime HD Camera", + "width": 1280, + "height": 720, + "fps": 30 + }, + "cursor": { + "mode": "editable-overlay" + }, + "outputs": { + "screenPath": "/Users/me/Library/Application Support/openscreen/recordings/recording-123.mp4", + "manifestPath": "/Users/me/Library/Application Support/openscreen/recordings/recording-123.session.json" + } +} +``` + +The helper emits newline-delimited JSON events to stdout: + +```json +{ "event": "ready", "schemaVersion": 1 } +{ "event": "recording-started", "timestampMs": 1234567890 } +{ "event": "warning", "code": "microphone-unavailable", "message": "..." } +{ "event": "recording-stopped", "screenPath": "..." } +{ "event": "error", "code": "screen-permission-denied", "message": "..." } +``` + +## Implementation Phases + +### 1. Native Session Boundary + +- Add a structured macOS native recording request type. +- Add a macOS helper resolver and build script placeholders. +- Keep the helper contract process-based, matching the Windows helper boundary. +- Do not route production macOS recording through this helper until the helper is available and validated. + +Acceptance: + +- TypeScript build passes. +- The macOS helper path and request contract are documented and testable without affecting Windows/Linux behavior. + +### 2. ScreenCaptureKit Display Capture + +- Implement a Swift helper using ScreenCaptureKit. +- Select display captures by `displayId`. +- Encode H.264 MP4 through AVFoundation/VideoToolbox. +- Set `showsCursor = false` when editable cursor overlay mode is selected. + +Acceptance: + +- Display-only recording produces a valid MP4. +- The real cursor is not baked into editable-cursor recordings. + +### 3. ScreenCaptureKit Window Capture + +- Resolve Electron `window:*` selections to ScreenCaptureKit window ids. +- Capture `SCContentFilter(desktopIndependentWindow:)`. +- Handle closed/minimized/protected windows with explicit errors. +- Keep window selection and capture source resolution in Electron/main, not the renderer. + +Acceptance: + +- Capturing a normal app window works with cursor/audio/webcam disabled. +- Unsupported windows return clear native errors. + +### 4. System Audio + +- Enable ScreenCaptureKit system audio on supported macOS versions. +- Keep audio format and timing owned by the helper. +- Encode or mux AAC audio into the primary MP4. + +Acceptance: + +- System-audio-only recordings produce a valid AAC track. +- Unsupported macOS versions return an explicit capability error. + +### 5. Microphone + +- Resolve the selected microphone device from the renderer-provided browser `deviceId` and user-visible label. +- Capture microphone audio in the helper timing domain. +- Apply OpenScreen microphone gain policy. +- Mix system and microphone audio before final AAC output. + +Acceptance: + +- Mic-only and mic-plus-system recordings produce a valid, balanced AAC track. +- Device selection honors the selected microphone, not only the default device. + +### 6. Webcam Composition + +- Capture the selected camera natively through AVFoundation. +- Match browser device id first where possible, then user-visible label. +- Compose an initial picture-in-picture overlay into the primary MP4. +- Hide webcam output until the first usable frame to avoid black startup flashes. + +Acceptance: + +- Native display/window recordings can include webcam without returning to Electron capture. +- Selected camera is honored. + +### 7. Runtime Controls + +- Add pause/resume commands to the helper. +- Add cancel command that removes partial outputs. +- Keep restart as stop-discard-start until the helper exposes a native restart operation. + +Acceptance: + +- Pause/resume keeps output duration coherent. +- Cancel leaves no stale media/session files. + +### 8. Test Pipeline + +- `npm run build:native:mac`: builds Swift helper binaries on macOS. +- `npm run test:sck-helper:mac`: display-only helper smoke test. +- `npm run test:sck-window:mac`: window capture smoke test. +- `npm run test:sck-audio:mac`: system audio smoke test when supported. +- `npm run test:sck-mic:mac`: microphone smoke test. +- `npm run test:sck-webcam:mac`: webcam smoke test when a webcam is available. +- Packaging check: confirms helpers are available under `electron/native/bin/darwin-${arch}` in packaged builds. + +## SSOT Rules + +- `src/lib/nativeMacRecording.ts` is the renderer/main TypeScript request contract. +- This document is the feature-level contract and phase checklist. +- The Swift helper owns ScreenCaptureKit/AVFoundation media timing. +- Electron owns output paths, session manifests, and selected source/device resolution. +- Renderer code must use existing hooks/client APIs and should not bind directly to helper process details. diff --git a/electron/native/README.md b/electron/native/README.md index 659829c..8d6ea46 100644 --- a/electron/native/README.md +++ b/electron/native/README.md @@ -1,5 +1,32 @@ # Native capture helpers +## macOS + +macOS native recording will use a ScreenCaptureKit helper with the same process boundary as the Windows WGC helper: + +1. Electron resolves the selected source, output paths, and user-selected devices. +2. The helper receives one structured JSON request. +3. The helper owns ScreenCaptureKit/AVFoundation capture, timing, encoding, and muxing. +4. Electron persists the resulting media/session manifest and reports helper errors explicitly. + +Expected development helper locations: + +1. `OPENSCREEN_SCK_CAPTURE_EXE`, for local development and diagnostics. +2. `electron/native/screencapturekit/build/openscreen-screencapturekit-helper`, for locally built Swift output. +3. `electron/native/bin/darwin-arm64/openscreen-screencapturekit-helper` or `electron/native/bin/darwin-x64/openscreen-screencapturekit-helper`, for packaged prebuilt helpers. + +The current macOS helper script is a placeholder: + +```bash +npm run build:native:mac +``` + +On non-macOS hosts this command exits successfully and does not affect Windows/Linux development. On macOS it fails until the Swift ScreenCaptureKit helper lands. + +See `docs/engineering/macos-native-recorder-roadmap.md` for the contract, rollout phases, and SSOT rules. + +## Windows + Windows native recording is resolved from one of these locations: 1. `OPENSCREEN_WGC_CAPTURE_EXE`, for local development and diagnostics. diff --git a/package.json b/package.json index 9388bcd..12d02d7 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "format": "biome format --write .", "i18n:check": "node scripts/i18n-check.mjs", "preview": "vite preview", + "build:native:mac": "node scripts/build-macos-screencapturekit-helper.mjs", "build:mac": "tsc && vite build && electron-builder --mac", "build:native:win": "node scripts/build-windows-wgc-helper.mjs", "build:win": "npm run build:native:win && tsc && vite build && electron-builder --win --config.npmRebuild=false", diff --git a/scripts/build-macos-screencapturekit-helper.mjs b/scripts/build-macos-screencapturekit-helper.mjs new file mode 100644 index 0000000..5e9292f --- /dev/null +++ b/scripts/build-macos-screencapturekit-helper.mjs @@ -0,0 +1,13 @@ +#!/usr/bin/env node + +import process from "node:process"; + +if (process.platform !== "darwin") { + console.log("Skipping macOS ScreenCaptureKit helper build: host platform is not macOS."); + process.exit(0); +} + +console.error( + "macOS ScreenCaptureKit helper sources are not implemented yet. See docs/engineering/macos-native-recorder-roadmap.md.", +); +process.exit(1); diff --git a/src/lib/nativeMacRecording.test.ts b/src/lib/nativeMacRecording.test.ts new file mode 100644 index 0000000..fce88f6 --- /dev/null +++ b/src/lib/nativeMacRecording.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vitest"; +import { parseMacDisplayIdFromSourceId, parseMacWindowIdFromSourceId } from "./nativeMacRecording"; + +describe("nativeMacRecording source parsing", () => { + it("parses Electron window source ids into ScreenCaptureKit window ids", () => { + expect(parseMacWindowIdFromSourceId("window:12345:0")).toBe(12345); + expect(parseMacWindowIdFromSourceId("window:987")).toBe(987); + }); + + it("rejects non-window source ids for window parsing", () => { + expect(parseMacWindowIdFromSourceId("screen:1:0")).toBeNull(); + expect(parseMacWindowIdFromSourceId("window:not-a-number:0")).toBeNull(); + expect(parseMacWindowIdFromSourceId(null)).toBeNull(); + }); + + it("parses Electron display source ids into ScreenCaptureKit display ids", () => { + expect(parseMacDisplayIdFromSourceId("screen:1:0")).toBe(1); + expect(parseMacDisplayIdFromSourceId("screen:69733248")).toBe(69733248); + }); + + it("rejects non-display source ids for display parsing", () => { + expect(parseMacDisplayIdFromSourceId("window:123:0")).toBeNull(); + expect(parseMacDisplayIdFromSourceId("screen:not-a-number:0")).toBeNull(); + expect(parseMacDisplayIdFromSourceId(undefined)).toBeNull(); + }); +}); diff --git a/src/lib/nativeMacRecording.ts b/src/lib/nativeMacRecording.ts new file mode 100644 index 0000000..0e596c6 --- /dev/null +++ b/src/lib/nativeMacRecording.ts @@ -0,0 +1,82 @@ +import type { Rectangle } from "electron"; +import type { CursorCaptureMode } from "./recordingSession"; + +export type NativeMacSourceType = "display" | "window"; + +export type NativeMacRecordingRequest = { + recordingId?: number; + source: { + type: NativeMacSourceType; + sourceId: string; + displayId?: number; + windowId?: number; + bounds?: Rectangle; + }; + video: { + fps: number; + width: number; + height: number; + bitrate?: number; + hideSystemCursor: boolean; + }; + audio: { + system: { + enabled: boolean; + }; + microphone: { + enabled: boolean; + deviceId?: string; + deviceName?: string; + gain: number; + }; + }; + webcam: { + enabled: boolean; + deviceId?: string; + deviceName?: string; + width: number; + height: number; + fps: number; + }; + cursor: { + mode: CursorCaptureMode; + }; + outputs: { + screenPath: string; + manifestPath?: string; + }; +}; + +export type NativeMacRecordingStartResult = { + success: boolean; + recordingId?: number; + path?: string; + helperPath?: string; + error?: string; +}; + +export function parseMacWindowIdFromSourceId(sourceId?: string | null) { + if (!sourceId?.startsWith("window:")) { + return null; + } + + const windowIdPart = sourceId.split(":")[1]; + if (!windowIdPart || !/^\d+$/.test(windowIdPart)) { + return null; + } + + return Number(windowIdPart); +} + +export function parseMacDisplayIdFromSourceId(sourceId?: string | null) { + if (!sourceId?.startsWith("screen:")) { + return null; + } + + const displayIdPart = sourceId.split(":")[1]; + if (!displayIdPart || !/^\d+$/.test(displayIdPart)) { + return null; + } + + return Number(displayIdPart); +}