diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 85e8f40..3530dcb 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -31,7 +31,16 @@ interface Window { switchToEditor: () => Promise; switchToHud: () => Promise; startNewRecording: () => Promise<{ success: boolean; error?: string }>; - openSourceSelector: () => Promise; + openSourceSelector: () => Promise<{ + opened: boolean; + reason?: string; + access?: { + success: boolean; + granted: boolean; + status: string; + error?: string; + }; + }>; selectSource: (source: ProcessedDesktopSource) => Promise; getSelectedSource: () => Promise; requestCameraAccess: () => Promise<{ @@ -40,6 +49,18 @@ interface Window { status: string; error?: string; }>; + requestScreenAccess: () => Promise<{ + success: boolean; + granted: boolean; + status: string; + error?: string; + }>; + requestNativeMacCursorAccess: () => Promise<{ + success: boolean; + granted: boolean; + status: string; + error?: string; + }>; assetBaseUrl: string; storeRecordedVideo: ( videoData: ArrayBuffer, diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 1dd3071..c653661 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -37,6 +37,7 @@ import type { import { mainT } from "../i18n"; import { RECORDINGS_DIR } from "../main"; import { createCursorRecordingSession } from "../native-bridge/cursor/recording/factory"; +import { requestMacCursorAccessibilityAccess } from "../native-bridge/cursor/recording/macNativeCursorRecordingSession"; import type { CursorRecordingSession } from "../native-bridge/cursor/recording/session"; import { registerNativeBridgeHandlers } from "./nativeBridge"; @@ -1044,6 +1045,43 @@ export function registerIpcHandlers( onRecordingStateChange?: (recording: boolean, sourceName: string) => void, _switchToHud?: () => void, ) { + async function requestScreenAccess() { + if (process.platform !== "darwin") { + return { success: true, granted: true, status: "granted" }; + } + + try { + const status = systemPreferences.getMediaAccessStatus("screen"); + if (status === "granted") { + return { success: true, granted: true, status }; + } + + // Screen recording has no askForMediaAccess equivalent. Trigger the + // TCC prompt without opening OpenScreen's source selector above it. + if (status === "not-determined") { + const mainWin = getMainWindow(); + if (mainWin && !mainWin.isDestroyed()) { + if (!mainWin.isVisible()) { + mainWin.show(); + } + mainWin.focus(); + } + app.focus({ steal: true }); + desktopCapturer + .getSources({ types: ["screen"], thumbnailSize: { width: 1, height: 1 } }) + .catch(() => { + // Permission probing failure is reported by the explicit status check below. + }); + return { success: true, granted: false, status: "not-determined" }; + } + + return { success: true, granted: false, status }; + } catch (error) { + console.error("Failed to request screen access:", error); + return { success: false, granted: false, status: "unknown", error: String(error) }; + } + } + ipcMain.handle("get-sources", async (_, opts) => { const sources = await desktopCapturer.getSources(opts); lastEnumeratedSources = new Map(sources.map((source) => [source.id, source])); @@ -1120,40 +1158,51 @@ export function registerIpcHandlers( }); ipcMain.handle("request-screen-access", async () => { - if (process.platform !== "darwin") { - return { success: true, granted: true, status: "granted" }; - } - - try { - const status = systemPreferences.getMediaAccessStatus("screen"); - if (status === "granted") { - return { success: true, granted: true, status }; - } - - // Screen recording has no askForMediaAccess equivalent — the TCC prompt - // is triggered by desktopCapturer.getSources(). Fire it and return so - // the renderer can re-check status after the user responds. - if (status === "not-determined") { - desktopCapturer.getSources({ types: ["screen"] }).catch(() => { - // Permission probing failure is reported by the explicit status check below. - }); - return { success: true, granted: false, status: "not-determined" }; - } - - return { success: true, granted: false, status }; - } catch (error) { - console.error("Failed to request screen access:", error); - return { success: false, granted: false, status: "unknown", error: String(error) }; - } + return requestScreenAccess(); }); - ipcMain.handle("open-source-selector", () => { + ipcMain.handle("request-native-mac-cursor-access", async () => { + return requestMacCursorAccessibilityAccess(); + }); + + ipcMain.handle("open-source-selector", async () => { + const access = await requestScreenAccess(); + if (!access.granted) { + if (process.platform === "darwin" && access.status !== "not-determined") { + const mainWin = getMainWindow(); + const messageOptions = { + type: "warning", + buttons: ["Open System Settings", "Cancel"], + defaultId: 0, + cancelId: 1, + message: "Screen Recording permission is required", + detail: + "Allow OpenScreen in macOS System Settings, then come back and choose a screen or window.", + } satisfies Electron.MessageBoxOptions; + const result = + mainWin && !mainWin.isDestroyed() + ? await dialog.showMessageBox(mainWin, messageOptions) + : await dialog.showMessageBox(messageOptions); + if (result.response === 0) { + await shell.openExternal( + "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture", + ); + } + } + return { + opened: false, + reason: "screen-access-required", + access, + }; + } + const sourceSelectorWin = getSourceSelectorWindow(); if (sourceSelectorWin) { sourceSelectorWin.focus(); - return; + return { opened: true }; } createSourceSelectorWindow(); + return { opened: true }; }); ipcMain.handle("switch-to-editor", () => { diff --git a/electron/main.ts b/electron/main.ts index 716d03b..3e2258f 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -4,7 +4,6 @@ import { fileURLToPath } from "node:url"; import { app, BrowserWindow, - desktopCapturer, ipcMain, Menu, nativeImage, @@ -493,23 +492,14 @@ app.whenReady().then(async () => { { useSystemPicker: false }, ); - // Request microphone and screen recording permissions from macOS + // Request microphone permission from macOS. Screen Recording is requested + // lazily from the source-picker action so the system prompt is not hidden + // behind OpenScreen's source selector window. if (process.platform === "darwin") { const micStatus = systemPreferences.getMediaAccessStatus("microphone"); if (micStatus !== "granted") { await systemPreferences.askForMediaAccess("microphone"); } - - // Screen recording has no askForMediaAccess equivalent — the TCC prompt is - // triggered by the first desktopCapturer.getSources() call. Firing it here - // at startup settles the permission state early and prevents repeated prompts - // driven by later getSources() calls (fixes repeated permission dialog). - const screenStatus = systemPreferences.getMediaAccessStatus("screen"); - if (screenStatus === "not-determined") { - desktopCapturer.getSources({ types: ["screen"] }).catch(() => { - // This only triggers the system prompt; permission state is read separately. - }); - } } // Listen for HUD overlay quit event (macOS only) diff --git a/electron/native-bridge/cursor/recording/factory.ts b/electron/native-bridge/cursor/recording/factory.ts index e072b75..0ba3077 100644 --- a/electron/native-bridge/cursor/recording/factory.ts +++ b/electron/native-bridge/cursor/recording/factory.ts @@ -1,4 +1,5 @@ import type { Rectangle } from "electron"; +import { MacNativeCursorRecordingSession } from "./macNativeCursorRecordingSession"; import type { CursorRecordingSession } from "./session"; import { TelemetryRecordingSession } from "./telemetryRecordingSession"; import { WindowsNativeRecordingSession } from "./windowsNativeRecordingSession"; @@ -25,9 +26,17 @@ export function createCursorRecordingSession( }); } - // macOS / Linux: capture cursor positions via Electron's `screen` API on an - // interval. No cursor sprites/assets and no clicks — just position telemetry, - // which is what auto-zoom and other features consume. + if (options.platform === "darwin") { + return new MacNativeCursorRecordingSession({ + getDisplayBounds: options.getDisplayBounds, + maxSamples: options.maxSamples, + sampleIntervalMs: options.sampleIntervalMs, + startTimeMs: options.startTimeMs, + }); + } + + // Linux: capture cursor positions via Electron's `screen` API on an interval. + // No cursor sprites/assets and no clicks — just position telemetry. return new TelemetryRecordingSession({ getDisplayBounds: options.getDisplayBounds, maxSamples: options.maxSamples, diff --git a/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts b/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts new file mode 100644 index 0000000..4164cb0 --- /dev/null +++ b/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts @@ -0,0 +1,393 @@ +import { type ChildProcessByStdio, spawn } from "node:child_process"; +import { accessSync, constants as fsConstants } from "node:fs"; +import path from "node:path"; +import type { Readable } from "node:stream"; +import { type Rectangle, screen, systemPreferences } from "electron"; +import type { + CursorRecordingData, + CursorRecordingSample, + NativeCursorType, +} from "../../../../src/native/contracts"; +import type { CursorRecordingSession } from "./session"; + +interface MacNativeCursorRecordingSessionOptions { + getDisplayBounds: () => Rectangle | null; + maxSamples: number; + sampleIntervalMs: number; + startTimeMs?: number; +} + +type MacCursorEvent = + | { + type: "ready"; + timestampMs: number; + accessibilityTrusted?: boolean; + mouseTapReady?: boolean; + } + | { + type: "sample"; + timestampMs: number; + cursorType?: NativeCursorType | null; + leftButtonDown?: boolean; + leftButtonPressed?: boolean; + leftButtonReleased?: boolean; + }; + +const HELPER_NAME = "openscreen-macos-cursor-helper"; +const READY_TIMEOUT_MS = 5_000; + +function helperCandidates() { + const envPath = process.env.OPENSCREEN_MAC_CURSOR_HELPER_EXE?.trim(); + const appRoot = process.env.APP_ROOT ? path.resolve(process.env.APP_ROOT) : process.cwd(); + const archTag = process.arch === "arm64" ? "darwin-arm64" : "darwin-x64"; + const resourceRoot = + typeof process.resourcesPath === "string" + ? process.resourcesPath + : path.join(appRoot, "resources"); + + return [ + envPath, + path.join(appRoot, "electron", "native", "screencapturekit", "build", HELPER_NAME), + path.join(appRoot, "electron", "native", "bin", archTag, HELPER_NAME), + path.join(resourceRoot, "electron", "native", "bin", archTag, HELPER_NAME), + ].filter((candidate): candidate is string => Boolean(candidate)); +} + +export function findMacCursorHelperPath() { + for (const candidate of helperCandidates()) { + try { + accessSync(candidate, fsConstants.X_OK); + return candidate; + } catch { + // Try the next helper location. + } + } + + return null; +} + +export async function requestMacCursorAccessibilityAccess() { + if (process.platform !== "darwin") { + return { success: true, granted: true, status: "granted" }; + } + + try { + systemPreferences.isTrustedAccessibilityClient(true); + } catch { + // Continue with helper probing; it can trigger the same macOS prompt. + } + + const helperPath = findMacCursorHelperPath(); + if (!helperPath) { + return { success: true, granted: false, status: "missing-helper" }; + } + + return new Promise<{ success: boolean; granted: boolean; status: string; error?: string }>( + (resolve) => { + const child = spawn(helperPath, [JSON.stringify({ sampleIntervalMs: 250 })], { + stdio: ["ignore", "pipe", "pipe"], + }); + let settled = false; + let lineBuffer = ""; + const finish = (result: { + success: boolean; + granted: boolean; + status: string; + error?: string; + }) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + if (!child.killed) { + child.kill("SIGTERM"); + } + resolve(result); + }; + const timer = setTimeout(() => { + finish({ + success: false, + granted: false, + status: "timeout", + error: "Timed out waiting for macOS cursor helper", + }); + }, READY_TIMEOUT_MS); + + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + lineBuffer += chunk; + const lines = lineBuffer.split(/\r?\n/); + lineBuffer = lines.pop() ?? ""; + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed) { + continue; + } + try { + const event = JSON.parse(trimmed) as MacCursorEvent; + if (event.type === "ready") { + finish({ + success: true, + granted: event.accessibilityTrusted === true, + status: event.accessibilityTrusted === true ? "granted" : "not-determined", + }); + return; + } + } catch { + // Ignore non-JSON helper output. + } + } + }); + + child.once("error", (error) => { + finish({ + success: false, + granted: false, + status: "error", + error: error.message, + }); + }); + child.once("exit", (code, signal) => { + finish({ + success: false, + granted: false, + status: "exited", + error: `macOS cursor helper exited before ready (code=${code}, signal=${signal})`, + }); + }); + }, + ); +} + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)); +} + +function normalizeCursorType(value: unknown): NativeCursorType | null { + return value === "arrow" || value === "pointer" || value === "text" ? value : null; +} + +export class MacNativeCursorRecordingSession implements CursorRecordingSession { + private samples: CursorRecordingSample[] = []; + private process: ChildProcessByStdio | null = null; + private lineBuffer = ""; + private startTimeMs = 0; + private fallbackInterval: NodeJS.Timeout | null = null; + private readyResolve: (() => void) | null = null; + private readyReject: ((error: Error) => void) | null = null; + private readyTimer: NodeJS.Timeout | null = null; + private previousLeftButtonDown = false; + + constructor(private readonly options: MacNativeCursorRecordingSessionOptions) {} + + async start(): Promise { + this.samples = []; + this.lineBuffer = ""; + this.startTimeMs = this.options.startTimeMs ?? Date.now(); + this.previousLeftButtonDown = false; + + try { + systemPreferences.isTrustedAccessibilityClient(true); + } catch { + // Link cursor detection degrades to arrow when Accessibility is unavailable. + } + + const helperPath = findMacCursorHelperPath(); + if (!helperPath) { + this.startPositionOnlyFallback(); + return; + } + + const child = spawn( + helperPath, + [ + JSON.stringify({ + sampleIntervalMs: this.options.sampleIntervalMs, + }), + ], + { + stdio: ["ignore", "pipe", "pipe"], + }, + ); + this.process = child; + + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => this.handleStdoutChunk(chunk)); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + const message = chunk.trim(); + if (message) { + console.error("[cursor-macos]", message); + } + }); + child.once("exit", (code, signal) => { + this.rejectReady( + new Error(`macOS cursor helper exited before ready (code=${code}, signal=${signal})`), + ); + this.process = null; + }); + child.once("error", (error) => { + this.rejectReady(error); + this.process = null; + }); + + try { + await this.waitUntilReady(); + } catch (error) { + this.killHelperProcess(child); + this.process = null; + console.warn("[cursor-macos] falling back to position-only cursor telemetry:", error); + this.startPositionOnlyFallback(); + } + } + + async stop(): Promise { + const child = this.process; + this.process = null; + this.clearReadyState(); + + if (this.fallbackInterval) { + clearInterval(this.fallbackInterval); + this.fallbackInterval = null; + } + + if (child) { + this.killHelperProcess(child); + } + + return { + version: 2, + provider: "none", + samples: this.samples, + assets: [], + }; + } + + private startPositionOnlyFallback() { + this.captureSample(Date.now(), null, false, false, false); + this.fallbackInterval = setInterval(() => { + this.captureSample(Date.now(), null, false, false, false); + }, this.options.sampleIntervalMs); + } + + private handleStdoutChunk(chunk: string) { + this.lineBuffer += chunk; + const lines = this.lineBuffer.split(/\r?\n/); + this.lineBuffer = lines.pop() ?? ""; + + for (const line of lines) { + const trimmedLine = line.trim(); + if (!trimmedLine) { + continue; + } + + try { + this.handleEvent(JSON.parse(trimmedLine) as MacCursorEvent); + } catch (error) { + console.error("Failed to parse macOS cursor helper output:", error, trimmedLine); + } + } + } + + private handleEvent(payload: MacCursorEvent) { + if (payload.type === "ready") { + if (payload.accessibilityTrusted === false) { + console.warn( + "[cursor-macos] Accessibility is not trusted; cursor shape detection will be arrow-only.", + ); + } + this.resolveReady(); + return; + } + + if (payload.type === "sample") { + this.captureSample( + payload.timestampMs, + normalizeCursorType(payload.cursorType), + payload.leftButtonDown === true, + payload.leftButtonPressed === true, + payload.leftButtonReleased === true, + ); + } + } + + private captureSample( + timestampMs: number, + cursorType: NativeCursorType | null, + leftButtonDown: boolean, + leftButtonPressed: boolean, + leftButtonReleased: boolean, + ) { + const cursor = screen.getCursorScreenPoint(); + const bounds = this.options.getDisplayBounds() ?? screen.getDisplayNearestPoint(cursor).bounds; + const width = Math.max(1, bounds.width); + const height = Math.max(1, bounds.height); + const normalizedX = (cursor.x - bounds.x) / width; + const normalizedY = (cursor.y - bounds.y) / height; + const interactionType = + leftButtonPressed || (leftButtonDown && !this.previousLeftButtonDown) + ? "click" + : leftButtonReleased || (!leftButtonDown && this.previousLeftButtonDown) + ? "mouseup" + : "move"; + this.previousLeftButtonDown = leftButtonDown; + + this.samples.push({ + timeMs: Math.max(0, timestampMs - this.startTimeMs), + cx: clamp(normalizedX, 0, 1), + cy: clamp(normalizedY, 0, 1), + visible: true, + interactionType, + ...(cursorType ? { cursorType } : {}), + }); + + if (this.samples.length > this.options.maxSamples) { + this.samples.shift(); + } + } + + private waitUntilReady() { + return new Promise((resolve, reject) => { + this.readyResolve = resolve; + this.readyReject = reject; + this.readyTimer = setTimeout(() => { + this.rejectReady(new Error("Timed out waiting for macOS cursor helper")); + }, READY_TIMEOUT_MS); + }); + } + + private resolveReady() { + const resolve = this.readyResolve; + this.clearReadyState(); + resolve?.(); + } + + private rejectReady(error: Error) { + const reject = this.readyReject; + this.clearReadyState(); + reject?.(error); + } + + private clearReadyState() { + if (this.readyTimer) { + clearTimeout(this.readyTimer); + this.readyTimer = null; + } + this.readyResolve = null; + this.readyReject = null; + } + + private killHelperProcess(child: ChildProcessByStdio) { + if (child.killed) { + return; + } + + child.kill("SIGTERM"); + setTimeout(() => { + if (!child.killed) { + child.kill("SIGKILL"); + } + }, 500).unref(); + } +} diff --git a/electron/native/README.md b/electron/native/README.md index bf8e6bc..59930ba 100644 --- a/electron/native/README.md +++ b/electron/native/README.md @@ -15,17 +15,19 @@ Helper locations: 2. `electron/native/screencapturekit/build/openscreen-screencapturekit-helper`, for locally built Swift output. 3. `electron/native/bin/darwin-arm64/openscreen-screencapturekit-helper` or `electron/native/bin/darwin-x64/openscreen-screencapturekit-helper`, for packaged prebuilt helpers. +The macOS cursor-shape helper is resolved from `OPENSCREEN_MAC_CURSOR_HELPER_EXE` first, then the matching `openscreen-macos-cursor-helper` binary in the same local build and packaged `electron/native/bin/darwin-${arch}` directories. + Build the macOS helper with: ```bash npm run build:native:mac ``` -On non-macOS hosts this command exits successfully and does not affect Windows/Linux development. On macOS it builds the Swift package at `electron/native/screencapturekit`, writes the development binary to `electron/native/screencapturekit/build/openscreen-screencapturekit-helper`, and copies the redistributable binary to `electron/native/bin/darwin-${arch}/openscreen-screencapturekit-helper`. +On non-macOS hosts this command exits successfully and does not affect Windows/Linux development. On macOS it builds the Swift package at `electron/native/screencapturekit`, writes the development binaries to `electron/native/screencapturekit/build`, and copies redistributable binaries to `electron/native/bin/darwin-${arch}`. The current helper implementation supports display/window ScreenCaptureKit video capture, cursor exclusion through `SCStreamConfiguration.showsCursor`, H.264 encoding, MP4 muxing, and ScreenCaptureKit system audio. It also attempts native ScreenCaptureKit microphone capture when the running macOS version exposes that capability. Webcam recording currently stays as an Electron sidecar and is attached to the same recording session after the native screen capture stops. -Electron exposes `is-native-mac-capture-available` for capability probing. It resolves the same helper locations listed above and reports `missing-helper` until a Swift helper binary is present. When available, macOS recording routes screen/window capture through the native helper so editable cursor recordings do not bake the system cursor into the video. +Electron exposes `is-native-mac-capture-available` for capability probing. It resolves the same helper locations listed above and reports `missing-helper` until a Swift helper binary is present. When available, macOS recording routes screen/window capture through the native helper so editable cursor recordings do not bake the system cursor into the video. Cursor positions are sampled in Electron; when the cursor helper is available and Accessibility is granted, samples are also tagged with link/text cursor hints such as `pointer`. See `docs/engineering/macos-native-recorder-roadmap.md` for the contract, rollout phases, and SSOT rules. diff --git a/electron/native/screencapturekit/Package.swift b/electron/native/screencapturekit/Package.swift index f040dd2..ec3b1d9 100644 --- a/electron/native/screencapturekit/Package.swift +++ b/electron/native/screencapturekit/Package.swift @@ -11,12 +11,20 @@ let package = Package( .executable( name: "openscreen-screencapturekit-helper", targets: ["OpenScreenScreenCaptureKitHelper"] + ), + .executable( + name: "openscreen-macos-cursor-helper", + targets: ["OpenScreenMacOSCursorHelper"] ) ], targets: [ .executableTarget( name: "OpenScreenScreenCaptureKitHelper", path: "Sources/OpenScreenScreenCaptureKitHelper" + ), + .executableTarget( + name: "OpenScreenMacOSCursorHelper", + path: "Sources/OpenScreenMacOSCursorHelper" ) ] ) diff --git a/electron/native/screencapturekit/Sources/OpenScreenMacOSCursorHelper/main.swift b/electron/native/screencapturekit/Sources/OpenScreenMacOSCursorHelper/main.swift new file mode 100644 index 0000000..7afebc8 --- /dev/null +++ b/electron/native/screencapturekit/Sources/OpenScreenMacOSCursorHelper/main.swift @@ -0,0 +1,256 @@ +import AppKit +import ApplicationServices +import Foundation + +struct CursorHelperRequest: Decodable { + let sampleIntervalMs: Int? +} + +final class MouseButtonTracker { + private let lock = NSLock() + private var leftDownCount = 0 + private var leftUpCount = 0 + private var eventTap: CFMachPort? + private var runLoopSource: CFRunLoopSource? + + struct Events { + let leftDownCount: Int + let leftUpCount: Int + } + + func start() -> Bool { + let mask = + (1 << CGEventType.leftMouseDown.rawValue) | + (1 << CGEventType.leftMouseUp.rawValue) + guard let tap = CGEvent.tapCreate( + tap: .cgSessionEventTap, + place: .headInsertEventTap, + options: .listenOnly, + eventsOfInterest: CGEventMask(mask), + callback: { _, type, event, userInfo in + if let userInfo { + let tracker = Unmanaged.fromOpaque(userInfo).takeUnretainedValue() + tracker.record(type) + } + return Unmanaged.passUnretained(event) + }, + userInfo: UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque()) + ) else { + return false + } + + guard let source = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, tap, 0) else { + return false + } + + eventTap = tap + runLoopSource = source + CFRunLoopAddSource(CFRunLoopGetCurrent(), source, .commonModes) + CGEvent.tapEnable(tap: tap, enable: true) + return true + } + + func pump() { + CFRunLoopRunInMode(.defaultMode, 0.001, false) + } + + func consume() -> Events { + lock.lock() + defer { lock.unlock() } + let events = Events(leftDownCount: leftDownCount, leftUpCount: leftUpCount) + leftDownCount = 0 + leftUpCount = 0 + return events + } + + private func record(_ type: CGEventType) { + lock.lock() + defer { lock.unlock() } + if type == .leftMouseDown { + leftDownCount += 1 + } else if type == .leftMouseUp { + leftUpCount += 1 + } + } +} + +func emit(_ fields: [String: Any?]) { + let compacted = fields.compactMapValues { $0 } + if let data = try? JSONSerialization.data(withJSONObject: compacted, options: []), + let line = String(data: data, encoding: .utf8) + { + print(line) + fflush(stdout) + } +} + +func stringAttribute(_ element: AXUIElement, _ attribute: String) -> String? { + var value: CFTypeRef? + let result = AXUIElementCopyAttributeValue(element, attribute as CFString, &value) + guard result == .success else { + return nil + } + + return value as? String +} + +func parentElement(_ element: AXUIElement) -> AXUIElement? { + var value: CFTypeRef? + let result = AXUIElementCopyAttributeValue(element, kAXParentAttribute as CFString, &value) + guard result == .success else { + return nil + } + + return (value as! AXUIElement) +} + +func roleDescription(_ element: AXUIElement) -> String? { + var value: CFTypeRef? + let result = AXUIElementCopyAttributeValue(element, kAXRoleDescriptionAttribute as CFString, &value) + guard result == .success else { + return nil + } + + return value as? String +} + +func actionNames(_ element: AXUIElement) -> [String] { + var value: CFArray? + let result = AXUIElementCopyActionNames(element, &value) + guard result == .success, let value else { + return [] + } + + return (value as NSArray).compactMap { $0 as? String } +} + +func isTextInputRole(_ role: String?) -> Bool { + role == "AXTextField" || + role == "AXTextArea" || + role == "AXTextView" || + role == "AXComboBox" +} + +func isPointerRole(_ role: String?, _ subrole: String?, _ description: String?) -> Bool { + if role == "AXLink" || + subrole?.localizedCaseInsensitiveContains("link") == true || + description?.contains("link") == true + { + return true + } + + return role == "AXButton" || + role == "AXMenuButton" || + role == "AXPopUpButton" || + role == "AXCheckBox" || + role == "AXRadioButton" || + role == "AXSwitch" || + role == "AXDisclosureTriangle" || + role == "AXTab" || + role == "AXMenuItem" || + role == "AXCell" +} + +func cursorTypeForElement(_ element: AXUIElement) -> String? { + var current: AXUIElement? = element + + for _ in 0..<5 { + guard let element = current else { + break + } + + let role = stringAttribute(element, kAXRoleAttribute) + let subrole = stringAttribute(element, kAXSubroleAttribute) + let description = roleDescription(element)?.lowercased() + + if isTextInputRole(role) { + return "text" + } + + if isPointerRole(role, subrole, description) || actionNames(element).contains(kAXPressAction) { + return "pointer" + } + + current = parentElement(element) + } + + return nil +} + +func accessibilityPointForMouse() -> CGPoint { + let mouse = NSEvent.mouseLocation + let maxY = NSScreen.screens.map { $0.frame.maxY }.max() ?? NSScreen.main?.frame.height ?? 0 + return CGPoint(x: mouse.x, y: maxY - mouse.y) +} + +func currentCursorType() -> String? { + guard AXIsProcessTrusted() else { + return nil + } + + let point = accessibilityPointForMouse() + let systemWide = AXUIElementCreateSystemWide() + var element: AXUIElement? + let result = AXUIElementCopyElementAtPosition( + systemWide, + Float(point.x), + Float(point.y), + &element + ) + + guard result == .success, let element else { + return "arrow" + } + + return cursorTypeForElement(element) ?? "arrow" +} + +func timestampMs() -> Int { + Int(Date().timeIntervalSince1970 * 1000) +} + +func leftButtonDown() -> Bool { + CGEventSource.buttonState(.hidSystemState, button: .left) +} + +func requestAccessibilityTrust() -> Bool { + let options = [ + kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true + ] as CFDictionary + return AXIsProcessTrustedWithOptions(options) +} + +let request: CursorHelperRequest +if CommandLine.arguments.count >= 2, + let data = CommandLine.arguments[1].data(using: .utf8), + let decoded = try? JSONDecoder().decode(CursorHelperRequest.self, from: data) +{ + request = decoded +} else { + request = CursorHelperRequest(sampleIntervalMs: nil) +} + +let intervalMs = max(8, request.sampleIntervalMs ?? 33) +let accessibilityTrusted = requestAccessibilityTrust() +let mouseTracker = MouseButtonTracker() +let mouseTapReady = mouseTracker.start() +emit([ + "type": "ready", + "timestampMs": timestampMs(), + "accessibilityTrusted": accessibilityTrusted, + "mouseTapReady": mouseTapReady, +]) + +while true { + mouseTracker.pump() + let mouseEvents = mouseTracker.consume() + emit([ + "type": "sample", + "timestampMs": timestampMs(), + "cursorType": currentCursorType(), + "leftButtonDown": leftButtonDown(), + "leftButtonPressed": mouseEvents.leftDownCount > 0, + "leftButtonReleased": mouseEvents.leftUpCount > 0, + ]) + Thread.sleep(forTimeInterval: Double(intervalMs) / 1000.0) +} diff --git a/electron/preload.ts b/electron/preload.ts index 861f3a4..d6af7b0 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -49,6 +49,12 @@ contextBridge.exposeInMainWorld("electronAPI", { requestCameraAccess: () => { return ipcRenderer.invoke("request-camera-access"); }, + requestScreenAccess: () => { + return ipcRenderer.invoke("request-screen-access"); + }, + requestNativeMacCursorAccess: () => { + return ipcRenderer.invoke("request-native-mac-cursor-access"); + }, storeRecordedVideo: (videoData: ArrayBuffer, fileName: string) => { return ipcRenderer.invoke("store-recorded-video", videoData, fileName); }, diff --git a/scripts/build-macos-screencapturekit-helper.mjs b/scripts/build-macos-screencapturekit-helper.mjs index f5f0c82..6a575d8 100644 --- a/scripts/build-macos-screencapturekit-helper.mjs +++ b/scripts/build-macos-screencapturekit-helper.mjs @@ -14,14 +14,18 @@ if (process.platform !== "darwin") { const __dirname = path.dirname(fileURLToPath(import.meta.url)); const root = path.resolve(__dirname, ".."); const helperName = "openscreen-screencapturekit-helper"; +const cursorHelperName = "openscreen-macos-cursor-helper"; const packageDir = path.join(root, "electron", "native", "screencapturekit"); const buildDir = path.join(packageDir, "build"); const swiftBuildDir = path.join(buildDir, "swiftpm"); const builtHelperPath = path.join(swiftBuildDir, "release", helperName); const localHelperPath = path.join(buildDir, helperName); +const builtCursorHelperPath = path.join(swiftBuildDir, "release", cursorHelperName); +const localCursorHelperPath = path.join(buildDir, cursorHelperName); const archTag = process.arch === "arm64" ? "darwin-arm64" : "darwin-x64"; const distributableDir = path.join(root, "electron", "native", "bin", archTag); const distributablePath = path.join(distributableDir, helperName); +const distributableCursorHelperPath = path.join(distributableDir, cursorHelperName); const xcodebuildVersion = spawnSync("xcodebuild", ["-version"], { cwd: root, @@ -68,8 +72,14 @@ fs.mkdirSync(buildDir, { recursive: true }); fs.mkdirSync(distributableDir, { recursive: true }); fs.copyFileSync(builtHelperPath, localHelperPath); fs.copyFileSync(builtHelperPath, distributablePath); +fs.copyFileSync(builtCursorHelperPath, localCursorHelperPath); +fs.copyFileSync(builtCursorHelperPath, distributableCursorHelperPath); fs.chmodSync(localHelperPath, 0o755); fs.chmodSync(distributablePath, 0o755); +fs.chmodSync(localCursorHelperPath, 0o755); +fs.chmodSync(distributableCursorHelperPath, 0o755); console.log(`Built macOS ScreenCaptureKit helper: ${localHelperPath}`); console.log(`Copied redistributable helper: ${distributablePath}`); +console.log(`Built macOS cursor helper: ${localCursorHelperPath}`); +console.log(`Copied redistributable cursor helper: ${distributableCursorHelperPath}`); diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index d4116f9..17b340d 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -253,13 +253,13 @@ export default function VideoEditor() { const nextSpeedIdRef = useRef(1); const { shortcuts, isMac } = useShortcuts(); - // Windows-only: the synthetic cursor overlay + cursor customization settings - // only apply when there's an actual native cursor recording (cursor frames + - // position samples produced by WindowsNativeRecordingSession). Mac and Linux - // keep their telemetry positions for auto-zoom but never render a synthetic - // cursor or expose cursor customization settings. + // Native Windows recordings include captured cursor assets. Native macOS + // recordings hide the system cursor in ScreenCaptureKit and use telemetry + // samples with OpenScreen's default arrow asset for the editable overlay. const hasEditableCursorRecording = - nativePlatform === "win32" && hasNativeCursorRecordingData(cursorRecordingData); + recordingCursorCaptureMode === "editable-overlay" && + (nativePlatform === "win32" || nativePlatform === "darwin") && + hasNativeCursorRecordingData(cursorRecordingData); const effectiveShowCursor = showCursor && hasEditableCursorRecording; const showCursorSettings = hasEditableCursorRecording; const { locale, setLocale, t: rawT } = useI18n(); diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 64dd1ad..34f3b9a 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -934,7 +934,6 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const runId = countdownRunId.current + 1; countdownRunId.current = runId; - setCountdownActive(true); let selectedSource: ProcessedDesktopSource | null = null; try { @@ -955,6 +954,27 @@ export function useScreenRecorder(): UseScreenRecorderReturn { return; } + try { + const platform = await window.electronAPI.getPlatform(); + if (platform === "darwin" && cursorCaptureMode === "editable-overlay") { + const access = await window.electronAPI.requestNativeMacCursorAccess(); + if (!access.granted) { + toast.info( + "Allow Accessibility access for OpenScreen, then press record again to start the countdown.", + ); + return; + } + } + } catch (error) { + console.warn("Failed to preflight macOS cursor accessibility before countdown:", error); + } + + if (!isCountdownRunActive(runId)) { + return; + } + + setCountdownActive(true); + let overlayHiddenBeforeStart = false; try { const values = [3, 2, 1]; diff --git a/src/lib/cursor/nativeCursor.test.ts b/src/lib/cursor/nativeCursor.test.ts index 1b919d6..75cdc77 100644 --- a/src/lib/cursor/nativeCursor.test.ts +++ b/src/lib/cursor/nativeCursor.test.ts @@ -2,6 +2,8 @@ import { describe, expect, it } from "vitest"; import { getNativeCursorClickBounceProgress, getNativeCursorClickBounceScale, + hasNativeCursorRecordingData, + resolveInterpolatedNativeCursorFrame, } from "./nativeCursor"; describe("native cursor click bounce", () => { @@ -31,4 +33,37 @@ describe("native cursor click bounce", () => { expect(getNativeCursorClickBounceScale(5, 0.28)).toBeGreaterThan(1.05); expect(getNativeCursorClickBounceScale(5, 0)).toBe(1); }); + + it("uses the default cursor asset for telemetry-only macOS recordings", () => { + const recordingData = { + version: 2, + provider: "none" as const, + assets: [], + samples: [ + { timeMs: 0, cx: 0.25, cy: 0.4, visible: true }, + { timeMs: 100, cx: 0.75, cy: 0.6, visible: true }, + ], + }; + + expect(hasNativeCursorRecordingData(recordingData)).toBe(true); + const frame = resolveInterpolatedNativeCursorFrame(recordingData, 50); + expect(frame?.asset.cursorType).toBe("arrow"); + expect(frame?.sample.cx).toBeCloseTo(0.5); + expect(frame?.sample.cy).toBeCloseTo(0.5); + }); + + it("applies click bounce to telemetry-only macOS recordings", () => { + const recordingData = { + version: 2, + provider: "none" as const, + assets: [], + samples: [ + { timeMs: 0, cx: 0.5, cy: 0.5, visible: true, interactionType: "move" as const }, + { timeMs: 100, cx: 0.5, cy: 0.5, visible: true, interactionType: "click" as const }, + { timeMs: 133, cx: 0.5, cy: 0.5, visible: true, interactionType: "move" as const }, + ], + }; + + expect(getNativeCursorClickBounceProgress(recordingData, 133)).toBeGreaterThan(0); + }); }); diff --git a/src/lib/cursor/nativeCursor.ts b/src/lib/cursor/nativeCursor.ts index 9ce308a..f20fd42 100644 --- a/src/lib/cursor/nativeCursor.ts +++ b/src/lib/cursor/nativeCursor.ts @@ -220,6 +220,33 @@ const PRETTY_NATIVE_CURSOR_ASSETS: Partial 0 && - recordingData.assets.length > 0, + (recordingData.assets.length > 0 || recordingData.provider === "none"), ); } @@ -329,7 +355,7 @@ export function getNativeCursorClickBounceProgress( recordingData: CursorRecordingData | null | undefined, timeMs: number, ) { - if (!recordingData || recordingData.provider !== "native" || recordingData.samples.length === 0) { + if (!recordingData || recordingData.samples.length === 0) { return 0; } @@ -444,11 +470,13 @@ export function resolveActiveNativeCursorFrame( if (index >= 0) { const sample = recordingData.samples[index]; - if (sample.visible === false || !sample.assetId) { + if (sample.visible === false) { return null; } - const asset = getNativeCursorAsset(recordingData, sample.assetId); + const asset = sample.assetId + ? getNativeCursorAsset(recordingData, sample.assetId) + : getTelemetryCursorAsset(sample); if (!asset) { return null; } @@ -475,11 +503,13 @@ export function resolveInterpolatedNativeCursorFrame( } const activeSample = samples[activeIndex]; - if (activeSample.visible === false || !activeSample.assetId) { + if (activeSample.visible === false) { return null; } - const asset = getNativeCursorAsset(recordingData, activeSample.assetId); + const asset = activeSample.assetId + ? getNativeCursorAsset(recordingData, activeSample.assetId) + : getTelemetryCursorAsset(activeSample); if (!asset) { return null; } @@ -489,7 +519,7 @@ export function resolveInterpolatedNativeCursorFrame( !nextSample || nextSample.timeMs <= activeSample.timeMs || nextSample.visible === false || - nextSample.assetId !== activeSample.assetId || + (nextSample.assetId ?? null) !== (activeSample.assetId ?? null) || timeMs <= activeSample.timeMs ) { return { asset, sample: activeSample };