feat: compose mac native capture with media

2026-05-12 09:32:14 +02:00
parent b9e2134749
commit 6a4ddc5dad
7 changed files with 453 additions and 81 deletions
@@ -1,4 +1,5 @@
 import AVFoundation
+import CoreGraphics
 import CoreMedia
 import Foundation
 import ScreenCaptureKit
@@ -77,6 +78,7 @@ enum HelperError: Error, CustomStringConvertible {
 	case unsupportedFeature(String)
 	case sourceNotFound(String)
 	case invalidSourceType(String)
+	case permissionDenied(String)
 	case writerSetupFailed(String)

 	var description: String {
@@ -91,6 +93,8 @@ enum HelperError: Error, CustomStringConvertible {
 			return message
 		case .invalidSourceType(let sourceType):
 			return "Unsupported source type: \(sourceType)."
+		case .permissionDenied(let message):
+			return message
 		case .writerSetupFailed(let message):
 			return message
 		}
@@ -116,40 +120,62 @@ func emitError(code: String, message: String) {

@available(macOS 13.0, *)
 final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
+	private struct CaptureTarget {
+		let filter: SCContentFilter
+		let width: Int
+		let height: Int
+	}
+
 	private let request: RecordingRequest
 	private let sampleQueue = DispatchQueue(label: "app.openscreen.sck-helper.samples")
 	private let stateQueue = DispatchQueue(label: "app.openscreen.sck-helper.state")
 	private var stream: SCStream?
 	private var writer: AVAssetWriter?
 	private var videoInput: AVAssetWriterInput?
+	private var systemAudioInput: AVAssetWriterInput?
+	private var microphoneAudioInput: AVAssetWriterInput?
 	private var didStartWriting = false
+	private var didEmitRecordingStarted = false
 	private var isStopping = false
+	private var nativeMicrophoneEnabled = false
+	private var outputWidth = 1920
+	private var outputHeight = 1080
+	private let microphoneOutputTypeRawValue = 2

 	init(request: RecordingRequest) {
 		self.request = request
 	}

 	func start() async throws {
-		try rejectUnsupportedPhaseFeatures()
+		try ensureRequestedPermissions()

 		let content = try await SCShareableContent.excludingDesktopWindows(
 			false,
 			onScreenWindowsOnly: true
 		)
-		let filter = try makeContentFilter(from: content)
+		let target = try makeCaptureTarget(from: content)
+		outputWidth = target.width
+		outputHeight = target.height
 		let configuration = makeStreamConfiguration()
-		let stream = SCStream(filter: filter, configuration: configuration, delegate: self)
+		let stream = SCStream(filter: target.filter, configuration: configuration, delegate: self)

 		try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: sampleQueue)
+		if request.audio.system.enabled {
+			try stream.addStreamOutput(self, type: .audio, sampleHandlerQueue: sampleQueue)
+		}
+		if nativeMicrophoneEnabled {
+			guard let microphoneOutputType = SCStreamOutputType(rawValue: microphoneOutputTypeRawValue) else {
+				throw HelperError.unsupportedFeature(
+					"Native microphone capture requires a macOS version with ScreenCaptureKit microphone output."
+				)
+			}
+			try stream.addStreamOutput(self, type: microphoneOutputType, sampleHandlerQueue: sampleQueue)
+		}
 		try setupWriter()

 		self.stream = stream
 		emit(["event": "ready", "schemaVersion": 1])
 		try await stream.startCapture()
-		emit([
-			"event": "recording-started",
-			"timestampMs": Int(Date().timeIntervalSince1970 * 1000),
-		])
 	}

 	func stop() async {
@@ -185,16 +211,29 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 	}

 	func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
+		guard CMSampleBufferDataIsReady(sampleBuffer) else {
+			return
+		}
+
+		if type == .audio {
+			appendAudioSampleBuffer(sampleBuffer, to: systemAudioInput)
+			return
+		}
+
+		if type.rawValue == microphoneOutputTypeRawValue {
+			appendAudioSampleBuffer(sampleBuffer, to: microphoneAudioInput)
+			return
+		}
+
 		guard type == .screen else {
 			return
 		}
-		guard CMSampleBufferDataIsReady(sampleBuffer) else {
+		guard isCompleteFrame(sampleBuffer) else {
 			return
 		}
 		guard let videoInput, let writer else {
 			return
 		}
-
 		let presentationTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
 		if !didStartWriting {
 			writer.startWriting()
@@ -203,29 +242,46 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 		}

 		if videoInput.isReadyForMoreMediaData {
-			videoInput.append(sampleBuffer)
+			if videoInput.append(sampleBuffer), !didEmitRecordingStarted {
+				didEmitRecordingStarted = true
+				emit([
+					"event": "recording-started",
+					"timestampMs": Int(Date().timeIntervalSince1970 * 1000),
+					"width": outputWidth,
+					"height": outputHeight,
+				])
+			}
 		}
 	}

-	private func rejectUnsupportedPhaseFeatures() throws {
-		if request.audio.system.enabled {
-			throw HelperError.unsupportedFeature(
-				"System audio capture is planned for the roadmap system-audio phase."
-			)
+	private func ensureRequestedPermissions() throws {
+		if !CGPreflightScreenCaptureAccess() {
+			let granted = CGRequestScreenCaptureAccess()
+			if !granted {
+				throw HelperError.permissionDenied("Screen recording permission is required for ScreenCaptureKit capture.")
+			}
 		}
+
 		if request.audio.microphone.enabled {
-			throw HelperError.unsupportedFeature(
-				"Microphone capture is planned for the roadmap microphone phase."
-			)
-		}
-		if request.webcam.enabled {
-			throw HelperError.unsupportedFeature(
-				"Webcam composition is planned for the roadmap webcam phase."
-			)
+			switch AVCaptureDevice.authorizationStatus(for: .audio) {
+			case .authorized:
+				break
+			case .notDetermined:
+				let semaphore = DispatchSemaphore(value: 0)
+				AVCaptureDevice.requestAccess(for: .audio) { _ in
+					semaphore.signal()
+				}
+				semaphore.wait()
+				if AVCaptureDevice.authorizationStatus(for: .audio) != .authorized {
+					throw HelperError.permissionDenied("Microphone permission is required for native microphone capture.")
+				}
+			default:
+				throw HelperError.permissionDenied("Microphone permission is required for native microphone capture.")
+			}
 		}
 	}

-	private func makeContentFilter(from content: SCShareableContent) throws -> SCContentFilter {
+	private func makeCaptureTarget(from content: SCShareableContent) throws -> CaptureTarget {
 		switch request.source.type {
 		case "display":
 			guard let displayId = request.source.displayId else {
@@ -234,7 +290,13 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 			guard let display = content.displays.first(where: { $0.displayID == displayId }) else {
 				throw HelperError.sourceNotFound("No ScreenCaptureKit display found for id \(displayId).")
 			}
-			return SCContentFilter(display: display, excludingWindows: [])
+			let width = Int(CGDisplayPixelsWide(display.displayID))
+			let height = Int(CGDisplayPixelsHigh(display.displayID))
+			return CaptureTarget(
+				filter: SCContentFilter(display: display, excludingWindows: []),
+				width: clampCaptureDimension(width, fallback: request.video.width),
+				height: clampCaptureDimension(height, fallback: request.video.height)
+			)
 		case "window":
 			guard let windowId = request.source.windowId else {
 				throw HelperError.sourceNotFound("Window capture requires source.windowId.")
@@ -242,7 +304,17 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 			guard let window = content.windows.first(where: { $0.windowID == windowId }) else {
 				throw HelperError.sourceNotFound("No ScreenCaptureKit window found for id \(windowId).")
 			}
-			return SCContentFilter(desktopIndependentWindow: window)
+			let candidateDisplay = content.displays.first {
+				$0.frame.intersects(window.frame) || $0.frame.contains(CGPoint(x: window.frame.midX, y: window.frame.midY))
+			}
+			let scaleFactor = Self.scaleFactor(for: candidateDisplay?.displayID ?? CGMainDisplayID())
+			let width = Int(window.frame.width) * scaleFactor
+			let height = Int(window.frame.height) * scaleFactor
+			return CaptureTarget(
+				filter: SCContentFilter(desktopIndependentWindow: window),
+				width: clampCaptureDimension(width, fallback: request.video.width),
+				height: clampCaptureDimension(height, fallback: request.video.height)
+			)
 		default:
 			throw HelperError.invalidSourceType(request.source.type)
 		}
@@ -250,12 +322,37 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {

 	private func makeStreamConfiguration() -> SCStreamConfiguration {
 		let configuration = SCStreamConfiguration()
-		configuration.width = request.video.width
-		configuration.height = request.video.height
+		configuration.width = outputWidth
+		configuration.height = outputHeight
 		configuration.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(max(1, request.video.fps)))
 		configuration.queueDepth = 6
 		configuration.showsCursor = !request.video.hideSystemCursor
 		configuration.pixelFormat = kCVPixelFormatType_32BGRA
+		configuration.sampleRate = 48_000
+		configuration.channelCount = 2
+		configuration.excludesCurrentProcessAudio = true
+		configuration.capturesAudio = request.audio.system.enabled
+
+		if request.audio.microphone.enabled {
+			guard supportsNativeMicrophoneCapture(streamConfig: configuration) else {
+				nativeMicrophoneEnabled = false
+				emit([
+					"event": "warning",
+					"code": "microphone-unavailable",
+					"message": "Native microphone capture requires ScreenCaptureKit microphone support on this macOS version.",
+				])
+				return configuration
+			}
+			nativeMicrophoneEnabled = true
+			configuration.capturesAudio = true
+			configuration.setValue(true, forKey: "captureMicrophone")
+			if let deviceId = resolveMicrophoneCaptureDeviceID() {
+				configuration.setValue(deviceId, forKey: "microphoneCaptureDeviceID")
+			}
+		} else {
+			nativeMicrophoneEnabled = false
+		}
+
 		return configuration
 	}

@@ -270,8 +367,8 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 		let writer = try AVAssetWriter(outputURL: outputUrl, fileType: .mp4)
 		let settings: [String: Any] = [
 			AVVideoCodecKey: AVVideoCodecType.h264,
-			AVVideoWidthKey: request.video.width,
-			AVVideoHeightKey: request.video.height,
+			AVVideoWidthKey: outputWidth,
+			AVVideoHeightKey: outputHeight,
 			AVVideoCompressionPropertiesKey: [
 				AVVideoAverageBitRateKey: request.video.bitrate ?? 18_000_000,
 				AVVideoExpectedSourceFrameRateKey: request.video.fps,
@@ -287,6 +384,13 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 		writer.add(input)
 		self.writer = writer
 		self.videoInput = input
+
+		if request.audio.system.enabled {
+			systemAudioInput = try addAudioInput(to: writer, bitRate: 192_000)
+		}
+		if nativeMicrophoneEnabled {
+			microphoneAudioInput = try addAudioInput(to: writer, bitRate: 128_000)
+		}
 	}

 	private func finishWriter() async {
@@ -295,6 +399,8 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 		}

 		videoInput?.markAsFinished()
+		systemAudioInput?.markAsFinished()
+		microphoneAudioInput?.markAsFinished()

 		await withCheckedContinuation { continuation in
 			writer.finishWriting {
@@ -314,6 +420,91 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 			)
 		}
 	}
+
+	private func addAudioInput(to writer: AVAssetWriter, bitRate: Int) throws -> AVAssetWriterInput {
+		let settings: [String: Any] = [
+			AVFormatIDKey: kAudioFormatMPEG4AAC,
+			AVSampleRateKey: 48_000,
+			AVNumberOfChannelsKey: 2,
+			AVEncoderBitRateKey: bitRate,
+		]
+		let input = AVAssetWriterInput(mediaType: .audio, outputSettings: settings)
+		input.expectsMediaDataInRealTime = true
+
+		guard writer.canAdd(input) else {
+			throw HelperError.writerSetupFailed("Unable to add AAC audio input to AVAssetWriter.")
+		}
+
+		writer.add(input)
+		return input
+	}
+
+	private func appendAudioSampleBuffer(_ sampleBuffer: CMSampleBuffer, to input: AVAssetWriterInput?) {
+		guard didStartWriting else {
+			return
+		}
+		guard let input, input.isReadyForMoreMediaData else {
+			return
+		}
+
+		input.append(sampleBuffer)
+	}
+
+	private func isCompleteFrame(_ sampleBuffer: CMSampleBuffer) -> Bool {
+		guard let attachments = CMSampleBufferGetSampleAttachmentsArray(
+			sampleBuffer,
+			createIfNecessary: false
+		) as? [[SCStreamFrameInfo: Any]],
+			let attachment = attachments.first,
+			let statusRawValue = attachment[SCStreamFrameInfo.status] as? Int,
+			let status = SCFrameStatus(rawValue: statusRawValue)
+		else {
+			return true
+		}
+
+		return status == .complete
+	}
+
+	private func clampCaptureDimension(_ value: Int, fallback: Int) -> Int {
+		let requested = max(2, fallback)
+		let candidate = value > 0 ? value : requested
+		let clamped = min(candidate, requested)
+		return max(2, clamped - (clamped % 2))
+	}
+
+	private static func scaleFactor(for displayId: CGDirectDisplayID) -> Int {
+		guard let mode = CGDisplayCopyDisplayMode(displayId) else {
+			return 1
+		}
+
+		return max(1, mode.pixelWidth / max(1, mode.width))
+	}
+
+	private func supportsNativeMicrophoneCapture(streamConfig: SCStreamConfiguration) -> Bool {
+		streamConfig.responds(to: Selector(("setCaptureMicrophone:"))) &&
+			streamConfig.responds(to: Selector(("setMicrophoneCaptureDeviceID:"))) &&
+			SCStreamOutputType(rawValue: microphoneOutputTypeRawValue) != nil
+	}
+
+	private func resolveMicrophoneCaptureDeviceID() -> String? {
+		let devices = AVCaptureDevice.devices(for: .audio)
+
+		if let deviceName = request.audio.microphone.deviceName?.trimmingCharacters(in: .whitespacesAndNewlines),
+			!deviceName.isEmpty,
+			let device = devices.first(where: { $0.localizedName == deviceName })
+		{
+			return device.uniqueID
+		}
+
+		if let deviceId = request.audio.microphone.deviceId?.trimmingCharacters(in: .whitespacesAndNewlines),
+			!deviceId.isEmpty,
+			devices.contains(where: { $0.uniqueID == deviceId })
+		{
+			return deviceId
+		}
+
+		return nil
+	}
 }

@main