huanld/openscreen
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases 2 Wiki Activity

fix: address native capture review feedback

Browse Source
This commit is contained in:
EtienneLescot
2026-05-05 20:57:52 +02:00
parent c7b43a50ef
commit ab3d38d90f
16 changed files with 260 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
electron/ipc/handlers.ts
+107 -4
View File
@@ -260,6 +260,7 @@ let nativeWindowsCaptureTargetPath: string | null = null;
let nativeWindowsCaptureWebcamTargetPath: string | null = null;
let nativeWindowsCaptureRecordingId: number | null = null;
let nativeWindowsCursorOffsetMs = 0;
const NATIVE_WINDOWS_CAPTURE_STOP_TIMEOUT_MS = 15_000;
function normalizeCursorSample(sample: unknown): CursorRecordingSample | null {
if (!sample || typeof sample !== "object") {
@@ -681,6 +682,19 @@ function waitForNativeWindowsCaptureStart(proc: ChildProcessWithoutNullStreams)
function waitForNativeWindowsCaptureStop(proc: ChildProcessWithoutNullStreams) {
return new Promise<string>((resolve, reject) => {
const timer = setTimeout(() => {
cleanup();
if (!proc.killed) {
proc.kill();
}
reject(
new Error(
`Timed out waiting for native Windows capture to stop. Output path: ${
nativeWindowsCaptureTargetPath ?? "unknown"
}. Output: ${nativeWindowsCaptureOutput.trim()}`,
),
);
}, NATIVE_WINDOWS_CAPTURE_STOP_TIMEOUT_MS);
const onOutput = (chunk: Buffer) => {
nativeWindowsCaptureOutput += chunk.toString();
};
@@ -707,6 +721,7 @@ function waitForNativeWindowsCaptureStop(proc: ChildProcessWithoutNullStreams) {
reject(error);
};
const cleanup = () => {
clearTimeout(timer);
proc.stdout.off("data", onOutput);
proc.stderr.off("data", onOutput);
proc.off("close", onClose);
@@ -744,6 +759,78 @@ function setCurrentRecordingSessionState(session: RecordingSession | null) {
currentVideoPath = session?.screenVideoPath ?? null;
}
function getSessionManifestPathForVideo(videoPath: string) {
const parsedPath = path.parse(videoPath);
const baseName = parsedPath.name.endsWith("-webcam")
? parsedPath.name.slice(0, -"-webcam".length)
: parsedPath.name;
return path.join(parsedPath.dir, `${baseName}${RECORDING_SESSION_SUFFIX}`);
}
async function loadRecordedSessionForVideoPath(
videoPath: string,
): Promise<RecordingSession | null> {
try {
const manifestPath = getSessionManifestPathForVideo(videoPath);
if (!isPathAllowed(manifestPath)) {
const parsedVideoPath = path.parse(videoPath);
if (!isPathWithinDir(path.resolve(manifestPath), parsedVideoPath.dir)) {
return null;
}
}
const content = await fs.readFile(manifestPath, "utf-8");
const session = normalizeRecordingSession(JSON.parse(content));
if (!session) {
return null;
}
const normalizedVideoPath = normalizePath(videoPath);
const matchesScreen = normalizePath(session.screenVideoPath) === normalizedVideoPath;
const matchesWebcam =
typeof session.webcamVideoPath === "string" &&
normalizePath(session.webcamVideoPath) === normalizedVideoPath;
if (!matchesScreen && !matchesWebcam) {
return null;
}
if (!isPathAllowed(session.screenVideoPath)) {
const approvedScreen = await approveReadableVideoPath(session.screenVideoPath, [
path.dirname(manifestPath),
RECORDINGS_DIR,
]);
if (!approvedScreen) {
return null;
}
session.screenVideoPath = approvedScreen;
}
if (session.webcamVideoPath && !isPathAllowed(session.webcamVideoPath)) {
const approvedWebcam = await approveReadableVideoPath(session.webcamVideoPath, [
path.dirname(manifestPath),
RECORDINGS_DIR,
]);
if (!approvedWebcam) {
session.webcamVideoPath = undefined;
} else {
session.webcamVideoPath = approvedWebcam;
}
}
approveFilePath(session.screenVideoPath);
if (session.webcamVideoPath) {
approveFilePath(session.webcamVideoPath);
}
return session;
} catch (error) {
const nodeError = error as NodeJS.ErrnoException;
if (nodeError.code !== "ENOENT") {
console.error("Failed to restore recording session manifest:", error);
}
return null;
}
}
export function registerIpcHandlers(
createEditorWindow: () => void,
createSourceSelectorWindow: () => BrowserWindow,
@@ -1629,7 +1716,7 @@ export function registerIpcHandlers(
}
}
ipcMain.handle("set-current-video-path", (_, path: string) => {
ipcMain.handle("set-current-video-path", async (_, path: string) => {
return setCurrentVideoPath(path);
});
@@ -1647,10 +1734,26 @@ export function registerIpcHandlers(
: { success: false };
});
function setCurrentVideoPath(path: string): ProjectPathResult {
currentVideoPath = normalizeVideoSourcePath(path) ?? path;
async function setCurrentVideoPath(path: string): Promise<ProjectPathResult> {
const normalizedPath = normalizeVideoSourcePath(path);
if (!normalizedPath || !isPathAllowed(normalizedPath)) {
return {
success: false,
message: "Video path has not been approved",
};
}
const restoredSession = await loadRecordedSessionForVideoPath(normalizedPath);
if (restoredSession) {
setCurrentRecordingSessionState(restoredSession);
} else {
setCurrentRecordingSessionState({
screenVideoPath: normalizedPath,
createdAt: Date.now(),
});
}
currentProjectPath = null;
return { success: true };
return { success: true, path: currentVideoPath ?? normalizedPath };
}
ipcMain.handle("get-current-video-path", () => {
electron/ipc/nativeBridge.ts
+2 -2
View File
@@ -27,7 +27,7 @@ export interface NativeBridgeContext {
) => Promise<ProjectFileResult>;
loadProjectFile: () => Promise<ProjectFileResult>;
loadCurrentProjectFile: () => Promise<ProjectFileResult>;
setCurrentVideoPath: (path: string) => ProjectPathResult;
setCurrentVideoPath: (path: string) => ProjectPathResult | Promise<ProjectPathResult>;
getCurrentVideoPathResult: () => ProjectPathResult;
clearCurrentVideoPath: () => ProjectPathResult;
resolveAssetBasePath: () => string | null;
@@ -171,7 +171,7 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) {
case "setCurrentVideoPath":
return createSuccessResponse(
requestId,
projectService.setCurrentVideoPath(request.payload.path),
await projectService.setCurrentVideoPath(request.payload.path),
);
case "getCurrentVideoPath":
return createSuccessResponse(requestId, projectService.getCurrentVideoPath());
electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts
+5 -1
View File
@@ -1,9 +1,13 @@
export function buildPowerShellCommand(sampleIntervalMs: number, windowHandle?: string | null) {
const targetWindowHandle =
typeof windowHandle === "string" && /^(?:0x[0-9a-fA-F]+|\d+)$/.test(windowHandle)
? `'${windowHandle}'`
: "$null";
const script = String.raw`
$ErrorActionPreference = 'Stop'
Add-Type -AssemblyName System.Drawing
$targetWindowHandle = ${windowHandle ? `'${windowHandle}'` : "$null"}
$targetWindowHandle = ${targetWindowHandle}
$source = @"
using System;
electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts
+10
View File
@@ -151,6 +151,7 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession {
if (payload.type === "error") {
this.logDiagnostic("helper-error", { message: payload.message });
console.error("Windows cursor helper error:", payload.message);
this.failHelper(new Error(payload.message));
return;
}
@@ -256,6 +257,15 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession {
reject?.(error);
}
private failHelper(error: Error) {
this.rejectReady(error);
const child = this.process;
this.process = null;
if (child && !child.killed) {
child.kill();
}
}
private clearReadyState() {
if (this.readyTimer) {
clearTimeout(this.readyTimer);
electron/native-bridge/cursor/telemetryCursorAdapter.ts
+2 -1
View File
@@ -38,7 +38,8 @@ export class TelemetryCursorAdapter implements CursorNativeAdapter {
const resolvedVideoPath = this.options.resolveVideoPath(videoPath);
if (!resolvedVideoPath) {
return {
success: true,
success: false,
message: "No video path is available for cursor telemetry",
samples: [],
} satisfies CursorTelemetryLoadResult;
}
electron/native-bridge/services/projectService.ts
+3 -3
View File
@@ -16,7 +16,7 @@ interface ProjectServiceOptions {
) => Promise<ProjectFileResult>;
loadProjectFile: () => Promise<ProjectFileResult>;
loadCurrentProjectFile: () => Promise<ProjectFileResult>;
setCurrentVideoPath: (path: string) => ProjectPathResult;
setCurrentVideoPath: (path: string) => ProjectPathResult | Promise<ProjectPathResult>;
getCurrentVideoPathResult: () => ProjectPathResult;
clearCurrentVideoPath: () => ProjectPathResult;
}
@@ -60,8 +60,8 @@ export class ProjectService {
return result;
}
setCurrentVideoPath(path: string) {
const result = this.options.setCurrentVideoPath(path);
async setCurrentVideoPath(path: string) {
const result = await this.options.setCurrentVideoPath(path);
this.getCurrentContext();
return result;
}
electron/native/wgc-capture/src/dshow_webcam_capture.cpp
+5 -2
View File
@@ -256,6 +256,7 @@ bool DirectShowWebcamCapture::initialize(
int requestedFps) {
stop();
delete impl_;
impl_ = nullptr;
impl_ = new Impl();
fps_ = std::clamp(requestedFps > 0 ? requestedFps : 30, 1, 60);
@@ -402,7 +403,7 @@ void DirectShowWebcamCapture::stop() {
}
void DirectShowWebcamCapture::captureLoop() {
CoInitializeEx(nullptr, COINIT_MULTITHREADED);
const HRESULT coinitHr = CoInitializeEx(nullptr, COINIT_MULTITHREADED);
while (!stopRequested_ && impl_ && impl_->sampleGrabber) {
long bufferSize = 0;
HRESULT hr = impl_->sampleGrabber->GetCurrentBuffer(&bufferSize, nullptr);
@@ -415,7 +416,9 @@ void DirectShowWebcamCapture::captureLoop() {
}
std::this_thread::sleep_for(std::chrono::milliseconds(1000 / std::max(1, fps_)));
}
CoUninitialize();
if (SUCCEEDED(coinitHr)) {
CoUninitialize();
}
}
void DirectShowWebcamCapture::storeFrame(const BYTE* buffer, long length) {
electron/native/wgc-capture/src/main.cpp
+23 -3
View File
@@ -463,13 +463,14 @@ int main(int argc, char* argv[]) {
std::atomic<bool> firstFrameWritten = false;
std::atomic<bool> encodeFailed = false;
Microsoft::WRL::ComPtr<ID3D11Texture2D> latestFrameTexture;
int64_t latestFrameTimestampHns = 0;
int64_t firstFrameTimestampHns = -1;
std::vector<BYTE> latestWebcamFrame;
int latestWebcamWidth = 0;
int latestWebcamHeight = 0;
bool hasVisibleWebcamFrame = false;
session.setFrameCallback([&](ID3D11Texture2D* texture, int64_t timestampHns) {
(void)timestampHns;
if (stopRequested) {
return;
}
@@ -490,6 +491,7 @@ int main(int argc, char* argv[]) {
}
session.context()->CopyResource(latestFrameTexture.Get(), texture);
latestFrameTimestampHns = timestampHns;
if (!firstFrameWritten.exchange(true)) {
cv.notify_all();
}
@@ -498,6 +500,7 @@ int main(int argc, char* argv[]) {
auto writeVideoFrames = [&]() {
const auto startedAt = std::chrono::steady_clock::now();
uint64_t frameIndex = 0;
int64_t lastEncodedVideoTimestampHns = -1;
while (!stopRequested && !encodeFailed) {
{
@@ -519,15 +522,32 @@ int main(int argc, char* argv[]) {
latestWebcamWidth,
latestWebcamHeight,
};
const int64_t syntheticTimestampHns =
static_cast<int64_t>((frameIndex * 10'000'000ULL) / config.fps);
const int64_t sourceTimestampHns =
latestFrameTimestampHns > 0 ? latestFrameTimestampHns : syntheticTimestampHns;
if (firstFrameTimestampHns < 0) {
firstFrameTimestampHns = sourceTimestampHns;
}
int64_t frameTimestampHns =
std::max<int64_t>(0, sourceTimestampHns - firstFrameTimestampHns);
if (lastEncodedVideoTimestampHns >= 0 &&
frameTimestampHns <= lastEncodedVideoTimestampHns) {
frameTimestampHns =
lastEncodedVideoTimestampHns + static_cast<int64_t>(10'000'000ULL / config.fps);
}
if (latestFrameTexture && !encoder.writeFrame(
latestFrameTexture.Get(),
static_cast<int64_t>((frameIndex * 10'000'000ULL) / config.fps),
frameTimestampHns,
webcamFrame.data ? &webcamFrame : nullptr)) {
encodeFailed = true;
stopRequested = true;
cv.notify_all();
return;
}
if (latestFrameTexture) {
lastEncodedVideoTimestampHns = frameTimestampHns;
}
}
frameIndex += 1;
@@ -714,7 +734,7 @@ int main(int argc, char* argv[]) {
}
if (stdinThread.joinable()) {
stdinThread.join();
stdinThread.detach();
}
if (encodeFailed) {
electron/native/wgc-capture/src/wasapi_loopback_capture.cpp
+34 -2
View File
@@ -293,6 +293,8 @@ bool WasapiLoopbackCapture::start(AudioCallback callback) {
callback_ = std::move(callback);
stopRequested_ = false;
writtenFrames_ = 0;
lastDevicePositionEnd_ = 0;
hasLastDevicePosition_ = false;
HRESULT hr = audioClient_->Start();
if (!succeeded(hr, "IAudioClient::Start")) {
@@ -324,6 +326,22 @@ const std::wstring& WasapiLoopbackCapture::selectedDeviceName() const {
}
void WasapiLoopbackCapture::captureLoop() {
auto emitSilenceFrames = [&](uint64_t frames, int64_t timestampHns) {
constexpr uint64_t MaxSilenceChunkFrames = 4800;
uint64_t remainingFrames = frames;
int64_t currentTimestampHns = timestampHns;
while (remainingFrames > 0 && !stopRequested_) {
const uint64_t chunkFrames = std::min<uint64_t>(remainingFrames, MaxSilenceChunkFrames);
const DWORD chunkBytes = static_cast<DWORD>(chunkFrames * inputFormat_.blockAlign);
const int64_t chunkDurationHns =
static_cast<int64_t>((chunkFrames * HnsPerSecond) / inputFormat_.sampleRate);
silenceBuffer_.assign(chunkBytes, 0);
callback_(silenceBuffer_.data(), chunkBytes, currentTimestampHns, chunkDurationHns);
remainingFrames -= chunkFrames;
currentTimestampHns += chunkDurationHns;
}
};
while (!stopRequested_) {
UINT32 packetFrames = 0;
HRESULT hr = captureClient_->GetNextPacketSize(&packetFrames);
@@ -337,17 +355,29 @@ void WasapiLoopbackCapture::captureLoop() {
BYTE* data = nullptr;
UINT32 framesAvailable = 0;
DWORD flags = 0;
UINT64 devicePosition = 0;
UINT64 qpcPosition = 0;
hr = captureClient_->GetBuffer(&data, &framesAvailable, &flags, nullptr, nullptr);
hr = captureClient_->GetBuffer(&data, &framesAvailable, &flags, &devicePosition, &qpcPosition);
if (FAILED(hr)) {
std::cerr << "ERROR: IAudioCaptureClient::GetBuffer failed (hr=0x" << std::hex
<< hr << std::dec << ")" << std::endl;
break;
}
(void)qpcPosition;
if (hasLastDevicePosition_ && devicePosition > lastDevicePositionEnd_) {
const uint64_t gapFrames = devicePosition - lastDevicePositionEnd_;
if ((flags & AUDCLNT_BUFFERFLAGS_DATA_DISCONTINUITY) != 0 || gapFrames > framesAvailable) {
const int64_t gapTimestampHns =
static_cast<int64_t>((lastDevicePositionEnd_ * HnsPerSecond) / inputFormat_.sampleRate);
emitSilenceFrames(gapFrames, gapTimestampHns);
}
}
const DWORD byteCount = framesAvailable * inputFormat_.blockAlign;
const int64_t timestampHns =
static_cast<int64_t>((writtenFrames_ * HnsPerSecond) / inputFormat_.sampleRate);
static_cast<int64_t>((devicePosition * HnsPerSecond) / inputFormat_.sampleRate);
const int64_t durationHns =
static_cast<int64_t>((static_cast<uint64_t>(framesAvailable) * HnsPerSecond) /
inputFormat_.sampleRate);
@@ -362,6 +392,8 @@ void WasapiLoopbackCapture::captureLoop() {
}
writtenFrames_ += framesAvailable;
lastDevicePositionEnd_ = devicePosition + framesAvailable;
hasLastDevicePosition_ = true;
captureClient_->ReleaseBuffer(framesAvailable);
hr = captureClient_->GetNextPacketSize(&packetFrames);
electron/native/wgc-capture/src/wasapi_loopback_capture.h
+2
View File
@@ -55,4 +55,6 @@ private:
std::atomic<bool> stopRequested_ = false;
std::vector<BYTE> silenceBuffer_;
uint64_t writtenFrames_ = 0;
uint64_t lastDevicePositionEnd_ = 0;
bool hasLastDevicePosition_ = false;
};
src/components/launch/LaunchWindow.tsx
+5 -1
View File
@@ -302,7 +302,11 @@ export function LaunchWindow() {
}
if (result.success && result.path) {
await nativeBridgeClient.project.setCurrentVideoPath(result.path);
const setVideoPathResult = await nativeBridgeClient.project.setCurrentVideoPath(result.path);
if (!setVideoPathResult.success) {
console.error("Failed to set current video path:", setVideoPathResult);
return;
}
await window.electronAPI.switchToEditor();
}
};
src/components/video-editor/projectPersistence.ts
+23 -2
View File
@@ -119,12 +119,33 @@ function clamp(value: number, min: number, max: number) {
return Math.min(max, Math.max(min, value));
}
function encodePathSegments(pathname: string, keepWindowsDrive = false): string {
return pathname
.split("/")
.map((segment, index) => {
if (!segment) {
return segment;
}
if (keepWindowsDrive && index === 0 && /^[a-zA-Z]:$/.test(segment)) {
return segment;
}
return encodeURIComponent(segment);
})
.join("/");
}
export function toFileUrl(filePath: string): string {
const normalized = filePath.replace(/\\/g, "/");
if (normalized.match(/^[a-zA-Z]:/)) {
return `file:///${encodeURI(normalized)}`;
return `file:///${encodePathSegments(normalized, true)}`;
}
return `file://${encodeURI(normalized)}`;
if (normalized.startsWith("//")) {
const withoutPrefix = normalized.slice(2);
const [host = "", ...segments] = withoutPrefix.split("/");
return `file://${host}/${encodePathSegments(segments.join("/"))}`;
}
const absolutePath = normalized.startsWith("/") ? normalized : `/${normalized}`;
return `file://${encodePathSegments(absolutePath)}`;
}
export function fromFileUrl(fileUrl: string): string {
src/components/video-editor/videoPlayback/cursorRenderer.ts
+2
View File
@@ -721,6 +721,8 @@ export class PixiCursorOverlay {
}
this.cursorMotionBlurFilter.destroy();
this.container.destroy({ children: true });
cursorAssetsPromise = null;
loadedCursorAssets = {};
}
}
src/hooks/useScreenRecorder.ts
+2 -2
View File
@@ -644,7 +644,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
finalizing: false,
};
accumulatedDurationMs.current = 0;
segmentStartedAt.current = result.recordingId;
segmentStartedAt.current = Date.now();
allowAutoFinalize.current = true;
setRecording(true);
setPaused(false);
@@ -918,7 +918,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
}
accumulatedDurationMs.current = 0;
segmentStartedAt.current = activeRecordingId;
segmentStartedAt.current = Date.now();
allowAutoFinalize.current = true;
setRecording(true);
setPaused(false);
src/lib/exporter/frameRenderer.ts
+16 -1
View File
@@ -145,6 +145,7 @@ export class FrameRenderer {
private threeDPass: ThreeDPass | null = null;
private currentRotation3D: Rotation3D = { ...DEFAULT_ROTATION_3D };
private cursorImageCache = new Map<string, HTMLImageElement>();
private warnedKeys = new Set<string>();
private config: FrameRenderConfig;
private animationState: AnimationState;
private layoutCache: LayoutCache | null = null;
@@ -585,7 +586,13 @@ export class FrameRenderer {
1,
activeNativeCursor.sample,
);
const image = await this.getCursorImage(renderAsset);
let image: HTMLImageElement;
try {
image = await this.getCursorImage(renderAsset);
} catch (error) {
this.warnOnce("native-cursor-image-load", "Failed to load native cursor asset", error);
return;
}
const scale = Math.max(0, this.config.cursorScale ?? 1);
const appliedScale = this.animationState.appliedScale;
const canvasX = projectedPoint.x * appliedScale + this.animationState.x;
@@ -616,6 +623,14 @@ export class FrameRenderer {
return image;
}
private warnOnce(key: string, message: string, error: unknown) {
if (this.warnedKeys.has(key)) {
return;
}
this.warnedKeys.add(key);
console.warn(`[FrameRenderer] ${message}:`, error);
}
private updateLayout(webcamFrame?: VideoFrame | null): void {
if (!this.app || !this.videoSprite || !this.maskGraphics || !this.videoContainer) return;
tests/e2e/windows-native-checklist.spec.ts
+19 -3
View File
@@ -1,3 +1,4 @@
import { once } from "node:events";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
@@ -18,12 +19,16 @@ async function launchApp() {
MAIN_JS,
"--no-sandbox",
"--enable-unsafe-swiftshader",
"--lang=en-US",
`--user-data-dir=${testUserDataDir}`,
],
env: {
...process.env,
ELECTRON_USER_DATA_DIR: testUserDataDir,
HEADLESS: process.env["HEADLESS"] ?? "true",
LANG: "en_US.UTF-8",
LC_ALL: "en_US.UTF-8",
LANGUAGE: "en_US",
},
});
@@ -53,13 +58,24 @@ async function closeApp(app: ElectronApplication) {
}
).__childProcess;
await Promise.race([app.close(), new Promise<void>((resolve) => setTimeout(resolve, 5_000))]);
if (childProcess && !childProcess.killed) {
childProcess.kill();
if (childProcess && childProcess.exitCode === null && childProcess.signalCode === null) {
if (!childProcess.killed) {
childProcess.kill();
}
await Promise.race([
once(childProcess, "close"),
new Promise<void>((resolve) => setTimeout(resolve, 5_000)),
]);
}
const testUserDataDir = (app as ElectronApplication & { __testUserDataDir?: string })
.__testUserDataDir;
if (testUserDataDir && fs.existsSync(testUserDataDir)) {
fs.rmSync(testUserDataDir, { recursive: true, force: true });
fs.rmSync(testUserDataDir, {
recursive: true,
force: true,
maxRetries: 5,
retryDelay: 100,
});
}
}