huanld/openscreen
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases 2 Wiki Activity

Release OpenScreen 1.4.2

Browse Source
This commit is contained in:
huanld
2026-05-28 10:01:22 +07:00
parent 69804c41c7
commit 198dc022b0
25 changed files with 844 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
electron/electron-env.d.ts
Vendored
+13
View File
@@ -53,6 +53,9 @@ interface Window {
import("../src/guide/contracts").CaptureGuidePointerMarkerResult import("../src/guide/contracts").CaptureGuidePointerMarkerResult
> >
>; >;
onMarkerCaptured: (
callback: (payload: import("../src/guide/contracts").GuideMarkerCapturedPayload) => void,
) => () => void;
finalizeEvents: ( finalizeEvents: (
input: import("../src/guide/contracts").FinalizeGuideEventsInput, input: import("../src/guide/contracts").FinalizeGuideEventsInput,
) => Promise< ) => Promise<
@@ -376,6 +379,16 @@ interface ProcessedDesktopSource {
display_id: string; display_id: string;
thumbnail: string | null; thumbnail: string | null;
appIcon: string | null; appIcon: string | null;
displayId?: number;
displayIndex?: number;
screenIndex?: number;
displayLabel?: string;
bounds?: {
x: number;
y: number;
width: number;
height: number;
};
} }
interface CursorTelemetryPoint { interface CursorTelemetryPoint {
electron/guide/guideStore.test.ts
+68
View File
@@ -228,6 +228,74 @@ describe("GuideStore", () => {
await expect(fs.readFile(html.path, "utf-8")).resolves.toContain("<!doctype html>"); await expect(fs.readFile(html.path, "utf-8")).resolves.toContain("<!doctype html>");
}); });
it("repairs generic hotkey marker text and attaches AI draft artifacts", async () => {
const store = new GuideStore(recordingsDir, {
ocrClient: {
recognize: async (snapshot) => [
{
id: `ocr-${snapshot.id}-1`,
snapshotId: snapshot.id,
text: "Save",
confidence: 0.95,
box: { x: 0.45, y: 0.45, width: 0.15, height: 0.08 },
},
],
},
draftClient: {
generate: async () => ({
title: "Guide",
steps: [
{
id: "guide-step-1",
order: 1,
title: "Step 1: Click Ctrl+F12 marker",
instruction: "Click Ctrl+F12 marker.",
},
],
}),
},
});
await store.startSession(114);
await store.addMarker({
recordingId: 114,
kind: "hotkey",
timeMs: 200,
label: "Ctrl+F12 marker",
normalizedX: 0.5,
normalizedY: 0.5,
});
const videoPath = path.join(recordingsDir, "recording-114.mp4");
await fs.writeFile(videoPath, "");
const eventsSession = await store.finalizeEvents({ recordingId: 114, videoPath });
await store.writeSnapshot({
recordingId: 114,
eventId: eventsSession.events[0]?.id ?? "",
timeMs: 700,
offsetMs: 500,
width: 800,
height: 600,
pngBytes: new Uint8Array([1, 2, 3]).buffer,
});
await store.runOcr({ recordingId: 114 });
const draftSession = await store.generateDraft({
recordingId: 114,
language: "en",
provider: "deepseek",
});
expect(draftSession.candidates[0]).toMatchObject({
targetText: "Save",
position: { xPercent: 50, yPercent: 50 },
});
expect(draftSession.generatedGuide?.steps[0]).toMatchObject({
title: "Step 1: Save",
instruction: 'Click "Save".',
sourceCandidateId: draftSession.candidates[0]?.id,
screenshotPath: draftSession.snapshots[0]?.path,
});
});
it("discards a guide session and output directory", async () => { it("discards a guide session and output directory", async () => {
const store = new GuideStore(recordingsDir); const store = new GuideStore(recordingsDir);
const session = await store.startSession(111); const session = await store.startSession(111);
electron/guide/guideStore.ts
+127 -1
View File
@@ -336,10 +336,11 @@ export class GuideStore {
} }
} }
const normalizedGuide = normalizeGeneratedGuide(generatedGuide) ?? generatedGuide;
const updatedSession = touchSession({ const updatedSession = touchSession({
...session, ...session,
candidates, candidates,
generatedGuide: normalizeGeneratedGuide(generatedGuide) ?? generatedGuide, generatedGuide: enrichGeneratedGuide(normalizedGuide, session, candidates, input.language),
status: "draft-ready", status: "draft-ready",
}); });
await this.writeSession(updatedSession); await this.writeSession(updatedSession);
@@ -743,11 +744,41 @@ function normalizeGuideStepCandidate(input: unknown): GuideStepCandidate | null
input.targetRole === "unknown" input.targetRole === "unknown"
? input.targetRole ? input.targetRole
: undefined, : undefined,
position: normalizeGuideCandidatePosition(input.position),
nearbyText, nearbyText,
confidence, confidence,
}; };
} }
function normalizeGuideCandidatePosition(
input: unknown,
): GuideStepCandidate["position"] | undefined {
if (!isRecord(input)) {
return undefined;
}
const normalizedX = normalizeOptionalNormalizedNumber(input.normalizedX);
const normalizedY = normalizeOptionalNormalizedNumber(input.normalizedY);
const xPercent = normalizeOptionalNumber(input.xPercent);
const yPercent = normalizeOptionalNumber(input.yPercent);
const description = normalizeOptionalString(input.description);
if (
normalizedX === undefined ||
normalizedY === undefined ||
xPercent === undefined ||
yPercent === undefined ||
!description
) {
return undefined;
}
return {
normalizedX,
normalizedY,
xPercent,
yPercent,
description,
};
}
function normalizeGeneratedGuide(input: unknown): GeneratedGuide | null { function normalizeGeneratedGuide(input: unknown): GeneratedGuide | null {
if (!isRecord(input)) { if (!isRecord(input)) {
return null; return null;
@@ -785,6 +816,101 @@ function normalizeGeneratedGuide(input: unknown): GeneratedGuide | null {
}; };
} }
function enrichGeneratedGuide(
guide: GeneratedGuide,
session: GuideSession,
candidates: GuideStepCandidate[],
language: GenerateGuideDraftInput["language"],
): GeneratedGuide {
const sortedCandidates = [...candidates].sort((left, right) => left.timeMs - right.timeMs);
const candidatesById = new Map(candidates.map((candidate) => [candidate.id, candidate]));
const snapshotsById = new Map(session.snapshots.map((snapshot) => [snapshot.id, snapshot]));
const snapshotsByEventId = new Map(
session.snapshots.map((snapshot) => [snapshot.eventId, snapshot]),
);
return {
...guide,
steps: guide.steps.map((step, index) => {
const candidate =
(step.sourceCandidateId ? candidatesById.get(step.sourceCandidateId) : undefined) ??
sortedCandidates[index];
const snapshot = candidate
? ((candidate.snapshotId ? snapshotsById.get(candidate.snapshotId) : undefined) ??
snapshotsByEventId.get(candidate.eventId))
: undefined;
const repairedStep = repairGenericMarkerStep(step, candidate, language);
return {
...repairedStep,
sourceCandidateId: candidate?.id ?? repairedStep.sourceCandidateId,
screenshotPath: repairedStep.screenshotPath ?? snapshot?.path,
};
}),
};
}
function repairGenericMarkerStep(
step: GeneratedGuideStep,
candidate: GuideStepCandidate | undefined,
language: GenerateGuideDraftInput["language"],
): GeneratedGuideStep {
if (
!candidate ||
(!containsGenericMarkerText(step.title) && !containsGenericMarkerText(step.instruction))
) {
return step;
}
return {
...step,
title: buildRepairedStepTitle(candidate, step.order, language),
instruction: buildRepairedStepInstruction(candidate, language),
};
}
function containsGenericMarkerText(value: string): boolean {
return /\b(?:ctrl|control)(?:\s*\+\s*f12)?\s+marker\b/i.test(value);
}
function buildRepairedStepTitle(
candidate: GuideStepCandidate,
order: number,
language: GenerateGuideDraftInput["language"],
): string {
if (candidate.targetText) {
return language === "vi"
? `Bước ${order}: ${candidate.targetText}`
: `Step ${order}: ${candidate.targetText}`;
}
if (candidate.position) {
return language === "vi"
? `Bước ${order}: Vị trí x ${candidate.position.xPercent}%, y ${candidate.position.yPercent}%`
: `Step ${order}: Position x ${candidate.position.xPercent}%, y ${candidate.position.yPercent}%`;
}
return stepTitleFallback(order, language);
}
function buildRepairedStepInstruction(
candidate: GuideStepCandidate,
language: GenerateGuideDraftInput["language"],
): string {
if (candidate.targetText) {
return language === "vi"
? `Nhấn vào "${candidate.targetText}".`
: `Click "${candidate.targetText}".`;
}
if (candidate.position) {
return language === "vi"
? `Nhấn tại vùng ${candidate.position.description} (x ${candidate.position.xPercent}%, y ${candidate.position.yPercent}%).`
: `Click the ${candidate.position.description} area (x ${candidate.position.xPercent}%, y ${candidate.position.yPercent}%).`;
}
return language === "vi" ? "Thực hiện thao tác tại mốc đã ghi." : "Perform the recorded action.";
}
function stepTitleFallback(order: number, language: GenerateGuideDraftInput["language"]): string {
return language === "vi" ? `Bước ${order}` : `Step ${order}`;
}
function normalizeArray<T>(input: unknown, normalize: (value: unknown) => T | null): T[] { function normalizeArray<T>(input: unknown, normalize: (value: unknown) => T | null): T[] {
return Array.isArray(input) return Array.isArray(input)
? input.map((value) => normalize(value)).filter((value): value is T => value !== null) ? input.map((value) => normalize(value)).filter((value): value is T => value !== null)
electron/ipc/handlers.ts
+304 -25
View File
@@ -5,7 +5,7 @@ import fs from "node:fs/promises";
import os from "node:os"; import os from "node:os";
import path from "node:path"; import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url"; import { fileURLToPath, pathToFileURL } from "node:url";
import type { DesktopCapturerSource } from "electron"; import type { DesktopCapturerSource, Rectangle } from "electron";
import { import {
app, app,
BrowserWindow, BrowserWindow,
@@ -17,6 +17,7 @@ import {
shell, shell,
systemPreferences, systemPreferences,
} from "electron"; } from "electron";
import type { GuideMarkerCapturedPayload } from "../../src/guide/contracts";
import type { NativeMacRecordingRequest } from "../../src/lib/nativeMacRecording"; import type { NativeMacRecordingRequest } from "../../src/lib/nativeMacRecording";
import type { NativeWindowsRecordingRequest } from "../../src/lib/nativeWindowsRecording"; import type { NativeWindowsRecordingRequest } from "../../src/lib/nativeWindowsRecording";
import { import {
@@ -344,9 +345,16 @@ type SelectedSource = {
name: string; name: string;
id?: string; id?: string;
display_id?: string; display_id?: string;
displayId?: number;
displayIndex?: number;
screenIndex?: number;
displayLabel?: string;
bounds?: SourceBounds;
[key: string]: unknown; [key: string]: unknown;
}; };
type SourceBounds = { x: number; y: number; width: number; height: number };
type AttachNativeMacWebcamRecordingInput = { type AttachNativeMacWebcamRecordingInput = {
screenVideoPath?: string; screenVideoPath?: string;
recordingId?: number; recordingId?: number;
@@ -429,8 +437,10 @@ let nativeMacCursorRecordingStartMs = 0;
let nativeMacPauseStartedAtMs: number | null = null; let nativeMacPauseStartedAtMs: number | null = null;
let nativeMacPauseRanges: Array<{ startMs: number; endMs: number }> = []; let nativeMacPauseRanges: Array<{ startMs: number; endMs: number }> = [];
let nativeMacIsPaused = false; let nativeMacIsPaused = false;
let guideHotkeyListenerProcess: ChildProcessWithoutNullStreams | null = null;
const GUIDE_MARKER_HOTKEY = "Control+F12"; const GUIDE_MARKER_HOTKEY = "Control+F12";
const GUIDE_MARKER_HOTKEY_LABEL = "Ctrl+F12"; const GUIDE_MARKER_HOTKEY_LABEL = "Ctrl+F12";
type GuideMarkerTrigger = GuideMarkerCapturedPayload["trigger"];
type GuideHotkeyBounds = { x: number; y: number; width: number; height: number }; type GuideHotkeyBounds = { x: number; y: number; width: number; height: number };
type GuideHotkeyRecordingState = { type GuideHotkeyRecordingState = {
recordingId: number; recordingId: number;
@@ -442,6 +452,8 @@ type GuideHotkeyRecordingState = {
let activeGuideHotkeyRecording: GuideHotkeyRecordingState | null = null; let activeGuideHotkeyRecording: GuideHotkeyRecordingState | null = null;
let activeGuideHotkeySessionId: number | null = null; let activeGuideHotkeySessionId: number | null = null;
let guideMarkerHotkeyRegistered = false; let guideMarkerHotkeyRegistered = false;
let lastGuideHotkeyCaptureAtMs = 0;
const GUIDE_HOTKEY_CAPTURE_DEBOUNCE_MS = 250;
function normalizeCursorSample(sample: unknown): CursorRecordingSample | null { function normalizeCursorSample(sample: unknown): CursorRecordingSample | null {
if (!sample || typeof sample !== "object") { if (!sample || typeof sample !== "object") {
@@ -590,12 +602,109 @@ function resolveAssetBasePath() {
} }
} }
function parseDesktopCapturerScreenIndex(sourceId?: string | null): number | null {
if (!sourceId?.startsWith("screen:")) {
return null;
}
const indexPart = sourceId.split(":")[1];
if (!indexPart || !/^\d+$/.test(indexPart)) {
return null;
}
const index = Number(indexPart);
return Number.isInteger(index) && index >= 0 ? index : null;
}
function normalizeSourceBounds(input: unknown): SourceBounds | undefined {
if (!input || typeof input !== "object") {
return undefined;
}
const bounds = input as Partial<SourceBounds>;
const x = Number(bounds.x);
const y = Number(bounds.y);
const width = Number(bounds.width);
const height = Number(bounds.height);
if (
!Number.isFinite(x) ||
!Number.isFinite(y) ||
!Number.isFinite(width) ||
!Number.isFinite(height) ||
width <= 0 ||
height <= 0
) {
return undefined;
}
return {
x: Math.round(x),
y: Math.round(y),
width: Math.round(width),
height: Math.round(height),
};
}
function toSourceBounds(bounds: Rectangle): SourceBounds {
return {
x: Math.round(bounds.x),
y: Math.round(bounds.y),
width: Math.round(bounds.width),
height: Math.round(bounds.height),
};
}
function findDisplayForSource(
source: Pick<DesktopCapturerSource, "id" | "display_id">,
screenSourceIndex?: number,
) {
const displays = screen.getAllDisplays();
const displayId = Number(source.display_id);
const displayById = Number.isFinite(displayId)
? displays.find((display) => display.id === displayId)
: undefined;
if (displayById) {
return { display: displayById, displayIndex: displays.indexOf(displayById) };
}
const sourceIndex = parseDesktopCapturerScreenIndex(source.id) ?? screenSourceIndex;
if (sourceIndex !== null && sourceIndex !== undefined && sourceIndex < displays.length) {
return { display: displays[sourceIndex], displayIndex: sourceIndex };
}
return { display: null, displayIndex: undefined };
}
function getSelectedSourceDisplay() {
const displays = screen.getAllDisplays();
const explicitDisplayId =
typeof selectedSource?.displayId === "number"
? selectedSource.displayId
: Number(selectedSource?.display_id);
const displayById = Number.isFinite(explicitDisplayId)
? displays.find((display) => display.id === explicitDisplayId)
: undefined;
if (displayById) {
return displayById;
}
const sourceIndex =
typeof selectedSource?.displayIndex === "number"
? selectedSource.displayIndex
: typeof selectedSource?.screenIndex === "number"
? selectedSource.screenIndex
: parseDesktopCapturerScreenIndex(selectedSource?.id);
if (sourceIndex !== null && sourceIndex !== undefined && sourceIndex < displays.length) {
return displays[sourceIndex];
}
return null;
}
function getSelectedSourceBounds() { function getSelectedSourceBounds() {
const cursor = screen.getCursorScreenPoint(); const cursor = screen.getCursorScreenPoint();
const sourceDisplayId = Number(selectedSource?.display_id); const selectedBounds = normalizeSourceBounds(selectedSource?.bounds);
const sourceDisplay = Number.isFinite(sourceDisplayId) if (selectedBounds) {
? (screen.getAllDisplays().find((display) => display.id === sourceDisplayId) ?? null) return selectedBounds;
: null; }
const sourceDisplay = getSelectedSourceDisplay();
return (sourceDisplay ?? screen.getDisplayNearestPoint(cursor)).bounds; return (sourceDisplay ?? screen.getDisplayNearestPoint(cursor)).bounds;
} }
@@ -698,12 +807,21 @@ function clampGuideHotkey01(value: number): number {
return Math.min(1, Math.max(0, value)); return Math.min(1, Math.max(0, value));
} }
async function captureGuideHotkeyMarker(guideStore: GuideStore) { async function captureGuideHotkeyMarker(
guideStore: GuideStore,
trigger: GuideMarkerTrigger = "global-shortcut",
) {
const recording = activeGuideHotkeyRecording; const recording = activeGuideHotkeyRecording;
if (!recording || activeGuideHotkeySessionId !== recording.recordingId) { if (!recording || activeGuideHotkeySessionId !== recording.recordingId) {
return { captured: false }; return { captured: false };
} }
const captureRequestedAtMs = Date.now();
if (captureRequestedAtMs - lastGuideHotkeyCaptureAtMs < GUIDE_HOTKEY_CAPTURE_DEBOUNCE_MS) {
return { captured: false };
}
lastGuideHotkeyCaptureAtMs = captureRequestedAtMs;
const point = getGuideHotkeyPoint(recording.bounds); const point = getGuideHotkeyPoint(recording.bounds);
try { try {
const result = await guideStore.addMarker({ const result = await guideStore.addMarker({
@@ -714,11 +832,21 @@ async function captureGuideHotkeyMarker(guideStore: GuideStore) {
y: point.normalizedY, y: point.normalizedY,
normalizedX: point.normalizedX, normalizedX: point.normalizedX,
normalizedY: point.normalizedY, normalizedY: point.normalizedY,
label: `${GUIDE_MARKER_HOTKEY_LABEL} marker`, });
notifyGuideMarkerCaptured({
recordingId: result.event.recordingId,
eventId: result.event.id,
timeMs: result.event.timeMs,
trigger,
normalizedX: result.event.normalizedX,
normalizedY: result.event.normalizedY,
rawX: point.rawX,
rawY: point.rawY,
}); });
console.info("[guide-hotkey] marker captured", { console.info("[guide-hotkey] marker captured", {
recordingId: recording.recordingId, recordingId: recording.recordingId,
timeMs: result.event.timeMs, timeMs: result.event.timeMs,
trigger,
normalizedX: result.event.normalizedX, normalizedX: result.event.normalizedX,
normalizedY: result.event.normalizedY, normalizedY: result.event.normalizedY,
rawX: point.rawX, rawX: point.rawX,
@@ -733,13 +861,110 @@ async function captureGuideHotkeyMarker(guideStore: GuideStore) {
} }
} }
function notifyGuideMarkerCaptured(payload: GuideMarkerCapturedPayload) {
for (const window of BrowserWindow.getAllWindows()) {
if (!window.isDestroyed()) {
window.webContents.send("guide:marker-captured", payload);
}
}
}
function handleGuideHotkeyListenerLine(line: string, guideStore: GuideStore) {
const text = line.trim();
if (!text) {
return;
}
try {
const event = JSON.parse(text) as {
event?: unknown;
key?: unknown;
state?: unknown;
};
if (event.event === "ready") {
console.info("[guide-hotkey] native Ctrl listener ready");
return;
}
if (event.event === "guide-hotkey" && event.key === "control" && event.state === "down") {
void captureGuideHotkeyMarker(guideStore, "global-control");
return;
}
} catch {
console.warn("[guide-hotkey] native listener emitted invalid JSON:", text);
}
}
async function startNativeGuideHotkeyListener(guideStore: GuideStore) {
if (process.platform !== "win32" || guideHotkeyListenerProcess) {
return;
}
const helperPath = await findNativeGuideHotkeyListenerPath();
if (!helperPath) {
console.warn("[guide-hotkey] native Ctrl listener is unavailable");
return;
}
const proc = spawn(helperPath, [], {
cwd: path.dirname(helperPath),
stdio: ["pipe", "pipe", "pipe"],
windowsHide: true,
});
proc.stdin.end();
guideHotkeyListenerProcess = proc;
let stdoutBuffer = "";
proc.stdout.setEncoding("utf-8");
proc.stdout.on("data", (chunk: string) => {
stdoutBuffer += chunk;
const lines = stdoutBuffer.split(/\r?\n/);
stdoutBuffer = lines.pop() ?? "";
for (const line of lines) {
handleGuideHotkeyListenerLine(line, guideStore);
}
});
proc.stderr.setEncoding("utf-8");
proc.stderr.on("data", (chunk: string) => {
const message = chunk.trim();
if (message) {
console.warn("[guide-hotkey] native listener:", message);
}
});
proc.once("error", (error) => {
console.warn("[guide-hotkey] failed to start native Ctrl listener:", error);
if (guideHotkeyListenerProcess === proc) {
guideHotkeyListenerProcess = null;
}
});
proc.once("exit", (code, signal) => {
if (guideHotkeyListenerProcess === proc) {
guideHotkeyListenerProcess = null;
}
if (code !== 0 && code !== null) {
console.warn("[guide-hotkey] native Ctrl listener exited", { code, signal });
}
});
}
function stopNativeGuideHotkeyListener() {
const proc = guideHotkeyListenerProcess;
guideHotkeyListenerProcess = null;
if (proc && !proc.killed) {
proc.kill();
}
}
function registerGuideMarkerHotkey(guideStore: GuideStore) { function registerGuideMarkerHotkey(guideStore: GuideStore) {
if (guideMarkerHotkeyRegistered) { if (guideMarkerHotkeyRegistered) {
return; return;
} }
void startNativeGuideHotkeyListener(guideStore);
guideMarkerHotkeyRegistered = globalShortcut.register(GUIDE_MARKER_HOTKEY, () => { guideMarkerHotkeyRegistered = globalShortcut.register(GUIDE_MARKER_HOTKEY, () => {
void captureGuideHotkeyMarker(guideStore); void captureGuideHotkeyMarker(guideStore, "global-shortcut");
}); });
if (!guideMarkerHotkeyRegistered) { if (!guideMarkerHotkeyRegistered) {
@@ -749,6 +974,7 @@ function registerGuideMarkerHotkey(guideStore: GuideStore) {
app.once("will-quit", () => { app.once("will-quit", () => {
globalShortcut.unregister(GUIDE_MARKER_HOTKEY); globalShortcut.unregister(GUIDE_MARKER_HOTKEY);
stopNativeGuideHotkeyListener();
guideMarkerHotkeyRegistered = false; guideMarkerHotkeyRegistered = false;
}); });
} }
@@ -758,12 +984,7 @@ function getSelectedSourceId() {
} }
function getSelectedDisplay() { function getSelectedDisplay() {
const sourceDisplayId = Number(selectedSource?.display_id); return getSelectedSourceDisplay();
if (!Number.isFinite(sourceDisplayId)) {
return null;
}
return screen.getAllDisplays().find((display) => display.id === sourceDisplayId) ?? null;
} }
function resolveUnpackedAppPath(...segments: string[]) { function resolveUnpackedAppPath(...segments: string[]) {
@@ -802,6 +1023,19 @@ function getNativeWindowsCaptureHelperCandidates() {
].filter((candidate): candidate is string => Boolean(candidate)); ].filter((candidate): candidate is string => Boolean(candidate));
} }
function getNativeGuideHotkeyListenerCandidates() {
const envPath = process.env.OPENSCREEN_GUIDE_HOTKEY_LISTENER_EXE?.trim();
const archTag = process.arch === "arm64" ? "win32-arm64" : "win32-x64";
const helperName = "guide-hotkey-listener.exe";
return [
envPath,
resolveUnpackedAppPath("electron", "native", "wgc-capture", "build", "Release", helperName),
resolveUnpackedAppPath("electron", "native", "wgc-capture", "build", helperName),
resolveUnpackedAppPath("electron", "native", "bin", archTag, helperName),
resolvePackagedResourcePath("electron", "native", "bin", archTag, helperName),
].filter((candidate): candidate is string => Boolean(candidate));
}
async function findNativeWindowsCaptureHelperPath() { async function findNativeWindowsCaptureHelperPath() {
if (process.platform !== "win32") { if (process.platform !== "win32") {
return null; return null;
@@ -819,6 +1053,23 @@ async function findNativeWindowsCaptureHelperPath() {
return null; return null;
} }
async function findNativeGuideHotkeyListenerPath() {
if (process.platform !== "win32") {
return null;
}
for (const candidate of getNativeGuideHotkeyListenerCandidates()) {
try {
await fs.access(candidate, fsConstants.X_OK);
return candidate;
} catch {
// Try the next configured helper location.
}
}
return null;
}
function getNativeMacCaptureHelperCandidates() { function getNativeMacCaptureHelperCandidates() {
const envPath = process.env.OPENSCREEN_SCK_CAPTURE_EXE?.trim(); const envPath = process.env.OPENSCREEN_SCK_CAPTURE_EXE?.trim();
const archTag = process.arch === "arm64" ? "darwin-arm64" : "darwin-x64"; const archTag = process.arch === "arm64" ? "darwin-arm64" : "darwin-x64";
@@ -1480,17 +1731,42 @@ export function registerIpcHandlers(
ipcMain.handle("get-sources", async (_, opts) => { ipcMain.handle("get-sources", async (_, opts) => {
const sources = await desktopCapturer.getSources(opts); const sources = await desktopCapturer.getSources(opts);
lastEnumeratedSources = new Map(sources.map((source) => [source.id, source])); lastEnumeratedSources = new Map(sources.map((source) => [source.id, source]));
return sources.map((source) => ({ let screenSourceIndex = 0;
id: source.id, return sources.map((source) => {
name: source.name, const isScreenSource = source.id.startsWith("screen:");
display_id: source.display_id, const sourceIndex = isScreenSource
thumbnail: source.thumbnail ? source.thumbnail.toDataURL() : null, ? (parseDesktopCapturerScreenIndex(source.id) ?? screenSourceIndex)
appIcon: source.appIcon ? source.appIcon.toDataURL() : null, : undefined;
})); const { display, displayIndex } = isScreenSource
? findDisplayForSource(source, screenSourceIndex)
: { display: null, displayIndex: undefined };
if (isScreenSource) {
screenSourceIndex += 1;
}
const bounds = display ? toSourceBounds(display.bounds) : undefined;
const displayLabel = bounds
? `Display ${(displayIndex ?? sourceIndex ?? 0) + 1} - ${bounds.width}x${bounds.height} @ ${bounds.x},${bounds.y}`
: undefined;
return {
id: source.id,
name: source.name,
display_id: source.display_id,
thumbnail: source.thumbnail ? source.thumbnail.toDataURL() : null,
appIcon: source.appIcon ? source.appIcon.toDataURL() : null,
displayId: display?.id,
displayIndex,
screenIndex: sourceIndex,
displayLabel,
bounds,
};
});
}); });
ipcMain.handle("select-source", async (_, source: SelectedSource) => { ipcMain.handle("select-source", async (_, source: SelectedSource) => {
selectedSource = source; selectedSource = {
...source,
bounds: normalizeSourceBounds(source.bounds),
};
// Reuse the exact source object returned during enumeration to avoid // Reuse the exact source object returned during enumeration to avoid
// Windows window-source id mismatches across separate getSources() calls. // Windows window-source id mismatches across separate getSources() calls.
selectedDesktopSource = selectedDesktopSource =
@@ -1713,16 +1989,19 @@ export function registerIpcHandlers(
RECORDINGS_DIR, RECORDINGS_DIR,
`${RECORDING_FILE_PREFIX}${recordingId}-webcam.mp4`, `${RECORDING_FILE_PREFIX}${recordingId}-webcam.mp4`,
); );
const requestBounds = normalizeSourceBounds(request.source.bounds);
const sourceDisplay = const sourceDisplay =
request.source.type === "display" && typeof request.source.displayId === "number" request.source.type === "display" && typeof request.source.displayId === "number"
? (screen.getAllDisplays().find((display) => display.id === request.source.displayId) ?? ? (screen.getAllDisplays().find((display) => display.id === request.source.displayId) ??
null) null)
: getSelectedDisplay(); : getSelectedDisplay();
const bounds = sourceDisplay?.bounds ?? getSelectedSourceBounds(); const bounds = requestBounds ?? sourceDisplay?.bounds ?? getSelectedSourceBounds();
const displayId = const displayId =
typeof request.source.displayId === "number" && Number.isFinite(request.source.displayId) typeof request.source.displayId === "number" && Number.isFinite(request.source.displayId)
? request.source.displayId ? request.source.displayId
: Number(selectedSource?.display_id); : typeof selectedSource?.displayId === "number"
? selectedSource.displayId
: Number(selectedSource?.display_id);
const webcamDirectShowClsid = request.webcam.enabled const webcamDirectShowClsid = request.webcam.enabled
? await resolveDirectShowWebcamClsid(request.webcam.deviceName) ? await resolveDirectShowWebcamClsid(request.webcam.deviceName)
: null; : null;
@@ -2365,7 +2644,7 @@ export function registerIpcHandlers(
onSessionEnded: (recordingId) => deactivateGuideHotkeySession(recordingId), onSessionEnded: (recordingId) => deactivateGuideHotkeySession(recordingId),
}); });
ipcMain.handle("guide:capture-pointer-marker", async () => { ipcMain.handle("guide:capture-pointer-marker", async () => {
const result = await captureGuideHotkeyMarker(guideStore); const result = await captureGuideHotkeyMarker(guideStore, "button");
if (result.error) { if (result.error) {
return { return {
success: false, success: false,
electron/native/README.md
+1 -1
View File
@@ -46,7 +46,7 @@ Build the Windows helper with:
npm run build:native:win npm run build:native:win
``` ```
The build writes the CMake output to `electron/native/wgc-capture/build/wgc-capture.exe` and copies the redistributable binary to `electron/native/bin/win32-x64/wgc-capture.exe`. The build writes the CMake output to `electron/native/wgc-capture/build/wgc-capture.exe` and copies the redistributable binary to `electron/native/bin/win32-x64/wgc-capture.exe`. It also builds `cursor-sampler.exe` for editable cursor telemetry and `guide-hotkey-listener.exe` for the Guide Mode global Ctrl capture hook.
The helper contract is process-based: the app starts the process with one JSON argument and sends commands on stdin. `stop\n` finalizes the recording. During migration the helper prints both newline-delimited JSON events and the legacy text messages `Recording started` / `Recording stopped. Output path: <path>`. The helper contract is process-based: the app starts the process with one JSON argument and sends commands on stdin. `stop\n` finalizes the recording. During migration the helper prints both newline-delimited JSON events and the legacy text messages `Recording started` / `Recording stopped. Output path: <path>`.
electron/native/wgc-capture/CMakeLists.txt
+16
View File
@@ -65,3 +65,19 @@ target_link_libraries(cursor-sampler PRIVATE
gdi32 gdi32
gdiplus gdiplus
) )
add_executable(guide-hotkey-listener
src/guide-hotkey-listener.cpp
)
target_compile_definitions(guide-hotkey-listener PRIVATE
NOMINMAX
WIN32_LEAN_AND_MEAN
_WIN32_WINNT=0x0A00
)
target_compile_options(guide-hotkey-listener PRIVATE /EHsc /W4 /utf-8)
target_link_libraries(guide-hotkey-listener PRIVATE
user32
)
electron/native/wgc-capture/src/guide-hotkey-listener.cpp
+91
View File
@@ -0,0 +1,91 @@
#include <windows.h>
#include <atomic>
#include <chrono>
#include <cstdint>
#include <iostream>
#include <mutex>
#include <string>
static HHOOK g_keyboardHook = nullptr;
static DWORD g_mainThreadId = 0;
static std::atomic<bool> g_ctrlDown{false};
static std::mutex g_stdoutMutex;
static int64_t nowMs() {
return static_cast<int64_t>(
std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::system_clock::now().time_since_epoch())
.count());
}
static void writeJsonLine(const std::string& json) {
std::lock_guard<std::mutex> lock(g_stdoutMutex);
std::cout << json << '\n';
std::cout.flush();
}
static bool isCtrlKey(DWORD vkCode) {
return vkCode == VK_CONTROL || vkCode == VK_LCONTROL || vkCode == VK_RCONTROL;
}
static LRESULT CALLBACK LowLevelKeyboardProc(int nCode, WPARAM wParam, LPARAM lParam) {
if (nCode >= 0) {
const auto* event = reinterpret_cast<KBDLLHOOKSTRUCT*>(lParam);
if (event && isCtrlKey(event->vkCode)) {
if (wParam == WM_KEYDOWN || wParam == WM_SYSKEYDOWN) {
const bool wasDown = g_ctrlDown.exchange(true, std::memory_order_acq_rel);
if (!wasDown) {
writeJsonLine(
"{\"event\":\"guide-hotkey\",\"key\":\"control\",\"state\":\"down\",\"timeMs\":" +
std::to_string(nowMs()) + "}");
}
} else if (wParam == WM_KEYUP || wParam == WM_SYSKEYUP) {
g_ctrlDown.store(false, std::memory_order_release);
}
}
}
return CallNextHookEx(g_keyboardHook, nCode, wParam, lParam);
}
static BOOL WINAPI consoleCtrlHandler(DWORD signal) {
if (
signal == CTRL_C_EVENT ||
signal == CTRL_BREAK_EVENT ||
signal == CTRL_CLOSE_EVENT ||
signal == CTRL_LOGOFF_EVENT ||
signal == CTRL_SHUTDOWN_EVENT
) {
PostThreadMessage(g_mainThreadId, WM_QUIT, 0, 0);
return TRUE;
}
return FALSE;
}
int main() {
g_mainThreadId = GetCurrentThreadId();
SetConsoleCtrlHandler(consoleCtrlHandler, TRUE);
g_keyboardHook = SetWindowsHookExW(WH_KEYBOARD_LL, LowLevelKeyboardProc, GetModuleHandleW(nullptr), 0);
if (!g_keyboardHook) {
std::cerr << "Failed to install guide hotkey keyboard hook. error=" << GetLastError() << std::endl;
return 1;
}
writeJsonLine("{\"event\":\"ready\"}");
MSG msg{};
while (GetMessageW(&msg, nullptr, 0, 0) > 0) {
TranslateMessage(&msg);
DispatchMessageW(&msg);
}
if (g_keyboardHook) {
UnhookWindowsHookEx(g_keyboardHook);
g_keyboardHook = nullptr;
}
return 0;
}
electron/native/wgc-capture/src/main.cpp
+1
View File
@@ -400,6 +400,7 @@ int main(int argc, char* argv[]) {
if (config.sourceType == "display") { if (config.sourceType == "display") {
HMONITOR monitor = findMonitorForCapture( HMONITOR monitor = findMonitorForCapture(
config.displayId, config.displayId,
config.sourceId,
config.hasDisplayBounds ? &config.bounds : nullptr); config.hasDisplayBounds ? &config.bounds : nullptr);
if (!monitor) { if (!monitor) {
std::cerr << "ERROR: Could not resolve monitor" << std::endl; std::cerr << "ERROR: Could not resolve monitor" << std::endl;
electron/native/wgc-capture/src/monitor_utils.cpp
+34 -1
View File
@@ -2,6 +2,7 @@
#include <algorithm> #include <algorithm>
#include <cmath> #include <cmath>
#include <string>
#include <vector> #include <vector>
namespace { namespace {
@@ -43,9 +44,36 @@ int64_t overlapArea(const RECT& rect, const MonitorBounds& bounds) {
return static_cast<int64_t>(right - left) * static_cast<int64_t>(bottom - top); return static_cast<int64_t>(right - left) * static_cast<int64_t>(bottom - top);
} }
int parseScreenSourceIndex(const std::string& sourceId) {
constexpr char prefix[] = "screen:";
if (sourceId.rfind(prefix, 0) != 0) {
return -1;
}
const size_t start = sizeof(prefix) - 1;
const size_t end = sourceId.find(':', start);
const std::string indexText = sourceId.substr(
start,
end == std::string::npos ? std::string::npos : end - start);
if (indexText.empty()) {
return -1;
}
try {
size_t parsed = 0;
const int index = std::stoi(indexText, &parsed, 10);
return parsed == indexText.size() && index >= 0 ? index : -1;
} catch (...) {
return -1;
}
}
} // namespace } // namespace
HMONITOR findMonitorForCapture(int64_t displayId, const MonitorBounds* bounds) { HMONITOR findMonitorForCapture(
int64_t displayId,
const std::string& sourceId,
const MonitorBounds* bounds) {
const auto monitors = enumerateMonitors(); const auto monitors = enumerateMonitors();
if (monitors.empty()) { if (monitors.empty()) {
return MonitorFromPoint({0, 0}, MONITOR_DEFAULTTOPRIMARY); return MonitorFromPoint({0, 0}, MONITOR_DEFAULTTOPRIMARY);
@@ -84,5 +112,10 @@ HMONITOR findMonitorForCapture(int64_t displayId, const MonitorBounds* bounds) {
} }
} }
const int sourceIndex = parseScreenSourceIndex(sourceId);
if (sourceIndex >= 0 && static_cast<size_t>(sourceIndex) < monitors.size()) {
return monitors[static_cast<size_t>(sourceIndex)].monitor;
}
return MonitorFromPoint({0, 0}, MONITOR_DEFAULTTOPRIMARY); return MonitorFromPoint({0, 0}, MONITOR_DEFAULTTOPRIMARY);
} }
electron/native/wgc-capture/src/monitor_utils.h
+5 -1
View File
@@ -3,6 +3,7 @@
#include <Windows.h> #include <Windows.h>
#include <cstdint> #include <cstdint>
#include <string>
struct MonitorBounds { struct MonitorBounds {
int x = 0; int x = 0;
@@ -11,4 +12,7 @@ struct MonitorBounds {
int height = 0; int height = 0;
}; };
HMONITOR findMonitorForCapture(int64_t displayId, const MonitorBounds* bounds); HMONITOR findMonitorForCapture(
int64_t displayId,
const std::string& sourceId,
const MonitorBounds* bounds);
electron/preload.ts
+8
View File
@@ -6,6 +6,7 @@ import type {
ExportGuideInput, ExportGuideInput,
FinalizeGuideEventsInput, FinalizeGuideEventsInput,
GenerateGuideDraftInput, GenerateGuideDraftInput,
GuideMarkerCapturedPayload,
RunGuideOcrInput, RunGuideOcrInput,
SaveGuideAiSettingsInput, SaveGuideAiSettingsInput,
SaveGuideInput, SaveGuideInput,
@@ -43,6 +44,13 @@ contextBridge.exposeInMainWorld("electronAPI", {
import("../src/guide/contracts").GuideIpcResult<CaptureGuidePointerMarkerResult> import("../src/guide/contracts").GuideIpcResult<CaptureGuidePointerMarkerResult>
>; >;
}, },
onMarkerCaptured: (callback: (payload: GuideMarkerCapturedPayload) => void) => {
const listener = (_event: Electron.IpcRendererEvent, payload: GuideMarkerCapturedPayload) => {
callback(payload);
};
ipcRenderer.on("guide:marker-captured", listener);
return () => ipcRenderer.removeListener("guide:marker-captured", listener);
},
finalizeEvents: (input: FinalizeGuideEventsInput) => { finalizeEvents: (input: FinalizeGuideEventsInput) => {
return ipcRenderer.invoke("guide:finalize-events", input); return ipcRenderer.invoke("guide:finalize-events", input);
}, },
package-lock.json
Generated
+2 -2
View File
@@ -1,12 +1,12 @@
{ {
"name": "openscreen", "name": "openscreen",
"version": "1.4.1", "version": "1.4.2",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "openscreen", "name": "openscreen",
"version": "1.4.1", "version": "1.4.2",
"dependencies": { "dependencies": {
"@fix-webm-duration/fix": "^1.0.1", "@fix-webm-duration/fix": "^1.0.1",
"@pixi/filter-drop-shadow": "^5.2.0", "@pixi/filter-drop-shadow": "^5.2.0",
package.json
+1 -1
View File
@@ -1,7 +1,7 @@
{ {
"name": "openscreen", "name": "openscreen",
"private": true, "private": true,
"version": "1.4.1", "version": "1.4.2",
"type": "module", "type": "module",
"packageManager": "npm@10.9.4", "packageManager": "npm@10.9.4",
"engines": { "engines": {
scripts/build-windows-wgc-helper.mjs
+10
View File
@@ -126,6 +126,11 @@ if (!fs.existsSync(cursorSamplerOutputPath)) {
throw new Error(`WGC helper build completed but ${cursorSamplerOutputPath} was not found.`); throw new Error(`WGC helper build completed but ${cursorSamplerOutputPath} was not found.`);
} }
const guideHotkeyListenerOutputPath = path.join(BUILD_DIR, "guide-hotkey-listener.exe");
if (!fs.existsSync(guideHotkeyListenerOutputPath)) {
throw new Error(`WGC helper build completed but ${guideHotkeyListenerOutputPath} was not found.`);
}
fs.mkdirSync(BIN_DIR, { recursive: true }); fs.mkdirSync(BIN_DIR, { recursive: true });
const distributablePath = path.join(BIN_DIR, "wgc-capture.exe"); const distributablePath = path.join(BIN_DIR, "wgc-capture.exe");
fs.copyFileSync(outputPath, distributablePath); fs.copyFileSync(outputPath, distributablePath);
@@ -133,7 +138,12 @@ fs.copyFileSync(outputPath, distributablePath);
const cursorSamplerDistributablePath = path.join(BIN_DIR, "cursor-sampler.exe"); const cursorSamplerDistributablePath = path.join(BIN_DIR, "cursor-sampler.exe");
fs.copyFileSync(cursorSamplerOutputPath, cursorSamplerDistributablePath); fs.copyFileSync(cursorSamplerOutputPath, cursorSamplerDistributablePath);
const guideHotkeyListenerDistributablePath = path.join(BIN_DIR, "guide-hotkey-listener.exe");
fs.copyFileSync(guideHotkeyListenerOutputPath, guideHotkeyListenerDistributablePath);
console.log(`Built ${outputPath}`); console.log(`Built ${outputPath}`);
console.log(`Copied ${distributablePath}`); console.log(`Copied ${distributablePath}`);
console.log(`Built ${cursorSamplerOutputPath}`); console.log(`Built ${cursorSamplerOutputPath}`);
console.log(`Copied ${cursorSamplerDistributablePath}`); console.log(`Copied ${cursorSamplerDistributablePath}`);
console.log(`Built ${guideHotkeyListenerOutputPath}`);
console.log(`Copied ${guideHotkeyListenerDistributablePath}`);
src/components/launch/LaunchWindow.tsx
+18 -42
View File
@@ -19,6 +19,7 @@ import {
MdVolumeUp, MdVolumeUp,
} from "react-icons/md"; } from "react-icons/md";
import { RxDragHandleDots2 } from "react-icons/rx"; import { RxDragHandleDots2 } from "react-icons/rx";
import { toast } from "sonner";
import { useI18n, useScopedT } from "@/contexts/I18nContext"; import { useI18n, useScopedT } from "@/contexts/I18nContext";
import { getAvailableLocales, getLocaleName } from "@/i18n/loader"; import { getAvailableLocales, getLocaleName } from "@/i18n/loader";
import { nativeBridgeClient } from "@/native"; import { nativeBridgeClient } from "@/native";
@@ -143,7 +144,6 @@ export function LaunchWindow() {
top: 12, top: 12,
maxHeight: 240, maxHeight: 240,
}); });
const guideCtrlMarkerArmedRef = useRef(false);
const { const {
devices: micDevices, devices: micDevices,
@@ -248,47 +248,6 @@ export function LaunchWindow() {
}; };
}, [isLanguageMenuOpen]); }, [isLanguageMenuOpen]);
useEffect(() => {
if (!recording || !guideModeEnabled) {
guideCtrlMarkerArmedRef.current = false;
return;
}
const isCtrlKey = (event: KeyboardEvent) =>
event.key === "Control" || event.code === "ControlLeft" || event.code === "ControlRight";
const handleKeyDown = (event: KeyboardEvent) => {
if (!isCtrlKey(event) || event.repeat || guideCtrlMarkerArmedRef.current) {
return;
}
guideCtrlMarkerArmedRef.current = true;
event.preventDefault();
event.stopPropagation();
addGuideMarker();
};
const releaseCtrlMarker = (event?: KeyboardEvent) => {
if (event && !isCtrlKey(event)) {
return;
}
guideCtrlMarkerArmedRef.current = false;
};
const handleWindowBlur = () => {
guideCtrlMarkerArmedRef.current = false;
};
window.addEventListener("keydown", handleKeyDown, { capture: true });
window.addEventListener("keyup", releaseCtrlMarker, { capture: true });
window.addEventListener("blur", handleWindowBlur);
return () => {
window.removeEventListener("keydown", handleKeyDown, { capture: true });
window.removeEventListener("keyup", releaseCtrlMarker, { capture: true });
window.removeEventListener("blur", handleWindowBlur);
};
}, [addGuideMarker, guideModeEnabled, recording]);
useEffect(() => { useEffect(() => {
if (!isLanguageMenuOpen || !languageTriggerRef.current) return; if (!isLanguageMenuOpen || !languageTriggerRef.current) return;
@@ -347,6 +306,23 @@ export function LaunchWindow() {
setHudMouseEventsEnabled(isLanguageMenuOpen); setHudMouseEventsEnabled(isLanguageMenuOpen);
}, [isLanguageMenuOpen, setHudMouseEventsEnabled]); }, [isLanguageMenuOpen, setHudMouseEventsEnabled]);
useEffect(() => {
const unsubscribe = window.electronAPI?.guide.onMarkerCaptured?.((payload) => {
const position =
typeof payload.normalizedX === "number" && typeof payload.normalizedY === "number"
? `x ${Math.round(payload.normalizedX * 100)}%, y ${Math.round(payload.normalizedY * 100)}%`
: undefined;
toast.success("Guide event captured", {
id: `guide-marker-${payload.eventId}`,
description: position,
duration: 1400,
});
});
return () => {
unsubscribe?.();
};
}, []);
const [selectedSource, setSelectedSource] = useState("Screen"); const [selectedSource, setSelectedSource] = useState("Screen");
const [hasSelectedSource, setHasSelectedSource] = useState(false); const [hasSelectedSource, setHasSelectedSource] = useState(false);
const [, setRecordPointerDownCount] = useState(0); const [, setRecordPointerDownCount] = useState(0);
src/components/launch/SourceSelector.tsx
+23 -1
View File
@@ -11,6 +11,16 @@ interface DesktopSource {
thumbnail: string | null; thumbnail: string | null;
display_id: string; display_id: string;
appIcon: string | null; appIcon: string | null;
displayId?: number;
displayIndex?: number;
screenIndex?: number;
displayLabel?: string;
bounds?: {
x: number;
y: number;
width: number;
height: number;
};
} }
export function SourceSelector() { export function SourceSelector() {
@@ -39,6 +49,11 @@ export function SourceSelector() {
thumbnail: source.thumbnail, thumbnail: source.thumbnail,
display_id: source.display_id, display_id: source.display_id,
appIcon: source.appIcon, appIcon: source.appIcon,
displayId: source.displayId,
displayIndex: source.displayIndex,
screenIndex: source.screenIndex,
displayLabel: source.displayLabel,
bounds: source.bounds,
})), })),
); );
} catch (error) { } catch (error) {
@@ -98,7 +113,14 @@ export function SourceSelector() {
{source.appIcon && ( {source.appIcon && (
<img src={source.appIcon} alt="" className={`${styles.icon} flex-shrink-0`} /> <img src={source.appIcon} alt="" className={`${styles.icon} flex-shrink-0`} />
)} )}
<div className={`${styles.name} truncate`}>{source.name}</div> <div className="min-w-0">
<div className={`${styles.name} truncate`}>{source.name}</div>
{source.displayLabel && (
<div className="truncate text-[9px] leading-3 text-zinc-500">
{source.displayLabel}
</div>
)}
</div>
</div> </div>
</div> </div>
); );
src/guide/contracts.ts
+18
View File
@@ -79,10 +79,28 @@ export interface GuideStepCandidate {
action: GuideAction; action: GuideAction;
targetText?: string; targetText?: string;
targetRole?: GuideTargetRole; targetRole?: GuideTargetRole;
position?: {
normalizedX: number;
normalizedY: number;
xPercent: number;
yPercent: number;
description: string;
};
nearbyText: string[]; nearbyText: string[];
confidence: number; confidence: number;
} }
export interface GuideMarkerCapturedPayload {
recordingId: string;
eventId: string;
timeMs: number;
trigger: "button" | "global-control" | "global-shortcut";
normalizedX?: number;
normalizedY?: number;
rawX?: number;
rawY?: number;
}
export interface GeneratedGuideStep { export interface GeneratedGuideStep {
id: string; id: string;
order: number; order: number;
src/guide/exporters.test.ts
+18
View File
@@ -83,4 +83,22 @@ describe("guide exporters", () => {
expect(html).toContain("click-marker"); expect(html).toContain("click-marker");
expect(html).toContain("left: 25.00%; top: 75.00%;"); expect(html).toContain("left: 25.00%; top: 75.00%;");
}); });
it("draws click markers for hotkey events with coordinates", () => {
const hotkeySession: GuideSession = {
...session,
events: [
{
...session.events[0],
kind: "hotkey",
source: "guide-hotkey",
},
],
};
const html = exportGuideToHtml(hotkeySession);
expect(html).toContain("click-marker");
expect(html).toContain("left: 25.00%; top: 75.00%;");
});
}); });
src/guide/exporters.ts
+1 -1
View File
@@ -97,7 +97,7 @@ function resolveStepClickPoint(
: undefined; : undefined;
const eventId = candidate?.eventId; const eventId = candidate?.eventId;
const event = eventId ? session.events.find((item) => item.id === eventId) : undefined; const event = eventId ? session.events.find((item) => item.id === eventId) : undefined;
if (!event || event.kind !== "click") { if (!event || (event.kind !== "click" && event.kind !== "hotkey")) {
return null; return null;
} }
if (isNormalizedNumber(event.normalizedX) && isNormalizedNumber(event.normalizedY)) { if (isNormalizedNumber(event.normalizedX) && isNormalizedNumber(event.normalizedY)) {
src/guide/promptBuilder.test.ts
+9
View File
@@ -36,6 +36,13 @@ const candidates: GuideStepCandidate[] = [
action: "click", action: "click",
targetText: "Save", targetText: "Save",
targetRole: "button", targetRole: "button",
position: {
normalizedX: 0.5,
normalizedY: 0.5,
xPercent: 50,
yPercent: 50,
description: "center",
},
nearbyText: ["Save"], nearbyText: ["Save"],
confidence: 0.9, confidence: 0.9,
}, },
@@ -46,7 +53,9 @@ describe("guide draft helpers", () => {
const prompt = buildGuideDraftPrompt({ session, candidates, language: "en" }); const prompt = buildGuideDraftPrompt({ session, candidates, language: "en" });
expect(prompt).toContain("Return JSON only"); expect(prompt).toContain("Return JSON only");
expect(prompt).toContain('"sourceCandidateId": "candidate-1"');
expect(prompt).toContain('"targetText": "Save"'); expect(prompt).toContain('"targetText": "Save"');
expect(prompt).toContain('"xPercent": 50');
expect(prompt).toContain('"id":"guide-step-1"'); expect(prompt).toContain('"id":"guide-step-1"');
}); });
src/guide/promptBuilder.ts
+11 -1
View File
@@ -17,10 +17,12 @@ export function buildGuideDraftPrompt(input: GuidePromptInput): string {
const candidatesJson = JSON.stringify( const candidatesJson = JSON.stringify(
input.candidates.map((candidate, index) => ({ input.candidates.map((candidate, index) => ({
order: index + 1, order: index + 1,
sourceCandidateId: candidate.id,
timeMs: Math.round(candidate.timeMs), timeMs: Math.round(candidate.timeMs),
action: candidate.action, action: candidate.action,
targetText: candidate.targetText, targetText: candidate.targetText,
targetRole: candidate.targetRole, targetRole: candidate.targetRole,
position: candidate.position,
nearbyText: candidate.nearbyText, nearbyText: candidate.nearbyText,
confidence: candidate.confidence, confidence: candidate.confidence,
})), })),
@@ -36,8 +38,10 @@ export function buildGuideDraftPrompt(input: GuidePromptInput): string {
"Rules:", "Rules:",
"- Use short, explicit step instructions.", "- Use short, explicit step instructions.",
"- Prefer visible target text from OCR when it is available.", "- Prefer visible target text from OCR when it is available.",
"- Return sourceCandidateId exactly from the chosen candidate.",
"- Never use generic marker text such as Ctrl+F12 marker or Ctrl marker as a UI target.",
"- Do not invent buttons or screens that are not in the candidates.", "- Do not invent buttons or screens that are not in the candidates.",
"- If a target is unclear, describe the action by screen position or timestamp.", "- If a target is unclear, describe the action by the candidate position and include the x/y percentages.",
"", "",
"Candidates:", "Candidates:",
candidatesJson, candidatesJson,
@@ -92,12 +96,18 @@ function buildInstruction(candidate: GuideStepCandidate, language: GuideLanguage
if (target) { if (target) {
return `${candidate.action === "click" ? "Nhấn" : "Thực hiện thao tác"} vào "${target}".`; return `${candidate.action === "click" ? "Nhấn" : "Thực hiện thao tác"} vào "${target}".`;
} }
if (candidate.position) {
return `Nhấn tại vùng ${candidate.position.description} (x ${candidate.position.xPercent}%, y ${candidate.position.yPercent}%).`;
}
return `Thực hiện thao tác tại mốc ${formatTimestamp(candidate.timeMs)}.`; return `Thực hiện thao tác tại mốc ${formatTimestamp(candidate.timeMs)}.`;
} }
if (target) { if (target) {
return `${candidate.action === "click" ? "Click" : "Use"} "${target}".`; return `${candidate.action === "click" ? "Click" : "Use"} "${target}".`;
} }
if (candidate.position) {
return `Click the ${candidate.position.description} area (x ${candidate.position.xPercent}%, y ${candidate.position.yPercent}%).`;
}
return `Perform the action at ${formatTimestamp(candidate.timeMs)}.`; return `Perform the action at ${formatTimestamp(candidate.timeMs)}.`;
} }
src/guide/targetMapper.test.ts
+7
View File
@@ -98,6 +98,7 @@ describe("buildGuideStepCandidates", () => {
source: "guide-hotkey", source: "guide-hotkey",
normalizedX: 0.5, normalizedX: 0.5,
normalizedY: 0.5, normalizedY: 0.5,
label: "Ctrl+F12 marker",
}; };
const candidates = buildGuideStepCandidates(session); const candidates = buildGuideStepCandidates(session);
@@ -106,6 +107,12 @@ describe("buildGuideStepCandidates", () => {
action: "click", action: "click",
targetText: "Save", targetText: "Save",
targetRole: "button", targetRole: "button",
position: {
normalizedX: 0.5,
normalizedY: 0.5,
xPercent: 50,
yPercent: 50,
},
}); });
}); });
src/guide/targetMapper.ts
+43 -3
View File
@@ -46,8 +46,11 @@ export function buildGuideStepCandidates(
0, 0,
maxNearbyText, maxNearbyText,
); );
const label = normalizeText(event.label); const label = normalizeEventLabelForTarget(event);
const targetText = label ?? normalizeText(targetRegion?.text); const point = getEventPoint(event);
const targetText = point
? (normalizeText(targetRegion?.text) ?? label)
: (label ?? normalizeText(targetRegion?.text));
return { return {
id: `candidate-${event.id}`, id: `candidate-${event.id}`,
@@ -57,6 +60,7 @@ export function buildGuideStepCandidates(
action: inferAction(event), action: inferAction(event),
targetText, targetText,
targetRole: inferTargetRole(targetText), targetRole: inferTargetRole(targetText),
position: point ? describeEventPosition(point) : undefined,
nearbyText, nearbyText,
confidence: calculateCandidateConfidence(event, targetRegion, rankedRegions[0]?.score), confidence: calculateCandidateConfidence(event, targetRegion, rankedRegions[0]?.score),
}; };
@@ -275,7 +279,7 @@ function calculateCandidateConfidence(
0.45 + clamp01(targetRegion.confidence) * 0.25 + clamp01(score ?? 0) * 0.3, 0.45 + clamp01(targetRegion.confidence) * 0.25 + clamp01(score ?? 0) * 0.3,
); );
} }
if (event.label) { if (normalizeEventLabelForTarget(event)) {
return 0.75; return 0.75;
} }
if (getEventPoint(event)) { if (getEventPoint(event)) {
@@ -307,6 +311,38 @@ function normalizeText(value: string | undefined): string | undefined {
return text ? text : undefined; return text ? text : undefined;
} }
function normalizeEventLabelForTarget(event: GuideEvent): string | undefined {
const label = normalizeText(event.label);
if (!label) {
return undefined;
}
if (/^(?:ctrl(?:\s*\+\s*f12)?|control)\s+marker$/i.test(label)) {
return undefined;
}
if (/^manual\s+marker$/i.test(label)) {
return undefined;
}
return label;
}
function describeEventPosition(point: { x: number; y: number }): GuideStepCandidate["position"] {
const normalizedX = clamp01(point.x);
const normalizedY = clamp01(point.y);
return {
normalizedX: roundPosition(normalizedX),
normalizedY: roundPosition(normalizedY),
xPercent: Math.round(normalizedX * 100),
yPercent: Math.round(normalizedY * 100),
description: describeScreenRegion(normalizedX, normalizedY),
};
}
function describeScreenRegion(x: number, y: number): string {
const vertical = y < 0.33 ? "top" : y > 0.66 ? "bottom" : "middle";
const horizontal = x < 0.33 ? "left" : x > 0.66 ? "right" : "center";
return vertical === "middle" && horizontal === "center" ? "center" : `${vertical} ${horizontal}`;
}
function isUsefulOcrText(text: string): boolean { function isUsefulOcrText(text: string): boolean {
if (!/[A-Za-z0-9À-ỹ]/.test(text)) { if (!/[A-Za-z0-9À-ỹ]/.test(text)) {
return false; return false;
@@ -346,6 +382,10 @@ function roundConfidence(value: number): number {
return Math.round(clamp01(value) * 100) / 100; return Math.round(clamp01(value) * 100) / 100;
} }
function roundPosition(value: number): number {
return Math.round(clamp01(value) * 1000) / 1000;
}
function clamp01(value: number): number { function clamp01(value: number): number {
if (!Number.isFinite(value)) { if (!Number.isFinite(value)) {
return 0; return 0;
src/hooks/useScreenRecorder.ts
+9 -2
View File
@@ -921,7 +921,10 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
} }
const activeRecordingId = Date.now(); const activeRecordingId = Date.now();
const displayId = Number(selectedSource.display_id); const displayId =
typeof selectedSource.displayId === "number"
? selectedSource.displayId
: Number(selectedSource.display_id);
const sourceType = selectedSource.id.startsWith("window:") ? "window" : "display"; const sourceType = selectedSource.id.startsWith("window:") ? "window" : "display";
const windowHandle = parseWindowHandleFromSourceId(selectedSource.id); const windowHandle = parseWindowHandleFromSourceId(selectedSource.id);
if (webcamEnabled) { if (webcamEnabled) {
@@ -946,6 +949,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
type: sourceType, type: sourceType,
sourceId: selectedSource.id, sourceId: selectedSource.id,
...(Number.isFinite(displayId) ? { displayId } : {}), ...(Number.isFinite(displayId) ? { displayId } : {}),
...(selectedSource.bounds ? { bounds: selectedSource.bounds } : {}),
...(windowHandle ? { windowHandle } : {}), ...(windowHandle ? { windowHandle } : {}),
}, },
video: { video: {
@@ -1039,7 +1043,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
const activeRecordingId = Date.now(); const activeRecordingId = Date.now();
const sourceType = selectedSource.id.startsWith("window:") ? "window" : "display"; const sourceType = selectedSource.id.startsWith("window:") ? "window" : "display";
const displayId = const displayId =
Number(selectedSource.display_id) || parseMacDisplayIdFromSourceId(selectedSource.id); typeof selectedSource.displayId === "number"
? selectedSource.displayId
: Number(selectedSource.display_id) || parseMacDisplayIdFromSourceId(selectedSource.id);
const windowId = parseMacWindowIdFromSourceId(selectedSource.id); const windowId = parseMacWindowIdFromSourceId(selectedSource.id);
let nativeWebcamRecorder: RecorderHandle | null = null; let nativeWebcamRecorder: RecorderHandle | null = null;
if (webcamEnabled) { if (webcamEnabled) {
@@ -1083,6 +1089,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
type: sourceType, type: sourceType,
sourceId: selectedSource.id, sourceId: selectedSource.id,
...(displayId ? { displayId } : {}), ...(displayId ? { displayId } : {}),
...(selectedSource.bounds ? { bounds: selectedSource.bounds } : {}),
...(windowId ? { windowId } : {}), ...(windowId ? { windowId } : {}),
}, },
video: { video: {
src/lib/nativeWindowsRecording.ts
+6
View File
@@ -6,6 +6,12 @@ export type NativeWindowsRecordingRequest = {
type: NativeWindowsSourceType; type: NativeWindowsSourceType;
sourceId: string; sourceId: string;
displayId?: number; displayId?: number;
bounds?: {
x: number;
y: number;
width: number;
height: number;
};
windowHandle?: string; windowHandle?: string;
}; };
video: { video: {