huanld/openscreen
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Release OpenScreen 1.4.2

Browse Source
This commit is contained in:
huanld
2026-05-28 10:01:22 +07:00
parent 69804c41c7
commit 198dc022b0
25 changed files with 844 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/components/launch/LaunchWindow.tsx
+18 -42
View File
@@ -19,6 +19,7 @@ import {
MdVolumeUp,
} from "react-icons/md";
import { RxDragHandleDots2 } from "react-icons/rx";
import { toast } from "sonner";
import { useI18n, useScopedT } from "@/contexts/I18nContext";
import { getAvailableLocales, getLocaleName } from "@/i18n/loader";
import { nativeBridgeClient } from "@/native";
@@ -143,7 +144,6 @@ export function LaunchWindow() {
top: 12,
maxHeight: 240,
});
const guideCtrlMarkerArmedRef = useRef(false);
const {
devices: micDevices,
@@ -248,47 +248,6 @@ export function LaunchWindow() {
};
}, [isLanguageMenuOpen]);
useEffect(() => {
if (!recording || !guideModeEnabled) {
guideCtrlMarkerArmedRef.current = false;
return;
}
const isCtrlKey = (event: KeyboardEvent) =>
event.key === "Control" || event.code === "ControlLeft" || event.code === "ControlRight";
const handleKeyDown = (event: KeyboardEvent) => {
if (!isCtrlKey(event) || event.repeat || guideCtrlMarkerArmedRef.current) {
return;
}
guideCtrlMarkerArmedRef.current = true;
event.preventDefault();
event.stopPropagation();
addGuideMarker();
};
const releaseCtrlMarker = (event?: KeyboardEvent) => {
if (event && !isCtrlKey(event)) {
return;
}
guideCtrlMarkerArmedRef.current = false;
};
const handleWindowBlur = () => {
guideCtrlMarkerArmedRef.current = false;
};
window.addEventListener("keydown", handleKeyDown, { capture: true });
window.addEventListener("keyup", releaseCtrlMarker, { capture: true });
window.addEventListener("blur", handleWindowBlur);
return () => {
window.removeEventListener("keydown", handleKeyDown, { capture: true });
window.removeEventListener("keyup", releaseCtrlMarker, { capture: true });
window.removeEventListener("blur", handleWindowBlur);
};
}, [addGuideMarker, guideModeEnabled, recording]);
useEffect(() => {
if (!isLanguageMenuOpen || !languageTriggerRef.current) return;
@@ -347,6 +306,23 @@ export function LaunchWindow() {
setHudMouseEventsEnabled(isLanguageMenuOpen);
}, [isLanguageMenuOpen, setHudMouseEventsEnabled]);
useEffect(() => {
const unsubscribe = window.electronAPI?.guide.onMarkerCaptured?.((payload) => {
const position =
typeof payload.normalizedX === "number" && typeof payload.normalizedY === "number"
? `x ${Math.round(payload.normalizedX * 100)}%, y ${Math.round(payload.normalizedY * 100)}%`
: undefined;
toast.success("Guide event captured", {
id: `guide-marker-${payload.eventId}`,
description: position,
duration: 1400,
});
});
return () => {
unsubscribe?.();
};
}, []);
const [selectedSource, setSelectedSource] = useState("Screen");
const [hasSelectedSource, setHasSelectedSource] = useState(false);
const [, setRecordPointerDownCount] = useState(0);
src/components/launch/SourceSelector.tsx
+23 -1
View File
@@ -11,6 +11,16 @@ interface DesktopSource {
thumbnail: string | null;
display_id: string;
appIcon: string | null;
displayId?: number;
displayIndex?: number;
screenIndex?: number;
displayLabel?: string;
bounds?: {
x: number;
y: number;
width: number;
height: number;
};
}
export function SourceSelector() {
@@ -39,6 +49,11 @@ export function SourceSelector() {
thumbnail: source.thumbnail,
display_id: source.display_id,
appIcon: source.appIcon,
displayId: source.displayId,
displayIndex: source.displayIndex,
screenIndex: source.screenIndex,
displayLabel: source.displayLabel,
bounds: source.bounds,
})),
);
} catch (error) {
@@ -98,7 +113,14 @@ export function SourceSelector() {
{source.appIcon && (
<img src={source.appIcon} alt="" className={`${styles.icon} flex-shrink-0`} />
)}
<div className={`${styles.name} truncate`}>{source.name}</div>
<div className="min-w-0">
<div className={`${styles.name} truncate`}>{source.name}</div>
{source.displayLabel && (
<div className="truncate text-[9px] leading-3 text-zinc-500">
{source.displayLabel}
</div>
)}
</div>
</div>
</div>
);
src/guide/contracts.ts
+18
View File
@@ -79,10 +79,28 @@ export interface GuideStepCandidate {
action: GuideAction;
targetText?: string;
targetRole?: GuideTargetRole;
position?: {
normalizedX: number;
normalizedY: number;
xPercent: number;
yPercent: number;
description: string;
};
nearbyText: string[];
confidence: number;
}
export interface GuideMarkerCapturedPayload {
recordingId: string;
eventId: string;
timeMs: number;
trigger: "button" | "global-control" | "global-shortcut";
normalizedX?: number;
normalizedY?: number;
rawX?: number;
rawY?: number;
}
export interface GeneratedGuideStep {
id: string;
order: number;
src/guide/exporters.test.ts
+18
View File
@@ -83,4 +83,22 @@ describe("guide exporters", () => {
expect(html).toContain("click-marker");
expect(html).toContain("left: 25.00%; top: 75.00%;");
});
it("draws click markers for hotkey events with coordinates", () => {
const hotkeySession: GuideSession = {
...session,
events: [
{
...session.events[0],
kind: "hotkey",
source: "guide-hotkey",
},
],
};
const html = exportGuideToHtml(hotkeySession);
expect(html).toContain("click-marker");
expect(html).toContain("left: 25.00%; top: 75.00%;");
});
});
src/guide/exporters.ts
+1 -1
View File
@@ -97,7 +97,7 @@ function resolveStepClickPoint(
: undefined;
const eventId = candidate?.eventId;
const event = eventId ? session.events.find((item) => item.id === eventId) : undefined;
if (!event || event.kind !== "click") {
if (!event || (event.kind !== "click" && event.kind !== "hotkey")) {
return null;
}
if (isNormalizedNumber(event.normalizedX) && isNormalizedNumber(event.normalizedY)) {
src/guide/promptBuilder.test.ts
+9
View File
@@ -36,6 +36,13 @@ const candidates: GuideStepCandidate[] = [
action: "click",
targetText: "Save",
targetRole: "button",
position: {
normalizedX: 0.5,
normalizedY: 0.5,
xPercent: 50,
yPercent: 50,
description: "center",
},
nearbyText: ["Save"],
confidence: 0.9,
},
@@ -46,7 +53,9 @@ describe("guide draft helpers", () => {
const prompt = buildGuideDraftPrompt({ session, candidates, language: "en" });
expect(prompt).toContain("Return JSON only");
expect(prompt).toContain('"sourceCandidateId": "candidate-1"');
expect(prompt).toContain('"targetText": "Save"');
expect(prompt).toContain('"xPercent": 50');
expect(prompt).toContain('"id":"guide-step-1"');
});
src/guide/promptBuilder.ts
+11 -1
View File
@@ -17,10 +17,12 @@ export function buildGuideDraftPrompt(input: GuidePromptInput): string {
const candidatesJson = JSON.stringify(
input.candidates.map((candidate, index) => ({
order: index + 1,
sourceCandidateId: candidate.id,
timeMs: Math.round(candidate.timeMs),
action: candidate.action,
targetText: candidate.targetText,
targetRole: candidate.targetRole,
position: candidate.position,
nearbyText: candidate.nearbyText,
confidence: candidate.confidence,
})),
@@ -36,8 +38,10 @@ export function buildGuideDraftPrompt(input: GuidePromptInput): string {
"Rules:",
"- Use short, explicit step instructions.",
"- Prefer visible target text from OCR when it is available.",
"- Return sourceCandidateId exactly from the chosen candidate.",
"- Never use generic marker text such as Ctrl+F12 marker or Ctrl marker as a UI target.",
"- Do not invent buttons or screens that are not in the candidates.",
"- If a target is unclear, describe the action by screen position or timestamp.",
"- If a target is unclear, describe the action by the candidate position and include the x/y percentages.",
"",
"Candidates:",
candidatesJson,
@@ -92,12 +96,18 @@ function buildInstruction(candidate: GuideStepCandidate, language: GuideLanguage
if (target) {
return `${candidate.action === "click" ? "Nhấn" : "Thực hiện thao tác"} vào "${target}".`;
}
if (candidate.position) {
return `Nhấn tại vùng ${candidate.position.description} (x ${candidate.position.xPercent}%, y ${candidate.position.yPercent}%).`;
}
return `Thực hiện thao tác tại mốc ${formatTimestamp(candidate.timeMs)}.`;
}
if (target) {
return `${candidate.action === "click" ? "Click" : "Use"} "${target}".`;
}
if (candidate.position) {
return `Click the ${candidate.position.description} area (x ${candidate.position.xPercent}%, y ${candidate.position.yPercent}%).`;
}
return `Perform the action at ${formatTimestamp(candidate.timeMs)}.`;
}
src/guide/targetMapper.test.ts
+7
View File
@@ -98,6 +98,7 @@ describe("buildGuideStepCandidates", () => {
source: "guide-hotkey",
normalizedX: 0.5,
normalizedY: 0.5,
label: "Ctrl+F12 marker",
};
const candidates = buildGuideStepCandidates(session);
@@ -106,6 +107,12 @@ describe("buildGuideStepCandidates", () => {
action: "click",
targetText: "Save",
targetRole: "button",
position: {
normalizedX: 0.5,
normalizedY: 0.5,
xPercent: 50,
yPercent: 50,
},
});
});
src/guide/targetMapper.ts
+43 -3
View File
@@ -46,8 +46,11 @@ export function buildGuideStepCandidates(
0,
maxNearbyText,
);
const label = normalizeText(event.label);
const targetText = label ?? normalizeText(targetRegion?.text);
const label = normalizeEventLabelForTarget(event);
const point = getEventPoint(event);
const targetText = point
? (normalizeText(targetRegion?.text) ?? label)
: (label ?? normalizeText(targetRegion?.text));
return {
id: `candidate-${event.id}`,
@@ -57,6 +60,7 @@ export function buildGuideStepCandidates(
action: inferAction(event),
targetText,
targetRole: inferTargetRole(targetText),
position: point ? describeEventPosition(point) : undefined,
nearbyText,
confidence: calculateCandidateConfidence(event, targetRegion, rankedRegions[0]?.score),
};
@@ -275,7 +279,7 @@ function calculateCandidateConfidence(
0.45 + clamp01(targetRegion.confidence) * 0.25 + clamp01(score ?? 0) * 0.3,
);
}
if (event.label) {
if (normalizeEventLabelForTarget(event)) {
return 0.75;
}
if (getEventPoint(event)) {
@@ -307,6 +311,38 @@ function normalizeText(value: string | undefined): string | undefined {
return text ? text : undefined;
}
function normalizeEventLabelForTarget(event: GuideEvent): string | undefined {
const label = normalizeText(event.label);
if (!label) {
return undefined;
}
if (/^(?:ctrl(?:\s*\+\s*f12)?|control)\s+marker$/i.test(label)) {
return undefined;
}
if (/^manual\s+marker$/i.test(label)) {
return undefined;
}
return label;
}
function describeEventPosition(point: { x: number; y: number }): GuideStepCandidate["position"] {
const normalizedX = clamp01(point.x);
const normalizedY = clamp01(point.y);
return {
normalizedX: roundPosition(normalizedX),
normalizedY: roundPosition(normalizedY),
xPercent: Math.round(normalizedX * 100),
yPercent: Math.round(normalizedY * 100),
description: describeScreenRegion(normalizedX, normalizedY),
};
}
function describeScreenRegion(x: number, y: number): string {
const vertical = y < 0.33 ? "top" : y > 0.66 ? "bottom" : "middle";
const horizontal = x < 0.33 ? "left" : x > 0.66 ? "right" : "center";
return vertical === "middle" && horizontal === "center" ? "center" : `${vertical} ${horizontal}`;
}
function isUsefulOcrText(text: string): boolean {
if (!/[A-Za-z0-9À-ỹ]/.test(text)) {
return false;
@@ -346,6 +382,10 @@ function roundConfidence(value: number): number {
return Math.round(clamp01(value) * 100) / 100;
}
function roundPosition(value: number): number {
return Math.round(clamp01(value) * 1000) / 1000;
}
function clamp01(value: number): number {
if (!Number.isFinite(value)) {
return 0;
src/hooks/useScreenRecorder.ts
+9 -2
View File
@@ -921,7 +921,10 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
}
const activeRecordingId = Date.now();
const displayId = Number(selectedSource.display_id);
const displayId =
typeof selectedSource.displayId === "number"
? selectedSource.displayId
: Number(selectedSource.display_id);
const sourceType = selectedSource.id.startsWith("window:") ? "window" : "display";
const windowHandle = parseWindowHandleFromSourceId(selectedSource.id);
if (webcamEnabled) {
@@ -946,6 +949,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
type: sourceType,
sourceId: selectedSource.id,
...(Number.isFinite(displayId) ? { displayId } : {}),
...(selectedSource.bounds ? { bounds: selectedSource.bounds } : {}),
...(windowHandle ? { windowHandle } : {}),
},
video: {
@@ -1039,7 +1043,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
const activeRecordingId = Date.now();
const sourceType = selectedSource.id.startsWith("window:") ? "window" : "display";
const displayId =
Number(selectedSource.display_id) || parseMacDisplayIdFromSourceId(selectedSource.id);
typeof selectedSource.displayId === "number"
? selectedSource.displayId
: Number(selectedSource.display_id) || parseMacDisplayIdFromSourceId(selectedSource.id);
const windowId = parseMacWindowIdFromSourceId(selectedSource.id);
let nativeWebcamRecorder: RecorderHandle | null = null;
if (webcamEnabled) {
@@ -1083,6 +1089,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
type: sourceType,
sourceId: selectedSource.id,
...(displayId ? { displayId } : {}),
...(selectedSource.bounds ? { bounds: selectedSource.bounds } : {}),
...(windowId ? { windowId } : {}),
},
video: {
src/lib/nativeWindowsRecording.ts
+6
View File
@@ -6,6 +6,12 @@ export type NativeWindowsRecordingRequest = {
type: NativeWindowsSourceType;
sourceId: string;
displayId?: number;
bounds?: {
x: number;
y: number;
width: number;
height: number;
};
windowHandle?: string;
};
video: {