Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5069354df3 | |||
| ee69df9222 | |||
| a235a0c50b | |||
| 94490a71af | |||
| 6ebabbaaaa | |||
| 0bd26eebf7 | |||
| cce81dd7c4 | |||
| 7823507a18 | |||
| 0b78ff6f7d |
@@ -0,0 +1,17 @@
|
||||
!macro customInstall
|
||||
DetailPrint "Installing OpenScreen OCR Windows service"
|
||||
nsExec::ExecToLog '"$SYSDIR\sc.exe" stop OpenScreenOCR'
|
||||
nsExec::ExecToLog '"$SYSDIR\sc.exe" delete OpenScreenOCR'
|
||||
Sleep 1000
|
||||
ExpandEnvStrings $0 "%ProgramData%\OpenScreen\ocr-runtime"
|
||||
CreateDirectory "$0"
|
||||
nsExec::ExecToLog '"$SYSDIR\sc.exe" create OpenScreenOCR binPath= "\"$INSTDIR\resources\electron\native\bin\win32-x64\openscreen-ocr-service-wrapper.exe\" --service --exe \"$INSTDIR\resources\ocr-service\openscreen-ocr-service.exe\" --resources \"$INSTDIR\resources\" --data \"$0\"" start= auto DisplayName= "OpenScreen OCR Service"'
|
||||
nsExec::ExecToLog '"$SYSDIR\sc.exe" description OpenScreenOCR "Local OCR service used by OpenScreen guide capture."'
|
||||
nsExec::ExecToLog '"$SYSDIR\sc.exe" start OpenScreenOCR'
|
||||
!macroend
|
||||
|
||||
!macro customUnInstall
|
||||
DetailPrint "Removing OpenScreen OCR Windows service"
|
||||
nsExec::ExecToLog '"$SYSDIR\sc.exe" stop OpenScreenOCR'
|
||||
nsExec::ExecToLog '"$SYSDIR\sc.exe" delete OpenScreenOCR'
|
||||
!macroend
|
||||
@@ -6,7 +6,7 @@ OpenScreen calls OCR through a local HTTP service. The default endpoint is:
|
||||
http://127.0.0.1:8866/ocr
|
||||
```
|
||||
|
||||
The app sends either `imageBase64` or `path` and expects OCR blocks:
|
||||
The app sends either `imageBase64` or `path`, plus optional `language` and `profile`, and expects OCR blocks:
|
||||
|
||||
```json
|
||||
{
|
||||
@@ -38,7 +38,7 @@ If `paddle` is still missing after installing `paddleocr`, install the CPU Paddl
|
||||
```powershell
|
||||
.\.venv-ocr\Scripts\Activate.ps1
|
||||
$env:PADDLEOCR_DEVICE="cpu"
|
||||
$env:PADDLEOCR_LANG="latin"
|
||||
$env:OPENSCREEN_OCR_PROFILE="vietnamese"
|
||||
npm run ocr:paddle
|
||||
```
|
||||
|
||||
@@ -58,7 +58,8 @@ Expected healthy environment:
|
||||
"paddleocrInstalled": true,
|
||||
"paddleInstalled": true,
|
||||
"engineReady": false,
|
||||
"defaultLanguage": "latin"
|
||||
"defaultLanguage": "vi,en",
|
||||
"defaultProfile": "vietnamese"
|
||||
}
|
||||
```
|
||||
|
||||
@@ -67,7 +68,10 @@ Expected healthy environment:
|
||||
## Configuration
|
||||
|
||||
- `PADDLEOCR_DEVICE`: `cpu`, `gpu:0`, or another PaddleOCR device string.
|
||||
- `PADDLEOCR_LANG`: defaults to `latin`; this is preferred for Vietnamese UI text because it uses a Latin-script recognition model.
|
||||
- `OPENSCREEN_OCR_PROFILE`: `fast`, `vietnamese`, or `hybrid`. The default `vietnamese` profile upscales and sharpens focused UI screenshots before OCR.
|
||||
- `OPENSCREEN_GUIDE_OCR_LANGUAGE`: defaults to `vi,en`.
|
||||
- `PADDLEOCR_LANG`: optional hard override. Leave unset for the app profile/language settings to work.
|
||||
- `PADDLEOCR_VERSION`: defaults to `PP-OCRv5`.
|
||||
- `PADDLEOCR_USE_MOBILE`: defaults to `1`; set to `0` to use the default/server models.
|
||||
- `PADDLEOCR_REC_MODEL`: optional recognizer model override. The bundled profile uses `latin_PP-OCRv5_mobile_rec`, which supports Vietnamese Latin-script text.
|
||||
- `OPENSCREEN_GUIDE_OCR_URL`: OpenScreen OCR endpoint override; defaults to `http://127.0.0.1:8866`.
|
||||
|
||||
+19
-10
@@ -13,11 +13,17 @@
|
||||
},
|
||||
"npmRebuild": true,
|
||||
"buildDependenciesFromSource": true,
|
||||
"compression": "normal",
|
||||
"directories": {
|
||||
"output": "release/${version}"
|
||||
},
|
||||
"files": [
|
||||
"compression": "normal",
|
||||
"directories": {
|
||||
"output": "release/${version}"
|
||||
},
|
||||
"publish": [
|
||||
{
|
||||
"provider": "generic",
|
||||
"url": "https://gittea.softs.business/huanld/openscreen/raw/branch/release-assets%2Flatest"
|
||||
}
|
||||
],
|
||||
"files": [
|
||||
"dist",
|
||||
"dist-electron",
|
||||
"!*.png",
|
||||
@@ -79,6 +85,7 @@
|
||||
"nsis"
|
||||
],
|
||||
"icon": "icons/icons/win/icon.ico",
|
||||
"requestedExecutionLevel": "requireAdministrator",
|
||||
"signAndEditExecutable": false,
|
||||
"signExts": ["!.exe"],
|
||||
"extraResources": [
|
||||
@@ -99,8 +106,10 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"nsis": {
|
||||
"oneClick": false,
|
||||
"allowToChangeInstallationDirectory": true
|
||||
}
|
||||
}
|
||||
"nsis": {
|
||||
"oneClick": false,
|
||||
"allowToChangeInstallationDirectory": true,
|
||||
"perMachine": true,
|
||||
"include": "build/installer.nsh"
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+8
@@ -24,6 +24,14 @@ declare namespace NodeJS {
|
||||
// Used in Renderer process, expose in `preload.ts`
|
||||
interface Window {
|
||||
electronAPI: {
|
||||
updates: {
|
||||
getStatus: () => Promise<import("../src/lib/updateStatus").UpdateStatus>;
|
||||
check: () => Promise<import("../src/lib/updateStatus").UpdateCheckResult>;
|
||||
install: () => Promise<import("../src/lib/updateStatus").UpdateCheckResult>;
|
||||
onStatus: (
|
||||
callback: (status: import("../src/lib/updateStatus").UpdateStatus) => void,
|
||||
) => () => void;
|
||||
};
|
||||
invokeNativeBridge: <TData = unknown>(
|
||||
request: import("../src/native/contracts").NativeBridgeRequest,
|
||||
) => Promise<import("../src/native/contracts").NativeBridgeResponse<TData>>;
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { DeepSeekSettingsStore } from "./deepseekSettingsStore";
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
const originalOcrProfile = process.env.OPENSCREEN_GUIDE_OCR_PROFILE;
|
||||
const originalOcrLanguage = process.env.OPENSCREEN_GUIDE_OCR_LANGUAGE;
|
||||
|
||||
beforeEach(() => {
|
||||
delete process.env.OPENSCREEN_GUIDE_OCR_PROFILE;
|
||||
delete process.env.OPENSCREEN_GUIDE_OCR_LANGUAGE;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
restoreEnv("OPENSCREEN_GUIDE_OCR_PROFILE", originalOcrProfile);
|
||||
restoreEnv("OPENSCREEN_GUIDE_OCR_LANGUAGE", originalOcrLanguage);
|
||||
await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
function restoreEnv(name: string, value: string | undefined): void {
|
||||
if (value === undefined) {
|
||||
delete process.env[name];
|
||||
return;
|
||||
}
|
||||
process.env[name] = value;
|
||||
}
|
||||
|
||||
async function createStore(): Promise<DeepSeekSettingsStore> {
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openscreen-guide-settings-"));
|
||||
tempDirs.push(dir);
|
||||
return new DeepSeekSettingsStore(path.join(dir, "guide-ai-settings.json"));
|
||||
}
|
||||
|
||||
describe("DeepSeekSettingsStore OCR settings", () => {
|
||||
it("defaults to the Vietnamese enhanced OCR profile", async () => {
|
||||
const store = await createStore();
|
||||
|
||||
await expect(store.getOcrConfig()).resolves.toEqual({
|
||||
profile: "vietnamese",
|
||||
language: "vi,en",
|
||||
});
|
||||
});
|
||||
|
||||
it("persists OCR profile changes alongside DeepSeek settings", async () => {
|
||||
const store = await createStore();
|
||||
|
||||
const status = await store.save({
|
||||
deepseekApiKeyEnvName: "DEEPSEEK_API_KEY",
|
||||
baseUrl: "https://api.deepseek.com",
|
||||
model: "deepseek-chat",
|
||||
ocrProfile: "hybrid",
|
||||
ocrLanguage: "vi,en",
|
||||
});
|
||||
|
||||
expect(status.ocr).toMatchObject({
|
||||
profile: "hybrid",
|
||||
language: "vi,en",
|
||||
});
|
||||
await expect(store.getOcrConfig()).resolves.toEqual({
|
||||
profile: "hybrid",
|
||||
language: "vi,en",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,10 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { GuideAiSettings, SaveGuideAiSettingsInput } from "../../../src/guide/contracts";
|
||||
import type {
|
||||
GuideAiSettings,
|
||||
GuideOcrProfile,
|
||||
SaveGuideAiSettingsInput,
|
||||
} from "../../../src/guide/contracts";
|
||||
|
||||
export interface DeepSeekGuideConfig {
|
||||
apiKey?: string;
|
||||
@@ -12,8 +16,22 @@ export interface DeepSeekGuideConfigProvider {
|
||||
getDeepSeekConfig(): Promise<DeepSeekGuideConfig>;
|
||||
}
|
||||
|
||||
export interface GuideOcrConfig {
|
||||
profile: GuideOcrProfile;
|
||||
language: string;
|
||||
}
|
||||
|
||||
export interface GuideOcrConfigProvider {
|
||||
getOcrConfig(): Promise<GuideOcrConfig>;
|
||||
}
|
||||
|
||||
interface PersistedGuideAiSettings {
|
||||
schemaVersion: 1;
|
||||
ocr?: {
|
||||
profile?: GuideOcrProfile;
|
||||
language?: string;
|
||||
updatedAt?: string;
|
||||
};
|
||||
deepseek?: {
|
||||
apiKeyEnvName?: string;
|
||||
baseUrl?: string;
|
||||
@@ -25,8 +43,10 @@ interface PersistedGuideAiSettings {
|
||||
const DEFAULT_DEEPSEEK_API_KEY_ENV_NAME = "DEEPSEEK_API_KEY";
|
||||
const DEFAULT_DEEPSEEK_BASE_URL = "https://api.deepseek.com";
|
||||
const DEFAULT_DEEPSEEK_MODEL = "deepseek-chat";
|
||||
const DEFAULT_OCR_PROFILE: GuideOcrProfile = "vietnamese";
|
||||
const DEFAULT_OCR_LANGUAGE = "vi,en";
|
||||
|
||||
export class DeepSeekSettingsStore implements DeepSeekGuideConfigProvider {
|
||||
export class DeepSeekSettingsStore implements DeepSeekGuideConfigProvider, GuideOcrConfigProvider {
|
||||
constructor(private readonly filePath: string) {}
|
||||
|
||||
async getStatus(): Promise<GuideAiSettings> {
|
||||
@@ -35,6 +55,13 @@ export class DeepSeekSettingsStore implements DeepSeekGuideConfigProvider {
|
||||
const activeApiKey = process.env[apiKeyEnvName];
|
||||
|
||||
return {
|
||||
ocr: {
|
||||
profile: normalizeOcrProfile(raw?.ocr?.profile ?? process.env.OPENSCREEN_GUIDE_OCR_PROFILE),
|
||||
language: normalizeOcrLanguage(
|
||||
raw?.ocr?.language ?? process.env.OPENSCREEN_GUIDE_OCR_LANGUAGE,
|
||||
),
|
||||
updatedAt: raw?.ocr?.updatedAt,
|
||||
},
|
||||
deepseek: {
|
||||
hasApiKey: Boolean(activeApiKey),
|
||||
apiKeyEnvName,
|
||||
@@ -49,7 +76,14 @@ export class DeepSeekSettingsStore implements DeepSeekGuideConfigProvider {
|
||||
|
||||
async save(input: SaveGuideAiSettingsInput): Promise<GuideAiSettings> {
|
||||
const current = (await this.readSettings()) ?? { schemaVersion: 1 };
|
||||
const currentOcr = current.ocr ?? {};
|
||||
const currentDeepSeek = current.deepseek ?? {};
|
||||
const nextOcr = {
|
||||
...currentOcr,
|
||||
profile: normalizeOcrProfile(input.ocrProfile ?? currentOcr.profile),
|
||||
language: normalizeOcrLanguage(input.ocrLanguage ?? currentOcr.language),
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
const nextDeepSeek = {
|
||||
...currentDeepSeek,
|
||||
baseUrl: normalizeBaseUrl(input.baseUrl ?? currentDeepSeek.baseUrl),
|
||||
@@ -65,6 +99,7 @@ export class DeepSeekSettingsStore implements DeepSeekGuideConfigProvider {
|
||||
|
||||
await this.writeSettings({
|
||||
schemaVersion: 1,
|
||||
ocr: nextOcr,
|
||||
deepseek: nextDeepSeek,
|
||||
});
|
||||
return await this.getStatus();
|
||||
@@ -80,6 +115,16 @@ export class DeepSeekSettingsStore implements DeepSeekGuideConfigProvider {
|
||||
};
|
||||
}
|
||||
|
||||
async getOcrConfig(): Promise<GuideOcrConfig> {
|
||||
const raw = await this.readSettings();
|
||||
return {
|
||||
profile: normalizeOcrProfile(raw?.ocr?.profile ?? process.env.OPENSCREEN_GUIDE_OCR_PROFILE),
|
||||
language: normalizeOcrLanguage(
|
||||
raw?.ocr?.language ?? process.env.OPENSCREEN_GUIDE_OCR_LANGUAGE,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
private async readSettings(): Promise<PersistedGuideAiSettings | null> {
|
||||
try {
|
||||
const content = await fs.readFile(this.filePath, "utf-8");
|
||||
@@ -120,6 +165,11 @@ function normalizePersistedSettings(input: unknown): PersistedGuideAiSettings |
|
||||
}
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
ocr: {
|
||||
profile: normalizeOcrProfile(raw.ocr?.profile),
|
||||
language: normalizeOcrLanguage(raw.ocr?.language),
|
||||
updatedAt: raw.ocr?.updatedAt,
|
||||
},
|
||||
deepseek: {
|
||||
apiKeyEnvName: normalizeEnvName(raw.deepseek?.apiKeyEnvName),
|
||||
baseUrl: raw.deepseek?.baseUrl,
|
||||
@@ -155,3 +205,19 @@ function normalizeBaseUrl(value: string | undefined): string {
|
||||
function normalizeModel(value: string | undefined): string {
|
||||
return value?.trim() || DEFAULT_DEEPSEEK_MODEL;
|
||||
}
|
||||
|
||||
function normalizeOcrProfile(value: string | undefined): GuideOcrProfile {
|
||||
if (value === "fast" || value === "vietnamese" || value === "hybrid") {
|
||||
return value;
|
||||
}
|
||||
return DEFAULT_OCR_PROFILE;
|
||||
}
|
||||
|
||||
function normalizeOcrLanguage(value: string | undefined): string {
|
||||
const normalized = value
|
||||
?.split(",")
|
||||
.map((part) => part.trim().toLowerCase())
|
||||
.filter(Boolean)
|
||||
.join(",");
|
||||
return normalized || DEFAULT_OCR_LANGUAGE;
|
||||
}
|
||||
|
||||
@@ -168,6 +168,7 @@ describe("GuideStore", () => {
|
||||
width: 800,
|
||||
height: 600,
|
||||
pngBytes: new Uint8Array([137, 80, 78, 71]).buffer,
|
||||
markedPngBytes: new Uint8Array([137, 80, 78, 71, 1]).buffer,
|
||||
});
|
||||
|
||||
expect(session.status).toBe("snapshots-ready");
|
||||
@@ -176,6 +177,9 @@ describe("GuideStore", () => {
|
||||
await expect(fs.readFile(session.snapshots[0]?.path ?? "")).resolves.toEqual(
|
||||
Buffer.from([137, 80, 78, 71]),
|
||||
);
|
||||
await expect(fs.readFile(session.snapshots[0]?.markedPath ?? "")).resolves.toEqual(
|
||||
Buffer.from([137, 80, 78, 71, 1]),
|
||||
);
|
||||
});
|
||||
|
||||
it("runs OCR, generates a local draft, and exports files", async () => {
|
||||
@@ -228,6 +232,74 @@ describe("GuideStore", () => {
|
||||
await expect(fs.readFile(html.path, "utf-8")).resolves.toContain("<!doctype html>");
|
||||
});
|
||||
|
||||
it("resumes OCR without reprocessing completed snapshots", async () => {
|
||||
const recognizedSnapshotIds: string[] = [];
|
||||
const store = new GuideStore(recordingsDir, {
|
||||
ocrClient: {
|
||||
recognize: async (snapshot) => {
|
||||
recognizedSnapshotIds.push(snapshot.id);
|
||||
return [];
|
||||
},
|
||||
},
|
||||
});
|
||||
await store.startSession(115);
|
||||
const firstMarker = await store.addMarker({
|
||||
recordingId: 115,
|
||||
kind: "hotkey",
|
||||
timeMs: 100,
|
||||
label: "Ctrl+F12 marker",
|
||||
normalizedX: 0.25,
|
||||
normalizedY: 0.35,
|
||||
});
|
||||
const secondMarker = await store.addMarker({
|
||||
recordingId: 115,
|
||||
kind: "hotkey",
|
||||
timeMs: 300,
|
||||
label: "Ctrl+F12 marker",
|
||||
normalizedX: 0.6,
|
||||
normalizedY: 0.7,
|
||||
});
|
||||
const firstEvent = firstMarker.event;
|
||||
const secondEvent = secondMarker.event;
|
||||
await store.writeSnapshot({
|
||||
recordingId: 115,
|
||||
eventId: firstEvent?.id ?? "",
|
||||
timeMs: 100,
|
||||
offsetMs: 0,
|
||||
width: 800,
|
||||
height: 600,
|
||||
pngBytes: new Uint8Array([1, 2, 3]).buffer,
|
||||
});
|
||||
await store.writeSnapshot({
|
||||
recordingId: 115,
|
||||
eventId: secondEvent?.id ?? "",
|
||||
timeMs: 300,
|
||||
offsetMs: 0,
|
||||
width: 800,
|
||||
height: 600,
|
||||
pngBytes: new Uint8Array([4, 5, 6]).buffer,
|
||||
});
|
||||
|
||||
await store.runOcr({
|
||||
recordingId: 115,
|
||||
snapshotIds: [`snapshot-${firstEvent?.id}`],
|
||||
});
|
||||
expect(recognizedSnapshotIds).toEqual([`snapshot-${firstEvent?.id}`]);
|
||||
|
||||
const resumedSession = await store.runOcr({ recordingId: 115 });
|
||||
expect(recognizedSnapshotIds).toEqual([
|
||||
`snapshot-${firstEvent?.id}`,
|
||||
`snapshot-${secondEvent?.id}`,
|
||||
]);
|
||||
expect(resumedSession.snapshots.every((snapshot) => snapshot.ocrCompletedAt)).toBe(true);
|
||||
|
||||
await store.runOcr({ recordingId: 115 });
|
||||
expect(recognizedSnapshotIds).toEqual([
|
||||
`snapshot-${firstEvent?.id}`,
|
||||
`snapshot-${secondEvent?.id}`,
|
||||
]);
|
||||
});
|
||||
|
||||
it("repairs generic hotkey marker text and attaches AI draft artifacts", async () => {
|
||||
const store = new GuideStore(recordingsDir, {
|
||||
ocrClient: {
|
||||
|
||||
+125
-36
@@ -34,7 +34,10 @@ import {
|
||||
DeepSeekGuideClientError,
|
||||
type GuideDraftClient,
|
||||
} from "./ai/deepseekGuideClient";
|
||||
import type { DeepSeekGuideConfigProvider } from "./ai/deepseekSettingsStore";
|
||||
import type {
|
||||
DeepSeekGuideConfigProvider,
|
||||
GuideOcrConfigProvider,
|
||||
} from "./ai/deepseekSettingsStore";
|
||||
import { type GuidePaths, normalizeGuideRecordingId, resolveGuidePaths } from "./guidePaths";
|
||||
import { createFocusedOcrSnapshot, remapFocusedOcrBlocks } from "./ocr/focusedOcrSnapshot";
|
||||
import { DefaultGuideOcrClient, type GuideOcrClient } from "./ocr/paddleOcrClient";
|
||||
@@ -55,6 +58,8 @@ const VALID_EVENT_SOURCES = new Set<GuideEventSource>([
|
||||
"review-ui",
|
||||
]);
|
||||
|
||||
const guideOcrJobsByRecordingId = new Map<string, Promise<GuideSession>>();
|
||||
|
||||
export class GuideStoreError extends Error {
|
||||
constructor(
|
||||
readonly code: GuideErrorCode,
|
||||
@@ -70,6 +75,7 @@ export interface GuideStoreDependencies {
|
||||
ocrClient?: GuideOcrClient;
|
||||
draftClient?: GuideDraftClient;
|
||||
deepSeekConfigProvider?: DeepSeekGuideConfigProvider;
|
||||
ocrConfigProvider?: GuideOcrConfigProvider;
|
||||
focusOcrSnapshots?: boolean;
|
||||
}
|
||||
|
||||
@@ -209,10 +215,19 @@ export class GuideStore {
|
||||
|
||||
this.assertGuidePathIsAllowed(session.outputDir);
|
||||
await fs.mkdir(session.outputDir, { recursive: true });
|
||||
const fileName = `step-${String(eventIndex + 1).padStart(3, "0")}.png`;
|
||||
const fileBaseName = `step-${String(eventIndex + 1).padStart(3, "0")}`;
|
||||
const fileName = `${fileBaseName}.png`;
|
||||
const snapshotPath = path.join(session.outputDir, fileName);
|
||||
const markedSnapshotPath = path.join(session.outputDir, `${fileBaseName}-marked.png`);
|
||||
this.assertGuidePathIsAllowed(snapshotPath);
|
||||
this.assertGuidePathIsAllowed(markedSnapshotPath);
|
||||
await fs.writeFile(snapshotPath, Buffer.from(new Uint8Array(input.pngBytes)));
|
||||
const hasMarkedSnapshot = Boolean(input.markedPngBytes?.byteLength);
|
||||
if (hasMarkedSnapshot && input.markedPngBytes) {
|
||||
await fs.writeFile(markedSnapshotPath, Buffer.from(new Uint8Array(input.markedPngBytes)));
|
||||
} else {
|
||||
await fs.unlink(markedSnapshotPath).catch(() => undefined);
|
||||
}
|
||||
|
||||
const snapshot: GuideSnapshot = {
|
||||
id: `snapshot-${input.eventId}`,
|
||||
@@ -220,6 +235,7 @@ export class GuideStore {
|
||||
timeMs: Math.max(0, input.timeMs),
|
||||
offsetMs: input.offsetMs,
|
||||
path: snapshotPath,
|
||||
markedPath: hasMarkedSnapshot ? markedSnapshotPath : undefined,
|
||||
width: Math.round(input.width),
|
||||
height: Math.round(input.height),
|
||||
};
|
||||
@@ -245,48 +261,103 @@ export class GuideStore {
|
||||
}
|
||||
|
||||
async runOcr(input: RunGuideOcrInput): Promise<GuideSession> {
|
||||
const session = await this.readSession(input.recordingId);
|
||||
const requestedIds = new Set(input.snapshotIds ?? []);
|
||||
const snapshots =
|
||||
requestedIds.size > 0
|
||||
? session.snapshots.filter((snapshot) => requestedIds.has(snapshot.id))
|
||||
: session.snapshots;
|
||||
if (snapshots.length === 0) {
|
||||
throw new GuideStoreError("guide-invalid-input", "No guide snapshots are available for OCR.");
|
||||
const recordingId = normalizeGuideRecordingId(input.recordingId);
|
||||
if (!recordingId) {
|
||||
throw new GuideStoreError("guide-invalid-input", "OCR run is missing recordingId.");
|
||||
}
|
||||
|
||||
const ocrClient = this.dependencies.ocrClient ?? new DefaultGuideOcrClient();
|
||||
const shouldFocusOcrSnapshots =
|
||||
this.dependencies.focusOcrSnapshots ?? this.dependencies.ocrClient === undefined;
|
||||
const eventsById = new Map(session.events.map((event) => [event.id, event]));
|
||||
const blocks: OcrBlock[] = [];
|
||||
try {
|
||||
for (const snapshot of snapshots) {
|
||||
const focusedSnapshot = shouldFocusOcrSnapshots
|
||||
? await createFocusedOcrSnapshot({
|
||||
snapshot,
|
||||
event: eventsById.get(snapshot.eventId),
|
||||
outputDir: session.outputDir,
|
||||
})
|
||||
: { snapshot };
|
||||
const recognizedBlocks = await ocrClient.recognize(focusedSnapshot.snapshot);
|
||||
blocks.push(...remapFocusedOcrBlocks(recognizedBlocks, focusedSnapshot.transform));
|
||||
const previousJob =
|
||||
guideOcrJobsByRecordingId.get(recordingId)?.catch(() => undefined) ?? Promise.resolve();
|
||||
const nextJob = previousJob.then(async () => {
|
||||
let session = await this.readSession(recordingId);
|
||||
const requestedIds = new Set(input.snapshotIds ?? []);
|
||||
const snapshots =
|
||||
requestedIds.size > 0
|
||||
? session.snapshots.filter((snapshot) => requestedIds.has(snapshot.id))
|
||||
: session.snapshots;
|
||||
if (snapshots.length === 0) {
|
||||
throw new GuideStoreError(
|
||||
"guide-invalid-input",
|
||||
"No guide snapshots are available for OCR.",
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
throw new GuideStoreError(
|
||||
"guide-ocr-unavailable",
|
||||
error instanceof Error ? error.message : "OCR failed.",
|
||||
true,
|
||||
);
|
||||
}
|
||||
|
||||
const snapshotIds = new Set(snapshots.map((snapshot) => snapshot.id));
|
||||
const completedSnapshotIds = new Set(
|
||||
session.snapshots
|
||||
.filter((snapshot) => isSnapshotOcrCompleted(snapshot, session.ocrBlocks))
|
||||
.map((snapshot) => snapshot.id),
|
||||
);
|
||||
const pendingSnapshots = snapshots.filter(
|
||||
(snapshot) => !completedSnapshotIds.has(snapshot.id),
|
||||
);
|
||||
if (pendingSnapshots.length === 0) {
|
||||
if (session.status === "ocr-ready") {
|
||||
return session;
|
||||
}
|
||||
const readySession = touchSession({
|
||||
...session,
|
||||
status: "ocr-ready",
|
||||
candidates: buildGuideStepCandidates(session),
|
||||
});
|
||||
await this.writeSession(readySession);
|
||||
return readySession;
|
||||
}
|
||||
|
||||
const ocrClient =
|
||||
this.dependencies.ocrClient ??
|
||||
DefaultGuideOcrClient.fromConfig(await this.dependencies.ocrConfigProvider?.getOcrConfig());
|
||||
const shouldFocusOcrSnapshots =
|
||||
this.dependencies.focusOcrSnapshots ?? this.dependencies.ocrClient === undefined;
|
||||
const eventsById = new Map(session.events.map((event) => [event.id, event]));
|
||||
try {
|
||||
for (const snapshot of pendingSnapshots) {
|
||||
const focusedSnapshot = shouldFocusOcrSnapshots
|
||||
? await createFocusedOcrSnapshot({
|
||||
snapshot,
|
||||
event: eventsById.get(snapshot.eventId),
|
||||
outputDir: session.outputDir,
|
||||
})
|
||||
: { snapshot };
|
||||
const recognizedBlocks = await ocrClient.recognize(focusedSnapshot.snapshot);
|
||||
const blocks = remapFocusedOcrBlocks(recognizedBlocks, focusedSnapshot.transform);
|
||||
session = await this.writeOcrSnapshotProgress(session, snapshot.id, blocks);
|
||||
}
|
||||
} catch (error) {
|
||||
throw new GuideStoreError(
|
||||
"guide-ocr-unavailable",
|
||||
error instanceof Error ? error.message : "OCR failed.",
|
||||
true,
|
||||
);
|
||||
}
|
||||
|
||||
return session;
|
||||
});
|
||||
guideOcrJobsByRecordingId.set(recordingId, nextJob);
|
||||
try {
|
||||
return await nextJob;
|
||||
} finally {
|
||||
if (guideOcrJobsByRecordingId.get(recordingId) === nextJob) {
|
||||
guideOcrJobsByRecordingId.delete(recordingId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async writeOcrSnapshotProgress(
|
||||
session: GuideSession,
|
||||
snapshotId: string,
|
||||
blocks: OcrBlock[],
|
||||
): Promise<GuideSession> {
|
||||
const updatedOcrBlocks = [
|
||||
...session.ocrBlocks.filter((block) => !snapshotIds.has(block.snapshotId)),
|
||||
...session.ocrBlocks.filter((block) => block.snapshotId !== snapshotId),
|
||||
...blocks,
|
||||
];
|
||||
const completedAt = new Date().toISOString();
|
||||
const updatedSnapshots = session.snapshots.map((snapshot) =>
|
||||
snapshot.id === snapshotId ? { ...snapshot, ocrCompletedAt: completedAt } : snapshot,
|
||||
);
|
||||
const draftSession = {
|
||||
...session,
|
||||
snapshots: updatedSnapshots,
|
||||
ocrBlocks: updatedOcrBlocks,
|
||||
};
|
||||
const updatedSession = touchSession({
|
||||
@@ -662,6 +733,8 @@ function normalizeGuideSnapshot(input: unknown): GuideSnapshot | null {
|
||||
const id = normalizeString(input.id);
|
||||
const eventId = normalizeString(input.eventId);
|
||||
const pathValue = normalizeString(input.path);
|
||||
const markedPath = normalizeOptionalString(input.markedPath);
|
||||
const ocrCompletedAt = normalizeOptionalString(input.ocrCompletedAt);
|
||||
const timeMs = normalizeNonNegativeNumber(input.timeMs);
|
||||
const offsetMs = normalizeOptionalNumber(input.offsetMs);
|
||||
const width = normalizePositiveInteger(input.width);
|
||||
@@ -677,7 +750,23 @@ function normalizeGuideSnapshot(input: unknown): GuideSnapshot | null {
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return { id, eventId, timeMs, offsetMs, path: pathValue, width, height };
|
||||
return {
|
||||
id,
|
||||
eventId,
|
||||
timeMs,
|
||||
offsetMs,
|
||||
path: pathValue,
|
||||
markedPath,
|
||||
ocrCompletedAt,
|
||||
width,
|
||||
height,
|
||||
};
|
||||
}
|
||||
|
||||
function isSnapshotOcrCompleted(snapshot: GuideSnapshot, ocrBlocks: OcrBlock[]): boolean {
|
||||
return (
|
||||
Boolean(snapshot.ocrCompletedAt) || ocrBlocks.some((block) => block.snapshotId === snapshot.id)
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeOcrBlock(input: unknown): OcrBlock | null {
|
||||
|
||||
@@ -1,14 +1,17 @@
|
||||
import { type ChildProcessWithoutNullStreams, spawn } from "node:child_process";
|
||||
import { type ChildProcessWithoutNullStreams, execFile, spawn } from "node:child_process";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { app } from "electron";
|
||||
|
||||
const DEFAULT_OCR_BASE_URL = "http://127.0.0.1:8866";
|
||||
const DEFAULT_OCR_PORT = "8866";
|
||||
const WINDOWS_SERVICE_NAME = "OpenScreenOCR";
|
||||
const SERVICE_EXE_NAME = "openscreen-ocr-service.exe";
|
||||
const HEALTH_TIMEOUT_MS = 1000;
|
||||
const STARTUP_TIMEOUT_MS = 90000;
|
||||
const PADDLEX_MODEL_NAMES = ["PP-OCRv5_mobile_det", "latin_PP-OCRv5_mobile_rec"];
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
let ocrProcess: ChildProcessWithoutNullStreams | null = null;
|
||||
let startupPromise: Promise<void> | null = null;
|
||||
@@ -24,6 +27,11 @@ export async function ensureBundledOcrServiceRunning(
|
||||
return;
|
||||
}
|
||||
|
||||
if (process.platform === "win32" && (await startInstalledWindowsOcrService())) {
|
||||
await waitForOcrServiceHealth(baseUrl, STARTUP_TIMEOUT_MS);
|
||||
return;
|
||||
}
|
||||
|
||||
const executablePath = await findBundledOcrServiceExecutable();
|
||||
if (!executablePath) {
|
||||
return;
|
||||
@@ -51,6 +59,39 @@ function shouldManageOcrService(baseUrl: string): boolean {
|
||||
}
|
||||
}
|
||||
|
||||
async function startInstalledWindowsOcrService(): Promise<boolean> {
|
||||
const query = await runSc(["query", WINDOWS_SERVICE_NAME]);
|
||||
if (!query.success) {
|
||||
return false;
|
||||
}
|
||||
if (/\bRUNNING\b/i.test(query.output)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const start = await runSc(["start", WINDOWS_SERVICE_NAME]);
|
||||
return start.success || /\b1056\b/.test(start.output) || /already running/i.test(start.output);
|
||||
}
|
||||
|
||||
async function runSc(args: string[]): Promise<{ success: boolean; output: string }> {
|
||||
try {
|
||||
const result = await execFileAsync("sc.exe", args, {
|
||||
windowsHide: true,
|
||||
timeout: 10000,
|
||||
maxBuffer: 512 * 1024,
|
||||
});
|
||||
return {
|
||||
success: true,
|
||||
output: `${result.stdout ?? ""}\n${result.stderr ?? ""}`,
|
||||
};
|
||||
} catch (error) {
|
||||
const failed = error as { stdout?: string; stderr?: string };
|
||||
return {
|
||||
success: false,
|
||||
output: `${failed.stdout ?? ""}\n${failed.stderr ?? ""}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function findBundledOcrServiceExecutable(): Promise<string | null> {
|
||||
const candidates = [
|
||||
process.env.OPENSCREEN_GUIDE_OCR_EXE,
|
||||
@@ -156,8 +197,11 @@ function startOcrServiceProcess(
|
||||
OPENSCREEN_OCR_PORT: DEFAULT_OCR_PORT,
|
||||
PADDLEOCR_DEVICE: process.env.PADDLEOCR_DEVICE ?? "cpu",
|
||||
PADDLEOCR_ENABLE_MKLDNN: process.env.PADDLEOCR_ENABLE_MKLDNN ?? "0",
|
||||
PADDLEOCR_LANG: process.env.PADDLEOCR_LANG ?? "latin",
|
||||
PADDLEOCR_LANG: process.env.PADDLEOCR_LANG ?? "",
|
||||
PADDLEOCR_USE_MOBILE: process.env.PADDLEOCR_USE_MOBILE ?? "1",
|
||||
OPENSCREEN_OCR_PROFILE:
|
||||
process.env.OPENSCREEN_OCR_PROFILE ?? process.env.OPENSCREEN_GUIDE_OCR_PROFILE ?? "",
|
||||
OPENSCREEN_OCR_WARMUP: process.env.OPENSCREEN_OCR_WARMUP ?? "1",
|
||||
PADDLE_PDX_ENABLE_MKLDNN_BYDEFAULT: process.env.PADDLE_PDX_ENABLE_MKLDNN_BYDEFAULT ?? "False",
|
||||
PADDLE_PDX_CACHE_HOME: process.env.PADDLE_PDX_CACHE_HOME ?? runtimePaths.paddlexCachePath,
|
||||
PADDLE_PDX_DISABLE_MODEL_SOURCE_CHECK:
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { GuideSnapshot, OcrBlock } from "../../../src/guide/contracts";
|
||||
import {
|
||||
DefaultGuideOcrClient,
|
||||
normalizeOcrResponse,
|
||||
PaddleOcrHttpClient,
|
||||
parseWindowsOcrPayload,
|
||||
} from "./paddleOcrClient";
|
||||
|
||||
@@ -16,6 +20,10 @@ const snapshot: GuideSnapshot = {
|
||||
height: 800,
|
||||
};
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
describe("normalizeOcrResponse", () => {
|
||||
it("normalizes pixel boxes into guide OCR blocks", () => {
|
||||
const blocks = normalizeOcrResponse(
|
||||
@@ -67,6 +75,35 @@ describe("normalizeOcrResponse", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("PaddleOcrHttpClient", () => {
|
||||
it("sends the selected OCR profile to the local service", async () => {
|
||||
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openscreen-ocr-client-"));
|
||||
const imagePath = path.join(tempDir, "step.png");
|
||||
await fs.writeFile(imagePath, Buffer.from([137, 80, 78, 71]));
|
||||
const requests: unknown[] = [];
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async (_url: string, init?: RequestInit) => {
|
||||
requests.push(JSON.parse(String(init?.body ?? "{}")));
|
||||
return new Response(JSON.stringify({ blocks: [] }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}),
|
||||
);
|
||||
|
||||
const client = new PaddleOcrHttpClient("https://ocr.example.test", "vi,en", "hybrid");
|
||||
await client.recognize({ ...snapshot, path: imagePath });
|
||||
|
||||
expect(requests[0]).toMatchObject({
|
||||
language: "vi,en",
|
||||
profile: "hybrid",
|
||||
path: imagePath,
|
||||
});
|
||||
await fs.rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe("DefaultGuideOcrClient", () => {
|
||||
it("falls back when the HTTP OCR service is unavailable", async () => {
|
||||
const fallbackBlock: OcrBlock = {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import fs from "node:fs/promises";
|
||||
import { promisify } from "node:util";
|
||||
import type { GuideSnapshot, OcrBlock } from "../../../src/guide/contracts";
|
||||
import type { GuideOcrProfile, GuideSnapshot, OcrBlock } from "../../../src/guide/contracts";
|
||||
import { ensureBundledOcrServiceRunning } from "./bundledOcrService";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
@@ -10,6 +10,11 @@ export interface GuideOcrClient {
|
||||
recognize(snapshot: GuideSnapshot): Promise<OcrBlock[]>;
|
||||
}
|
||||
|
||||
export interface GuideOcrClientConfig {
|
||||
profile: GuideOcrProfile;
|
||||
language: string;
|
||||
}
|
||||
|
||||
interface PaddleOcrResponseBlock {
|
||||
text?: unknown;
|
||||
confidence?: unknown;
|
||||
@@ -21,7 +26,8 @@ interface PaddleOcrResponseBlock {
|
||||
export class PaddleOcrHttpClient implements GuideOcrClient {
|
||||
constructor(
|
||||
private readonly baseUrl = process.env.OPENSCREEN_GUIDE_OCR_URL ?? "http://127.0.0.1:8866",
|
||||
private readonly language = process.env.OPENSCREEN_GUIDE_OCR_LANGUAGE ?? "vi,en",
|
||||
private readonly language = normalizeOcrLanguage(process.env.OPENSCREEN_GUIDE_OCR_LANGUAGE),
|
||||
private readonly profile = normalizeOcrProfile(process.env.OPENSCREEN_GUIDE_OCR_PROFILE),
|
||||
) {}
|
||||
|
||||
async recognize(snapshot: GuideSnapshot): Promise<OcrBlock[]> {
|
||||
@@ -36,6 +42,7 @@ export class PaddleOcrHttpClient implements GuideOcrClient {
|
||||
imageBase64,
|
||||
path: snapshot.path,
|
||||
language: this.language,
|
||||
profile: this.profile,
|
||||
}),
|
||||
});
|
||||
} catch (error) {
|
||||
@@ -54,7 +61,9 @@ export class PaddleOcrHttpClient implements GuideOcrClient {
|
||||
}
|
||||
|
||||
export class WindowsOcrClient implements GuideOcrClient {
|
||||
constructor(private readonly language = process.env.OPENSCREEN_GUIDE_OCR_LANGUAGE ?? "vi,en") {}
|
||||
constructor(
|
||||
private readonly language = normalizeOcrLanguage(process.env.OPENSCREEN_GUIDE_OCR_LANGUAGE),
|
||||
) {}
|
||||
|
||||
async recognize(snapshot: GuideSnapshot): Promise<OcrBlock[]> {
|
||||
if (process.platform !== "win32") {
|
||||
@@ -96,6 +105,14 @@ export class WindowsOcrClient implements GuideOcrClient {
|
||||
}
|
||||
|
||||
export class DefaultGuideOcrClient implements GuideOcrClient {
|
||||
static fromConfig(config?: Partial<GuideOcrClientConfig>): DefaultGuideOcrClient {
|
||||
const normalizedConfig = normalizeOcrClientConfig(config);
|
||||
return new DefaultGuideOcrClient(
|
||||
new PaddleOcrHttpClient(undefined, normalizedConfig.language, normalizedConfig.profile),
|
||||
new WindowsOcrClient(normalizedConfig.language),
|
||||
);
|
||||
}
|
||||
|
||||
constructor(
|
||||
private readonly httpClient = new PaddleOcrHttpClient(),
|
||||
private readonly windowsClient = new WindowsOcrClient(),
|
||||
@@ -119,6 +136,31 @@ export class DefaultGuideOcrClient implements GuideOcrClient {
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeOcrClientConfig(
|
||||
config: Partial<GuideOcrClientConfig> | undefined,
|
||||
): GuideOcrClientConfig {
|
||||
return {
|
||||
profile: normalizeOcrProfile(config?.profile ?? process.env.OPENSCREEN_GUIDE_OCR_PROFILE),
|
||||
language: normalizeOcrLanguage(config?.language ?? process.env.OPENSCREEN_GUIDE_OCR_LANGUAGE),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeOcrProfile(value: string | undefined): GuideOcrProfile {
|
||||
if (value === "fast" || value === "vietnamese" || value === "hybrid") {
|
||||
return value;
|
||||
}
|
||||
return "vietnamese";
|
||||
}
|
||||
|
||||
function normalizeOcrLanguage(value: string | undefined): string {
|
||||
const normalized = value
|
||||
?.split(",")
|
||||
.map((part) => part.trim().toLowerCase())
|
||||
.filter(Boolean)
|
||||
.join(",");
|
||||
return normalized || "vi,en";
|
||||
}
|
||||
|
||||
export function parseWindowsOcrPayload(stdout: string): unknown {
|
||||
const normalized = stdout.replace(/^\uFEFF/, "").trim();
|
||||
try {
|
||||
|
||||
+122
-25
@@ -426,6 +426,7 @@ let nativeWindowsCursorRecordingStartMs = 0;
|
||||
let nativeWindowsPauseStartedAtMs: number | null = null;
|
||||
let nativeWindowsPauseRanges: Array<{ startMs: number; endMs: number }> = [];
|
||||
let nativeWindowsIsPaused = false;
|
||||
let nativeWindowsCaptureStopping = false;
|
||||
const NATIVE_WINDOWS_CAPTURE_STOP_TIMEOUT_MS = 15_000;
|
||||
let nativeMacCaptureProcess: ChildProcessWithoutNullStreams | null = null;
|
||||
let nativeMacCaptureOutput = "";
|
||||
@@ -1337,6 +1338,81 @@ function completeNativeWindowsCursorPauseRange(endMs = Date.now()) {
|
||||
nativeWindowsPauseStartedAtMs = null;
|
||||
}
|
||||
|
||||
function resetNativeWindowsCaptureState() {
|
||||
nativeWindowsCaptureProcess = null;
|
||||
nativeWindowsCaptureTargetPath = null;
|
||||
nativeWindowsCaptureWebcamTargetPath = null;
|
||||
nativeWindowsCaptureRecordingId = null;
|
||||
nativeWindowsCursorOffsetMs = 0;
|
||||
nativeWindowsCursorCaptureMode = "editable-overlay";
|
||||
nativeWindowsCursorRecordingStartMs = 0;
|
||||
nativeWindowsPauseStartedAtMs = null;
|
||||
nativeWindowsPauseRanges = [];
|
||||
nativeWindowsIsPaused = false;
|
||||
nativeWindowsCaptureStopping = false;
|
||||
clearGuideHotkeyRecording();
|
||||
}
|
||||
|
||||
function hasActiveNativeWindowsCaptureProcess() {
|
||||
const proc = nativeWindowsCaptureProcess;
|
||||
if (!proc) {
|
||||
return false;
|
||||
}
|
||||
if (proc.exitCode === null && !proc.killed) {
|
||||
return true;
|
||||
}
|
||||
|
||||
console.warn("[native-wgc] clearing stale Windows capture process state", {
|
||||
exitCode: proc.exitCode,
|
||||
killed: proc.killed,
|
||||
});
|
||||
resetNativeWindowsCaptureState();
|
||||
return false;
|
||||
}
|
||||
|
||||
function attachNativeWindowsCaptureLifecycle(
|
||||
proc: ChildProcessWithoutNullStreams,
|
||||
sourceName: string,
|
||||
onRecordingStateChange?: (recording: boolean, sourceName: string) => void,
|
||||
) {
|
||||
const cleanupAfterUnexpectedExit = async () => {
|
||||
try {
|
||||
await stopCursorRecording();
|
||||
} catch (error) {
|
||||
console.warn("[native-wgc] failed to stop cursor recording after helper exit", error);
|
||||
}
|
||||
pendingCursorRecordingData = null;
|
||||
resetNativeWindowsCaptureState();
|
||||
onRecordingStateChange?.(false, sourceName);
|
||||
};
|
||||
|
||||
function onClose(code: number | null, signal: NodeJS.Signals | null) {
|
||||
proc.off("error", onError);
|
||||
if (nativeWindowsCaptureProcess !== proc || nativeWindowsCaptureStopping) {
|
||||
return;
|
||||
}
|
||||
|
||||
console.warn("[native-wgc] Windows capture helper exited before stop was requested", {
|
||||
code,
|
||||
signal,
|
||||
output: nativeWindowsCaptureOutput.trim(),
|
||||
});
|
||||
void cleanupAfterUnexpectedExit();
|
||||
}
|
||||
function onError(error: Error) {
|
||||
proc.off("close", onClose);
|
||||
if (nativeWindowsCaptureProcess !== proc || nativeWindowsCaptureStopping) {
|
||||
return;
|
||||
}
|
||||
|
||||
console.warn("[native-wgc] Windows capture helper errored before stop was requested", error);
|
||||
void cleanupAfterUnexpectedExit();
|
||||
}
|
||||
|
||||
proc.once("close", onClose);
|
||||
proc.once("error", onError);
|
||||
}
|
||||
|
||||
function waitForNativeWindowsCaptureStart(proc: ChildProcessWithoutNullStreams) {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
@@ -1732,7 +1808,7 @@ export function registerIpcHandlers(
|
||||
const sources = await desktopCapturer.getSources(opts);
|
||||
lastEnumeratedSources = new Map(sources.map((source) => [source.id, source]));
|
||||
let screenSourceIndex = 0;
|
||||
return sources.map((source) => {
|
||||
const processedSources = sources.map((source) => {
|
||||
const isScreenSource = source.id.startsWith("screen:");
|
||||
const sourceIndex = isScreenSource
|
||||
? (parseDesktopCapturerScreenIndex(source.id) ?? screenSourceIndex)
|
||||
@@ -1760,6 +1836,43 @@ export function registerIpcHandlers(
|
||||
bounds,
|
||||
};
|
||||
});
|
||||
const screenDisplays = screen.getAllDisplays();
|
||||
const mappedDisplayIds = new Set(
|
||||
processedSources
|
||||
.filter((source) => source.id.startsWith("screen:") && typeof source.displayId === "number")
|
||||
.map((source) => source.displayId),
|
||||
);
|
||||
const fallbackScreenSources = screenDisplays
|
||||
.map((display, displayIndex) => ({ display, displayIndex }))
|
||||
.filter(({ display }) => !mappedDisplayIds.has(display.id))
|
||||
.map(({ display, displayIndex }) => {
|
||||
const bounds = toSourceBounds(display.bounds);
|
||||
return {
|
||||
id: `screen:${displayIndex}:fallback:${display.id}`,
|
||||
name: `Screen ${displayIndex + 1}`,
|
||||
display_id: String(display.id),
|
||||
thumbnail: null,
|
||||
appIcon: null,
|
||||
displayId: display.id,
|
||||
displayIndex,
|
||||
screenIndex: displayIndex,
|
||||
displayLabel: `Display ${displayIndex + 1} - ${bounds.width}x${bounds.height} @ ${bounds.x},${bounds.y}`,
|
||||
bounds,
|
||||
};
|
||||
});
|
||||
if (fallbackScreenSources.length > 0) {
|
||||
console.warn("[desktop-capturer] added fallback display sources", {
|
||||
capturerScreens: processedSources.filter((source) => source.id.startsWith("screen:"))
|
||||
.length,
|
||||
electronDisplays: screenDisplays.length,
|
||||
fallbackScreens: fallbackScreenSources.map((source) => ({
|
||||
id: source.id,
|
||||
displayId: source.displayId,
|
||||
bounds: source.bounds,
|
||||
})),
|
||||
});
|
||||
}
|
||||
return [...processedSources, ...fallbackScreenSources];
|
||||
});
|
||||
|
||||
ipcMain.handle("select-source", async (_, source: SelectedSource) => {
|
||||
@@ -1964,7 +2077,7 @@ export function registerIpcHandlers(
|
||||
error: "Windows Graphics Capture requires Windows 10 build 19041 or newer.",
|
||||
};
|
||||
}
|
||||
if (nativeWindowsCaptureProcess) {
|
||||
if (hasActiveNativeWindowsCaptureProcess()) {
|
||||
return { success: false, error: "Native Windows capture is already running." };
|
||||
}
|
||||
|
||||
@@ -2113,6 +2226,7 @@ export function registerIpcHandlers(
|
||||
});
|
||||
|
||||
const source = selectedSource || { name: "Screen" };
|
||||
attachNativeWindowsCaptureLifecycle(proc, source.name, onRecordingStateChange);
|
||||
startGuideHotkeyRecording(recordingId, bounds);
|
||||
if (onRecordingStateChange) {
|
||||
onRecordingStateChange(true, source.name);
|
||||
@@ -2127,17 +2241,7 @@ export function registerIpcHandlers(
|
||||
} catch (error) {
|
||||
console.error("Failed to start native Windows recording:", error);
|
||||
nativeWindowsCaptureProcess?.kill();
|
||||
nativeWindowsCaptureProcess = null;
|
||||
nativeWindowsCaptureTargetPath = null;
|
||||
nativeWindowsCaptureWebcamTargetPath = null;
|
||||
nativeWindowsCaptureRecordingId = null;
|
||||
nativeWindowsCursorOffsetMs = 0;
|
||||
nativeWindowsCursorCaptureMode = "editable-overlay";
|
||||
nativeWindowsCursorRecordingStartMs = 0;
|
||||
nativeWindowsPauseStartedAtMs = null;
|
||||
nativeWindowsPauseRanges = [];
|
||||
nativeWindowsIsPaused = false;
|
||||
clearGuideHotkeyRecording();
|
||||
resetNativeWindowsCaptureState();
|
||||
await stopCursorRecording();
|
||||
return { success: false, error: String(error) };
|
||||
}
|
||||
@@ -2396,11 +2500,13 @@ export function registerIpcHandlers(
|
||||
const recordingId = nativeWindowsCaptureRecordingId ?? Date.now();
|
||||
const cursorCaptureMode = nativeWindowsCursorCaptureMode;
|
||||
|
||||
if (!proc) {
|
||||
if (!proc || proc.exitCode !== null || proc.killed) {
|
||||
resetNativeWindowsCaptureState();
|
||||
return { success: false, error: "Native Windows capture is not running." };
|
||||
}
|
||||
|
||||
try {
|
||||
nativeWindowsCaptureStopping = true;
|
||||
completeNativeWindowsCursorPauseRange();
|
||||
const stoppedPathPromise = waitForNativeWindowsCaptureStop(proc);
|
||||
proc.stdin.write("stop\n");
|
||||
@@ -2462,17 +2568,7 @@ export function registerIpcHandlers(
|
||||
await stopCursorRecording();
|
||||
return { success: false, error: String(error) };
|
||||
} finally {
|
||||
nativeWindowsCaptureProcess = null;
|
||||
nativeWindowsCaptureTargetPath = null;
|
||||
nativeWindowsCaptureWebcamTargetPath = null;
|
||||
nativeWindowsCaptureRecordingId = null;
|
||||
nativeWindowsCursorOffsetMs = 0;
|
||||
nativeWindowsCursorCaptureMode = "editable-overlay";
|
||||
nativeWindowsCursorRecordingStartMs = 0;
|
||||
nativeWindowsPauseStartedAtMs = null;
|
||||
nativeWindowsPauseRanges = [];
|
||||
nativeWindowsIsPaused = false;
|
||||
clearGuideHotkeyRecording();
|
||||
resetNativeWindowsCaptureState();
|
||||
const source = selectedSource || { name: "Screen" };
|
||||
if (onRecordingStateChange) {
|
||||
onRecordingStateChange(false, source.name);
|
||||
@@ -2637,6 +2733,7 @@ export function registerIpcHandlers(
|
||||
);
|
||||
const guideStore = new GuideStore(RECORDINGS_DIR, {
|
||||
deepSeekConfigProvider: guideAiSettingsStore,
|
||||
ocrConfigProvider: guideAiSettingsStore,
|
||||
});
|
||||
registerGuideMarkerHotkey(guideStore);
|
||||
registerGuideIpcHandlers(ipcMain, guideStore, guideAiSettingsStore, {
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
} from "electron";
|
||||
import { mainT, setMainLocale } from "./i18n";
|
||||
import { getSelectedDesktopSource, registerIpcHandlers } from "./ipc/handlers";
|
||||
import { initializeAutoUpdates } from "./updater";
|
||||
import {
|
||||
createCountdownOverlayWindow,
|
||||
createEditorWindow,
|
||||
@@ -515,6 +516,7 @@ app.whenReady().then(async () => {
|
||||
createTray();
|
||||
updateTrayMenu();
|
||||
setupApplicationMenu();
|
||||
initializeAutoUpdates();
|
||||
// Ensure recordings directory exists
|
||||
await ensureRecordingsDir();
|
||||
|
||||
|
||||
@@ -81,3 +81,21 @@ target_compile_options(guide-hotkey-listener PRIVATE /EHsc /W4 /utf-8)
|
||||
target_link_libraries(guide-hotkey-listener PRIVATE
|
||||
user32
|
||||
)
|
||||
|
||||
add_executable(openscreen-ocr-service-wrapper
|
||||
src/ocr-service-wrapper.cpp
|
||||
)
|
||||
|
||||
target_compile_definitions(openscreen-ocr-service-wrapper PRIVATE
|
||||
NOMINMAX
|
||||
WIN32_LEAN_AND_MEAN
|
||||
UNICODE
|
||||
_UNICODE
|
||||
_WIN32_WINNT=0x0A00
|
||||
)
|
||||
|
||||
target_compile_options(openscreen-ocr-service-wrapper PRIVATE /EHsc /W4 /utf-8)
|
||||
|
||||
target_link_libraries(openscreen-ocr-service-wrapper PRIVATE
|
||||
advapi32
|
||||
)
|
||||
|
||||
@@ -0,0 +1,263 @@
|
||||
#include <Windows.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr const wchar_t* SERVICE_NAME = L"OpenScreenOCR";
|
||||
|
||||
struct ServiceConfig {
|
||||
std::wstring exePath;
|
||||
std::wstring resourcesPath;
|
||||
std::wstring dataPath;
|
||||
};
|
||||
|
||||
SERVICE_STATUS_HANDLE g_statusHandle = nullptr;
|
||||
SERVICE_STATUS g_status{};
|
||||
HANDLE g_stopEvent = nullptr;
|
||||
PROCESS_INFORMATION g_childProcess{};
|
||||
ServiceConfig g_config;
|
||||
|
||||
std::wstring quoteArg(const std::wstring& value) {
|
||||
std::wstring result = L"\"";
|
||||
for (wchar_t ch : value) {
|
||||
if (ch == L'"') {
|
||||
result += L"\\\"";
|
||||
} else {
|
||||
result.push_back(ch);
|
||||
}
|
||||
}
|
||||
result += L"\"";
|
||||
return result;
|
||||
}
|
||||
|
||||
std::wstring directoryName(const std::wstring& path) {
|
||||
const size_t slash = path.find_last_of(L"\\/");
|
||||
return slash == std::wstring::npos ? L"." : path.substr(0, slash);
|
||||
}
|
||||
|
||||
void createDirectoryRecursive(const std::wstring& path) {
|
||||
if (path.empty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
std::wstring current;
|
||||
for (size_t i = 0; i < path.size(); ++i) {
|
||||
current.push_back(path[i]);
|
||||
if (path[i] != L'\\' && path[i] != L'/') {
|
||||
continue;
|
||||
}
|
||||
if (current.size() > 3) {
|
||||
CreateDirectoryW(current.c_str(), nullptr);
|
||||
}
|
||||
}
|
||||
CreateDirectoryW(path.c_str(), nullptr);
|
||||
}
|
||||
|
||||
void setEnv(const wchar_t* name, const std::wstring& value) {
|
||||
SetEnvironmentVariableW(name, value.empty() ? nullptr : value.c_str());
|
||||
}
|
||||
|
||||
void setServiceStatus(DWORD state, DWORD win32ExitCode = NO_ERROR, DWORD waitHint = 0) {
|
||||
if (!g_statusHandle) {
|
||||
return;
|
||||
}
|
||||
|
||||
g_status.dwServiceType = SERVICE_WIN32_OWN_PROCESS;
|
||||
g_status.dwCurrentState = state;
|
||||
g_status.dwWin32ExitCode = win32ExitCode;
|
||||
g_status.dwWaitHint = waitHint;
|
||||
g_status.dwControlsAccepted =
|
||||
state == SERVICE_RUNNING ? SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN : 0;
|
||||
static DWORD checkpoint = 1;
|
||||
g_status.dwCheckPoint =
|
||||
state == SERVICE_START_PENDING || state == SERVICE_STOP_PENDING ? checkpoint++ : 0;
|
||||
SetServiceStatus(g_statusHandle, &g_status);
|
||||
}
|
||||
|
||||
HANDLE openServiceLog(const std::wstring& dataPath) {
|
||||
const std::wstring logDir = dataPath + L"\\logs";
|
||||
createDirectoryRecursive(logDir);
|
||||
const std::wstring logPath = logDir + L"\\ocr-service.log";
|
||||
SECURITY_ATTRIBUTES securityAttributes{};
|
||||
securityAttributes.nLength = sizeof(securityAttributes);
|
||||
securityAttributes.bInheritHandle = TRUE;
|
||||
HANDLE file = CreateFileW(
|
||||
logPath.c_str(),
|
||||
FILE_APPEND_DATA,
|
||||
FILE_SHARE_READ | FILE_SHARE_WRITE,
|
||||
&securityAttributes,
|
||||
OPEN_ALWAYS,
|
||||
FILE_ATTRIBUTE_NORMAL,
|
||||
nullptr);
|
||||
if (file != INVALID_HANDLE_VALUE) {
|
||||
SetFilePointer(file, 0, nullptr, FILE_END);
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
bool startOcrProcess(const ServiceConfig& config) {
|
||||
if (config.exePath.empty()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const std::wstring dataPath = config.dataPath.empty()
|
||||
? directoryName(config.exePath) + L"\\ocr-runtime"
|
||||
: config.dataPath;
|
||||
const std::wstring resourcesPath = config.resourcesPath.empty()
|
||||
? directoryName(directoryName(config.exePath))
|
||||
: config.resourcesPath;
|
||||
const std::wstring modelCachePath = dataPath + L"\\ocr-models";
|
||||
const std::wstring paddlexCachePath = resourcesPath + L"\\ocr-models\\paddlex";
|
||||
|
||||
createDirectoryRecursive(dataPath);
|
||||
createDirectoryRecursive(modelCachePath);
|
||||
|
||||
setEnv(L"OPENSCREEN_OCR_HOST", L"127.0.0.1");
|
||||
setEnv(L"OPENSCREEN_OCR_PORT", L"8866");
|
||||
setEnv(L"PADDLEOCR_DEVICE", L"cpu");
|
||||
setEnv(L"PADDLEOCR_ENABLE_MKLDNN", L"0");
|
||||
setEnv(L"PADDLEOCR_LANG", L"");
|
||||
setEnv(L"PADDLEOCR_USE_MOBILE", L"1");
|
||||
setEnv(L"OPENSCREEN_OCR_PROFILE", L"vietnamese");
|
||||
setEnv(L"OPENSCREEN_OCR_WARMUP", L"1");
|
||||
setEnv(L"PADDLE_PDX_ENABLE_MKLDNN_BYDEFAULT", L"False");
|
||||
setEnv(L"PADDLE_PDX_CACHE_HOME", paddlexCachePath);
|
||||
setEnv(L"PADDLE_PDX_DISABLE_MODEL_SOURCE_CHECK", L"True");
|
||||
setEnv(L"PADDLE_HOME", modelCachePath + L"\\paddle");
|
||||
setEnv(L"PADDLEOCR_HOME", modelCachePath + L"\\paddleocr");
|
||||
setEnv(L"PYTHONUTF8", L"1");
|
||||
|
||||
STARTUPINFOW startupInfo{};
|
||||
startupInfo.cb = sizeof(startupInfo);
|
||||
HANDLE logFile = openServiceLog(dataPath);
|
||||
if (logFile != INVALID_HANDLE_VALUE) {
|
||||
startupInfo.dwFlags |= STARTF_USESTDHANDLES;
|
||||
startupInfo.hStdOutput = logFile;
|
||||
startupInfo.hStdError = logFile;
|
||||
startupInfo.hStdInput = GetStdHandle(STD_INPUT_HANDLE);
|
||||
}
|
||||
|
||||
std::wstring commandLine = quoteArg(config.exePath);
|
||||
const std::wstring cwd = directoryName(config.exePath);
|
||||
ZeroMemory(&g_childProcess, sizeof(g_childProcess));
|
||||
const BOOL created = CreateProcessW(
|
||||
config.exePath.c_str(),
|
||||
commandLine.data(),
|
||||
nullptr,
|
||||
nullptr,
|
||||
TRUE,
|
||||
CREATE_NO_WINDOW,
|
||||
nullptr,
|
||||
cwd.c_str(),
|
||||
&startupInfo,
|
||||
&g_childProcess);
|
||||
|
||||
if (logFile != INVALID_HANDLE_VALUE) {
|
||||
CloseHandle(logFile);
|
||||
}
|
||||
return created == TRUE;
|
||||
}
|
||||
|
||||
void stopOcrProcess() {
|
||||
if (g_childProcess.hProcess) {
|
||||
TerminateProcess(g_childProcess.hProcess, 0);
|
||||
WaitForSingleObject(g_childProcess.hProcess, 10000);
|
||||
CloseHandle(g_childProcess.hProcess);
|
||||
g_childProcess.hProcess = nullptr;
|
||||
}
|
||||
if (g_childProcess.hThread) {
|
||||
CloseHandle(g_childProcess.hThread);
|
||||
g_childProcess.hThread = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
DWORD WINAPI serviceControlHandler(DWORD control, DWORD, LPVOID, LPVOID) {
|
||||
if (control == SERVICE_CONTROL_STOP || control == SERVICE_CONTROL_SHUTDOWN) {
|
||||
setServiceStatus(SERVICE_STOP_PENDING, NO_ERROR, 10000);
|
||||
if (g_stopEvent) {
|
||||
SetEvent(g_stopEvent);
|
||||
}
|
||||
stopOcrProcess();
|
||||
return NO_ERROR;
|
||||
}
|
||||
return NO_ERROR;
|
||||
}
|
||||
|
||||
void WINAPI serviceMain(DWORD, LPWSTR*) {
|
||||
g_statusHandle = RegisterServiceCtrlHandlerExW(SERVICE_NAME, serviceControlHandler, nullptr);
|
||||
if (!g_statusHandle) {
|
||||
return;
|
||||
}
|
||||
|
||||
setServiceStatus(SERVICE_START_PENDING, NO_ERROR, 30000);
|
||||
g_stopEvent = CreateEventW(nullptr, TRUE, FALSE, nullptr);
|
||||
if (!g_stopEvent || !startOcrProcess(g_config)) {
|
||||
setServiceStatus(SERVICE_STOPPED, ERROR_SERVICE_SPECIFIC_ERROR);
|
||||
return;
|
||||
}
|
||||
|
||||
setServiceStatus(SERVICE_RUNNING);
|
||||
HANDLE waitHandles[] = {g_stopEvent, g_childProcess.hProcess};
|
||||
WaitForMultipleObjects(2, waitHandles, FALSE, INFINITE);
|
||||
stopOcrProcess();
|
||||
if (g_stopEvent) {
|
||||
CloseHandle(g_stopEvent);
|
||||
g_stopEvent = nullptr;
|
||||
}
|
||||
setServiceStatus(SERVICE_STOPPED);
|
||||
}
|
||||
|
||||
ServiceConfig parseConfig(int argc, wchar_t* argv[]) {
|
||||
ServiceConfig config;
|
||||
for (int i = 1; i < argc; ++i) {
|
||||
const std::wstring arg = argv[i];
|
||||
auto readNext = [&](std::wstring& target) {
|
||||
if (i + 1 < argc) {
|
||||
target = argv[++i];
|
||||
}
|
||||
};
|
||||
if (arg == L"--exe") {
|
||||
readNext(config.exePath);
|
||||
} else if (arg == L"--resources") {
|
||||
readNext(config.resourcesPath);
|
||||
} else if (arg == L"--data") {
|
||||
readNext(config.dataPath);
|
||||
}
|
||||
}
|
||||
return config;
|
||||
}
|
||||
|
||||
bool hasServiceFlag(int argc, wchar_t* argv[]) {
|
||||
for (int i = 1; i < argc; ++i) {
|
||||
if (std::wstring(argv[i]) == L"--service") {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int wmain(int argc, wchar_t* argv[]) {
|
||||
g_config = parseConfig(argc, argv);
|
||||
|
||||
if (hasServiceFlag(argc, argv)) {
|
||||
SERVICE_TABLE_ENTRYW serviceTable[] = {
|
||||
{const_cast<LPWSTR>(SERVICE_NAME), serviceMain},
|
||||
{nullptr, nullptr},
|
||||
};
|
||||
return StartServiceCtrlDispatcherW(serviceTable) ? 0 : 1;
|
||||
}
|
||||
|
||||
if (!startOcrProcess(g_config)) {
|
||||
std::wcerr << L"Failed to start OCR service process." << std::endl;
|
||||
return 1;
|
||||
}
|
||||
WaitForSingleObject(g_childProcess.hProcess, INFINITE);
|
||||
stopOcrProcess();
|
||||
return 0;
|
||||
}
|
||||
@@ -28,6 +28,60 @@ bool succeeded(HRESULT hr, const char* label) {
|
||||
return false;
|
||||
}
|
||||
|
||||
Microsoft::WRL::ComPtr<IDXGIAdapter1> findAdapterForMonitor(HMONITOR monitor) {
|
||||
if (!monitor) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
Microsoft::WRL::ComPtr<IDXGIFactory1> factory;
|
||||
HRESULT hr = CreateDXGIFactory1(IID_PPV_ARGS(&factory));
|
||||
if (FAILED(hr) || !factory) {
|
||||
std::cerr << "WARNING: CreateDXGIFactory1 failed while resolving monitor adapter (hr=0x"
|
||||
<< std::hex << hr << std::dec << ")" << std::endl;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
for (UINT adapterIndex = 0;; ++adapterIndex) {
|
||||
Microsoft::WRL::ComPtr<IDXGIAdapter1> adapter;
|
||||
hr = factory->EnumAdapters1(adapterIndex, adapter.GetAddressOf());
|
||||
if (hr == DXGI_ERROR_NOT_FOUND) {
|
||||
break;
|
||||
}
|
||||
if (FAILED(hr) || !adapter) {
|
||||
continue;
|
||||
}
|
||||
|
||||
DXGI_ADAPTER_DESC1 adapterDesc{};
|
||||
if (SUCCEEDED(adapter->GetDesc1(&adapterDesc)) &&
|
||||
(adapterDesc.Flags & DXGI_ADAPTER_FLAG_SOFTWARE) != 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (UINT outputIndex = 0;; ++outputIndex) {
|
||||
Microsoft::WRL::ComPtr<IDXGIOutput> output;
|
||||
hr = adapter->EnumOutputs(outputIndex, output.GetAddressOf());
|
||||
if (hr == DXGI_ERROR_NOT_FOUND) {
|
||||
break;
|
||||
}
|
||||
if (FAILED(hr) || !output) {
|
||||
continue;
|
||||
}
|
||||
|
||||
DXGI_OUTPUT_DESC outputDesc{};
|
||||
if (SUCCEEDED(output->GetDesc(&outputDesc)) && outputDesc.Monitor == monitor) {
|
||||
std::cout << "{\"event\":\"display-adapter-resolved\",\"schemaVersion\":2,"
|
||||
<< "\"adapterIndex\":" << adapterIndex
|
||||
<< ",\"outputIndex\":" << outputIndex << "}" << std::endl;
|
||||
return adapter;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::cerr << "WARNING: Could not resolve DXGI adapter for selected monitor; using default adapter"
|
||||
<< std::endl;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
int64_t timeSpanToHns(wf::TimeSpan const& value) {
|
||||
return value.count();
|
||||
}
|
||||
@@ -38,7 +92,7 @@ WgcSession::~WgcSession() {
|
||||
stop();
|
||||
}
|
||||
|
||||
bool WgcSession::createD3DDevice() {
|
||||
bool WgcSession::createD3DDevice(IDXGIAdapter* adapter) {
|
||||
UINT flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT;
|
||||
#if defined(_DEBUG)
|
||||
flags |= D3D11_CREATE_DEVICE_DEBUG;
|
||||
@@ -53,8 +107,8 @@ bool WgcSession::createD3DDevice() {
|
||||
D3D_FEATURE_LEVEL featureLevel{};
|
||||
|
||||
HRESULT hr = D3D11CreateDevice(
|
||||
nullptr,
|
||||
D3D_DRIVER_TYPE_HARDWARE,
|
||||
adapter,
|
||||
adapter ? D3D_DRIVER_TYPE_UNKNOWN : D3D_DRIVER_TYPE_HARDWARE,
|
||||
nullptr,
|
||||
flags,
|
||||
featureLevels,
|
||||
@@ -67,6 +121,23 @@ bool WgcSession::createD3DDevice() {
|
||||
#if defined(_DEBUG)
|
||||
if (FAILED(hr)) {
|
||||
flags &= ~D3D11_CREATE_DEVICE_DEBUG;
|
||||
hr = D3D11CreateDevice(
|
||||
adapter,
|
||||
adapter ? D3D_DRIVER_TYPE_UNKNOWN : D3D_DRIVER_TYPE_HARDWARE,
|
||||
nullptr,
|
||||
flags,
|
||||
featureLevels,
|
||||
ARRAYSIZE(featureLevels),
|
||||
D3D11_SDK_VERSION,
|
||||
&d3dDevice_,
|
||||
&featureLevel,
|
||||
&d3dContext_);
|
||||
}
|
||||
#endif
|
||||
|
||||
if (FAILED(hr) && adapter) {
|
||||
std::cerr << "WARNING: D3D11CreateDevice failed for selected monitor adapter (hr=0x"
|
||||
<< std::hex << hr << std::dec << "); retrying default adapter" << std::endl;
|
||||
hr = D3D11CreateDevice(
|
||||
nullptr,
|
||||
D3D_DRIVER_TYPE_HARDWARE,
|
||||
@@ -79,7 +150,6 @@ bool WgcSession::createD3DDevice() {
|
||||
&featureLevel,
|
||||
&d3dContext_);
|
||||
}
|
||||
#endif
|
||||
|
||||
if (!succeeded(hr, "D3D11CreateDevice")) {
|
||||
return false;
|
||||
@@ -100,6 +170,11 @@ bool WgcSession::createD3DDevice() {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool WgcSession::createD3DDeviceForMonitor(HMONITOR monitor) {
|
||||
auto adapter = findAdapterForMonitor(monitor);
|
||||
return createD3DDevice(adapter.Get());
|
||||
}
|
||||
|
||||
bool WgcSession::createCaptureItem(HMONITOR monitor) {
|
||||
auto factory = winrt::get_activation_factory<wgcap::GraphicsCaptureItem>();
|
||||
auto interop = factory.as<IGraphicsCaptureItemInterop>();
|
||||
@@ -188,7 +263,7 @@ bool WgcSession::applySessionOptions(bool captureCursor) {
|
||||
|
||||
bool WgcSession::initialize(HMONITOR monitor, int fps, bool captureCursor) {
|
||||
fps_ = fps > 0 ? fps : 60;
|
||||
if (!createD3DDevice()) {
|
||||
if (!createD3DDeviceForMonitor(monitor)) {
|
||||
return false;
|
||||
}
|
||||
if (!createCaptureItem(monitor)) {
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
#include <Windows.h>
|
||||
#include <d3d11.h>
|
||||
#include <dxgi.h>
|
||||
#include <windows.graphics.capture.h>
|
||||
#include <windows.graphics.directx.direct3d11.interop.h>
|
||||
#include <winrt/Windows.Foundation.h>
|
||||
@@ -34,7 +35,8 @@ public:
|
||||
ID3D11DeviceContext* context() const;
|
||||
|
||||
private:
|
||||
bool createD3DDevice();
|
||||
bool createD3DDevice(IDXGIAdapter* adapter = nullptr);
|
||||
bool createD3DDeviceForMonitor(HMONITOR monitor);
|
||||
bool createCaptureItem(HMONITOR monitor);
|
||||
bool createCaptureItem(HWND window);
|
||||
bool applySessionOptions(bool captureCursor);
|
||||
|
||||
@@ -26,6 +26,27 @@ const assetBaseUrl = assetBaseUrlArg ? assetBaseUrlArg.slice(ASSET_BASE_URL_ARG_
|
||||
|
||||
contextBridge.exposeInMainWorld("electronAPI", {
|
||||
assetBaseUrl,
|
||||
updates: {
|
||||
getStatus: () => {
|
||||
return ipcRenderer.invoke("updates:get-status");
|
||||
},
|
||||
check: () => {
|
||||
return ipcRenderer.invoke("updates:check");
|
||||
},
|
||||
install: () => {
|
||||
return ipcRenderer.invoke("updates:install");
|
||||
},
|
||||
onStatus: (callback: (status: import("../src/lib/updateStatus").UpdateStatus) => void) => {
|
||||
const listener = (
|
||||
_event: Electron.IpcRendererEvent,
|
||||
status: import("../src/lib/updateStatus").UpdateStatus,
|
||||
) => {
|
||||
callback(status);
|
||||
};
|
||||
ipcRenderer.on("updates:status", listener);
|
||||
return () => ipcRenderer.removeListener("updates:status", listener);
|
||||
},
|
||||
},
|
||||
invokeNativeBridge: <TData>(request: NativeBridgeRequest) => {
|
||||
return ipcRenderer.invoke(NATIVE_BRIDGE_CHANNEL, request) as Promise<TData>;
|
||||
},
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
import { app, BrowserWindow, ipcMain } from "electron";
|
||||
import { autoUpdater, type ProgressInfo, type UpdateInfo } from "electron-updater";
|
||||
import type { UpdateCheckResult, UpdateStatus } from "../src/lib/updateStatus";
|
||||
|
||||
const DEFAULT_UPDATE_FEED_URL =
|
||||
"https://gittea.softs.business/huanld/openscreen/raw/branch/release-assets%2Flatest";
|
||||
const AUTO_CHECK_DELAY_MS = 10_000;
|
||||
const AUTO_CHECK_INTERVAL_MS = 4 * 60 * 60 * 1000;
|
||||
|
||||
let status: UpdateStatus = createStatus("idle");
|
||||
let handlersRegistered = false;
|
||||
let initialized = false;
|
||||
let checkInFlight: Promise<UpdateCheckResult> | null = null;
|
||||
|
||||
function createStatus(
|
||||
phase: UpdateStatus["phase"],
|
||||
patch: Partial<UpdateStatus> = {},
|
||||
): UpdateStatus {
|
||||
return {
|
||||
phase,
|
||||
currentVersion: app.getVersion(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
...patch,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeReleaseNotes(releaseNotes: UpdateInfo["releaseNotes"]): string | undefined {
|
||||
if (typeof releaseNotes === "string") {
|
||||
return releaseNotes;
|
||||
}
|
||||
if (Array.isArray(releaseNotes)) {
|
||||
return releaseNotes
|
||||
.map((note) => note.note)
|
||||
.filter(Boolean)
|
||||
.join("\n\n");
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function updateStatus(next: UpdateStatus) {
|
||||
status = next;
|
||||
for (const window of BrowserWindow.getAllWindows()) {
|
||||
if (!window.isDestroyed()) {
|
||||
window.webContents.send("updates:status", status);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function statusFromInfo(phase: UpdateStatus["phase"], info: UpdateInfo): UpdateStatus {
|
||||
return createStatus(phase, {
|
||||
version: info.version,
|
||||
releaseName: info.releaseName ?? undefined,
|
||||
releaseNotes: normalizeReleaseNotes(info.releaseNotes),
|
||||
});
|
||||
}
|
||||
|
||||
async function checkForUpdates(): Promise<UpdateCheckResult> {
|
||||
if (!initialized) {
|
||||
updateStatus(
|
||||
createStatus("unsupported", {
|
||||
error: "Update service is not initialized.",
|
||||
}),
|
||||
);
|
||||
return { success: false, status, error: status.error };
|
||||
}
|
||||
|
||||
if (!app.isPackaged && process.env.OPENSCREEN_ALLOW_DEV_UPDATE_CHECK !== "1") {
|
||||
updateStatus(
|
||||
createStatus("unsupported", {
|
||||
error: "Update checks only run in packaged builds.",
|
||||
}),
|
||||
);
|
||||
return { success: false, status, error: status.error };
|
||||
}
|
||||
|
||||
if (checkInFlight) {
|
||||
return checkInFlight;
|
||||
}
|
||||
|
||||
updateStatus(createStatus("checking"));
|
||||
checkInFlight = autoUpdater
|
||||
.checkForUpdates()
|
||||
.then(() => ({ success: true, status }))
|
||||
.catch((error) => {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
updateStatus(createStatus("error", { error: message }));
|
||||
return { success: false, status, error: message };
|
||||
})
|
||||
.finally(() => {
|
||||
checkInFlight = null;
|
||||
});
|
||||
|
||||
return checkInFlight;
|
||||
}
|
||||
|
||||
function registerUpdateIpcHandlers() {
|
||||
if (handlersRegistered) {
|
||||
return;
|
||||
}
|
||||
|
||||
handlersRegistered = true;
|
||||
ipcMain.handle("updates:get-status", () => status);
|
||||
ipcMain.handle("updates:check", () => checkForUpdates());
|
||||
ipcMain.handle("updates:install", () => {
|
||||
if (status.phase !== "downloaded") {
|
||||
return {
|
||||
success: false,
|
||||
status,
|
||||
error: "No downloaded update is ready to install.",
|
||||
};
|
||||
}
|
||||
setImmediate(() => autoUpdater.quitAndInstall(false, true));
|
||||
return { success: true, status };
|
||||
});
|
||||
}
|
||||
|
||||
export function initializeAutoUpdates() {
|
||||
registerUpdateIpcHandlers();
|
||||
|
||||
if (initialized) {
|
||||
return;
|
||||
}
|
||||
|
||||
initialized = true;
|
||||
autoUpdater.autoDownload = true;
|
||||
autoUpdater.autoInstallOnAppQuit = true;
|
||||
autoUpdater.logger = console;
|
||||
|
||||
const feedUrl = process.env.OPENSCREEN_UPDATE_FEED_URL?.trim() || DEFAULT_UPDATE_FEED_URL;
|
||||
const updateToken = process.env.OPENSCREEN_UPDATE_TOKEN?.trim();
|
||||
if (updateToken) {
|
||||
autoUpdater.requestHeaders = {
|
||||
Authorization: `token ${updateToken}`,
|
||||
};
|
||||
}
|
||||
autoUpdater.setFeedURL({
|
||||
provider: "generic",
|
||||
url: feedUrl,
|
||||
});
|
||||
|
||||
autoUpdater.on("checking-for-update", () => {
|
||||
updateStatus(createStatus("checking"));
|
||||
});
|
||||
autoUpdater.on("update-available", (info) => {
|
||||
updateStatus(statusFromInfo("available", info));
|
||||
});
|
||||
autoUpdater.on("update-not-available", (info) => {
|
||||
updateStatus(statusFromInfo("not-available", info));
|
||||
});
|
||||
autoUpdater.on("download-progress", (progress: ProgressInfo) => {
|
||||
updateStatus(
|
||||
createStatus("downloading", {
|
||||
version: status.version,
|
||||
releaseName: status.releaseName,
|
||||
releaseNotes: status.releaseNotes,
|
||||
percent: progress.percent,
|
||||
bytesPerSecond: progress.bytesPerSecond,
|
||||
transferred: progress.transferred,
|
||||
total: progress.total,
|
||||
}),
|
||||
);
|
||||
});
|
||||
autoUpdater.on("update-downloaded", (info) => {
|
||||
updateStatus(statusFromInfo("downloaded", info));
|
||||
});
|
||||
autoUpdater.on("error", (error) => {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
updateStatus(createStatus("error", { error: message }));
|
||||
});
|
||||
|
||||
setTimeout(() => {
|
||||
void checkForUpdates();
|
||||
}, AUTO_CHECK_DELAY_MS);
|
||||
setInterval(() => {
|
||||
void checkForUpdates();
|
||||
}, AUTO_CHECK_INTERVAL_MS).unref();
|
||||
}
|
||||
Generated
+85
-10
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "openscreen",
|
||||
"version": "1.4.2",
|
||||
"version": "1.4.12",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "openscreen",
|
||||
"version": "1.4.2",
|
||||
"version": "1.4.12",
|
||||
"dependencies": {
|
||||
"@fix-webm-duration/fix": "^1.0.1",
|
||||
"@pixi/filter-drop-shadow": "^5.2.0",
|
||||
@@ -29,6 +29,7 @@
|
||||
"class-variance-authority": "^0.7.1",
|
||||
"clsx": "^2.1.1",
|
||||
"dnd-timeline": "^2.4.0",
|
||||
"electron-updater": "^6.8.3",
|
||||
"emoji-picker-react": "^4.18.0",
|
||||
"fix-webm-duration": "^1.0.6",
|
||||
"gif.js": "^0.2.0",
|
||||
@@ -4625,7 +4626,6 @@
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
|
||||
"integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
|
||||
"dev": true,
|
||||
"license": "Python-2.0"
|
||||
},
|
||||
"node_modules/aria-hidden": {
|
||||
@@ -4959,7 +4959,6 @@
|
||||
"version": "9.5.1",
|
||||
"resolved": "https://registry.npmjs.org/builder-util-runtime/-/builder-util-runtime-9.5.1.tgz",
|
||||
"integrity": "sha512-qt41tMfgHTllhResqM5DcnHyDIWNgzHvuY2jDcYP9iaGpkWxTUzV6GQjDeLnlR1/DtdlcsWQbA7sByMpmJFTLQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"debug": "^4.3.4",
|
||||
@@ -5504,7 +5503,6 @@
|
||||
"version": "4.4.3",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
|
||||
"integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ms": "^2.1.3"
|
||||
@@ -6015,6 +6013,69 @@
|
||||
"dev": true,
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/electron-updater": {
|
||||
"version": "6.8.3",
|
||||
"resolved": "https://registry.npmjs.org/electron-updater/-/electron-updater-6.8.3.tgz",
|
||||
"integrity": "sha512-Z6sgw3jgbikWKXei1ENdqFOxBP0WlXg3TtKfz0rgw2vIZFJUyI4pD7ZN7jrkm7EoMK+tcm/qTnPUdqfZukBlBQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"builder-util-runtime": "9.5.1",
|
||||
"fs-extra": "^10.1.0",
|
||||
"js-yaml": "^4.1.0",
|
||||
"lazy-val": "^1.0.5",
|
||||
"lodash.escaperegexp": "^4.1.2",
|
||||
"lodash.isequal": "^4.5.0",
|
||||
"semver": "~7.7.3",
|
||||
"tiny-typed-emitter": "^2.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/electron-updater/node_modules/fs-extra": {
|
||||
"version": "10.1.0",
|
||||
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz",
|
||||
"integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"graceful-fs": "^4.2.0",
|
||||
"jsonfile": "^6.0.1",
|
||||
"universalify": "^2.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/electron-updater/node_modules/jsonfile": {
|
||||
"version": "6.2.1",
|
||||
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.1.tgz",
|
||||
"integrity": "sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"universalify": "^2.0.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"graceful-fs": "^4.1.6"
|
||||
}
|
||||
},
|
||||
"node_modules/electron-updater/node_modules/semver": {
|
||||
"version": "7.7.4",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz",
|
||||
"integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==",
|
||||
"license": "ISC",
|
||||
"bin": {
|
||||
"semver": "bin/semver.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/electron-updater/node_modules/universalify": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz",
|
||||
"integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/electron-winstaller": {
|
||||
"version": "5.4.0",
|
||||
"resolved": "https://registry.npmjs.org/electron-winstaller/-/electron-winstaller-5.4.0.tgz",
|
||||
@@ -6874,7 +6935,6 @@
|
||||
"version": "4.2.11",
|
||||
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
|
||||
"integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==",
|
||||
"dev": true,
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/gsap": {
|
||||
@@ -7287,7 +7347,6 @@
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz",
|
||||
"integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"argparse": "^2.0.1"
|
||||
@@ -7419,7 +7478,6 @@
|
||||
"version": "1.0.5",
|
||||
"resolved": "https://registry.npmjs.org/lazy-val/-/lazy-val-1.0.5.tgz",
|
||||
"integrity": "sha512-0/BnGCCfyUMkBpeDgWihanIAF9JmZhHBgUhEqzvf+adhNGLoP6TaiI5oF8oyb3I45P+PcnrqihSf01M0l0G5+Q==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lilconfig": {
|
||||
@@ -7586,6 +7644,19 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lodash.escaperegexp": {
|
||||
"version": "4.1.2",
|
||||
"resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz",
|
||||
"integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lodash.isequal": {
|
||||
"version": "4.5.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz",
|
||||
"integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==",
|
||||
"deprecated": "This package is deprecated. Use require('node:util').isDeepStrictEqual instead.",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/log-update": {
|
||||
"version": "6.1.0",
|
||||
"resolved": "https://registry.npmjs.org/log-update/-/log-update-6.1.0.tgz",
|
||||
@@ -7991,7 +8062,6 @@
|
||||
"version": "2.1.3",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
|
||||
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/mz": {
|
||||
@@ -9394,7 +9464,6 @@
|
||||
"version": "1.6.0",
|
||||
"resolved": "https://registry.npmjs.org/sax/-/sax-1.6.0.tgz",
|
||||
"integrity": "sha512-6R3J5M4AcbtLUdZmRv2SygeVaM7IhrLXu9BmnOGmmACak8fiUtOsYNWUS4uK7upbmHIBbLBeFeI//477BKLBzA==",
|
||||
"dev": true,
|
||||
"license": "BlueOak-1.0.0",
|
||||
"engines": {
|
||||
"node": ">=11.0.0"
|
||||
@@ -10099,6 +10168,12 @@
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/tiny-typed-emitter": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/tiny-typed-emitter/-/tiny-typed-emitter-2.1.0.tgz",
|
||||
"integrity": "sha512-qVtvMxeXbVej0cQWKqVSSAHmKZEHAvxdF8HEUBFWts8h+xEo5m/lEiPakuyZ3BnCBjOD8i24kzNOiOLLgsSxhA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/tinybench": {
|
||||
"version": "2.9.0",
|
||||
"resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz",
|
||||
|
||||
+2
-1
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "openscreen",
|
||||
"private": true,
|
||||
"version": "1.4.2",
|
||||
"version": "1.4.12",
|
||||
"type": "module",
|
||||
"packageManager": "npm@10.9.4",
|
||||
"engines": {
|
||||
@@ -70,6 +70,7 @@
|
||||
"class-variance-authority": "^0.7.1",
|
||||
"clsx": "^2.1.1",
|
||||
"dnd-timeline": "^2.4.0",
|
||||
"electron-updater": "^6.8.3",
|
||||
"emoji-picker-react": "^4.18.0",
|
||||
"fix-webm-duration": "^1.0.6",
|
||||
"gif.js": "^0.2.0",
|
||||
|
||||
@@ -131,6 +131,11 @@ if (!fs.existsSync(guideHotkeyListenerOutputPath)) {
|
||||
throw new Error(`WGC helper build completed but ${guideHotkeyListenerOutputPath} was not found.`);
|
||||
}
|
||||
|
||||
const ocrServiceWrapperOutputPath = path.join(BUILD_DIR, "openscreen-ocr-service-wrapper.exe");
|
||||
if (!fs.existsSync(ocrServiceWrapperOutputPath)) {
|
||||
throw new Error(`WGC helper build completed but ${ocrServiceWrapperOutputPath} was not found.`);
|
||||
}
|
||||
|
||||
fs.mkdirSync(BIN_DIR, { recursive: true });
|
||||
const distributablePath = path.join(BIN_DIR, "wgc-capture.exe");
|
||||
fs.copyFileSync(outputPath, distributablePath);
|
||||
@@ -141,9 +146,14 @@ fs.copyFileSync(cursorSamplerOutputPath, cursorSamplerDistributablePath);
|
||||
const guideHotkeyListenerDistributablePath = path.join(BIN_DIR, "guide-hotkey-listener.exe");
|
||||
fs.copyFileSync(guideHotkeyListenerOutputPath, guideHotkeyListenerDistributablePath);
|
||||
|
||||
const ocrServiceWrapperDistributablePath = path.join(BIN_DIR, "openscreen-ocr-service-wrapper.exe");
|
||||
fs.copyFileSync(ocrServiceWrapperOutputPath, ocrServiceWrapperDistributablePath);
|
||||
|
||||
console.log(`Built ${outputPath}`);
|
||||
console.log(`Copied ${distributablePath}`);
|
||||
console.log(`Built ${cursorSamplerOutputPath}`);
|
||||
console.log(`Copied ${cursorSamplerDistributablePath}`);
|
||||
console.log(`Built ${guideHotkeyListenerOutputPath}`);
|
||||
console.log(`Copied ${guideHotkeyListenerDistributablePath}`);
|
||||
console.log(`Built ${ocrServiceWrapperOutputPath}`);
|
||||
console.log(`Copied ${ocrServiceWrapperDistributablePath}`);
|
||||
|
||||
+70
-1
@@ -1,4 +1,5 @@
|
||||
import { lazy, Suspense, useEffect, useState } from "react";
|
||||
import { lazy, Suspense, useEffect, useRef, useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
import { CountdownOverlay } from "./components/launch/CountdownOverlay.tsx";
|
||||
import { LaunchWindow } from "./components/launch/LaunchWindow";
|
||||
import { SourceSelector } from "./components/launch/SourceSelector";
|
||||
@@ -6,6 +7,7 @@ import { Toaster } from "./components/ui/sonner";
|
||||
import { TooltipProvider } from "./components/ui/tooltip";
|
||||
import { ShortcutsProvider } from "./contexts/ShortcutsContext";
|
||||
import { loadAllCustomFonts } from "./lib/customFonts";
|
||||
import type { UpdateStatus } from "./lib/updateStatus";
|
||||
|
||||
const VideoEditor = lazy(() => import("./components/video-editor/VideoEditor"));
|
||||
const ShortcutsConfigDialog = lazy(() =>
|
||||
@@ -79,11 +81,78 @@ export default function App() {
|
||||
return (
|
||||
<TooltipProvider>
|
||||
{content}
|
||||
<UpdateNotifier
|
||||
enabled={
|
||||
hasElectronBridge &&
|
||||
windowType !== "hud-overlay" &&
|
||||
windowType !== "source-selector" &&
|
||||
windowType !== "countdown-overlay"
|
||||
}
|
||||
/>
|
||||
<Toaster theme="dark" className="pointer-events-auto" />
|
||||
</TooltipProvider>
|
||||
);
|
||||
}
|
||||
|
||||
function UpdateNotifier({ enabled }: { enabled: boolean }) {
|
||||
const lastPhaseRef = useRef<UpdateStatus["phase"]>("idle");
|
||||
|
||||
useEffect(() => {
|
||||
if (!enabled || !window.electronAPI?.updates) {
|
||||
return;
|
||||
}
|
||||
|
||||
const applyStatus = (status: UpdateStatus) => {
|
||||
const version = status.version ? ` ${status.version}` : "";
|
||||
if (status.phase === "available") {
|
||||
toast.loading(`Downloading OpenScreen${version} update...`, {
|
||||
id: "openscreen-update",
|
||||
duration: Number.POSITIVE_INFINITY,
|
||||
});
|
||||
} else if (status.phase === "downloading") {
|
||||
const percent = typeof status.percent === "number" ? ` ${Math.round(status.percent)}%` : "";
|
||||
toast.loading(`Downloading OpenScreen${version} update${percent}...`, {
|
||||
id: "openscreen-update",
|
||||
duration: Number.POSITIVE_INFINITY,
|
||||
});
|
||||
} else if (status.phase === "downloaded") {
|
||||
toast.success(`OpenScreen${version} is ready to install.`, {
|
||||
id: "openscreen-update",
|
||||
duration: Number.POSITIVE_INFINITY,
|
||||
action: {
|
||||
label: "Restart",
|
||||
onClick: () => {
|
||||
void window.electronAPI.updates.install();
|
||||
},
|
||||
},
|
||||
});
|
||||
} else if (
|
||||
status.phase === "error" &&
|
||||
(lastPhaseRef.current === "available" ||
|
||||
lastPhaseRef.current === "downloading" ||
|
||||
lastPhaseRef.current === "downloaded")
|
||||
) {
|
||||
toast.error(status.error || "OpenScreen update failed.", {
|
||||
id: "openscreen-update",
|
||||
});
|
||||
} else if (status.phase === "not-available" || status.phase === "unsupported") {
|
||||
toast.dismiss("openscreen-update");
|
||||
}
|
||||
lastPhaseRef.current = status.phase;
|
||||
};
|
||||
|
||||
const unsubscribe = window.electronAPI.updates.onStatus(applyStatus);
|
||||
void window.electronAPI.updates
|
||||
.getStatus()
|
||||
.then(applyStatus)
|
||||
.catch(() => undefined);
|
||||
|
||||
return unsubscribe;
|
||||
}, [enabled]);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function BrowserDevFallback() {
|
||||
return (
|
||||
<div className="flex h-screen w-screen items-center justify-center bg-[#08090b] px-6 text-slate-100">
|
||||
|
||||
@@ -65,7 +65,13 @@ export function SourceSelector() {
|
||||
fetchSources();
|
||||
}, []);
|
||||
|
||||
const screenSources = sources.filter((s) => s.id.startsWith("screen:"));
|
||||
const screenSources = sources
|
||||
.filter((s) => s.id.startsWith("screen:"))
|
||||
.sort(
|
||||
(left, right) =>
|
||||
(left.displayIndex ?? left.screenIndex ?? Number.MAX_SAFE_INTEGER) -
|
||||
(right.displayIndex ?? right.screenIndex ?? Number.MAX_SAFE_INTEGER),
|
||||
);
|
||||
const windowSources = sources.filter((s) => s.id.startsWith("window:"));
|
||||
|
||||
const handleSourceSelect = (source: DesktopSource) => setSelectedSource(source);
|
||||
@@ -96,11 +102,17 @@ export function SourceSelector() {
|
||||
onClick={() => handleSourceSelect(source)}
|
||||
>
|
||||
<div className="relative mb-1.5 overflow-hidden rounded-lg border border-white/[0.06] bg-black/30">
|
||||
<img
|
||||
src={source.thumbnail || ""}
|
||||
alt={source.name}
|
||||
className="w-full aspect-video object-cover"
|
||||
/>
|
||||
{source.thumbnail ? (
|
||||
<img
|
||||
src={source.thumbnail}
|
||||
alt={source.name}
|
||||
className="w-full aspect-video object-cover"
|
||||
/>
|
||||
) : (
|
||||
<div className="flex aspect-video w-full items-center justify-center bg-zinc-950 text-center text-[11px] font-medium text-zinc-400">
|
||||
{source.displayLabel ?? source.name}
|
||||
</div>
|
||||
)}
|
||||
{isSelected && (
|
||||
<div className="absolute right-1.5 top-1.5">
|
||||
<div className={styles.checkBadge}>
|
||||
|
||||
@@ -82,6 +82,7 @@ export function AnnotationOverlay({
|
||||
);
|
||||
const [livePointerPoint, setLivePointerPoint] = useState<{ x: number; y: number } | null>(null);
|
||||
const mosaicCanvasRef = useRef<HTMLCanvasElement | null>(null);
|
||||
const magnifierCanvasRef = useRef<HTMLCanvasElement | null>(null);
|
||||
const blurType = "mosaic";
|
||||
const blurOverlayColor =
|
||||
annotation.type === "blur" ? getBlurOverlayColor(annotation.blurData) : "";
|
||||
@@ -183,6 +184,79 @@ export function AnnotationOverlay({
|
||||
y,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
if (annotation.type !== "magnifier") {
|
||||
return;
|
||||
}
|
||||
void previewFrameVersion;
|
||||
|
||||
const canvas = magnifierCanvasRef.current;
|
||||
const sourceCanvas = previewSourceCanvas;
|
||||
if (!canvas || !sourceCanvas) {
|
||||
return;
|
||||
}
|
||||
|
||||
const sourceWidth = sourceCanvas.width;
|
||||
const sourceHeight = sourceCanvas.height;
|
||||
const sourceClientWidth = sourceCanvas.clientWidth || containerWidth || sourceWidth;
|
||||
const sourceClientHeight = sourceCanvas.clientHeight || containerHeight || sourceHeight;
|
||||
if (
|
||||
sourceWidth <= 0 ||
|
||||
sourceHeight <= 0 ||
|
||||
sourceClientWidth <= 0 ||
|
||||
sourceClientHeight <= 0
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const drawWidth = Math.max(1, Math.round(width));
|
||||
const drawHeight = Math.max(1, Math.round(height));
|
||||
canvas.width = drawWidth;
|
||||
canvas.height = drawHeight;
|
||||
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) {
|
||||
return;
|
||||
}
|
||||
|
||||
const zoom = Math.max(1, annotation.magnifierData?.zoom ?? 2.2);
|
||||
const target = annotation.magnifierData?.target ?? {
|
||||
x: annotation.position.x + annotation.size.width / 2,
|
||||
y: annotation.position.y + annotation.size.height / 2,
|
||||
};
|
||||
const scaleX = sourceWidth / sourceClientWidth;
|
||||
const scaleY = sourceHeight / sourceClientHeight;
|
||||
const targetX = (target.x / 100) * sourceClientWidth * scaleX;
|
||||
const targetY = (target.y / 100) * sourceClientHeight * scaleY;
|
||||
const sampleWidth = Math.max(1, drawWidth / zoom);
|
||||
const sampleHeight = Math.max(1, drawHeight / zoom);
|
||||
const sx = Math.max(0, Math.min(sourceWidth - sampleWidth, targetX - sampleWidth / 2));
|
||||
const sy = Math.max(0, Math.min(sourceHeight - sampleHeight, targetY - sampleHeight / 2));
|
||||
|
||||
context.clearRect(0, 0, drawWidth, drawHeight);
|
||||
context.imageSmoothingEnabled = true;
|
||||
context.imageSmoothingQuality = "high";
|
||||
context.drawImage(
|
||||
sourceCanvas as CanvasImageSource,
|
||||
sx,
|
||||
sy,
|
||||
Math.min(sampleWidth, sourceWidth - sx),
|
||||
Math.min(sampleHeight, sourceHeight - sy),
|
||||
0,
|
||||
0,
|
||||
drawWidth,
|
||||
drawHeight,
|
||||
);
|
||||
}, [
|
||||
annotation,
|
||||
containerHeight,
|
||||
containerWidth,
|
||||
height,
|
||||
previewFrameVersion,
|
||||
previewSourceCanvas,
|
||||
width,
|
||||
]);
|
||||
|
||||
const renderArrow = () => {
|
||||
const direction = annotation.figureData?.arrowDirection || "right";
|
||||
const color = annotation.figureData?.color || "#34B27B";
|
||||
@@ -351,6 +425,30 @@ export function AnnotationOverlay({
|
||||
<div className="w-full h-full flex items-center justify-center p-2">{renderArrow()}</div>
|
||||
);
|
||||
|
||||
case "magnifier": {
|
||||
const shape = annotation.magnifierData?.shape ?? "circle";
|
||||
const caption = annotation.magnifierData?.caption;
|
||||
return (
|
||||
<div className="relative h-full w-full">
|
||||
<canvas
|
||||
ref={magnifierCanvasRef}
|
||||
className="absolute inset-0 h-full w-full"
|
||||
style={{
|
||||
borderRadius: shape === "circle" ? "9999px" : "10px",
|
||||
border: "3px solid rgba(248,250,252,0.96)",
|
||||
boxShadow: "0 14px 36px rgba(0,0,0,0.38), 0 0 0 1px rgba(52,178,123,0.55)",
|
||||
backgroundColor: "rgba(15, 23, 42, 0.9)",
|
||||
}}
|
||||
/>
|
||||
{caption && (
|
||||
<div className="absolute left-1/2 top-full mt-1 max-w-[220px] -translate-x-1/2 rounded-md bg-slate-950/90 px-2 py-1 text-center text-[11px] font-semibold leading-4 text-slate-100 shadow-lg">
|
||||
{caption}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
case "blur": {
|
||||
const shape = annotation.blurData?.shape ?? "rectangle";
|
||||
const blurIntensity = Math.max(
|
||||
@@ -623,6 +721,7 @@ export function AnnotationOverlay({
|
||||
annotation.type === "text" && "bg-transparent",
|
||||
annotation.type === "image" && "bg-transparent",
|
||||
annotation.type === "figure" && "bg-transparent",
|
||||
annotation.type === "magnifier" && "bg-transparent",
|
||||
annotation.type === "blur" && "bg-transparent",
|
||||
isSelected && annotation.type !== "blur" && "shadow-lg",
|
||||
)}
|
||||
|
||||
@@ -13,6 +13,8 @@ import {
|
||||
} from "@/components/ui/dialog";
|
||||
import { useI18n, useScopedT } from "@/contexts/I18nContext";
|
||||
import { useShortcuts } from "@/contexts/ShortcutsContext";
|
||||
import type { GuideSession } from "@/guide/contracts";
|
||||
import { buildGuideVideoAnnotations, buildGuideVideoSpeedRegions } from "@/guide/videoAnnotations";
|
||||
import { INITIAL_EDITOR_STATE, useEditorHistory } from "@/hooks/useEditorHistory";
|
||||
import { type Locale } from "@/i18n/config";
|
||||
import { getAvailableLocales, getLocaleName } from "@/i18n/loader";
|
||||
@@ -1374,6 +1376,34 @@ export default function VideoEditor() {
|
||||
[pushState],
|
||||
);
|
||||
|
||||
const handleGuideAttachToVideo = useCallback(
|
||||
(session: GuideSession) => {
|
||||
const guideAnnotations = buildGuideVideoAnnotations(session, {
|
||||
nextId: () => `annotation-${nextAnnotationIdRef.current++}`,
|
||||
nextZIndex: () => nextAnnotationZIndexRef.current++,
|
||||
});
|
||||
const guideSpeedRegions = buildGuideVideoSpeedRegions(session, {
|
||||
nextId: () => `speed-${nextSpeedIdRef.current++}`,
|
||||
});
|
||||
if (guideAnnotations.length === 0 && guideSpeedRegions.length === 0) {
|
||||
toast.error("Generate a guide draft before attaching steps to the video.");
|
||||
return;
|
||||
}
|
||||
|
||||
pushState((prev) => ({
|
||||
annotationRegions: [...prev.annotationRegions, ...guideAnnotations],
|
||||
speedRegions: [...prev.speedRegions, ...guideSpeedRegions],
|
||||
}));
|
||||
const firstTextAnnotation = guideAnnotations.find((annotation) => annotation.type === "text");
|
||||
setSelectedAnnotationId(firstTextAnnotation?.id ?? guideAnnotations[0]?.id ?? null);
|
||||
setSelectedBlurId(null);
|
||||
setSelectedZoomId(null);
|
||||
setSelectedTrimId(null);
|
||||
setSelectedSpeedId(null);
|
||||
},
|
||||
[pushState],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const handleKeyDown = (e: KeyboardEvent) => {
|
||||
const mod = e.ctrlKey || e.metaKey;
|
||||
@@ -2162,6 +2192,7 @@ export default function VideoEditor() {
|
||||
videoPath={videoPath}
|
||||
videoSourcePath={videoSourcePath}
|
||||
currentTimeMs={currentTime * 1000}
|
||||
onAttachToVideo={handleGuideAttachToVideo}
|
||||
/>
|
||||
)}
|
||||
<div className="min-h-0 flex-1 overflow-hidden">
|
||||
|
||||
@@ -1963,18 +1963,20 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
region: blurRegion,
|
||||
})),
|
||||
].sort((a, b) => a.region.zIndex - b.region.zIndex);
|
||||
const previewSnapshotCanvas =
|
||||
filteredBlurRegions.length > 0
|
||||
? (() => {
|
||||
const app = appRef.current;
|
||||
if (!app?.renderer?.extract) return null;
|
||||
try {
|
||||
return app.renderer.extract.canvas(app.stage);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})()
|
||||
: null;
|
||||
const needsPreviewSnapshot =
|
||||
filteredBlurRegions.length > 0 ||
|
||||
filteredAnnotations.some((annotation) => annotation.type === "magnifier");
|
||||
const previewSnapshotCanvas = needsPreviewSnapshot
|
||||
? (() => {
|
||||
const app = appRef.current;
|
||||
if (!app?.renderer?.extract) return null;
|
||||
try {
|
||||
return app.renderer.extract.canvas(app.stage);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})()
|
||||
: null;
|
||||
|
||||
// Handle click-through cycling: when clicking same annotation, cycle to next
|
||||
const handleAnnotationClick = (clickedId: string) => {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { KeyRound, ListChecks, RefreshCw, Save, Trash2, Wand2 } from "lucide-react";
|
||||
import { KeyRound, ListChecks, RefreshCw, Save, Trash2, Video, Wand2 } from "lucide-react";
|
||||
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
import { Button } from "@/components/ui/button";
|
||||
@@ -7,7 +7,9 @@ import type {
|
||||
GuideAiProvider,
|
||||
GuideAiSettings,
|
||||
GuideLanguage,
|
||||
GuideOcrProfile,
|
||||
GuideSession,
|
||||
GuideSnapshot,
|
||||
} from "@/guide/contracts";
|
||||
import { captureGuideSnapshots } from "@/guide/snapshot/extractGuideSnapshots";
|
||||
|
||||
@@ -16,9 +18,17 @@ interface GuidePanelProps {
|
||||
videoPath: string | null;
|
||||
videoSourcePath: string | null;
|
||||
currentTimeMs: number;
|
||||
onAttachToVideo?: (session: GuideSession) => void;
|
||||
}
|
||||
|
||||
type BusyAction = "load" | "generate";
|
||||
type BusyAction = "load" | "generate" | "attach";
|
||||
|
||||
interface GuideProgressState {
|
||||
label: string;
|
||||
current: number;
|
||||
total: number;
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
const COPY = {
|
||||
en: {
|
||||
@@ -42,13 +52,19 @@ const COPY = {
|
||||
captureStep: "Capture step",
|
||||
captureLabel: "Manual capture",
|
||||
settings: "Settings",
|
||||
guideSettings: "Guide settings",
|
||||
apiKey: "API key env",
|
||||
apiKeyPlaceholder: "DEEPSEEK_API_KEY",
|
||||
baseUrl: "Base URL",
|
||||
model: "Model",
|
||||
ocrProfile: "OCR profile",
|
||||
ocrLanguage: "OCR languages",
|
||||
ocrFast: "Fast Latin",
|
||||
ocrVietnamese: "Vietnamese Enhanced",
|
||||
ocrHybrid: "Hybrid Vi + Latin",
|
||||
saveSettings: "Save",
|
||||
clearKey: "Reset env",
|
||||
keySaved: "DeepSeek settings saved.",
|
||||
settingsSaved: "Guide settings saved.",
|
||||
keyMissing: "Set a DeepSeek API key environment variable before generating with DeepSeek.",
|
||||
keyConfigured: "Env ready",
|
||||
keyNotConfigured: "Env value missing",
|
||||
@@ -56,6 +72,14 @@ const COPY = {
|
||||
noEvents: "No click events were captured for this guide.",
|
||||
ocrUnavailable: "Local OCR service is unavailable. You can still create a local draft.",
|
||||
exported: "Guide exported",
|
||||
attachToVideo: "Attach to video",
|
||||
attachedToVideo: "Guide steps attached to the video timeline.",
|
||||
noDraft: "Generate a guide draft before attaching steps to the video.",
|
||||
progressPreparing: "Preparing events",
|
||||
progressSnapshots: "Capturing snapshots",
|
||||
progressOcr: "Running OCR",
|
||||
progressDraft: "Writing draft",
|
||||
progressExport: "Exporting files",
|
||||
},
|
||||
vi: {
|
||||
title: "Hướng dẫn",
|
||||
@@ -78,13 +102,19 @@ const COPY = {
|
||||
captureStep: "Chụp bước",
|
||||
captureLabel: "Chụp thủ công",
|
||||
settings: "Cài đặt",
|
||||
guideSettings: "Guide settings",
|
||||
apiKey: "API key env",
|
||||
apiKeyPlaceholder: "DEEPSEEK_API_KEY",
|
||||
baseUrl: "Base URL",
|
||||
model: "Model",
|
||||
ocrProfile: "OCR profile",
|
||||
ocrLanguage: "OCR languages",
|
||||
ocrFast: "Fast Latin",
|
||||
ocrVietnamese: "Vietnamese Enhanced",
|
||||
ocrHybrid: "Hybrid Vi + Latin",
|
||||
saveSettings: "Lưu",
|
||||
clearKey: "Reset env",
|
||||
keySaved: "Đã lưu cài đặt DeepSeek.",
|
||||
settingsSaved: "Da luu cai dat guide.",
|
||||
keyMissing: "Hãy set biến môi trường DeepSeek API key trước khi tạo draft bằng DeepSeek.",
|
||||
keyConfigured: "Env ready",
|
||||
keyNotConfigured: "Chưa thấy giá trị env",
|
||||
@@ -92,10 +122,41 @@ const COPY = {
|
||||
noEvents: "Chưa ghi nhận click event nào cho guide này.",
|
||||
ocrUnavailable: "OCR local chưa chạy. Vẫn có thể tạo draft local.",
|
||||
exported: "Đã export hướng dẫn",
|
||||
attachToVideo: "Gắn vào video",
|
||||
attachedToVideo: "Đã gắn các bước guide vào timeline video.",
|
||||
noDraft: "Hãy tạo draft guide trước khi gắn vào video.",
|
||||
progressPreparing: "Đang chuẩn bị events",
|
||||
progressSnapshots: "Đang chụp ảnh",
|
||||
progressOcr: "Đang OCR",
|
||||
progressDraft: "Đang tạo draft",
|
||||
progressExport: "Đang export file",
|
||||
},
|
||||
} as const;
|
||||
|
||||
export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePanelProps) {
|
||||
function getPendingOcrSnapshots(session: GuideSession): GuideSnapshot[] {
|
||||
const ocrCompletedSnapshotIds = new Set(session.ocrBlocks.map((block) => block.snapshotId));
|
||||
return session.snapshots.filter(
|
||||
(snapshot) => !snapshot.ocrCompletedAt && !ocrCompletedSnapshotIds.has(snapshot.id),
|
||||
);
|
||||
}
|
||||
|
||||
function getProgressPercent(progress: GuideProgressState | null): number {
|
||||
if (!progress) {
|
||||
return 0;
|
||||
}
|
||||
if (progress.total <= 0) {
|
||||
return 100;
|
||||
}
|
||||
const percent = Math.round((progress.current / progress.total) * 100);
|
||||
return Math.min(100, Math.max(progress.current > 0 ? 8 : 4, percent));
|
||||
}
|
||||
|
||||
export function GuidePanel({
|
||||
recordingId,
|
||||
videoPath,
|
||||
videoSourcePath,
|
||||
onAttachToVideo,
|
||||
}: GuidePanelProps) {
|
||||
const { locale } = useI18n();
|
||||
const copy = useMemo(() => (locale.startsWith("vi") ? COPY.vi : COPY.en), [locale]);
|
||||
const guideLanguage: GuideLanguage = locale.startsWith("vi") ? "vi" : "en";
|
||||
@@ -108,9 +169,13 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
const [deepSeekApiKeyEnvName, setDeepSeekApiKeyEnvName] = useState("DEEPSEEK_API_KEY");
|
||||
const [deepSeekBaseUrl, setDeepSeekBaseUrl] = useState("https://api.deepseek.com");
|
||||
const [deepSeekModel, setDeepSeekModel] = useState("deepseek-chat");
|
||||
const [ocrProfile, setOcrProfile] = useState<GuideOcrProfile>("vietnamese");
|
||||
const [ocrLanguage, setOcrLanguage] = useState("vi,en");
|
||||
const [message, setMessage] = useState<string | null>(null);
|
||||
const [progress, setProgress] = useState<GuideProgressState | null>(null);
|
||||
|
||||
const isBusy = busyAction !== null;
|
||||
const progressPercent = getProgressPercent(progress);
|
||||
const canUseGuide = Boolean(recordingId && videoSourcePath && window.electronAPI?.guide);
|
||||
const generatedSteps = session?.generatedGuide?.steps ?? [];
|
||||
const statusLabel = useMemo(() => {
|
||||
@@ -138,6 +203,8 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
setDeepSeekBaseUrl(result.data.deepseek.baseUrl);
|
||||
setDeepSeekModel(result.data.deepseek.model);
|
||||
setDeepSeekApiKeyEnvName(result.data.deepseek.apiKeyEnvName);
|
||||
setOcrProfile(result.data.ocr.profile);
|
||||
setOcrLanguage(result.data.ocr.language);
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
@@ -203,6 +270,15 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
}
|
||||
|
||||
let current = session;
|
||||
const readResult = await window.electronAPI.guide.readSession(recordingId);
|
||||
if (readResult.success) {
|
||||
current = readResult.data;
|
||||
} else if (readResult.code === "guide-session-not-found") {
|
||||
current = null;
|
||||
} else if (!current) {
|
||||
throw new Error(readResult.error);
|
||||
}
|
||||
|
||||
if (!current) {
|
||||
const startResult = await window.electronAPI.guide.startSession(recordingId);
|
||||
if (!startResult.success) {
|
||||
@@ -234,6 +310,7 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
}
|
||||
setBusyAction(action);
|
||||
setMessage(null);
|
||||
setProgress(null);
|
||||
try {
|
||||
await task();
|
||||
} catch (error) {
|
||||
@@ -269,6 +346,8 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
deepseekApiKeyEnvName: deepSeekApiKeyEnvName,
|
||||
baseUrl: deepSeekBaseUrl,
|
||||
model: deepSeekModel,
|
||||
ocrProfile,
|
||||
ocrLanguage,
|
||||
});
|
||||
if (!result.success) {
|
||||
throw new Error(result.error);
|
||||
@@ -277,7 +356,9 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
setDeepSeekApiKeyEnvName(result.data.deepseek.apiKeyEnvName);
|
||||
setDeepSeekBaseUrl(result.data.deepseek.baseUrl);
|
||||
setDeepSeekModel(result.data.deepseek.model);
|
||||
toast.success(copy.keySaved);
|
||||
setOcrProfile(result.data.ocr.profile);
|
||||
setOcrLanguage(result.data.ocr.language);
|
||||
toast.success(copy.settingsSaved);
|
||||
} catch (error) {
|
||||
const text = error instanceof Error ? error.message : String(error);
|
||||
setMessage(text);
|
||||
@@ -285,7 +366,14 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
} finally {
|
||||
setSettingsBusy(false);
|
||||
}
|
||||
}, [copy.keySaved, deepSeekApiKeyEnvName, deepSeekBaseUrl, deepSeekModel]);
|
||||
}, [
|
||||
copy.settingsSaved,
|
||||
deepSeekApiKeyEnvName,
|
||||
deepSeekBaseUrl,
|
||||
deepSeekModel,
|
||||
ocrLanguage,
|
||||
ocrProfile,
|
||||
]);
|
||||
|
||||
const handleClearDeepSeekKey = useCallback(async () => {
|
||||
if (!window.electronAPI?.guide?.saveAiSettings) {
|
||||
@@ -298,13 +386,17 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
clearDeepseekApiKeyEnvName: true,
|
||||
baseUrl: deepSeekBaseUrl,
|
||||
model: deepSeekModel,
|
||||
ocrProfile,
|
||||
ocrLanguage,
|
||||
});
|
||||
if (!result.success) {
|
||||
throw new Error(result.error);
|
||||
}
|
||||
setAiSettings(result.data);
|
||||
setDeepSeekApiKeyEnvName(result.data.deepseek.apiKeyEnvName);
|
||||
toast.success(copy.keySaved);
|
||||
setOcrProfile(result.data.ocr.profile);
|
||||
setOcrLanguage(result.data.ocr.language);
|
||||
toast.success(copy.settingsSaved);
|
||||
} catch (error) {
|
||||
const text = error instanceof Error ? error.message : String(error);
|
||||
setMessage(text);
|
||||
@@ -312,7 +404,7 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
} finally {
|
||||
setSettingsBusy(false);
|
||||
}
|
||||
}, [copy.keySaved, deepSeekBaseUrl, deepSeekModel]);
|
||||
}, [copy.settingsSaved, deepSeekBaseUrl, deepSeekModel, ocrLanguage, ocrProfile]);
|
||||
|
||||
const handleGenerateGuide = useCallback(() => {
|
||||
void runAction("generate", async () => {
|
||||
@@ -323,21 +415,59 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
if (!videoPath) {
|
||||
throw new Error("Video URL is not available.");
|
||||
}
|
||||
setProgress({
|
||||
label: copy.progressPreparing,
|
||||
current: 0,
|
||||
total: 1,
|
||||
detail: "0/1",
|
||||
});
|
||||
let current = await ensureEventsSession();
|
||||
setProgress({
|
||||
label: copy.progressPreparing,
|
||||
current: 1,
|
||||
total: 1,
|
||||
detail: "1/1",
|
||||
});
|
||||
if (current.events.length === 0) {
|
||||
throw new Error(copy.noEvents);
|
||||
}
|
||||
if (current.snapshots.length < current.events.length) {
|
||||
const snapshotEventIds = new Set(current.snapshots.map((snapshot) => snapshot.eventId));
|
||||
const pendingSnapshotTotal = current.events.filter(
|
||||
(event) => !snapshotEventIds.has(event.id),
|
||||
).length;
|
||||
if (pendingSnapshotTotal > 0) {
|
||||
setProgress({
|
||||
label: copy.progressSnapshots,
|
||||
current: 0,
|
||||
total: pendingSnapshotTotal,
|
||||
detail: `0/${pendingSnapshotTotal}`,
|
||||
});
|
||||
current = await captureGuideSnapshots({
|
||||
session: current,
|
||||
videoUrl: videoPath,
|
||||
maxWidth: 1280,
|
||||
onProgress: ({ completed, total }) => {
|
||||
setProgress({
|
||||
label: copy.progressSnapshots,
|
||||
current: completed,
|
||||
total,
|
||||
detail: `${completed}/${total}`,
|
||||
});
|
||||
},
|
||||
});
|
||||
setSession(current);
|
||||
}
|
||||
if (current.ocrBlocks.length === 0 && current.snapshots.length > 0) {
|
||||
const pendingOcrSnapshots = getPendingOcrSnapshots(current);
|
||||
for (const [index, snapshot] of pendingOcrSnapshots.entries()) {
|
||||
setProgress({
|
||||
label: copy.progressOcr,
|
||||
current: index,
|
||||
total: pendingOcrSnapshots.length,
|
||||
detail: `${index + 1}/${pendingOcrSnapshots.length}`,
|
||||
});
|
||||
const ocrResult = await window.electronAPI.guide.runOcr({
|
||||
recordingId: current.recordingId,
|
||||
snapshotIds: [snapshot.id],
|
||||
});
|
||||
if (!ocrResult.success) {
|
||||
if (ocrResult.code === "guide-ocr-unavailable") {
|
||||
@@ -347,7 +477,19 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
}
|
||||
current = ocrResult.data;
|
||||
setSession(current);
|
||||
setProgress({
|
||||
label: copy.progressOcr,
|
||||
current: index + 1,
|
||||
total: pendingOcrSnapshots.length,
|
||||
detail: `${index + 1}/${pendingOcrSnapshots.length}`,
|
||||
});
|
||||
}
|
||||
setProgress({
|
||||
label: copy.progressDraft,
|
||||
current: 0,
|
||||
total: 1,
|
||||
detail: "0/1",
|
||||
});
|
||||
const result = await window.electronAPI.guide.generateDraft({
|
||||
recordingId: current.recordingId,
|
||||
language: guideLanguage,
|
||||
@@ -356,18 +498,44 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
if (!result.success) {
|
||||
throw new Error(result.error);
|
||||
}
|
||||
current = result.data;
|
||||
setSession(current);
|
||||
setProgress({
|
||||
label: copy.progressDraft,
|
||||
current: 1,
|
||||
total: 1,
|
||||
detail: "1/1",
|
||||
});
|
||||
setProgress({
|
||||
label: copy.progressExport,
|
||||
current: 0,
|
||||
total: 2,
|
||||
detail: "0/2",
|
||||
});
|
||||
const markdownResult = await window.electronAPI.guide.exportMarkdown({
|
||||
recordingId: current.recordingId,
|
||||
});
|
||||
if (!markdownResult.success) {
|
||||
throw new Error(markdownResult.error);
|
||||
}
|
||||
setProgress({
|
||||
label: copy.progressExport,
|
||||
current: 1,
|
||||
total: 2,
|
||||
detail: "1/2",
|
||||
});
|
||||
const htmlResult = await window.electronAPI.guide.exportHtml({
|
||||
recordingId: current.recordingId,
|
||||
});
|
||||
if (!htmlResult.success) {
|
||||
throw new Error(htmlResult.error);
|
||||
}
|
||||
setProgress({
|
||||
label: copy.progressExport,
|
||||
current: 2,
|
||||
total: 2,
|
||||
detail: "2/2",
|
||||
});
|
||||
const revealResult = await window.electronAPI.revealInFolder(htmlResult.data.path);
|
||||
if (!revealResult.success) {
|
||||
toast.warning(revealResult.error ?? "Unable to open guide folder.");
|
||||
@@ -383,6 +551,11 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
copy.keyMissing,
|
||||
copy.noEvents,
|
||||
copy.ocrUnavailable,
|
||||
copy.progressDraft,
|
||||
copy.progressExport,
|
||||
copy.progressOcr,
|
||||
copy.progressPreparing,
|
||||
copy.progressSnapshots,
|
||||
ensureEventsSession,
|
||||
guideLanguage,
|
||||
provider,
|
||||
@@ -390,6 +563,25 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
videoPath,
|
||||
]);
|
||||
|
||||
const handleAttachToVideo = useCallback(() => {
|
||||
if (!session?.generatedGuide || session.generatedGuide.steps.length === 0) {
|
||||
setMessage(copy.noDraft);
|
||||
toast.error(copy.noDraft);
|
||||
return;
|
||||
}
|
||||
if (!onAttachToVideo) {
|
||||
return;
|
||||
}
|
||||
setBusyAction("attach");
|
||||
try {
|
||||
onAttachToVideo(session);
|
||||
setMessage(null);
|
||||
toast.success(copy.attachedToVideo);
|
||||
} finally {
|
||||
setBusyAction(null);
|
||||
}
|
||||
}, [copy.attachedToVideo, copy.noDraft, onAttachToVideo, session]);
|
||||
|
||||
return (
|
||||
<section className="editor-inspector-shell flex max-h-[320px] min-h-[246px] shrink-0 flex-col overflow-hidden rounded-[18px] border border-white/[0.075] bg-[#090a0c]">
|
||||
<div className="flex items-center justify-between border-b border-white/[0.07] px-3 py-2">
|
||||
@@ -413,6 +605,24 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
{canUseGuide ? statusLabel : copy.noRecording}
|
||||
</p>
|
||||
{message && <p className="mb-2 text-[11px] leading-4 text-amber-300">{message}</p>}
|
||||
{progress && (
|
||||
<div className="mb-2 rounded-md border border-white/[0.07] bg-white/[0.035] px-2 py-1.5">
|
||||
<div className="mb-1 flex items-center justify-between gap-2 text-[10px] leading-4">
|
||||
<span className="min-w-0 truncate font-semibold text-slate-200">
|
||||
{progress.label}
|
||||
</span>
|
||||
<span className="shrink-0 text-slate-500">
|
||||
{progress.detail ?? `${progress.current}/${progress.total}`}
|
||||
</span>
|
||||
</div>
|
||||
<div className="h-1.5 overflow-hidden rounded-full bg-white/[0.06]">
|
||||
<div
|
||||
className="h-full rounded-full bg-[#34B27B] transition-all duration-200"
|
||||
style={{ width: `${progressPercent}%` }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="mb-2 flex items-center gap-1.5">
|
||||
<select
|
||||
@@ -450,12 +660,22 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
</span>
|
||||
</button>
|
||||
|
||||
<button
|
||||
type="button"
|
||||
disabled={!generatedSteps.length || isBusy || !onAttachToVideo}
|
||||
onClick={handleAttachToVideo}
|
||||
className="mb-2 flex h-9 w-full items-center justify-center gap-2 rounded-md border border-sky-400/25 bg-sky-400/10 px-3 text-xs font-semibold text-sky-100 transition-all hover:border-sky-300/45 hover:bg-sky-400/18 disabled:cursor-not-allowed disabled:opacity-40"
|
||||
>
|
||||
<Video className="h-3.5 w-3.5 shrink-0" />
|
||||
<span className="truncate">{copy.attachToVideo}</span>
|
||||
</button>
|
||||
|
||||
{settingsOpen && (
|
||||
<div className="mb-2 space-y-2 rounded-md border border-white/[0.07] bg-white/[0.035] p-2">
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<div className="min-w-0">
|
||||
<div className="truncate text-[11px] font-semibold text-slate-100">
|
||||
{copy.deepseek} {copy.settings}
|
||||
{copy.guideSettings}
|
||||
</div>
|
||||
<div className="truncate text-[10px] text-slate-500">
|
||||
{aiSettings?.deepseek.hasApiKey
|
||||
@@ -470,6 +690,33 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-2 gap-1.5">
|
||||
<label className="block min-w-0 text-[10px] font-medium text-slate-400">
|
||||
{copy.ocrProfile}
|
||||
<select
|
||||
value={ocrProfile}
|
||||
onChange={(event) => setOcrProfile(event.target.value as GuideOcrProfile)}
|
||||
disabled={settingsBusy}
|
||||
className="mt-1 h-8 w-full rounded-md border border-white/[0.08] bg-black/20 px-2 text-[11px] text-slate-100 outline-none"
|
||||
>
|
||||
<option value="vietnamese">{copy.ocrVietnamese}</option>
|
||||
<option value="hybrid">{copy.ocrHybrid}</option>
|
||||
<option value="fast">{copy.ocrFast}</option>
|
||||
</select>
|
||||
</label>
|
||||
<label className="block min-w-0 text-[10px] font-medium text-slate-400">
|
||||
{copy.ocrLanguage}
|
||||
<input
|
||||
type="text"
|
||||
value={ocrLanguage}
|
||||
onChange={(event) => setOcrLanguage(event.target.value)}
|
||||
placeholder="vi,en"
|
||||
disabled={settingsBusy}
|
||||
className="mt-1 h-8 w-full rounded-md border border-white/[0.08] bg-black/20 px-2 text-[11px] text-slate-100 outline-none placeholder:text-slate-600"
|
||||
/>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<label className="block text-[10px] font-medium text-slate-400">
|
||||
{copy.apiKey}
|
||||
<input
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
DEFAULT_BLUR_FREEHAND_POINTS,
|
||||
DEFAULT_BLUR_INTENSITY,
|
||||
DEFAULT_FIGURE_DATA,
|
||||
DEFAULT_MAGNIFIER_DATA,
|
||||
DEFAULT_PLAYBACK_SPEED,
|
||||
DEFAULT_ZOOM_DEPTH,
|
||||
DEFAULT_ZOOM_MOTION_BLUR,
|
||||
@@ -325,7 +326,10 @@ export function normalizeProjectEditor(editor: Partial<ProjectEditorState>): Pro
|
||||
startMs,
|
||||
endMs,
|
||||
type:
|
||||
region.type === "image" || region.type === "figure" || region.type === "blur"
|
||||
region.type === "image" ||
|
||||
region.type === "figure" ||
|
||||
region.type === "blur" ||
|
||||
region.type === "magnifier"
|
||||
? region.type
|
||||
: "text",
|
||||
content: typeof region.content === "string" ? region.content : "",
|
||||
@@ -410,6 +414,45 @@ export function normalizeProjectEditor(editor: Partial<ProjectEditorState>): Pro
|
||||
: DEFAULT_BLUR_FREEHAND_POINTS,
|
||||
}
|
||||
: undefined,
|
||||
magnifierData:
|
||||
region.magnifierData && typeof region.magnifierData === "object"
|
||||
? {
|
||||
...DEFAULT_MAGNIFIER_DATA,
|
||||
...region.magnifierData,
|
||||
target: {
|
||||
x: clamp(
|
||||
isFiniteNumber(region.magnifierData.target?.x)
|
||||
? region.magnifierData.target.x
|
||||
: DEFAULT_MAGNIFIER_DATA.target.x,
|
||||
0,
|
||||
100,
|
||||
),
|
||||
y: clamp(
|
||||
isFiniteNumber(region.magnifierData.target?.y)
|
||||
? region.magnifierData.target.y
|
||||
: DEFAULT_MAGNIFIER_DATA.target.y,
|
||||
0,
|
||||
100,
|
||||
),
|
||||
},
|
||||
zoom: clamp(
|
||||
isFiniteNumber(region.magnifierData.zoom)
|
||||
? region.magnifierData.zoom
|
||||
: DEFAULT_MAGNIFIER_DATA.zoom,
|
||||
1,
|
||||
6,
|
||||
),
|
||||
shape:
|
||||
region.magnifierData.shape === "rounded" ||
|
||||
region.magnifierData.shape === "circle"
|
||||
? region.magnifierData.shape
|
||||
: DEFAULT_MAGNIFIER_DATA.shape,
|
||||
caption:
|
||||
typeof region.magnifierData.caption === "string"
|
||||
? region.magnifierData.caption
|
||||
: undefined,
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
})
|
||||
: [];
|
||||
|
||||
@@ -206,7 +206,7 @@ export interface TrimRegion {
|
||||
endMs: number;
|
||||
}
|
||||
|
||||
export type AnnotationType = "text" | "image" | "figure" | "blur";
|
||||
export type AnnotationType = "text" | "image" | "figure" | "blur" | "magnifier";
|
||||
|
||||
export type ArrowDirection =
|
||||
| "up"
|
||||
@@ -245,6 +245,13 @@ export interface BlurData {
|
||||
freehandPoints?: Array<{ x: number; y: number }>;
|
||||
}
|
||||
|
||||
export interface MagnifierData {
|
||||
target: AnnotationPosition;
|
||||
zoom: number;
|
||||
shape: "circle" | "rounded";
|
||||
caption?: string;
|
||||
}
|
||||
|
||||
export interface AnnotationPosition {
|
||||
x: number;
|
||||
y: number;
|
||||
@@ -280,6 +287,7 @@ export interface AnnotationRegion {
|
||||
zIndex: number;
|
||||
figureData?: FigureData;
|
||||
blurData?: BlurData;
|
||||
magnifierData?: MagnifierData;
|
||||
}
|
||||
|
||||
export const DEFAULT_ANNOTATION_POSITION: AnnotationPosition = {
|
||||
@@ -330,6 +338,12 @@ export const DEFAULT_BLUR_DATA: BlurData = {
|
||||
freehandPoints: DEFAULT_BLUR_FREEHAND_POINTS,
|
||||
};
|
||||
|
||||
export const DEFAULT_MAGNIFIER_DATA: MagnifierData = {
|
||||
target: { x: 50, y: 50 },
|
||||
zoom: 2.2,
|
||||
shape: "circle",
|
||||
};
|
||||
|
||||
export interface CropRegion {
|
||||
x: number;
|
||||
y: number;
|
||||
|
||||
@@ -9,6 +9,7 @@ export type GuideTargetRole = "button" | "menu" | "tab" | "field" | "link" | "un
|
||||
export type GuideLanguage = "vi" | "en";
|
||||
export type GuideAiProvider = "deepseek" | "local";
|
||||
export type GuideSecretStorage = "environment" | "none";
|
||||
export type GuideOcrProfile = "fast" | "vietnamese" | "hybrid";
|
||||
|
||||
export type GuideSessionStatus =
|
||||
| "recording"
|
||||
@@ -54,6 +55,8 @@ export interface GuideSnapshot {
|
||||
timeMs: number;
|
||||
offsetMs: number;
|
||||
path: string;
|
||||
markedPath?: string;
|
||||
ocrCompletedAt?: string;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
@@ -162,6 +165,7 @@ export interface WriteGuideSnapshotInput {
|
||||
timeMs: number;
|
||||
offsetMs: number;
|
||||
pngBytes: ArrayBuffer;
|
||||
markedPngBytes?: ArrayBuffer;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
@@ -178,6 +182,11 @@ export interface GenerateGuideDraftInput {
|
||||
}
|
||||
|
||||
export interface GuideAiSettings {
|
||||
ocr: {
|
||||
profile: GuideOcrProfile;
|
||||
language: string;
|
||||
updatedAt?: string;
|
||||
};
|
||||
deepseek: {
|
||||
hasApiKey: boolean;
|
||||
apiKeyEnvName: string;
|
||||
@@ -194,6 +203,8 @@ export interface SaveGuideAiSettingsInput {
|
||||
clearDeepseekApiKeyEnvName?: boolean;
|
||||
baseUrl?: string;
|
||||
model?: string;
|
||||
ocrProfile?: GuideOcrProfile;
|
||||
ocrLanguage?: string;
|
||||
}
|
||||
|
||||
export interface SaveGuideInput {
|
||||
|
||||
@@ -29,6 +29,7 @@ const session: GuideSession = {
|
||||
timeMs: 1500,
|
||||
offsetMs: 500,
|
||||
path: "/tmp/recording-guide/step-001.png",
|
||||
markedPath: "/tmp/recording-guide/step-001-marked.png",
|
||||
width: 1280,
|
||||
height: 720,
|
||||
},
|
||||
@@ -71,7 +72,7 @@ describe("guide exporters", () => {
|
||||
|
||||
expect(markdown).toContain("# User guide");
|
||||
expect(markdown).toContain("## 1. Open Settings");
|
||||
expect(markdown).toContain("](step-001.png)");
|
||||
expect(markdown).toContain("](step-001-marked.png)");
|
||||
});
|
||||
|
||||
it("exports escaped HTML", () => {
|
||||
@@ -79,12 +80,11 @@ describe("guide exporters", () => {
|
||||
|
||||
expect(html).toContain("<!doctype html>");
|
||||
expect(html).toContain("<h1>User guide</h1>");
|
||||
expect(html).toContain('src="step-001.png"');
|
||||
expect(html).toContain("click-marker");
|
||||
expect(html).toContain("left: 25.00%; top: 75.00%;");
|
||||
expect(html).toContain('src="step-001-marked.png"');
|
||||
expect(html).not.toContain("click-marker");
|
||||
});
|
||||
|
||||
it("draws click markers for hotkey events with coordinates", () => {
|
||||
it("uses marker snapshots for hotkey events with coordinates", () => {
|
||||
const hotkeySession: GuideSession = {
|
||||
...session,
|
||||
events: [
|
||||
@@ -98,7 +98,21 @@ describe("guide exporters", () => {
|
||||
|
||||
const html = exportGuideToHtml(hotkeySession);
|
||||
|
||||
expect(html).toContain("click-marker");
|
||||
expect(html).toContain("left: 25.00%; top: 75.00%;");
|
||||
expect(html).toContain('src="step-001-marked.png"');
|
||||
expect(html).not.toContain("click-marker");
|
||||
});
|
||||
|
||||
it("falls back to the unmarked screenshot when no marker snapshot exists", () => {
|
||||
const unmarkedSession: GuideSession = {
|
||||
...session,
|
||||
snapshots: session.snapshots.map((snapshot) => ({
|
||||
...snapshot,
|
||||
markedPath: undefined,
|
||||
})),
|
||||
};
|
||||
|
||||
const markdown = exportGuideToMarkdown(unmarkedSession);
|
||||
|
||||
expect(markdown).toContain("](step-001.png)");
|
||||
});
|
||||
});
|
||||
|
||||
+25
-51
@@ -10,8 +10,9 @@ export function exportGuideToMarkdown(session: GuideSession): string {
|
||||
|
||||
for (const step of guide.steps) {
|
||||
lines.push(`## ${step.order}. ${step.title}`, "", step.instruction, "");
|
||||
if (step.screenshotPath) {
|
||||
lines.push(`})`, "");
|
||||
const screenshotPath = resolveStepScreenshotPath(step, session);
|
||||
if (screenshotPath) {
|
||||
lines.push(`})`, "");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,10 +37,8 @@ export function exportGuideToHtml(session: GuideSession): string {
|
||||
.step { border-top: 1px solid #e5e7eb; padding: 22px 0; }
|
||||
.step h2 { font-size: 18px; margin: 0 0 8px; }
|
||||
.step p { margin: 0 0 12px; }
|
||||
.shot { display: inline-block; position: relative; max-width: 100%; margin: 0; }
|
||||
.shot { display: inline-block; max-width: 100%; margin: 0; }
|
||||
img { display: block; max-width: 100%; border: 1px solid #e5e7eb; border-radius: 6px; }
|
||||
.click-marker { position: absolute; width: 26px; height: 26px; border: 3px solid #ef4444; border-radius: 999px; box-shadow: 0 0 0 4px rgba(239, 68, 68, 0.18), 0 2px 8px rgba(17, 24, 39, 0.28); transform: translate(-50%, -50%); pointer-events: none; }
|
||||
.click-marker::after { content: ""; position: absolute; left: 50%; top: 50%; width: 6px; height: 6px; border-radius: 999px; background: #ef4444; transform: translate(-50%, -50%); }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
@@ -54,12 +53,9 @@ export function exportGuideToHtml(session: GuideSession): string {
|
||||
}
|
||||
|
||||
function renderStepHtml(step: GeneratedGuideStep, session: GuideSession): string {
|
||||
const clickPoint = resolveStepClickPoint(step, session);
|
||||
const marker = clickPoint
|
||||
? `<span class="click-marker" style="left: ${formatPercent(clickPoint.x)}%; top: ${formatPercent(clickPoint.y)}%;" aria-label="Click position"></span>`
|
||||
: "";
|
||||
const image = step.screenshotPath
|
||||
? `<figure class="shot"><img src="${escapeHtml(path.basename(step.screenshotPath))}" alt="${escapeHtml(step.title)}">${marker}</figure>`
|
||||
const screenshotPath = resolveStepScreenshotPath(step, session);
|
||||
const image = screenshotPath
|
||||
? `<figure class="shot"><img src="${escapeHtml(path.basename(screenshotPath))}" alt="${escapeHtml(step.title)}"></figure>`
|
||||
: "";
|
||||
return `<section class="step">
|
||||
<h2>${step.order}. ${escapeHtml(step.title)}</h2>
|
||||
@@ -88,54 +84,32 @@ function escapeHtml(value: string): string {
|
||||
.replace(/'/g, "'");
|
||||
}
|
||||
|
||||
function resolveStepClickPoint(
|
||||
function resolveStepScreenshotPath(
|
||||
step: GeneratedGuideStep,
|
||||
session: GuideSession,
|
||||
): { x: number; y: number } | null {
|
||||
): string | undefined {
|
||||
const snapshot = resolveStepSnapshot(step, session);
|
||||
return snapshot?.markedPath ?? step.screenshotPath ?? snapshot?.path;
|
||||
}
|
||||
|
||||
function resolveStepSnapshot(step: GeneratedGuideStep, session: GuideSession) {
|
||||
const candidate = step.sourceCandidateId
|
||||
? session.candidates.find((item) => item.id === step.sourceCandidateId)
|
||||
: undefined;
|
||||
const eventId = candidate?.eventId;
|
||||
const event = eventId ? session.events.find((item) => item.id === eventId) : undefined;
|
||||
if (!event || (event.kind !== "click" && event.kind !== "hotkey")) {
|
||||
return null;
|
||||
}
|
||||
if (isNormalizedNumber(event.normalizedX) && isNormalizedNumber(event.normalizedY)) {
|
||||
return { x: clamp01(event.normalizedX), y: clamp01(event.normalizedY) };
|
||||
}
|
||||
|
||||
const screenshotFileName = step.screenshotPath ? path.basename(step.screenshotPath) : undefined;
|
||||
const snapshot =
|
||||
return (
|
||||
(candidate?.snapshotId
|
||||
? session.snapshots.find((item) => item.id === candidate.snapshotId)
|
||||
: undefined) ??
|
||||
(candidate?.eventId
|
||||
? session.snapshots.find((item) => item.eventId === candidate.eventId)
|
||||
: undefined) ??
|
||||
(screenshotFileName
|
||||
? session.snapshots.find((item) => path.basename(item.path) === screenshotFileName)
|
||||
: undefined);
|
||||
if (
|
||||
!snapshot ||
|
||||
typeof event.x !== "number" ||
|
||||
typeof event.y !== "number" ||
|
||||
snapshot.width <= 0 ||
|
||||
snapshot.height <= 0
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
x: clamp01(event.x / snapshot.width),
|
||||
y: clamp01(event.y / snapshot.height),
|
||||
};
|
||||
}
|
||||
|
||||
function formatPercent(value: number): string {
|
||||
return (clamp01(value) * 100).toFixed(2);
|
||||
}
|
||||
|
||||
function isNormalizedNumber(value: unknown): value is number {
|
||||
return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
|
||||
}
|
||||
|
||||
function clamp01(value: number): number {
|
||||
return Math.min(1, Math.max(0, value));
|
||||
? session.snapshots.find(
|
||||
(item) =>
|
||||
path.basename(item.path) === screenshotFileName ||
|
||||
(item.markedPath ? path.basename(item.markedPath) === screenshotFileName : false),
|
||||
)
|
||||
: undefined)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -4,6 +4,13 @@ export interface CaptureGuideSnapshotsInput {
|
||||
session: GuideSession;
|
||||
videoUrl: string;
|
||||
maxWidth?: number;
|
||||
onProgress?: (progress: CaptureGuideSnapshotsProgress) => void;
|
||||
}
|
||||
|
||||
export interface CaptureGuideSnapshotsProgress {
|
||||
event: GuideEvent;
|
||||
completed: number;
|
||||
total: number;
|
||||
}
|
||||
|
||||
export async function captureGuideSnapshots(
|
||||
@@ -13,6 +20,13 @@ export async function captureGuideSnapshots(
|
||||
if (events.length === 0) {
|
||||
return input.session;
|
||||
}
|
||||
const existingSnapshotsByEventId = new Set(
|
||||
input.session.snapshots.map((snapshot) => snapshot.eventId),
|
||||
);
|
||||
const pendingEvents = events.filter((event) => !existingSnapshotsByEventId.has(event.id));
|
||||
if (pendingEvents.length === 0) {
|
||||
return input.session;
|
||||
}
|
||||
|
||||
const video = document.createElement("video");
|
||||
video.preload = "auto";
|
||||
@@ -35,18 +49,24 @@ export async function captureGuideSnapshots(
|
||||
canvas.height = Math.max(1, Math.round(sourceHeight * scale));
|
||||
|
||||
let latestSession = input.session;
|
||||
for (const event of events) {
|
||||
let completed = 0;
|
||||
for (const event of pendingEvents) {
|
||||
const offsetMs = event.screenshotOffsetMs ?? 500;
|
||||
const timeMs = getSnapshotTimeMs(event, offsetMs, video.duration);
|
||||
await seekVideo(video, timeMs / 1000);
|
||||
context.drawImage(video, 0, 0, canvas.width, canvas.height);
|
||||
const pngBytes = await canvasToPngBytes(canvas);
|
||||
const markerPoint = getSnapshotMarkerPoint(event, canvas.width, canvas.height);
|
||||
const markedPngBytes = markerPoint
|
||||
? await canvasToMarkedPngBytes(canvas, markerPoint)
|
||||
: undefined;
|
||||
const result = await window.electronAPI.guide.writeSnapshot({
|
||||
recordingId: input.session.recordingId,
|
||||
eventId: event.id,
|
||||
timeMs,
|
||||
offsetMs,
|
||||
pngBytes,
|
||||
markedPngBytes,
|
||||
width: canvas.width,
|
||||
height: canvas.height,
|
||||
});
|
||||
@@ -54,6 +74,12 @@ export async function captureGuideSnapshots(
|
||||
throw new Error(result.error);
|
||||
}
|
||||
latestSession = result.data;
|
||||
completed += 1;
|
||||
input.onProgress?.({
|
||||
event,
|
||||
completed,
|
||||
total: pendingEvents.length,
|
||||
});
|
||||
}
|
||||
|
||||
return latestSession;
|
||||
@@ -143,3 +169,75 @@ function canvasToPngBytes(canvas: HTMLCanvasElement): Promise<ArrayBuffer> {
|
||||
}, "image/png");
|
||||
});
|
||||
}
|
||||
|
||||
async function canvasToMarkedPngBytes(
|
||||
canvas: HTMLCanvasElement,
|
||||
point: { x: number; y: number },
|
||||
): Promise<ArrayBuffer> {
|
||||
const markedCanvas = document.createElement("canvas");
|
||||
markedCanvas.width = canvas.width;
|
||||
markedCanvas.height = canvas.height;
|
||||
const markedContext = markedCanvas.getContext("2d");
|
||||
if (!markedContext) {
|
||||
throw new Error("Canvas 2D context is unavailable.");
|
||||
}
|
||||
markedContext.drawImage(canvas, 0, 0);
|
||||
drawSnapshotMarker(markedContext, markedCanvas, point);
|
||||
return await canvasToPngBytes(markedCanvas);
|
||||
}
|
||||
|
||||
function drawSnapshotMarker(
|
||||
context: CanvasRenderingContext2D,
|
||||
canvas: HTMLCanvasElement,
|
||||
point: { x: number; y: number },
|
||||
) {
|
||||
const shortSide = Math.max(1, Math.min(canvas.width, canvas.height));
|
||||
const dotRadius = clampNumber(Math.round(shortSide * 0.005), 4, 7);
|
||||
|
||||
context.beginPath();
|
||||
context.arc(point.x, point.y, dotRadius, 0, Math.PI * 2);
|
||||
context.fillStyle = "rgba(220, 38, 38, 0.92)";
|
||||
context.fill();
|
||||
}
|
||||
|
||||
function getSnapshotMarkerPoint(
|
||||
event: GuideEvent,
|
||||
width: number,
|
||||
height: number,
|
||||
): { x: number; y: number } | null {
|
||||
if (event.kind !== "click" && event.kind !== "hotkey") {
|
||||
return null;
|
||||
}
|
||||
if (isNormalizedNumber(event.normalizedX) && isNormalizedNumber(event.normalizedY)) {
|
||||
return {
|
||||
x: clampNumber(event.normalizedX * width, 0, width),
|
||||
y: clampNumber(event.normalizedY * height, 0, height),
|
||||
};
|
||||
}
|
||||
if (isNormalizedNumber(event.x) && isNormalizedNumber(event.y)) {
|
||||
return {
|
||||
x: clampNumber(event.x * width, 0, width),
|
||||
y: clampNumber(event.y * height, 0, height),
|
||||
};
|
||||
}
|
||||
if (
|
||||
typeof event.x === "number" &&
|
||||
typeof event.y === "number" &&
|
||||
Number.isFinite(event.x) &&
|
||||
Number.isFinite(event.y)
|
||||
) {
|
||||
return {
|
||||
x: clampNumber(event.x, 0, width),
|
||||
y: clampNumber(event.y, 0, height),
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function isNormalizedNumber(value: unknown): value is number {
|
||||
return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
|
||||
}
|
||||
|
||||
function clampNumber(value: number, min = 0, max = Number.POSITIVE_INFINITY): number {
|
||||
return Math.min(max, Math.max(min, value));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { GUIDE_SCHEMA_VERSION, type GuideSession } from "./contracts";
|
||||
import { buildGuideVideoAnnotations, buildGuideVideoSpeedRegions } from "./videoAnnotations";
|
||||
|
||||
function createSession(): GuideSession {
|
||||
return {
|
||||
schemaVersion: GUIDE_SCHEMA_VERSION,
|
||||
recordingId: "recording-1",
|
||||
videoPath: "recording.mp4",
|
||||
guidePath: "recording.guide.json",
|
||||
outputDir: "recording-guide",
|
||||
status: "draft-ready",
|
||||
events: [],
|
||||
snapshots: [],
|
||||
ocrBlocks: [],
|
||||
candidates: [
|
||||
{
|
||||
id: "candidate-1",
|
||||
eventId: "event-1",
|
||||
timeMs: 1200,
|
||||
action: "click",
|
||||
targetText: "Settings",
|
||||
targetRole: "button",
|
||||
position: {
|
||||
normalizedX: 0.2,
|
||||
normalizedY: 0.25,
|
||||
xPercent: 20,
|
||||
yPercent: 25,
|
||||
description: "top left",
|
||||
},
|
||||
nearbyText: ["Settings"],
|
||||
confidence: 0.91,
|
||||
},
|
||||
],
|
||||
generatedGuide: {
|
||||
title: "Guide",
|
||||
steps: [
|
||||
{
|
||||
id: "step-1",
|
||||
order: 1,
|
||||
title: "Open settings",
|
||||
instruction: "Click Settings.",
|
||||
sourceCandidateId: "candidate-1",
|
||||
},
|
||||
],
|
||||
},
|
||||
createdAt: "2026-06-04T00:00:00.000Z",
|
||||
updatedAt: "2026-06-04T00:00:00.000Z",
|
||||
};
|
||||
}
|
||||
|
||||
describe("buildGuideVideoAnnotations", () => {
|
||||
it("creates caption and pointer annotations from generated guide candidates", () => {
|
||||
let id = 1;
|
||||
let zIndex = 1;
|
||||
const annotations = buildGuideVideoAnnotations(createSession(), {
|
||||
nextId: () => `guide-video-${id++}`,
|
||||
nextZIndex: () => zIndex++,
|
||||
});
|
||||
|
||||
expect(annotations).toHaveLength(3);
|
||||
expect(annotations[0]).toMatchObject({
|
||||
id: "guide-video-1",
|
||||
type: "text",
|
||||
startMs: 1200,
|
||||
content: "1. Click Settings.",
|
||||
});
|
||||
expect(annotations[0]?.endMs).toBe(3200);
|
||||
expect(annotations[0]?.position.x).toBeGreaterThan(20);
|
||||
expect(annotations[1]?.endMs).toBe(3200);
|
||||
expect(annotations[1]?.position.x).toBeGreaterThan((annotations[0]?.position.x ?? 0) + 34);
|
||||
expect(annotations[1]?.position.y).toBeCloseTo(30.5);
|
||||
expect(annotations[1]).toMatchObject({
|
||||
id: "guide-video-2",
|
||||
type: "magnifier",
|
||||
magnifierData: {
|
||||
target: { x: 20, y: 25 },
|
||||
zoom: 2.2,
|
||||
shape: "circle",
|
||||
caption: "Settings",
|
||||
},
|
||||
});
|
||||
expect(annotations[2]).toMatchObject({
|
||||
id: "guide-video-3",
|
||||
type: "figure",
|
||||
endMs: 3200,
|
||||
figureData: {
|
||||
arrowDirection: "left",
|
||||
color: "#34B27B",
|
||||
},
|
||||
});
|
||||
expect(annotations[2]?.position.x).toBeGreaterThan(20);
|
||||
});
|
||||
|
||||
it("returns an empty list when no draft exists", () => {
|
||||
const session = createSession();
|
||||
session.generatedGuide = undefined;
|
||||
|
||||
const annotations = buildGuideVideoAnnotations(session, {
|
||||
nextId: () => "unused",
|
||||
nextZIndex: () => 1,
|
||||
});
|
||||
|
||||
expect(annotations).toEqual([]);
|
||||
});
|
||||
|
||||
it("creates 0.3x speed regions for two seconds at each guide point", () => {
|
||||
let id = 1;
|
||||
const speedRegions = buildGuideVideoSpeedRegions(createSession(), {
|
||||
nextId: () => `guide-speed-${id++}`,
|
||||
});
|
||||
|
||||
expect(speedRegions).toEqual([
|
||||
{
|
||||
id: "guide-speed-1",
|
||||
startMs: 1200,
|
||||
endMs: 3200,
|
||||
speed: 0.3,
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,249 @@
|
||||
import {
|
||||
type AnnotationRegion,
|
||||
type ArrowDirection,
|
||||
DEFAULT_ANNOTATION_STYLE,
|
||||
DEFAULT_FIGURE_DATA,
|
||||
DEFAULT_MAGNIFIER_DATA,
|
||||
type SpeedRegion,
|
||||
} from "@/components/video-editor/types";
|
||||
import type { GeneratedGuideStep, GuideSession, GuideStepCandidate } from "./contracts";
|
||||
|
||||
export interface BuildGuideVideoAnnotationsOptions {
|
||||
nextId: () => string;
|
||||
nextZIndex: () => number;
|
||||
defaultDurationMs?: number;
|
||||
}
|
||||
|
||||
const DEFAULT_STEP_DURATION_MS = 2000;
|
||||
const DEFAULT_STEP_SLOW_MOTION_DURATION_MS = 2000;
|
||||
const DEFAULT_STEP_SLOW_MOTION_SPEED = 0.3;
|
||||
const CAPTION_WIDTH = 34;
|
||||
const CAPTION_HEIGHT = 13;
|
||||
const MAGNIFIER_SIZE = 18;
|
||||
const ARROW_SIZE = 10;
|
||||
const ANNOTATION_GAP = 2;
|
||||
|
||||
function clamp(value: number, min: number, max: number) {
|
||||
return Math.min(max, Math.max(min, value));
|
||||
}
|
||||
|
||||
function findCandidate(
|
||||
step: GeneratedGuideStep,
|
||||
stepIndex: number,
|
||||
candidates: GuideStepCandidate[],
|
||||
): GuideStepCandidate | undefined {
|
||||
if (step.sourceCandidateId) {
|
||||
const matched = candidates.find((candidate) => candidate.id === step.sourceCandidateId);
|
||||
if (matched) return matched;
|
||||
}
|
||||
const sorted = [...candidates].sort((left, right) => left.timeMs - right.timeMs);
|
||||
return sorted[stepIndex];
|
||||
}
|
||||
|
||||
function getCaptionPosition(candidate: GuideStepCandidate | undefined) {
|
||||
const target = candidate?.position;
|
||||
if (!target) {
|
||||
return { x: 8, y: 8 };
|
||||
}
|
||||
|
||||
const targetX = target.normalizedX * 100;
|
||||
const targetY = target.normalizedY * 100;
|
||||
const x = target.normalizedX < 0.5 ? targetX + 8 : targetX - CAPTION_WIDTH - 8;
|
||||
const y = target.normalizedY < 0.5 ? targetY + 8 : targetY - CAPTION_HEIGHT - 8;
|
||||
|
||||
return {
|
||||
x: clamp(x, 2, 100 - CAPTION_WIDTH - 2),
|
||||
y: clamp(y, 2, 100 - CAPTION_HEIGHT - 2),
|
||||
};
|
||||
}
|
||||
|
||||
function getArrowDirection(
|
||||
candidate: GuideStepCandidate | undefined,
|
||||
originPosition: { x: number; y: number },
|
||||
originSize: { width: number; height: number } = {
|
||||
width: CAPTION_WIDTH,
|
||||
height: CAPTION_HEIGHT,
|
||||
},
|
||||
): ArrowDirection {
|
||||
const target = candidate?.position;
|
||||
if (!target) return "right";
|
||||
|
||||
const originCenterX = originPosition.x + originSize.width / 2;
|
||||
const originCenterY = originPosition.y + originSize.height / 2;
|
||||
const dx = target.normalizedX * 100 - originCenterX;
|
||||
const dy = target.normalizedY * 100 - originCenterY;
|
||||
const horizontal = dx > 8 ? "right" : dx < -8 ? "left" : "";
|
||||
const vertical = dy > 8 ? "down" : dy < -8 ? "up" : "";
|
||||
|
||||
if (vertical && horizontal) return `${vertical}-${horizontal}` as ArrowDirection;
|
||||
return (horizontal || vertical || "right") as ArrowDirection;
|
||||
}
|
||||
|
||||
function getMagnifierPosition(captionPosition: { x: number; y: number }) {
|
||||
const canPlaceRight = captionPosition.x + CAPTION_WIDTH + ANNOTATION_GAP + MAGNIFIER_SIZE <= 98;
|
||||
const x = canPlaceRight
|
||||
? captionPosition.x + CAPTION_WIDTH + ANNOTATION_GAP
|
||||
: captionPosition.x - MAGNIFIER_SIZE - ANNOTATION_GAP;
|
||||
const y = captionPosition.y + (CAPTION_HEIGHT - MAGNIFIER_SIZE) / 2;
|
||||
|
||||
return {
|
||||
x: clamp(x, 2, 100 - MAGNIFIER_SIZE - 2),
|
||||
y: clamp(y, 2, 100 - MAGNIFIER_SIZE - 2),
|
||||
};
|
||||
}
|
||||
|
||||
function getArrowPosition(
|
||||
position: NonNullable<GuideStepCandidate["position"]>,
|
||||
originPosition: { x: number; y: number },
|
||||
originSize: { width: number; height: number },
|
||||
) {
|
||||
const targetX = position.normalizedX * 100;
|
||||
const targetY = position.normalizedY * 100;
|
||||
const originCenterX = originPosition.x + originSize.width / 2;
|
||||
const originCenterY = originPosition.y + originSize.height / 2;
|
||||
const distance = Math.hypot(targetX - originCenterX, targetY - originCenterY);
|
||||
const targetOffset = Math.min(18, Math.max(10, distance * 0.35));
|
||||
const ratio = distance > 0 ? Math.max(0, (distance - targetOffset) / distance) : 0;
|
||||
const arrowCenterX = originCenterX + (targetX - originCenterX) * ratio;
|
||||
const arrowCenterY = originCenterY + (targetY - originCenterY) * ratio;
|
||||
|
||||
return {
|
||||
x: clamp(arrowCenterX - ARROW_SIZE / 2, 0, 100 - ARROW_SIZE),
|
||||
y: clamp(arrowCenterY - ARROW_SIZE / 2, 0, 100 - ARROW_SIZE),
|
||||
};
|
||||
}
|
||||
|
||||
function buildCaption(step: GeneratedGuideStep) {
|
||||
const instruction = step.instruction.trim();
|
||||
const title = step.title.trim();
|
||||
if (instruction) {
|
||||
return `${step.order}. ${instruction}`;
|
||||
}
|
||||
return title ? `${step.order}. ${title}` : `Step ${step.order}`;
|
||||
}
|
||||
|
||||
export function buildGuideVideoAnnotations(
|
||||
session: GuideSession,
|
||||
options: BuildGuideVideoAnnotationsOptions,
|
||||
): AnnotationRegion[] {
|
||||
const guide = session.generatedGuide;
|
||||
if (!guide || guide.steps.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const durationMs = Math.max(1000, options.defaultDurationMs ?? DEFAULT_STEP_DURATION_MS);
|
||||
const sortedSteps = [...guide.steps].sort((left, right) => left.order - right.order);
|
||||
const annotations: AnnotationRegion[] = [];
|
||||
|
||||
for (const [index, step] of sortedSteps.entries()) {
|
||||
const candidate = findCandidate(step, index, session.candidates);
|
||||
const startMs = Math.max(0, Math.round(candidate?.timeMs ?? index * durationMs));
|
||||
const endMs = Math.max(startMs + 750, startMs + durationMs);
|
||||
const captionPosition = getCaptionPosition(candidate);
|
||||
|
||||
annotations.push({
|
||||
id: options.nextId(),
|
||||
startMs,
|
||||
endMs,
|
||||
type: "text",
|
||||
content: buildCaption(step),
|
||||
textContent: buildCaption(step),
|
||||
position: captionPosition,
|
||||
size: { width: CAPTION_WIDTH, height: CAPTION_HEIGHT },
|
||||
style: {
|
||||
...DEFAULT_ANNOTATION_STYLE,
|
||||
color: "#f8fafc",
|
||||
backgroundColor: "rgba(15, 23, 42, 0.88)",
|
||||
fontSize: 18,
|
||||
fontWeight: "bold",
|
||||
textAlign: "left",
|
||||
},
|
||||
zIndex: options.nextZIndex(),
|
||||
});
|
||||
|
||||
if (candidate?.position) {
|
||||
const magnifierPosition = getMagnifierPosition(captionPosition);
|
||||
const arrowPosition = getArrowPosition(candidate.position, magnifierPosition, {
|
||||
width: MAGNIFIER_SIZE,
|
||||
height: MAGNIFIER_SIZE,
|
||||
});
|
||||
const arrowDirection = getArrowDirection(candidate, arrowPosition, {
|
||||
width: ARROW_SIZE,
|
||||
height: ARROW_SIZE,
|
||||
});
|
||||
|
||||
annotations.push({
|
||||
id: options.nextId(),
|
||||
startMs,
|
||||
endMs,
|
||||
type: "magnifier",
|
||||
content: buildCaption(step),
|
||||
position: magnifierPosition,
|
||||
size: { width: MAGNIFIER_SIZE, height: MAGNIFIER_SIZE },
|
||||
style: { ...DEFAULT_ANNOTATION_STYLE },
|
||||
zIndex: options.nextZIndex(),
|
||||
magnifierData: {
|
||||
...DEFAULT_MAGNIFIER_DATA,
|
||||
target: {
|
||||
x: candidate.position.normalizedX * 100,
|
||||
y: candidate.position.normalizedY * 100,
|
||||
},
|
||||
caption: candidate.targetText,
|
||||
},
|
||||
});
|
||||
annotations.push({
|
||||
id: options.nextId(),
|
||||
startMs,
|
||||
endMs,
|
||||
type: "figure",
|
||||
content: "",
|
||||
position: arrowPosition,
|
||||
size: { width: ARROW_SIZE, height: ARROW_SIZE },
|
||||
style: { ...DEFAULT_ANNOTATION_STYLE },
|
||||
zIndex: options.nextZIndex(),
|
||||
figureData: {
|
||||
...DEFAULT_FIGURE_DATA,
|
||||
arrowDirection,
|
||||
color: "#34B27B",
|
||||
strokeWidth: 5,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return annotations;
|
||||
}
|
||||
|
||||
export interface BuildGuideVideoSpeedRegionsOptions {
|
||||
nextId: () => string;
|
||||
durationMs?: number;
|
||||
speed?: number;
|
||||
}
|
||||
|
||||
export function buildGuideVideoSpeedRegions(
|
||||
session: GuideSession,
|
||||
options: BuildGuideVideoSpeedRegionsOptions,
|
||||
): SpeedRegion[] {
|
||||
const guide = session.generatedGuide;
|
||||
if (!guide || guide.steps.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const durationMs = Math.max(
|
||||
100,
|
||||
Math.round(options.durationMs ?? DEFAULT_STEP_SLOW_MOTION_DURATION_MS),
|
||||
);
|
||||
const speed = options.speed ?? DEFAULT_STEP_SLOW_MOTION_SPEED;
|
||||
const sortedSteps = [...guide.steps].sort((left, right) => left.order - right.order);
|
||||
|
||||
return sortedSteps.map((step, index) => {
|
||||
const candidate = findCandidate(step, index, session.candidates);
|
||||
const startMs = Math.max(0, Math.round(candidate?.timeMs ?? index * durationMs));
|
||||
return {
|
||||
id: options.nextId(),
|
||||
startMs,
|
||||
endMs: startMs + durationMs,
|
||||
speed,
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -9,6 +9,8 @@ import {
|
||||
|
||||
let blurScratchCanvas: HTMLCanvasElement | null = null;
|
||||
let blurScratchCtx: CanvasRenderingContext2D | null = null;
|
||||
let magnifierScratchCanvas: HTMLCanvasElement | null = null;
|
||||
let magnifierScratchCtx: CanvasRenderingContext2D | null = null;
|
||||
|
||||
// Matches a single code point whose script is Han (including non-BMP
|
||||
// Extension A-F), Hiragana, Katakana (including halfwidth forms), or
|
||||
@@ -396,6 +398,130 @@ async function renderImage(
|
||||
});
|
||||
}
|
||||
|
||||
function renderMagnifier(
|
||||
ctx: CanvasRenderingContext2D,
|
||||
annotation: AnnotationRegion,
|
||||
x: number,
|
||||
y: number,
|
||||
width: number,
|
||||
height: number,
|
||||
canvasWidth: number,
|
||||
canvasHeight: number,
|
||||
scaleFactor: number,
|
||||
) {
|
||||
if (!magnifierScratchCanvas || !magnifierScratchCtx) {
|
||||
magnifierScratchCanvas = document.createElement("canvas");
|
||||
magnifierScratchCtx = magnifierScratchCanvas.getContext("2d");
|
||||
}
|
||||
if (!magnifierScratchCanvas || !magnifierScratchCtx) return;
|
||||
|
||||
const data = annotation.magnifierData;
|
||||
const zoom = Math.max(1, data?.zoom ?? 2.2);
|
||||
const target = data?.target ?? {
|
||||
x: annotation.position.x + annotation.size.width / 2,
|
||||
y: annotation.position.y + annotation.size.height / 2,
|
||||
};
|
||||
const targetX = (target.x / 100) * canvasWidth;
|
||||
const targetY = (target.y / 100) * canvasHeight;
|
||||
const sampleWidth = Math.max(1, width / zoom);
|
||||
const sampleHeight = Math.max(1, height / zoom);
|
||||
const sx = Math.max(0, Math.min(canvasWidth - sampleWidth, targetX - sampleWidth / 2));
|
||||
const sy = Math.max(0, Math.min(canvasHeight - sampleHeight, targetY - sampleHeight / 2));
|
||||
const sw = Math.max(1, Math.min(sampleWidth, canvasWidth - sx));
|
||||
const sh = Math.max(1, Math.min(sampleHeight, canvasHeight - sy));
|
||||
|
||||
magnifierScratchCanvas.width = Math.max(1, Math.round(width));
|
||||
magnifierScratchCanvas.height = Math.max(1, Math.round(height));
|
||||
magnifierScratchCtx.clearRect(0, 0, magnifierScratchCanvas.width, magnifierScratchCanvas.height);
|
||||
magnifierScratchCtx.imageSmoothingEnabled = true;
|
||||
magnifierScratchCtx.imageSmoothingQuality = "high";
|
||||
magnifierScratchCtx.drawImage(
|
||||
ctx.canvas,
|
||||
sx,
|
||||
sy,
|
||||
sw,
|
||||
sh,
|
||||
0,
|
||||
0,
|
||||
magnifierScratchCanvas.width,
|
||||
magnifierScratchCanvas.height,
|
||||
);
|
||||
|
||||
const centerX = x + width / 2;
|
||||
const centerY = y + height / 2;
|
||||
const shape = data?.shape ?? "circle";
|
||||
const radius = Math.min(width, height) / 2;
|
||||
const cornerRadius = shape === "circle" ? radius : Math.min(18 * scaleFactor, radius);
|
||||
|
||||
ctx.save();
|
||||
ctx.strokeStyle = "rgba(52,178,123,0.85)";
|
||||
ctx.lineWidth = Math.max(2, 2 * scaleFactor);
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(centerX, centerY);
|
||||
ctx.lineTo(targetX, targetY);
|
||||
ctx.stroke();
|
||||
ctx.fillStyle = "#34B27B";
|
||||
ctx.beginPath();
|
||||
ctx.arc(targetX, targetY, Math.max(4, 4 * scaleFactor), 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
|
||||
ctx.shadowColor = "rgba(0,0,0,0.38)";
|
||||
ctx.shadowBlur = 24 * scaleFactor;
|
||||
ctx.shadowOffsetY = 12 * scaleFactor;
|
||||
ctx.fillStyle = "rgba(15,23,42,0.92)";
|
||||
ctx.beginPath();
|
||||
ctx.roundRect(x, y, width, height, cornerRadius);
|
||||
ctx.fill();
|
||||
ctx.restore();
|
||||
|
||||
ctx.save();
|
||||
ctx.beginPath();
|
||||
ctx.roundRect(x, y, width, height, cornerRadius);
|
||||
ctx.clip();
|
||||
ctx.drawImage(magnifierScratchCanvas, x, y, width, height);
|
||||
ctx.restore();
|
||||
|
||||
ctx.save();
|
||||
ctx.strokeStyle = "rgba(248,250,252,0.96)";
|
||||
ctx.lineWidth = Math.max(3, 3 * scaleFactor);
|
||||
ctx.beginPath();
|
||||
ctx.roundRect(x, y, width, height, cornerRadius);
|
||||
ctx.stroke();
|
||||
ctx.strokeStyle = "rgba(52,178,123,0.58)";
|
||||
ctx.lineWidth = Math.max(1, 1.5 * scaleFactor);
|
||||
ctx.beginPath();
|
||||
ctx.roundRect(
|
||||
x + 2 * scaleFactor,
|
||||
y + 2 * scaleFactor,
|
||||
width - 4 * scaleFactor,
|
||||
height - 4 * scaleFactor,
|
||||
cornerRadius,
|
||||
);
|
||||
ctx.stroke();
|
||||
|
||||
const caption = data?.caption || "";
|
||||
if (caption) {
|
||||
const fontSize = Math.max(12, 13 * scaleFactor);
|
||||
ctx.font = `bold ${fontSize}px Inter, sans-serif`;
|
||||
ctx.textAlign = "center";
|
||||
ctx.textBaseline = "middle";
|
||||
const paddingX = 8 * scaleFactor;
|
||||
const paddingY = 5 * scaleFactor;
|
||||
const metrics = ctx.measureText(caption);
|
||||
const captionWidth = Math.min(width * 1.6, metrics.width + paddingX * 2);
|
||||
const captionHeight = fontSize + paddingY * 2;
|
||||
const captionX = centerX - captionWidth / 2;
|
||||
const captionY = y + height + 8 * scaleFactor;
|
||||
ctx.fillStyle = "rgba(15,23,42,0.92)";
|
||||
ctx.beginPath();
|
||||
ctx.roundRect(captionX, captionY, captionWidth, captionHeight, 6 * scaleFactor);
|
||||
ctx.fill();
|
||||
ctx.fillStyle = "#f8fafc";
|
||||
ctx.fillText(caption, centerX, captionY + captionHeight / 2, captionWidth - paddingX * 2);
|
||||
}
|
||||
ctx.restore();
|
||||
}
|
||||
|
||||
export async function renderAnnotations(
|
||||
ctx: CanvasRenderingContext2D,
|
||||
annotations: AnnotationRegion[],
|
||||
@@ -443,6 +569,20 @@ export async function renderAnnotations(
|
||||
}
|
||||
break;
|
||||
|
||||
case "magnifier":
|
||||
renderMagnifier(
|
||||
ctx,
|
||||
annotation,
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
canvasWidth,
|
||||
canvasHeight,
|
||||
scaleFactor,
|
||||
);
|
||||
break;
|
||||
|
||||
case "blur":
|
||||
renderBlur(ctx, annotation, x, y, width, height, scaleFactor);
|
||||
break;
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
export type UpdateStatusPhase =
|
||||
| "idle"
|
||||
| "checking"
|
||||
| "available"
|
||||
| "not-available"
|
||||
| "downloading"
|
||||
| "downloaded"
|
||||
| "error"
|
||||
| "unsupported";
|
||||
|
||||
export interface UpdateStatus {
|
||||
phase: UpdateStatusPhase;
|
||||
currentVersion: string;
|
||||
version?: string;
|
||||
releaseName?: string;
|
||||
releaseNotes?: string;
|
||||
percent?: number;
|
||||
bytesPerSecond?: number;
|
||||
transferred?: number;
|
||||
total?: number;
|
||||
error?: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface UpdateCheckResult {
|
||||
success: boolean;
|
||||
status: UpdateStatus;
|
||||
error?: string;
|
||||
}
|
||||
+327
-19
@@ -5,8 +5,9 @@ import importlib.util
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from threading import Lock
|
||||
from threading import Lock, Thread
|
||||
from typing import Any
|
||||
|
||||
from fastapi import FastAPI, HTTPException
|
||||
@@ -17,6 +18,67 @@ app = FastAPI(title="OpenScreen PaddleOCR service")
|
||||
|
||||
_engines: dict[str, Any] = {}
|
||||
_engine_lock = Lock()
|
||||
_warmup_lock = Lock()
|
||||
_warmup_started = False
|
||||
_LATIN_RECOGNITION_LANGS = {
|
||||
"af",
|
||||
"az",
|
||||
"bs",
|
||||
"ca",
|
||||
"cs",
|
||||
"cy",
|
||||
"da",
|
||||
"de",
|
||||
"en",
|
||||
"es",
|
||||
"et",
|
||||
"eu",
|
||||
"fi",
|
||||
"fr",
|
||||
"ga",
|
||||
"gl",
|
||||
"hr",
|
||||
"hu",
|
||||
"id",
|
||||
"is",
|
||||
"it",
|
||||
"ku",
|
||||
"la",
|
||||
"latin",
|
||||
"lb",
|
||||
"lt",
|
||||
"lv",
|
||||
"mi",
|
||||
"ms",
|
||||
"mt",
|
||||
"nl",
|
||||
"no",
|
||||
"oc",
|
||||
"pi",
|
||||
"pl",
|
||||
"pt",
|
||||
"qu",
|
||||
"rm",
|
||||
"ro",
|
||||
"rs_latin",
|
||||
"rslatin",
|
||||
"sk",
|
||||
"sl",
|
||||
"sq",
|
||||
"sv",
|
||||
"sw",
|
||||
"tl",
|
||||
"tr",
|
||||
"uz",
|
||||
"vi",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PreparedImage:
|
||||
path: str
|
||||
scale: float = 1.0
|
||||
should_delete: bool = False
|
||||
|
||||
|
||||
class OcrRequest(BaseModel):
|
||||
@@ -24,6 +86,21 @@ class OcrRequest(BaseModel):
|
||||
path: str | None = None
|
||||
imagePath: str | None = None
|
||||
language: str | None = None
|
||||
profile: str | None = None
|
||||
|
||||
|
||||
@app.on_event("startup")
|
||||
def start_ocr_warmup() -> None:
|
||||
if os.getenv("OPENSCREEN_OCR_WARMUP", "0") != "1":
|
||||
return
|
||||
|
||||
global _warmup_started
|
||||
with _warmup_lock:
|
||||
if _warmup_started:
|
||||
return
|
||||
_warmup_started = True
|
||||
|
||||
Thread(target=_warmup_default_engines, name="openscreen-ocr-warmup", daemon=True).start()
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
@@ -33,16 +110,31 @@ def health() -> dict[str, Any]:
|
||||
"paddleocrInstalled": importlib.util.find_spec("paddleocr") is not None,
|
||||
"paddleInstalled": importlib.util.find_spec("paddle") is not None,
|
||||
"engineReady": bool(_engines),
|
||||
"defaultLanguage": os.getenv("PADDLEOCR_LANG", "latin"),
|
||||
"defaultLanguage": os.getenv("PADDLEOCR_LANG") or "vi,en",
|
||||
"defaultProfile": os.getenv("OPENSCREEN_OCR_PROFILE") or "vietnamese",
|
||||
"loadedEngines": sorted(_engines.keys()),
|
||||
}
|
||||
|
||||
|
||||
def _warmup_default_engines() -> None:
|
||||
try:
|
||||
profile = _resolve_ocr_profile(None)
|
||||
for paddle_lang in _resolve_paddle_languages(None, profile):
|
||||
_get_engine(paddle_lang)
|
||||
except Exception as error:
|
||||
print(f"OpenScreen OCR warmup failed: {error}", file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
@app.post("/ocr")
|
||||
async def ocr(request: OcrRequest) -> dict[str, Any]:
|
||||
image_path, should_delete = _resolve_image_path(request)
|
||||
try:
|
||||
engine = _get_engine(request.language)
|
||||
blocks = await run_in_threadpool(_recognize_blocks, engine, image_path)
|
||||
blocks = await run_in_threadpool(
|
||||
_recognize_profile_blocks,
|
||||
image_path,
|
||||
request.language,
|
||||
request.profile,
|
||||
)
|
||||
return {"blocks": blocks}
|
||||
finally:
|
||||
if should_delete:
|
||||
@@ -73,8 +165,7 @@ def _resolve_image_path(request: OcrRequest) -> tuple[str, bool]:
|
||||
return handle.name, True
|
||||
|
||||
|
||||
def _get_engine(language: str | None) -> Any:
|
||||
paddle_lang = _resolve_paddle_language(language)
|
||||
def _get_engine(paddle_lang: str) -> Any:
|
||||
cache_key = f"{paddle_lang}|{os.getenv('PADDLEOCR_DEVICE', 'cpu')}"
|
||||
with _engine_lock:
|
||||
if cache_key not in _engines:
|
||||
@@ -105,13 +196,17 @@ def _create_engine(paddle_lang: str) -> Any:
|
||||
"enable_mkldnn": os.getenv("PADDLEOCR_ENABLE_MKLDNN", "0") == "1",
|
||||
"use_doc_orientation_classify": False,
|
||||
"use_doc_unwarping": False,
|
||||
"use_textline_orientation": False,
|
||||
"use_textline_orientation": os.getenv("PADDLEOCR_USE_TEXTLINE_ORIENTATION", "0") == "1",
|
||||
}
|
||||
if os.getenv("PADDLEOCR_USE_MOBILE", "1") != "0":
|
||||
modern_kwargs.update(
|
||||
{
|
||||
"text_detection_model_name": "PP-OCRv5_mobile_det",
|
||||
"text_recognition_model_name": _mobile_recognition_model(paddle_lang),
|
||||
"text_detection_model_name": os.getenv(
|
||||
"PADDLEOCR_DET_MODEL",
|
||||
"PP-OCRv5_mobile_det",
|
||||
),
|
||||
"text_recognition_model_name": os.getenv("PADDLEOCR_REC_MODEL")
|
||||
or _mobile_recognition_model(paddle_lang),
|
||||
}
|
||||
)
|
||||
|
||||
@@ -150,23 +245,236 @@ def _patch_paddlex_frozen_ocr_extra_gate() -> None:
|
||||
deps._openscreen_ocr_extra_patch = True
|
||||
|
||||
|
||||
def _resolve_paddle_language(language: str | None) -> str:
|
||||
explicit = os.getenv("PADDLEOCR_LANG")
|
||||
def _recognize_profile_blocks(
|
||||
image_path: str,
|
||||
language: str | None,
|
||||
profile: str | None,
|
||||
) -> list[dict[str, Any]]:
|
||||
ocr_profile = _resolve_ocr_profile(profile)
|
||||
languages = _resolve_paddle_languages(language, ocr_profile)
|
||||
prepared = _prepare_image_for_profile(image_path, ocr_profile)
|
||||
try:
|
||||
blocks: list[dict[str, Any]] = []
|
||||
for paddle_lang in languages:
|
||||
engine = _get_engine(paddle_lang)
|
||||
recognized = _recognize_blocks(engine, prepared.path)
|
||||
blocks.extend(_scale_blocks(recognized, prepared.scale))
|
||||
return _merge_blocks(blocks)
|
||||
finally:
|
||||
if prepared.should_delete:
|
||||
Path(prepared.path).unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _resolve_ocr_profile(profile: str | None) -> str:
|
||||
explicit = (os.getenv("OPENSCREEN_OCR_PROFILE") or "").strip().lower()
|
||||
value = explicit or (profile or "").strip().lower()
|
||||
if value in {"fast", "vietnamese", "hybrid"}:
|
||||
return value
|
||||
return "vietnamese"
|
||||
|
||||
|
||||
def _resolve_paddle_languages(language: str | None, profile: str) -> list[str]:
|
||||
explicit = (os.getenv("PADDLEOCR_LANG") or "").strip().lower()
|
||||
if explicit:
|
||||
return explicit
|
||||
return [explicit]
|
||||
|
||||
language_value = (language or "vi,en").lower()
|
||||
if "vi" in language_value or "latin" in language_value:
|
||||
has_vietnamese = "vi" in _split_language_tags(language_value)
|
||||
if profile == "fast":
|
||||
return [_resolve_primary_paddle_language(language_value, prefer_vietnamese=False)]
|
||||
if profile == "hybrid":
|
||||
languages = ["vi"] if has_vietnamese else []
|
||||
languages.append("latin")
|
||||
return _dedupe_languages(languages)
|
||||
return [_resolve_primary_paddle_language(language_value, prefer_vietnamese=True)]
|
||||
|
||||
|
||||
def _split_language_tags(language: str) -> set[str]:
|
||||
return {part.strip().lower() for part in language.split(",") if part.strip()}
|
||||
|
||||
|
||||
def _dedupe_languages(languages: list[str]) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
result: list[str] = []
|
||||
for language in languages:
|
||||
if language not in seen:
|
||||
seen.add(language)
|
||||
result.append(language)
|
||||
return result
|
||||
|
||||
|
||||
def _resolve_primary_paddle_language(language_value: str, *, prefer_vietnamese: bool) -> str:
|
||||
tags = _split_language_tags(language_value)
|
||||
if prefer_vietnamese and "vi" in tags:
|
||||
return "vi"
|
||||
if "latin" in tags or "vi" in tags or "en" in tags:
|
||||
return "latin"
|
||||
if "en" in language_value:
|
||||
return "en"
|
||||
return language_value.split(",")[0].strip() or "latin"
|
||||
for tag in tags:
|
||||
return tag
|
||||
return "latin"
|
||||
|
||||
|
||||
def _prepare_image_for_profile(image_path: str, profile: str) -> PreparedImage:
|
||||
if profile == "fast":
|
||||
return PreparedImage(image_path)
|
||||
|
||||
try:
|
||||
from PIL import Image, ImageEnhance, ImageOps
|
||||
except Exception:
|
||||
return PreparedImage(image_path)
|
||||
|
||||
try:
|
||||
with Image.open(image_path) as source:
|
||||
image = source.convert("RGB")
|
||||
except Exception:
|
||||
return PreparedImage(image_path)
|
||||
|
||||
scale = _resolve_enhancement_scale(image.width, image.height)
|
||||
if scale <= 1:
|
||||
return PreparedImage(image_path)
|
||||
|
||||
resampling = getattr(getattr(Image, "Resampling", Image), "LANCZOS")
|
||||
enhanced = image.resize((round(image.width * scale), round(image.height * scale)), resampling)
|
||||
enhanced = ImageOps.autocontrast(enhanced)
|
||||
enhanced = ImageEnhance.Contrast(enhanced).enhance(1.25)
|
||||
enhanced = ImageEnhance.Sharpness(enhanced).enhance(1.35)
|
||||
|
||||
handle = tempfile.NamedTemporaryFile(prefix="openscreen-ocr-enhanced-", suffix=".png", delete=False)
|
||||
try:
|
||||
handle.close()
|
||||
enhanced.save(handle.name, format="PNG")
|
||||
return PreparedImage(handle.name, scale=scale, should_delete=True)
|
||||
except Exception:
|
||||
Path(handle.name).unlink(missing_ok=True)
|
||||
return PreparedImage(image_path)
|
||||
|
||||
|
||||
def _resolve_enhancement_scale(width: int, height: int) -> float:
|
||||
try:
|
||||
requested_scale = float(os.getenv("OPENSCREEN_OCR_ENHANCE_SCALE", "2"))
|
||||
except ValueError:
|
||||
requested_scale = 2.0
|
||||
scale = max(1.0, min(3.0, requested_scale))
|
||||
try:
|
||||
max_side = int(os.getenv("OPENSCREEN_OCR_ENHANCE_MAX_SIDE", "2400"))
|
||||
except ValueError:
|
||||
max_side = 2400
|
||||
largest_side = max(width, height)
|
||||
if largest_side <= 0:
|
||||
return 1.0
|
||||
return max(1.0, min(scale, max_side / largest_side))
|
||||
|
||||
|
||||
def _scale_blocks(blocks: list[dict[str, Any]], scale: float) -> list[dict[str, Any]]:
|
||||
if scale <= 1:
|
||||
return blocks
|
||||
|
||||
scaled_blocks: list[dict[str, Any]] = []
|
||||
for block in blocks:
|
||||
box = block.get("box")
|
||||
if not isinstance(box, dict) or not _box_uses_pixels(box):
|
||||
scaled_blocks.append(block)
|
||||
continue
|
||||
scaled_box = {
|
||||
"x": float(box["x"]) / scale,
|
||||
"y": float(box["y"]) / scale,
|
||||
"width": float(box["width"]) / scale,
|
||||
"height": float(box["height"]) / scale,
|
||||
}
|
||||
scaled_blocks.append({**block, "box": scaled_box})
|
||||
return scaled_blocks
|
||||
|
||||
|
||||
def _box_uses_pixels(box: dict[str, Any]) -> bool:
|
||||
try:
|
||||
x = float(box["x"])
|
||||
y = float(box["y"])
|
||||
width = float(box["width"])
|
||||
height = float(box["height"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return False
|
||||
return x > 1 or y > 1 or width > 1 or height > 1 or x + width > 1 or y + height > 1
|
||||
|
||||
|
||||
def _merge_blocks(blocks: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
merged: list[dict[str, Any]] = []
|
||||
for block in sorted(blocks, key=_block_quality, reverse=True):
|
||||
box = block.get("box")
|
||||
if not isinstance(box, dict):
|
||||
continue
|
||||
overlapping_index = next(
|
||||
(
|
||||
index
|
||||
for index, existing in enumerate(merged)
|
||||
if _box_iou(box, existing.get("box")) >= 0.62
|
||||
),
|
||||
None,
|
||||
)
|
||||
if overlapping_index is None:
|
||||
merged.append(block)
|
||||
continue
|
||||
if _block_quality(block) > _block_quality(merged[overlapping_index]):
|
||||
merged[overlapping_index] = block
|
||||
return sorted(merged, key=lambda block: _box_sort_key(block.get("box")))
|
||||
|
||||
|
||||
def _block_quality(block: dict[str, Any]) -> float:
|
||||
text = str(block.get("text") or "")
|
||||
score = _score_to_float(block.get("confidence"))
|
||||
if _has_vietnamese_diacritics(text):
|
||||
score += 0.08
|
||||
if len(text) >= 2:
|
||||
score += min(0.04, len(text) * 0.002)
|
||||
return score
|
||||
|
||||
|
||||
def _has_vietnamese_diacritics(text: str) -> bool:
|
||||
return any(
|
||||
character
|
||||
in "ăâđêôơưĂÂĐÊÔƠƯáàảãạắằẳẵặấầẩẫậéèẻẽẹếềểễệíìỉĩịóòỏõọốồổỗộớờởỡợúùủũụứừửữựýỳỷỹỵ"
|
||||
for character in text
|
||||
)
|
||||
|
||||
|
||||
def _box_iou(left: Any, right: Any) -> float:
|
||||
if not isinstance(left, dict) or not isinstance(right, dict):
|
||||
return 0.0
|
||||
try:
|
||||
left_x = float(left["x"])
|
||||
left_y = float(left["y"])
|
||||
left_width = float(left["width"])
|
||||
left_height = float(left["height"])
|
||||
right_x = float(right["x"])
|
||||
right_y = float(right["y"])
|
||||
right_width = float(right["width"])
|
||||
right_height = float(right["height"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return 0.0
|
||||
|
||||
intersection_left = max(left_x, right_x)
|
||||
intersection_top = max(left_y, right_y)
|
||||
intersection_right = min(left_x + left_width, right_x + right_width)
|
||||
intersection_bottom = min(left_y + left_height, right_y + right_height)
|
||||
intersection_width = max(0.0, intersection_right - intersection_left)
|
||||
intersection_height = max(0.0, intersection_bottom - intersection_top)
|
||||
intersection_area = intersection_width * intersection_height
|
||||
if intersection_area <= 0:
|
||||
return 0.0
|
||||
union_area = left_width * left_height + right_width * right_height - intersection_area
|
||||
return intersection_area / union_area if union_area > 0 else 0.0
|
||||
|
||||
|
||||
def _box_sort_key(box: Any) -> tuple[float, float]:
|
||||
if not isinstance(box, dict):
|
||||
return (0.0, 0.0)
|
||||
try:
|
||||
return (float(box["y"]), float(box["x"]))
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return (0.0, 0.0)
|
||||
|
||||
|
||||
def _mobile_recognition_model(paddle_lang: str) -> str:
|
||||
if paddle_lang == "en":
|
||||
return "en_PP-OCRv5_mobile_rec"
|
||||
if paddle_lang == "latin":
|
||||
if paddle_lang in _LATIN_RECOGNITION_LANGS:
|
||||
return "latin_PP-OCRv5_mobile_rec"
|
||||
return "PP-OCRv5_mobile_rec"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user