Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5069354df3 | |||
| ee69df9222 | |||
| a235a0c50b | |||
| 94490a71af | |||
| 6ebabbaaaa | |||
| 0bd26eebf7 | |||
| cce81dd7c4 | |||
| 7823507a18 | |||
| 0b78ff6f7d |
@@ -0,0 +1,17 @@
|
|||||||
|
!macro customInstall
|
||||||
|
DetailPrint "Installing OpenScreen OCR Windows service"
|
||||||
|
nsExec::ExecToLog '"$SYSDIR\sc.exe" stop OpenScreenOCR'
|
||||||
|
nsExec::ExecToLog '"$SYSDIR\sc.exe" delete OpenScreenOCR'
|
||||||
|
Sleep 1000
|
||||||
|
ExpandEnvStrings $0 "%ProgramData%\OpenScreen\ocr-runtime"
|
||||||
|
CreateDirectory "$0"
|
||||||
|
nsExec::ExecToLog '"$SYSDIR\sc.exe" create OpenScreenOCR binPath= "\"$INSTDIR\resources\electron\native\bin\win32-x64\openscreen-ocr-service-wrapper.exe\" --service --exe \"$INSTDIR\resources\ocr-service\openscreen-ocr-service.exe\" --resources \"$INSTDIR\resources\" --data \"$0\"" start= auto DisplayName= "OpenScreen OCR Service"'
|
||||||
|
nsExec::ExecToLog '"$SYSDIR\sc.exe" description OpenScreenOCR "Local OCR service used by OpenScreen guide capture."'
|
||||||
|
nsExec::ExecToLog '"$SYSDIR\sc.exe" start OpenScreenOCR'
|
||||||
|
!macroend
|
||||||
|
|
||||||
|
!macro customUnInstall
|
||||||
|
DetailPrint "Removing OpenScreen OCR Windows service"
|
||||||
|
nsExec::ExecToLog '"$SYSDIR\sc.exe" stop OpenScreenOCR'
|
||||||
|
nsExec::ExecToLog '"$SYSDIR\sc.exe" delete OpenScreenOCR'
|
||||||
|
!macroend
|
||||||
@@ -6,7 +6,7 @@ OpenScreen calls OCR through a local HTTP service. The default endpoint is:
|
|||||||
http://127.0.0.1:8866/ocr
|
http://127.0.0.1:8866/ocr
|
||||||
```
|
```
|
||||||
|
|
||||||
The app sends either `imageBase64` or `path` and expects OCR blocks:
|
The app sends either `imageBase64` or `path`, plus optional `language` and `profile`, and expects OCR blocks:
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
@@ -38,7 +38,7 @@ If `paddle` is still missing after installing `paddleocr`, install the CPU Paddl
|
|||||||
```powershell
|
```powershell
|
||||||
.\.venv-ocr\Scripts\Activate.ps1
|
.\.venv-ocr\Scripts\Activate.ps1
|
||||||
$env:PADDLEOCR_DEVICE="cpu"
|
$env:PADDLEOCR_DEVICE="cpu"
|
||||||
$env:PADDLEOCR_LANG="latin"
|
$env:OPENSCREEN_OCR_PROFILE="vietnamese"
|
||||||
npm run ocr:paddle
|
npm run ocr:paddle
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -58,7 +58,8 @@ Expected healthy environment:
|
|||||||
"paddleocrInstalled": true,
|
"paddleocrInstalled": true,
|
||||||
"paddleInstalled": true,
|
"paddleInstalled": true,
|
||||||
"engineReady": false,
|
"engineReady": false,
|
||||||
"defaultLanguage": "latin"
|
"defaultLanguage": "vi,en",
|
||||||
|
"defaultProfile": "vietnamese"
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -67,7 +68,10 @@ Expected healthy environment:
|
|||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
- `PADDLEOCR_DEVICE`: `cpu`, `gpu:0`, or another PaddleOCR device string.
|
- `PADDLEOCR_DEVICE`: `cpu`, `gpu:0`, or another PaddleOCR device string.
|
||||||
- `PADDLEOCR_LANG`: defaults to `latin`; this is preferred for Vietnamese UI text because it uses a Latin-script recognition model.
|
- `OPENSCREEN_OCR_PROFILE`: `fast`, `vietnamese`, or `hybrid`. The default `vietnamese` profile upscales and sharpens focused UI screenshots before OCR.
|
||||||
|
- `OPENSCREEN_GUIDE_OCR_LANGUAGE`: defaults to `vi,en`.
|
||||||
|
- `PADDLEOCR_LANG`: optional hard override. Leave unset for the app profile/language settings to work.
|
||||||
- `PADDLEOCR_VERSION`: defaults to `PP-OCRv5`.
|
- `PADDLEOCR_VERSION`: defaults to `PP-OCRv5`.
|
||||||
- `PADDLEOCR_USE_MOBILE`: defaults to `1`; set to `0` to use the default/server models.
|
- `PADDLEOCR_USE_MOBILE`: defaults to `1`; set to `0` to use the default/server models.
|
||||||
|
- `PADDLEOCR_REC_MODEL`: optional recognizer model override. The bundled profile uses `latin_PP-OCRv5_mobile_rec`, which supports Vietnamese Latin-script text.
|
||||||
- `OPENSCREEN_GUIDE_OCR_URL`: OpenScreen OCR endpoint override; defaults to `http://127.0.0.1:8866`.
|
- `OPENSCREEN_GUIDE_OCR_URL`: OpenScreen OCR endpoint override; defaults to `http://127.0.0.1:8866`.
|
||||||
|
|||||||
+19
-10
@@ -13,11 +13,17 @@
|
|||||||
},
|
},
|
||||||
"npmRebuild": true,
|
"npmRebuild": true,
|
||||||
"buildDependenciesFromSource": true,
|
"buildDependenciesFromSource": true,
|
||||||
"compression": "normal",
|
"compression": "normal",
|
||||||
"directories": {
|
"directories": {
|
||||||
"output": "release/${version}"
|
"output": "release/${version}"
|
||||||
},
|
},
|
||||||
"files": [
|
"publish": [
|
||||||
|
{
|
||||||
|
"provider": "generic",
|
||||||
|
"url": "https://gittea.softs.business/huanld/openscreen/raw/branch/release-assets%2Flatest"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"files": [
|
||||||
"dist",
|
"dist",
|
||||||
"dist-electron",
|
"dist-electron",
|
||||||
"!*.png",
|
"!*.png",
|
||||||
@@ -79,6 +85,7 @@
|
|||||||
"nsis"
|
"nsis"
|
||||||
],
|
],
|
||||||
"icon": "icons/icons/win/icon.ico",
|
"icon": "icons/icons/win/icon.ico",
|
||||||
|
"requestedExecutionLevel": "requireAdministrator",
|
||||||
"signAndEditExecutable": false,
|
"signAndEditExecutable": false,
|
||||||
"signExts": ["!.exe"],
|
"signExts": ["!.exe"],
|
||||||
"extraResources": [
|
"extraResources": [
|
||||||
@@ -99,8 +106,10 @@
|
|||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"nsis": {
|
"nsis": {
|
||||||
"oneClick": false,
|
"oneClick": false,
|
||||||
"allowToChangeInstallationDirectory": true
|
"allowToChangeInstallationDirectory": true,
|
||||||
}
|
"perMachine": true,
|
||||||
}
|
"include": "build/installer.nsh"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Vendored
+8
@@ -24,6 +24,14 @@ declare namespace NodeJS {
|
|||||||
// Used in Renderer process, expose in `preload.ts`
|
// Used in Renderer process, expose in `preload.ts`
|
||||||
interface Window {
|
interface Window {
|
||||||
electronAPI: {
|
electronAPI: {
|
||||||
|
updates: {
|
||||||
|
getStatus: () => Promise<import("../src/lib/updateStatus").UpdateStatus>;
|
||||||
|
check: () => Promise<import("../src/lib/updateStatus").UpdateCheckResult>;
|
||||||
|
install: () => Promise<import("../src/lib/updateStatus").UpdateCheckResult>;
|
||||||
|
onStatus: (
|
||||||
|
callback: (status: import("../src/lib/updateStatus").UpdateStatus) => void,
|
||||||
|
) => () => void;
|
||||||
|
};
|
||||||
invokeNativeBridge: <TData = unknown>(
|
invokeNativeBridge: <TData = unknown>(
|
||||||
request: import("../src/native/contracts").NativeBridgeRequest,
|
request: import("../src/native/contracts").NativeBridgeRequest,
|
||||||
) => Promise<import("../src/native/contracts").NativeBridgeResponse<TData>>;
|
) => Promise<import("../src/native/contracts").NativeBridgeResponse<TData>>;
|
||||||
|
|||||||
@@ -0,0 +1,66 @@
|
|||||||
|
import fs from "node:fs/promises";
|
||||||
|
import os from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||||
|
import { DeepSeekSettingsStore } from "./deepseekSettingsStore";
|
||||||
|
|
||||||
|
const tempDirs: string[] = [];
|
||||||
|
const originalOcrProfile = process.env.OPENSCREEN_GUIDE_OCR_PROFILE;
|
||||||
|
const originalOcrLanguage = process.env.OPENSCREEN_GUIDE_OCR_LANGUAGE;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
delete process.env.OPENSCREEN_GUIDE_OCR_PROFILE;
|
||||||
|
delete process.env.OPENSCREEN_GUIDE_OCR_LANGUAGE;
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
restoreEnv("OPENSCREEN_GUIDE_OCR_PROFILE", originalOcrProfile);
|
||||||
|
restoreEnv("OPENSCREEN_GUIDE_OCR_LANGUAGE", originalOcrLanguage);
|
||||||
|
await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true })));
|
||||||
|
});
|
||||||
|
|
||||||
|
function restoreEnv(name: string, value: string | undefined): void {
|
||||||
|
if (value === undefined) {
|
||||||
|
delete process.env[name];
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
process.env[name] = value;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function createStore(): Promise<DeepSeekSettingsStore> {
|
||||||
|
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openscreen-guide-settings-"));
|
||||||
|
tempDirs.push(dir);
|
||||||
|
return new DeepSeekSettingsStore(path.join(dir, "guide-ai-settings.json"));
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("DeepSeekSettingsStore OCR settings", () => {
|
||||||
|
it("defaults to the Vietnamese enhanced OCR profile", async () => {
|
||||||
|
const store = await createStore();
|
||||||
|
|
||||||
|
await expect(store.getOcrConfig()).resolves.toEqual({
|
||||||
|
profile: "vietnamese",
|
||||||
|
language: "vi,en",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("persists OCR profile changes alongside DeepSeek settings", async () => {
|
||||||
|
const store = await createStore();
|
||||||
|
|
||||||
|
const status = await store.save({
|
||||||
|
deepseekApiKeyEnvName: "DEEPSEEK_API_KEY",
|
||||||
|
baseUrl: "https://api.deepseek.com",
|
||||||
|
model: "deepseek-chat",
|
||||||
|
ocrProfile: "hybrid",
|
||||||
|
ocrLanguage: "vi,en",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(status.ocr).toMatchObject({
|
||||||
|
profile: "hybrid",
|
||||||
|
language: "vi,en",
|
||||||
|
});
|
||||||
|
await expect(store.getOcrConfig()).resolves.toEqual({
|
||||||
|
profile: "hybrid",
|
||||||
|
language: "vi,en",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,6 +1,10 @@
|
|||||||
import fs from "node:fs/promises";
|
import fs from "node:fs/promises";
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
import type { GuideAiSettings, SaveGuideAiSettingsInput } from "../../../src/guide/contracts";
|
import type {
|
||||||
|
GuideAiSettings,
|
||||||
|
GuideOcrProfile,
|
||||||
|
SaveGuideAiSettingsInput,
|
||||||
|
} from "../../../src/guide/contracts";
|
||||||
|
|
||||||
export interface DeepSeekGuideConfig {
|
export interface DeepSeekGuideConfig {
|
||||||
apiKey?: string;
|
apiKey?: string;
|
||||||
@@ -12,8 +16,22 @@ export interface DeepSeekGuideConfigProvider {
|
|||||||
getDeepSeekConfig(): Promise<DeepSeekGuideConfig>;
|
getDeepSeekConfig(): Promise<DeepSeekGuideConfig>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface GuideOcrConfig {
|
||||||
|
profile: GuideOcrProfile;
|
||||||
|
language: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface GuideOcrConfigProvider {
|
||||||
|
getOcrConfig(): Promise<GuideOcrConfig>;
|
||||||
|
}
|
||||||
|
|
||||||
interface PersistedGuideAiSettings {
|
interface PersistedGuideAiSettings {
|
||||||
schemaVersion: 1;
|
schemaVersion: 1;
|
||||||
|
ocr?: {
|
||||||
|
profile?: GuideOcrProfile;
|
||||||
|
language?: string;
|
||||||
|
updatedAt?: string;
|
||||||
|
};
|
||||||
deepseek?: {
|
deepseek?: {
|
||||||
apiKeyEnvName?: string;
|
apiKeyEnvName?: string;
|
||||||
baseUrl?: string;
|
baseUrl?: string;
|
||||||
@@ -25,8 +43,10 @@ interface PersistedGuideAiSettings {
|
|||||||
const DEFAULT_DEEPSEEK_API_KEY_ENV_NAME = "DEEPSEEK_API_KEY";
|
const DEFAULT_DEEPSEEK_API_KEY_ENV_NAME = "DEEPSEEK_API_KEY";
|
||||||
const DEFAULT_DEEPSEEK_BASE_URL = "https://api.deepseek.com";
|
const DEFAULT_DEEPSEEK_BASE_URL = "https://api.deepseek.com";
|
||||||
const DEFAULT_DEEPSEEK_MODEL = "deepseek-chat";
|
const DEFAULT_DEEPSEEK_MODEL = "deepseek-chat";
|
||||||
|
const DEFAULT_OCR_PROFILE: GuideOcrProfile = "vietnamese";
|
||||||
|
const DEFAULT_OCR_LANGUAGE = "vi,en";
|
||||||
|
|
||||||
export class DeepSeekSettingsStore implements DeepSeekGuideConfigProvider {
|
export class DeepSeekSettingsStore implements DeepSeekGuideConfigProvider, GuideOcrConfigProvider {
|
||||||
constructor(private readonly filePath: string) {}
|
constructor(private readonly filePath: string) {}
|
||||||
|
|
||||||
async getStatus(): Promise<GuideAiSettings> {
|
async getStatus(): Promise<GuideAiSettings> {
|
||||||
@@ -35,6 +55,13 @@ export class DeepSeekSettingsStore implements DeepSeekGuideConfigProvider {
|
|||||||
const activeApiKey = process.env[apiKeyEnvName];
|
const activeApiKey = process.env[apiKeyEnvName];
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
ocr: {
|
||||||
|
profile: normalizeOcrProfile(raw?.ocr?.profile ?? process.env.OPENSCREEN_GUIDE_OCR_PROFILE),
|
||||||
|
language: normalizeOcrLanguage(
|
||||||
|
raw?.ocr?.language ?? process.env.OPENSCREEN_GUIDE_OCR_LANGUAGE,
|
||||||
|
),
|
||||||
|
updatedAt: raw?.ocr?.updatedAt,
|
||||||
|
},
|
||||||
deepseek: {
|
deepseek: {
|
||||||
hasApiKey: Boolean(activeApiKey),
|
hasApiKey: Boolean(activeApiKey),
|
||||||
apiKeyEnvName,
|
apiKeyEnvName,
|
||||||
@@ -49,7 +76,14 @@ export class DeepSeekSettingsStore implements DeepSeekGuideConfigProvider {
|
|||||||
|
|
||||||
async save(input: SaveGuideAiSettingsInput): Promise<GuideAiSettings> {
|
async save(input: SaveGuideAiSettingsInput): Promise<GuideAiSettings> {
|
||||||
const current = (await this.readSettings()) ?? { schemaVersion: 1 };
|
const current = (await this.readSettings()) ?? { schemaVersion: 1 };
|
||||||
|
const currentOcr = current.ocr ?? {};
|
||||||
const currentDeepSeek = current.deepseek ?? {};
|
const currentDeepSeek = current.deepseek ?? {};
|
||||||
|
const nextOcr = {
|
||||||
|
...currentOcr,
|
||||||
|
profile: normalizeOcrProfile(input.ocrProfile ?? currentOcr.profile),
|
||||||
|
language: normalizeOcrLanguage(input.ocrLanguage ?? currentOcr.language),
|
||||||
|
updatedAt: new Date().toISOString(),
|
||||||
|
};
|
||||||
const nextDeepSeek = {
|
const nextDeepSeek = {
|
||||||
...currentDeepSeek,
|
...currentDeepSeek,
|
||||||
baseUrl: normalizeBaseUrl(input.baseUrl ?? currentDeepSeek.baseUrl),
|
baseUrl: normalizeBaseUrl(input.baseUrl ?? currentDeepSeek.baseUrl),
|
||||||
@@ -65,6 +99,7 @@ export class DeepSeekSettingsStore implements DeepSeekGuideConfigProvider {
|
|||||||
|
|
||||||
await this.writeSettings({
|
await this.writeSettings({
|
||||||
schemaVersion: 1,
|
schemaVersion: 1,
|
||||||
|
ocr: nextOcr,
|
||||||
deepseek: nextDeepSeek,
|
deepseek: nextDeepSeek,
|
||||||
});
|
});
|
||||||
return await this.getStatus();
|
return await this.getStatus();
|
||||||
@@ -80,6 +115,16 @@ export class DeepSeekSettingsStore implements DeepSeekGuideConfigProvider {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async getOcrConfig(): Promise<GuideOcrConfig> {
|
||||||
|
const raw = await this.readSettings();
|
||||||
|
return {
|
||||||
|
profile: normalizeOcrProfile(raw?.ocr?.profile ?? process.env.OPENSCREEN_GUIDE_OCR_PROFILE),
|
||||||
|
language: normalizeOcrLanguage(
|
||||||
|
raw?.ocr?.language ?? process.env.OPENSCREEN_GUIDE_OCR_LANGUAGE,
|
||||||
|
),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
private async readSettings(): Promise<PersistedGuideAiSettings | null> {
|
private async readSettings(): Promise<PersistedGuideAiSettings | null> {
|
||||||
try {
|
try {
|
||||||
const content = await fs.readFile(this.filePath, "utf-8");
|
const content = await fs.readFile(this.filePath, "utf-8");
|
||||||
@@ -120,6 +165,11 @@ function normalizePersistedSettings(input: unknown): PersistedGuideAiSettings |
|
|||||||
}
|
}
|
||||||
return {
|
return {
|
||||||
schemaVersion: 1,
|
schemaVersion: 1,
|
||||||
|
ocr: {
|
||||||
|
profile: normalizeOcrProfile(raw.ocr?.profile),
|
||||||
|
language: normalizeOcrLanguage(raw.ocr?.language),
|
||||||
|
updatedAt: raw.ocr?.updatedAt,
|
||||||
|
},
|
||||||
deepseek: {
|
deepseek: {
|
||||||
apiKeyEnvName: normalizeEnvName(raw.deepseek?.apiKeyEnvName),
|
apiKeyEnvName: normalizeEnvName(raw.deepseek?.apiKeyEnvName),
|
||||||
baseUrl: raw.deepseek?.baseUrl,
|
baseUrl: raw.deepseek?.baseUrl,
|
||||||
@@ -155,3 +205,19 @@ function normalizeBaseUrl(value: string | undefined): string {
|
|||||||
function normalizeModel(value: string | undefined): string {
|
function normalizeModel(value: string | undefined): string {
|
||||||
return value?.trim() || DEFAULT_DEEPSEEK_MODEL;
|
return value?.trim() || DEFAULT_DEEPSEEK_MODEL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function normalizeOcrProfile(value: string | undefined): GuideOcrProfile {
|
||||||
|
if (value === "fast" || value === "vietnamese" || value === "hybrid") {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
return DEFAULT_OCR_PROFILE;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeOcrLanguage(value: string | undefined): string {
|
||||||
|
const normalized = value
|
||||||
|
?.split(",")
|
||||||
|
.map((part) => part.trim().toLowerCase())
|
||||||
|
.filter(Boolean)
|
||||||
|
.join(",");
|
||||||
|
return normalized || DEFAULT_OCR_LANGUAGE;
|
||||||
|
}
|
||||||
|
|||||||
@@ -168,6 +168,7 @@ describe("GuideStore", () => {
|
|||||||
width: 800,
|
width: 800,
|
||||||
height: 600,
|
height: 600,
|
||||||
pngBytes: new Uint8Array([137, 80, 78, 71]).buffer,
|
pngBytes: new Uint8Array([137, 80, 78, 71]).buffer,
|
||||||
|
markedPngBytes: new Uint8Array([137, 80, 78, 71, 1]).buffer,
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(session.status).toBe("snapshots-ready");
|
expect(session.status).toBe("snapshots-ready");
|
||||||
@@ -176,6 +177,9 @@ describe("GuideStore", () => {
|
|||||||
await expect(fs.readFile(session.snapshots[0]?.path ?? "")).resolves.toEqual(
|
await expect(fs.readFile(session.snapshots[0]?.path ?? "")).resolves.toEqual(
|
||||||
Buffer.from([137, 80, 78, 71]),
|
Buffer.from([137, 80, 78, 71]),
|
||||||
);
|
);
|
||||||
|
await expect(fs.readFile(session.snapshots[0]?.markedPath ?? "")).resolves.toEqual(
|
||||||
|
Buffer.from([137, 80, 78, 71, 1]),
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("runs OCR, generates a local draft, and exports files", async () => {
|
it("runs OCR, generates a local draft, and exports files", async () => {
|
||||||
@@ -228,6 +232,74 @@ describe("GuideStore", () => {
|
|||||||
await expect(fs.readFile(html.path, "utf-8")).resolves.toContain("<!doctype html>");
|
await expect(fs.readFile(html.path, "utf-8")).resolves.toContain("<!doctype html>");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("resumes OCR without reprocessing completed snapshots", async () => {
|
||||||
|
const recognizedSnapshotIds: string[] = [];
|
||||||
|
const store = new GuideStore(recordingsDir, {
|
||||||
|
ocrClient: {
|
||||||
|
recognize: async (snapshot) => {
|
||||||
|
recognizedSnapshotIds.push(snapshot.id);
|
||||||
|
return [];
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await store.startSession(115);
|
||||||
|
const firstMarker = await store.addMarker({
|
||||||
|
recordingId: 115,
|
||||||
|
kind: "hotkey",
|
||||||
|
timeMs: 100,
|
||||||
|
label: "Ctrl+F12 marker",
|
||||||
|
normalizedX: 0.25,
|
||||||
|
normalizedY: 0.35,
|
||||||
|
});
|
||||||
|
const secondMarker = await store.addMarker({
|
||||||
|
recordingId: 115,
|
||||||
|
kind: "hotkey",
|
||||||
|
timeMs: 300,
|
||||||
|
label: "Ctrl+F12 marker",
|
||||||
|
normalizedX: 0.6,
|
||||||
|
normalizedY: 0.7,
|
||||||
|
});
|
||||||
|
const firstEvent = firstMarker.event;
|
||||||
|
const secondEvent = secondMarker.event;
|
||||||
|
await store.writeSnapshot({
|
||||||
|
recordingId: 115,
|
||||||
|
eventId: firstEvent?.id ?? "",
|
||||||
|
timeMs: 100,
|
||||||
|
offsetMs: 0,
|
||||||
|
width: 800,
|
||||||
|
height: 600,
|
||||||
|
pngBytes: new Uint8Array([1, 2, 3]).buffer,
|
||||||
|
});
|
||||||
|
await store.writeSnapshot({
|
||||||
|
recordingId: 115,
|
||||||
|
eventId: secondEvent?.id ?? "",
|
||||||
|
timeMs: 300,
|
||||||
|
offsetMs: 0,
|
||||||
|
width: 800,
|
||||||
|
height: 600,
|
||||||
|
pngBytes: new Uint8Array([4, 5, 6]).buffer,
|
||||||
|
});
|
||||||
|
|
||||||
|
await store.runOcr({
|
||||||
|
recordingId: 115,
|
||||||
|
snapshotIds: [`snapshot-${firstEvent?.id}`],
|
||||||
|
});
|
||||||
|
expect(recognizedSnapshotIds).toEqual([`snapshot-${firstEvent?.id}`]);
|
||||||
|
|
||||||
|
const resumedSession = await store.runOcr({ recordingId: 115 });
|
||||||
|
expect(recognizedSnapshotIds).toEqual([
|
||||||
|
`snapshot-${firstEvent?.id}`,
|
||||||
|
`snapshot-${secondEvent?.id}`,
|
||||||
|
]);
|
||||||
|
expect(resumedSession.snapshots.every((snapshot) => snapshot.ocrCompletedAt)).toBe(true);
|
||||||
|
|
||||||
|
await store.runOcr({ recordingId: 115 });
|
||||||
|
expect(recognizedSnapshotIds).toEqual([
|
||||||
|
`snapshot-${firstEvent?.id}`,
|
||||||
|
`snapshot-${secondEvent?.id}`,
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
it("repairs generic hotkey marker text and attaches AI draft artifacts", async () => {
|
it("repairs generic hotkey marker text and attaches AI draft artifacts", async () => {
|
||||||
const store = new GuideStore(recordingsDir, {
|
const store = new GuideStore(recordingsDir, {
|
||||||
ocrClient: {
|
ocrClient: {
|
||||||
|
|||||||
+125
-36
@@ -34,7 +34,10 @@ import {
|
|||||||
DeepSeekGuideClientError,
|
DeepSeekGuideClientError,
|
||||||
type GuideDraftClient,
|
type GuideDraftClient,
|
||||||
} from "./ai/deepseekGuideClient";
|
} from "./ai/deepseekGuideClient";
|
||||||
import type { DeepSeekGuideConfigProvider } from "./ai/deepseekSettingsStore";
|
import type {
|
||||||
|
DeepSeekGuideConfigProvider,
|
||||||
|
GuideOcrConfigProvider,
|
||||||
|
} from "./ai/deepseekSettingsStore";
|
||||||
import { type GuidePaths, normalizeGuideRecordingId, resolveGuidePaths } from "./guidePaths";
|
import { type GuidePaths, normalizeGuideRecordingId, resolveGuidePaths } from "./guidePaths";
|
||||||
import { createFocusedOcrSnapshot, remapFocusedOcrBlocks } from "./ocr/focusedOcrSnapshot";
|
import { createFocusedOcrSnapshot, remapFocusedOcrBlocks } from "./ocr/focusedOcrSnapshot";
|
||||||
import { DefaultGuideOcrClient, type GuideOcrClient } from "./ocr/paddleOcrClient";
|
import { DefaultGuideOcrClient, type GuideOcrClient } from "./ocr/paddleOcrClient";
|
||||||
@@ -55,6 +58,8 @@ const VALID_EVENT_SOURCES = new Set<GuideEventSource>([
|
|||||||
"review-ui",
|
"review-ui",
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
const guideOcrJobsByRecordingId = new Map<string, Promise<GuideSession>>();
|
||||||
|
|
||||||
export class GuideStoreError extends Error {
|
export class GuideStoreError extends Error {
|
||||||
constructor(
|
constructor(
|
||||||
readonly code: GuideErrorCode,
|
readonly code: GuideErrorCode,
|
||||||
@@ -70,6 +75,7 @@ export interface GuideStoreDependencies {
|
|||||||
ocrClient?: GuideOcrClient;
|
ocrClient?: GuideOcrClient;
|
||||||
draftClient?: GuideDraftClient;
|
draftClient?: GuideDraftClient;
|
||||||
deepSeekConfigProvider?: DeepSeekGuideConfigProvider;
|
deepSeekConfigProvider?: DeepSeekGuideConfigProvider;
|
||||||
|
ocrConfigProvider?: GuideOcrConfigProvider;
|
||||||
focusOcrSnapshots?: boolean;
|
focusOcrSnapshots?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -209,10 +215,19 @@ export class GuideStore {
|
|||||||
|
|
||||||
this.assertGuidePathIsAllowed(session.outputDir);
|
this.assertGuidePathIsAllowed(session.outputDir);
|
||||||
await fs.mkdir(session.outputDir, { recursive: true });
|
await fs.mkdir(session.outputDir, { recursive: true });
|
||||||
const fileName = `step-${String(eventIndex + 1).padStart(3, "0")}.png`;
|
const fileBaseName = `step-${String(eventIndex + 1).padStart(3, "0")}`;
|
||||||
|
const fileName = `${fileBaseName}.png`;
|
||||||
const snapshotPath = path.join(session.outputDir, fileName);
|
const snapshotPath = path.join(session.outputDir, fileName);
|
||||||
|
const markedSnapshotPath = path.join(session.outputDir, `${fileBaseName}-marked.png`);
|
||||||
this.assertGuidePathIsAllowed(snapshotPath);
|
this.assertGuidePathIsAllowed(snapshotPath);
|
||||||
|
this.assertGuidePathIsAllowed(markedSnapshotPath);
|
||||||
await fs.writeFile(snapshotPath, Buffer.from(new Uint8Array(input.pngBytes)));
|
await fs.writeFile(snapshotPath, Buffer.from(new Uint8Array(input.pngBytes)));
|
||||||
|
const hasMarkedSnapshot = Boolean(input.markedPngBytes?.byteLength);
|
||||||
|
if (hasMarkedSnapshot && input.markedPngBytes) {
|
||||||
|
await fs.writeFile(markedSnapshotPath, Buffer.from(new Uint8Array(input.markedPngBytes)));
|
||||||
|
} else {
|
||||||
|
await fs.unlink(markedSnapshotPath).catch(() => undefined);
|
||||||
|
}
|
||||||
|
|
||||||
const snapshot: GuideSnapshot = {
|
const snapshot: GuideSnapshot = {
|
||||||
id: `snapshot-${input.eventId}`,
|
id: `snapshot-${input.eventId}`,
|
||||||
@@ -220,6 +235,7 @@ export class GuideStore {
|
|||||||
timeMs: Math.max(0, input.timeMs),
|
timeMs: Math.max(0, input.timeMs),
|
||||||
offsetMs: input.offsetMs,
|
offsetMs: input.offsetMs,
|
||||||
path: snapshotPath,
|
path: snapshotPath,
|
||||||
|
markedPath: hasMarkedSnapshot ? markedSnapshotPath : undefined,
|
||||||
width: Math.round(input.width),
|
width: Math.round(input.width),
|
||||||
height: Math.round(input.height),
|
height: Math.round(input.height),
|
||||||
};
|
};
|
||||||
@@ -245,48 +261,103 @@ export class GuideStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async runOcr(input: RunGuideOcrInput): Promise<GuideSession> {
|
async runOcr(input: RunGuideOcrInput): Promise<GuideSession> {
|
||||||
const session = await this.readSession(input.recordingId);
|
const recordingId = normalizeGuideRecordingId(input.recordingId);
|
||||||
const requestedIds = new Set(input.snapshotIds ?? []);
|
if (!recordingId) {
|
||||||
const snapshots =
|
throw new GuideStoreError("guide-invalid-input", "OCR run is missing recordingId.");
|
||||||
requestedIds.size > 0
|
|
||||||
? session.snapshots.filter((snapshot) => requestedIds.has(snapshot.id))
|
|
||||||
: session.snapshots;
|
|
||||||
if (snapshots.length === 0) {
|
|
||||||
throw new GuideStoreError("guide-invalid-input", "No guide snapshots are available for OCR.");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const ocrClient = this.dependencies.ocrClient ?? new DefaultGuideOcrClient();
|
const previousJob =
|
||||||
const shouldFocusOcrSnapshots =
|
guideOcrJobsByRecordingId.get(recordingId)?.catch(() => undefined) ?? Promise.resolve();
|
||||||
this.dependencies.focusOcrSnapshots ?? this.dependencies.ocrClient === undefined;
|
const nextJob = previousJob.then(async () => {
|
||||||
const eventsById = new Map(session.events.map((event) => [event.id, event]));
|
let session = await this.readSession(recordingId);
|
||||||
const blocks: OcrBlock[] = [];
|
const requestedIds = new Set(input.snapshotIds ?? []);
|
||||||
try {
|
const snapshots =
|
||||||
for (const snapshot of snapshots) {
|
requestedIds.size > 0
|
||||||
const focusedSnapshot = shouldFocusOcrSnapshots
|
? session.snapshots.filter((snapshot) => requestedIds.has(snapshot.id))
|
||||||
? await createFocusedOcrSnapshot({
|
: session.snapshots;
|
||||||
snapshot,
|
if (snapshots.length === 0) {
|
||||||
event: eventsById.get(snapshot.eventId),
|
throw new GuideStoreError(
|
||||||
outputDir: session.outputDir,
|
"guide-invalid-input",
|
||||||
})
|
"No guide snapshots are available for OCR.",
|
||||||
: { snapshot };
|
);
|
||||||
const recognizedBlocks = await ocrClient.recognize(focusedSnapshot.snapshot);
|
|
||||||
blocks.push(...remapFocusedOcrBlocks(recognizedBlocks, focusedSnapshot.transform));
|
|
||||||
}
|
}
|
||||||
} catch (error) {
|
|
||||||
throw new GuideStoreError(
|
|
||||||
"guide-ocr-unavailable",
|
|
||||||
error instanceof Error ? error.message : "OCR failed.",
|
|
||||||
true,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const snapshotIds = new Set(snapshots.map((snapshot) => snapshot.id));
|
const completedSnapshotIds = new Set(
|
||||||
|
session.snapshots
|
||||||
|
.filter((snapshot) => isSnapshotOcrCompleted(snapshot, session.ocrBlocks))
|
||||||
|
.map((snapshot) => snapshot.id),
|
||||||
|
);
|
||||||
|
const pendingSnapshots = snapshots.filter(
|
||||||
|
(snapshot) => !completedSnapshotIds.has(snapshot.id),
|
||||||
|
);
|
||||||
|
if (pendingSnapshots.length === 0) {
|
||||||
|
if (session.status === "ocr-ready") {
|
||||||
|
return session;
|
||||||
|
}
|
||||||
|
const readySession = touchSession({
|
||||||
|
...session,
|
||||||
|
status: "ocr-ready",
|
||||||
|
candidates: buildGuideStepCandidates(session),
|
||||||
|
});
|
||||||
|
await this.writeSession(readySession);
|
||||||
|
return readySession;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ocrClient =
|
||||||
|
this.dependencies.ocrClient ??
|
||||||
|
DefaultGuideOcrClient.fromConfig(await this.dependencies.ocrConfigProvider?.getOcrConfig());
|
||||||
|
const shouldFocusOcrSnapshots =
|
||||||
|
this.dependencies.focusOcrSnapshots ?? this.dependencies.ocrClient === undefined;
|
||||||
|
const eventsById = new Map(session.events.map((event) => [event.id, event]));
|
||||||
|
try {
|
||||||
|
for (const snapshot of pendingSnapshots) {
|
||||||
|
const focusedSnapshot = shouldFocusOcrSnapshots
|
||||||
|
? await createFocusedOcrSnapshot({
|
||||||
|
snapshot,
|
||||||
|
event: eventsById.get(snapshot.eventId),
|
||||||
|
outputDir: session.outputDir,
|
||||||
|
})
|
||||||
|
: { snapshot };
|
||||||
|
const recognizedBlocks = await ocrClient.recognize(focusedSnapshot.snapshot);
|
||||||
|
const blocks = remapFocusedOcrBlocks(recognizedBlocks, focusedSnapshot.transform);
|
||||||
|
session = await this.writeOcrSnapshotProgress(session, snapshot.id, blocks);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
throw new GuideStoreError(
|
||||||
|
"guide-ocr-unavailable",
|
||||||
|
error instanceof Error ? error.message : "OCR failed.",
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return session;
|
||||||
|
});
|
||||||
|
guideOcrJobsByRecordingId.set(recordingId, nextJob);
|
||||||
|
try {
|
||||||
|
return await nextJob;
|
||||||
|
} finally {
|
||||||
|
if (guideOcrJobsByRecordingId.get(recordingId) === nextJob) {
|
||||||
|
guideOcrJobsByRecordingId.delete(recordingId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async writeOcrSnapshotProgress(
|
||||||
|
session: GuideSession,
|
||||||
|
snapshotId: string,
|
||||||
|
blocks: OcrBlock[],
|
||||||
|
): Promise<GuideSession> {
|
||||||
const updatedOcrBlocks = [
|
const updatedOcrBlocks = [
|
||||||
...session.ocrBlocks.filter((block) => !snapshotIds.has(block.snapshotId)),
|
...session.ocrBlocks.filter((block) => block.snapshotId !== snapshotId),
|
||||||
...blocks,
|
...blocks,
|
||||||
];
|
];
|
||||||
|
const completedAt = new Date().toISOString();
|
||||||
|
const updatedSnapshots = session.snapshots.map((snapshot) =>
|
||||||
|
snapshot.id === snapshotId ? { ...snapshot, ocrCompletedAt: completedAt } : snapshot,
|
||||||
|
);
|
||||||
const draftSession = {
|
const draftSession = {
|
||||||
...session,
|
...session,
|
||||||
|
snapshots: updatedSnapshots,
|
||||||
ocrBlocks: updatedOcrBlocks,
|
ocrBlocks: updatedOcrBlocks,
|
||||||
};
|
};
|
||||||
const updatedSession = touchSession({
|
const updatedSession = touchSession({
|
||||||
@@ -662,6 +733,8 @@ function normalizeGuideSnapshot(input: unknown): GuideSnapshot | null {
|
|||||||
const id = normalizeString(input.id);
|
const id = normalizeString(input.id);
|
||||||
const eventId = normalizeString(input.eventId);
|
const eventId = normalizeString(input.eventId);
|
||||||
const pathValue = normalizeString(input.path);
|
const pathValue = normalizeString(input.path);
|
||||||
|
const markedPath = normalizeOptionalString(input.markedPath);
|
||||||
|
const ocrCompletedAt = normalizeOptionalString(input.ocrCompletedAt);
|
||||||
const timeMs = normalizeNonNegativeNumber(input.timeMs);
|
const timeMs = normalizeNonNegativeNumber(input.timeMs);
|
||||||
const offsetMs = normalizeOptionalNumber(input.offsetMs);
|
const offsetMs = normalizeOptionalNumber(input.offsetMs);
|
||||||
const width = normalizePositiveInteger(input.width);
|
const width = normalizePositiveInteger(input.width);
|
||||||
@@ -677,7 +750,23 @@ function normalizeGuideSnapshot(input: unknown): GuideSnapshot | null {
|
|||||||
) {
|
) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
return { id, eventId, timeMs, offsetMs, path: pathValue, width, height };
|
return {
|
||||||
|
id,
|
||||||
|
eventId,
|
||||||
|
timeMs,
|
||||||
|
offsetMs,
|
||||||
|
path: pathValue,
|
||||||
|
markedPath,
|
||||||
|
ocrCompletedAt,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function isSnapshotOcrCompleted(snapshot: GuideSnapshot, ocrBlocks: OcrBlock[]): boolean {
|
||||||
|
return (
|
||||||
|
Boolean(snapshot.ocrCompletedAt) || ocrBlocks.some((block) => block.snapshotId === snapshot.id)
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function normalizeOcrBlock(input: unknown): OcrBlock | null {
|
function normalizeOcrBlock(input: unknown): OcrBlock | null {
|
||||||
|
|||||||
@@ -1,14 +1,17 @@
|
|||||||
import { type ChildProcessWithoutNullStreams, spawn } from "node:child_process";
|
import { type ChildProcessWithoutNullStreams, execFile, spawn } from "node:child_process";
|
||||||
import fs from "node:fs/promises";
|
import fs from "node:fs/promises";
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
|
import { promisify } from "node:util";
|
||||||
import { app } from "electron";
|
import { app } from "electron";
|
||||||
|
|
||||||
const DEFAULT_OCR_BASE_URL = "http://127.0.0.1:8866";
|
const DEFAULT_OCR_BASE_URL = "http://127.0.0.1:8866";
|
||||||
const DEFAULT_OCR_PORT = "8866";
|
const DEFAULT_OCR_PORT = "8866";
|
||||||
|
const WINDOWS_SERVICE_NAME = "OpenScreenOCR";
|
||||||
const SERVICE_EXE_NAME = "openscreen-ocr-service.exe";
|
const SERVICE_EXE_NAME = "openscreen-ocr-service.exe";
|
||||||
const HEALTH_TIMEOUT_MS = 1000;
|
const HEALTH_TIMEOUT_MS = 1000;
|
||||||
const STARTUP_TIMEOUT_MS = 90000;
|
const STARTUP_TIMEOUT_MS = 90000;
|
||||||
const PADDLEX_MODEL_NAMES = ["PP-OCRv5_mobile_det", "latin_PP-OCRv5_mobile_rec"];
|
const PADDLEX_MODEL_NAMES = ["PP-OCRv5_mobile_det", "latin_PP-OCRv5_mobile_rec"];
|
||||||
|
const execFileAsync = promisify(execFile);
|
||||||
|
|
||||||
let ocrProcess: ChildProcessWithoutNullStreams | null = null;
|
let ocrProcess: ChildProcessWithoutNullStreams | null = null;
|
||||||
let startupPromise: Promise<void> | null = null;
|
let startupPromise: Promise<void> | null = null;
|
||||||
@@ -24,6 +27,11 @@ export async function ensureBundledOcrServiceRunning(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (process.platform === "win32" && (await startInstalledWindowsOcrService())) {
|
||||||
|
await waitForOcrServiceHealth(baseUrl, STARTUP_TIMEOUT_MS);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const executablePath = await findBundledOcrServiceExecutable();
|
const executablePath = await findBundledOcrServiceExecutable();
|
||||||
if (!executablePath) {
|
if (!executablePath) {
|
||||||
return;
|
return;
|
||||||
@@ -51,6 +59,39 @@ function shouldManageOcrService(baseUrl: string): boolean {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function startInstalledWindowsOcrService(): Promise<boolean> {
|
||||||
|
const query = await runSc(["query", WINDOWS_SERVICE_NAME]);
|
||||||
|
if (!query.success) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (/\bRUNNING\b/i.test(query.output)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
const start = await runSc(["start", WINDOWS_SERVICE_NAME]);
|
||||||
|
return start.success || /\b1056\b/.test(start.output) || /already running/i.test(start.output);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runSc(args: string[]): Promise<{ success: boolean; output: string }> {
|
||||||
|
try {
|
||||||
|
const result = await execFileAsync("sc.exe", args, {
|
||||||
|
windowsHide: true,
|
||||||
|
timeout: 10000,
|
||||||
|
maxBuffer: 512 * 1024,
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
success: true,
|
||||||
|
output: `${result.stdout ?? ""}\n${result.stderr ?? ""}`,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
const failed = error as { stdout?: string; stderr?: string };
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
output: `${failed.stdout ?? ""}\n${failed.stderr ?? ""}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function findBundledOcrServiceExecutable(): Promise<string | null> {
|
async function findBundledOcrServiceExecutable(): Promise<string | null> {
|
||||||
const candidates = [
|
const candidates = [
|
||||||
process.env.OPENSCREEN_GUIDE_OCR_EXE,
|
process.env.OPENSCREEN_GUIDE_OCR_EXE,
|
||||||
@@ -156,8 +197,11 @@ function startOcrServiceProcess(
|
|||||||
OPENSCREEN_OCR_PORT: DEFAULT_OCR_PORT,
|
OPENSCREEN_OCR_PORT: DEFAULT_OCR_PORT,
|
||||||
PADDLEOCR_DEVICE: process.env.PADDLEOCR_DEVICE ?? "cpu",
|
PADDLEOCR_DEVICE: process.env.PADDLEOCR_DEVICE ?? "cpu",
|
||||||
PADDLEOCR_ENABLE_MKLDNN: process.env.PADDLEOCR_ENABLE_MKLDNN ?? "0",
|
PADDLEOCR_ENABLE_MKLDNN: process.env.PADDLEOCR_ENABLE_MKLDNN ?? "0",
|
||||||
PADDLEOCR_LANG: process.env.PADDLEOCR_LANG ?? "latin",
|
PADDLEOCR_LANG: process.env.PADDLEOCR_LANG ?? "",
|
||||||
PADDLEOCR_USE_MOBILE: process.env.PADDLEOCR_USE_MOBILE ?? "1",
|
PADDLEOCR_USE_MOBILE: process.env.PADDLEOCR_USE_MOBILE ?? "1",
|
||||||
|
OPENSCREEN_OCR_PROFILE:
|
||||||
|
process.env.OPENSCREEN_OCR_PROFILE ?? process.env.OPENSCREEN_GUIDE_OCR_PROFILE ?? "",
|
||||||
|
OPENSCREEN_OCR_WARMUP: process.env.OPENSCREEN_OCR_WARMUP ?? "1",
|
||||||
PADDLE_PDX_ENABLE_MKLDNN_BYDEFAULT: process.env.PADDLE_PDX_ENABLE_MKLDNN_BYDEFAULT ?? "False",
|
PADDLE_PDX_ENABLE_MKLDNN_BYDEFAULT: process.env.PADDLE_PDX_ENABLE_MKLDNN_BYDEFAULT ?? "False",
|
||||||
PADDLE_PDX_CACHE_HOME: process.env.PADDLE_PDX_CACHE_HOME ?? runtimePaths.paddlexCachePath,
|
PADDLE_PDX_CACHE_HOME: process.env.PADDLE_PDX_CACHE_HOME ?? runtimePaths.paddlexCachePath,
|
||||||
PADDLE_PDX_DISABLE_MODEL_SOURCE_CHECK:
|
PADDLE_PDX_DISABLE_MODEL_SOURCE_CHECK:
|
||||||
|
|||||||
@@ -1,8 +1,12 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import fs from "node:fs/promises";
|
||||||
|
import os from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
import type { GuideSnapshot, OcrBlock } from "../../../src/guide/contracts";
|
import type { GuideSnapshot, OcrBlock } from "../../../src/guide/contracts";
|
||||||
import {
|
import {
|
||||||
DefaultGuideOcrClient,
|
DefaultGuideOcrClient,
|
||||||
normalizeOcrResponse,
|
normalizeOcrResponse,
|
||||||
|
PaddleOcrHttpClient,
|
||||||
parseWindowsOcrPayload,
|
parseWindowsOcrPayload,
|
||||||
} from "./paddleOcrClient";
|
} from "./paddleOcrClient";
|
||||||
|
|
||||||
@@ -16,6 +20,10 @@ const snapshot: GuideSnapshot = {
|
|||||||
height: 800,
|
height: 800,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.unstubAllGlobals();
|
||||||
|
});
|
||||||
|
|
||||||
describe("normalizeOcrResponse", () => {
|
describe("normalizeOcrResponse", () => {
|
||||||
it("normalizes pixel boxes into guide OCR blocks", () => {
|
it("normalizes pixel boxes into guide OCR blocks", () => {
|
||||||
const blocks = normalizeOcrResponse(
|
const blocks = normalizeOcrResponse(
|
||||||
@@ -67,6 +75,35 @@ describe("normalizeOcrResponse", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("PaddleOcrHttpClient", () => {
|
||||||
|
it("sends the selected OCR profile to the local service", async () => {
|
||||||
|
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openscreen-ocr-client-"));
|
||||||
|
const imagePath = path.join(tempDir, "step.png");
|
||||||
|
await fs.writeFile(imagePath, Buffer.from([137, 80, 78, 71]));
|
||||||
|
const requests: unknown[] = [];
|
||||||
|
vi.stubGlobal(
|
||||||
|
"fetch",
|
||||||
|
vi.fn(async (_url: string, init?: RequestInit) => {
|
||||||
|
requests.push(JSON.parse(String(init?.body ?? "{}")));
|
||||||
|
return new Response(JSON.stringify({ blocks: [] }), {
|
||||||
|
status: 200,
|
||||||
|
headers: { "content-type": "application/json" },
|
||||||
|
});
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const client = new PaddleOcrHttpClient("https://ocr.example.test", "vi,en", "hybrid");
|
||||||
|
await client.recognize({ ...snapshot, path: imagePath });
|
||||||
|
|
||||||
|
expect(requests[0]).toMatchObject({
|
||||||
|
language: "vi,en",
|
||||||
|
profile: "hybrid",
|
||||||
|
path: imagePath,
|
||||||
|
});
|
||||||
|
await fs.rm(tempDir, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("DefaultGuideOcrClient", () => {
|
describe("DefaultGuideOcrClient", () => {
|
||||||
it("falls back when the HTTP OCR service is unavailable", async () => {
|
it("falls back when the HTTP OCR service is unavailable", async () => {
|
||||||
const fallbackBlock: OcrBlock = {
|
const fallbackBlock: OcrBlock = {
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import { execFile } from "node:child_process";
|
import { execFile } from "node:child_process";
|
||||||
import fs from "node:fs/promises";
|
import fs from "node:fs/promises";
|
||||||
import { promisify } from "node:util";
|
import { promisify } from "node:util";
|
||||||
import type { GuideSnapshot, OcrBlock } from "../../../src/guide/contracts";
|
import type { GuideOcrProfile, GuideSnapshot, OcrBlock } from "../../../src/guide/contracts";
|
||||||
import { ensureBundledOcrServiceRunning } from "./bundledOcrService";
|
import { ensureBundledOcrServiceRunning } from "./bundledOcrService";
|
||||||
|
|
||||||
const execFileAsync = promisify(execFile);
|
const execFileAsync = promisify(execFile);
|
||||||
@@ -10,6 +10,11 @@ export interface GuideOcrClient {
|
|||||||
recognize(snapshot: GuideSnapshot): Promise<OcrBlock[]>;
|
recognize(snapshot: GuideSnapshot): Promise<OcrBlock[]>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface GuideOcrClientConfig {
|
||||||
|
profile: GuideOcrProfile;
|
||||||
|
language: string;
|
||||||
|
}
|
||||||
|
|
||||||
interface PaddleOcrResponseBlock {
|
interface PaddleOcrResponseBlock {
|
||||||
text?: unknown;
|
text?: unknown;
|
||||||
confidence?: unknown;
|
confidence?: unknown;
|
||||||
@@ -21,7 +26,8 @@ interface PaddleOcrResponseBlock {
|
|||||||
export class PaddleOcrHttpClient implements GuideOcrClient {
|
export class PaddleOcrHttpClient implements GuideOcrClient {
|
||||||
constructor(
|
constructor(
|
||||||
private readonly baseUrl = process.env.OPENSCREEN_GUIDE_OCR_URL ?? "http://127.0.0.1:8866",
|
private readonly baseUrl = process.env.OPENSCREEN_GUIDE_OCR_URL ?? "http://127.0.0.1:8866",
|
||||||
private readonly language = process.env.OPENSCREEN_GUIDE_OCR_LANGUAGE ?? "vi,en",
|
private readonly language = normalizeOcrLanguage(process.env.OPENSCREEN_GUIDE_OCR_LANGUAGE),
|
||||||
|
private readonly profile = normalizeOcrProfile(process.env.OPENSCREEN_GUIDE_OCR_PROFILE),
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
async recognize(snapshot: GuideSnapshot): Promise<OcrBlock[]> {
|
async recognize(snapshot: GuideSnapshot): Promise<OcrBlock[]> {
|
||||||
@@ -36,6 +42,7 @@ export class PaddleOcrHttpClient implements GuideOcrClient {
|
|||||||
imageBase64,
|
imageBase64,
|
||||||
path: snapshot.path,
|
path: snapshot.path,
|
||||||
language: this.language,
|
language: this.language,
|
||||||
|
profile: this.profile,
|
||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -54,7 +61,9 @@ export class PaddleOcrHttpClient implements GuideOcrClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export class WindowsOcrClient implements GuideOcrClient {
|
export class WindowsOcrClient implements GuideOcrClient {
|
||||||
constructor(private readonly language = process.env.OPENSCREEN_GUIDE_OCR_LANGUAGE ?? "vi,en") {}
|
constructor(
|
||||||
|
private readonly language = normalizeOcrLanguage(process.env.OPENSCREEN_GUIDE_OCR_LANGUAGE),
|
||||||
|
) {}
|
||||||
|
|
||||||
async recognize(snapshot: GuideSnapshot): Promise<OcrBlock[]> {
|
async recognize(snapshot: GuideSnapshot): Promise<OcrBlock[]> {
|
||||||
if (process.platform !== "win32") {
|
if (process.platform !== "win32") {
|
||||||
@@ -96,6 +105,14 @@ export class WindowsOcrClient implements GuideOcrClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export class DefaultGuideOcrClient implements GuideOcrClient {
|
export class DefaultGuideOcrClient implements GuideOcrClient {
|
||||||
|
static fromConfig(config?: Partial<GuideOcrClientConfig>): DefaultGuideOcrClient {
|
||||||
|
const normalizedConfig = normalizeOcrClientConfig(config);
|
||||||
|
return new DefaultGuideOcrClient(
|
||||||
|
new PaddleOcrHttpClient(undefined, normalizedConfig.language, normalizedConfig.profile),
|
||||||
|
new WindowsOcrClient(normalizedConfig.language),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
private readonly httpClient = new PaddleOcrHttpClient(),
|
private readonly httpClient = new PaddleOcrHttpClient(),
|
||||||
private readonly windowsClient = new WindowsOcrClient(),
|
private readonly windowsClient = new WindowsOcrClient(),
|
||||||
@@ -119,6 +136,31 @@ export class DefaultGuideOcrClient implements GuideOcrClient {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function normalizeOcrClientConfig(
|
||||||
|
config: Partial<GuideOcrClientConfig> | undefined,
|
||||||
|
): GuideOcrClientConfig {
|
||||||
|
return {
|
||||||
|
profile: normalizeOcrProfile(config?.profile ?? process.env.OPENSCREEN_GUIDE_OCR_PROFILE),
|
||||||
|
language: normalizeOcrLanguage(config?.language ?? process.env.OPENSCREEN_GUIDE_OCR_LANGUAGE),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeOcrProfile(value: string | undefined): GuideOcrProfile {
|
||||||
|
if (value === "fast" || value === "vietnamese" || value === "hybrid") {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
return "vietnamese";
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeOcrLanguage(value: string | undefined): string {
|
||||||
|
const normalized = value
|
||||||
|
?.split(",")
|
||||||
|
.map((part) => part.trim().toLowerCase())
|
||||||
|
.filter(Boolean)
|
||||||
|
.join(",");
|
||||||
|
return normalized || "vi,en";
|
||||||
|
}
|
||||||
|
|
||||||
export function parseWindowsOcrPayload(stdout: string): unknown {
|
export function parseWindowsOcrPayload(stdout: string): unknown {
|
||||||
const normalized = stdout.replace(/^\uFEFF/, "").trim();
|
const normalized = stdout.replace(/^\uFEFF/, "").trim();
|
||||||
try {
|
try {
|
||||||
|
|||||||
+122
-25
@@ -426,6 +426,7 @@ let nativeWindowsCursorRecordingStartMs = 0;
|
|||||||
let nativeWindowsPauseStartedAtMs: number | null = null;
|
let nativeWindowsPauseStartedAtMs: number | null = null;
|
||||||
let nativeWindowsPauseRanges: Array<{ startMs: number; endMs: number }> = [];
|
let nativeWindowsPauseRanges: Array<{ startMs: number; endMs: number }> = [];
|
||||||
let nativeWindowsIsPaused = false;
|
let nativeWindowsIsPaused = false;
|
||||||
|
let nativeWindowsCaptureStopping = false;
|
||||||
const NATIVE_WINDOWS_CAPTURE_STOP_TIMEOUT_MS = 15_000;
|
const NATIVE_WINDOWS_CAPTURE_STOP_TIMEOUT_MS = 15_000;
|
||||||
let nativeMacCaptureProcess: ChildProcessWithoutNullStreams | null = null;
|
let nativeMacCaptureProcess: ChildProcessWithoutNullStreams | null = null;
|
||||||
let nativeMacCaptureOutput = "";
|
let nativeMacCaptureOutput = "";
|
||||||
@@ -1337,6 +1338,81 @@ function completeNativeWindowsCursorPauseRange(endMs = Date.now()) {
|
|||||||
nativeWindowsPauseStartedAtMs = null;
|
nativeWindowsPauseStartedAtMs = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function resetNativeWindowsCaptureState() {
|
||||||
|
nativeWindowsCaptureProcess = null;
|
||||||
|
nativeWindowsCaptureTargetPath = null;
|
||||||
|
nativeWindowsCaptureWebcamTargetPath = null;
|
||||||
|
nativeWindowsCaptureRecordingId = null;
|
||||||
|
nativeWindowsCursorOffsetMs = 0;
|
||||||
|
nativeWindowsCursorCaptureMode = "editable-overlay";
|
||||||
|
nativeWindowsCursorRecordingStartMs = 0;
|
||||||
|
nativeWindowsPauseStartedAtMs = null;
|
||||||
|
nativeWindowsPauseRanges = [];
|
||||||
|
nativeWindowsIsPaused = false;
|
||||||
|
nativeWindowsCaptureStopping = false;
|
||||||
|
clearGuideHotkeyRecording();
|
||||||
|
}
|
||||||
|
|
||||||
|
function hasActiveNativeWindowsCaptureProcess() {
|
||||||
|
const proc = nativeWindowsCaptureProcess;
|
||||||
|
if (!proc) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (proc.exitCode === null && !proc.killed) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.warn("[native-wgc] clearing stale Windows capture process state", {
|
||||||
|
exitCode: proc.exitCode,
|
||||||
|
killed: proc.killed,
|
||||||
|
});
|
||||||
|
resetNativeWindowsCaptureState();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function attachNativeWindowsCaptureLifecycle(
|
||||||
|
proc: ChildProcessWithoutNullStreams,
|
||||||
|
sourceName: string,
|
||||||
|
onRecordingStateChange?: (recording: boolean, sourceName: string) => void,
|
||||||
|
) {
|
||||||
|
const cleanupAfterUnexpectedExit = async () => {
|
||||||
|
try {
|
||||||
|
await stopCursorRecording();
|
||||||
|
} catch (error) {
|
||||||
|
console.warn("[native-wgc] failed to stop cursor recording after helper exit", error);
|
||||||
|
}
|
||||||
|
pendingCursorRecordingData = null;
|
||||||
|
resetNativeWindowsCaptureState();
|
||||||
|
onRecordingStateChange?.(false, sourceName);
|
||||||
|
};
|
||||||
|
|
||||||
|
function onClose(code: number | null, signal: NodeJS.Signals | null) {
|
||||||
|
proc.off("error", onError);
|
||||||
|
if (nativeWindowsCaptureProcess !== proc || nativeWindowsCaptureStopping) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.warn("[native-wgc] Windows capture helper exited before stop was requested", {
|
||||||
|
code,
|
||||||
|
signal,
|
||||||
|
output: nativeWindowsCaptureOutput.trim(),
|
||||||
|
});
|
||||||
|
void cleanupAfterUnexpectedExit();
|
||||||
|
}
|
||||||
|
function onError(error: Error) {
|
||||||
|
proc.off("close", onClose);
|
||||||
|
if (nativeWindowsCaptureProcess !== proc || nativeWindowsCaptureStopping) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.warn("[native-wgc] Windows capture helper errored before stop was requested", error);
|
||||||
|
void cleanupAfterUnexpectedExit();
|
||||||
|
}
|
||||||
|
|
||||||
|
proc.once("close", onClose);
|
||||||
|
proc.once("error", onError);
|
||||||
|
}
|
||||||
|
|
||||||
function waitForNativeWindowsCaptureStart(proc: ChildProcessWithoutNullStreams) {
|
function waitForNativeWindowsCaptureStart(proc: ChildProcessWithoutNullStreams) {
|
||||||
return new Promise<void>((resolve, reject) => {
|
return new Promise<void>((resolve, reject) => {
|
||||||
const timer = setTimeout(() => {
|
const timer = setTimeout(() => {
|
||||||
@@ -1732,7 +1808,7 @@ export function registerIpcHandlers(
|
|||||||
const sources = await desktopCapturer.getSources(opts);
|
const sources = await desktopCapturer.getSources(opts);
|
||||||
lastEnumeratedSources = new Map(sources.map((source) => [source.id, source]));
|
lastEnumeratedSources = new Map(sources.map((source) => [source.id, source]));
|
||||||
let screenSourceIndex = 0;
|
let screenSourceIndex = 0;
|
||||||
return sources.map((source) => {
|
const processedSources = sources.map((source) => {
|
||||||
const isScreenSource = source.id.startsWith("screen:");
|
const isScreenSource = source.id.startsWith("screen:");
|
||||||
const sourceIndex = isScreenSource
|
const sourceIndex = isScreenSource
|
||||||
? (parseDesktopCapturerScreenIndex(source.id) ?? screenSourceIndex)
|
? (parseDesktopCapturerScreenIndex(source.id) ?? screenSourceIndex)
|
||||||
@@ -1760,6 +1836,43 @@ export function registerIpcHandlers(
|
|||||||
bounds,
|
bounds,
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
const screenDisplays = screen.getAllDisplays();
|
||||||
|
const mappedDisplayIds = new Set(
|
||||||
|
processedSources
|
||||||
|
.filter((source) => source.id.startsWith("screen:") && typeof source.displayId === "number")
|
||||||
|
.map((source) => source.displayId),
|
||||||
|
);
|
||||||
|
const fallbackScreenSources = screenDisplays
|
||||||
|
.map((display, displayIndex) => ({ display, displayIndex }))
|
||||||
|
.filter(({ display }) => !mappedDisplayIds.has(display.id))
|
||||||
|
.map(({ display, displayIndex }) => {
|
||||||
|
const bounds = toSourceBounds(display.bounds);
|
||||||
|
return {
|
||||||
|
id: `screen:${displayIndex}:fallback:${display.id}`,
|
||||||
|
name: `Screen ${displayIndex + 1}`,
|
||||||
|
display_id: String(display.id),
|
||||||
|
thumbnail: null,
|
||||||
|
appIcon: null,
|
||||||
|
displayId: display.id,
|
||||||
|
displayIndex,
|
||||||
|
screenIndex: displayIndex,
|
||||||
|
displayLabel: `Display ${displayIndex + 1} - ${bounds.width}x${bounds.height} @ ${bounds.x},${bounds.y}`,
|
||||||
|
bounds,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
if (fallbackScreenSources.length > 0) {
|
||||||
|
console.warn("[desktop-capturer] added fallback display sources", {
|
||||||
|
capturerScreens: processedSources.filter((source) => source.id.startsWith("screen:"))
|
||||||
|
.length,
|
||||||
|
electronDisplays: screenDisplays.length,
|
||||||
|
fallbackScreens: fallbackScreenSources.map((source) => ({
|
||||||
|
id: source.id,
|
||||||
|
displayId: source.displayId,
|
||||||
|
bounds: source.bounds,
|
||||||
|
})),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return [...processedSources, ...fallbackScreenSources];
|
||||||
});
|
});
|
||||||
|
|
||||||
ipcMain.handle("select-source", async (_, source: SelectedSource) => {
|
ipcMain.handle("select-source", async (_, source: SelectedSource) => {
|
||||||
@@ -1964,7 +2077,7 @@ export function registerIpcHandlers(
|
|||||||
error: "Windows Graphics Capture requires Windows 10 build 19041 or newer.",
|
error: "Windows Graphics Capture requires Windows 10 build 19041 or newer.",
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
if (nativeWindowsCaptureProcess) {
|
if (hasActiveNativeWindowsCaptureProcess()) {
|
||||||
return { success: false, error: "Native Windows capture is already running." };
|
return { success: false, error: "Native Windows capture is already running." };
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2113,6 +2226,7 @@ export function registerIpcHandlers(
|
|||||||
});
|
});
|
||||||
|
|
||||||
const source = selectedSource || { name: "Screen" };
|
const source = selectedSource || { name: "Screen" };
|
||||||
|
attachNativeWindowsCaptureLifecycle(proc, source.name, onRecordingStateChange);
|
||||||
startGuideHotkeyRecording(recordingId, bounds);
|
startGuideHotkeyRecording(recordingId, bounds);
|
||||||
if (onRecordingStateChange) {
|
if (onRecordingStateChange) {
|
||||||
onRecordingStateChange(true, source.name);
|
onRecordingStateChange(true, source.name);
|
||||||
@@ -2127,17 +2241,7 @@ export function registerIpcHandlers(
|
|||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Failed to start native Windows recording:", error);
|
console.error("Failed to start native Windows recording:", error);
|
||||||
nativeWindowsCaptureProcess?.kill();
|
nativeWindowsCaptureProcess?.kill();
|
||||||
nativeWindowsCaptureProcess = null;
|
resetNativeWindowsCaptureState();
|
||||||
nativeWindowsCaptureTargetPath = null;
|
|
||||||
nativeWindowsCaptureWebcamTargetPath = null;
|
|
||||||
nativeWindowsCaptureRecordingId = null;
|
|
||||||
nativeWindowsCursorOffsetMs = 0;
|
|
||||||
nativeWindowsCursorCaptureMode = "editable-overlay";
|
|
||||||
nativeWindowsCursorRecordingStartMs = 0;
|
|
||||||
nativeWindowsPauseStartedAtMs = null;
|
|
||||||
nativeWindowsPauseRanges = [];
|
|
||||||
nativeWindowsIsPaused = false;
|
|
||||||
clearGuideHotkeyRecording();
|
|
||||||
await stopCursorRecording();
|
await stopCursorRecording();
|
||||||
return { success: false, error: String(error) };
|
return { success: false, error: String(error) };
|
||||||
}
|
}
|
||||||
@@ -2396,11 +2500,13 @@ export function registerIpcHandlers(
|
|||||||
const recordingId = nativeWindowsCaptureRecordingId ?? Date.now();
|
const recordingId = nativeWindowsCaptureRecordingId ?? Date.now();
|
||||||
const cursorCaptureMode = nativeWindowsCursorCaptureMode;
|
const cursorCaptureMode = nativeWindowsCursorCaptureMode;
|
||||||
|
|
||||||
if (!proc) {
|
if (!proc || proc.exitCode !== null || proc.killed) {
|
||||||
|
resetNativeWindowsCaptureState();
|
||||||
return { success: false, error: "Native Windows capture is not running." };
|
return { success: false, error: "Native Windows capture is not running." };
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
nativeWindowsCaptureStopping = true;
|
||||||
completeNativeWindowsCursorPauseRange();
|
completeNativeWindowsCursorPauseRange();
|
||||||
const stoppedPathPromise = waitForNativeWindowsCaptureStop(proc);
|
const stoppedPathPromise = waitForNativeWindowsCaptureStop(proc);
|
||||||
proc.stdin.write("stop\n");
|
proc.stdin.write("stop\n");
|
||||||
@@ -2462,17 +2568,7 @@ export function registerIpcHandlers(
|
|||||||
await stopCursorRecording();
|
await stopCursorRecording();
|
||||||
return { success: false, error: String(error) };
|
return { success: false, error: String(error) };
|
||||||
} finally {
|
} finally {
|
||||||
nativeWindowsCaptureProcess = null;
|
resetNativeWindowsCaptureState();
|
||||||
nativeWindowsCaptureTargetPath = null;
|
|
||||||
nativeWindowsCaptureWebcamTargetPath = null;
|
|
||||||
nativeWindowsCaptureRecordingId = null;
|
|
||||||
nativeWindowsCursorOffsetMs = 0;
|
|
||||||
nativeWindowsCursorCaptureMode = "editable-overlay";
|
|
||||||
nativeWindowsCursorRecordingStartMs = 0;
|
|
||||||
nativeWindowsPauseStartedAtMs = null;
|
|
||||||
nativeWindowsPauseRanges = [];
|
|
||||||
nativeWindowsIsPaused = false;
|
|
||||||
clearGuideHotkeyRecording();
|
|
||||||
const source = selectedSource || { name: "Screen" };
|
const source = selectedSource || { name: "Screen" };
|
||||||
if (onRecordingStateChange) {
|
if (onRecordingStateChange) {
|
||||||
onRecordingStateChange(false, source.name);
|
onRecordingStateChange(false, source.name);
|
||||||
@@ -2637,6 +2733,7 @@ export function registerIpcHandlers(
|
|||||||
);
|
);
|
||||||
const guideStore = new GuideStore(RECORDINGS_DIR, {
|
const guideStore = new GuideStore(RECORDINGS_DIR, {
|
||||||
deepSeekConfigProvider: guideAiSettingsStore,
|
deepSeekConfigProvider: guideAiSettingsStore,
|
||||||
|
ocrConfigProvider: guideAiSettingsStore,
|
||||||
});
|
});
|
||||||
registerGuideMarkerHotkey(guideStore);
|
registerGuideMarkerHotkey(guideStore);
|
||||||
registerGuideIpcHandlers(ipcMain, guideStore, guideAiSettingsStore, {
|
registerGuideIpcHandlers(ipcMain, guideStore, guideAiSettingsStore, {
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import {
|
|||||||
} from "electron";
|
} from "electron";
|
||||||
import { mainT, setMainLocale } from "./i18n";
|
import { mainT, setMainLocale } from "./i18n";
|
||||||
import { getSelectedDesktopSource, registerIpcHandlers } from "./ipc/handlers";
|
import { getSelectedDesktopSource, registerIpcHandlers } from "./ipc/handlers";
|
||||||
|
import { initializeAutoUpdates } from "./updater";
|
||||||
import {
|
import {
|
||||||
createCountdownOverlayWindow,
|
createCountdownOverlayWindow,
|
||||||
createEditorWindow,
|
createEditorWindow,
|
||||||
@@ -515,6 +516,7 @@ app.whenReady().then(async () => {
|
|||||||
createTray();
|
createTray();
|
||||||
updateTrayMenu();
|
updateTrayMenu();
|
||||||
setupApplicationMenu();
|
setupApplicationMenu();
|
||||||
|
initializeAutoUpdates();
|
||||||
// Ensure recordings directory exists
|
// Ensure recordings directory exists
|
||||||
await ensureRecordingsDir();
|
await ensureRecordingsDir();
|
||||||
|
|
||||||
|
|||||||
@@ -81,3 +81,21 @@ target_compile_options(guide-hotkey-listener PRIVATE /EHsc /W4 /utf-8)
|
|||||||
target_link_libraries(guide-hotkey-listener PRIVATE
|
target_link_libraries(guide-hotkey-listener PRIVATE
|
||||||
user32
|
user32
|
||||||
)
|
)
|
||||||
|
|
||||||
|
add_executable(openscreen-ocr-service-wrapper
|
||||||
|
src/ocr-service-wrapper.cpp
|
||||||
|
)
|
||||||
|
|
||||||
|
target_compile_definitions(openscreen-ocr-service-wrapper PRIVATE
|
||||||
|
NOMINMAX
|
||||||
|
WIN32_LEAN_AND_MEAN
|
||||||
|
UNICODE
|
||||||
|
_UNICODE
|
||||||
|
_WIN32_WINNT=0x0A00
|
||||||
|
)
|
||||||
|
|
||||||
|
target_compile_options(openscreen-ocr-service-wrapper PRIVATE /EHsc /W4 /utf-8)
|
||||||
|
|
||||||
|
target_link_libraries(openscreen-ocr-service-wrapper PRIVATE
|
||||||
|
advapi32
|
||||||
|
)
|
||||||
|
|||||||
@@ -0,0 +1,263 @@
|
|||||||
|
#include <Windows.h>
|
||||||
|
|
||||||
|
#include <algorithm>
|
||||||
|
#include <iostream>
|
||||||
|
#include <string>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
|
||||||
|
constexpr const wchar_t* SERVICE_NAME = L"OpenScreenOCR";
|
||||||
|
|
||||||
|
struct ServiceConfig {
|
||||||
|
std::wstring exePath;
|
||||||
|
std::wstring resourcesPath;
|
||||||
|
std::wstring dataPath;
|
||||||
|
};
|
||||||
|
|
||||||
|
SERVICE_STATUS_HANDLE g_statusHandle = nullptr;
|
||||||
|
SERVICE_STATUS g_status{};
|
||||||
|
HANDLE g_stopEvent = nullptr;
|
||||||
|
PROCESS_INFORMATION g_childProcess{};
|
||||||
|
ServiceConfig g_config;
|
||||||
|
|
||||||
|
std::wstring quoteArg(const std::wstring& value) {
|
||||||
|
std::wstring result = L"\"";
|
||||||
|
for (wchar_t ch : value) {
|
||||||
|
if (ch == L'"') {
|
||||||
|
result += L"\\\"";
|
||||||
|
} else {
|
||||||
|
result.push_back(ch);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result += L"\"";
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::wstring directoryName(const std::wstring& path) {
|
||||||
|
const size_t slash = path.find_last_of(L"\\/");
|
||||||
|
return slash == std::wstring::npos ? L"." : path.substr(0, slash);
|
||||||
|
}
|
||||||
|
|
||||||
|
void createDirectoryRecursive(const std::wstring& path) {
|
||||||
|
if (path.empty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::wstring current;
|
||||||
|
for (size_t i = 0; i < path.size(); ++i) {
|
||||||
|
current.push_back(path[i]);
|
||||||
|
if (path[i] != L'\\' && path[i] != L'/') {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (current.size() > 3) {
|
||||||
|
CreateDirectoryW(current.c_str(), nullptr);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
CreateDirectoryW(path.c_str(), nullptr);
|
||||||
|
}
|
||||||
|
|
||||||
|
void setEnv(const wchar_t* name, const std::wstring& value) {
|
||||||
|
SetEnvironmentVariableW(name, value.empty() ? nullptr : value.c_str());
|
||||||
|
}
|
||||||
|
|
||||||
|
void setServiceStatus(DWORD state, DWORD win32ExitCode = NO_ERROR, DWORD waitHint = 0) {
|
||||||
|
if (!g_statusHandle) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
g_status.dwServiceType = SERVICE_WIN32_OWN_PROCESS;
|
||||||
|
g_status.dwCurrentState = state;
|
||||||
|
g_status.dwWin32ExitCode = win32ExitCode;
|
||||||
|
g_status.dwWaitHint = waitHint;
|
||||||
|
g_status.dwControlsAccepted =
|
||||||
|
state == SERVICE_RUNNING ? SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN : 0;
|
||||||
|
static DWORD checkpoint = 1;
|
||||||
|
g_status.dwCheckPoint =
|
||||||
|
state == SERVICE_START_PENDING || state == SERVICE_STOP_PENDING ? checkpoint++ : 0;
|
||||||
|
SetServiceStatus(g_statusHandle, &g_status);
|
||||||
|
}
|
||||||
|
|
||||||
|
HANDLE openServiceLog(const std::wstring& dataPath) {
|
||||||
|
const std::wstring logDir = dataPath + L"\\logs";
|
||||||
|
createDirectoryRecursive(logDir);
|
||||||
|
const std::wstring logPath = logDir + L"\\ocr-service.log";
|
||||||
|
SECURITY_ATTRIBUTES securityAttributes{};
|
||||||
|
securityAttributes.nLength = sizeof(securityAttributes);
|
||||||
|
securityAttributes.bInheritHandle = TRUE;
|
||||||
|
HANDLE file = CreateFileW(
|
||||||
|
logPath.c_str(),
|
||||||
|
FILE_APPEND_DATA,
|
||||||
|
FILE_SHARE_READ | FILE_SHARE_WRITE,
|
||||||
|
&securityAttributes,
|
||||||
|
OPEN_ALWAYS,
|
||||||
|
FILE_ATTRIBUTE_NORMAL,
|
||||||
|
nullptr);
|
||||||
|
if (file != INVALID_HANDLE_VALUE) {
|
||||||
|
SetFilePointer(file, 0, nullptr, FILE_END);
|
||||||
|
}
|
||||||
|
return file;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool startOcrProcess(const ServiceConfig& config) {
|
||||||
|
if (config.exePath.empty()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const std::wstring dataPath = config.dataPath.empty()
|
||||||
|
? directoryName(config.exePath) + L"\\ocr-runtime"
|
||||||
|
: config.dataPath;
|
||||||
|
const std::wstring resourcesPath = config.resourcesPath.empty()
|
||||||
|
? directoryName(directoryName(config.exePath))
|
||||||
|
: config.resourcesPath;
|
||||||
|
const std::wstring modelCachePath = dataPath + L"\\ocr-models";
|
||||||
|
const std::wstring paddlexCachePath = resourcesPath + L"\\ocr-models\\paddlex";
|
||||||
|
|
||||||
|
createDirectoryRecursive(dataPath);
|
||||||
|
createDirectoryRecursive(modelCachePath);
|
||||||
|
|
||||||
|
setEnv(L"OPENSCREEN_OCR_HOST", L"127.0.0.1");
|
||||||
|
setEnv(L"OPENSCREEN_OCR_PORT", L"8866");
|
||||||
|
setEnv(L"PADDLEOCR_DEVICE", L"cpu");
|
||||||
|
setEnv(L"PADDLEOCR_ENABLE_MKLDNN", L"0");
|
||||||
|
setEnv(L"PADDLEOCR_LANG", L"");
|
||||||
|
setEnv(L"PADDLEOCR_USE_MOBILE", L"1");
|
||||||
|
setEnv(L"OPENSCREEN_OCR_PROFILE", L"vietnamese");
|
||||||
|
setEnv(L"OPENSCREEN_OCR_WARMUP", L"1");
|
||||||
|
setEnv(L"PADDLE_PDX_ENABLE_MKLDNN_BYDEFAULT", L"False");
|
||||||
|
setEnv(L"PADDLE_PDX_CACHE_HOME", paddlexCachePath);
|
||||||
|
setEnv(L"PADDLE_PDX_DISABLE_MODEL_SOURCE_CHECK", L"True");
|
||||||
|
setEnv(L"PADDLE_HOME", modelCachePath + L"\\paddle");
|
||||||
|
setEnv(L"PADDLEOCR_HOME", modelCachePath + L"\\paddleocr");
|
||||||
|
setEnv(L"PYTHONUTF8", L"1");
|
||||||
|
|
||||||
|
STARTUPINFOW startupInfo{};
|
||||||
|
startupInfo.cb = sizeof(startupInfo);
|
||||||
|
HANDLE logFile = openServiceLog(dataPath);
|
||||||
|
if (logFile != INVALID_HANDLE_VALUE) {
|
||||||
|
startupInfo.dwFlags |= STARTF_USESTDHANDLES;
|
||||||
|
startupInfo.hStdOutput = logFile;
|
||||||
|
startupInfo.hStdError = logFile;
|
||||||
|
startupInfo.hStdInput = GetStdHandle(STD_INPUT_HANDLE);
|
||||||
|
}
|
||||||
|
|
||||||
|
std::wstring commandLine = quoteArg(config.exePath);
|
||||||
|
const std::wstring cwd = directoryName(config.exePath);
|
||||||
|
ZeroMemory(&g_childProcess, sizeof(g_childProcess));
|
||||||
|
const BOOL created = CreateProcessW(
|
||||||
|
config.exePath.c_str(),
|
||||||
|
commandLine.data(),
|
||||||
|
nullptr,
|
||||||
|
nullptr,
|
||||||
|
TRUE,
|
||||||
|
CREATE_NO_WINDOW,
|
||||||
|
nullptr,
|
||||||
|
cwd.c_str(),
|
||||||
|
&startupInfo,
|
||||||
|
&g_childProcess);
|
||||||
|
|
||||||
|
if (logFile != INVALID_HANDLE_VALUE) {
|
||||||
|
CloseHandle(logFile);
|
||||||
|
}
|
||||||
|
return created == TRUE;
|
||||||
|
}
|
||||||
|
|
||||||
|
void stopOcrProcess() {
|
||||||
|
if (g_childProcess.hProcess) {
|
||||||
|
TerminateProcess(g_childProcess.hProcess, 0);
|
||||||
|
WaitForSingleObject(g_childProcess.hProcess, 10000);
|
||||||
|
CloseHandle(g_childProcess.hProcess);
|
||||||
|
g_childProcess.hProcess = nullptr;
|
||||||
|
}
|
||||||
|
if (g_childProcess.hThread) {
|
||||||
|
CloseHandle(g_childProcess.hThread);
|
||||||
|
g_childProcess.hThread = nullptr;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
DWORD WINAPI serviceControlHandler(DWORD control, DWORD, LPVOID, LPVOID) {
|
||||||
|
if (control == SERVICE_CONTROL_STOP || control == SERVICE_CONTROL_SHUTDOWN) {
|
||||||
|
setServiceStatus(SERVICE_STOP_PENDING, NO_ERROR, 10000);
|
||||||
|
if (g_stopEvent) {
|
||||||
|
SetEvent(g_stopEvent);
|
||||||
|
}
|
||||||
|
stopOcrProcess();
|
||||||
|
return NO_ERROR;
|
||||||
|
}
|
||||||
|
return NO_ERROR;
|
||||||
|
}
|
||||||
|
|
||||||
|
void WINAPI serviceMain(DWORD, LPWSTR*) {
|
||||||
|
g_statusHandle = RegisterServiceCtrlHandlerExW(SERVICE_NAME, serviceControlHandler, nullptr);
|
||||||
|
if (!g_statusHandle) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
setServiceStatus(SERVICE_START_PENDING, NO_ERROR, 30000);
|
||||||
|
g_stopEvent = CreateEventW(nullptr, TRUE, FALSE, nullptr);
|
||||||
|
if (!g_stopEvent || !startOcrProcess(g_config)) {
|
||||||
|
setServiceStatus(SERVICE_STOPPED, ERROR_SERVICE_SPECIFIC_ERROR);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
setServiceStatus(SERVICE_RUNNING);
|
||||||
|
HANDLE waitHandles[] = {g_stopEvent, g_childProcess.hProcess};
|
||||||
|
WaitForMultipleObjects(2, waitHandles, FALSE, INFINITE);
|
||||||
|
stopOcrProcess();
|
||||||
|
if (g_stopEvent) {
|
||||||
|
CloseHandle(g_stopEvent);
|
||||||
|
g_stopEvent = nullptr;
|
||||||
|
}
|
||||||
|
setServiceStatus(SERVICE_STOPPED);
|
||||||
|
}
|
||||||
|
|
||||||
|
ServiceConfig parseConfig(int argc, wchar_t* argv[]) {
|
||||||
|
ServiceConfig config;
|
||||||
|
for (int i = 1; i < argc; ++i) {
|
||||||
|
const std::wstring arg = argv[i];
|
||||||
|
auto readNext = [&](std::wstring& target) {
|
||||||
|
if (i + 1 < argc) {
|
||||||
|
target = argv[++i];
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if (arg == L"--exe") {
|
||||||
|
readNext(config.exePath);
|
||||||
|
} else if (arg == L"--resources") {
|
||||||
|
readNext(config.resourcesPath);
|
||||||
|
} else if (arg == L"--data") {
|
||||||
|
readNext(config.dataPath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return config;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool hasServiceFlag(int argc, wchar_t* argv[]) {
|
||||||
|
for (int i = 1; i < argc; ++i) {
|
||||||
|
if (std::wstring(argv[i]) == L"--service") {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
int wmain(int argc, wchar_t* argv[]) {
|
||||||
|
g_config = parseConfig(argc, argv);
|
||||||
|
|
||||||
|
if (hasServiceFlag(argc, argv)) {
|
||||||
|
SERVICE_TABLE_ENTRYW serviceTable[] = {
|
||||||
|
{const_cast<LPWSTR>(SERVICE_NAME), serviceMain},
|
||||||
|
{nullptr, nullptr},
|
||||||
|
};
|
||||||
|
return StartServiceCtrlDispatcherW(serviceTable) ? 0 : 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!startOcrProcess(g_config)) {
|
||||||
|
std::wcerr << L"Failed to start OCR service process." << std::endl;
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
WaitForSingleObject(g_childProcess.hProcess, INFINITE);
|
||||||
|
stopOcrProcess();
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
@@ -28,6 +28,60 @@ bool succeeded(HRESULT hr, const char* label) {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Microsoft::WRL::ComPtr<IDXGIAdapter1> findAdapterForMonitor(HMONITOR monitor) {
|
||||||
|
if (!monitor) {
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
Microsoft::WRL::ComPtr<IDXGIFactory1> factory;
|
||||||
|
HRESULT hr = CreateDXGIFactory1(IID_PPV_ARGS(&factory));
|
||||||
|
if (FAILED(hr) || !factory) {
|
||||||
|
std::cerr << "WARNING: CreateDXGIFactory1 failed while resolving monitor adapter (hr=0x"
|
||||||
|
<< std::hex << hr << std::dec << ")" << std::endl;
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (UINT adapterIndex = 0;; ++adapterIndex) {
|
||||||
|
Microsoft::WRL::ComPtr<IDXGIAdapter1> adapter;
|
||||||
|
hr = factory->EnumAdapters1(adapterIndex, adapter.GetAddressOf());
|
||||||
|
if (hr == DXGI_ERROR_NOT_FOUND) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (FAILED(hr) || !adapter) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
DXGI_ADAPTER_DESC1 adapterDesc{};
|
||||||
|
if (SUCCEEDED(adapter->GetDesc1(&adapterDesc)) &&
|
||||||
|
(adapterDesc.Flags & DXGI_ADAPTER_FLAG_SOFTWARE) != 0) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (UINT outputIndex = 0;; ++outputIndex) {
|
||||||
|
Microsoft::WRL::ComPtr<IDXGIOutput> output;
|
||||||
|
hr = adapter->EnumOutputs(outputIndex, output.GetAddressOf());
|
||||||
|
if (hr == DXGI_ERROR_NOT_FOUND) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (FAILED(hr) || !output) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
DXGI_OUTPUT_DESC outputDesc{};
|
||||||
|
if (SUCCEEDED(output->GetDesc(&outputDesc)) && outputDesc.Monitor == monitor) {
|
||||||
|
std::cout << "{\"event\":\"display-adapter-resolved\",\"schemaVersion\":2,"
|
||||||
|
<< "\"adapterIndex\":" << adapterIndex
|
||||||
|
<< ",\"outputIndex\":" << outputIndex << "}" << std::endl;
|
||||||
|
return adapter;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
std::cerr << "WARNING: Could not resolve DXGI adapter for selected monitor; using default adapter"
|
||||||
|
<< std::endl;
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
int64_t timeSpanToHns(wf::TimeSpan const& value) {
|
int64_t timeSpanToHns(wf::TimeSpan const& value) {
|
||||||
return value.count();
|
return value.count();
|
||||||
}
|
}
|
||||||
@@ -38,7 +92,7 @@ WgcSession::~WgcSession() {
|
|||||||
stop();
|
stop();
|
||||||
}
|
}
|
||||||
|
|
||||||
bool WgcSession::createD3DDevice() {
|
bool WgcSession::createD3DDevice(IDXGIAdapter* adapter) {
|
||||||
UINT flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT;
|
UINT flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT;
|
||||||
#if defined(_DEBUG)
|
#if defined(_DEBUG)
|
||||||
flags |= D3D11_CREATE_DEVICE_DEBUG;
|
flags |= D3D11_CREATE_DEVICE_DEBUG;
|
||||||
@@ -53,8 +107,8 @@ bool WgcSession::createD3DDevice() {
|
|||||||
D3D_FEATURE_LEVEL featureLevel{};
|
D3D_FEATURE_LEVEL featureLevel{};
|
||||||
|
|
||||||
HRESULT hr = D3D11CreateDevice(
|
HRESULT hr = D3D11CreateDevice(
|
||||||
nullptr,
|
adapter,
|
||||||
D3D_DRIVER_TYPE_HARDWARE,
|
adapter ? D3D_DRIVER_TYPE_UNKNOWN : D3D_DRIVER_TYPE_HARDWARE,
|
||||||
nullptr,
|
nullptr,
|
||||||
flags,
|
flags,
|
||||||
featureLevels,
|
featureLevels,
|
||||||
@@ -67,6 +121,23 @@ bool WgcSession::createD3DDevice() {
|
|||||||
#if defined(_DEBUG)
|
#if defined(_DEBUG)
|
||||||
if (FAILED(hr)) {
|
if (FAILED(hr)) {
|
||||||
flags &= ~D3D11_CREATE_DEVICE_DEBUG;
|
flags &= ~D3D11_CREATE_DEVICE_DEBUG;
|
||||||
|
hr = D3D11CreateDevice(
|
||||||
|
adapter,
|
||||||
|
adapter ? D3D_DRIVER_TYPE_UNKNOWN : D3D_DRIVER_TYPE_HARDWARE,
|
||||||
|
nullptr,
|
||||||
|
flags,
|
||||||
|
featureLevels,
|
||||||
|
ARRAYSIZE(featureLevels),
|
||||||
|
D3D11_SDK_VERSION,
|
||||||
|
&d3dDevice_,
|
||||||
|
&featureLevel,
|
||||||
|
&d3dContext_);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
if (FAILED(hr) && adapter) {
|
||||||
|
std::cerr << "WARNING: D3D11CreateDevice failed for selected monitor adapter (hr=0x"
|
||||||
|
<< std::hex << hr << std::dec << "); retrying default adapter" << std::endl;
|
||||||
hr = D3D11CreateDevice(
|
hr = D3D11CreateDevice(
|
||||||
nullptr,
|
nullptr,
|
||||||
D3D_DRIVER_TYPE_HARDWARE,
|
D3D_DRIVER_TYPE_HARDWARE,
|
||||||
@@ -79,7 +150,6 @@ bool WgcSession::createD3DDevice() {
|
|||||||
&featureLevel,
|
&featureLevel,
|
||||||
&d3dContext_);
|
&d3dContext_);
|
||||||
}
|
}
|
||||||
#endif
|
|
||||||
|
|
||||||
if (!succeeded(hr, "D3D11CreateDevice")) {
|
if (!succeeded(hr, "D3D11CreateDevice")) {
|
||||||
return false;
|
return false;
|
||||||
@@ -100,6 +170,11 @@ bool WgcSession::createD3DDevice() {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool WgcSession::createD3DDeviceForMonitor(HMONITOR monitor) {
|
||||||
|
auto adapter = findAdapterForMonitor(monitor);
|
||||||
|
return createD3DDevice(adapter.Get());
|
||||||
|
}
|
||||||
|
|
||||||
bool WgcSession::createCaptureItem(HMONITOR monitor) {
|
bool WgcSession::createCaptureItem(HMONITOR monitor) {
|
||||||
auto factory = winrt::get_activation_factory<wgcap::GraphicsCaptureItem>();
|
auto factory = winrt::get_activation_factory<wgcap::GraphicsCaptureItem>();
|
||||||
auto interop = factory.as<IGraphicsCaptureItemInterop>();
|
auto interop = factory.as<IGraphicsCaptureItemInterop>();
|
||||||
@@ -188,7 +263,7 @@ bool WgcSession::applySessionOptions(bool captureCursor) {
|
|||||||
|
|
||||||
bool WgcSession::initialize(HMONITOR monitor, int fps, bool captureCursor) {
|
bool WgcSession::initialize(HMONITOR monitor, int fps, bool captureCursor) {
|
||||||
fps_ = fps > 0 ? fps : 60;
|
fps_ = fps > 0 ? fps : 60;
|
||||||
if (!createD3DDevice()) {
|
if (!createD3DDeviceForMonitor(monitor)) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (!createCaptureItem(monitor)) {
|
if (!createCaptureItem(monitor)) {
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
#include <Windows.h>
|
#include <Windows.h>
|
||||||
#include <d3d11.h>
|
#include <d3d11.h>
|
||||||
|
#include <dxgi.h>
|
||||||
#include <windows.graphics.capture.h>
|
#include <windows.graphics.capture.h>
|
||||||
#include <windows.graphics.directx.direct3d11.interop.h>
|
#include <windows.graphics.directx.direct3d11.interop.h>
|
||||||
#include <winrt/Windows.Foundation.h>
|
#include <winrt/Windows.Foundation.h>
|
||||||
@@ -34,7 +35,8 @@ public:
|
|||||||
ID3D11DeviceContext* context() const;
|
ID3D11DeviceContext* context() const;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
bool createD3DDevice();
|
bool createD3DDevice(IDXGIAdapter* adapter = nullptr);
|
||||||
|
bool createD3DDeviceForMonitor(HMONITOR monitor);
|
||||||
bool createCaptureItem(HMONITOR monitor);
|
bool createCaptureItem(HMONITOR monitor);
|
||||||
bool createCaptureItem(HWND window);
|
bool createCaptureItem(HWND window);
|
||||||
bool applySessionOptions(bool captureCursor);
|
bool applySessionOptions(bool captureCursor);
|
||||||
|
|||||||
@@ -26,6 +26,27 @@ const assetBaseUrl = assetBaseUrlArg ? assetBaseUrlArg.slice(ASSET_BASE_URL_ARG_
|
|||||||
|
|
||||||
contextBridge.exposeInMainWorld("electronAPI", {
|
contextBridge.exposeInMainWorld("electronAPI", {
|
||||||
assetBaseUrl,
|
assetBaseUrl,
|
||||||
|
updates: {
|
||||||
|
getStatus: () => {
|
||||||
|
return ipcRenderer.invoke("updates:get-status");
|
||||||
|
},
|
||||||
|
check: () => {
|
||||||
|
return ipcRenderer.invoke("updates:check");
|
||||||
|
},
|
||||||
|
install: () => {
|
||||||
|
return ipcRenderer.invoke("updates:install");
|
||||||
|
},
|
||||||
|
onStatus: (callback: (status: import("../src/lib/updateStatus").UpdateStatus) => void) => {
|
||||||
|
const listener = (
|
||||||
|
_event: Electron.IpcRendererEvent,
|
||||||
|
status: import("../src/lib/updateStatus").UpdateStatus,
|
||||||
|
) => {
|
||||||
|
callback(status);
|
||||||
|
};
|
||||||
|
ipcRenderer.on("updates:status", listener);
|
||||||
|
return () => ipcRenderer.removeListener("updates:status", listener);
|
||||||
|
},
|
||||||
|
},
|
||||||
invokeNativeBridge: <TData>(request: NativeBridgeRequest) => {
|
invokeNativeBridge: <TData>(request: NativeBridgeRequest) => {
|
||||||
return ipcRenderer.invoke(NATIVE_BRIDGE_CHANNEL, request) as Promise<TData>;
|
return ipcRenderer.invoke(NATIVE_BRIDGE_CHANNEL, request) as Promise<TData>;
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -0,0 +1,177 @@
|
|||||||
|
import { app, BrowserWindow, ipcMain } from "electron";
|
||||||
|
import { autoUpdater, type ProgressInfo, type UpdateInfo } from "electron-updater";
|
||||||
|
import type { UpdateCheckResult, UpdateStatus } from "../src/lib/updateStatus";
|
||||||
|
|
||||||
|
const DEFAULT_UPDATE_FEED_URL =
|
||||||
|
"https://gittea.softs.business/huanld/openscreen/raw/branch/release-assets%2Flatest";
|
||||||
|
const AUTO_CHECK_DELAY_MS = 10_000;
|
||||||
|
const AUTO_CHECK_INTERVAL_MS = 4 * 60 * 60 * 1000;
|
||||||
|
|
||||||
|
let status: UpdateStatus = createStatus("idle");
|
||||||
|
let handlersRegistered = false;
|
||||||
|
let initialized = false;
|
||||||
|
let checkInFlight: Promise<UpdateCheckResult> | null = null;
|
||||||
|
|
||||||
|
function createStatus(
|
||||||
|
phase: UpdateStatus["phase"],
|
||||||
|
patch: Partial<UpdateStatus> = {},
|
||||||
|
): UpdateStatus {
|
||||||
|
return {
|
||||||
|
phase,
|
||||||
|
currentVersion: app.getVersion(),
|
||||||
|
updatedAt: new Date().toISOString(),
|
||||||
|
...patch,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeReleaseNotes(releaseNotes: UpdateInfo["releaseNotes"]): string | undefined {
|
||||||
|
if (typeof releaseNotes === "string") {
|
||||||
|
return releaseNotes;
|
||||||
|
}
|
||||||
|
if (Array.isArray(releaseNotes)) {
|
||||||
|
return releaseNotes
|
||||||
|
.map((note) => note.note)
|
||||||
|
.filter(Boolean)
|
||||||
|
.join("\n\n");
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateStatus(next: UpdateStatus) {
|
||||||
|
status = next;
|
||||||
|
for (const window of BrowserWindow.getAllWindows()) {
|
||||||
|
if (!window.isDestroyed()) {
|
||||||
|
window.webContents.send("updates:status", status);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function statusFromInfo(phase: UpdateStatus["phase"], info: UpdateInfo): UpdateStatus {
|
||||||
|
return createStatus(phase, {
|
||||||
|
version: info.version,
|
||||||
|
releaseName: info.releaseName ?? undefined,
|
||||||
|
releaseNotes: normalizeReleaseNotes(info.releaseNotes),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function checkForUpdates(): Promise<UpdateCheckResult> {
|
||||||
|
if (!initialized) {
|
||||||
|
updateStatus(
|
||||||
|
createStatus("unsupported", {
|
||||||
|
error: "Update service is not initialized.",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return { success: false, status, error: status.error };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!app.isPackaged && process.env.OPENSCREEN_ALLOW_DEV_UPDATE_CHECK !== "1") {
|
||||||
|
updateStatus(
|
||||||
|
createStatus("unsupported", {
|
||||||
|
error: "Update checks only run in packaged builds.",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return { success: false, status, error: status.error };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (checkInFlight) {
|
||||||
|
return checkInFlight;
|
||||||
|
}
|
||||||
|
|
||||||
|
updateStatus(createStatus("checking"));
|
||||||
|
checkInFlight = autoUpdater
|
||||||
|
.checkForUpdates()
|
||||||
|
.then(() => ({ success: true, status }))
|
||||||
|
.catch((error) => {
|
||||||
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
|
updateStatus(createStatus("error", { error: message }));
|
||||||
|
return { success: false, status, error: message };
|
||||||
|
})
|
||||||
|
.finally(() => {
|
||||||
|
checkInFlight = null;
|
||||||
|
});
|
||||||
|
|
||||||
|
return checkInFlight;
|
||||||
|
}
|
||||||
|
|
||||||
|
function registerUpdateIpcHandlers() {
|
||||||
|
if (handlersRegistered) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
handlersRegistered = true;
|
||||||
|
ipcMain.handle("updates:get-status", () => status);
|
||||||
|
ipcMain.handle("updates:check", () => checkForUpdates());
|
||||||
|
ipcMain.handle("updates:install", () => {
|
||||||
|
if (status.phase !== "downloaded") {
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
status,
|
||||||
|
error: "No downloaded update is ready to install.",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
setImmediate(() => autoUpdater.quitAndInstall(false, true));
|
||||||
|
return { success: true, status };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function initializeAutoUpdates() {
|
||||||
|
registerUpdateIpcHandlers();
|
||||||
|
|
||||||
|
if (initialized) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
initialized = true;
|
||||||
|
autoUpdater.autoDownload = true;
|
||||||
|
autoUpdater.autoInstallOnAppQuit = true;
|
||||||
|
autoUpdater.logger = console;
|
||||||
|
|
||||||
|
const feedUrl = process.env.OPENSCREEN_UPDATE_FEED_URL?.trim() || DEFAULT_UPDATE_FEED_URL;
|
||||||
|
const updateToken = process.env.OPENSCREEN_UPDATE_TOKEN?.trim();
|
||||||
|
if (updateToken) {
|
||||||
|
autoUpdater.requestHeaders = {
|
||||||
|
Authorization: `token ${updateToken}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
autoUpdater.setFeedURL({
|
||||||
|
provider: "generic",
|
||||||
|
url: feedUrl,
|
||||||
|
});
|
||||||
|
|
||||||
|
autoUpdater.on("checking-for-update", () => {
|
||||||
|
updateStatus(createStatus("checking"));
|
||||||
|
});
|
||||||
|
autoUpdater.on("update-available", (info) => {
|
||||||
|
updateStatus(statusFromInfo("available", info));
|
||||||
|
});
|
||||||
|
autoUpdater.on("update-not-available", (info) => {
|
||||||
|
updateStatus(statusFromInfo("not-available", info));
|
||||||
|
});
|
||||||
|
autoUpdater.on("download-progress", (progress: ProgressInfo) => {
|
||||||
|
updateStatus(
|
||||||
|
createStatus("downloading", {
|
||||||
|
version: status.version,
|
||||||
|
releaseName: status.releaseName,
|
||||||
|
releaseNotes: status.releaseNotes,
|
||||||
|
percent: progress.percent,
|
||||||
|
bytesPerSecond: progress.bytesPerSecond,
|
||||||
|
transferred: progress.transferred,
|
||||||
|
total: progress.total,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
autoUpdater.on("update-downloaded", (info) => {
|
||||||
|
updateStatus(statusFromInfo("downloaded", info));
|
||||||
|
});
|
||||||
|
autoUpdater.on("error", (error) => {
|
||||||
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
|
updateStatus(createStatus("error", { error: message }));
|
||||||
|
});
|
||||||
|
|
||||||
|
setTimeout(() => {
|
||||||
|
void checkForUpdates();
|
||||||
|
}, AUTO_CHECK_DELAY_MS);
|
||||||
|
setInterval(() => {
|
||||||
|
void checkForUpdates();
|
||||||
|
}, AUTO_CHECK_INTERVAL_MS).unref();
|
||||||
|
}
|
||||||
Generated
+85
-10
@@ -1,12 +1,12 @@
|
|||||||
{
|
{
|
||||||
"name": "openscreen",
|
"name": "openscreen",
|
||||||
"version": "1.4.2",
|
"version": "1.4.12",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "openscreen",
|
"name": "openscreen",
|
||||||
"version": "1.4.2",
|
"version": "1.4.12",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@fix-webm-duration/fix": "^1.0.1",
|
"@fix-webm-duration/fix": "^1.0.1",
|
||||||
"@pixi/filter-drop-shadow": "^5.2.0",
|
"@pixi/filter-drop-shadow": "^5.2.0",
|
||||||
@@ -29,6 +29,7 @@
|
|||||||
"class-variance-authority": "^0.7.1",
|
"class-variance-authority": "^0.7.1",
|
||||||
"clsx": "^2.1.1",
|
"clsx": "^2.1.1",
|
||||||
"dnd-timeline": "^2.4.0",
|
"dnd-timeline": "^2.4.0",
|
||||||
|
"electron-updater": "^6.8.3",
|
||||||
"emoji-picker-react": "^4.18.0",
|
"emoji-picker-react": "^4.18.0",
|
||||||
"fix-webm-duration": "^1.0.6",
|
"fix-webm-duration": "^1.0.6",
|
||||||
"gif.js": "^0.2.0",
|
"gif.js": "^0.2.0",
|
||||||
@@ -4625,7 +4626,6 @@
|
|||||||
"version": "2.0.1",
|
"version": "2.0.1",
|
||||||
"resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
|
"resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
|
||||||
"integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
|
"integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
|
||||||
"dev": true,
|
|
||||||
"license": "Python-2.0"
|
"license": "Python-2.0"
|
||||||
},
|
},
|
||||||
"node_modules/aria-hidden": {
|
"node_modules/aria-hidden": {
|
||||||
@@ -4959,7 +4959,6 @@
|
|||||||
"version": "9.5.1",
|
"version": "9.5.1",
|
||||||
"resolved": "https://registry.npmjs.org/builder-util-runtime/-/builder-util-runtime-9.5.1.tgz",
|
"resolved": "https://registry.npmjs.org/builder-util-runtime/-/builder-util-runtime-9.5.1.tgz",
|
||||||
"integrity": "sha512-qt41tMfgHTllhResqM5DcnHyDIWNgzHvuY2jDcYP9iaGpkWxTUzV6GQjDeLnlR1/DtdlcsWQbA7sByMpmJFTLQ==",
|
"integrity": "sha512-qt41tMfgHTllhResqM5DcnHyDIWNgzHvuY2jDcYP9iaGpkWxTUzV6GQjDeLnlR1/DtdlcsWQbA7sByMpmJFTLQ==",
|
||||||
"dev": true,
|
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"debug": "^4.3.4",
|
"debug": "^4.3.4",
|
||||||
@@ -5504,7 +5503,6 @@
|
|||||||
"version": "4.4.3",
|
"version": "4.4.3",
|
||||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
|
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
|
||||||
"integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
|
"integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
|
||||||
"dev": true,
|
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"ms": "^2.1.3"
|
"ms": "^2.1.3"
|
||||||
@@ -6015,6 +6013,69 @@
|
|||||||
"dev": true,
|
"dev": true,
|
||||||
"license": "ISC"
|
"license": "ISC"
|
||||||
},
|
},
|
||||||
|
"node_modules/electron-updater": {
|
||||||
|
"version": "6.8.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/electron-updater/-/electron-updater-6.8.3.tgz",
|
||||||
|
"integrity": "sha512-Z6sgw3jgbikWKXei1ENdqFOxBP0WlXg3TtKfz0rgw2vIZFJUyI4pD7ZN7jrkm7EoMK+tcm/qTnPUdqfZukBlBQ==",
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"builder-util-runtime": "9.5.1",
|
||||||
|
"fs-extra": "^10.1.0",
|
||||||
|
"js-yaml": "^4.1.0",
|
||||||
|
"lazy-val": "^1.0.5",
|
||||||
|
"lodash.escaperegexp": "^4.1.2",
|
||||||
|
"lodash.isequal": "^4.5.0",
|
||||||
|
"semver": "~7.7.3",
|
||||||
|
"tiny-typed-emitter": "^2.1.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/electron-updater/node_modules/fs-extra": {
|
||||||
|
"version": "10.1.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz",
|
||||||
|
"integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==",
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"graceful-fs": "^4.2.0",
|
||||||
|
"jsonfile": "^6.0.1",
|
||||||
|
"universalify": "^2.0.0"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=12"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/electron-updater/node_modules/jsonfile": {
|
||||||
|
"version": "6.2.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.1.tgz",
|
||||||
|
"integrity": "sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==",
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"universalify": "^2.0.0"
|
||||||
|
},
|
||||||
|
"optionalDependencies": {
|
||||||
|
"graceful-fs": "^4.1.6"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/electron-updater/node_modules/semver": {
|
||||||
|
"version": "7.7.4",
|
||||||
|
"resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz",
|
||||||
|
"integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==",
|
||||||
|
"license": "ISC",
|
||||||
|
"bin": {
|
||||||
|
"semver": "bin/semver.js"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=10"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/electron-updater/node_modules/universalify": {
|
||||||
|
"version": "2.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz",
|
||||||
|
"integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==",
|
||||||
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 10.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/electron-winstaller": {
|
"node_modules/electron-winstaller": {
|
||||||
"version": "5.4.0",
|
"version": "5.4.0",
|
||||||
"resolved": "https://registry.npmjs.org/electron-winstaller/-/electron-winstaller-5.4.0.tgz",
|
"resolved": "https://registry.npmjs.org/electron-winstaller/-/electron-winstaller-5.4.0.tgz",
|
||||||
@@ -6874,7 +6935,6 @@
|
|||||||
"version": "4.2.11",
|
"version": "4.2.11",
|
||||||
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
|
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
|
||||||
"integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==",
|
"integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==",
|
||||||
"dev": true,
|
|
||||||
"license": "ISC"
|
"license": "ISC"
|
||||||
},
|
},
|
||||||
"node_modules/gsap": {
|
"node_modules/gsap": {
|
||||||
@@ -7287,7 +7347,6 @@
|
|||||||
"version": "4.1.1",
|
"version": "4.1.1",
|
||||||
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz",
|
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz",
|
||||||
"integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==",
|
"integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==",
|
||||||
"dev": true,
|
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"argparse": "^2.0.1"
|
"argparse": "^2.0.1"
|
||||||
@@ -7419,7 +7478,6 @@
|
|||||||
"version": "1.0.5",
|
"version": "1.0.5",
|
||||||
"resolved": "https://registry.npmjs.org/lazy-val/-/lazy-val-1.0.5.tgz",
|
"resolved": "https://registry.npmjs.org/lazy-val/-/lazy-val-1.0.5.tgz",
|
||||||
"integrity": "sha512-0/BnGCCfyUMkBpeDgWihanIAF9JmZhHBgUhEqzvf+adhNGLoP6TaiI5oF8oyb3I45P+PcnrqihSf01M0l0G5+Q==",
|
"integrity": "sha512-0/BnGCCfyUMkBpeDgWihanIAF9JmZhHBgUhEqzvf+adhNGLoP6TaiI5oF8oyb3I45P+PcnrqihSf01M0l0G5+Q==",
|
||||||
"dev": true,
|
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/lilconfig": {
|
"node_modules/lilconfig": {
|
||||||
@@ -7586,6 +7644,19 @@
|
|||||||
"dev": true,
|
"dev": true,
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
|
"node_modules/lodash.escaperegexp": {
|
||||||
|
"version": "4.1.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz",
|
||||||
|
"integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==",
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
|
"node_modules/lodash.isequal": {
|
||||||
|
"version": "4.5.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz",
|
||||||
|
"integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==",
|
||||||
|
"deprecated": "This package is deprecated. Use require('node:util').isDeepStrictEqual instead.",
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
"node_modules/log-update": {
|
"node_modules/log-update": {
|
||||||
"version": "6.1.0",
|
"version": "6.1.0",
|
||||||
"resolved": "https://registry.npmjs.org/log-update/-/log-update-6.1.0.tgz",
|
"resolved": "https://registry.npmjs.org/log-update/-/log-update-6.1.0.tgz",
|
||||||
@@ -7991,7 +8062,6 @@
|
|||||||
"version": "2.1.3",
|
"version": "2.1.3",
|
||||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
|
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
|
||||||
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
|
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
|
||||||
"dev": true,
|
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/mz": {
|
"node_modules/mz": {
|
||||||
@@ -9394,7 +9464,6 @@
|
|||||||
"version": "1.6.0",
|
"version": "1.6.0",
|
||||||
"resolved": "https://registry.npmjs.org/sax/-/sax-1.6.0.tgz",
|
"resolved": "https://registry.npmjs.org/sax/-/sax-1.6.0.tgz",
|
||||||
"integrity": "sha512-6R3J5M4AcbtLUdZmRv2SygeVaM7IhrLXu9BmnOGmmACak8fiUtOsYNWUS4uK7upbmHIBbLBeFeI//477BKLBzA==",
|
"integrity": "sha512-6R3J5M4AcbtLUdZmRv2SygeVaM7IhrLXu9BmnOGmmACak8fiUtOsYNWUS4uK7upbmHIBbLBeFeI//477BKLBzA==",
|
||||||
"dev": true,
|
|
||||||
"license": "BlueOak-1.0.0",
|
"license": "BlueOak-1.0.0",
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">=11.0.0"
|
"node": ">=11.0.0"
|
||||||
@@ -10099,6 +10168,12 @@
|
|||||||
"node": ">=12"
|
"node": ">=12"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/tiny-typed-emitter": {
|
||||||
|
"version": "2.1.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/tiny-typed-emitter/-/tiny-typed-emitter-2.1.0.tgz",
|
||||||
|
"integrity": "sha512-qVtvMxeXbVej0cQWKqVSSAHmKZEHAvxdF8HEUBFWts8h+xEo5m/lEiPakuyZ3BnCBjOD8i24kzNOiOLLgsSxhA==",
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
"node_modules/tinybench": {
|
"node_modules/tinybench": {
|
||||||
"version": "2.9.0",
|
"version": "2.9.0",
|
||||||
"resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz",
|
"resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz",
|
||||||
|
|||||||
+2
-1
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"name": "openscreen",
|
"name": "openscreen",
|
||||||
"private": true,
|
"private": true,
|
||||||
"version": "1.4.2",
|
"version": "1.4.12",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"packageManager": "npm@10.9.4",
|
"packageManager": "npm@10.9.4",
|
||||||
"engines": {
|
"engines": {
|
||||||
@@ -70,6 +70,7 @@
|
|||||||
"class-variance-authority": "^0.7.1",
|
"class-variance-authority": "^0.7.1",
|
||||||
"clsx": "^2.1.1",
|
"clsx": "^2.1.1",
|
||||||
"dnd-timeline": "^2.4.0",
|
"dnd-timeline": "^2.4.0",
|
||||||
|
"electron-updater": "^6.8.3",
|
||||||
"emoji-picker-react": "^4.18.0",
|
"emoji-picker-react": "^4.18.0",
|
||||||
"fix-webm-duration": "^1.0.6",
|
"fix-webm-duration": "^1.0.6",
|
||||||
"gif.js": "^0.2.0",
|
"gif.js": "^0.2.0",
|
||||||
|
|||||||
@@ -131,6 +131,11 @@ if (!fs.existsSync(guideHotkeyListenerOutputPath)) {
|
|||||||
throw new Error(`WGC helper build completed but ${guideHotkeyListenerOutputPath} was not found.`);
|
throw new Error(`WGC helper build completed but ${guideHotkeyListenerOutputPath} was not found.`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const ocrServiceWrapperOutputPath = path.join(BUILD_DIR, "openscreen-ocr-service-wrapper.exe");
|
||||||
|
if (!fs.existsSync(ocrServiceWrapperOutputPath)) {
|
||||||
|
throw new Error(`WGC helper build completed but ${ocrServiceWrapperOutputPath} was not found.`);
|
||||||
|
}
|
||||||
|
|
||||||
fs.mkdirSync(BIN_DIR, { recursive: true });
|
fs.mkdirSync(BIN_DIR, { recursive: true });
|
||||||
const distributablePath = path.join(BIN_DIR, "wgc-capture.exe");
|
const distributablePath = path.join(BIN_DIR, "wgc-capture.exe");
|
||||||
fs.copyFileSync(outputPath, distributablePath);
|
fs.copyFileSync(outputPath, distributablePath);
|
||||||
@@ -141,9 +146,14 @@ fs.copyFileSync(cursorSamplerOutputPath, cursorSamplerDistributablePath);
|
|||||||
const guideHotkeyListenerDistributablePath = path.join(BIN_DIR, "guide-hotkey-listener.exe");
|
const guideHotkeyListenerDistributablePath = path.join(BIN_DIR, "guide-hotkey-listener.exe");
|
||||||
fs.copyFileSync(guideHotkeyListenerOutputPath, guideHotkeyListenerDistributablePath);
|
fs.copyFileSync(guideHotkeyListenerOutputPath, guideHotkeyListenerDistributablePath);
|
||||||
|
|
||||||
|
const ocrServiceWrapperDistributablePath = path.join(BIN_DIR, "openscreen-ocr-service-wrapper.exe");
|
||||||
|
fs.copyFileSync(ocrServiceWrapperOutputPath, ocrServiceWrapperDistributablePath);
|
||||||
|
|
||||||
console.log(`Built ${outputPath}`);
|
console.log(`Built ${outputPath}`);
|
||||||
console.log(`Copied ${distributablePath}`);
|
console.log(`Copied ${distributablePath}`);
|
||||||
console.log(`Built ${cursorSamplerOutputPath}`);
|
console.log(`Built ${cursorSamplerOutputPath}`);
|
||||||
console.log(`Copied ${cursorSamplerDistributablePath}`);
|
console.log(`Copied ${cursorSamplerDistributablePath}`);
|
||||||
console.log(`Built ${guideHotkeyListenerOutputPath}`);
|
console.log(`Built ${guideHotkeyListenerOutputPath}`);
|
||||||
console.log(`Copied ${guideHotkeyListenerDistributablePath}`);
|
console.log(`Copied ${guideHotkeyListenerDistributablePath}`);
|
||||||
|
console.log(`Built ${ocrServiceWrapperOutputPath}`);
|
||||||
|
console.log(`Copied ${ocrServiceWrapperDistributablePath}`);
|
||||||
|
|||||||
+70
-1
@@ -1,4 +1,5 @@
|
|||||||
import { lazy, Suspense, useEffect, useState } from "react";
|
import { lazy, Suspense, useEffect, useRef, useState } from "react";
|
||||||
|
import { toast } from "sonner";
|
||||||
import { CountdownOverlay } from "./components/launch/CountdownOverlay.tsx";
|
import { CountdownOverlay } from "./components/launch/CountdownOverlay.tsx";
|
||||||
import { LaunchWindow } from "./components/launch/LaunchWindow";
|
import { LaunchWindow } from "./components/launch/LaunchWindow";
|
||||||
import { SourceSelector } from "./components/launch/SourceSelector";
|
import { SourceSelector } from "./components/launch/SourceSelector";
|
||||||
@@ -6,6 +7,7 @@ import { Toaster } from "./components/ui/sonner";
|
|||||||
import { TooltipProvider } from "./components/ui/tooltip";
|
import { TooltipProvider } from "./components/ui/tooltip";
|
||||||
import { ShortcutsProvider } from "./contexts/ShortcutsContext";
|
import { ShortcutsProvider } from "./contexts/ShortcutsContext";
|
||||||
import { loadAllCustomFonts } from "./lib/customFonts";
|
import { loadAllCustomFonts } from "./lib/customFonts";
|
||||||
|
import type { UpdateStatus } from "./lib/updateStatus";
|
||||||
|
|
||||||
const VideoEditor = lazy(() => import("./components/video-editor/VideoEditor"));
|
const VideoEditor = lazy(() => import("./components/video-editor/VideoEditor"));
|
||||||
const ShortcutsConfigDialog = lazy(() =>
|
const ShortcutsConfigDialog = lazy(() =>
|
||||||
@@ -79,11 +81,78 @@ export default function App() {
|
|||||||
return (
|
return (
|
||||||
<TooltipProvider>
|
<TooltipProvider>
|
||||||
{content}
|
{content}
|
||||||
|
<UpdateNotifier
|
||||||
|
enabled={
|
||||||
|
hasElectronBridge &&
|
||||||
|
windowType !== "hud-overlay" &&
|
||||||
|
windowType !== "source-selector" &&
|
||||||
|
windowType !== "countdown-overlay"
|
||||||
|
}
|
||||||
|
/>
|
||||||
<Toaster theme="dark" className="pointer-events-auto" />
|
<Toaster theme="dark" className="pointer-events-auto" />
|
||||||
</TooltipProvider>
|
</TooltipProvider>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function UpdateNotifier({ enabled }: { enabled: boolean }) {
|
||||||
|
const lastPhaseRef = useRef<UpdateStatus["phase"]>("idle");
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!enabled || !window.electronAPI?.updates) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const applyStatus = (status: UpdateStatus) => {
|
||||||
|
const version = status.version ? ` ${status.version}` : "";
|
||||||
|
if (status.phase === "available") {
|
||||||
|
toast.loading(`Downloading OpenScreen${version} update...`, {
|
||||||
|
id: "openscreen-update",
|
||||||
|
duration: Number.POSITIVE_INFINITY,
|
||||||
|
});
|
||||||
|
} else if (status.phase === "downloading") {
|
||||||
|
const percent = typeof status.percent === "number" ? ` ${Math.round(status.percent)}%` : "";
|
||||||
|
toast.loading(`Downloading OpenScreen${version} update${percent}...`, {
|
||||||
|
id: "openscreen-update",
|
||||||
|
duration: Number.POSITIVE_INFINITY,
|
||||||
|
});
|
||||||
|
} else if (status.phase === "downloaded") {
|
||||||
|
toast.success(`OpenScreen${version} is ready to install.`, {
|
||||||
|
id: "openscreen-update",
|
||||||
|
duration: Number.POSITIVE_INFINITY,
|
||||||
|
action: {
|
||||||
|
label: "Restart",
|
||||||
|
onClick: () => {
|
||||||
|
void window.electronAPI.updates.install();
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} else if (
|
||||||
|
status.phase === "error" &&
|
||||||
|
(lastPhaseRef.current === "available" ||
|
||||||
|
lastPhaseRef.current === "downloading" ||
|
||||||
|
lastPhaseRef.current === "downloaded")
|
||||||
|
) {
|
||||||
|
toast.error(status.error || "OpenScreen update failed.", {
|
||||||
|
id: "openscreen-update",
|
||||||
|
});
|
||||||
|
} else if (status.phase === "not-available" || status.phase === "unsupported") {
|
||||||
|
toast.dismiss("openscreen-update");
|
||||||
|
}
|
||||||
|
lastPhaseRef.current = status.phase;
|
||||||
|
};
|
||||||
|
|
||||||
|
const unsubscribe = window.electronAPI.updates.onStatus(applyStatus);
|
||||||
|
void window.electronAPI.updates
|
||||||
|
.getStatus()
|
||||||
|
.then(applyStatus)
|
||||||
|
.catch(() => undefined);
|
||||||
|
|
||||||
|
return unsubscribe;
|
||||||
|
}, [enabled]);
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
function BrowserDevFallback() {
|
function BrowserDevFallback() {
|
||||||
return (
|
return (
|
||||||
<div className="flex h-screen w-screen items-center justify-center bg-[#08090b] px-6 text-slate-100">
|
<div className="flex h-screen w-screen items-center justify-center bg-[#08090b] px-6 text-slate-100">
|
||||||
|
|||||||
@@ -65,7 +65,13 @@ export function SourceSelector() {
|
|||||||
fetchSources();
|
fetchSources();
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const screenSources = sources.filter((s) => s.id.startsWith("screen:"));
|
const screenSources = sources
|
||||||
|
.filter((s) => s.id.startsWith("screen:"))
|
||||||
|
.sort(
|
||||||
|
(left, right) =>
|
||||||
|
(left.displayIndex ?? left.screenIndex ?? Number.MAX_SAFE_INTEGER) -
|
||||||
|
(right.displayIndex ?? right.screenIndex ?? Number.MAX_SAFE_INTEGER),
|
||||||
|
);
|
||||||
const windowSources = sources.filter((s) => s.id.startsWith("window:"));
|
const windowSources = sources.filter((s) => s.id.startsWith("window:"));
|
||||||
|
|
||||||
const handleSourceSelect = (source: DesktopSource) => setSelectedSource(source);
|
const handleSourceSelect = (source: DesktopSource) => setSelectedSource(source);
|
||||||
@@ -96,11 +102,17 @@ export function SourceSelector() {
|
|||||||
onClick={() => handleSourceSelect(source)}
|
onClick={() => handleSourceSelect(source)}
|
||||||
>
|
>
|
||||||
<div className="relative mb-1.5 overflow-hidden rounded-lg border border-white/[0.06] bg-black/30">
|
<div className="relative mb-1.5 overflow-hidden rounded-lg border border-white/[0.06] bg-black/30">
|
||||||
<img
|
{source.thumbnail ? (
|
||||||
src={source.thumbnail || ""}
|
<img
|
||||||
alt={source.name}
|
src={source.thumbnail}
|
||||||
className="w-full aspect-video object-cover"
|
alt={source.name}
|
||||||
/>
|
className="w-full aspect-video object-cover"
|
||||||
|
/>
|
||||||
|
) : (
|
||||||
|
<div className="flex aspect-video w-full items-center justify-center bg-zinc-950 text-center text-[11px] font-medium text-zinc-400">
|
||||||
|
{source.displayLabel ?? source.name}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
{isSelected && (
|
{isSelected && (
|
||||||
<div className="absolute right-1.5 top-1.5">
|
<div className="absolute right-1.5 top-1.5">
|
||||||
<div className={styles.checkBadge}>
|
<div className={styles.checkBadge}>
|
||||||
|
|||||||
@@ -82,6 +82,7 @@ export function AnnotationOverlay({
|
|||||||
);
|
);
|
||||||
const [livePointerPoint, setLivePointerPoint] = useState<{ x: number; y: number } | null>(null);
|
const [livePointerPoint, setLivePointerPoint] = useState<{ x: number; y: number } | null>(null);
|
||||||
const mosaicCanvasRef = useRef<HTMLCanvasElement | null>(null);
|
const mosaicCanvasRef = useRef<HTMLCanvasElement | null>(null);
|
||||||
|
const magnifierCanvasRef = useRef<HTMLCanvasElement | null>(null);
|
||||||
const blurType = "mosaic";
|
const blurType = "mosaic";
|
||||||
const blurOverlayColor =
|
const blurOverlayColor =
|
||||||
annotation.type === "blur" ? getBlurOverlayColor(annotation.blurData) : "";
|
annotation.type === "blur" ? getBlurOverlayColor(annotation.blurData) : "";
|
||||||
@@ -183,6 +184,79 @@ export function AnnotationOverlay({
|
|||||||
y,
|
y,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (annotation.type !== "magnifier") {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
void previewFrameVersion;
|
||||||
|
|
||||||
|
const canvas = magnifierCanvasRef.current;
|
||||||
|
const sourceCanvas = previewSourceCanvas;
|
||||||
|
if (!canvas || !sourceCanvas) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const sourceWidth = sourceCanvas.width;
|
||||||
|
const sourceHeight = sourceCanvas.height;
|
||||||
|
const sourceClientWidth = sourceCanvas.clientWidth || containerWidth || sourceWidth;
|
||||||
|
const sourceClientHeight = sourceCanvas.clientHeight || containerHeight || sourceHeight;
|
||||||
|
if (
|
||||||
|
sourceWidth <= 0 ||
|
||||||
|
sourceHeight <= 0 ||
|
||||||
|
sourceClientWidth <= 0 ||
|
||||||
|
sourceClientHeight <= 0
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const drawWidth = Math.max(1, Math.round(width));
|
||||||
|
const drawHeight = Math.max(1, Math.round(height));
|
||||||
|
canvas.width = drawWidth;
|
||||||
|
canvas.height = drawHeight;
|
||||||
|
|
||||||
|
const context = canvas.getContext("2d");
|
||||||
|
if (!context) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const zoom = Math.max(1, annotation.magnifierData?.zoom ?? 2.2);
|
||||||
|
const target = annotation.magnifierData?.target ?? {
|
||||||
|
x: annotation.position.x + annotation.size.width / 2,
|
||||||
|
y: annotation.position.y + annotation.size.height / 2,
|
||||||
|
};
|
||||||
|
const scaleX = sourceWidth / sourceClientWidth;
|
||||||
|
const scaleY = sourceHeight / sourceClientHeight;
|
||||||
|
const targetX = (target.x / 100) * sourceClientWidth * scaleX;
|
||||||
|
const targetY = (target.y / 100) * sourceClientHeight * scaleY;
|
||||||
|
const sampleWidth = Math.max(1, drawWidth / zoom);
|
||||||
|
const sampleHeight = Math.max(1, drawHeight / zoom);
|
||||||
|
const sx = Math.max(0, Math.min(sourceWidth - sampleWidth, targetX - sampleWidth / 2));
|
||||||
|
const sy = Math.max(0, Math.min(sourceHeight - sampleHeight, targetY - sampleHeight / 2));
|
||||||
|
|
||||||
|
context.clearRect(0, 0, drawWidth, drawHeight);
|
||||||
|
context.imageSmoothingEnabled = true;
|
||||||
|
context.imageSmoothingQuality = "high";
|
||||||
|
context.drawImage(
|
||||||
|
sourceCanvas as CanvasImageSource,
|
||||||
|
sx,
|
||||||
|
sy,
|
||||||
|
Math.min(sampleWidth, sourceWidth - sx),
|
||||||
|
Math.min(sampleHeight, sourceHeight - sy),
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
drawWidth,
|
||||||
|
drawHeight,
|
||||||
|
);
|
||||||
|
}, [
|
||||||
|
annotation,
|
||||||
|
containerHeight,
|
||||||
|
containerWidth,
|
||||||
|
height,
|
||||||
|
previewFrameVersion,
|
||||||
|
previewSourceCanvas,
|
||||||
|
width,
|
||||||
|
]);
|
||||||
|
|
||||||
const renderArrow = () => {
|
const renderArrow = () => {
|
||||||
const direction = annotation.figureData?.arrowDirection || "right";
|
const direction = annotation.figureData?.arrowDirection || "right";
|
||||||
const color = annotation.figureData?.color || "#34B27B";
|
const color = annotation.figureData?.color || "#34B27B";
|
||||||
@@ -351,6 +425,30 @@ export function AnnotationOverlay({
|
|||||||
<div className="w-full h-full flex items-center justify-center p-2">{renderArrow()}</div>
|
<div className="w-full h-full flex items-center justify-center p-2">{renderArrow()}</div>
|
||||||
);
|
);
|
||||||
|
|
||||||
|
case "magnifier": {
|
||||||
|
const shape = annotation.magnifierData?.shape ?? "circle";
|
||||||
|
const caption = annotation.magnifierData?.caption;
|
||||||
|
return (
|
||||||
|
<div className="relative h-full w-full">
|
||||||
|
<canvas
|
||||||
|
ref={magnifierCanvasRef}
|
||||||
|
className="absolute inset-0 h-full w-full"
|
||||||
|
style={{
|
||||||
|
borderRadius: shape === "circle" ? "9999px" : "10px",
|
||||||
|
border: "3px solid rgba(248,250,252,0.96)",
|
||||||
|
boxShadow: "0 14px 36px rgba(0,0,0,0.38), 0 0 0 1px rgba(52,178,123,0.55)",
|
||||||
|
backgroundColor: "rgba(15, 23, 42, 0.9)",
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
{caption && (
|
||||||
|
<div className="absolute left-1/2 top-full mt-1 max-w-[220px] -translate-x-1/2 rounded-md bg-slate-950/90 px-2 py-1 text-center text-[11px] font-semibold leading-4 text-slate-100 shadow-lg">
|
||||||
|
{caption}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
case "blur": {
|
case "blur": {
|
||||||
const shape = annotation.blurData?.shape ?? "rectangle";
|
const shape = annotation.blurData?.shape ?? "rectangle";
|
||||||
const blurIntensity = Math.max(
|
const blurIntensity = Math.max(
|
||||||
@@ -623,6 +721,7 @@ export function AnnotationOverlay({
|
|||||||
annotation.type === "text" && "bg-transparent",
|
annotation.type === "text" && "bg-transparent",
|
||||||
annotation.type === "image" && "bg-transparent",
|
annotation.type === "image" && "bg-transparent",
|
||||||
annotation.type === "figure" && "bg-transparent",
|
annotation.type === "figure" && "bg-transparent",
|
||||||
|
annotation.type === "magnifier" && "bg-transparent",
|
||||||
annotation.type === "blur" && "bg-transparent",
|
annotation.type === "blur" && "bg-transparent",
|
||||||
isSelected && annotation.type !== "blur" && "shadow-lg",
|
isSelected && annotation.type !== "blur" && "shadow-lg",
|
||||||
)}
|
)}
|
||||||
|
|||||||
@@ -13,6 +13,8 @@ import {
|
|||||||
} from "@/components/ui/dialog";
|
} from "@/components/ui/dialog";
|
||||||
import { useI18n, useScopedT } from "@/contexts/I18nContext";
|
import { useI18n, useScopedT } from "@/contexts/I18nContext";
|
||||||
import { useShortcuts } from "@/contexts/ShortcutsContext";
|
import { useShortcuts } from "@/contexts/ShortcutsContext";
|
||||||
|
import type { GuideSession } from "@/guide/contracts";
|
||||||
|
import { buildGuideVideoAnnotations, buildGuideVideoSpeedRegions } from "@/guide/videoAnnotations";
|
||||||
import { INITIAL_EDITOR_STATE, useEditorHistory } from "@/hooks/useEditorHistory";
|
import { INITIAL_EDITOR_STATE, useEditorHistory } from "@/hooks/useEditorHistory";
|
||||||
import { type Locale } from "@/i18n/config";
|
import { type Locale } from "@/i18n/config";
|
||||||
import { getAvailableLocales, getLocaleName } from "@/i18n/loader";
|
import { getAvailableLocales, getLocaleName } from "@/i18n/loader";
|
||||||
@@ -1374,6 +1376,34 @@ export default function VideoEditor() {
|
|||||||
[pushState],
|
[pushState],
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const handleGuideAttachToVideo = useCallback(
|
||||||
|
(session: GuideSession) => {
|
||||||
|
const guideAnnotations = buildGuideVideoAnnotations(session, {
|
||||||
|
nextId: () => `annotation-${nextAnnotationIdRef.current++}`,
|
||||||
|
nextZIndex: () => nextAnnotationZIndexRef.current++,
|
||||||
|
});
|
||||||
|
const guideSpeedRegions = buildGuideVideoSpeedRegions(session, {
|
||||||
|
nextId: () => `speed-${nextSpeedIdRef.current++}`,
|
||||||
|
});
|
||||||
|
if (guideAnnotations.length === 0 && guideSpeedRegions.length === 0) {
|
||||||
|
toast.error("Generate a guide draft before attaching steps to the video.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
pushState((prev) => ({
|
||||||
|
annotationRegions: [...prev.annotationRegions, ...guideAnnotations],
|
||||||
|
speedRegions: [...prev.speedRegions, ...guideSpeedRegions],
|
||||||
|
}));
|
||||||
|
const firstTextAnnotation = guideAnnotations.find((annotation) => annotation.type === "text");
|
||||||
|
setSelectedAnnotationId(firstTextAnnotation?.id ?? guideAnnotations[0]?.id ?? null);
|
||||||
|
setSelectedBlurId(null);
|
||||||
|
setSelectedZoomId(null);
|
||||||
|
setSelectedTrimId(null);
|
||||||
|
setSelectedSpeedId(null);
|
||||||
|
},
|
||||||
|
[pushState],
|
||||||
|
);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const handleKeyDown = (e: KeyboardEvent) => {
|
const handleKeyDown = (e: KeyboardEvent) => {
|
||||||
const mod = e.ctrlKey || e.metaKey;
|
const mod = e.ctrlKey || e.metaKey;
|
||||||
@@ -2162,6 +2192,7 @@ export default function VideoEditor() {
|
|||||||
videoPath={videoPath}
|
videoPath={videoPath}
|
||||||
videoSourcePath={videoSourcePath}
|
videoSourcePath={videoSourcePath}
|
||||||
currentTimeMs={currentTime * 1000}
|
currentTimeMs={currentTime * 1000}
|
||||||
|
onAttachToVideo={handleGuideAttachToVideo}
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
<div className="min-h-0 flex-1 overflow-hidden">
|
<div className="min-h-0 flex-1 overflow-hidden">
|
||||||
|
|||||||
@@ -1963,18 +1963,20 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
|||||||
region: blurRegion,
|
region: blurRegion,
|
||||||
})),
|
})),
|
||||||
].sort((a, b) => a.region.zIndex - b.region.zIndex);
|
].sort((a, b) => a.region.zIndex - b.region.zIndex);
|
||||||
const previewSnapshotCanvas =
|
const needsPreviewSnapshot =
|
||||||
filteredBlurRegions.length > 0
|
filteredBlurRegions.length > 0 ||
|
||||||
? (() => {
|
filteredAnnotations.some((annotation) => annotation.type === "magnifier");
|
||||||
const app = appRef.current;
|
const previewSnapshotCanvas = needsPreviewSnapshot
|
||||||
if (!app?.renderer?.extract) return null;
|
? (() => {
|
||||||
try {
|
const app = appRef.current;
|
||||||
return app.renderer.extract.canvas(app.stage);
|
if (!app?.renderer?.extract) return null;
|
||||||
} catch {
|
try {
|
||||||
return null;
|
return app.renderer.extract.canvas(app.stage);
|
||||||
}
|
} catch {
|
||||||
})()
|
return null;
|
||||||
: null;
|
}
|
||||||
|
})()
|
||||||
|
: null;
|
||||||
|
|
||||||
// Handle click-through cycling: when clicking same annotation, cycle to next
|
// Handle click-through cycling: when clicking same annotation, cycle to next
|
||||||
const handleAnnotationClick = (clickedId: string) => {
|
const handleAnnotationClick = (clickedId: string) => {
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { KeyRound, ListChecks, RefreshCw, Save, Trash2, Wand2 } from "lucide-react";
|
import { KeyRound, ListChecks, RefreshCw, Save, Trash2, Video, Wand2 } from "lucide-react";
|
||||||
import { useCallback, useEffect, useMemo, useState } from "react";
|
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||||
import { toast } from "sonner";
|
import { toast } from "sonner";
|
||||||
import { Button } from "@/components/ui/button";
|
import { Button } from "@/components/ui/button";
|
||||||
@@ -7,7 +7,9 @@ import type {
|
|||||||
GuideAiProvider,
|
GuideAiProvider,
|
||||||
GuideAiSettings,
|
GuideAiSettings,
|
||||||
GuideLanguage,
|
GuideLanguage,
|
||||||
|
GuideOcrProfile,
|
||||||
GuideSession,
|
GuideSession,
|
||||||
|
GuideSnapshot,
|
||||||
} from "@/guide/contracts";
|
} from "@/guide/contracts";
|
||||||
import { captureGuideSnapshots } from "@/guide/snapshot/extractGuideSnapshots";
|
import { captureGuideSnapshots } from "@/guide/snapshot/extractGuideSnapshots";
|
||||||
|
|
||||||
@@ -16,9 +18,17 @@ interface GuidePanelProps {
|
|||||||
videoPath: string | null;
|
videoPath: string | null;
|
||||||
videoSourcePath: string | null;
|
videoSourcePath: string | null;
|
||||||
currentTimeMs: number;
|
currentTimeMs: number;
|
||||||
|
onAttachToVideo?: (session: GuideSession) => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
type BusyAction = "load" | "generate";
|
type BusyAction = "load" | "generate" | "attach";
|
||||||
|
|
||||||
|
interface GuideProgressState {
|
||||||
|
label: string;
|
||||||
|
current: number;
|
||||||
|
total: number;
|
||||||
|
detail?: string;
|
||||||
|
}
|
||||||
|
|
||||||
const COPY = {
|
const COPY = {
|
||||||
en: {
|
en: {
|
||||||
@@ -42,13 +52,19 @@ const COPY = {
|
|||||||
captureStep: "Capture step",
|
captureStep: "Capture step",
|
||||||
captureLabel: "Manual capture",
|
captureLabel: "Manual capture",
|
||||||
settings: "Settings",
|
settings: "Settings",
|
||||||
|
guideSettings: "Guide settings",
|
||||||
apiKey: "API key env",
|
apiKey: "API key env",
|
||||||
apiKeyPlaceholder: "DEEPSEEK_API_KEY",
|
apiKeyPlaceholder: "DEEPSEEK_API_KEY",
|
||||||
baseUrl: "Base URL",
|
baseUrl: "Base URL",
|
||||||
model: "Model",
|
model: "Model",
|
||||||
|
ocrProfile: "OCR profile",
|
||||||
|
ocrLanguage: "OCR languages",
|
||||||
|
ocrFast: "Fast Latin",
|
||||||
|
ocrVietnamese: "Vietnamese Enhanced",
|
||||||
|
ocrHybrid: "Hybrid Vi + Latin",
|
||||||
saveSettings: "Save",
|
saveSettings: "Save",
|
||||||
clearKey: "Reset env",
|
clearKey: "Reset env",
|
||||||
keySaved: "DeepSeek settings saved.",
|
settingsSaved: "Guide settings saved.",
|
||||||
keyMissing: "Set a DeepSeek API key environment variable before generating with DeepSeek.",
|
keyMissing: "Set a DeepSeek API key environment variable before generating with DeepSeek.",
|
||||||
keyConfigured: "Env ready",
|
keyConfigured: "Env ready",
|
||||||
keyNotConfigured: "Env value missing",
|
keyNotConfigured: "Env value missing",
|
||||||
@@ -56,6 +72,14 @@ const COPY = {
|
|||||||
noEvents: "No click events were captured for this guide.",
|
noEvents: "No click events were captured for this guide.",
|
||||||
ocrUnavailable: "Local OCR service is unavailable. You can still create a local draft.",
|
ocrUnavailable: "Local OCR service is unavailable. You can still create a local draft.",
|
||||||
exported: "Guide exported",
|
exported: "Guide exported",
|
||||||
|
attachToVideo: "Attach to video",
|
||||||
|
attachedToVideo: "Guide steps attached to the video timeline.",
|
||||||
|
noDraft: "Generate a guide draft before attaching steps to the video.",
|
||||||
|
progressPreparing: "Preparing events",
|
||||||
|
progressSnapshots: "Capturing snapshots",
|
||||||
|
progressOcr: "Running OCR",
|
||||||
|
progressDraft: "Writing draft",
|
||||||
|
progressExport: "Exporting files",
|
||||||
},
|
},
|
||||||
vi: {
|
vi: {
|
||||||
title: "Hướng dẫn",
|
title: "Hướng dẫn",
|
||||||
@@ -78,13 +102,19 @@ const COPY = {
|
|||||||
captureStep: "Chụp bước",
|
captureStep: "Chụp bước",
|
||||||
captureLabel: "Chụp thủ công",
|
captureLabel: "Chụp thủ công",
|
||||||
settings: "Cài đặt",
|
settings: "Cài đặt",
|
||||||
|
guideSettings: "Guide settings",
|
||||||
apiKey: "API key env",
|
apiKey: "API key env",
|
||||||
apiKeyPlaceholder: "DEEPSEEK_API_KEY",
|
apiKeyPlaceholder: "DEEPSEEK_API_KEY",
|
||||||
baseUrl: "Base URL",
|
baseUrl: "Base URL",
|
||||||
model: "Model",
|
model: "Model",
|
||||||
|
ocrProfile: "OCR profile",
|
||||||
|
ocrLanguage: "OCR languages",
|
||||||
|
ocrFast: "Fast Latin",
|
||||||
|
ocrVietnamese: "Vietnamese Enhanced",
|
||||||
|
ocrHybrid: "Hybrid Vi + Latin",
|
||||||
saveSettings: "Lưu",
|
saveSettings: "Lưu",
|
||||||
clearKey: "Reset env",
|
clearKey: "Reset env",
|
||||||
keySaved: "Đã lưu cài đặt DeepSeek.",
|
settingsSaved: "Da luu cai dat guide.",
|
||||||
keyMissing: "Hãy set biến môi trường DeepSeek API key trước khi tạo draft bằng DeepSeek.",
|
keyMissing: "Hãy set biến môi trường DeepSeek API key trước khi tạo draft bằng DeepSeek.",
|
||||||
keyConfigured: "Env ready",
|
keyConfigured: "Env ready",
|
||||||
keyNotConfigured: "Chưa thấy giá trị env",
|
keyNotConfigured: "Chưa thấy giá trị env",
|
||||||
@@ -92,10 +122,41 @@ const COPY = {
|
|||||||
noEvents: "Chưa ghi nhận click event nào cho guide này.",
|
noEvents: "Chưa ghi nhận click event nào cho guide này.",
|
||||||
ocrUnavailable: "OCR local chưa chạy. Vẫn có thể tạo draft local.",
|
ocrUnavailable: "OCR local chưa chạy. Vẫn có thể tạo draft local.",
|
||||||
exported: "Đã export hướng dẫn",
|
exported: "Đã export hướng dẫn",
|
||||||
|
attachToVideo: "Gắn vào video",
|
||||||
|
attachedToVideo: "Đã gắn các bước guide vào timeline video.",
|
||||||
|
noDraft: "Hãy tạo draft guide trước khi gắn vào video.",
|
||||||
|
progressPreparing: "Đang chuẩn bị events",
|
||||||
|
progressSnapshots: "Đang chụp ảnh",
|
||||||
|
progressOcr: "Đang OCR",
|
||||||
|
progressDraft: "Đang tạo draft",
|
||||||
|
progressExport: "Đang export file",
|
||||||
},
|
},
|
||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePanelProps) {
|
function getPendingOcrSnapshots(session: GuideSession): GuideSnapshot[] {
|
||||||
|
const ocrCompletedSnapshotIds = new Set(session.ocrBlocks.map((block) => block.snapshotId));
|
||||||
|
return session.snapshots.filter(
|
||||||
|
(snapshot) => !snapshot.ocrCompletedAt && !ocrCompletedSnapshotIds.has(snapshot.id),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function getProgressPercent(progress: GuideProgressState | null): number {
|
||||||
|
if (!progress) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
if (progress.total <= 0) {
|
||||||
|
return 100;
|
||||||
|
}
|
||||||
|
const percent = Math.round((progress.current / progress.total) * 100);
|
||||||
|
return Math.min(100, Math.max(progress.current > 0 ? 8 : 4, percent));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function GuidePanel({
|
||||||
|
recordingId,
|
||||||
|
videoPath,
|
||||||
|
videoSourcePath,
|
||||||
|
onAttachToVideo,
|
||||||
|
}: GuidePanelProps) {
|
||||||
const { locale } = useI18n();
|
const { locale } = useI18n();
|
||||||
const copy = useMemo(() => (locale.startsWith("vi") ? COPY.vi : COPY.en), [locale]);
|
const copy = useMemo(() => (locale.startsWith("vi") ? COPY.vi : COPY.en), [locale]);
|
||||||
const guideLanguage: GuideLanguage = locale.startsWith("vi") ? "vi" : "en";
|
const guideLanguage: GuideLanguage = locale.startsWith("vi") ? "vi" : "en";
|
||||||
@@ -108,9 +169,13 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
|||||||
const [deepSeekApiKeyEnvName, setDeepSeekApiKeyEnvName] = useState("DEEPSEEK_API_KEY");
|
const [deepSeekApiKeyEnvName, setDeepSeekApiKeyEnvName] = useState("DEEPSEEK_API_KEY");
|
||||||
const [deepSeekBaseUrl, setDeepSeekBaseUrl] = useState("https://api.deepseek.com");
|
const [deepSeekBaseUrl, setDeepSeekBaseUrl] = useState("https://api.deepseek.com");
|
||||||
const [deepSeekModel, setDeepSeekModel] = useState("deepseek-chat");
|
const [deepSeekModel, setDeepSeekModel] = useState("deepseek-chat");
|
||||||
|
const [ocrProfile, setOcrProfile] = useState<GuideOcrProfile>("vietnamese");
|
||||||
|
const [ocrLanguage, setOcrLanguage] = useState("vi,en");
|
||||||
const [message, setMessage] = useState<string | null>(null);
|
const [message, setMessage] = useState<string | null>(null);
|
||||||
|
const [progress, setProgress] = useState<GuideProgressState | null>(null);
|
||||||
|
|
||||||
const isBusy = busyAction !== null;
|
const isBusy = busyAction !== null;
|
||||||
|
const progressPercent = getProgressPercent(progress);
|
||||||
const canUseGuide = Boolean(recordingId && videoSourcePath && window.electronAPI?.guide);
|
const canUseGuide = Boolean(recordingId && videoSourcePath && window.electronAPI?.guide);
|
||||||
const generatedSteps = session?.generatedGuide?.steps ?? [];
|
const generatedSteps = session?.generatedGuide?.steps ?? [];
|
||||||
const statusLabel = useMemo(() => {
|
const statusLabel = useMemo(() => {
|
||||||
@@ -138,6 +203,8 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
|||||||
setDeepSeekBaseUrl(result.data.deepseek.baseUrl);
|
setDeepSeekBaseUrl(result.data.deepseek.baseUrl);
|
||||||
setDeepSeekModel(result.data.deepseek.model);
|
setDeepSeekModel(result.data.deepseek.model);
|
||||||
setDeepSeekApiKeyEnvName(result.data.deepseek.apiKeyEnvName);
|
setDeepSeekApiKeyEnvName(result.data.deepseek.apiKeyEnvName);
|
||||||
|
setOcrProfile(result.data.ocr.profile);
|
||||||
|
setOcrLanguage(result.data.ocr.language);
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
@@ -203,6 +270,15 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
|||||||
}
|
}
|
||||||
|
|
||||||
let current = session;
|
let current = session;
|
||||||
|
const readResult = await window.electronAPI.guide.readSession(recordingId);
|
||||||
|
if (readResult.success) {
|
||||||
|
current = readResult.data;
|
||||||
|
} else if (readResult.code === "guide-session-not-found") {
|
||||||
|
current = null;
|
||||||
|
} else if (!current) {
|
||||||
|
throw new Error(readResult.error);
|
||||||
|
}
|
||||||
|
|
||||||
if (!current) {
|
if (!current) {
|
||||||
const startResult = await window.electronAPI.guide.startSession(recordingId);
|
const startResult = await window.electronAPI.guide.startSession(recordingId);
|
||||||
if (!startResult.success) {
|
if (!startResult.success) {
|
||||||
@@ -234,6 +310,7 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
|||||||
}
|
}
|
||||||
setBusyAction(action);
|
setBusyAction(action);
|
||||||
setMessage(null);
|
setMessage(null);
|
||||||
|
setProgress(null);
|
||||||
try {
|
try {
|
||||||
await task();
|
await task();
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -269,6 +346,8 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
|||||||
deepseekApiKeyEnvName: deepSeekApiKeyEnvName,
|
deepseekApiKeyEnvName: deepSeekApiKeyEnvName,
|
||||||
baseUrl: deepSeekBaseUrl,
|
baseUrl: deepSeekBaseUrl,
|
||||||
model: deepSeekModel,
|
model: deepSeekModel,
|
||||||
|
ocrProfile,
|
||||||
|
ocrLanguage,
|
||||||
});
|
});
|
||||||
if (!result.success) {
|
if (!result.success) {
|
||||||
throw new Error(result.error);
|
throw new Error(result.error);
|
||||||
@@ -277,7 +356,9 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
|||||||
setDeepSeekApiKeyEnvName(result.data.deepseek.apiKeyEnvName);
|
setDeepSeekApiKeyEnvName(result.data.deepseek.apiKeyEnvName);
|
||||||
setDeepSeekBaseUrl(result.data.deepseek.baseUrl);
|
setDeepSeekBaseUrl(result.data.deepseek.baseUrl);
|
||||||
setDeepSeekModel(result.data.deepseek.model);
|
setDeepSeekModel(result.data.deepseek.model);
|
||||||
toast.success(copy.keySaved);
|
setOcrProfile(result.data.ocr.profile);
|
||||||
|
setOcrLanguage(result.data.ocr.language);
|
||||||
|
toast.success(copy.settingsSaved);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const text = error instanceof Error ? error.message : String(error);
|
const text = error instanceof Error ? error.message : String(error);
|
||||||
setMessage(text);
|
setMessage(text);
|
||||||
@@ -285,7 +366,14 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
|||||||
} finally {
|
} finally {
|
||||||
setSettingsBusy(false);
|
setSettingsBusy(false);
|
||||||
}
|
}
|
||||||
}, [copy.keySaved, deepSeekApiKeyEnvName, deepSeekBaseUrl, deepSeekModel]);
|
}, [
|
||||||
|
copy.settingsSaved,
|
||||||
|
deepSeekApiKeyEnvName,
|
||||||
|
deepSeekBaseUrl,
|
||||||
|
deepSeekModel,
|
||||||
|
ocrLanguage,
|
||||||
|
ocrProfile,
|
||||||
|
]);
|
||||||
|
|
||||||
const handleClearDeepSeekKey = useCallback(async () => {
|
const handleClearDeepSeekKey = useCallback(async () => {
|
||||||
if (!window.electronAPI?.guide?.saveAiSettings) {
|
if (!window.electronAPI?.guide?.saveAiSettings) {
|
||||||
@@ -298,13 +386,17 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
|||||||
clearDeepseekApiKeyEnvName: true,
|
clearDeepseekApiKeyEnvName: true,
|
||||||
baseUrl: deepSeekBaseUrl,
|
baseUrl: deepSeekBaseUrl,
|
||||||
model: deepSeekModel,
|
model: deepSeekModel,
|
||||||
|
ocrProfile,
|
||||||
|
ocrLanguage,
|
||||||
});
|
});
|
||||||
if (!result.success) {
|
if (!result.success) {
|
||||||
throw new Error(result.error);
|
throw new Error(result.error);
|
||||||
}
|
}
|
||||||
setAiSettings(result.data);
|
setAiSettings(result.data);
|
||||||
setDeepSeekApiKeyEnvName(result.data.deepseek.apiKeyEnvName);
|
setDeepSeekApiKeyEnvName(result.data.deepseek.apiKeyEnvName);
|
||||||
toast.success(copy.keySaved);
|
setOcrProfile(result.data.ocr.profile);
|
||||||
|
setOcrLanguage(result.data.ocr.language);
|
||||||
|
toast.success(copy.settingsSaved);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const text = error instanceof Error ? error.message : String(error);
|
const text = error instanceof Error ? error.message : String(error);
|
||||||
setMessage(text);
|
setMessage(text);
|
||||||
@@ -312,7 +404,7 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
|||||||
} finally {
|
} finally {
|
||||||
setSettingsBusy(false);
|
setSettingsBusy(false);
|
||||||
}
|
}
|
||||||
}, [copy.keySaved, deepSeekBaseUrl, deepSeekModel]);
|
}, [copy.settingsSaved, deepSeekBaseUrl, deepSeekModel, ocrLanguage, ocrProfile]);
|
||||||
|
|
||||||
const handleGenerateGuide = useCallback(() => {
|
const handleGenerateGuide = useCallback(() => {
|
||||||
void runAction("generate", async () => {
|
void runAction("generate", async () => {
|
||||||
@@ -323,21 +415,59 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
|||||||
if (!videoPath) {
|
if (!videoPath) {
|
||||||
throw new Error("Video URL is not available.");
|
throw new Error("Video URL is not available.");
|
||||||
}
|
}
|
||||||
|
setProgress({
|
||||||
|
label: copy.progressPreparing,
|
||||||
|
current: 0,
|
||||||
|
total: 1,
|
||||||
|
detail: "0/1",
|
||||||
|
});
|
||||||
let current = await ensureEventsSession();
|
let current = await ensureEventsSession();
|
||||||
|
setProgress({
|
||||||
|
label: copy.progressPreparing,
|
||||||
|
current: 1,
|
||||||
|
total: 1,
|
||||||
|
detail: "1/1",
|
||||||
|
});
|
||||||
if (current.events.length === 0) {
|
if (current.events.length === 0) {
|
||||||
throw new Error(copy.noEvents);
|
throw new Error(copy.noEvents);
|
||||||
}
|
}
|
||||||
if (current.snapshots.length < current.events.length) {
|
const snapshotEventIds = new Set(current.snapshots.map((snapshot) => snapshot.eventId));
|
||||||
|
const pendingSnapshotTotal = current.events.filter(
|
||||||
|
(event) => !snapshotEventIds.has(event.id),
|
||||||
|
).length;
|
||||||
|
if (pendingSnapshotTotal > 0) {
|
||||||
|
setProgress({
|
||||||
|
label: copy.progressSnapshots,
|
||||||
|
current: 0,
|
||||||
|
total: pendingSnapshotTotal,
|
||||||
|
detail: `0/${pendingSnapshotTotal}`,
|
||||||
|
});
|
||||||
current = await captureGuideSnapshots({
|
current = await captureGuideSnapshots({
|
||||||
session: current,
|
session: current,
|
||||||
videoUrl: videoPath,
|
videoUrl: videoPath,
|
||||||
maxWidth: 1280,
|
maxWidth: 1280,
|
||||||
|
onProgress: ({ completed, total }) => {
|
||||||
|
setProgress({
|
||||||
|
label: copy.progressSnapshots,
|
||||||
|
current: completed,
|
||||||
|
total,
|
||||||
|
detail: `${completed}/${total}`,
|
||||||
|
});
|
||||||
|
},
|
||||||
});
|
});
|
||||||
setSession(current);
|
setSession(current);
|
||||||
}
|
}
|
||||||
if (current.ocrBlocks.length === 0 && current.snapshots.length > 0) {
|
const pendingOcrSnapshots = getPendingOcrSnapshots(current);
|
||||||
|
for (const [index, snapshot] of pendingOcrSnapshots.entries()) {
|
||||||
|
setProgress({
|
||||||
|
label: copy.progressOcr,
|
||||||
|
current: index,
|
||||||
|
total: pendingOcrSnapshots.length,
|
||||||
|
detail: `${index + 1}/${pendingOcrSnapshots.length}`,
|
||||||
|
});
|
||||||
const ocrResult = await window.electronAPI.guide.runOcr({
|
const ocrResult = await window.electronAPI.guide.runOcr({
|
||||||
recordingId: current.recordingId,
|
recordingId: current.recordingId,
|
||||||
|
snapshotIds: [snapshot.id],
|
||||||
});
|
});
|
||||||
if (!ocrResult.success) {
|
if (!ocrResult.success) {
|
||||||
if (ocrResult.code === "guide-ocr-unavailable") {
|
if (ocrResult.code === "guide-ocr-unavailable") {
|
||||||
@@ -347,7 +477,19 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
|||||||
}
|
}
|
||||||
current = ocrResult.data;
|
current = ocrResult.data;
|
||||||
setSession(current);
|
setSession(current);
|
||||||
|
setProgress({
|
||||||
|
label: copy.progressOcr,
|
||||||
|
current: index + 1,
|
||||||
|
total: pendingOcrSnapshots.length,
|
||||||
|
detail: `${index + 1}/${pendingOcrSnapshots.length}`,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
setProgress({
|
||||||
|
label: copy.progressDraft,
|
||||||
|
current: 0,
|
||||||
|
total: 1,
|
||||||
|
detail: "0/1",
|
||||||
|
});
|
||||||
const result = await window.electronAPI.guide.generateDraft({
|
const result = await window.electronAPI.guide.generateDraft({
|
||||||
recordingId: current.recordingId,
|
recordingId: current.recordingId,
|
||||||
language: guideLanguage,
|
language: guideLanguage,
|
||||||
@@ -356,18 +498,44 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
|||||||
if (!result.success) {
|
if (!result.success) {
|
||||||
throw new Error(result.error);
|
throw new Error(result.error);
|
||||||
}
|
}
|
||||||
|
current = result.data;
|
||||||
|
setSession(current);
|
||||||
|
setProgress({
|
||||||
|
label: copy.progressDraft,
|
||||||
|
current: 1,
|
||||||
|
total: 1,
|
||||||
|
detail: "1/1",
|
||||||
|
});
|
||||||
|
setProgress({
|
||||||
|
label: copy.progressExport,
|
||||||
|
current: 0,
|
||||||
|
total: 2,
|
||||||
|
detail: "0/2",
|
||||||
|
});
|
||||||
const markdownResult = await window.electronAPI.guide.exportMarkdown({
|
const markdownResult = await window.electronAPI.guide.exportMarkdown({
|
||||||
recordingId: current.recordingId,
|
recordingId: current.recordingId,
|
||||||
});
|
});
|
||||||
if (!markdownResult.success) {
|
if (!markdownResult.success) {
|
||||||
throw new Error(markdownResult.error);
|
throw new Error(markdownResult.error);
|
||||||
}
|
}
|
||||||
|
setProgress({
|
||||||
|
label: copy.progressExport,
|
||||||
|
current: 1,
|
||||||
|
total: 2,
|
||||||
|
detail: "1/2",
|
||||||
|
});
|
||||||
const htmlResult = await window.electronAPI.guide.exportHtml({
|
const htmlResult = await window.electronAPI.guide.exportHtml({
|
||||||
recordingId: current.recordingId,
|
recordingId: current.recordingId,
|
||||||
});
|
});
|
||||||
if (!htmlResult.success) {
|
if (!htmlResult.success) {
|
||||||
throw new Error(htmlResult.error);
|
throw new Error(htmlResult.error);
|
||||||
}
|
}
|
||||||
|
setProgress({
|
||||||
|
label: copy.progressExport,
|
||||||
|
current: 2,
|
||||||
|
total: 2,
|
||||||
|
detail: "2/2",
|
||||||
|
});
|
||||||
const revealResult = await window.electronAPI.revealInFolder(htmlResult.data.path);
|
const revealResult = await window.electronAPI.revealInFolder(htmlResult.data.path);
|
||||||
if (!revealResult.success) {
|
if (!revealResult.success) {
|
||||||
toast.warning(revealResult.error ?? "Unable to open guide folder.");
|
toast.warning(revealResult.error ?? "Unable to open guide folder.");
|
||||||
@@ -383,6 +551,11 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
|||||||
copy.keyMissing,
|
copy.keyMissing,
|
||||||
copy.noEvents,
|
copy.noEvents,
|
||||||
copy.ocrUnavailable,
|
copy.ocrUnavailable,
|
||||||
|
copy.progressDraft,
|
||||||
|
copy.progressExport,
|
||||||
|
copy.progressOcr,
|
||||||
|
copy.progressPreparing,
|
||||||
|
copy.progressSnapshots,
|
||||||
ensureEventsSession,
|
ensureEventsSession,
|
||||||
guideLanguage,
|
guideLanguage,
|
||||||
provider,
|
provider,
|
||||||
@@ -390,6 +563,25 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
|||||||
videoPath,
|
videoPath,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
const handleAttachToVideo = useCallback(() => {
|
||||||
|
if (!session?.generatedGuide || session.generatedGuide.steps.length === 0) {
|
||||||
|
setMessage(copy.noDraft);
|
||||||
|
toast.error(copy.noDraft);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!onAttachToVideo) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setBusyAction("attach");
|
||||||
|
try {
|
||||||
|
onAttachToVideo(session);
|
||||||
|
setMessage(null);
|
||||||
|
toast.success(copy.attachedToVideo);
|
||||||
|
} finally {
|
||||||
|
setBusyAction(null);
|
||||||
|
}
|
||||||
|
}, [copy.attachedToVideo, copy.noDraft, onAttachToVideo, session]);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<section className="editor-inspector-shell flex max-h-[320px] min-h-[246px] shrink-0 flex-col overflow-hidden rounded-[18px] border border-white/[0.075] bg-[#090a0c]">
|
<section className="editor-inspector-shell flex max-h-[320px] min-h-[246px] shrink-0 flex-col overflow-hidden rounded-[18px] border border-white/[0.075] bg-[#090a0c]">
|
||||||
<div className="flex items-center justify-between border-b border-white/[0.07] px-3 py-2">
|
<div className="flex items-center justify-between border-b border-white/[0.07] px-3 py-2">
|
||||||
@@ -413,6 +605,24 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
|||||||
{canUseGuide ? statusLabel : copy.noRecording}
|
{canUseGuide ? statusLabel : copy.noRecording}
|
||||||
</p>
|
</p>
|
||||||
{message && <p className="mb-2 text-[11px] leading-4 text-amber-300">{message}</p>}
|
{message && <p className="mb-2 text-[11px] leading-4 text-amber-300">{message}</p>}
|
||||||
|
{progress && (
|
||||||
|
<div className="mb-2 rounded-md border border-white/[0.07] bg-white/[0.035] px-2 py-1.5">
|
||||||
|
<div className="mb-1 flex items-center justify-between gap-2 text-[10px] leading-4">
|
||||||
|
<span className="min-w-0 truncate font-semibold text-slate-200">
|
||||||
|
{progress.label}
|
||||||
|
</span>
|
||||||
|
<span className="shrink-0 text-slate-500">
|
||||||
|
{progress.detail ?? `${progress.current}/${progress.total}`}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<div className="h-1.5 overflow-hidden rounded-full bg-white/[0.06]">
|
||||||
|
<div
|
||||||
|
className="h-full rounded-full bg-[#34B27B] transition-all duration-200"
|
||||||
|
style={{ width: `${progressPercent}%` }}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
<div className="mb-2 flex items-center gap-1.5">
|
<div className="mb-2 flex items-center gap-1.5">
|
||||||
<select
|
<select
|
||||||
@@ -450,12 +660,22 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
|||||||
</span>
|
</span>
|
||||||
</button>
|
</button>
|
||||||
|
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
disabled={!generatedSteps.length || isBusy || !onAttachToVideo}
|
||||||
|
onClick={handleAttachToVideo}
|
||||||
|
className="mb-2 flex h-9 w-full items-center justify-center gap-2 rounded-md border border-sky-400/25 bg-sky-400/10 px-3 text-xs font-semibold text-sky-100 transition-all hover:border-sky-300/45 hover:bg-sky-400/18 disabled:cursor-not-allowed disabled:opacity-40"
|
||||||
|
>
|
||||||
|
<Video className="h-3.5 w-3.5 shrink-0" />
|
||||||
|
<span className="truncate">{copy.attachToVideo}</span>
|
||||||
|
</button>
|
||||||
|
|
||||||
{settingsOpen && (
|
{settingsOpen && (
|
||||||
<div className="mb-2 space-y-2 rounded-md border border-white/[0.07] bg-white/[0.035] p-2">
|
<div className="mb-2 space-y-2 rounded-md border border-white/[0.07] bg-white/[0.035] p-2">
|
||||||
<div className="flex items-center justify-between gap-2">
|
<div className="flex items-center justify-between gap-2">
|
||||||
<div className="min-w-0">
|
<div className="min-w-0">
|
||||||
<div className="truncate text-[11px] font-semibold text-slate-100">
|
<div className="truncate text-[11px] font-semibold text-slate-100">
|
||||||
{copy.deepseek} {copy.settings}
|
{copy.guideSettings}
|
||||||
</div>
|
</div>
|
||||||
<div className="truncate text-[10px] text-slate-500">
|
<div className="truncate text-[10px] text-slate-500">
|
||||||
{aiSettings?.deepseek.hasApiKey
|
{aiSettings?.deepseek.hasApiKey
|
||||||
@@ -470,6 +690,33 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
|||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div className="grid grid-cols-2 gap-1.5">
|
||||||
|
<label className="block min-w-0 text-[10px] font-medium text-slate-400">
|
||||||
|
{copy.ocrProfile}
|
||||||
|
<select
|
||||||
|
value={ocrProfile}
|
||||||
|
onChange={(event) => setOcrProfile(event.target.value as GuideOcrProfile)}
|
||||||
|
disabled={settingsBusy}
|
||||||
|
className="mt-1 h-8 w-full rounded-md border border-white/[0.08] bg-black/20 px-2 text-[11px] text-slate-100 outline-none"
|
||||||
|
>
|
||||||
|
<option value="vietnamese">{copy.ocrVietnamese}</option>
|
||||||
|
<option value="hybrid">{copy.ocrHybrid}</option>
|
||||||
|
<option value="fast">{copy.ocrFast}</option>
|
||||||
|
</select>
|
||||||
|
</label>
|
||||||
|
<label className="block min-w-0 text-[10px] font-medium text-slate-400">
|
||||||
|
{copy.ocrLanguage}
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
value={ocrLanguage}
|
||||||
|
onChange={(event) => setOcrLanguage(event.target.value)}
|
||||||
|
placeholder="vi,en"
|
||||||
|
disabled={settingsBusy}
|
||||||
|
className="mt-1 h-8 w-full rounded-md border border-white/[0.08] bg-black/20 px-2 text-[11px] text-slate-100 outline-none placeholder:text-slate-600"
|
||||||
|
/>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
|
||||||
<label className="block text-[10px] font-medium text-slate-400">
|
<label className="block text-[10px] font-medium text-slate-400">
|
||||||
{copy.apiKey}
|
{copy.apiKey}
|
||||||
<input
|
<input
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ import {
|
|||||||
DEFAULT_BLUR_FREEHAND_POINTS,
|
DEFAULT_BLUR_FREEHAND_POINTS,
|
||||||
DEFAULT_BLUR_INTENSITY,
|
DEFAULT_BLUR_INTENSITY,
|
||||||
DEFAULT_FIGURE_DATA,
|
DEFAULT_FIGURE_DATA,
|
||||||
|
DEFAULT_MAGNIFIER_DATA,
|
||||||
DEFAULT_PLAYBACK_SPEED,
|
DEFAULT_PLAYBACK_SPEED,
|
||||||
DEFAULT_ZOOM_DEPTH,
|
DEFAULT_ZOOM_DEPTH,
|
||||||
DEFAULT_ZOOM_MOTION_BLUR,
|
DEFAULT_ZOOM_MOTION_BLUR,
|
||||||
@@ -325,7 +326,10 @@ export function normalizeProjectEditor(editor: Partial<ProjectEditorState>): Pro
|
|||||||
startMs,
|
startMs,
|
||||||
endMs,
|
endMs,
|
||||||
type:
|
type:
|
||||||
region.type === "image" || region.type === "figure" || region.type === "blur"
|
region.type === "image" ||
|
||||||
|
region.type === "figure" ||
|
||||||
|
region.type === "blur" ||
|
||||||
|
region.type === "magnifier"
|
||||||
? region.type
|
? region.type
|
||||||
: "text",
|
: "text",
|
||||||
content: typeof region.content === "string" ? region.content : "",
|
content: typeof region.content === "string" ? region.content : "",
|
||||||
@@ -410,6 +414,45 @@ export function normalizeProjectEditor(editor: Partial<ProjectEditorState>): Pro
|
|||||||
: DEFAULT_BLUR_FREEHAND_POINTS,
|
: DEFAULT_BLUR_FREEHAND_POINTS,
|
||||||
}
|
}
|
||||||
: undefined,
|
: undefined,
|
||||||
|
magnifierData:
|
||||||
|
region.magnifierData && typeof region.magnifierData === "object"
|
||||||
|
? {
|
||||||
|
...DEFAULT_MAGNIFIER_DATA,
|
||||||
|
...region.magnifierData,
|
||||||
|
target: {
|
||||||
|
x: clamp(
|
||||||
|
isFiniteNumber(region.magnifierData.target?.x)
|
||||||
|
? region.magnifierData.target.x
|
||||||
|
: DEFAULT_MAGNIFIER_DATA.target.x,
|
||||||
|
0,
|
||||||
|
100,
|
||||||
|
),
|
||||||
|
y: clamp(
|
||||||
|
isFiniteNumber(region.magnifierData.target?.y)
|
||||||
|
? region.magnifierData.target.y
|
||||||
|
: DEFAULT_MAGNIFIER_DATA.target.y,
|
||||||
|
0,
|
||||||
|
100,
|
||||||
|
),
|
||||||
|
},
|
||||||
|
zoom: clamp(
|
||||||
|
isFiniteNumber(region.magnifierData.zoom)
|
||||||
|
? region.magnifierData.zoom
|
||||||
|
: DEFAULT_MAGNIFIER_DATA.zoom,
|
||||||
|
1,
|
||||||
|
6,
|
||||||
|
),
|
||||||
|
shape:
|
||||||
|
region.magnifierData.shape === "rounded" ||
|
||||||
|
region.magnifierData.shape === "circle"
|
||||||
|
? region.magnifierData.shape
|
||||||
|
: DEFAULT_MAGNIFIER_DATA.shape,
|
||||||
|
caption:
|
||||||
|
typeof region.magnifierData.caption === "string"
|
||||||
|
? region.magnifierData.caption
|
||||||
|
: undefined,
|
||||||
|
}
|
||||||
|
: undefined,
|
||||||
};
|
};
|
||||||
})
|
})
|
||||||
: [];
|
: [];
|
||||||
|
|||||||
@@ -206,7 +206,7 @@ export interface TrimRegion {
|
|||||||
endMs: number;
|
endMs: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type AnnotationType = "text" | "image" | "figure" | "blur";
|
export type AnnotationType = "text" | "image" | "figure" | "blur" | "magnifier";
|
||||||
|
|
||||||
export type ArrowDirection =
|
export type ArrowDirection =
|
||||||
| "up"
|
| "up"
|
||||||
@@ -245,6 +245,13 @@ export interface BlurData {
|
|||||||
freehandPoints?: Array<{ x: number; y: number }>;
|
freehandPoints?: Array<{ x: number; y: number }>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface MagnifierData {
|
||||||
|
target: AnnotationPosition;
|
||||||
|
zoom: number;
|
||||||
|
shape: "circle" | "rounded";
|
||||||
|
caption?: string;
|
||||||
|
}
|
||||||
|
|
||||||
export interface AnnotationPosition {
|
export interface AnnotationPosition {
|
||||||
x: number;
|
x: number;
|
||||||
y: number;
|
y: number;
|
||||||
@@ -280,6 +287,7 @@ export interface AnnotationRegion {
|
|||||||
zIndex: number;
|
zIndex: number;
|
||||||
figureData?: FigureData;
|
figureData?: FigureData;
|
||||||
blurData?: BlurData;
|
blurData?: BlurData;
|
||||||
|
magnifierData?: MagnifierData;
|
||||||
}
|
}
|
||||||
|
|
||||||
export const DEFAULT_ANNOTATION_POSITION: AnnotationPosition = {
|
export const DEFAULT_ANNOTATION_POSITION: AnnotationPosition = {
|
||||||
@@ -330,6 +338,12 @@ export const DEFAULT_BLUR_DATA: BlurData = {
|
|||||||
freehandPoints: DEFAULT_BLUR_FREEHAND_POINTS,
|
freehandPoints: DEFAULT_BLUR_FREEHAND_POINTS,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const DEFAULT_MAGNIFIER_DATA: MagnifierData = {
|
||||||
|
target: { x: 50, y: 50 },
|
||||||
|
zoom: 2.2,
|
||||||
|
shape: "circle",
|
||||||
|
};
|
||||||
|
|
||||||
export interface CropRegion {
|
export interface CropRegion {
|
||||||
x: number;
|
x: number;
|
||||||
y: number;
|
y: number;
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ export type GuideTargetRole = "button" | "menu" | "tab" | "field" | "link" | "un
|
|||||||
export type GuideLanguage = "vi" | "en";
|
export type GuideLanguage = "vi" | "en";
|
||||||
export type GuideAiProvider = "deepseek" | "local";
|
export type GuideAiProvider = "deepseek" | "local";
|
||||||
export type GuideSecretStorage = "environment" | "none";
|
export type GuideSecretStorage = "environment" | "none";
|
||||||
|
export type GuideOcrProfile = "fast" | "vietnamese" | "hybrid";
|
||||||
|
|
||||||
export type GuideSessionStatus =
|
export type GuideSessionStatus =
|
||||||
| "recording"
|
| "recording"
|
||||||
@@ -54,6 +55,8 @@ export interface GuideSnapshot {
|
|||||||
timeMs: number;
|
timeMs: number;
|
||||||
offsetMs: number;
|
offsetMs: number;
|
||||||
path: string;
|
path: string;
|
||||||
|
markedPath?: string;
|
||||||
|
ocrCompletedAt?: string;
|
||||||
width: number;
|
width: number;
|
||||||
height: number;
|
height: number;
|
||||||
}
|
}
|
||||||
@@ -162,6 +165,7 @@ export interface WriteGuideSnapshotInput {
|
|||||||
timeMs: number;
|
timeMs: number;
|
||||||
offsetMs: number;
|
offsetMs: number;
|
||||||
pngBytes: ArrayBuffer;
|
pngBytes: ArrayBuffer;
|
||||||
|
markedPngBytes?: ArrayBuffer;
|
||||||
width: number;
|
width: number;
|
||||||
height: number;
|
height: number;
|
||||||
}
|
}
|
||||||
@@ -178,6 +182,11 @@ export interface GenerateGuideDraftInput {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface GuideAiSettings {
|
export interface GuideAiSettings {
|
||||||
|
ocr: {
|
||||||
|
profile: GuideOcrProfile;
|
||||||
|
language: string;
|
||||||
|
updatedAt?: string;
|
||||||
|
};
|
||||||
deepseek: {
|
deepseek: {
|
||||||
hasApiKey: boolean;
|
hasApiKey: boolean;
|
||||||
apiKeyEnvName: string;
|
apiKeyEnvName: string;
|
||||||
@@ -194,6 +203,8 @@ export interface SaveGuideAiSettingsInput {
|
|||||||
clearDeepseekApiKeyEnvName?: boolean;
|
clearDeepseekApiKeyEnvName?: boolean;
|
||||||
baseUrl?: string;
|
baseUrl?: string;
|
||||||
model?: string;
|
model?: string;
|
||||||
|
ocrProfile?: GuideOcrProfile;
|
||||||
|
ocrLanguage?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface SaveGuideInput {
|
export interface SaveGuideInput {
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ const session: GuideSession = {
|
|||||||
timeMs: 1500,
|
timeMs: 1500,
|
||||||
offsetMs: 500,
|
offsetMs: 500,
|
||||||
path: "/tmp/recording-guide/step-001.png",
|
path: "/tmp/recording-guide/step-001.png",
|
||||||
|
markedPath: "/tmp/recording-guide/step-001-marked.png",
|
||||||
width: 1280,
|
width: 1280,
|
||||||
height: 720,
|
height: 720,
|
||||||
},
|
},
|
||||||
@@ -71,7 +72,7 @@ describe("guide exporters", () => {
|
|||||||
|
|
||||||
expect(markdown).toContain("# User guide");
|
expect(markdown).toContain("# User guide");
|
||||||
expect(markdown).toContain("## 1. Open Settings");
|
expect(markdown).toContain("## 1. Open Settings");
|
||||||
expect(markdown).toContain("](step-001.png)");
|
expect(markdown).toContain("](step-001-marked.png)");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("exports escaped HTML", () => {
|
it("exports escaped HTML", () => {
|
||||||
@@ -79,12 +80,11 @@ describe("guide exporters", () => {
|
|||||||
|
|
||||||
expect(html).toContain("<!doctype html>");
|
expect(html).toContain("<!doctype html>");
|
||||||
expect(html).toContain("<h1>User guide</h1>");
|
expect(html).toContain("<h1>User guide</h1>");
|
||||||
expect(html).toContain('src="step-001.png"');
|
expect(html).toContain('src="step-001-marked.png"');
|
||||||
expect(html).toContain("click-marker");
|
expect(html).not.toContain("click-marker");
|
||||||
expect(html).toContain("left: 25.00%; top: 75.00%;");
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it("draws click markers for hotkey events with coordinates", () => {
|
it("uses marker snapshots for hotkey events with coordinates", () => {
|
||||||
const hotkeySession: GuideSession = {
|
const hotkeySession: GuideSession = {
|
||||||
...session,
|
...session,
|
||||||
events: [
|
events: [
|
||||||
@@ -98,7 +98,21 @@ describe("guide exporters", () => {
|
|||||||
|
|
||||||
const html = exportGuideToHtml(hotkeySession);
|
const html = exportGuideToHtml(hotkeySession);
|
||||||
|
|
||||||
expect(html).toContain("click-marker");
|
expect(html).toContain('src="step-001-marked.png"');
|
||||||
expect(html).toContain("left: 25.00%; top: 75.00%;");
|
expect(html).not.toContain("click-marker");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("falls back to the unmarked screenshot when no marker snapshot exists", () => {
|
||||||
|
const unmarkedSession: GuideSession = {
|
||||||
|
...session,
|
||||||
|
snapshots: session.snapshots.map((snapshot) => ({
|
||||||
|
...snapshot,
|
||||||
|
markedPath: undefined,
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
|
||||||
|
const markdown = exportGuideToMarkdown(unmarkedSession);
|
||||||
|
|
||||||
|
expect(markdown).toContain("](step-001.png)");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
+25
-51
@@ -10,8 +10,9 @@ export function exportGuideToMarkdown(session: GuideSession): string {
|
|||||||
|
|
||||||
for (const step of guide.steps) {
|
for (const step of guide.steps) {
|
||||||
lines.push(`## ${step.order}. ${step.title}`, "", step.instruction, "");
|
lines.push(`## ${step.order}. ${step.title}`, "", step.instruction, "");
|
||||||
if (step.screenshotPath) {
|
const screenshotPath = resolveStepScreenshotPath(step, session);
|
||||||
lines.push(`})`, "");
|
if (screenshotPath) {
|
||||||
|
lines.push(`})`, "");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -36,10 +37,8 @@ export function exportGuideToHtml(session: GuideSession): string {
|
|||||||
.step { border-top: 1px solid #e5e7eb; padding: 22px 0; }
|
.step { border-top: 1px solid #e5e7eb; padding: 22px 0; }
|
||||||
.step h2 { font-size: 18px; margin: 0 0 8px; }
|
.step h2 { font-size: 18px; margin: 0 0 8px; }
|
||||||
.step p { margin: 0 0 12px; }
|
.step p { margin: 0 0 12px; }
|
||||||
.shot { display: inline-block; position: relative; max-width: 100%; margin: 0; }
|
.shot { display: inline-block; max-width: 100%; margin: 0; }
|
||||||
img { display: block; max-width: 100%; border: 1px solid #e5e7eb; border-radius: 6px; }
|
img { display: block; max-width: 100%; border: 1px solid #e5e7eb; border-radius: 6px; }
|
||||||
.click-marker { position: absolute; width: 26px; height: 26px; border: 3px solid #ef4444; border-radius: 999px; box-shadow: 0 0 0 4px rgba(239, 68, 68, 0.18), 0 2px 8px rgba(17, 24, 39, 0.28); transform: translate(-50%, -50%); pointer-events: none; }
|
|
||||||
.click-marker::after { content: ""; position: absolute; left: 50%; top: 50%; width: 6px; height: 6px; border-radius: 999px; background: #ef4444; transform: translate(-50%, -50%); }
|
|
||||||
</style>
|
</style>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
@@ -54,12 +53,9 @@ export function exportGuideToHtml(session: GuideSession): string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function renderStepHtml(step: GeneratedGuideStep, session: GuideSession): string {
|
function renderStepHtml(step: GeneratedGuideStep, session: GuideSession): string {
|
||||||
const clickPoint = resolveStepClickPoint(step, session);
|
const screenshotPath = resolveStepScreenshotPath(step, session);
|
||||||
const marker = clickPoint
|
const image = screenshotPath
|
||||||
? `<span class="click-marker" style="left: ${formatPercent(clickPoint.x)}%; top: ${formatPercent(clickPoint.y)}%;" aria-label="Click position"></span>`
|
? `<figure class="shot"><img src="${escapeHtml(path.basename(screenshotPath))}" alt="${escapeHtml(step.title)}"></figure>`
|
||||||
: "";
|
|
||||||
const image = step.screenshotPath
|
|
||||||
? `<figure class="shot"><img src="${escapeHtml(path.basename(step.screenshotPath))}" alt="${escapeHtml(step.title)}">${marker}</figure>`
|
|
||||||
: "";
|
: "";
|
||||||
return `<section class="step">
|
return `<section class="step">
|
||||||
<h2>${step.order}. ${escapeHtml(step.title)}</h2>
|
<h2>${step.order}. ${escapeHtml(step.title)}</h2>
|
||||||
@@ -88,54 +84,32 @@ function escapeHtml(value: string): string {
|
|||||||
.replace(/'/g, "'");
|
.replace(/'/g, "'");
|
||||||
}
|
}
|
||||||
|
|
||||||
function resolveStepClickPoint(
|
function resolveStepScreenshotPath(
|
||||||
step: GeneratedGuideStep,
|
step: GeneratedGuideStep,
|
||||||
session: GuideSession,
|
session: GuideSession,
|
||||||
): { x: number; y: number } | null {
|
): string | undefined {
|
||||||
|
const snapshot = resolveStepSnapshot(step, session);
|
||||||
|
return snapshot?.markedPath ?? step.screenshotPath ?? snapshot?.path;
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveStepSnapshot(step: GeneratedGuideStep, session: GuideSession) {
|
||||||
const candidate = step.sourceCandidateId
|
const candidate = step.sourceCandidateId
|
||||||
? session.candidates.find((item) => item.id === step.sourceCandidateId)
|
? session.candidates.find((item) => item.id === step.sourceCandidateId)
|
||||||
: undefined;
|
: undefined;
|
||||||
const eventId = candidate?.eventId;
|
|
||||||
const event = eventId ? session.events.find((item) => item.id === eventId) : undefined;
|
|
||||||
if (!event || (event.kind !== "click" && event.kind !== "hotkey")) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
if (isNormalizedNumber(event.normalizedX) && isNormalizedNumber(event.normalizedY)) {
|
|
||||||
return { x: clamp01(event.normalizedX), y: clamp01(event.normalizedY) };
|
|
||||||
}
|
|
||||||
|
|
||||||
const screenshotFileName = step.screenshotPath ? path.basename(step.screenshotPath) : undefined;
|
const screenshotFileName = step.screenshotPath ? path.basename(step.screenshotPath) : undefined;
|
||||||
const snapshot =
|
return (
|
||||||
(candidate?.snapshotId
|
(candidate?.snapshotId
|
||||||
? session.snapshots.find((item) => item.id === candidate.snapshotId)
|
? session.snapshots.find((item) => item.id === candidate.snapshotId)
|
||||||
: undefined) ??
|
: undefined) ??
|
||||||
|
(candidate?.eventId
|
||||||
|
? session.snapshots.find((item) => item.eventId === candidate.eventId)
|
||||||
|
: undefined) ??
|
||||||
(screenshotFileName
|
(screenshotFileName
|
||||||
? session.snapshots.find((item) => path.basename(item.path) === screenshotFileName)
|
? session.snapshots.find(
|
||||||
: undefined);
|
(item) =>
|
||||||
if (
|
path.basename(item.path) === screenshotFileName ||
|
||||||
!snapshot ||
|
(item.markedPath ? path.basename(item.markedPath) === screenshotFileName : false),
|
||||||
typeof event.x !== "number" ||
|
)
|
||||||
typeof event.y !== "number" ||
|
: undefined)
|
||||||
snapshot.width <= 0 ||
|
);
|
||||||
snapshot.height <= 0
|
|
||||||
) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
x: clamp01(event.x / snapshot.width),
|
|
||||||
y: clamp01(event.y / snapshot.height),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function formatPercent(value: number): string {
|
|
||||||
return (clamp01(value) * 100).toFixed(2);
|
|
||||||
}
|
|
||||||
|
|
||||||
function isNormalizedNumber(value: unknown): value is number {
|
|
||||||
return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
function clamp01(value: number): number {
|
|
||||||
return Math.min(1, Math.max(0, value));
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,13 @@ export interface CaptureGuideSnapshotsInput {
|
|||||||
session: GuideSession;
|
session: GuideSession;
|
||||||
videoUrl: string;
|
videoUrl: string;
|
||||||
maxWidth?: number;
|
maxWidth?: number;
|
||||||
|
onProgress?: (progress: CaptureGuideSnapshotsProgress) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CaptureGuideSnapshotsProgress {
|
||||||
|
event: GuideEvent;
|
||||||
|
completed: number;
|
||||||
|
total: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function captureGuideSnapshots(
|
export async function captureGuideSnapshots(
|
||||||
@@ -13,6 +20,13 @@ export async function captureGuideSnapshots(
|
|||||||
if (events.length === 0) {
|
if (events.length === 0) {
|
||||||
return input.session;
|
return input.session;
|
||||||
}
|
}
|
||||||
|
const existingSnapshotsByEventId = new Set(
|
||||||
|
input.session.snapshots.map((snapshot) => snapshot.eventId),
|
||||||
|
);
|
||||||
|
const pendingEvents = events.filter((event) => !existingSnapshotsByEventId.has(event.id));
|
||||||
|
if (pendingEvents.length === 0) {
|
||||||
|
return input.session;
|
||||||
|
}
|
||||||
|
|
||||||
const video = document.createElement("video");
|
const video = document.createElement("video");
|
||||||
video.preload = "auto";
|
video.preload = "auto";
|
||||||
@@ -35,18 +49,24 @@ export async function captureGuideSnapshots(
|
|||||||
canvas.height = Math.max(1, Math.round(sourceHeight * scale));
|
canvas.height = Math.max(1, Math.round(sourceHeight * scale));
|
||||||
|
|
||||||
let latestSession = input.session;
|
let latestSession = input.session;
|
||||||
for (const event of events) {
|
let completed = 0;
|
||||||
|
for (const event of pendingEvents) {
|
||||||
const offsetMs = event.screenshotOffsetMs ?? 500;
|
const offsetMs = event.screenshotOffsetMs ?? 500;
|
||||||
const timeMs = getSnapshotTimeMs(event, offsetMs, video.duration);
|
const timeMs = getSnapshotTimeMs(event, offsetMs, video.duration);
|
||||||
await seekVideo(video, timeMs / 1000);
|
await seekVideo(video, timeMs / 1000);
|
||||||
context.drawImage(video, 0, 0, canvas.width, canvas.height);
|
context.drawImage(video, 0, 0, canvas.width, canvas.height);
|
||||||
const pngBytes = await canvasToPngBytes(canvas);
|
const pngBytes = await canvasToPngBytes(canvas);
|
||||||
|
const markerPoint = getSnapshotMarkerPoint(event, canvas.width, canvas.height);
|
||||||
|
const markedPngBytes = markerPoint
|
||||||
|
? await canvasToMarkedPngBytes(canvas, markerPoint)
|
||||||
|
: undefined;
|
||||||
const result = await window.electronAPI.guide.writeSnapshot({
|
const result = await window.electronAPI.guide.writeSnapshot({
|
||||||
recordingId: input.session.recordingId,
|
recordingId: input.session.recordingId,
|
||||||
eventId: event.id,
|
eventId: event.id,
|
||||||
timeMs,
|
timeMs,
|
||||||
offsetMs,
|
offsetMs,
|
||||||
pngBytes,
|
pngBytes,
|
||||||
|
markedPngBytes,
|
||||||
width: canvas.width,
|
width: canvas.width,
|
||||||
height: canvas.height,
|
height: canvas.height,
|
||||||
});
|
});
|
||||||
@@ -54,6 +74,12 @@ export async function captureGuideSnapshots(
|
|||||||
throw new Error(result.error);
|
throw new Error(result.error);
|
||||||
}
|
}
|
||||||
latestSession = result.data;
|
latestSession = result.data;
|
||||||
|
completed += 1;
|
||||||
|
input.onProgress?.({
|
||||||
|
event,
|
||||||
|
completed,
|
||||||
|
total: pendingEvents.length,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
return latestSession;
|
return latestSession;
|
||||||
@@ -143,3 +169,75 @@ function canvasToPngBytes(canvas: HTMLCanvasElement): Promise<ArrayBuffer> {
|
|||||||
}, "image/png");
|
}, "image/png");
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function canvasToMarkedPngBytes(
|
||||||
|
canvas: HTMLCanvasElement,
|
||||||
|
point: { x: number; y: number },
|
||||||
|
): Promise<ArrayBuffer> {
|
||||||
|
const markedCanvas = document.createElement("canvas");
|
||||||
|
markedCanvas.width = canvas.width;
|
||||||
|
markedCanvas.height = canvas.height;
|
||||||
|
const markedContext = markedCanvas.getContext("2d");
|
||||||
|
if (!markedContext) {
|
||||||
|
throw new Error("Canvas 2D context is unavailable.");
|
||||||
|
}
|
||||||
|
markedContext.drawImage(canvas, 0, 0);
|
||||||
|
drawSnapshotMarker(markedContext, markedCanvas, point);
|
||||||
|
return await canvasToPngBytes(markedCanvas);
|
||||||
|
}
|
||||||
|
|
||||||
|
function drawSnapshotMarker(
|
||||||
|
context: CanvasRenderingContext2D,
|
||||||
|
canvas: HTMLCanvasElement,
|
||||||
|
point: { x: number; y: number },
|
||||||
|
) {
|
||||||
|
const shortSide = Math.max(1, Math.min(canvas.width, canvas.height));
|
||||||
|
const dotRadius = clampNumber(Math.round(shortSide * 0.005), 4, 7);
|
||||||
|
|
||||||
|
context.beginPath();
|
||||||
|
context.arc(point.x, point.y, dotRadius, 0, Math.PI * 2);
|
||||||
|
context.fillStyle = "rgba(220, 38, 38, 0.92)";
|
||||||
|
context.fill();
|
||||||
|
}
|
||||||
|
|
||||||
|
function getSnapshotMarkerPoint(
|
||||||
|
event: GuideEvent,
|
||||||
|
width: number,
|
||||||
|
height: number,
|
||||||
|
): { x: number; y: number } | null {
|
||||||
|
if (event.kind !== "click" && event.kind !== "hotkey") {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (isNormalizedNumber(event.normalizedX) && isNormalizedNumber(event.normalizedY)) {
|
||||||
|
return {
|
||||||
|
x: clampNumber(event.normalizedX * width, 0, width),
|
||||||
|
y: clampNumber(event.normalizedY * height, 0, height),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (isNormalizedNumber(event.x) && isNormalizedNumber(event.y)) {
|
||||||
|
return {
|
||||||
|
x: clampNumber(event.x * width, 0, width),
|
||||||
|
y: clampNumber(event.y * height, 0, height),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
typeof event.x === "number" &&
|
||||||
|
typeof event.y === "number" &&
|
||||||
|
Number.isFinite(event.x) &&
|
||||||
|
Number.isFinite(event.y)
|
||||||
|
) {
|
||||||
|
return {
|
||||||
|
x: clampNumber(event.x, 0, width),
|
||||||
|
y: clampNumber(event.y, 0, height),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isNormalizedNumber(value: unknown): value is number {
|
||||||
|
return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
function clampNumber(value: number, min = 0, max = Number.POSITIVE_INFINITY): number {
|
||||||
|
return Math.min(max, Math.max(min, value));
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,122 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { GUIDE_SCHEMA_VERSION, type GuideSession } from "./contracts";
|
||||||
|
import { buildGuideVideoAnnotations, buildGuideVideoSpeedRegions } from "./videoAnnotations";
|
||||||
|
|
||||||
|
function createSession(): GuideSession {
|
||||||
|
return {
|
||||||
|
schemaVersion: GUIDE_SCHEMA_VERSION,
|
||||||
|
recordingId: "recording-1",
|
||||||
|
videoPath: "recording.mp4",
|
||||||
|
guidePath: "recording.guide.json",
|
||||||
|
outputDir: "recording-guide",
|
||||||
|
status: "draft-ready",
|
||||||
|
events: [],
|
||||||
|
snapshots: [],
|
||||||
|
ocrBlocks: [],
|
||||||
|
candidates: [
|
||||||
|
{
|
||||||
|
id: "candidate-1",
|
||||||
|
eventId: "event-1",
|
||||||
|
timeMs: 1200,
|
||||||
|
action: "click",
|
||||||
|
targetText: "Settings",
|
||||||
|
targetRole: "button",
|
||||||
|
position: {
|
||||||
|
normalizedX: 0.2,
|
||||||
|
normalizedY: 0.25,
|
||||||
|
xPercent: 20,
|
||||||
|
yPercent: 25,
|
||||||
|
description: "top left",
|
||||||
|
},
|
||||||
|
nearbyText: ["Settings"],
|
||||||
|
confidence: 0.91,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
generatedGuide: {
|
||||||
|
title: "Guide",
|
||||||
|
steps: [
|
||||||
|
{
|
||||||
|
id: "step-1",
|
||||||
|
order: 1,
|
||||||
|
title: "Open settings",
|
||||||
|
instruction: "Click Settings.",
|
||||||
|
sourceCandidateId: "candidate-1",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
createdAt: "2026-06-04T00:00:00.000Z",
|
||||||
|
updatedAt: "2026-06-04T00:00:00.000Z",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("buildGuideVideoAnnotations", () => {
|
||||||
|
it("creates caption and pointer annotations from generated guide candidates", () => {
|
||||||
|
let id = 1;
|
||||||
|
let zIndex = 1;
|
||||||
|
const annotations = buildGuideVideoAnnotations(createSession(), {
|
||||||
|
nextId: () => `guide-video-${id++}`,
|
||||||
|
nextZIndex: () => zIndex++,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(annotations).toHaveLength(3);
|
||||||
|
expect(annotations[0]).toMatchObject({
|
||||||
|
id: "guide-video-1",
|
||||||
|
type: "text",
|
||||||
|
startMs: 1200,
|
||||||
|
content: "1. Click Settings.",
|
||||||
|
});
|
||||||
|
expect(annotations[0]?.endMs).toBe(3200);
|
||||||
|
expect(annotations[0]?.position.x).toBeGreaterThan(20);
|
||||||
|
expect(annotations[1]?.endMs).toBe(3200);
|
||||||
|
expect(annotations[1]?.position.x).toBeGreaterThan((annotations[0]?.position.x ?? 0) + 34);
|
||||||
|
expect(annotations[1]?.position.y).toBeCloseTo(30.5);
|
||||||
|
expect(annotations[1]).toMatchObject({
|
||||||
|
id: "guide-video-2",
|
||||||
|
type: "magnifier",
|
||||||
|
magnifierData: {
|
||||||
|
target: { x: 20, y: 25 },
|
||||||
|
zoom: 2.2,
|
||||||
|
shape: "circle",
|
||||||
|
caption: "Settings",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(annotations[2]).toMatchObject({
|
||||||
|
id: "guide-video-3",
|
||||||
|
type: "figure",
|
||||||
|
endMs: 3200,
|
||||||
|
figureData: {
|
||||||
|
arrowDirection: "left",
|
||||||
|
color: "#34B27B",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(annotations[2]?.position.x).toBeGreaterThan(20);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns an empty list when no draft exists", () => {
|
||||||
|
const session = createSession();
|
||||||
|
session.generatedGuide = undefined;
|
||||||
|
|
||||||
|
const annotations = buildGuideVideoAnnotations(session, {
|
||||||
|
nextId: () => "unused",
|
||||||
|
nextZIndex: () => 1,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(annotations).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("creates 0.3x speed regions for two seconds at each guide point", () => {
|
||||||
|
let id = 1;
|
||||||
|
const speedRegions = buildGuideVideoSpeedRegions(createSession(), {
|
||||||
|
nextId: () => `guide-speed-${id++}`,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(speedRegions).toEqual([
|
||||||
|
{
|
||||||
|
id: "guide-speed-1",
|
||||||
|
startMs: 1200,
|
||||||
|
endMs: 3200,
|
||||||
|
speed: 0.3,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,249 @@
|
|||||||
|
import {
|
||||||
|
type AnnotationRegion,
|
||||||
|
type ArrowDirection,
|
||||||
|
DEFAULT_ANNOTATION_STYLE,
|
||||||
|
DEFAULT_FIGURE_DATA,
|
||||||
|
DEFAULT_MAGNIFIER_DATA,
|
||||||
|
type SpeedRegion,
|
||||||
|
} from "@/components/video-editor/types";
|
||||||
|
import type { GeneratedGuideStep, GuideSession, GuideStepCandidate } from "./contracts";
|
||||||
|
|
||||||
|
export interface BuildGuideVideoAnnotationsOptions {
|
||||||
|
nextId: () => string;
|
||||||
|
nextZIndex: () => number;
|
||||||
|
defaultDurationMs?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
const DEFAULT_STEP_DURATION_MS = 2000;
|
||||||
|
const DEFAULT_STEP_SLOW_MOTION_DURATION_MS = 2000;
|
||||||
|
const DEFAULT_STEP_SLOW_MOTION_SPEED = 0.3;
|
||||||
|
const CAPTION_WIDTH = 34;
|
||||||
|
const CAPTION_HEIGHT = 13;
|
||||||
|
const MAGNIFIER_SIZE = 18;
|
||||||
|
const ARROW_SIZE = 10;
|
||||||
|
const ANNOTATION_GAP = 2;
|
||||||
|
|
||||||
|
function clamp(value: number, min: number, max: number) {
|
||||||
|
return Math.min(max, Math.max(min, value));
|
||||||
|
}
|
||||||
|
|
||||||
|
function findCandidate(
|
||||||
|
step: GeneratedGuideStep,
|
||||||
|
stepIndex: number,
|
||||||
|
candidates: GuideStepCandidate[],
|
||||||
|
): GuideStepCandidate | undefined {
|
||||||
|
if (step.sourceCandidateId) {
|
||||||
|
const matched = candidates.find((candidate) => candidate.id === step.sourceCandidateId);
|
||||||
|
if (matched) return matched;
|
||||||
|
}
|
||||||
|
const sorted = [...candidates].sort((left, right) => left.timeMs - right.timeMs);
|
||||||
|
return sorted[stepIndex];
|
||||||
|
}
|
||||||
|
|
||||||
|
function getCaptionPosition(candidate: GuideStepCandidate | undefined) {
|
||||||
|
const target = candidate?.position;
|
||||||
|
if (!target) {
|
||||||
|
return { x: 8, y: 8 };
|
||||||
|
}
|
||||||
|
|
||||||
|
const targetX = target.normalizedX * 100;
|
||||||
|
const targetY = target.normalizedY * 100;
|
||||||
|
const x = target.normalizedX < 0.5 ? targetX + 8 : targetX - CAPTION_WIDTH - 8;
|
||||||
|
const y = target.normalizedY < 0.5 ? targetY + 8 : targetY - CAPTION_HEIGHT - 8;
|
||||||
|
|
||||||
|
return {
|
||||||
|
x: clamp(x, 2, 100 - CAPTION_WIDTH - 2),
|
||||||
|
y: clamp(y, 2, 100 - CAPTION_HEIGHT - 2),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function getArrowDirection(
|
||||||
|
candidate: GuideStepCandidate | undefined,
|
||||||
|
originPosition: { x: number; y: number },
|
||||||
|
originSize: { width: number; height: number } = {
|
||||||
|
width: CAPTION_WIDTH,
|
||||||
|
height: CAPTION_HEIGHT,
|
||||||
|
},
|
||||||
|
): ArrowDirection {
|
||||||
|
const target = candidate?.position;
|
||||||
|
if (!target) return "right";
|
||||||
|
|
||||||
|
const originCenterX = originPosition.x + originSize.width / 2;
|
||||||
|
const originCenterY = originPosition.y + originSize.height / 2;
|
||||||
|
const dx = target.normalizedX * 100 - originCenterX;
|
||||||
|
const dy = target.normalizedY * 100 - originCenterY;
|
||||||
|
const horizontal = dx > 8 ? "right" : dx < -8 ? "left" : "";
|
||||||
|
const vertical = dy > 8 ? "down" : dy < -8 ? "up" : "";
|
||||||
|
|
||||||
|
if (vertical && horizontal) return `${vertical}-${horizontal}` as ArrowDirection;
|
||||||
|
return (horizontal || vertical || "right") as ArrowDirection;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getMagnifierPosition(captionPosition: { x: number; y: number }) {
|
||||||
|
const canPlaceRight = captionPosition.x + CAPTION_WIDTH + ANNOTATION_GAP + MAGNIFIER_SIZE <= 98;
|
||||||
|
const x = canPlaceRight
|
||||||
|
? captionPosition.x + CAPTION_WIDTH + ANNOTATION_GAP
|
||||||
|
: captionPosition.x - MAGNIFIER_SIZE - ANNOTATION_GAP;
|
||||||
|
const y = captionPosition.y + (CAPTION_HEIGHT - MAGNIFIER_SIZE) / 2;
|
||||||
|
|
||||||
|
return {
|
||||||
|
x: clamp(x, 2, 100 - MAGNIFIER_SIZE - 2),
|
||||||
|
y: clamp(y, 2, 100 - MAGNIFIER_SIZE - 2),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function getArrowPosition(
|
||||||
|
position: NonNullable<GuideStepCandidate["position"]>,
|
||||||
|
originPosition: { x: number; y: number },
|
||||||
|
originSize: { width: number; height: number },
|
||||||
|
) {
|
||||||
|
const targetX = position.normalizedX * 100;
|
||||||
|
const targetY = position.normalizedY * 100;
|
||||||
|
const originCenterX = originPosition.x + originSize.width / 2;
|
||||||
|
const originCenterY = originPosition.y + originSize.height / 2;
|
||||||
|
const distance = Math.hypot(targetX - originCenterX, targetY - originCenterY);
|
||||||
|
const targetOffset = Math.min(18, Math.max(10, distance * 0.35));
|
||||||
|
const ratio = distance > 0 ? Math.max(0, (distance - targetOffset) / distance) : 0;
|
||||||
|
const arrowCenterX = originCenterX + (targetX - originCenterX) * ratio;
|
||||||
|
const arrowCenterY = originCenterY + (targetY - originCenterY) * ratio;
|
||||||
|
|
||||||
|
return {
|
||||||
|
x: clamp(arrowCenterX - ARROW_SIZE / 2, 0, 100 - ARROW_SIZE),
|
||||||
|
y: clamp(arrowCenterY - ARROW_SIZE / 2, 0, 100 - ARROW_SIZE),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildCaption(step: GeneratedGuideStep) {
|
||||||
|
const instruction = step.instruction.trim();
|
||||||
|
const title = step.title.trim();
|
||||||
|
if (instruction) {
|
||||||
|
return `${step.order}. ${instruction}`;
|
||||||
|
}
|
||||||
|
return title ? `${step.order}. ${title}` : `Step ${step.order}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildGuideVideoAnnotations(
|
||||||
|
session: GuideSession,
|
||||||
|
options: BuildGuideVideoAnnotationsOptions,
|
||||||
|
): AnnotationRegion[] {
|
||||||
|
const guide = session.generatedGuide;
|
||||||
|
if (!guide || guide.steps.length === 0) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const durationMs = Math.max(1000, options.defaultDurationMs ?? DEFAULT_STEP_DURATION_MS);
|
||||||
|
const sortedSteps = [...guide.steps].sort((left, right) => left.order - right.order);
|
||||||
|
const annotations: AnnotationRegion[] = [];
|
||||||
|
|
||||||
|
for (const [index, step] of sortedSteps.entries()) {
|
||||||
|
const candidate = findCandidate(step, index, session.candidates);
|
||||||
|
const startMs = Math.max(0, Math.round(candidate?.timeMs ?? index * durationMs));
|
||||||
|
const endMs = Math.max(startMs + 750, startMs + durationMs);
|
||||||
|
const captionPosition = getCaptionPosition(candidate);
|
||||||
|
|
||||||
|
annotations.push({
|
||||||
|
id: options.nextId(),
|
||||||
|
startMs,
|
||||||
|
endMs,
|
||||||
|
type: "text",
|
||||||
|
content: buildCaption(step),
|
||||||
|
textContent: buildCaption(step),
|
||||||
|
position: captionPosition,
|
||||||
|
size: { width: CAPTION_WIDTH, height: CAPTION_HEIGHT },
|
||||||
|
style: {
|
||||||
|
...DEFAULT_ANNOTATION_STYLE,
|
||||||
|
color: "#f8fafc",
|
||||||
|
backgroundColor: "rgba(15, 23, 42, 0.88)",
|
||||||
|
fontSize: 18,
|
||||||
|
fontWeight: "bold",
|
||||||
|
textAlign: "left",
|
||||||
|
},
|
||||||
|
zIndex: options.nextZIndex(),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (candidate?.position) {
|
||||||
|
const magnifierPosition = getMagnifierPosition(captionPosition);
|
||||||
|
const arrowPosition = getArrowPosition(candidate.position, magnifierPosition, {
|
||||||
|
width: MAGNIFIER_SIZE,
|
||||||
|
height: MAGNIFIER_SIZE,
|
||||||
|
});
|
||||||
|
const arrowDirection = getArrowDirection(candidate, arrowPosition, {
|
||||||
|
width: ARROW_SIZE,
|
||||||
|
height: ARROW_SIZE,
|
||||||
|
});
|
||||||
|
|
||||||
|
annotations.push({
|
||||||
|
id: options.nextId(),
|
||||||
|
startMs,
|
||||||
|
endMs,
|
||||||
|
type: "magnifier",
|
||||||
|
content: buildCaption(step),
|
||||||
|
position: magnifierPosition,
|
||||||
|
size: { width: MAGNIFIER_SIZE, height: MAGNIFIER_SIZE },
|
||||||
|
style: { ...DEFAULT_ANNOTATION_STYLE },
|
||||||
|
zIndex: options.nextZIndex(),
|
||||||
|
magnifierData: {
|
||||||
|
...DEFAULT_MAGNIFIER_DATA,
|
||||||
|
target: {
|
||||||
|
x: candidate.position.normalizedX * 100,
|
||||||
|
y: candidate.position.normalizedY * 100,
|
||||||
|
},
|
||||||
|
caption: candidate.targetText,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
annotations.push({
|
||||||
|
id: options.nextId(),
|
||||||
|
startMs,
|
||||||
|
endMs,
|
||||||
|
type: "figure",
|
||||||
|
content: "",
|
||||||
|
position: arrowPosition,
|
||||||
|
size: { width: ARROW_SIZE, height: ARROW_SIZE },
|
||||||
|
style: { ...DEFAULT_ANNOTATION_STYLE },
|
||||||
|
zIndex: options.nextZIndex(),
|
||||||
|
figureData: {
|
||||||
|
...DEFAULT_FIGURE_DATA,
|
||||||
|
arrowDirection,
|
||||||
|
color: "#34B27B",
|
||||||
|
strokeWidth: 5,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return annotations;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface BuildGuideVideoSpeedRegionsOptions {
|
||||||
|
nextId: () => string;
|
||||||
|
durationMs?: number;
|
||||||
|
speed?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildGuideVideoSpeedRegions(
|
||||||
|
session: GuideSession,
|
||||||
|
options: BuildGuideVideoSpeedRegionsOptions,
|
||||||
|
): SpeedRegion[] {
|
||||||
|
const guide = session.generatedGuide;
|
||||||
|
if (!guide || guide.steps.length === 0) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const durationMs = Math.max(
|
||||||
|
100,
|
||||||
|
Math.round(options.durationMs ?? DEFAULT_STEP_SLOW_MOTION_DURATION_MS),
|
||||||
|
);
|
||||||
|
const speed = options.speed ?? DEFAULT_STEP_SLOW_MOTION_SPEED;
|
||||||
|
const sortedSteps = [...guide.steps].sort((left, right) => left.order - right.order);
|
||||||
|
|
||||||
|
return sortedSteps.map((step, index) => {
|
||||||
|
const candidate = findCandidate(step, index, session.candidates);
|
||||||
|
const startMs = Math.max(0, Math.round(candidate?.timeMs ?? index * durationMs));
|
||||||
|
return {
|
||||||
|
id: options.nextId(),
|
||||||
|
startMs,
|
||||||
|
endMs: startMs + durationMs,
|
||||||
|
speed,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -9,6 +9,8 @@ import {
|
|||||||
|
|
||||||
let blurScratchCanvas: HTMLCanvasElement | null = null;
|
let blurScratchCanvas: HTMLCanvasElement | null = null;
|
||||||
let blurScratchCtx: CanvasRenderingContext2D | null = null;
|
let blurScratchCtx: CanvasRenderingContext2D | null = null;
|
||||||
|
let magnifierScratchCanvas: HTMLCanvasElement | null = null;
|
||||||
|
let magnifierScratchCtx: CanvasRenderingContext2D | null = null;
|
||||||
|
|
||||||
// Matches a single code point whose script is Han (including non-BMP
|
// Matches a single code point whose script is Han (including non-BMP
|
||||||
// Extension A-F), Hiragana, Katakana (including halfwidth forms), or
|
// Extension A-F), Hiragana, Katakana (including halfwidth forms), or
|
||||||
@@ -396,6 +398,130 @@ async function renderImage(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function renderMagnifier(
|
||||||
|
ctx: CanvasRenderingContext2D,
|
||||||
|
annotation: AnnotationRegion,
|
||||||
|
x: number,
|
||||||
|
y: number,
|
||||||
|
width: number,
|
||||||
|
height: number,
|
||||||
|
canvasWidth: number,
|
||||||
|
canvasHeight: number,
|
||||||
|
scaleFactor: number,
|
||||||
|
) {
|
||||||
|
if (!magnifierScratchCanvas || !magnifierScratchCtx) {
|
||||||
|
magnifierScratchCanvas = document.createElement("canvas");
|
||||||
|
magnifierScratchCtx = magnifierScratchCanvas.getContext("2d");
|
||||||
|
}
|
||||||
|
if (!magnifierScratchCanvas || !magnifierScratchCtx) return;
|
||||||
|
|
||||||
|
const data = annotation.magnifierData;
|
||||||
|
const zoom = Math.max(1, data?.zoom ?? 2.2);
|
||||||
|
const target = data?.target ?? {
|
||||||
|
x: annotation.position.x + annotation.size.width / 2,
|
||||||
|
y: annotation.position.y + annotation.size.height / 2,
|
||||||
|
};
|
||||||
|
const targetX = (target.x / 100) * canvasWidth;
|
||||||
|
const targetY = (target.y / 100) * canvasHeight;
|
||||||
|
const sampleWidth = Math.max(1, width / zoom);
|
||||||
|
const sampleHeight = Math.max(1, height / zoom);
|
||||||
|
const sx = Math.max(0, Math.min(canvasWidth - sampleWidth, targetX - sampleWidth / 2));
|
||||||
|
const sy = Math.max(0, Math.min(canvasHeight - sampleHeight, targetY - sampleHeight / 2));
|
||||||
|
const sw = Math.max(1, Math.min(sampleWidth, canvasWidth - sx));
|
||||||
|
const sh = Math.max(1, Math.min(sampleHeight, canvasHeight - sy));
|
||||||
|
|
||||||
|
magnifierScratchCanvas.width = Math.max(1, Math.round(width));
|
||||||
|
magnifierScratchCanvas.height = Math.max(1, Math.round(height));
|
||||||
|
magnifierScratchCtx.clearRect(0, 0, magnifierScratchCanvas.width, magnifierScratchCanvas.height);
|
||||||
|
magnifierScratchCtx.imageSmoothingEnabled = true;
|
||||||
|
magnifierScratchCtx.imageSmoothingQuality = "high";
|
||||||
|
magnifierScratchCtx.drawImage(
|
||||||
|
ctx.canvas,
|
||||||
|
sx,
|
||||||
|
sy,
|
||||||
|
sw,
|
||||||
|
sh,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
magnifierScratchCanvas.width,
|
||||||
|
magnifierScratchCanvas.height,
|
||||||
|
);
|
||||||
|
|
||||||
|
const centerX = x + width / 2;
|
||||||
|
const centerY = y + height / 2;
|
||||||
|
const shape = data?.shape ?? "circle";
|
||||||
|
const radius = Math.min(width, height) / 2;
|
||||||
|
const cornerRadius = shape === "circle" ? radius : Math.min(18 * scaleFactor, radius);
|
||||||
|
|
||||||
|
ctx.save();
|
||||||
|
ctx.strokeStyle = "rgba(52,178,123,0.85)";
|
||||||
|
ctx.lineWidth = Math.max(2, 2 * scaleFactor);
|
||||||
|
ctx.beginPath();
|
||||||
|
ctx.moveTo(centerX, centerY);
|
||||||
|
ctx.lineTo(targetX, targetY);
|
||||||
|
ctx.stroke();
|
||||||
|
ctx.fillStyle = "#34B27B";
|
||||||
|
ctx.beginPath();
|
||||||
|
ctx.arc(targetX, targetY, Math.max(4, 4 * scaleFactor), 0, Math.PI * 2);
|
||||||
|
ctx.fill();
|
||||||
|
|
||||||
|
ctx.shadowColor = "rgba(0,0,0,0.38)";
|
||||||
|
ctx.shadowBlur = 24 * scaleFactor;
|
||||||
|
ctx.shadowOffsetY = 12 * scaleFactor;
|
||||||
|
ctx.fillStyle = "rgba(15,23,42,0.92)";
|
||||||
|
ctx.beginPath();
|
||||||
|
ctx.roundRect(x, y, width, height, cornerRadius);
|
||||||
|
ctx.fill();
|
||||||
|
ctx.restore();
|
||||||
|
|
||||||
|
ctx.save();
|
||||||
|
ctx.beginPath();
|
||||||
|
ctx.roundRect(x, y, width, height, cornerRadius);
|
||||||
|
ctx.clip();
|
||||||
|
ctx.drawImage(magnifierScratchCanvas, x, y, width, height);
|
||||||
|
ctx.restore();
|
||||||
|
|
||||||
|
ctx.save();
|
||||||
|
ctx.strokeStyle = "rgba(248,250,252,0.96)";
|
||||||
|
ctx.lineWidth = Math.max(3, 3 * scaleFactor);
|
||||||
|
ctx.beginPath();
|
||||||
|
ctx.roundRect(x, y, width, height, cornerRadius);
|
||||||
|
ctx.stroke();
|
||||||
|
ctx.strokeStyle = "rgba(52,178,123,0.58)";
|
||||||
|
ctx.lineWidth = Math.max(1, 1.5 * scaleFactor);
|
||||||
|
ctx.beginPath();
|
||||||
|
ctx.roundRect(
|
||||||
|
x + 2 * scaleFactor,
|
||||||
|
y + 2 * scaleFactor,
|
||||||
|
width - 4 * scaleFactor,
|
||||||
|
height - 4 * scaleFactor,
|
||||||
|
cornerRadius,
|
||||||
|
);
|
||||||
|
ctx.stroke();
|
||||||
|
|
||||||
|
const caption = data?.caption || "";
|
||||||
|
if (caption) {
|
||||||
|
const fontSize = Math.max(12, 13 * scaleFactor);
|
||||||
|
ctx.font = `bold ${fontSize}px Inter, sans-serif`;
|
||||||
|
ctx.textAlign = "center";
|
||||||
|
ctx.textBaseline = "middle";
|
||||||
|
const paddingX = 8 * scaleFactor;
|
||||||
|
const paddingY = 5 * scaleFactor;
|
||||||
|
const metrics = ctx.measureText(caption);
|
||||||
|
const captionWidth = Math.min(width * 1.6, metrics.width + paddingX * 2);
|
||||||
|
const captionHeight = fontSize + paddingY * 2;
|
||||||
|
const captionX = centerX - captionWidth / 2;
|
||||||
|
const captionY = y + height + 8 * scaleFactor;
|
||||||
|
ctx.fillStyle = "rgba(15,23,42,0.92)";
|
||||||
|
ctx.beginPath();
|
||||||
|
ctx.roundRect(captionX, captionY, captionWidth, captionHeight, 6 * scaleFactor);
|
||||||
|
ctx.fill();
|
||||||
|
ctx.fillStyle = "#f8fafc";
|
||||||
|
ctx.fillText(caption, centerX, captionY + captionHeight / 2, captionWidth - paddingX * 2);
|
||||||
|
}
|
||||||
|
ctx.restore();
|
||||||
|
}
|
||||||
|
|
||||||
export async function renderAnnotations(
|
export async function renderAnnotations(
|
||||||
ctx: CanvasRenderingContext2D,
|
ctx: CanvasRenderingContext2D,
|
||||||
annotations: AnnotationRegion[],
|
annotations: AnnotationRegion[],
|
||||||
@@ -443,6 +569,20 @@ export async function renderAnnotations(
|
|||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
|
|
||||||
|
case "magnifier":
|
||||||
|
renderMagnifier(
|
||||||
|
ctx,
|
||||||
|
annotation,
|
||||||
|
x,
|
||||||
|
y,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
canvasWidth,
|
||||||
|
canvasHeight,
|
||||||
|
scaleFactor,
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
|
||||||
case "blur":
|
case "blur":
|
||||||
renderBlur(ctx, annotation, x, y, width, height, scaleFactor);
|
renderBlur(ctx, annotation, x, y, width, height, scaleFactor);
|
||||||
break;
|
break;
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
export type UpdateStatusPhase =
|
||||||
|
| "idle"
|
||||||
|
| "checking"
|
||||||
|
| "available"
|
||||||
|
| "not-available"
|
||||||
|
| "downloading"
|
||||||
|
| "downloaded"
|
||||||
|
| "error"
|
||||||
|
| "unsupported";
|
||||||
|
|
||||||
|
export interface UpdateStatus {
|
||||||
|
phase: UpdateStatusPhase;
|
||||||
|
currentVersion: string;
|
||||||
|
version?: string;
|
||||||
|
releaseName?: string;
|
||||||
|
releaseNotes?: string;
|
||||||
|
percent?: number;
|
||||||
|
bytesPerSecond?: number;
|
||||||
|
transferred?: number;
|
||||||
|
total?: number;
|
||||||
|
error?: string;
|
||||||
|
updatedAt: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface UpdateCheckResult {
|
||||||
|
success: boolean;
|
||||||
|
status: UpdateStatus;
|
||||||
|
error?: string;
|
||||||
|
}
|
||||||
+327
-19
@@ -5,8 +5,9 @@ import importlib.util
|
|||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from threading import Lock
|
from threading import Lock, Thread
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from fastapi import FastAPI, HTTPException
|
from fastapi import FastAPI, HTTPException
|
||||||
@@ -17,6 +18,67 @@ app = FastAPI(title="OpenScreen PaddleOCR service")
|
|||||||
|
|
||||||
_engines: dict[str, Any] = {}
|
_engines: dict[str, Any] = {}
|
||||||
_engine_lock = Lock()
|
_engine_lock = Lock()
|
||||||
|
_warmup_lock = Lock()
|
||||||
|
_warmup_started = False
|
||||||
|
_LATIN_RECOGNITION_LANGS = {
|
||||||
|
"af",
|
||||||
|
"az",
|
||||||
|
"bs",
|
||||||
|
"ca",
|
||||||
|
"cs",
|
||||||
|
"cy",
|
||||||
|
"da",
|
||||||
|
"de",
|
||||||
|
"en",
|
||||||
|
"es",
|
||||||
|
"et",
|
||||||
|
"eu",
|
||||||
|
"fi",
|
||||||
|
"fr",
|
||||||
|
"ga",
|
||||||
|
"gl",
|
||||||
|
"hr",
|
||||||
|
"hu",
|
||||||
|
"id",
|
||||||
|
"is",
|
||||||
|
"it",
|
||||||
|
"ku",
|
||||||
|
"la",
|
||||||
|
"latin",
|
||||||
|
"lb",
|
||||||
|
"lt",
|
||||||
|
"lv",
|
||||||
|
"mi",
|
||||||
|
"ms",
|
||||||
|
"mt",
|
||||||
|
"nl",
|
||||||
|
"no",
|
||||||
|
"oc",
|
||||||
|
"pi",
|
||||||
|
"pl",
|
||||||
|
"pt",
|
||||||
|
"qu",
|
||||||
|
"rm",
|
||||||
|
"ro",
|
||||||
|
"rs_latin",
|
||||||
|
"rslatin",
|
||||||
|
"sk",
|
||||||
|
"sl",
|
||||||
|
"sq",
|
||||||
|
"sv",
|
||||||
|
"sw",
|
||||||
|
"tl",
|
||||||
|
"tr",
|
||||||
|
"uz",
|
||||||
|
"vi",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class PreparedImage:
|
||||||
|
path: str
|
||||||
|
scale: float = 1.0
|
||||||
|
should_delete: bool = False
|
||||||
|
|
||||||
|
|
||||||
class OcrRequest(BaseModel):
|
class OcrRequest(BaseModel):
|
||||||
@@ -24,6 +86,21 @@ class OcrRequest(BaseModel):
|
|||||||
path: str | None = None
|
path: str | None = None
|
||||||
imagePath: str | None = None
|
imagePath: str | None = None
|
||||||
language: str | None = None
|
language: str | None = None
|
||||||
|
profile: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
@app.on_event("startup")
|
||||||
|
def start_ocr_warmup() -> None:
|
||||||
|
if os.getenv("OPENSCREEN_OCR_WARMUP", "0") != "1":
|
||||||
|
return
|
||||||
|
|
||||||
|
global _warmup_started
|
||||||
|
with _warmup_lock:
|
||||||
|
if _warmup_started:
|
||||||
|
return
|
||||||
|
_warmup_started = True
|
||||||
|
|
||||||
|
Thread(target=_warmup_default_engines, name="openscreen-ocr-warmup", daemon=True).start()
|
||||||
|
|
||||||
|
|
||||||
@app.get("/health")
|
@app.get("/health")
|
||||||
@@ -33,16 +110,31 @@ def health() -> dict[str, Any]:
|
|||||||
"paddleocrInstalled": importlib.util.find_spec("paddleocr") is not None,
|
"paddleocrInstalled": importlib.util.find_spec("paddleocr") is not None,
|
||||||
"paddleInstalled": importlib.util.find_spec("paddle") is not None,
|
"paddleInstalled": importlib.util.find_spec("paddle") is not None,
|
||||||
"engineReady": bool(_engines),
|
"engineReady": bool(_engines),
|
||||||
"defaultLanguage": os.getenv("PADDLEOCR_LANG", "latin"),
|
"defaultLanguage": os.getenv("PADDLEOCR_LANG") or "vi,en",
|
||||||
|
"defaultProfile": os.getenv("OPENSCREEN_OCR_PROFILE") or "vietnamese",
|
||||||
|
"loadedEngines": sorted(_engines.keys()),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _warmup_default_engines() -> None:
|
||||||
|
try:
|
||||||
|
profile = _resolve_ocr_profile(None)
|
||||||
|
for paddle_lang in _resolve_paddle_languages(None, profile):
|
||||||
|
_get_engine(paddle_lang)
|
||||||
|
except Exception as error:
|
||||||
|
print(f"OpenScreen OCR warmup failed: {error}", file=sys.stderr, flush=True)
|
||||||
|
|
||||||
|
|
||||||
@app.post("/ocr")
|
@app.post("/ocr")
|
||||||
async def ocr(request: OcrRequest) -> dict[str, Any]:
|
async def ocr(request: OcrRequest) -> dict[str, Any]:
|
||||||
image_path, should_delete = _resolve_image_path(request)
|
image_path, should_delete = _resolve_image_path(request)
|
||||||
try:
|
try:
|
||||||
engine = _get_engine(request.language)
|
blocks = await run_in_threadpool(
|
||||||
blocks = await run_in_threadpool(_recognize_blocks, engine, image_path)
|
_recognize_profile_blocks,
|
||||||
|
image_path,
|
||||||
|
request.language,
|
||||||
|
request.profile,
|
||||||
|
)
|
||||||
return {"blocks": blocks}
|
return {"blocks": blocks}
|
||||||
finally:
|
finally:
|
||||||
if should_delete:
|
if should_delete:
|
||||||
@@ -73,8 +165,7 @@ def _resolve_image_path(request: OcrRequest) -> tuple[str, bool]:
|
|||||||
return handle.name, True
|
return handle.name, True
|
||||||
|
|
||||||
|
|
||||||
def _get_engine(language: str | None) -> Any:
|
def _get_engine(paddle_lang: str) -> Any:
|
||||||
paddle_lang = _resolve_paddle_language(language)
|
|
||||||
cache_key = f"{paddle_lang}|{os.getenv('PADDLEOCR_DEVICE', 'cpu')}"
|
cache_key = f"{paddle_lang}|{os.getenv('PADDLEOCR_DEVICE', 'cpu')}"
|
||||||
with _engine_lock:
|
with _engine_lock:
|
||||||
if cache_key not in _engines:
|
if cache_key not in _engines:
|
||||||
@@ -105,13 +196,17 @@ def _create_engine(paddle_lang: str) -> Any:
|
|||||||
"enable_mkldnn": os.getenv("PADDLEOCR_ENABLE_MKLDNN", "0") == "1",
|
"enable_mkldnn": os.getenv("PADDLEOCR_ENABLE_MKLDNN", "0") == "1",
|
||||||
"use_doc_orientation_classify": False,
|
"use_doc_orientation_classify": False,
|
||||||
"use_doc_unwarping": False,
|
"use_doc_unwarping": False,
|
||||||
"use_textline_orientation": False,
|
"use_textline_orientation": os.getenv("PADDLEOCR_USE_TEXTLINE_ORIENTATION", "0") == "1",
|
||||||
}
|
}
|
||||||
if os.getenv("PADDLEOCR_USE_MOBILE", "1") != "0":
|
if os.getenv("PADDLEOCR_USE_MOBILE", "1") != "0":
|
||||||
modern_kwargs.update(
|
modern_kwargs.update(
|
||||||
{
|
{
|
||||||
"text_detection_model_name": "PP-OCRv5_mobile_det",
|
"text_detection_model_name": os.getenv(
|
||||||
"text_recognition_model_name": _mobile_recognition_model(paddle_lang),
|
"PADDLEOCR_DET_MODEL",
|
||||||
|
"PP-OCRv5_mobile_det",
|
||||||
|
),
|
||||||
|
"text_recognition_model_name": os.getenv("PADDLEOCR_REC_MODEL")
|
||||||
|
or _mobile_recognition_model(paddle_lang),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -150,23 +245,236 @@ def _patch_paddlex_frozen_ocr_extra_gate() -> None:
|
|||||||
deps._openscreen_ocr_extra_patch = True
|
deps._openscreen_ocr_extra_patch = True
|
||||||
|
|
||||||
|
|
||||||
def _resolve_paddle_language(language: str | None) -> str:
|
def _recognize_profile_blocks(
|
||||||
explicit = os.getenv("PADDLEOCR_LANG")
|
image_path: str,
|
||||||
|
language: str | None,
|
||||||
|
profile: str | None,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
ocr_profile = _resolve_ocr_profile(profile)
|
||||||
|
languages = _resolve_paddle_languages(language, ocr_profile)
|
||||||
|
prepared = _prepare_image_for_profile(image_path, ocr_profile)
|
||||||
|
try:
|
||||||
|
blocks: list[dict[str, Any]] = []
|
||||||
|
for paddle_lang in languages:
|
||||||
|
engine = _get_engine(paddle_lang)
|
||||||
|
recognized = _recognize_blocks(engine, prepared.path)
|
||||||
|
blocks.extend(_scale_blocks(recognized, prepared.scale))
|
||||||
|
return _merge_blocks(blocks)
|
||||||
|
finally:
|
||||||
|
if prepared.should_delete:
|
||||||
|
Path(prepared.path).unlink(missing_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_ocr_profile(profile: str | None) -> str:
|
||||||
|
explicit = (os.getenv("OPENSCREEN_OCR_PROFILE") or "").strip().lower()
|
||||||
|
value = explicit or (profile or "").strip().lower()
|
||||||
|
if value in {"fast", "vietnamese", "hybrid"}:
|
||||||
|
return value
|
||||||
|
return "vietnamese"
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_paddle_languages(language: str | None, profile: str) -> list[str]:
|
||||||
|
explicit = (os.getenv("PADDLEOCR_LANG") or "").strip().lower()
|
||||||
if explicit:
|
if explicit:
|
||||||
return explicit
|
return [explicit]
|
||||||
|
|
||||||
language_value = (language or "vi,en").lower()
|
language_value = (language or "vi,en").lower()
|
||||||
if "vi" in language_value or "latin" in language_value:
|
has_vietnamese = "vi" in _split_language_tags(language_value)
|
||||||
|
if profile == "fast":
|
||||||
|
return [_resolve_primary_paddle_language(language_value, prefer_vietnamese=False)]
|
||||||
|
if profile == "hybrid":
|
||||||
|
languages = ["vi"] if has_vietnamese else []
|
||||||
|
languages.append("latin")
|
||||||
|
return _dedupe_languages(languages)
|
||||||
|
return [_resolve_primary_paddle_language(language_value, prefer_vietnamese=True)]
|
||||||
|
|
||||||
|
|
||||||
|
def _split_language_tags(language: str) -> set[str]:
|
||||||
|
return {part.strip().lower() for part in language.split(",") if part.strip()}
|
||||||
|
|
||||||
|
|
||||||
|
def _dedupe_languages(languages: list[str]) -> list[str]:
|
||||||
|
seen: set[str] = set()
|
||||||
|
result: list[str] = []
|
||||||
|
for language in languages:
|
||||||
|
if language not in seen:
|
||||||
|
seen.add(language)
|
||||||
|
result.append(language)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_primary_paddle_language(language_value: str, *, prefer_vietnamese: bool) -> str:
|
||||||
|
tags = _split_language_tags(language_value)
|
||||||
|
if prefer_vietnamese and "vi" in tags:
|
||||||
|
return "vi"
|
||||||
|
if "latin" in tags or "vi" in tags or "en" in tags:
|
||||||
return "latin"
|
return "latin"
|
||||||
if "en" in language_value:
|
for tag in tags:
|
||||||
return "en"
|
return tag
|
||||||
return language_value.split(",")[0].strip() or "latin"
|
return "latin"
|
||||||
|
|
||||||
|
|
||||||
|
def _prepare_image_for_profile(image_path: str, profile: str) -> PreparedImage:
|
||||||
|
if profile == "fast":
|
||||||
|
return PreparedImage(image_path)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from PIL import Image, ImageEnhance, ImageOps
|
||||||
|
except Exception:
|
||||||
|
return PreparedImage(image_path)
|
||||||
|
|
||||||
|
try:
|
||||||
|
with Image.open(image_path) as source:
|
||||||
|
image = source.convert("RGB")
|
||||||
|
except Exception:
|
||||||
|
return PreparedImage(image_path)
|
||||||
|
|
||||||
|
scale = _resolve_enhancement_scale(image.width, image.height)
|
||||||
|
if scale <= 1:
|
||||||
|
return PreparedImage(image_path)
|
||||||
|
|
||||||
|
resampling = getattr(getattr(Image, "Resampling", Image), "LANCZOS")
|
||||||
|
enhanced = image.resize((round(image.width * scale), round(image.height * scale)), resampling)
|
||||||
|
enhanced = ImageOps.autocontrast(enhanced)
|
||||||
|
enhanced = ImageEnhance.Contrast(enhanced).enhance(1.25)
|
||||||
|
enhanced = ImageEnhance.Sharpness(enhanced).enhance(1.35)
|
||||||
|
|
||||||
|
handle = tempfile.NamedTemporaryFile(prefix="openscreen-ocr-enhanced-", suffix=".png", delete=False)
|
||||||
|
try:
|
||||||
|
handle.close()
|
||||||
|
enhanced.save(handle.name, format="PNG")
|
||||||
|
return PreparedImage(handle.name, scale=scale, should_delete=True)
|
||||||
|
except Exception:
|
||||||
|
Path(handle.name).unlink(missing_ok=True)
|
||||||
|
return PreparedImage(image_path)
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_enhancement_scale(width: int, height: int) -> float:
|
||||||
|
try:
|
||||||
|
requested_scale = float(os.getenv("OPENSCREEN_OCR_ENHANCE_SCALE", "2"))
|
||||||
|
except ValueError:
|
||||||
|
requested_scale = 2.0
|
||||||
|
scale = max(1.0, min(3.0, requested_scale))
|
||||||
|
try:
|
||||||
|
max_side = int(os.getenv("OPENSCREEN_OCR_ENHANCE_MAX_SIDE", "2400"))
|
||||||
|
except ValueError:
|
||||||
|
max_side = 2400
|
||||||
|
largest_side = max(width, height)
|
||||||
|
if largest_side <= 0:
|
||||||
|
return 1.0
|
||||||
|
return max(1.0, min(scale, max_side / largest_side))
|
||||||
|
|
||||||
|
|
||||||
|
def _scale_blocks(blocks: list[dict[str, Any]], scale: float) -> list[dict[str, Any]]:
|
||||||
|
if scale <= 1:
|
||||||
|
return blocks
|
||||||
|
|
||||||
|
scaled_blocks: list[dict[str, Any]] = []
|
||||||
|
for block in blocks:
|
||||||
|
box = block.get("box")
|
||||||
|
if not isinstance(box, dict) or not _box_uses_pixels(box):
|
||||||
|
scaled_blocks.append(block)
|
||||||
|
continue
|
||||||
|
scaled_box = {
|
||||||
|
"x": float(box["x"]) / scale,
|
||||||
|
"y": float(box["y"]) / scale,
|
||||||
|
"width": float(box["width"]) / scale,
|
||||||
|
"height": float(box["height"]) / scale,
|
||||||
|
}
|
||||||
|
scaled_blocks.append({**block, "box": scaled_box})
|
||||||
|
return scaled_blocks
|
||||||
|
|
||||||
|
|
||||||
|
def _box_uses_pixels(box: dict[str, Any]) -> bool:
|
||||||
|
try:
|
||||||
|
x = float(box["x"])
|
||||||
|
y = float(box["y"])
|
||||||
|
width = float(box["width"])
|
||||||
|
height = float(box["height"])
|
||||||
|
except (KeyError, TypeError, ValueError):
|
||||||
|
return False
|
||||||
|
return x > 1 or y > 1 or width > 1 or height > 1 or x + width > 1 or y + height > 1
|
||||||
|
|
||||||
|
|
||||||
|
def _merge_blocks(blocks: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||||
|
merged: list[dict[str, Any]] = []
|
||||||
|
for block in sorted(blocks, key=_block_quality, reverse=True):
|
||||||
|
box = block.get("box")
|
||||||
|
if not isinstance(box, dict):
|
||||||
|
continue
|
||||||
|
overlapping_index = next(
|
||||||
|
(
|
||||||
|
index
|
||||||
|
for index, existing in enumerate(merged)
|
||||||
|
if _box_iou(box, existing.get("box")) >= 0.62
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if overlapping_index is None:
|
||||||
|
merged.append(block)
|
||||||
|
continue
|
||||||
|
if _block_quality(block) > _block_quality(merged[overlapping_index]):
|
||||||
|
merged[overlapping_index] = block
|
||||||
|
return sorted(merged, key=lambda block: _box_sort_key(block.get("box")))
|
||||||
|
|
||||||
|
|
||||||
|
def _block_quality(block: dict[str, Any]) -> float:
|
||||||
|
text = str(block.get("text") or "")
|
||||||
|
score = _score_to_float(block.get("confidence"))
|
||||||
|
if _has_vietnamese_diacritics(text):
|
||||||
|
score += 0.08
|
||||||
|
if len(text) >= 2:
|
||||||
|
score += min(0.04, len(text) * 0.002)
|
||||||
|
return score
|
||||||
|
|
||||||
|
|
||||||
|
def _has_vietnamese_diacritics(text: str) -> bool:
|
||||||
|
return any(
|
||||||
|
character
|
||||||
|
in "ăâđêôơưĂÂĐÊÔƠƯáàảãạắằẳẵặấầẩẫậéèẻẽẹếềểễệíìỉĩịóòỏõọốồổỗộớờởỡợúùủũụứừửữựýỳỷỹỵ"
|
||||||
|
for character in text
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _box_iou(left: Any, right: Any) -> float:
|
||||||
|
if not isinstance(left, dict) or not isinstance(right, dict):
|
||||||
|
return 0.0
|
||||||
|
try:
|
||||||
|
left_x = float(left["x"])
|
||||||
|
left_y = float(left["y"])
|
||||||
|
left_width = float(left["width"])
|
||||||
|
left_height = float(left["height"])
|
||||||
|
right_x = float(right["x"])
|
||||||
|
right_y = float(right["y"])
|
||||||
|
right_width = float(right["width"])
|
||||||
|
right_height = float(right["height"])
|
||||||
|
except (KeyError, TypeError, ValueError):
|
||||||
|
return 0.0
|
||||||
|
|
||||||
|
intersection_left = max(left_x, right_x)
|
||||||
|
intersection_top = max(left_y, right_y)
|
||||||
|
intersection_right = min(left_x + left_width, right_x + right_width)
|
||||||
|
intersection_bottom = min(left_y + left_height, right_y + right_height)
|
||||||
|
intersection_width = max(0.0, intersection_right - intersection_left)
|
||||||
|
intersection_height = max(0.0, intersection_bottom - intersection_top)
|
||||||
|
intersection_area = intersection_width * intersection_height
|
||||||
|
if intersection_area <= 0:
|
||||||
|
return 0.0
|
||||||
|
union_area = left_width * left_height + right_width * right_height - intersection_area
|
||||||
|
return intersection_area / union_area if union_area > 0 else 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def _box_sort_key(box: Any) -> tuple[float, float]:
|
||||||
|
if not isinstance(box, dict):
|
||||||
|
return (0.0, 0.0)
|
||||||
|
try:
|
||||||
|
return (float(box["y"]), float(box["x"]))
|
||||||
|
except (KeyError, TypeError, ValueError):
|
||||||
|
return (0.0, 0.0)
|
||||||
|
|
||||||
|
|
||||||
def _mobile_recognition_model(paddle_lang: str) -> str:
|
def _mobile_recognition_model(paddle_lang: str) -> str:
|
||||||
if paddle_lang == "en":
|
if paddle_lang in _LATIN_RECOGNITION_LANGS:
|
||||||
return "en_PP-OCRv5_mobile_rec"
|
|
||||||
if paddle_lang == "latin":
|
|
||||||
return "latin_PP-OCRv5_mobile_rec"
|
return "latin_PP-OCRv5_mobile_rec"
|
||||||
return "PP-OCRv5_mobile_rec"
|
return "PP-OCRv5_mobile_rec"
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user