Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| aae562f146 | |||
| 5069354df3 | |||
| ee69df9222 | |||
| a235a0c50b | |||
| 94490a71af | |||
| 6ebabbaaaa |
+11
-5
@@ -13,11 +13,17 @@
|
||||
},
|
||||
"npmRebuild": true,
|
||||
"buildDependenciesFromSource": true,
|
||||
"compression": "normal",
|
||||
"directories": {
|
||||
"output": "release/${version}"
|
||||
},
|
||||
"files": [
|
||||
"compression": "normal",
|
||||
"directories": {
|
||||
"output": "release/${version}"
|
||||
},
|
||||
"publish": [
|
||||
{
|
||||
"provider": "generic",
|
||||
"url": "https://gittea.softs.business/huanld/openscreen/raw/branch/release-assets%2Flatest"
|
||||
}
|
||||
],
|
||||
"files": [
|
||||
"dist",
|
||||
"dist-electron",
|
||||
"!*.png",
|
||||
|
||||
Vendored
+15
@@ -24,6 +24,14 @@ declare namespace NodeJS {
|
||||
// Used in Renderer process, expose in `preload.ts`
|
||||
interface Window {
|
||||
electronAPI: {
|
||||
updates: {
|
||||
getStatus: () => Promise<import("../src/lib/updateStatus").UpdateStatus>;
|
||||
check: () => Promise<import("../src/lib/updateStatus").UpdateCheckResult>;
|
||||
install: () => Promise<import("../src/lib/updateStatus").UpdateCheckResult>;
|
||||
onStatus: (
|
||||
callback: (status: import("../src/lib/updateStatus").UpdateStatus) => void,
|
||||
) => () => void;
|
||||
};
|
||||
invokeNativeBridge: <TData = unknown>(
|
||||
request: import("../src/native/contracts").NativeBridgeRequest,
|
||||
) => Promise<import("../src/native/contracts").NativeBridgeResponse<TData>>;
|
||||
@@ -357,6 +365,13 @@ interface Window {
|
||||
showCountdownOverlay: (value: number, runId: number) => Promise<void>;
|
||||
setCountdownOverlayValue: (value: number, runId: number) => Promise<void>;
|
||||
hideCountdownOverlay: (runId: number) => Promise<void>;
|
||||
onMcpControlRequest: (
|
||||
callback: (
|
||||
request: import("../src/lib/mcpControl").McpControlRequest,
|
||||
) =>
|
||||
| Promise<import("../src/lib/mcpControl").McpControlResult>
|
||||
| import("../src/lib/mcpControl").McpControlResult,
|
||||
) => () => void;
|
||||
onCountdownOverlayValue: (callback: (value: number | null) => void) => () => void;
|
||||
setMicrophoneExpanded: (expanded: boolean) => void;
|
||||
setHasUnsavedChanges: (hasChanges: boolean) => void;
|
||||
|
||||
+2
-195
@@ -1,11 +1,10 @@
|
||||
import { type ChildProcessWithoutNullStreams, execFile, spawn } from "node:child_process";
|
||||
import { type ChildProcessWithoutNullStreams, spawn } from "node:child_process";
|
||||
import { EventEmitter } from "node:events";
|
||||
import { constants as fsConstants } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import type { DesktopCapturerSource, Rectangle } from "electron";
|
||||
import {
|
||||
app,
|
||||
@@ -18,7 +17,7 @@ import {
|
||||
shell,
|
||||
systemPreferences,
|
||||
} from "electron";
|
||||
import type { GuideEvent, GuideMarkerCapturedPayload } from "../../src/guide/contracts";
|
||||
import type { GuideMarkerCapturedPayload } from "../../src/guide/contracts";
|
||||
import type { NativeMacRecordingRequest } from "../../src/lib/nativeMacRecording";
|
||||
import type { NativeWindowsRecordingRequest } from "../../src/lib/nativeWindowsRecording";
|
||||
import {
|
||||
@@ -57,7 +56,6 @@ const RECORDING_SESSION_SUFFIX = ".session.json";
|
||||
const ALLOWED_IMPORT_VIDEO_EXTENSIONS = new Set([".webm", ".mp4", ".mov", ".avi", ".mkv"]);
|
||||
const PREVIEW_AUDIO_DIR = path.join(app.getPath("userData"), "preview-audio");
|
||||
const nativeMacCaptureEvents = new EventEmitter();
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
/**
|
||||
* Paths explicitly approved by the user via file picker dialogs or project loads.
|
||||
@@ -456,7 +454,6 @@ let activeGuideHotkeyRecording: GuideHotkeyRecordingState | null = null;
|
||||
let activeGuideHotkeySessionId: number | null = null;
|
||||
let guideMarkerHotkeyRegistered = false;
|
||||
let lastGuideHotkeyCaptureAtMs = 0;
|
||||
const guideHotkeyBackgroundJobs = new Map<string, Promise<void>>();
|
||||
const GUIDE_HOTKEY_CAPTURE_DEBOUNCE_MS = 250;
|
||||
|
||||
function normalizeCursorSample(sample: unknown): CursorRecordingSample | null {
|
||||
@@ -811,195 +808,6 @@ function clampGuideHotkey01(value: number): number {
|
||||
return Math.min(1, Math.max(0, value));
|
||||
}
|
||||
|
||||
async function captureGuideHotkeySnapshotAndRunOcr(
|
||||
guideStore: GuideStore,
|
||||
event: GuideEvent,
|
||||
boundsInput: GuideHotkeyBounds,
|
||||
point: { normalizedX: number; normalizedY: number },
|
||||
) {
|
||||
try {
|
||||
const bounds = sanitizeGuideHotkeyBounds(boundsInput);
|
||||
const sources = await desktopCapturer.getSources({
|
||||
types: ["screen"],
|
||||
thumbnailSize: {
|
||||
width: Math.max(1, Math.round(bounds.width)),
|
||||
height: Math.max(1, Math.round(bounds.height)),
|
||||
},
|
||||
});
|
||||
const source = findScreenSourceForGuideBounds(sources, bounds);
|
||||
if (!source || source.thumbnail.isEmpty()) {
|
||||
console.warn("[guide-hotkey] no live screen thumbnail was available for OCR");
|
||||
return;
|
||||
}
|
||||
|
||||
const pngBuffer = source.thumbnail.toPNG();
|
||||
const imageSize = source.thumbnail.getSize();
|
||||
const markedPngBuffer = await createMarkedGuideSnapshotPng(pngBuffer, {
|
||||
width: imageSize.width,
|
||||
height: imageSize.height,
|
||||
x: point.normalizedX * imageSize.width,
|
||||
y: point.normalizedY * imageSize.height,
|
||||
}).catch((error) => {
|
||||
console.warn("[guide-hotkey] failed to create marked live snapshot:", error);
|
||||
return undefined;
|
||||
});
|
||||
|
||||
enqueueGuideHotkeyBackgroundJob(event.recordingId, async () => {
|
||||
const session = await guideStore.writeSnapshot({
|
||||
recordingId: event.recordingId,
|
||||
eventId: event.id,
|
||||
timeMs: event.timeMs,
|
||||
offsetMs: 0,
|
||||
pngBytes: bufferToArrayBuffer(pngBuffer),
|
||||
markedPngBytes: markedPngBuffer ? bufferToArrayBuffer(markedPngBuffer) : undefined,
|
||||
width: imageSize.width,
|
||||
height: imageSize.height,
|
||||
});
|
||||
const snapshot = session.snapshots.find((item) => item.eventId === event.id);
|
||||
if (!snapshot) {
|
||||
return;
|
||||
}
|
||||
|
||||
await guideStore.runOcr({
|
||||
recordingId: event.recordingId,
|
||||
snapshotIds: [snapshot.id],
|
||||
});
|
||||
console.info("[guide-hotkey] live snapshot OCR completed", {
|
||||
recordingId: event.recordingId,
|
||||
eventId: event.id,
|
||||
snapshotId: snapshot.id,
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
console.warn("[guide-hotkey] live snapshot OCR failed:", error);
|
||||
}
|
||||
}
|
||||
|
||||
function enqueueGuideHotkeyBackgroundJob(recordingId: string, job: () => Promise<void>) {
|
||||
const previousJob =
|
||||
guideHotkeyBackgroundJobs.get(recordingId)?.catch(() => undefined) ?? Promise.resolve();
|
||||
const nextJob = previousJob
|
||||
.then(job)
|
||||
.catch((error) => {
|
||||
console.warn("[guide-hotkey] background OCR job failed:", error);
|
||||
})
|
||||
.finally(() => {
|
||||
if (guideHotkeyBackgroundJobs.get(recordingId) === nextJob) {
|
||||
guideHotkeyBackgroundJobs.delete(recordingId);
|
||||
}
|
||||
});
|
||||
guideHotkeyBackgroundJobs.set(recordingId, nextJob);
|
||||
}
|
||||
|
||||
function findScreenSourceForGuideBounds(
|
||||
sources: DesktopCapturerSource[],
|
||||
bounds: GuideHotkeyBounds,
|
||||
): DesktopCapturerSource | undefined {
|
||||
const displays = screen.getAllDisplays();
|
||||
const displayIndex = displays.findIndex((display) =>
|
||||
rectMatchesGuideBounds(display.bounds, bounds),
|
||||
);
|
||||
const display = displayIndex >= 0 ? displays[displayIndex] : undefined;
|
||||
if (display) {
|
||||
const byDisplayId = sources.find((source) => Number(source.display_id) === display.id);
|
||||
if (byDisplayId) {
|
||||
return byDisplayId;
|
||||
}
|
||||
const bySourceIndex = sources.find(
|
||||
(source) => parseDesktopCapturerScreenIndex(source.id) === displayIndex,
|
||||
);
|
||||
if (bySourceIndex) {
|
||||
return bySourceIndex;
|
||||
}
|
||||
}
|
||||
return sources.find((source) => source.id.startsWith("screen:")) ?? sources[0];
|
||||
}
|
||||
|
||||
function rectMatchesGuideBounds(rect: Rectangle, bounds: GuideHotkeyBounds): boolean {
|
||||
return (
|
||||
Math.round(rect.x) === Math.round(bounds.x) &&
|
||||
Math.round(rect.y) === Math.round(bounds.y) &&
|
||||
Math.round(rect.width) === Math.round(bounds.width) &&
|
||||
Math.round(rect.height) === Math.round(bounds.height)
|
||||
);
|
||||
}
|
||||
|
||||
async function createMarkedGuideSnapshotPng(
|
||||
pngBuffer: Buffer,
|
||||
marker: { width: number; height: number; x: number; y: number },
|
||||
): Promise<Buffer> {
|
||||
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openscreen-guide-marker-"));
|
||||
const sourcePath = path.join(tempDir, "source.png");
|
||||
const outputPath = path.join(tempDir, "marked.png");
|
||||
try {
|
||||
await fs.writeFile(sourcePath, pngBuffer);
|
||||
await execFileAsync(
|
||||
"powershell.exe",
|
||||
[
|
||||
"-NoProfile",
|
||||
"-ExecutionPolicy",
|
||||
"Bypass",
|
||||
"-EncodedCommand",
|
||||
buildMarkerScript(sourcePath, outputPath, marker),
|
||||
],
|
||||
{
|
||||
timeout: 30000,
|
||||
windowsHide: true,
|
||||
maxBuffer: 1024 * 1024,
|
||||
},
|
||||
);
|
||||
return await fs.readFile(outputPath);
|
||||
} finally {
|
||||
await fs.rm(tempDir, { recursive: true, force: true }).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
function buildMarkerScript(
|
||||
sourcePath: string,
|
||||
outputPath: string,
|
||||
marker: { width: number; height: number; x: number; y: number },
|
||||
): string {
|
||||
const sourcePathBase64 = Buffer.from(sourcePath, "utf8").toString("base64");
|
||||
const outputPathBase64 = Buffer.from(outputPath, "utf8").toString("base64");
|
||||
const script = `
|
||||
$ErrorActionPreference = "Stop"
|
||||
$sourcePath = [System.Text.Encoding]::UTF8.GetString([Convert]::FromBase64String("${sourcePathBase64}"))
|
||||
$outputPath = [System.Text.Encoding]::UTF8.GetString([Convert]::FromBase64String("${outputPathBase64}"))
|
||||
Add-Type -AssemblyName System.Drawing
|
||||
|
||||
$source = [System.Drawing.Image]::FromFile($sourcePath)
|
||||
$bitmap = [System.Drawing.Bitmap]::new($source.Width, $source.Height)
|
||||
$graphics = [System.Drawing.Graphics]::FromImage($bitmap)
|
||||
try {
|
||||
$graphics.SmoothingMode = [System.Drawing.Drawing2D.SmoothingMode]::AntiAlias
|
||||
$graphics.DrawImage($source, 0, 0, $source.Width, $source.Height)
|
||||
$shortSide = [Math]::Max(1, [Math]::Min($source.Width, $source.Height))
|
||||
$dotRadius = [Math]::Min(7, [Math]::Max(4, [Math]::Round($shortSide * 0.005)))
|
||||
$x = [Math]::Min($source.Width, [Math]::Max(0, ${marker.x.toFixed(4)}))
|
||||
$y = [Math]::Min($source.Height, [Math]::Max(0, ${marker.y.toFixed(4)}))
|
||||
$dotBrush = [System.Drawing.SolidBrush]::new([System.Drawing.Color]::FromArgb(235, 220, 38, 38))
|
||||
try {
|
||||
$graphics.FillEllipse($dotBrush, $x - $dotRadius, $y - $dotRadius, $dotRadius * 2, $dotRadius * 2)
|
||||
} finally {
|
||||
$dotBrush.Dispose()
|
||||
}
|
||||
$bitmap.Save($outputPath, [System.Drawing.Imaging.ImageFormat]::Png)
|
||||
} finally {
|
||||
$graphics.Dispose()
|
||||
$bitmap.Dispose()
|
||||
$source.Dispose()
|
||||
}
|
||||
`;
|
||||
return Buffer.from(script, "utf16le").toString("base64");
|
||||
}
|
||||
|
||||
function bufferToArrayBuffer(buffer: Buffer): ArrayBuffer {
|
||||
return buffer.buffer.slice(
|
||||
buffer.byteOffset,
|
||||
buffer.byteOffset + buffer.byteLength,
|
||||
) as ArrayBuffer;
|
||||
}
|
||||
|
||||
async function captureGuideHotkeyMarker(
|
||||
guideStore: GuideStore,
|
||||
trigger: GuideMarkerTrigger = "global-shortcut",
|
||||
@@ -1046,7 +854,6 @@ async function captureGuideHotkeyMarker(
|
||||
rawY: point.rawY,
|
||||
bounds: point.bounds,
|
||||
});
|
||||
void captureGuideHotkeySnapshotAndRunOcr(guideStore, result.event, recording.bounds, point);
|
||||
return { captured: true, ...result };
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
|
||||
+14
-1
@@ -13,6 +13,8 @@ import {
|
||||
} from "electron";
|
||||
import { mainT, setMainLocale } from "./i18n";
|
||||
import { getSelectedDesktopSource, registerIpcHandlers } from "./ipc/handlers";
|
||||
import { startMcpControlServer } from "./mcpControlServer";
|
||||
import { initializeAutoUpdates } from "./updater";
|
||||
import {
|
||||
createCountdownOverlayWindow,
|
||||
createEditorWindow,
|
||||
@@ -515,9 +517,9 @@ app.whenReady().then(async () => {
|
||||
createTray();
|
||||
updateTrayMenu();
|
||||
setupApplicationMenu();
|
||||
initializeAutoUpdates();
|
||||
// Ensure recordings directory exists
|
||||
await ensureRecordingsDir();
|
||||
|
||||
function switchToHudWrapper() {
|
||||
if (mainWindow) {
|
||||
isForceClosing = true;
|
||||
@@ -528,6 +530,17 @@ app.whenReady().then(async () => {
|
||||
showMainWindow();
|
||||
}
|
||||
|
||||
startMcpControlServer({
|
||||
getMainWindow: () => mainWindow,
|
||||
ensureWindow: (action) => {
|
||||
if (action === "list_sources" || action === "record_video") {
|
||||
switchToHudWrapper();
|
||||
return;
|
||||
}
|
||||
showMainWindow();
|
||||
},
|
||||
});
|
||||
|
||||
registerIpcHandlers(
|
||||
createEditorWindowWrapper,
|
||||
createSourceSelectorWindowWrapper,
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { app, type BrowserWindow, ipcMain } from "electron";
|
||||
import type { McpControlAction, McpControlRequest, McpControlResult } from "../src/lib/mcpControl";
|
||||
import { isMcpControlAction } from "../src/lib/mcpControl";
|
||||
|
||||
const DEFAULT_MCP_CONTROL_PORT = 52347;
|
||||
const MCP_CONTROL_REQUEST_TIMEOUT_MS = 120_000;
|
||||
|
||||
type GetWindow = () => BrowserWindow | null;
|
||||
|
||||
interface PendingMcpRequest {
|
||||
resolve: (result: McpControlResult) => void;
|
||||
reject: (error: Error) => void;
|
||||
timeout: NodeJS.Timeout;
|
||||
}
|
||||
|
||||
interface StartMcpControlServerOptions {
|
||||
getMainWindow: GetWindow;
|
||||
ensureWindow: (action: McpControlAction) => void;
|
||||
}
|
||||
|
||||
const pendingRequests = new Map<string, PendingMcpRequest>();
|
||||
|
||||
function sendJson(response: ServerResponse, statusCode: number, body: unknown) {
|
||||
response.writeHead(statusCode, {
|
||||
"content-type": "application/json; charset=utf-8",
|
||||
});
|
||||
response.end(JSON.stringify(body));
|
||||
}
|
||||
|
||||
async function readJsonBody(request: IncomingMessage) {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of request) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
}
|
||||
const text = Buffer.concat(chunks).toString("utf-8").trim();
|
||||
return text ? JSON.parse(text) : {};
|
||||
}
|
||||
|
||||
function withFileUrl(result: McpControlResult): McpControlResult {
|
||||
if (!result.success || result.url || !result.path) {
|
||||
return result;
|
||||
}
|
||||
|
||||
return {
|
||||
...result,
|
||||
url: pathToFileURL(result.path).toString(),
|
||||
};
|
||||
}
|
||||
|
||||
function resolveDefaultExportPath(payload: unknown) {
|
||||
if (
|
||||
!payload ||
|
||||
typeof payload !== "object" ||
|
||||
("outputPath" in payload && typeof payload.outputPath === "string" && payload.outputPath)
|
||||
) {
|
||||
return payload;
|
||||
}
|
||||
|
||||
const requested = payload as { outputPath?: unknown; settings?: { format?: unknown } };
|
||||
const format = requested.settings?.format === "gif" ? "gif" : "mp4";
|
||||
const fileName = `openscreen-export-${Date.now()}.${format}`;
|
||||
return {
|
||||
...requested,
|
||||
outputPath: path.join(app.getPath("downloads"), fileName),
|
||||
};
|
||||
}
|
||||
|
||||
function dispatchRendererRequest(
|
||||
window: BrowserWindow,
|
||||
action: McpControlAction,
|
||||
payload: unknown,
|
||||
): Promise<McpControlResult> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const id = `${Date.now()}-${Math.random().toString(16).slice(2)}`;
|
||||
const timeout = setTimeout(() => {
|
||||
pendingRequests.delete(id);
|
||||
reject(new Error(`Timed out waiting for renderer to handle ${action}`));
|
||||
}, MCP_CONTROL_REQUEST_TIMEOUT_MS);
|
||||
|
||||
pendingRequests.set(id, { resolve, reject, timeout });
|
||||
const request: McpControlRequest = { id, action, payload };
|
||||
window.webContents.send("mcp-control-request", request);
|
||||
});
|
||||
}
|
||||
|
||||
function registerMcpControlIpc() {
|
||||
ipcMain.on(
|
||||
"mcp-control-response",
|
||||
(_event, response: { id?: unknown; result?: McpControlResult }) => {
|
||||
if (typeof response.id !== "string") {
|
||||
return;
|
||||
}
|
||||
const pending = pendingRequests.get(response.id);
|
||||
if (!pending) {
|
||||
return;
|
||||
}
|
||||
clearTimeout(pending.timeout);
|
||||
pendingRequests.delete(response.id);
|
||||
pending.resolve(
|
||||
response.result ?? { success: false, error: "Renderer returned an empty result" },
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export function startMcpControlServer({
|
||||
getMainWindow,
|
||||
ensureWindow,
|
||||
}: StartMcpControlServerOptions): Server {
|
||||
registerMcpControlIpc();
|
||||
|
||||
const token = process.env.OPENSCREEN_MCP_CONTROL_TOKEN;
|
||||
const port = Number(process.env.OPENSCREEN_MCP_CONTROL_PORT) || DEFAULT_MCP_CONTROL_PORT;
|
||||
|
||||
const server = createServer(async (request, response) => {
|
||||
try {
|
||||
const url = new URL(request.url ?? "/", "http://127.0.0.1");
|
||||
if (request.method === "GET" && url.pathname === "/health") {
|
||||
sendJson(response, 200, { success: true, app: "OpenScreen" });
|
||||
return;
|
||||
}
|
||||
|
||||
if (request.method !== "POST" || !url.pathname.startsWith("/mcp/")) {
|
||||
sendJson(response, 404, { success: false, error: "Not found" });
|
||||
return;
|
||||
}
|
||||
|
||||
if (token && request.headers.authorization !== `Bearer ${token}`) {
|
||||
sendJson(response, 401, { success: false, error: "Unauthorized" });
|
||||
return;
|
||||
}
|
||||
|
||||
const action = url.pathname.slice("/mcp/".length);
|
||||
if (!isMcpControlAction(action)) {
|
||||
sendJson(response, 400, { success: false, error: `Unsupported MCP action: ${action}` });
|
||||
return;
|
||||
}
|
||||
|
||||
const body = await readJsonBody(request);
|
||||
const payload = action === "export_video" ? resolveDefaultExportPath(body) : body;
|
||||
|
||||
ensureWindow(action);
|
||||
const window = getMainWindow();
|
||||
if (!window || window.isDestroyed()) {
|
||||
sendJson(response, 503, { success: false, error: "OpenScreen window is not available" });
|
||||
return;
|
||||
}
|
||||
|
||||
if (window.webContents.isLoading()) {
|
||||
await new Promise<void>((resolve) => {
|
||||
window.webContents.once("did-finish-load", () => resolve());
|
||||
});
|
||||
}
|
||||
|
||||
const result = withFileUrl(await dispatchRendererRequest(window, action, payload));
|
||||
sendJson(response, result.success ? 200 : 409, result);
|
||||
} catch (error) {
|
||||
sendJson(response, 500, {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
server.listen(port, "127.0.0.1", () => {
|
||||
console.info(`[mcp-control] listening on http://127.0.0.1:${port}`);
|
||||
});
|
||||
server.on("error", (error) => {
|
||||
console.error("[mcp-control] failed to start:", error);
|
||||
});
|
||||
app.once("before-quit", () => {
|
||||
server.close();
|
||||
});
|
||||
|
||||
return server;
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import type {
|
||||
SaveGuideInput,
|
||||
WriteGuideSnapshotInput,
|
||||
} from "../src/guide/contracts";
|
||||
import type { McpControlRequest, McpControlResult } from "../src/lib/mcpControl";
|
||||
import type { NativeMacRecordingRequest } from "../src/lib/nativeMacRecording";
|
||||
import type { NativeWindowsRecordingRequest } from "../src/lib/nativeWindowsRecording";
|
||||
import type { RecordingSession, StoreRecordedSessionInput } from "../src/lib/recordingSession";
|
||||
@@ -26,6 +27,27 @@ const assetBaseUrl = assetBaseUrlArg ? assetBaseUrlArg.slice(ASSET_BASE_URL_ARG_
|
||||
|
||||
contextBridge.exposeInMainWorld("electronAPI", {
|
||||
assetBaseUrl,
|
||||
updates: {
|
||||
getStatus: () => {
|
||||
return ipcRenderer.invoke("updates:get-status");
|
||||
},
|
||||
check: () => {
|
||||
return ipcRenderer.invoke("updates:check");
|
||||
},
|
||||
install: () => {
|
||||
return ipcRenderer.invoke("updates:install");
|
||||
},
|
||||
onStatus: (callback: (status: import("../src/lib/updateStatus").UpdateStatus) => void) => {
|
||||
const listener = (
|
||||
_event: Electron.IpcRendererEvent,
|
||||
status: import("../src/lib/updateStatus").UpdateStatus,
|
||||
) => {
|
||||
callback(status);
|
||||
};
|
||||
ipcRenderer.on("updates:status", listener);
|
||||
return () => ipcRenderer.removeListener("updates:status", listener);
|
||||
},
|
||||
},
|
||||
invokeNativeBridge: <TData>(request: NativeBridgeRequest) => {
|
||||
return ipcRenderer.invoke(NATIVE_BRIDGE_CHANNEL, request) as Promise<TData>;
|
||||
},
|
||||
@@ -294,6 +316,26 @@ contextBridge.exposeInMainWorld("electronAPI", {
|
||||
hideCountdownOverlay: (runId: number) => {
|
||||
return ipcRenderer.invoke("countdown-overlay-hide", runId);
|
||||
},
|
||||
onMcpControlRequest: (
|
||||
callback: (request: McpControlRequest) => Promise<McpControlResult> | McpControlResult,
|
||||
) => {
|
||||
const listener = async (_event: unknown, request: McpControlRequest) => {
|
||||
try {
|
||||
const result = await callback(request);
|
||||
ipcRenderer.send("mcp-control-response", { id: request.id, result });
|
||||
} catch (error) {
|
||||
ipcRenderer.send("mcp-control-response", {
|
||||
id: request.id,
|
||||
result: {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
});
|
||||
}
|
||||
};
|
||||
ipcRenderer.on("mcp-control-request", listener);
|
||||
return () => ipcRenderer.removeListener("mcp-control-request", listener);
|
||||
},
|
||||
onCountdownOverlayValue: (callback: (value: number | null) => void) => {
|
||||
const listener = (_event: unknown, value: number | null) => callback(value);
|
||||
ipcRenderer.on("countdown-overlay-value", listener);
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
import { app, BrowserWindow, ipcMain } from "electron";
|
||||
import { autoUpdater, type ProgressInfo, type UpdateInfo } from "electron-updater";
|
||||
import type { UpdateCheckResult, UpdateStatus } from "../src/lib/updateStatus";
|
||||
|
||||
const DEFAULT_UPDATE_FEED_URL =
|
||||
"https://gittea.softs.business/huanld/openscreen/raw/branch/release-assets%2Flatest";
|
||||
const AUTO_CHECK_DELAY_MS = 10_000;
|
||||
const AUTO_CHECK_INTERVAL_MS = 4 * 60 * 60 * 1000;
|
||||
|
||||
let status: UpdateStatus = createStatus("idle");
|
||||
let handlersRegistered = false;
|
||||
let initialized = false;
|
||||
let checkInFlight: Promise<UpdateCheckResult> | null = null;
|
||||
|
||||
function createStatus(
|
||||
phase: UpdateStatus["phase"],
|
||||
patch: Partial<UpdateStatus> = {},
|
||||
): UpdateStatus {
|
||||
return {
|
||||
phase,
|
||||
currentVersion: app.getVersion(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
...patch,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeReleaseNotes(releaseNotes: UpdateInfo["releaseNotes"]): string | undefined {
|
||||
if (typeof releaseNotes === "string") {
|
||||
return releaseNotes;
|
||||
}
|
||||
if (Array.isArray(releaseNotes)) {
|
||||
return releaseNotes
|
||||
.map((note) => note.note)
|
||||
.filter(Boolean)
|
||||
.join("\n\n");
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function updateStatus(next: UpdateStatus) {
|
||||
status = next;
|
||||
for (const window of BrowserWindow.getAllWindows()) {
|
||||
if (!window.isDestroyed()) {
|
||||
window.webContents.send("updates:status", status);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function statusFromInfo(phase: UpdateStatus["phase"], info: UpdateInfo): UpdateStatus {
|
||||
return createStatus(phase, {
|
||||
version: info.version,
|
||||
releaseName: info.releaseName ?? undefined,
|
||||
releaseNotes: normalizeReleaseNotes(info.releaseNotes),
|
||||
});
|
||||
}
|
||||
|
||||
async function checkForUpdates(): Promise<UpdateCheckResult> {
|
||||
if (!initialized) {
|
||||
updateStatus(
|
||||
createStatus("unsupported", {
|
||||
error: "Update service is not initialized.",
|
||||
}),
|
||||
);
|
||||
return { success: false, status, error: status.error };
|
||||
}
|
||||
|
||||
if (!app.isPackaged && process.env.OPENSCREEN_ALLOW_DEV_UPDATE_CHECK !== "1") {
|
||||
updateStatus(
|
||||
createStatus("unsupported", {
|
||||
error: "Update checks only run in packaged builds.",
|
||||
}),
|
||||
);
|
||||
return { success: false, status, error: status.error };
|
||||
}
|
||||
|
||||
if (checkInFlight) {
|
||||
return checkInFlight;
|
||||
}
|
||||
|
||||
updateStatus(createStatus("checking"));
|
||||
checkInFlight = autoUpdater
|
||||
.checkForUpdates()
|
||||
.then(() => ({ success: true, status }))
|
||||
.catch((error) => {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
updateStatus(createStatus("error", { error: message }));
|
||||
return { success: false, status, error: message };
|
||||
})
|
||||
.finally(() => {
|
||||
checkInFlight = null;
|
||||
});
|
||||
|
||||
return checkInFlight;
|
||||
}
|
||||
|
||||
function registerUpdateIpcHandlers() {
|
||||
if (handlersRegistered) {
|
||||
return;
|
||||
}
|
||||
|
||||
handlersRegistered = true;
|
||||
ipcMain.handle("updates:get-status", () => status);
|
||||
ipcMain.handle("updates:check", () => checkForUpdates());
|
||||
ipcMain.handle("updates:install", () => {
|
||||
if (status.phase !== "downloaded") {
|
||||
return {
|
||||
success: false,
|
||||
status,
|
||||
error: "No downloaded update is ready to install.",
|
||||
};
|
||||
}
|
||||
setImmediate(() => autoUpdater.quitAndInstall(false, true));
|
||||
return { success: true, status };
|
||||
});
|
||||
}
|
||||
|
||||
export function initializeAutoUpdates() {
|
||||
registerUpdateIpcHandlers();
|
||||
|
||||
if (initialized) {
|
||||
return;
|
||||
}
|
||||
|
||||
initialized = true;
|
||||
autoUpdater.autoDownload = true;
|
||||
autoUpdater.autoInstallOnAppQuit = true;
|
||||
autoUpdater.logger = console;
|
||||
|
||||
const feedUrl = process.env.OPENSCREEN_UPDATE_FEED_URL?.trim() || DEFAULT_UPDATE_FEED_URL;
|
||||
const updateToken = process.env.OPENSCREEN_UPDATE_TOKEN?.trim();
|
||||
if (updateToken) {
|
||||
autoUpdater.requestHeaders = {
|
||||
Authorization: `token ${updateToken}`,
|
||||
};
|
||||
}
|
||||
autoUpdater.setFeedURL({
|
||||
provider: "generic",
|
||||
url: feedUrl,
|
||||
});
|
||||
|
||||
autoUpdater.on("checking-for-update", () => {
|
||||
updateStatus(createStatus("checking"));
|
||||
});
|
||||
autoUpdater.on("update-available", (info) => {
|
||||
updateStatus(statusFromInfo("available", info));
|
||||
});
|
||||
autoUpdater.on("update-not-available", (info) => {
|
||||
updateStatus(statusFromInfo("not-available", info));
|
||||
});
|
||||
autoUpdater.on("download-progress", (progress: ProgressInfo) => {
|
||||
updateStatus(
|
||||
createStatus("downloading", {
|
||||
version: status.version,
|
||||
releaseName: status.releaseName,
|
||||
releaseNotes: status.releaseNotes,
|
||||
percent: progress.percent,
|
||||
bytesPerSecond: progress.bytesPerSecond,
|
||||
transferred: progress.transferred,
|
||||
total: progress.total,
|
||||
}),
|
||||
);
|
||||
});
|
||||
autoUpdater.on("update-downloaded", (info) => {
|
||||
updateStatus(statusFromInfo("downloaded", info));
|
||||
});
|
||||
autoUpdater.on("error", (error) => {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
updateStatus(createStatus("error", { error: message }));
|
||||
});
|
||||
|
||||
setTimeout(() => {
|
||||
void checkForUpdates();
|
||||
}, AUTO_CHECK_DELAY_MS);
|
||||
setInterval(() => {
|
||||
void checkForUpdates();
|
||||
}, AUTO_CHECK_INTERVAL_MS).unref();
|
||||
}
|
||||
Generated
+953
-32
File diff suppressed because it is too large
Load Diff
+4
-1
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "openscreen",
|
||||
"private": true,
|
||||
"version": "1.4.9",
|
||||
"version": "1.4.13",
|
||||
"type": "module",
|
||||
"packageManager": "npm@10.9.4",
|
||||
"engines": {
|
||||
@@ -20,6 +20,7 @@
|
||||
"format": "biome format --write .",
|
||||
"i18n:check": "node scripts/i18n-check.mjs",
|
||||
"preview": "vite preview",
|
||||
"mcp": "node scripts/openscreen-mcp-server.mjs",
|
||||
"build:native:mac": "node scripts/build-macos-screencapturekit-helper.mjs",
|
||||
"build:mac": "npm run build:native:mac && tsc && vite build && electron-builder --mac --config electron-builder.json5",
|
||||
"build:native:win": "node scripts/build-windows-wgc-helper.mjs",
|
||||
@@ -50,6 +51,7 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@fix-webm-duration/fix": "^1.0.1",
|
||||
"@modelcontextprotocol/sdk": "^1.29.0",
|
||||
"@pixi/filter-drop-shadow": "^5.2.0",
|
||||
"@radix-ui/react-accordion": "^1.2.12",
|
||||
"@radix-ui/react-dialog": "^1.1.15",
|
||||
@@ -70,6 +72,7 @@
|
||||
"class-variance-authority": "^0.7.1",
|
||||
"clsx": "^2.1.1",
|
||||
"dnd-timeline": "^2.4.0",
|
||||
"electron-updater": "^6.8.3",
|
||||
"emoji-picker-react": "^4.18.0",
|
||||
"fix-webm-duration": "^1.0.6",
|
||||
"gif.js": "^0.2.0",
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env node
|
||||
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
||||
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
||||
import { z } from "zod";
|
||||
|
||||
const controlUrl = process.env.OPENSCREEN_MCP_CONTROL_URL || "http://127.0.0.1:52347";
|
||||
const token = process.env.OPENSCREEN_MCP_CONTROL_TOKEN;
|
||||
|
||||
async function callOpenScreen(action, payload = {}) {
|
||||
const response = await fetch(`${controlUrl.replace(/\/$/, "")}/mcp/${action}`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
...(token ? { authorization: `Bearer ${token}` } : {}),
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
const result = await response.json().catch(() => ({
|
||||
success: false,
|
||||
error: `OpenScreen returned HTTP ${response.status}`,
|
||||
}));
|
||||
if (!response.ok || !result.success) {
|
||||
throw new Error(
|
||||
result.error || result.message || `OpenScreen returned HTTP ${response.status}`,
|
||||
);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function textResult(result) {
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: JSON.stringify(result, null, 2),
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
const server = new McpServer({
|
||||
name: "openscreen",
|
||||
version: "1.0.0",
|
||||
});
|
||||
|
||||
server.registerTool(
|
||||
"list_sources",
|
||||
{
|
||||
title: "List OpenScreen capture sources",
|
||||
description: "List available screen and window sources that can be passed to record_video.",
|
||||
inputSchema: {},
|
||||
},
|
||||
async () => textResult(await callOpenScreen("list_sources")),
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"record_video",
|
||||
{
|
||||
title: "Start OpenScreen recording",
|
||||
description:
|
||||
"Start recording with a selected screen/window source, or the current/default source.",
|
||||
inputSchema: {
|
||||
guideMode: z.boolean().optional().describe("Enable Guide Mode for this recording."),
|
||||
sourceType: z
|
||||
.enum(["screen", "window"])
|
||||
.optional()
|
||||
.describe("Capture a screen/display or a window."),
|
||||
sourceId: z
|
||||
.string()
|
||||
.optional()
|
||||
.describe("Exact source id returned by list_sources, for example screen:0:0."),
|
||||
sourceName: z
|
||||
.string()
|
||||
.optional()
|
||||
.describe("Exact or partial source/window name to match when sourceId is not supplied."),
|
||||
displayIndex: z
|
||||
.number()
|
||||
.int()
|
||||
.nonnegative()
|
||||
.optional()
|
||||
.describe("Zero-based display index for screen capture."),
|
||||
},
|
||||
},
|
||||
async (input) => textResult(await callOpenScreen("record_video", input)),
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"stop_recording",
|
||||
{
|
||||
title: "Stop OpenScreen recording",
|
||||
description: "Stop the active OpenScreen recording and return the saved video file URL.",
|
||||
inputSchema: {
|
||||
discard: z.boolean().optional().describe("Discard the recording instead of saving it."),
|
||||
},
|
||||
},
|
||||
async (input) => textResult(await callOpenScreen("stop_recording", input)),
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"export_video",
|
||||
{
|
||||
title: "Export OpenScreen video",
|
||||
description:
|
||||
"Export the currently loaded OpenScreen editor project and return the exported file URL.",
|
||||
inputSchema: {
|
||||
outputPath: z.string().optional().describe("Absolute output path. Defaults to Downloads."),
|
||||
format: z.enum(["mp4", "gif"]).optional().describe("Export format. Defaults to mp4."),
|
||||
quality: z.enum(["medium", "good", "source"]).optional().describe("MP4 quality preset."),
|
||||
},
|
||||
},
|
||||
async ({ outputPath, format, quality }) =>
|
||||
textResult(
|
||||
await callOpenScreen("export_video", {
|
||||
outputPath,
|
||||
settings: {
|
||||
format: format ?? "mp4",
|
||||
quality: quality ?? "good",
|
||||
},
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"status",
|
||||
{
|
||||
title: "Get OpenScreen status",
|
||||
description: "Return whether OpenScreen is currently recording.",
|
||||
inputSchema: {},
|
||||
},
|
||||
async () => textResult(await callOpenScreen("status")),
|
||||
);
|
||||
|
||||
await server.connect(new StdioServerTransport());
|
||||
+70
-1
@@ -1,4 +1,5 @@
|
||||
import { lazy, Suspense, useEffect, useState } from "react";
|
||||
import { lazy, Suspense, useEffect, useRef, useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
import { CountdownOverlay } from "./components/launch/CountdownOverlay.tsx";
|
||||
import { LaunchWindow } from "./components/launch/LaunchWindow";
|
||||
import { SourceSelector } from "./components/launch/SourceSelector";
|
||||
@@ -6,6 +7,7 @@ import { Toaster } from "./components/ui/sonner";
|
||||
import { TooltipProvider } from "./components/ui/tooltip";
|
||||
import { ShortcutsProvider } from "./contexts/ShortcutsContext";
|
||||
import { loadAllCustomFonts } from "./lib/customFonts";
|
||||
import type { UpdateStatus } from "./lib/updateStatus";
|
||||
|
||||
const VideoEditor = lazy(() => import("./components/video-editor/VideoEditor"));
|
||||
const ShortcutsConfigDialog = lazy(() =>
|
||||
@@ -79,11 +81,78 @@ export default function App() {
|
||||
return (
|
||||
<TooltipProvider>
|
||||
{content}
|
||||
<UpdateNotifier
|
||||
enabled={
|
||||
hasElectronBridge &&
|
||||
windowType !== "hud-overlay" &&
|
||||
windowType !== "source-selector" &&
|
||||
windowType !== "countdown-overlay"
|
||||
}
|
||||
/>
|
||||
<Toaster theme="dark" className="pointer-events-auto" />
|
||||
</TooltipProvider>
|
||||
);
|
||||
}
|
||||
|
||||
function UpdateNotifier({ enabled }: { enabled: boolean }) {
|
||||
const lastPhaseRef = useRef<UpdateStatus["phase"]>("idle");
|
||||
|
||||
useEffect(() => {
|
||||
if (!enabled || !window.electronAPI?.updates) {
|
||||
return;
|
||||
}
|
||||
|
||||
const applyStatus = (status: UpdateStatus) => {
|
||||
const version = status.version ? ` ${status.version}` : "";
|
||||
if (status.phase === "available") {
|
||||
toast.loading(`Downloading OpenScreen${version} update...`, {
|
||||
id: "openscreen-update",
|
||||
duration: Number.POSITIVE_INFINITY,
|
||||
});
|
||||
} else if (status.phase === "downloading") {
|
||||
const percent = typeof status.percent === "number" ? ` ${Math.round(status.percent)}%` : "";
|
||||
toast.loading(`Downloading OpenScreen${version} update${percent}...`, {
|
||||
id: "openscreen-update",
|
||||
duration: Number.POSITIVE_INFINITY,
|
||||
});
|
||||
} else if (status.phase === "downloaded") {
|
||||
toast.success(`OpenScreen${version} is ready to install.`, {
|
||||
id: "openscreen-update",
|
||||
duration: Number.POSITIVE_INFINITY,
|
||||
action: {
|
||||
label: "Restart",
|
||||
onClick: () => {
|
||||
void window.electronAPI.updates.install();
|
||||
},
|
||||
},
|
||||
});
|
||||
} else if (
|
||||
status.phase === "error" &&
|
||||
(lastPhaseRef.current === "available" ||
|
||||
lastPhaseRef.current === "downloading" ||
|
||||
lastPhaseRef.current === "downloaded")
|
||||
) {
|
||||
toast.error(status.error || "OpenScreen update failed.", {
|
||||
id: "openscreen-update",
|
||||
});
|
||||
} else if (status.phase === "not-available" || status.phase === "unsupported") {
|
||||
toast.dismiss("openscreen-update");
|
||||
}
|
||||
lastPhaseRef.current = status.phase;
|
||||
};
|
||||
|
||||
const unsubscribe = window.electronAPI.updates.onStatus(applyStatus);
|
||||
void window.electronAPI.updates
|
||||
.getStatus()
|
||||
.then(applyStatus)
|
||||
.catch(() => undefined);
|
||||
|
||||
return unsubscribe;
|
||||
}, [enabled]);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function BrowserDevFallback() {
|
||||
return (
|
||||
<div className="flex h-screen w-screen items-center justify-center bg-[#08090b] px-6 text-slate-100">
|
||||
|
||||
@@ -22,6 +22,11 @@ import { RxDragHandleDots2 } from "react-icons/rx";
|
||||
import { toast } from "sonner";
|
||||
import { useI18n, useScopedT } from "@/contexts/I18nContext";
|
||||
import { getAvailableLocales, getLocaleName } from "@/i18n/loader";
|
||||
import type {
|
||||
McpControlResult,
|
||||
McpRecordVideoPayload,
|
||||
McpStopRecordingPayload,
|
||||
} from "@/lib/mcpControl";
|
||||
import { nativeBridgeClient } from "@/native";
|
||||
import { useAudioLevelMeter } from "../../hooks/useAudioLevelMeter";
|
||||
import { useCameraDevices } from "../../hooks/useCameraDevices";
|
||||
@@ -96,6 +101,7 @@ export function LaunchWindow() {
|
||||
const {
|
||||
recording,
|
||||
paused,
|
||||
countdownActive,
|
||||
elapsedSeconds,
|
||||
toggleRecording,
|
||||
togglePaused,
|
||||
@@ -326,6 +332,16 @@ export function LaunchWindow() {
|
||||
const [selectedSource, setSelectedSource] = useState("Screen");
|
||||
const [hasSelectedSource, setHasSelectedSource] = useState(false);
|
||||
const [, setRecordPointerDownCount] = useState(0);
|
||||
const recordingRef = useRef(recording);
|
||||
const countdownActiveRef = useRef(countdownActive);
|
||||
|
||||
useEffect(() => {
|
||||
recordingRef.current = recording;
|
||||
}, [recording]);
|
||||
|
||||
useEffect(() => {
|
||||
countdownActiveRef.current = countdownActive;
|
||||
}, [countdownActive]);
|
||||
|
||||
useEffect(() => {
|
||||
const checkSelectedSource = async () => {
|
||||
@@ -376,6 +392,162 @@ export function LaunchWindow() {
|
||||
await window.electronAPI.switchToEditor();
|
||||
};
|
||||
|
||||
const getMcpSources = useCallback(async () => {
|
||||
const sources = await window.electronAPI.getSources({
|
||||
types: ["screen", "window"],
|
||||
thumbnailSize: { width: 0, height: 0 },
|
||||
fetchWindowIcons: true,
|
||||
});
|
||||
return sources;
|
||||
}, []);
|
||||
|
||||
const toMcpSourceSummary = useCallback((source: ProcessedDesktopSource) => {
|
||||
return {
|
||||
id: source.id,
|
||||
name: source.name,
|
||||
type: source.id.startsWith("window:") ? "window" : "screen",
|
||||
displayId: source.displayId,
|
||||
displayIndex: source.displayIndex ?? source.screenIndex,
|
||||
displayLabel: source.displayLabel,
|
||||
bounds: source.bounds,
|
||||
};
|
||||
}, []);
|
||||
|
||||
const selectMcpSource = useCallback(
|
||||
async (payload: McpRecordVideoPayload = {}) => {
|
||||
const sources = await getMcpSources();
|
||||
const requestedType = payload.sourceType;
|
||||
const sourceName = payload.sourceName?.trim().toLowerCase();
|
||||
const typedSources = requestedType
|
||||
? sources.filter((source) =>
|
||||
requestedType === "window"
|
||||
? source.id.startsWith("window:")
|
||||
: source.id.startsWith("screen:"),
|
||||
)
|
||||
: sources;
|
||||
const source =
|
||||
(payload.sourceId
|
||||
? typedSources.find((item) => item.id === payload.sourceId)
|
||||
: undefined) ??
|
||||
(typeof payload.displayIndex === "number"
|
||||
? typedSources.find(
|
||||
(item) =>
|
||||
item.displayIndex === payload.displayIndex ||
|
||||
item.screenIndex === payload.displayIndex,
|
||||
)
|
||||
: undefined) ??
|
||||
(sourceName
|
||||
? (typedSources.find((item) => item.name.toLowerCase() === sourceName) ??
|
||||
typedSources.find((item) => item.name.toLowerCase().includes(sourceName)))
|
||||
: undefined) ??
|
||||
typedSources.find((item) => item.id.startsWith("screen:")) ??
|
||||
typedSources[0] ??
|
||||
sources.find((item) => item.id.startsWith("screen:")) ??
|
||||
sources[0];
|
||||
|
||||
if (!source) {
|
||||
return null;
|
||||
}
|
||||
const selected = await window.electronAPI.selectSource(source);
|
||||
if (selected) {
|
||||
setSelectedSource(selected.name);
|
||||
setHasSelectedSource(true);
|
||||
}
|
||||
return selected;
|
||||
},
|
||||
[getMcpSources],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const unsubscribe = window.electronAPI?.onMcpControlRequest?.(async (request) => {
|
||||
if (request.action === "list_sources") {
|
||||
const sources = await getMcpSources();
|
||||
return {
|
||||
success: true,
|
||||
data: sources.map(toMcpSourceSummary),
|
||||
};
|
||||
}
|
||||
|
||||
if (request.action === "status") {
|
||||
return {
|
||||
success: true,
|
||||
recording: recordingRef.current,
|
||||
message: recordingRef.current ? "Recording is active" : "Recording is idle",
|
||||
} satisfies McpControlResult;
|
||||
}
|
||||
|
||||
if (request.action === "record_video") {
|
||||
if (recordingRef.current) {
|
||||
return { success: true, recording: true, message: "Recording is already active" };
|
||||
}
|
||||
const payload = (request.payload ?? {}) as McpRecordVideoPayload;
|
||||
let sourceForRecording: ProcessedDesktopSource | null = null;
|
||||
if (!hasSelectedSource || payload.sourceId || payload.sourceName || payload.sourceType) {
|
||||
const selected = await selectMcpSource(payload);
|
||||
if (!selected) {
|
||||
return {
|
||||
success: false,
|
||||
error: "No screen or window source is available for recording.",
|
||||
};
|
||||
}
|
||||
sourceForRecording = selected;
|
||||
} else {
|
||||
sourceForRecording = await window.electronAPI.getSelectedSource();
|
||||
}
|
||||
|
||||
if (payload.guideMode === true) {
|
||||
setGuideModeEnabled(true);
|
||||
}
|
||||
|
||||
toggleRecording();
|
||||
return {
|
||||
success: true,
|
||||
recording: true,
|
||||
message: "Recording start requested",
|
||||
data: sourceForRecording ? toMcpSourceSummary(sourceForRecording) : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
if (request.action === "stop_recording") {
|
||||
if (!recordingRef.current && !countdownActiveRef.current) {
|
||||
return { success: false, recording: false, error: "No active recording to stop" };
|
||||
}
|
||||
|
||||
const payload = (request.payload ?? {}) as McpStopRecordingPayload;
|
||||
if (payload.discard === true) {
|
||||
window.setTimeout(() => cancelRecording(), 0);
|
||||
return { success: true, recording: false, message: "Recording discard requested" };
|
||||
}
|
||||
|
||||
window.setTimeout(() => toggleRecording(), 0);
|
||||
return {
|
||||
success: true,
|
||||
recording: false,
|
||||
message: countdownActiveRef.current
|
||||
? "Recording countdown cancel requested"
|
||||
: "Recording stop requested",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: `The HUD cannot handle MCP action: ${request.action}`,
|
||||
};
|
||||
});
|
||||
|
||||
return () => {
|
||||
unsubscribe?.();
|
||||
};
|
||||
}, [
|
||||
cancelRecording,
|
||||
getMcpSources,
|
||||
hasSelectedSource,
|
||||
selectMcpSource,
|
||||
setGuideModeEnabled,
|
||||
toMcpSourceSummary,
|
||||
toggleRecording,
|
||||
]);
|
||||
|
||||
const sendHudOverlayHide = () => {
|
||||
if (window.electronAPI && window.electronAPI.hudOverlayHide) {
|
||||
window.electronAPI.hudOverlayHide();
|
||||
|
||||
@@ -82,6 +82,7 @@ export function AnnotationOverlay({
|
||||
);
|
||||
const [livePointerPoint, setLivePointerPoint] = useState<{ x: number; y: number } | null>(null);
|
||||
const mosaicCanvasRef = useRef<HTMLCanvasElement | null>(null);
|
||||
const magnifierCanvasRef = useRef<HTMLCanvasElement | null>(null);
|
||||
const blurType = "mosaic";
|
||||
const blurOverlayColor =
|
||||
annotation.type === "blur" ? getBlurOverlayColor(annotation.blurData) : "";
|
||||
@@ -183,6 +184,79 @@ export function AnnotationOverlay({
|
||||
y,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
if (annotation.type !== "magnifier") {
|
||||
return;
|
||||
}
|
||||
void previewFrameVersion;
|
||||
|
||||
const canvas = magnifierCanvasRef.current;
|
||||
const sourceCanvas = previewSourceCanvas;
|
||||
if (!canvas || !sourceCanvas) {
|
||||
return;
|
||||
}
|
||||
|
||||
const sourceWidth = sourceCanvas.width;
|
||||
const sourceHeight = sourceCanvas.height;
|
||||
const sourceClientWidth = sourceCanvas.clientWidth || containerWidth || sourceWidth;
|
||||
const sourceClientHeight = sourceCanvas.clientHeight || containerHeight || sourceHeight;
|
||||
if (
|
||||
sourceWidth <= 0 ||
|
||||
sourceHeight <= 0 ||
|
||||
sourceClientWidth <= 0 ||
|
||||
sourceClientHeight <= 0
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const drawWidth = Math.max(1, Math.round(width));
|
||||
const drawHeight = Math.max(1, Math.round(height));
|
||||
canvas.width = drawWidth;
|
||||
canvas.height = drawHeight;
|
||||
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) {
|
||||
return;
|
||||
}
|
||||
|
||||
const zoom = Math.max(1, annotation.magnifierData?.zoom ?? 2.2);
|
||||
const target = annotation.magnifierData?.target ?? {
|
||||
x: annotation.position.x + annotation.size.width / 2,
|
||||
y: annotation.position.y + annotation.size.height / 2,
|
||||
};
|
||||
const scaleX = sourceWidth / sourceClientWidth;
|
||||
const scaleY = sourceHeight / sourceClientHeight;
|
||||
const targetX = (target.x / 100) * sourceClientWidth * scaleX;
|
||||
const targetY = (target.y / 100) * sourceClientHeight * scaleY;
|
||||
const sampleWidth = Math.max(1, drawWidth / zoom);
|
||||
const sampleHeight = Math.max(1, drawHeight / zoom);
|
||||
const sx = Math.max(0, Math.min(sourceWidth - sampleWidth, targetX - sampleWidth / 2));
|
||||
const sy = Math.max(0, Math.min(sourceHeight - sampleHeight, targetY - sampleHeight / 2));
|
||||
|
||||
context.clearRect(0, 0, drawWidth, drawHeight);
|
||||
context.imageSmoothingEnabled = true;
|
||||
context.imageSmoothingQuality = "high";
|
||||
context.drawImage(
|
||||
sourceCanvas as CanvasImageSource,
|
||||
sx,
|
||||
sy,
|
||||
Math.min(sampleWidth, sourceWidth - sx),
|
||||
Math.min(sampleHeight, sourceHeight - sy),
|
||||
0,
|
||||
0,
|
||||
drawWidth,
|
||||
drawHeight,
|
||||
);
|
||||
}, [
|
||||
annotation,
|
||||
containerHeight,
|
||||
containerWidth,
|
||||
height,
|
||||
previewFrameVersion,
|
||||
previewSourceCanvas,
|
||||
width,
|
||||
]);
|
||||
|
||||
const renderArrow = () => {
|
||||
const direction = annotation.figureData?.arrowDirection || "right";
|
||||
const color = annotation.figureData?.color || "#34B27B";
|
||||
@@ -351,6 +425,30 @@ export function AnnotationOverlay({
|
||||
<div className="w-full h-full flex items-center justify-center p-2">{renderArrow()}</div>
|
||||
);
|
||||
|
||||
case "magnifier": {
|
||||
const shape = annotation.magnifierData?.shape ?? "circle";
|
||||
const caption = annotation.magnifierData?.caption;
|
||||
return (
|
||||
<div className="relative h-full w-full">
|
||||
<canvas
|
||||
ref={magnifierCanvasRef}
|
||||
className="absolute inset-0 h-full w-full"
|
||||
style={{
|
||||
borderRadius: shape === "circle" ? "9999px" : "10px",
|
||||
border: "3px solid rgba(248,250,252,0.96)",
|
||||
boxShadow: "0 14px 36px rgba(0,0,0,0.38), 0 0 0 1px rgba(52,178,123,0.55)",
|
||||
backgroundColor: "rgba(15, 23, 42, 0.9)",
|
||||
}}
|
||||
/>
|
||||
{caption && (
|
||||
<div className="absolute left-1/2 top-full mt-1 max-w-[220px] -translate-x-1/2 rounded-md bg-slate-950/90 px-2 py-1 text-center text-[11px] font-semibold leading-4 text-slate-100 shadow-lg">
|
||||
{caption}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
case "blur": {
|
||||
const shape = annotation.blurData?.shape ?? "rectangle";
|
||||
const blurIntensity = Math.max(
|
||||
@@ -623,6 +721,7 @@ export function AnnotationOverlay({
|
||||
annotation.type === "text" && "bg-transparent",
|
||||
annotation.type === "image" && "bg-transparent",
|
||||
annotation.type === "figure" && "bg-transparent",
|
||||
annotation.type === "magnifier" && "bg-transparent",
|
||||
annotation.type === "blur" && "bg-transparent",
|
||||
isSelected && annotation.type !== "blur" && "shadow-lg",
|
||||
)}
|
||||
|
||||
@@ -13,6 +13,8 @@ import {
|
||||
} from "@/components/ui/dialog";
|
||||
import { useI18n, useScopedT } from "@/contexts/I18nContext";
|
||||
import { useShortcuts } from "@/contexts/ShortcutsContext";
|
||||
import type { GuideSession } from "@/guide/contracts";
|
||||
import { buildGuideVideoAnnotations, buildGuideVideoSpeedRegions } from "@/guide/videoAnnotations";
|
||||
import { INITIAL_EDITOR_STATE, useEditorHistory } from "@/hooks/useEditorHistory";
|
||||
import { type Locale } from "@/i18n/config";
|
||||
import { getAvailableLocales, getLocaleName } from "@/i18n/loader";
|
||||
@@ -32,6 +34,7 @@ import {
|
||||
VideoExporter,
|
||||
} from "@/lib/exporter";
|
||||
import { computeFrameStepTime } from "@/lib/frameStep";
|
||||
import type { McpControlResult, McpExportVideoPayload } from "@/lib/mcpControl";
|
||||
import type { CursorCaptureMode, ProjectMedia } from "@/lib/recordingSession";
|
||||
import { matchesShortcut } from "@/lib/shortcuts";
|
||||
import {
|
||||
@@ -1374,6 +1377,34 @@ export default function VideoEditor() {
|
||||
[pushState],
|
||||
);
|
||||
|
||||
const handleGuideAttachToVideo = useCallback(
|
||||
(session: GuideSession) => {
|
||||
const guideAnnotations = buildGuideVideoAnnotations(session, {
|
||||
nextId: () => `annotation-${nextAnnotationIdRef.current++}`,
|
||||
nextZIndex: () => nextAnnotationZIndexRef.current++,
|
||||
});
|
||||
const guideSpeedRegions = buildGuideVideoSpeedRegions(session, {
|
||||
nextId: () => `speed-${nextSpeedIdRef.current++}`,
|
||||
});
|
||||
if (guideAnnotations.length === 0 && guideSpeedRegions.length === 0) {
|
||||
toast.error("Generate a guide draft before attaching steps to the video.");
|
||||
return;
|
||||
}
|
||||
|
||||
pushState((prev) => ({
|
||||
annotationRegions: [...prev.annotationRegions, ...guideAnnotations],
|
||||
speedRegions: [...prev.speedRegions, ...guideSpeedRegions],
|
||||
}));
|
||||
const firstTextAnnotation = guideAnnotations.find((annotation) => annotation.type === "text");
|
||||
setSelectedAnnotationId(firstTextAnnotation?.id ?? guideAnnotations[0]?.id ?? null);
|
||||
setSelectedBlurId(null);
|
||||
setSelectedZoomId(null);
|
||||
setSelectedTrimId(null);
|
||||
setSelectedSpeedId(null);
|
||||
},
|
||||
[pushState],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const handleKeyDown = (e: KeyboardEvent) => {
|
||||
const mod = e.ctrlKey || e.metaKey;
|
||||
@@ -1558,16 +1589,19 @@ export default function VideoEditor() {
|
||||
}, [unsavedExport, handleExportSaved]);
|
||||
|
||||
const handleExport = useCallback(
|
||||
async (settings: ExportSettings) => {
|
||||
async (
|
||||
settings: ExportSettings,
|
||||
options?: { targetPath?: string },
|
||||
): Promise<McpControlResult> => {
|
||||
if (!videoPath) {
|
||||
toast.error("No video loaded");
|
||||
return;
|
||||
return { success: false, error: "No video loaded" };
|
||||
}
|
||||
|
||||
const video = videoPlaybackRef.current?.video;
|
||||
if (!video) {
|
||||
toast.error("Video not ready");
|
||||
return;
|
||||
return { success: false, error: "Video not ready" };
|
||||
}
|
||||
|
||||
// Ask the user where to save BEFORE starting the export. This avoids the
|
||||
@@ -1575,20 +1609,27 @@ export default function VideoEditor() {
|
||||
// long-running export.
|
||||
const isGifFormat = settings.format === "gif";
|
||||
const targetFileName = `export-${Date.now()}.${isGifFormat ? "gif" : "mp4"}`;
|
||||
const pickResult = await window.electronAPI.pickExportSavePath(
|
||||
targetFileName,
|
||||
getExportFolder(),
|
||||
);
|
||||
if (pickResult.canceled || !pickResult.success || !pickResult.path) {
|
||||
setShowExportDialog(false);
|
||||
return;
|
||||
let targetPath = options?.targetPath;
|
||||
if (!targetPath) {
|
||||
const pickResult = await window.electronAPI.pickExportSavePath(
|
||||
targetFileName,
|
||||
getExportFolder(),
|
||||
);
|
||||
if (pickResult.canceled || !pickResult.success || !pickResult.path) {
|
||||
setShowExportDialog(false);
|
||||
return { success: false, message: "Export canceled" };
|
||||
}
|
||||
targetPath = pickResult.path;
|
||||
}
|
||||
const targetPath = pickResult.path;
|
||||
|
||||
setIsExporting(true);
|
||||
setExportProgress(null);
|
||||
setExportError(null);
|
||||
setExportedFilePath(null);
|
||||
let mcpResult: McpControlResult = {
|
||||
success: false,
|
||||
error: "Export did not complete",
|
||||
};
|
||||
|
||||
try {
|
||||
const wasPlaying = isPlaying;
|
||||
@@ -1673,6 +1714,11 @@ export default function VideoEditor() {
|
||||
if (saveResult.success && saveResult.path) {
|
||||
setUnsavedExport(null);
|
||||
handleExportSaved("GIF", saveResult.path);
|
||||
mcpResult = {
|
||||
success: true,
|
||||
path: saveResult.path,
|
||||
message: "GIF exported successfully",
|
||||
};
|
||||
} else {
|
||||
setUnsavedExport({ arrayBuffer, fileName: targetFileName, format: "gif" });
|
||||
const message = buildSaveDiagnosticMessage(
|
||||
@@ -1681,6 +1727,7 @@ export default function VideoEditor() {
|
||||
);
|
||||
setExportError(message);
|
||||
toast.error(message);
|
||||
mcpResult = { success: false, error: message };
|
||||
}
|
||||
} else {
|
||||
const message = buildExportDiagnosticMessage({
|
||||
@@ -1693,6 +1740,7 @@ export default function VideoEditor() {
|
||||
});
|
||||
setExportError(message);
|
||||
toast.error(message);
|
||||
mcpResult = { success: false, error: message };
|
||||
}
|
||||
} else {
|
||||
// MP4 Export
|
||||
@@ -1764,6 +1812,11 @@ export default function VideoEditor() {
|
||||
if (saveResult.success && saveResult.path) {
|
||||
setUnsavedExport(null);
|
||||
handleExportSaved("Video", saveResult.path);
|
||||
mcpResult = {
|
||||
success: true,
|
||||
path: saveResult.path,
|
||||
message: "Video exported successfully",
|
||||
};
|
||||
} else {
|
||||
setUnsavedExport({ arrayBuffer, fileName: targetFileName, format: "mp4" });
|
||||
const message = buildSaveDiagnosticMessage(
|
||||
@@ -1772,6 +1825,7 @@ export default function VideoEditor() {
|
||||
);
|
||||
setExportError(message);
|
||||
toast.error(message);
|
||||
mcpResult = { success: false, error: message };
|
||||
}
|
||||
} else {
|
||||
const message = buildExportDiagnosticMessage({
|
||||
@@ -1786,6 +1840,7 @@ export default function VideoEditor() {
|
||||
});
|
||||
setExportError(message);
|
||||
toast.error(message);
|
||||
mcpResult = { success: false, error: message };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1798,6 +1853,7 @@ export default function VideoEditor() {
|
||||
const message = t("errors.exportBackgroundLoadFailed", { url: error.displayUrl });
|
||||
setExportError(message);
|
||||
toast.error(message);
|
||||
mcpResult = { success: false, error: message };
|
||||
} else {
|
||||
const errorMessage = error instanceof Error ? error.message : "Unknown error";
|
||||
const message = buildExportDiagnosticMessage({
|
||||
@@ -1807,6 +1863,7 @@ export default function VideoEditor() {
|
||||
});
|
||||
setExportError(message);
|
||||
toast.error(t("errors.exportFailedWithError", { error: message }));
|
||||
mcpResult = { success: false, error: message };
|
||||
}
|
||||
} finally {
|
||||
setIsExporting(false);
|
||||
@@ -1816,6 +1873,7 @@ export default function VideoEditor() {
|
||||
setShowExportDialog(false);
|
||||
setExportProgress(null);
|
||||
}
|
||||
return mcpResult;
|
||||
},
|
||||
[
|
||||
videoPath,
|
||||
@@ -1918,6 +1976,48 @@ export default function VideoEditor() {
|
||||
handleExport,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
const unsubscribe = window.electronAPI?.onMcpControlRequest?.(async (request) => {
|
||||
if (request.action === "status") {
|
||||
return {
|
||||
success: true,
|
||||
recording: false,
|
||||
data: { videoPath, editorReady: Boolean(videoPlaybackRef.current?.video) },
|
||||
};
|
||||
}
|
||||
|
||||
if (request.action !== "export_video") {
|
||||
return {
|
||||
success: false,
|
||||
error: `The editor cannot handle MCP action: ${request.action}`,
|
||||
};
|
||||
}
|
||||
|
||||
const payload = (request.payload ?? {}) as McpExportVideoPayload;
|
||||
const requestedSettings = payload.settings ?? {};
|
||||
const settings: ExportSettings = {
|
||||
format: requestedSettings.format ?? "mp4",
|
||||
quality: requestedSettings.quality ?? exportQuality,
|
||||
gifConfig:
|
||||
requestedSettings.format === "gif" && requestedSettings.gifConfig
|
||||
? requestedSettings.gifConfig
|
||||
: undefined,
|
||||
};
|
||||
if (settings.format === "gif" && !settings.gifConfig) {
|
||||
return {
|
||||
success: false,
|
||||
error: "MCP GIF export requires gifConfig dimensions and frame settings.",
|
||||
};
|
||||
}
|
||||
|
||||
return handleExport(settings, { targetPath: payload.outputPath });
|
||||
});
|
||||
|
||||
return () => {
|
||||
unsubscribe?.();
|
||||
};
|
||||
}, [exportQuality, handleExport, videoPath]);
|
||||
|
||||
const handleCancelExport = useCallback(() => {
|
||||
if (exporterRef.current) {
|
||||
exporterRef.current.cancel();
|
||||
@@ -2162,6 +2262,7 @@ export default function VideoEditor() {
|
||||
videoPath={videoPath}
|
||||
videoSourcePath={videoSourcePath}
|
||||
currentTimeMs={currentTime * 1000}
|
||||
onAttachToVideo={handleGuideAttachToVideo}
|
||||
/>
|
||||
)}
|
||||
<div className="min-h-0 flex-1 overflow-hidden">
|
||||
|
||||
@@ -1963,18 +1963,20 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
region: blurRegion,
|
||||
})),
|
||||
].sort((a, b) => a.region.zIndex - b.region.zIndex);
|
||||
const previewSnapshotCanvas =
|
||||
filteredBlurRegions.length > 0
|
||||
? (() => {
|
||||
const app = appRef.current;
|
||||
if (!app?.renderer?.extract) return null;
|
||||
try {
|
||||
return app.renderer.extract.canvas(app.stage);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})()
|
||||
: null;
|
||||
const needsPreviewSnapshot =
|
||||
filteredBlurRegions.length > 0 ||
|
||||
filteredAnnotations.some((annotation) => annotation.type === "magnifier");
|
||||
const previewSnapshotCanvas = needsPreviewSnapshot
|
||||
? (() => {
|
||||
const app = appRef.current;
|
||||
if (!app?.renderer?.extract) return null;
|
||||
try {
|
||||
return app.renderer.extract.canvas(app.stage);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})()
|
||||
: null;
|
||||
|
||||
// Handle click-through cycling: when clicking same annotation, cycle to next
|
||||
const handleAnnotationClick = (clickedId: string) => {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { KeyRound, ListChecks, RefreshCw, Save, Trash2, Wand2 } from "lucide-react";
|
||||
import { KeyRound, ListChecks, RefreshCw, Save, Trash2, Video, Wand2 } from "lucide-react";
|
||||
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
import { Button } from "@/components/ui/button";
|
||||
@@ -9,6 +9,7 @@ import type {
|
||||
GuideLanguage,
|
||||
GuideOcrProfile,
|
||||
GuideSession,
|
||||
GuideSnapshot,
|
||||
} from "@/guide/contracts";
|
||||
import { captureGuideSnapshots } from "@/guide/snapshot/extractGuideSnapshots";
|
||||
|
||||
@@ -17,9 +18,17 @@ interface GuidePanelProps {
|
||||
videoPath: string | null;
|
||||
videoSourcePath: string | null;
|
||||
currentTimeMs: number;
|
||||
onAttachToVideo?: (session: GuideSession) => void;
|
||||
}
|
||||
|
||||
type BusyAction = "load" | "generate";
|
||||
type BusyAction = "load" | "generate" | "attach";
|
||||
|
||||
interface GuideProgressState {
|
||||
label: string;
|
||||
current: number;
|
||||
total: number;
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
const COPY = {
|
||||
en: {
|
||||
@@ -63,6 +72,14 @@ const COPY = {
|
||||
noEvents: "No click events were captured for this guide.",
|
||||
ocrUnavailable: "Local OCR service is unavailable. You can still create a local draft.",
|
||||
exported: "Guide exported",
|
||||
attachToVideo: "Attach to video",
|
||||
attachedToVideo: "Guide steps attached to the video timeline.",
|
||||
noDraft: "Generate a guide draft before attaching steps to the video.",
|
||||
progressPreparing: "Preparing events",
|
||||
progressSnapshots: "Capturing snapshots",
|
||||
progressOcr: "Running OCR",
|
||||
progressDraft: "Writing draft",
|
||||
progressExport: "Exporting files",
|
||||
},
|
||||
vi: {
|
||||
title: "Hướng dẫn",
|
||||
@@ -105,10 +122,41 @@ const COPY = {
|
||||
noEvents: "Chưa ghi nhận click event nào cho guide này.",
|
||||
ocrUnavailable: "OCR local chưa chạy. Vẫn có thể tạo draft local.",
|
||||
exported: "Đã export hướng dẫn",
|
||||
attachToVideo: "Gắn vào video",
|
||||
attachedToVideo: "Đã gắn các bước guide vào timeline video.",
|
||||
noDraft: "Hãy tạo draft guide trước khi gắn vào video.",
|
||||
progressPreparing: "Đang chuẩn bị events",
|
||||
progressSnapshots: "Đang chụp ảnh",
|
||||
progressOcr: "Đang OCR",
|
||||
progressDraft: "Đang tạo draft",
|
||||
progressExport: "Đang export file",
|
||||
},
|
||||
} as const;
|
||||
|
||||
export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePanelProps) {
|
||||
function getPendingOcrSnapshots(session: GuideSession): GuideSnapshot[] {
|
||||
const ocrCompletedSnapshotIds = new Set(session.ocrBlocks.map((block) => block.snapshotId));
|
||||
return session.snapshots.filter(
|
||||
(snapshot) => !snapshot.ocrCompletedAt && !ocrCompletedSnapshotIds.has(snapshot.id),
|
||||
);
|
||||
}
|
||||
|
||||
function getProgressPercent(progress: GuideProgressState | null): number {
|
||||
if (!progress) {
|
||||
return 0;
|
||||
}
|
||||
if (progress.total <= 0) {
|
||||
return 100;
|
||||
}
|
||||
const percent = Math.round((progress.current / progress.total) * 100);
|
||||
return Math.min(100, Math.max(progress.current > 0 ? 8 : 4, percent));
|
||||
}
|
||||
|
||||
export function GuidePanel({
|
||||
recordingId,
|
||||
videoPath,
|
||||
videoSourcePath,
|
||||
onAttachToVideo,
|
||||
}: GuidePanelProps) {
|
||||
const { locale } = useI18n();
|
||||
const copy = useMemo(() => (locale.startsWith("vi") ? COPY.vi : COPY.en), [locale]);
|
||||
const guideLanguage: GuideLanguage = locale.startsWith("vi") ? "vi" : "en";
|
||||
@@ -124,8 +172,10 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
const [ocrProfile, setOcrProfile] = useState<GuideOcrProfile>("vietnamese");
|
||||
const [ocrLanguage, setOcrLanguage] = useState("vi,en");
|
||||
const [message, setMessage] = useState<string | null>(null);
|
||||
const [progress, setProgress] = useState<GuideProgressState | null>(null);
|
||||
|
||||
const isBusy = busyAction !== null;
|
||||
const progressPercent = getProgressPercent(progress);
|
||||
const canUseGuide = Boolean(recordingId && videoSourcePath && window.electronAPI?.guide);
|
||||
const generatedSteps = session?.generatedGuide?.steps ?? [];
|
||||
const statusLabel = useMemo(() => {
|
||||
@@ -220,6 +270,15 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
}
|
||||
|
||||
let current = session;
|
||||
const readResult = await window.electronAPI.guide.readSession(recordingId);
|
||||
if (readResult.success) {
|
||||
current = readResult.data;
|
||||
} else if (readResult.code === "guide-session-not-found") {
|
||||
current = null;
|
||||
} else if (!current) {
|
||||
throw new Error(readResult.error);
|
||||
}
|
||||
|
||||
if (!current) {
|
||||
const startResult = await window.electronAPI.guide.startSession(recordingId);
|
||||
if (!startResult.success) {
|
||||
@@ -251,6 +310,7 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
}
|
||||
setBusyAction(action);
|
||||
setMessage(null);
|
||||
setProgress(null);
|
||||
try {
|
||||
await task();
|
||||
} catch (error) {
|
||||
@@ -355,25 +415,59 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
if (!videoPath) {
|
||||
throw new Error("Video URL is not available.");
|
||||
}
|
||||
setProgress({
|
||||
label: copy.progressPreparing,
|
||||
current: 0,
|
||||
total: 1,
|
||||
detail: "0/1",
|
||||
});
|
||||
let current = await ensureEventsSession();
|
||||
setProgress({
|
||||
label: copy.progressPreparing,
|
||||
current: 1,
|
||||
total: 1,
|
||||
detail: "1/1",
|
||||
});
|
||||
if (current.events.length === 0) {
|
||||
throw new Error(copy.noEvents);
|
||||
}
|
||||
if (current.snapshots.length < current.events.length) {
|
||||
const snapshotEventIds = new Set(current.snapshots.map((snapshot) => snapshot.eventId));
|
||||
const pendingSnapshotTotal = current.events.filter(
|
||||
(event) => !snapshotEventIds.has(event.id),
|
||||
).length;
|
||||
if (pendingSnapshotTotal > 0) {
|
||||
setProgress({
|
||||
label: copy.progressSnapshots,
|
||||
current: 0,
|
||||
total: pendingSnapshotTotal,
|
||||
detail: `0/${pendingSnapshotTotal}`,
|
||||
});
|
||||
current = await captureGuideSnapshots({
|
||||
session: current,
|
||||
videoUrl: videoPath,
|
||||
maxWidth: 1280,
|
||||
onProgress: ({ completed, total }) => {
|
||||
setProgress({
|
||||
label: copy.progressSnapshots,
|
||||
current: completed,
|
||||
total,
|
||||
detail: `${completed}/${total}`,
|
||||
});
|
||||
},
|
||||
});
|
||||
setSession(current);
|
||||
}
|
||||
const ocrCompletedSnapshotIds = new Set(current.ocrBlocks.map((block) => block.snapshotId));
|
||||
const hasPendingOcr = current.snapshots.some(
|
||||
(snapshot) => !snapshot.ocrCompletedAt && !ocrCompletedSnapshotIds.has(snapshot.id),
|
||||
);
|
||||
if (hasPendingOcr) {
|
||||
const pendingOcrSnapshots = getPendingOcrSnapshots(current);
|
||||
for (const [index, snapshot] of pendingOcrSnapshots.entries()) {
|
||||
setProgress({
|
||||
label: copy.progressOcr,
|
||||
current: index,
|
||||
total: pendingOcrSnapshots.length,
|
||||
detail: `${index + 1}/${pendingOcrSnapshots.length}`,
|
||||
});
|
||||
const ocrResult = await window.electronAPI.guide.runOcr({
|
||||
recordingId: current.recordingId,
|
||||
snapshotIds: [snapshot.id],
|
||||
});
|
||||
if (!ocrResult.success) {
|
||||
if (ocrResult.code === "guide-ocr-unavailable") {
|
||||
@@ -383,7 +477,19 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
}
|
||||
current = ocrResult.data;
|
||||
setSession(current);
|
||||
setProgress({
|
||||
label: copy.progressOcr,
|
||||
current: index + 1,
|
||||
total: pendingOcrSnapshots.length,
|
||||
detail: `${index + 1}/${pendingOcrSnapshots.length}`,
|
||||
});
|
||||
}
|
||||
setProgress({
|
||||
label: copy.progressDraft,
|
||||
current: 0,
|
||||
total: 1,
|
||||
detail: "0/1",
|
||||
});
|
||||
const result = await window.electronAPI.guide.generateDraft({
|
||||
recordingId: current.recordingId,
|
||||
language: guideLanguage,
|
||||
@@ -392,18 +498,44 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
if (!result.success) {
|
||||
throw new Error(result.error);
|
||||
}
|
||||
current = result.data;
|
||||
setSession(current);
|
||||
setProgress({
|
||||
label: copy.progressDraft,
|
||||
current: 1,
|
||||
total: 1,
|
||||
detail: "1/1",
|
||||
});
|
||||
setProgress({
|
||||
label: copy.progressExport,
|
||||
current: 0,
|
||||
total: 2,
|
||||
detail: "0/2",
|
||||
});
|
||||
const markdownResult = await window.electronAPI.guide.exportMarkdown({
|
||||
recordingId: current.recordingId,
|
||||
});
|
||||
if (!markdownResult.success) {
|
||||
throw new Error(markdownResult.error);
|
||||
}
|
||||
setProgress({
|
||||
label: copy.progressExport,
|
||||
current: 1,
|
||||
total: 2,
|
||||
detail: "1/2",
|
||||
});
|
||||
const htmlResult = await window.electronAPI.guide.exportHtml({
|
||||
recordingId: current.recordingId,
|
||||
});
|
||||
if (!htmlResult.success) {
|
||||
throw new Error(htmlResult.error);
|
||||
}
|
||||
setProgress({
|
||||
label: copy.progressExport,
|
||||
current: 2,
|
||||
total: 2,
|
||||
detail: "2/2",
|
||||
});
|
||||
const revealResult = await window.electronAPI.revealInFolder(htmlResult.data.path);
|
||||
if (!revealResult.success) {
|
||||
toast.warning(revealResult.error ?? "Unable to open guide folder.");
|
||||
@@ -419,6 +551,11 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
copy.keyMissing,
|
||||
copy.noEvents,
|
||||
copy.ocrUnavailable,
|
||||
copy.progressDraft,
|
||||
copy.progressExport,
|
||||
copy.progressOcr,
|
||||
copy.progressPreparing,
|
||||
copy.progressSnapshots,
|
||||
ensureEventsSession,
|
||||
guideLanguage,
|
||||
provider,
|
||||
@@ -426,6 +563,25 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
videoPath,
|
||||
]);
|
||||
|
||||
const handleAttachToVideo = useCallback(() => {
|
||||
if (!session?.generatedGuide || session.generatedGuide.steps.length === 0) {
|
||||
setMessage(copy.noDraft);
|
||||
toast.error(copy.noDraft);
|
||||
return;
|
||||
}
|
||||
if (!onAttachToVideo) {
|
||||
return;
|
||||
}
|
||||
setBusyAction("attach");
|
||||
try {
|
||||
onAttachToVideo(session);
|
||||
setMessage(null);
|
||||
toast.success(copy.attachedToVideo);
|
||||
} finally {
|
||||
setBusyAction(null);
|
||||
}
|
||||
}, [copy.attachedToVideo, copy.noDraft, onAttachToVideo, session]);
|
||||
|
||||
return (
|
||||
<section className="editor-inspector-shell flex max-h-[320px] min-h-[246px] shrink-0 flex-col overflow-hidden rounded-[18px] border border-white/[0.075] bg-[#090a0c]">
|
||||
<div className="flex items-center justify-between border-b border-white/[0.07] px-3 py-2">
|
||||
@@ -449,6 +605,24 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
{canUseGuide ? statusLabel : copy.noRecording}
|
||||
</p>
|
||||
{message && <p className="mb-2 text-[11px] leading-4 text-amber-300">{message}</p>}
|
||||
{progress && (
|
||||
<div className="mb-2 rounded-md border border-white/[0.07] bg-white/[0.035] px-2 py-1.5">
|
||||
<div className="mb-1 flex items-center justify-between gap-2 text-[10px] leading-4">
|
||||
<span className="min-w-0 truncate font-semibold text-slate-200">
|
||||
{progress.label}
|
||||
</span>
|
||||
<span className="shrink-0 text-slate-500">
|
||||
{progress.detail ?? `${progress.current}/${progress.total}`}
|
||||
</span>
|
||||
</div>
|
||||
<div className="h-1.5 overflow-hidden rounded-full bg-white/[0.06]">
|
||||
<div
|
||||
className="h-full rounded-full bg-[#34B27B] transition-all duration-200"
|
||||
style={{ width: `${progressPercent}%` }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="mb-2 flex items-center gap-1.5">
|
||||
<select
|
||||
@@ -486,6 +660,16 @@ export function GuidePanel({ recordingId, videoPath, videoSourcePath }: GuidePan
|
||||
</span>
|
||||
</button>
|
||||
|
||||
<button
|
||||
type="button"
|
||||
disabled={!generatedSteps.length || isBusy || !onAttachToVideo}
|
||||
onClick={handleAttachToVideo}
|
||||
className="mb-2 flex h-9 w-full items-center justify-center gap-2 rounded-md border border-sky-400/25 bg-sky-400/10 px-3 text-xs font-semibold text-sky-100 transition-all hover:border-sky-300/45 hover:bg-sky-400/18 disabled:cursor-not-allowed disabled:opacity-40"
|
||||
>
|
||||
<Video className="h-3.5 w-3.5 shrink-0" />
|
||||
<span className="truncate">{copy.attachToVideo}</span>
|
||||
</button>
|
||||
|
||||
{settingsOpen && (
|
||||
<div className="mb-2 space-y-2 rounded-md border border-white/[0.07] bg-white/[0.035] p-2">
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
DEFAULT_BLUR_FREEHAND_POINTS,
|
||||
DEFAULT_BLUR_INTENSITY,
|
||||
DEFAULT_FIGURE_DATA,
|
||||
DEFAULT_MAGNIFIER_DATA,
|
||||
DEFAULT_PLAYBACK_SPEED,
|
||||
DEFAULT_ZOOM_DEPTH,
|
||||
DEFAULT_ZOOM_MOTION_BLUR,
|
||||
@@ -325,7 +326,10 @@ export function normalizeProjectEditor(editor: Partial<ProjectEditorState>): Pro
|
||||
startMs,
|
||||
endMs,
|
||||
type:
|
||||
region.type === "image" || region.type === "figure" || region.type === "blur"
|
||||
region.type === "image" ||
|
||||
region.type === "figure" ||
|
||||
region.type === "blur" ||
|
||||
region.type === "magnifier"
|
||||
? region.type
|
||||
: "text",
|
||||
content: typeof region.content === "string" ? region.content : "",
|
||||
@@ -410,6 +414,45 @@ export function normalizeProjectEditor(editor: Partial<ProjectEditorState>): Pro
|
||||
: DEFAULT_BLUR_FREEHAND_POINTS,
|
||||
}
|
||||
: undefined,
|
||||
magnifierData:
|
||||
region.magnifierData && typeof region.magnifierData === "object"
|
||||
? {
|
||||
...DEFAULT_MAGNIFIER_DATA,
|
||||
...region.magnifierData,
|
||||
target: {
|
||||
x: clamp(
|
||||
isFiniteNumber(region.magnifierData.target?.x)
|
||||
? region.magnifierData.target.x
|
||||
: DEFAULT_MAGNIFIER_DATA.target.x,
|
||||
0,
|
||||
100,
|
||||
),
|
||||
y: clamp(
|
||||
isFiniteNumber(region.magnifierData.target?.y)
|
||||
? region.magnifierData.target.y
|
||||
: DEFAULT_MAGNIFIER_DATA.target.y,
|
||||
0,
|
||||
100,
|
||||
),
|
||||
},
|
||||
zoom: clamp(
|
||||
isFiniteNumber(region.magnifierData.zoom)
|
||||
? region.magnifierData.zoom
|
||||
: DEFAULT_MAGNIFIER_DATA.zoom,
|
||||
1,
|
||||
6,
|
||||
),
|
||||
shape:
|
||||
region.magnifierData.shape === "rounded" ||
|
||||
region.magnifierData.shape === "circle"
|
||||
? region.magnifierData.shape
|
||||
: DEFAULT_MAGNIFIER_DATA.shape,
|
||||
caption:
|
||||
typeof region.magnifierData.caption === "string"
|
||||
? region.magnifierData.caption
|
||||
: undefined,
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
})
|
||||
: [];
|
||||
|
||||
@@ -206,7 +206,7 @@ export interface TrimRegion {
|
||||
endMs: number;
|
||||
}
|
||||
|
||||
export type AnnotationType = "text" | "image" | "figure" | "blur";
|
||||
export type AnnotationType = "text" | "image" | "figure" | "blur" | "magnifier";
|
||||
|
||||
export type ArrowDirection =
|
||||
| "up"
|
||||
@@ -245,6 +245,13 @@ export interface BlurData {
|
||||
freehandPoints?: Array<{ x: number; y: number }>;
|
||||
}
|
||||
|
||||
export interface MagnifierData {
|
||||
target: AnnotationPosition;
|
||||
zoom: number;
|
||||
shape: "circle" | "rounded";
|
||||
caption?: string;
|
||||
}
|
||||
|
||||
export interface AnnotationPosition {
|
||||
x: number;
|
||||
y: number;
|
||||
@@ -280,6 +287,7 @@ export interface AnnotationRegion {
|
||||
zIndex: number;
|
||||
figureData?: FigureData;
|
||||
blurData?: BlurData;
|
||||
magnifierData?: MagnifierData;
|
||||
}
|
||||
|
||||
export const DEFAULT_ANNOTATION_POSITION: AnnotationPosition = {
|
||||
@@ -330,6 +338,12 @@ export const DEFAULT_BLUR_DATA: BlurData = {
|
||||
freehandPoints: DEFAULT_BLUR_FREEHAND_POINTS,
|
||||
};
|
||||
|
||||
export const DEFAULT_MAGNIFIER_DATA: MagnifierData = {
|
||||
target: { x: 50, y: 50 },
|
||||
zoom: 2.2,
|
||||
shape: "circle",
|
||||
};
|
||||
|
||||
export interface CropRegion {
|
||||
x: number;
|
||||
y: number;
|
||||
|
||||
@@ -4,6 +4,13 @@ export interface CaptureGuideSnapshotsInput {
|
||||
session: GuideSession;
|
||||
videoUrl: string;
|
||||
maxWidth?: number;
|
||||
onProgress?: (progress: CaptureGuideSnapshotsProgress) => void;
|
||||
}
|
||||
|
||||
export interface CaptureGuideSnapshotsProgress {
|
||||
event: GuideEvent;
|
||||
completed: number;
|
||||
total: number;
|
||||
}
|
||||
|
||||
export async function captureGuideSnapshots(
|
||||
@@ -13,6 +20,13 @@ export async function captureGuideSnapshots(
|
||||
if (events.length === 0) {
|
||||
return input.session;
|
||||
}
|
||||
const existingSnapshotsByEventId = new Set(
|
||||
input.session.snapshots.map((snapshot) => snapshot.eventId),
|
||||
);
|
||||
const pendingEvents = events.filter((event) => !existingSnapshotsByEventId.has(event.id));
|
||||
if (pendingEvents.length === 0) {
|
||||
return input.session;
|
||||
}
|
||||
|
||||
const video = document.createElement("video");
|
||||
video.preload = "auto";
|
||||
@@ -35,13 +49,8 @@ export async function captureGuideSnapshots(
|
||||
canvas.height = Math.max(1, Math.round(sourceHeight * scale));
|
||||
|
||||
let latestSession = input.session;
|
||||
const existingSnapshotsByEventId = new Set(
|
||||
input.session.snapshots.map((snapshot) => snapshot.eventId),
|
||||
);
|
||||
for (const event of events) {
|
||||
if (existingSnapshotsByEventId.has(event.id)) {
|
||||
continue;
|
||||
}
|
||||
let completed = 0;
|
||||
for (const event of pendingEvents) {
|
||||
const offsetMs = event.screenshotOffsetMs ?? 500;
|
||||
const timeMs = getSnapshotTimeMs(event, offsetMs, video.duration);
|
||||
await seekVideo(video, timeMs / 1000);
|
||||
@@ -65,6 +74,12 @@ export async function captureGuideSnapshots(
|
||||
throw new Error(result.error);
|
||||
}
|
||||
latestSession = result.data;
|
||||
completed += 1;
|
||||
input.onProgress?.({
|
||||
event,
|
||||
completed,
|
||||
total: pendingEvents.length,
|
||||
});
|
||||
}
|
||||
|
||||
return latestSession;
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { GUIDE_SCHEMA_VERSION, type GuideSession } from "./contracts";
|
||||
import { buildGuideVideoAnnotations, buildGuideVideoSpeedRegions } from "./videoAnnotations";
|
||||
|
||||
function createSession(): GuideSession {
|
||||
return {
|
||||
schemaVersion: GUIDE_SCHEMA_VERSION,
|
||||
recordingId: "recording-1",
|
||||
videoPath: "recording.mp4",
|
||||
guidePath: "recording.guide.json",
|
||||
outputDir: "recording-guide",
|
||||
status: "draft-ready",
|
||||
events: [],
|
||||
snapshots: [],
|
||||
ocrBlocks: [],
|
||||
candidates: [
|
||||
{
|
||||
id: "candidate-1",
|
||||
eventId: "event-1",
|
||||
timeMs: 1200,
|
||||
action: "click",
|
||||
targetText: "Settings",
|
||||
targetRole: "button",
|
||||
position: {
|
||||
normalizedX: 0.2,
|
||||
normalizedY: 0.25,
|
||||
xPercent: 20,
|
||||
yPercent: 25,
|
||||
description: "top left",
|
||||
},
|
||||
nearbyText: ["Settings"],
|
||||
confidence: 0.91,
|
||||
},
|
||||
],
|
||||
generatedGuide: {
|
||||
title: "Guide",
|
||||
steps: [
|
||||
{
|
||||
id: "step-1",
|
||||
order: 1,
|
||||
title: "Open settings",
|
||||
instruction: "Click Settings.",
|
||||
sourceCandidateId: "candidate-1",
|
||||
},
|
||||
],
|
||||
},
|
||||
createdAt: "2026-06-04T00:00:00.000Z",
|
||||
updatedAt: "2026-06-04T00:00:00.000Z",
|
||||
};
|
||||
}
|
||||
|
||||
describe("buildGuideVideoAnnotations", () => {
|
||||
it("creates caption and pointer annotations from generated guide candidates", () => {
|
||||
let id = 1;
|
||||
let zIndex = 1;
|
||||
const annotations = buildGuideVideoAnnotations(createSession(), {
|
||||
nextId: () => `guide-video-${id++}`,
|
||||
nextZIndex: () => zIndex++,
|
||||
});
|
||||
|
||||
expect(annotations).toHaveLength(3);
|
||||
expect(annotations[0]).toMatchObject({
|
||||
id: "guide-video-1",
|
||||
type: "text",
|
||||
startMs: 1200,
|
||||
content: "1. Click Settings.",
|
||||
});
|
||||
expect(annotations[0]?.endMs).toBe(3200);
|
||||
expect(annotations[0]?.position.x).toBeGreaterThan(20);
|
||||
expect(annotations[1]?.endMs).toBe(3200);
|
||||
expect(annotations[1]?.position.x).toBeGreaterThan((annotations[0]?.position.x ?? 0) + 34);
|
||||
expect(annotations[1]?.position.y).toBeCloseTo(30.5);
|
||||
expect(annotations[1]).toMatchObject({
|
||||
id: "guide-video-2",
|
||||
type: "magnifier",
|
||||
magnifierData: {
|
||||
target: { x: 20, y: 25 },
|
||||
zoom: 2.2,
|
||||
shape: "circle",
|
||||
caption: "Settings",
|
||||
},
|
||||
});
|
||||
expect(annotations[2]).toMatchObject({
|
||||
id: "guide-video-3",
|
||||
type: "figure",
|
||||
endMs: 3200,
|
||||
figureData: {
|
||||
arrowDirection: "left",
|
||||
color: "#34B27B",
|
||||
},
|
||||
});
|
||||
expect(annotations[2]?.position.x).toBeGreaterThan(20);
|
||||
});
|
||||
|
||||
it("returns an empty list when no draft exists", () => {
|
||||
const session = createSession();
|
||||
session.generatedGuide = undefined;
|
||||
|
||||
const annotations = buildGuideVideoAnnotations(session, {
|
||||
nextId: () => "unused",
|
||||
nextZIndex: () => 1,
|
||||
});
|
||||
|
||||
expect(annotations).toEqual([]);
|
||||
});
|
||||
|
||||
it("creates 0.3x speed regions for two seconds at each guide point", () => {
|
||||
let id = 1;
|
||||
const speedRegions = buildGuideVideoSpeedRegions(createSession(), {
|
||||
nextId: () => `guide-speed-${id++}`,
|
||||
});
|
||||
|
||||
expect(speedRegions).toEqual([
|
||||
{
|
||||
id: "guide-speed-1",
|
||||
startMs: 1200,
|
||||
endMs: 3200,
|
||||
speed: 0.3,
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,249 @@
|
||||
import {
|
||||
type AnnotationRegion,
|
||||
type ArrowDirection,
|
||||
DEFAULT_ANNOTATION_STYLE,
|
||||
DEFAULT_FIGURE_DATA,
|
||||
DEFAULT_MAGNIFIER_DATA,
|
||||
type SpeedRegion,
|
||||
} from "@/components/video-editor/types";
|
||||
import type { GeneratedGuideStep, GuideSession, GuideStepCandidate } from "./contracts";
|
||||
|
||||
export interface BuildGuideVideoAnnotationsOptions {
|
||||
nextId: () => string;
|
||||
nextZIndex: () => number;
|
||||
defaultDurationMs?: number;
|
||||
}
|
||||
|
||||
const DEFAULT_STEP_DURATION_MS = 2000;
|
||||
const DEFAULT_STEP_SLOW_MOTION_DURATION_MS = 2000;
|
||||
const DEFAULT_STEP_SLOW_MOTION_SPEED = 0.3;
|
||||
const CAPTION_WIDTH = 34;
|
||||
const CAPTION_HEIGHT = 13;
|
||||
const MAGNIFIER_SIZE = 18;
|
||||
const ARROW_SIZE = 10;
|
||||
const ANNOTATION_GAP = 2;
|
||||
|
||||
function clamp(value: number, min: number, max: number) {
|
||||
return Math.min(max, Math.max(min, value));
|
||||
}
|
||||
|
||||
function findCandidate(
|
||||
step: GeneratedGuideStep,
|
||||
stepIndex: number,
|
||||
candidates: GuideStepCandidate[],
|
||||
): GuideStepCandidate | undefined {
|
||||
if (step.sourceCandidateId) {
|
||||
const matched = candidates.find((candidate) => candidate.id === step.sourceCandidateId);
|
||||
if (matched) return matched;
|
||||
}
|
||||
const sorted = [...candidates].sort((left, right) => left.timeMs - right.timeMs);
|
||||
return sorted[stepIndex];
|
||||
}
|
||||
|
||||
function getCaptionPosition(candidate: GuideStepCandidate | undefined) {
|
||||
const target = candidate?.position;
|
||||
if (!target) {
|
||||
return { x: 8, y: 8 };
|
||||
}
|
||||
|
||||
const targetX = target.normalizedX * 100;
|
||||
const targetY = target.normalizedY * 100;
|
||||
const x = target.normalizedX < 0.5 ? targetX + 8 : targetX - CAPTION_WIDTH - 8;
|
||||
const y = target.normalizedY < 0.5 ? targetY + 8 : targetY - CAPTION_HEIGHT - 8;
|
||||
|
||||
return {
|
||||
x: clamp(x, 2, 100 - CAPTION_WIDTH - 2),
|
||||
y: clamp(y, 2, 100 - CAPTION_HEIGHT - 2),
|
||||
};
|
||||
}
|
||||
|
||||
function getArrowDirection(
|
||||
candidate: GuideStepCandidate | undefined,
|
||||
originPosition: { x: number; y: number },
|
||||
originSize: { width: number; height: number } = {
|
||||
width: CAPTION_WIDTH,
|
||||
height: CAPTION_HEIGHT,
|
||||
},
|
||||
): ArrowDirection {
|
||||
const target = candidate?.position;
|
||||
if (!target) return "right";
|
||||
|
||||
const originCenterX = originPosition.x + originSize.width / 2;
|
||||
const originCenterY = originPosition.y + originSize.height / 2;
|
||||
const dx = target.normalizedX * 100 - originCenterX;
|
||||
const dy = target.normalizedY * 100 - originCenterY;
|
||||
const horizontal = dx > 8 ? "right" : dx < -8 ? "left" : "";
|
||||
const vertical = dy > 8 ? "down" : dy < -8 ? "up" : "";
|
||||
|
||||
if (vertical && horizontal) return `${vertical}-${horizontal}` as ArrowDirection;
|
||||
return (horizontal || vertical || "right") as ArrowDirection;
|
||||
}
|
||||
|
||||
function getMagnifierPosition(captionPosition: { x: number; y: number }) {
|
||||
const canPlaceRight = captionPosition.x + CAPTION_WIDTH + ANNOTATION_GAP + MAGNIFIER_SIZE <= 98;
|
||||
const x = canPlaceRight
|
||||
? captionPosition.x + CAPTION_WIDTH + ANNOTATION_GAP
|
||||
: captionPosition.x - MAGNIFIER_SIZE - ANNOTATION_GAP;
|
||||
const y = captionPosition.y + (CAPTION_HEIGHT - MAGNIFIER_SIZE) / 2;
|
||||
|
||||
return {
|
||||
x: clamp(x, 2, 100 - MAGNIFIER_SIZE - 2),
|
||||
y: clamp(y, 2, 100 - MAGNIFIER_SIZE - 2),
|
||||
};
|
||||
}
|
||||
|
||||
function getArrowPosition(
|
||||
position: NonNullable<GuideStepCandidate["position"]>,
|
||||
originPosition: { x: number; y: number },
|
||||
originSize: { width: number; height: number },
|
||||
) {
|
||||
const targetX = position.normalizedX * 100;
|
||||
const targetY = position.normalizedY * 100;
|
||||
const originCenterX = originPosition.x + originSize.width / 2;
|
||||
const originCenterY = originPosition.y + originSize.height / 2;
|
||||
const distance = Math.hypot(targetX - originCenterX, targetY - originCenterY);
|
||||
const targetOffset = Math.min(18, Math.max(10, distance * 0.35));
|
||||
const ratio = distance > 0 ? Math.max(0, (distance - targetOffset) / distance) : 0;
|
||||
const arrowCenterX = originCenterX + (targetX - originCenterX) * ratio;
|
||||
const arrowCenterY = originCenterY + (targetY - originCenterY) * ratio;
|
||||
|
||||
return {
|
||||
x: clamp(arrowCenterX - ARROW_SIZE / 2, 0, 100 - ARROW_SIZE),
|
||||
y: clamp(arrowCenterY - ARROW_SIZE / 2, 0, 100 - ARROW_SIZE),
|
||||
};
|
||||
}
|
||||
|
||||
function buildCaption(step: GeneratedGuideStep) {
|
||||
const instruction = step.instruction.trim();
|
||||
const title = step.title.trim();
|
||||
if (instruction) {
|
||||
return `${step.order}. ${instruction}`;
|
||||
}
|
||||
return title ? `${step.order}. ${title}` : `Step ${step.order}`;
|
||||
}
|
||||
|
||||
export function buildGuideVideoAnnotations(
|
||||
session: GuideSession,
|
||||
options: BuildGuideVideoAnnotationsOptions,
|
||||
): AnnotationRegion[] {
|
||||
const guide = session.generatedGuide;
|
||||
if (!guide || guide.steps.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const durationMs = Math.max(1000, options.defaultDurationMs ?? DEFAULT_STEP_DURATION_MS);
|
||||
const sortedSteps = [...guide.steps].sort((left, right) => left.order - right.order);
|
||||
const annotations: AnnotationRegion[] = [];
|
||||
|
||||
for (const [index, step] of sortedSteps.entries()) {
|
||||
const candidate = findCandidate(step, index, session.candidates);
|
||||
const startMs = Math.max(0, Math.round(candidate?.timeMs ?? index * durationMs));
|
||||
const endMs = Math.max(startMs + 750, startMs + durationMs);
|
||||
const captionPosition = getCaptionPosition(candidate);
|
||||
|
||||
annotations.push({
|
||||
id: options.nextId(),
|
||||
startMs,
|
||||
endMs,
|
||||
type: "text",
|
||||
content: buildCaption(step),
|
||||
textContent: buildCaption(step),
|
||||
position: captionPosition,
|
||||
size: { width: CAPTION_WIDTH, height: CAPTION_HEIGHT },
|
||||
style: {
|
||||
...DEFAULT_ANNOTATION_STYLE,
|
||||
color: "#f8fafc",
|
||||
backgroundColor: "rgba(15, 23, 42, 0.88)",
|
||||
fontSize: 18,
|
||||
fontWeight: "bold",
|
||||
textAlign: "left",
|
||||
},
|
||||
zIndex: options.nextZIndex(),
|
||||
});
|
||||
|
||||
if (candidate?.position) {
|
||||
const magnifierPosition = getMagnifierPosition(captionPosition);
|
||||
const arrowPosition = getArrowPosition(candidate.position, magnifierPosition, {
|
||||
width: MAGNIFIER_SIZE,
|
||||
height: MAGNIFIER_SIZE,
|
||||
});
|
||||
const arrowDirection = getArrowDirection(candidate, arrowPosition, {
|
||||
width: ARROW_SIZE,
|
||||
height: ARROW_SIZE,
|
||||
});
|
||||
|
||||
annotations.push({
|
||||
id: options.nextId(),
|
||||
startMs,
|
||||
endMs,
|
||||
type: "magnifier",
|
||||
content: buildCaption(step),
|
||||
position: magnifierPosition,
|
||||
size: { width: MAGNIFIER_SIZE, height: MAGNIFIER_SIZE },
|
||||
style: { ...DEFAULT_ANNOTATION_STYLE },
|
||||
zIndex: options.nextZIndex(),
|
||||
magnifierData: {
|
||||
...DEFAULT_MAGNIFIER_DATA,
|
||||
target: {
|
||||
x: candidate.position.normalizedX * 100,
|
||||
y: candidate.position.normalizedY * 100,
|
||||
},
|
||||
caption: candidate.targetText,
|
||||
},
|
||||
});
|
||||
annotations.push({
|
||||
id: options.nextId(),
|
||||
startMs,
|
||||
endMs,
|
||||
type: "figure",
|
||||
content: "",
|
||||
position: arrowPosition,
|
||||
size: { width: ARROW_SIZE, height: ARROW_SIZE },
|
||||
style: { ...DEFAULT_ANNOTATION_STYLE },
|
||||
zIndex: options.nextZIndex(),
|
||||
figureData: {
|
||||
...DEFAULT_FIGURE_DATA,
|
||||
arrowDirection,
|
||||
color: "#34B27B",
|
||||
strokeWidth: 5,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return annotations;
|
||||
}
|
||||
|
||||
export interface BuildGuideVideoSpeedRegionsOptions {
|
||||
nextId: () => string;
|
||||
durationMs?: number;
|
||||
speed?: number;
|
||||
}
|
||||
|
||||
export function buildGuideVideoSpeedRegions(
|
||||
session: GuideSession,
|
||||
options: BuildGuideVideoSpeedRegionsOptions,
|
||||
): SpeedRegion[] {
|
||||
const guide = session.generatedGuide;
|
||||
if (!guide || guide.steps.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const durationMs = Math.max(
|
||||
100,
|
||||
Math.round(options.durationMs ?? DEFAULT_STEP_SLOW_MOTION_DURATION_MS),
|
||||
);
|
||||
const speed = options.speed ?? DEFAULT_STEP_SLOW_MOTION_SPEED;
|
||||
const sortedSteps = [...guide.steps].sort((left, right) => left.order - right.order);
|
||||
|
||||
return sortedSteps.map((step, index) => {
|
||||
const candidate = findCandidate(step, index, session.candidates);
|
||||
const startMs = Math.max(0, Math.round(candidate?.timeMs ?? index * durationMs));
|
||||
return {
|
||||
id: options.nextId(),
|
||||
startMs,
|
||||
endMs: startMs + durationMs,
|
||||
speed,
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -50,6 +50,7 @@ const WEBCAM_TARGET_FRAME_RATE = 30;
|
||||
type UseScreenRecorderReturn = {
|
||||
recording: boolean;
|
||||
paused: boolean;
|
||||
countdownActive: boolean;
|
||||
elapsedSeconds: number;
|
||||
toggleRecording: () => void;
|
||||
togglePaused: () => void;
|
||||
@@ -1783,6 +1784,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
|
||||
return {
|
||||
recording,
|
||||
paused,
|
||||
countdownActive,
|
||||
elapsedSeconds,
|
||||
toggleRecording,
|
||||
togglePaused,
|
||||
|
||||
@@ -9,6 +9,8 @@ import {
|
||||
|
||||
let blurScratchCanvas: HTMLCanvasElement | null = null;
|
||||
let blurScratchCtx: CanvasRenderingContext2D | null = null;
|
||||
let magnifierScratchCanvas: HTMLCanvasElement | null = null;
|
||||
let magnifierScratchCtx: CanvasRenderingContext2D | null = null;
|
||||
|
||||
// Matches a single code point whose script is Han (including non-BMP
|
||||
// Extension A-F), Hiragana, Katakana (including halfwidth forms), or
|
||||
@@ -396,6 +398,130 @@ async function renderImage(
|
||||
});
|
||||
}
|
||||
|
||||
function renderMagnifier(
|
||||
ctx: CanvasRenderingContext2D,
|
||||
annotation: AnnotationRegion,
|
||||
x: number,
|
||||
y: number,
|
||||
width: number,
|
||||
height: number,
|
||||
canvasWidth: number,
|
||||
canvasHeight: number,
|
||||
scaleFactor: number,
|
||||
) {
|
||||
if (!magnifierScratchCanvas || !magnifierScratchCtx) {
|
||||
magnifierScratchCanvas = document.createElement("canvas");
|
||||
magnifierScratchCtx = magnifierScratchCanvas.getContext("2d");
|
||||
}
|
||||
if (!magnifierScratchCanvas || !magnifierScratchCtx) return;
|
||||
|
||||
const data = annotation.magnifierData;
|
||||
const zoom = Math.max(1, data?.zoom ?? 2.2);
|
||||
const target = data?.target ?? {
|
||||
x: annotation.position.x + annotation.size.width / 2,
|
||||
y: annotation.position.y + annotation.size.height / 2,
|
||||
};
|
||||
const targetX = (target.x / 100) * canvasWidth;
|
||||
const targetY = (target.y / 100) * canvasHeight;
|
||||
const sampleWidth = Math.max(1, width / zoom);
|
||||
const sampleHeight = Math.max(1, height / zoom);
|
||||
const sx = Math.max(0, Math.min(canvasWidth - sampleWidth, targetX - sampleWidth / 2));
|
||||
const sy = Math.max(0, Math.min(canvasHeight - sampleHeight, targetY - sampleHeight / 2));
|
||||
const sw = Math.max(1, Math.min(sampleWidth, canvasWidth - sx));
|
||||
const sh = Math.max(1, Math.min(sampleHeight, canvasHeight - sy));
|
||||
|
||||
magnifierScratchCanvas.width = Math.max(1, Math.round(width));
|
||||
magnifierScratchCanvas.height = Math.max(1, Math.round(height));
|
||||
magnifierScratchCtx.clearRect(0, 0, magnifierScratchCanvas.width, magnifierScratchCanvas.height);
|
||||
magnifierScratchCtx.imageSmoothingEnabled = true;
|
||||
magnifierScratchCtx.imageSmoothingQuality = "high";
|
||||
magnifierScratchCtx.drawImage(
|
||||
ctx.canvas,
|
||||
sx,
|
||||
sy,
|
||||
sw,
|
||||
sh,
|
||||
0,
|
||||
0,
|
||||
magnifierScratchCanvas.width,
|
||||
magnifierScratchCanvas.height,
|
||||
);
|
||||
|
||||
const centerX = x + width / 2;
|
||||
const centerY = y + height / 2;
|
||||
const shape = data?.shape ?? "circle";
|
||||
const radius = Math.min(width, height) / 2;
|
||||
const cornerRadius = shape === "circle" ? radius : Math.min(18 * scaleFactor, radius);
|
||||
|
||||
ctx.save();
|
||||
ctx.strokeStyle = "rgba(52,178,123,0.85)";
|
||||
ctx.lineWidth = Math.max(2, 2 * scaleFactor);
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(centerX, centerY);
|
||||
ctx.lineTo(targetX, targetY);
|
||||
ctx.stroke();
|
||||
ctx.fillStyle = "#34B27B";
|
||||
ctx.beginPath();
|
||||
ctx.arc(targetX, targetY, Math.max(4, 4 * scaleFactor), 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
|
||||
ctx.shadowColor = "rgba(0,0,0,0.38)";
|
||||
ctx.shadowBlur = 24 * scaleFactor;
|
||||
ctx.shadowOffsetY = 12 * scaleFactor;
|
||||
ctx.fillStyle = "rgba(15,23,42,0.92)";
|
||||
ctx.beginPath();
|
||||
ctx.roundRect(x, y, width, height, cornerRadius);
|
||||
ctx.fill();
|
||||
ctx.restore();
|
||||
|
||||
ctx.save();
|
||||
ctx.beginPath();
|
||||
ctx.roundRect(x, y, width, height, cornerRadius);
|
||||
ctx.clip();
|
||||
ctx.drawImage(magnifierScratchCanvas, x, y, width, height);
|
||||
ctx.restore();
|
||||
|
||||
ctx.save();
|
||||
ctx.strokeStyle = "rgba(248,250,252,0.96)";
|
||||
ctx.lineWidth = Math.max(3, 3 * scaleFactor);
|
||||
ctx.beginPath();
|
||||
ctx.roundRect(x, y, width, height, cornerRadius);
|
||||
ctx.stroke();
|
||||
ctx.strokeStyle = "rgba(52,178,123,0.58)";
|
||||
ctx.lineWidth = Math.max(1, 1.5 * scaleFactor);
|
||||
ctx.beginPath();
|
||||
ctx.roundRect(
|
||||
x + 2 * scaleFactor,
|
||||
y + 2 * scaleFactor,
|
||||
width - 4 * scaleFactor,
|
||||
height - 4 * scaleFactor,
|
||||
cornerRadius,
|
||||
);
|
||||
ctx.stroke();
|
||||
|
||||
const caption = data?.caption || "";
|
||||
if (caption) {
|
||||
const fontSize = Math.max(12, 13 * scaleFactor);
|
||||
ctx.font = `bold ${fontSize}px Inter, sans-serif`;
|
||||
ctx.textAlign = "center";
|
||||
ctx.textBaseline = "middle";
|
||||
const paddingX = 8 * scaleFactor;
|
||||
const paddingY = 5 * scaleFactor;
|
||||
const metrics = ctx.measureText(caption);
|
||||
const captionWidth = Math.min(width * 1.6, metrics.width + paddingX * 2);
|
||||
const captionHeight = fontSize + paddingY * 2;
|
||||
const captionX = centerX - captionWidth / 2;
|
||||
const captionY = y + height + 8 * scaleFactor;
|
||||
ctx.fillStyle = "rgba(15,23,42,0.92)";
|
||||
ctx.beginPath();
|
||||
ctx.roundRect(captionX, captionY, captionWidth, captionHeight, 6 * scaleFactor);
|
||||
ctx.fill();
|
||||
ctx.fillStyle = "#f8fafc";
|
||||
ctx.fillText(caption, centerX, captionY + captionHeight / 2, captionWidth - paddingX * 2);
|
||||
}
|
||||
ctx.restore();
|
||||
}
|
||||
|
||||
export async function renderAnnotations(
|
||||
ctx: CanvasRenderingContext2D,
|
||||
annotations: AnnotationRegion[],
|
||||
@@ -443,6 +569,20 @@ export async function renderAnnotations(
|
||||
}
|
||||
break;
|
||||
|
||||
case "magnifier":
|
||||
renderMagnifier(
|
||||
ctx,
|
||||
annotation,
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
canvasWidth,
|
||||
canvasHeight,
|
||||
scaleFactor,
|
||||
);
|
||||
break;
|
||||
|
||||
case "blur":
|
||||
renderBlur(ctx, annotation, x, y, width, height, scaleFactor);
|
||||
break;
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import type { ExportSettings } from "@/lib/exporter";
|
||||
|
||||
export type McpControlAction =
|
||||
| "list_sources"
|
||||
| "record_video"
|
||||
| "stop_recording"
|
||||
| "export_video"
|
||||
| "status";
|
||||
|
||||
export interface McpControlRequest {
|
||||
id: string;
|
||||
action: McpControlAction;
|
||||
payload?: unknown;
|
||||
}
|
||||
|
||||
export interface McpRecordVideoPayload {
|
||||
guideMode?: boolean;
|
||||
sourceType?: "screen" | "window";
|
||||
sourceId?: string;
|
||||
sourceName?: string;
|
||||
displayIndex?: number;
|
||||
}
|
||||
|
||||
export interface McpStopRecordingPayload {
|
||||
discard?: boolean;
|
||||
}
|
||||
|
||||
export interface McpExportVideoPayload {
|
||||
outputPath?: string;
|
||||
settings?: Partial<ExportSettings>;
|
||||
}
|
||||
|
||||
export interface McpControlResult {
|
||||
success: boolean;
|
||||
message?: string;
|
||||
recording?: boolean;
|
||||
path?: string;
|
||||
url?: string;
|
||||
data?: unknown;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export function isMcpControlAction(value: unknown): value is McpControlAction {
|
||||
return (
|
||||
value === "list_sources" ||
|
||||
value === "record_video" ||
|
||||
value === "stop_recording" ||
|
||||
value === "export_video" ||
|
||||
value === "status"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
export type UpdateStatusPhase =
|
||||
| "idle"
|
||||
| "checking"
|
||||
| "available"
|
||||
| "not-available"
|
||||
| "downloading"
|
||||
| "downloaded"
|
||||
| "error"
|
||||
| "unsupported";
|
||||
|
||||
export interface UpdateStatus {
|
||||
phase: UpdateStatusPhase;
|
||||
currentVersion: string;
|
||||
version?: string;
|
||||
releaseName?: string;
|
||||
releaseNotes?: string;
|
||||
percent?: number;
|
||||
bytesPerSecond?: number;
|
||||
transferred?: number;
|
||||
total?: number;
|
||||
error?: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface UpdateCheckResult {
|
||||
success: boolean;
|
||||
status: UpdateStatus;
|
||||
error?: string;
|
||||
}
|
||||
Reference in New Issue
Block a user