Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| aae562f146 | |||
| 5069354df3 |
Vendored
+7
@@ -365,6 +365,13 @@ interface Window {
|
||||
showCountdownOverlay: (value: number, runId: number) => Promise<void>;
|
||||
setCountdownOverlayValue: (value: number, runId: number) => Promise<void>;
|
||||
hideCountdownOverlay: (runId: number) => Promise<void>;
|
||||
onMcpControlRequest: (
|
||||
callback: (
|
||||
request: import("../src/lib/mcpControl").McpControlRequest,
|
||||
) =>
|
||||
| Promise<import("../src/lib/mcpControl").McpControlResult>
|
||||
| import("../src/lib/mcpControl").McpControlResult,
|
||||
) => () => void;
|
||||
onCountdownOverlayValue: (callback: (value: number | null) => void) => () => void;
|
||||
setMicrophoneExpanded: (expanded: boolean) => void;
|
||||
setHasUnsavedChanges: (hasChanges: boolean) => void;
|
||||
|
||||
+12
-1
@@ -13,6 +13,7 @@ import {
|
||||
} from "electron";
|
||||
import { mainT, setMainLocale } from "./i18n";
|
||||
import { getSelectedDesktopSource, registerIpcHandlers } from "./ipc/handlers";
|
||||
import { startMcpControlServer } from "./mcpControlServer";
|
||||
import { initializeAutoUpdates } from "./updater";
|
||||
import {
|
||||
createCountdownOverlayWindow,
|
||||
@@ -519,7 +520,6 @@ app.whenReady().then(async () => {
|
||||
initializeAutoUpdates();
|
||||
// Ensure recordings directory exists
|
||||
await ensureRecordingsDir();
|
||||
|
||||
function switchToHudWrapper() {
|
||||
if (mainWindow) {
|
||||
isForceClosing = true;
|
||||
@@ -530,6 +530,17 @@ app.whenReady().then(async () => {
|
||||
showMainWindow();
|
||||
}
|
||||
|
||||
startMcpControlServer({
|
||||
getMainWindow: () => mainWindow,
|
||||
ensureWindow: (action) => {
|
||||
if (action === "list_sources" || action === "record_video") {
|
||||
switchToHudWrapper();
|
||||
return;
|
||||
}
|
||||
showMainWindow();
|
||||
},
|
||||
});
|
||||
|
||||
registerIpcHandlers(
|
||||
createEditorWindowWrapper,
|
||||
createSourceSelectorWindowWrapper,
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { app, type BrowserWindow, ipcMain } from "electron";
|
||||
import type { McpControlAction, McpControlRequest, McpControlResult } from "../src/lib/mcpControl";
|
||||
import { isMcpControlAction } from "../src/lib/mcpControl";
|
||||
|
||||
const DEFAULT_MCP_CONTROL_PORT = 52347;
|
||||
const MCP_CONTROL_REQUEST_TIMEOUT_MS = 120_000;
|
||||
|
||||
type GetWindow = () => BrowserWindow | null;
|
||||
|
||||
interface PendingMcpRequest {
|
||||
resolve: (result: McpControlResult) => void;
|
||||
reject: (error: Error) => void;
|
||||
timeout: NodeJS.Timeout;
|
||||
}
|
||||
|
||||
interface StartMcpControlServerOptions {
|
||||
getMainWindow: GetWindow;
|
||||
ensureWindow: (action: McpControlAction) => void;
|
||||
}
|
||||
|
||||
const pendingRequests = new Map<string, PendingMcpRequest>();
|
||||
|
||||
function sendJson(response: ServerResponse, statusCode: number, body: unknown) {
|
||||
response.writeHead(statusCode, {
|
||||
"content-type": "application/json; charset=utf-8",
|
||||
});
|
||||
response.end(JSON.stringify(body));
|
||||
}
|
||||
|
||||
async function readJsonBody(request: IncomingMessage) {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of request) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
}
|
||||
const text = Buffer.concat(chunks).toString("utf-8").trim();
|
||||
return text ? JSON.parse(text) : {};
|
||||
}
|
||||
|
||||
function withFileUrl(result: McpControlResult): McpControlResult {
|
||||
if (!result.success || result.url || !result.path) {
|
||||
return result;
|
||||
}
|
||||
|
||||
return {
|
||||
...result,
|
||||
url: pathToFileURL(result.path).toString(),
|
||||
};
|
||||
}
|
||||
|
||||
function resolveDefaultExportPath(payload: unknown) {
|
||||
if (
|
||||
!payload ||
|
||||
typeof payload !== "object" ||
|
||||
("outputPath" in payload && typeof payload.outputPath === "string" && payload.outputPath)
|
||||
) {
|
||||
return payload;
|
||||
}
|
||||
|
||||
const requested = payload as { outputPath?: unknown; settings?: { format?: unknown } };
|
||||
const format = requested.settings?.format === "gif" ? "gif" : "mp4";
|
||||
const fileName = `openscreen-export-${Date.now()}.${format}`;
|
||||
return {
|
||||
...requested,
|
||||
outputPath: path.join(app.getPath("downloads"), fileName),
|
||||
};
|
||||
}
|
||||
|
||||
function dispatchRendererRequest(
|
||||
window: BrowserWindow,
|
||||
action: McpControlAction,
|
||||
payload: unknown,
|
||||
): Promise<McpControlResult> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const id = `${Date.now()}-${Math.random().toString(16).slice(2)}`;
|
||||
const timeout = setTimeout(() => {
|
||||
pendingRequests.delete(id);
|
||||
reject(new Error(`Timed out waiting for renderer to handle ${action}`));
|
||||
}, MCP_CONTROL_REQUEST_TIMEOUT_MS);
|
||||
|
||||
pendingRequests.set(id, { resolve, reject, timeout });
|
||||
const request: McpControlRequest = { id, action, payload };
|
||||
window.webContents.send("mcp-control-request", request);
|
||||
});
|
||||
}
|
||||
|
||||
function registerMcpControlIpc() {
|
||||
ipcMain.on(
|
||||
"mcp-control-response",
|
||||
(_event, response: { id?: unknown; result?: McpControlResult }) => {
|
||||
if (typeof response.id !== "string") {
|
||||
return;
|
||||
}
|
||||
const pending = pendingRequests.get(response.id);
|
||||
if (!pending) {
|
||||
return;
|
||||
}
|
||||
clearTimeout(pending.timeout);
|
||||
pendingRequests.delete(response.id);
|
||||
pending.resolve(
|
||||
response.result ?? { success: false, error: "Renderer returned an empty result" },
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export function startMcpControlServer({
|
||||
getMainWindow,
|
||||
ensureWindow,
|
||||
}: StartMcpControlServerOptions): Server {
|
||||
registerMcpControlIpc();
|
||||
|
||||
const token = process.env.OPENSCREEN_MCP_CONTROL_TOKEN;
|
||||
const port = Number(process.env.OPENSCREEN_MCP_CONTROL_PORT) || DEFAULT_MCP_CONTROL_PORT;
|
||||
|
||||
const server = createServer(async (request, response) => {
|
||||
try {
|
||||
const url = new URL(request.url ?? "/", "http://127.0.0.1");
|
||||
if (request.method === "GET" && url.pathname === "/health") {
|
||||
sendJson(response, 200, { success: true, app: "OpenScreen" });
|
||||
return;
|
||||
}
|
||||
|
||||
if (request.method !== "POST" || !url.pathname.startsWith("/mcp/")) {
|
||||
sendJson(response, 404, { success: false, error: "Not found" });
|
||||
return;
|
||||
}
|
||||
|
||||
if (token && request.headers.authorization !== `Bearer ${token}`) {
|
||||
sendJson(response, 401, { success: false, error: "Unauthorized" });
|
||||
return;
|
||||
}
|
||||
|
||||
const action = url.pathname.slice("/mcp/".length);
|
||||
if (!isMcpControlAction(action)) {
|
||||
sendJson(response, 400, { success: false, error: `Unsupported MCP action: ${action}` });
|
||||
return;
|
||||
}
|
||||
|
||||
const body = await readJsonBody(request);
|
||||
const payload = action === "export_video" ? resolveDefaultExportPath(body) : body;
|
||||
|
||||
ensureWindow(action);
|
||||
const window = getMainWindow();
|
||||
if (!window || window.isDestroyed()) {
|
||||
sendJson(response, 503, { success: false, error: "OpenScreen window is not available" });
|
||||
return;
|
||||
}
|
||||
|
||||
if (window.webContents.isLoading()) {
|
||||
await new Promise<void>((resolve) => {
|
||||
window.webContents.once("did-finish-load", () => resolve());
|
||||
});
|
||||
}
|
||||
|
||||
const result = withFileUrl(await dispatchRendererRequest(window, action, payload));
|
||||
sendJson(response, result.success ? 200 : 409, result);
|
||||
} catch (error) {
|
||||
sendJson(response, 500, {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
server.listen(port, "127.0.0.1", () => {
|
||||
console.info(`[mcp-control] listening on http://127.0.0.1:${port}`);
|
||||
});
|
||||
server.on("error", (error) => {
|
||||
console.error("[mcp-control] failed to start:", error);
|
||||
});
|
||||
app.once("before-quit", () => {
|
||||
server.close();
|
||||
});
|
||||
|
||||
return server;
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import type {
|
||||
SaveGuideInput,
|
||||
WriteGuideSnapshotInput,
|
||||
} from "../src/guide/contracts";
|
||||
import type { McpControlRequest, McpControlResult } from "../src/lib/mcpControl";
|
||||
import type { NativeMacRecordingRequest } from "../src/lib/nativeMacRecording";
|
||||
import type { NativeWindowsRecordingRequest } from "../src/lib/nativeWindowsRecording";
|
||||
import type { RecordingSession, StoreRecordedSessionInput } from "../src/lib/recordingSession";
|
||||
@@ -315,6 +316,26 @@ contextBridge.exposeInMainWorld("electronAPI", {
|
||||
hideCountdownOverlay: (runId: number) => {
|
||||
return ipcRenderer.invoke("countdown-overlay-hide", runId);
|
||||
},
|
||||
onMcpControlRequest: (
|
||||
callback: (request: McpControlRequest) => Promise<McpControlResult> | McpControlResult,
|
||||
) => {
|
||||
const listener = async (_event: unknown, request: McpControlRequest) => {
|
||||
try {
|
||||
const result = await callback(request);
|
||||
ipcRenderer.send("mcp-control-response", { id: request.id, result });
|
||||
} catch (error) {
|
||||
ipcRenderer.send("mcp-control-response", {
|
||||
id: request.id,
|
||||
result: {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
});
|
||||
}
|
||||
};
|
||||
ipcRenderer.on("mcp-control-request", listener);
|
||||
return () => ipcRenderer.removeListener("mcp-control-request", listener);
|
||||
},
|
||||
onCountdownOverlayValue: (callback: (value: number | null) => void) => {
|
||||
const listener = (_event: unknown, value: number | null) => callback(value);
|
||||
ipcRenderer.on("countdown-overlay-value", listener);
|
||||
|
||||
Generated
+870
-24
File diff suppressed because it is too large
Load Diff
+3
-1
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "openscreen",
|
||||
"private": true,
|
||||
"version": "1.4.11",
|
||||
"version": "1.4.13",
|
||||
"type": "module",
|
||||
"packageManager": "npm@10.9.4",
|
||||
"engines": {
|
||||
@@ -20,6 +20,7 @@
|
||||
"format": "biome format --write .",
|
||||
"i18n:check": "node scripts/i18n-check.mjs",
|
||||
"preview": "vite preview",
|
||||
"mcp": "node scripts/openscreen-mcp-server.mjs",
|
||||
"build:native:mac": "node scripts/build-macos-screencapturekit-helper.mjs",
|
||||
"build:mac": "npm run build:native:mac && tsc && vite build && electron-builder --mac --config electron-builder.json5",
|
||||
"build:native:win": "node scripts/build-windows-wgc-helper.mjs",
|
||||
@@ -50,6 +51,7 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@fix-webm-duration/fix": "^1.0.1",
|
||||
"@modelcontextprotocol/sdk": "^1.29.0",
|
||||
"@pixi/filter-drop-shadow": "^5.2.0",
|
||||
"@radix-ui/react-accordion": "^1.2.12",
|
||||
"@radix-ui/react-dialog": "^1.1.15",
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env node
|
||||
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
||||
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
||||
import { z } from "zod";
|
||||
|
||||
const controlUrl = process.env.OPENSCREEN_MCP_CONTROL_URL || "http://127.0.0.1:52347";
|
||||
const token = process.env.OPENSCREEN_MCP_CONTROL_TOKEN;
|
||||
|
||||
async function callOpenScreen(action, payload = {}) {
|
||||
const response = await fetch(`${controlUrl.replace(/\/$/, "")}/mcp/${action}`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
...(token ? { authorization: `Bearer ${token}` } : {}),
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
const result = await response.json().catch(() => ({
|
||||
success: false,
|
||||
error: `OpenScreen returned HTTP ${response.status}`,
|
||||
}));
|
||||
if (!response.ok || !result.success) {
|
||||
throw new Error(
|
||||
result.error || result.message || `OpenScreen returned HTTP ${response.status}`,
|
||||
);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function textResult(result) {
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: JSON.stringify(result, null, 2),
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
const server = new McpServer({
|
||||
name: "openscreen",
|
||||
version: "1.0.0",
|
||||
});
|
||||
|
||||
server.registerTool(
|
||||
"list_sources",
|
||||
{
|
||||
title: "List OpenScreen capture sources",
|
||||
description: "List available screen and window sources that can be passed to record_video.",
|
||||
inputSchema: {},
|
||||
},
|
||||
async () => textResult(await callOpenScreen("list_sources")),
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"record_video",
|
||||
{
|
||||
title: "Start OpenScreen recording",
|
||||
description:
|
||||
"Start recording with a selected screen/window source, or the current/default source.",
|
||||
inputSchema: {
|
||||
guideMode: z.boolean().optional().describe("Enable Guide Mode for this recording."),
|
||||
sourceType: z
|
||||
.enum(["screen", "window"])
|
||||
.optional()
|
||||
.describe("Capture a screen/display or a window."),
|
||||
sourceId: z
|
||||
.string()
|
||||
.optional()
|
||||
.describe("Exact source id returned by list_sources, for example screen:0:0."),
|
||||
sourceName: z
|
||||
.string()
|
||||
.optional()
|
||||
.describe("Exact or partial source/window name to match when sourceId is not supplied."),
|
||||
displayIndex: z
|
||||
.number()
|
||||
.int()
|
||||
.nonnegative()
|
||||
.optional()
|
||||
.describe("Zero-based display index for screen capture."),
|
||||
},
|
||||
},
|
||||
async (input) => textResult(await callOpenScreen("record_video", input)),
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"stop_recording",
|
||||
{
|
||||
title: "Stop OpenScreen recording",
|
||||
description: "Stop the active OpenScreen recording and return the saved video file URL.",
|
||||
inputSchema: {
|
||||
discard: z.boolean().optional().describe("Discard the recording instead of saving it."),
|
||||
},
|
||||
},
|
||||
async (input) => textResult(await callOpenScreen("stop_recording", input)),
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"export_video",
|
||||
{
|
||||
title: "Export OpenScreen video",
|
||||
description:
|
||||
"Export the currently loaded OpenScreen editor project and return the exported file URL.",
|
||||
inputSchema: {
|
||||
outputPath: z.string().optional().describe("Absolute output path. Defaults to Downloads."),
|
||||
format: z.enum(["mp4", "gif"]).optional().describe("Export format. Defaults to mp4."),
|
||||
quality: z.enum(["medium", "good", "source"]).optional().describe("MP4 quality preset."),
|
||||
},
|
||||
},
|
||||
async ({ outputPath, format, quality }) =>
|
||||
textResult(
|
||||
await callOpenScreen("export_video", {
|
||||
outputPath,
|
||||
settings: {
|
||||
format: format ?? "mp4",
|
||||
quality: quality ?? "good",
|
||||
},
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"status",
|
||||
{
|
||||
title: "Get OpenScreen status",
|
||||
description: "Return whether OpenScreen is currently recording.",
|
||||
inputSchema: {},
|
||||
},
|
||||
async () => textResult(await callOpenScreen("status")),
|
||||
);
|
||||
|
||||
await server.connect(new StdioServerTransport());
|
||||
@@ -22,6 +22,11 @@ import { RxDragHandleDots2 } from "react-icons/rx";
|
||||
import { toast } from "sonner";
|
||||
import { useI18n, useScopedT } from "@/contexts/I18nContext";
|
||||
import { getAvailableLocales, getLocaleName } from "@/i18n/loader";
|
||||
import type {
|
||||
McpControlResult,
|
||||
McpRecordVideoPayload,
|
||||
McpStopRecordingPayload,
|
||||
} from "@/lib/mcpControl";
|
||||
import { nativeBridgeClient } from "@/native";
|
||||
import { useAudioLevelMeter } from "../../hooks/useAudioLevelMeter";
|
||||
import { useCameraDevices } from "../../hooks/useCameraDevices";
|
||||
@@ -96,6 +101,7 @@ export function LaunchWindow() {
|
||||
const {
|
||||
recording,
|
||||
paused,
|
||||
countdownActive,
|
||||
elapsedSeconds,
|
||||
toggleRecording,
|
||||
togglePaused,
|
||||
@@ -326,6 +332,16 @@ export function LaunchWindow() {
|
||||
const [selectedSource, setSelectedSource] = useState("Screen");
|
||||
const [hasSelectedSource, setHasSelectedSource] = useState(false);
|
||||
const [, setRecordPointerDownCount] = useState(0);
|
||||
const recordingRef = useRef(recording);
|
||||
const countdownActiveRef = useRef(countdownActive);
|
||||
|
||||
useEffect(() => {
|
||||
recordingRef.current = recording;
|
||||
}, [recording]);
|
||||
|
||||
useEffect(() => {
|
||||
countdownActiveRef.current = countdownActive;
|
||||
}, [countdownActive]);
|
||||
|
||||
useEffect(() => {
|
||||
const checkSelectedSource = async () => {
|
||||
@@ -376,6 +392,162 @@ export function LaunchWindow() {
|
||||
await window.electronAPI.switchToEditor();
|
||||
};
|
||||
|
||||
const getMcpSources = useCallback(async () => {
|
||||
const sources = await window.electronAPI.getSources({
|
||||
types: ["screen", "window"],
|
||||
thumbnailSize: { width: 0, height: 0 },
|
||||
fetchWindowIcons: true,
|
||||
});
|
||||
return sources;
|
||||
}, []);
|
||||
|
||||
const toMcpSourceSummary = useCallback((source: ProcessedDesktopSource) => {
|
||||
return {
|
||||
id: source.id,
|
||||
name: source.name,
|
||||
type: source.id.startsWith("window:") ? "window" : "screen",
|
||||
displayId: source.displayId,
|
||||
displayIndex: source.displayIndex ?? source.screenIndex,
|
||||
displayLabel: source.displayLabel,
|
||||
bounds: source.bounds,
|
||||
};
|
||||
}, []);
|
||||
|
||||
const selectMcpSource = useCallback(
|
||||
async (payload: McpRecordVideoPayload = {}) => {
|
||||
const sources = await getMcpSources();
|
||||
const requestedType = payload.sourceType;
|
||||
const sourceName = payload.sourceName?.trim().toLowerCase();
|
||||
const typedSources = requestedType
|
||||
? sources.filter((source) =>
|
||||
requestedType === "window"
|
||||
? source.id.startsWith("window:")
|
||||
: source.id.startsWith("screen:"),
|
||||
)
|
||||
: sources;
|
||||
const source =
|
||||
(payload.sourceId
|
||||
? typedSources.find((item) => item.id === payload.sourceId)
|
||||
: undefined) ??
|
||||
(typeof payload.displayIndex === "number"
|
||||
? typedSources.find(
|
||||
(item) =>
|
||||
item.displayIndex === payload.displayIndex ||
|
||||
item.screenIndex === payload.displayIndex,
|
||||
)
|
||||
: undefined) ??
|
||||
(sourceName
|
||||
? (typedSources.find((item) => item.name.toLowerCase() === sourceName) ??
|
||||
typedSources.find((item) => item.name.toLowerCase().includes(sourceName)))
|
||||
: undefined) ??
|
||||
typedSources.find((item) => item.id.startsWith("screen:")) ??
|
||||
typedSources[0] ??
|
||||
sources.find((item) => item.id.startsWith("screen:")) ??
|
||||
sources[0];
|
||||
|
||||
if (!source) {
|
||||
return null;
|
||||
}
|
||||
const selected = await window.electronAPI.selectSource(source);
|
||||
if (selected) {
|
||||
setSelectedSource(selected.name);
|
||||
setHasSelectedSource(true);
|
||||
}
|
||||
return selected;
|
||||
},
|
||||
[getMcpSources],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const unsubscribe = window.electronAPI?.onMcpControlRequest?.(async (request) => {
|
||||
if (request.action === "list_sources") {
|
||||
const sources = await getMcpSources();
|
||||
return {
|
||||
success: true,
|
||||
data: sources.map(toMcpSourceSummary),
|
||||
};
|
||||
}
|
||||
|
||||
if (request.action === "status") {
|
||||
return {
|
||||
success: true,
|
||||
recording: recordingRef.current,
|
||||
message: recordingRef.current ? "Recording is active" : "Recording is idle",
|
||||
} satisfies McpControlResult;
|
||||
}
|
||||
|
||||
if (request.action === "record_video") {
|
||||
if (recordingRef.current) {
|
||||
return { success: true, recording: true, message: "Recording is already active" };
|
||||
}
|
||||
const payload = (request.payload ?? {}) as McpRecordVideoPayload;
|
||||
let sourceForRecording: ProcessedDesktopSource | null = null;
|
||||
if (!hasSelectedSource || payload.sourceId || payload.sourceName || payload.sourceType) {
|
||||
const selected = await selectMcpSource(payload);
|
||||
if (!selected) {
|
||||
return {
|
||||
success: false,
|
||||
error: "No screen or window source is available for recording.",
|
||||
};
|
||||
}
|
||||
sourceForRecording = selected;
|
||||
} else {
|
||||
sourceForRecording = await window.electronAPI.getSelectedSource();
|
||||
}
|
||||
|
||||
if (payload.guideMode === true) {
|
||||
setGuideModeEnabled(true);
|
||||
}
|
||||
|
||||
toggleRecording();
|
||||
return {
|
||||
success: true,
|
||||
recording: true,
|
||||
message: "Recording start requested",
|
||||
data: sourceForRecording ? toMcpSourceSummary(sourceForRecording) : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
if (request.action === "stop_recording") {
|
||||
if (!recordingRef.current && !countdownActiveRef.current) {
|
||||
return { success: false, recording: false, error: "No active recording to stop" };
|
||||
}
|
||||
|
||||
const payload = (request.payload ?? {}) as McpStopRecordingPayload;
|
||||
if (payload.discard === true) {
|
||||
window.setTimeout(() => cancelRecording(), 0);
|
||||
return { success: true, recording: false, message: "Recording discard requested" };
|
||||
}
|
||||
|
||||
window.setTimeout(() => toggleRecording(), 0);
|
||||
return {
|
||||
success: true,
|
||||
recording: false,
|
||||
message: countdownActiveRef.current
|
||||
? "Recording countdown cancel requested"
|
||||
: "Recording stop requested",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: `The HUD cannot handle MCP action: ${request.action}`,
|
||||
};
|
||||
});
|
||||
|
||||
return () => {
|
||||
unsubscribe?.();
|
||||
};
|
||||
}, [
|
||||
cancelRecording,
|
||||
getMcpSources,
|
||||
hasSelectedSource,
|
||||
selectMcpSource,
|
||||
setGuideModeEnabled,
|
||||
toMcpSourceSummary,
|
||||
toggleRecording,
|
||||
]);
|
||||
|
||||
const sendHudOverlayHide = () => {
|
||||
if (window.electronAPI && window.electronAPI.hudOverlayHide) {
|
||||
window.electronAPI.hudOverlayHide();
|
||||
|
||||
@@ -34,6 +34,7 @@ import {
|
||||
VideoExporter,
|
||||
} from "@/lib/exporter";
|
||||
import { computeFrameStepTime } from "@/lib/frameStep";
|
||||
import type { McpControlResult, McpExportVideoPayload } from "@/lib/mcpControl";
|
||||
import type { CursorCaptureMode, ProjectMedia } from "@/lib/recordingSession";
|
||||
import { matchesShortcut } from "@/lib/shortcuts";
|
||||
import {
|
||||
@@ -1588,16 +1589,19 @@ export default function VideoEditor() {
|
||||
}, [unsavedExport, handleExportSaved]);
|
||||
|
||||
const handleExport = useCallback(
|
||||
async (settings: ExportSettings) => {
|
||||
async (
|
||||
settings: ExportSettings,
|
||||
options?: { targetPath?: string },
|
||||
): Promise<McpControlResult> => {
|
||||
if (!videoPath) {
|
||||
toast.error("No video loaded");
|
||||
return;
|
||||
return { success: false, error: "No video loaded" };
|
||||
}
|
||||
|
||||
const video = videoPlaybackRef.current?.video;
|
||||
if (!video) {
|
||||
toast.error("Video not ready");
|
||||
return;
|
||||
return { success: false, error: "Video not ready" };
|
||||
}
|
||||
|
||||
// Ask the user where to save BEFORE starting the export. This avoids the
|
||||
@@ -1605,20 +1609,27 @@ export default function VideoEditor() {
|
||||
// long-running export.
|
||||
const isGifFormat = settings.format === "gif";
|
||||
const targetFileName = `export-${Date.now()}.${isGifFormat ? "gif" : "mp4"}`;
|
||||
const pickResult = await window.electronAPI.pickExportSavePath(
|
||||
targetFileName,
|
||||
getExportFolder(),
|
||||
);
|
||||
if (pickResult.canceled || !pickResult.success || !pickResult.path) {
|
||||
setShowExportDialog(false);
|
||||
return;
|
||||
let targetPath = options?.targetPath;
|
||||
if (!targetPath) {
|
||||
const pickResult = await window.electronAPI.pickExportSavePath(
|
||||
targetFileName,
|
||||
getExportFolder(),
|
||||
);
|
||||
if (pickResult.canceled || !pickResult.success || !pickResult.path) {
|
||||
setShowExportDialog(false);
|
||||
return { success: false, message: "Export canceled" };
|
||||
}
|
||||
targetPath = pickResult.path;
|
||||
}
|
||||
const targetPath = pickResult.path;
|
||||
|
||||
setIsExporting(true);
|
||||
setExportProgress(null);
|
||||
setExportError(null);
|
||||
setExportedFilePath(null);
|
||||
let mcpResult: McpControlResult = {
|
||||
success: false,
|
||||
error: "Export did not complete",
|
||||
};
|
||||
|
||||
try {
|
||||
const wasPlaying = isPlaying;
|
||||
@@ -1703,6 +1714,11 @@ export default function VideoEditor() {
|
||||
if (saveResult.success && saveResult.path) {
|
||||
setUnsavedExport(null);
|
||||
handleExportSaved("GIF", saveResult.path);
|
||||
mcpResult = {
|
||||
success: true,
|
||||
path: saveResult.path,
|
||||
message: "GIF exported successfully",
|
||||
};
|
||||
} else {
|
||||
setUnsavedExport({ arrayBuffer, fileName: targetFileName, format: "gif" });
|
||||
const message = buildSaveDiagnosticMessage(
|
||||
@@ -1711,6 +1727,7 @@ export default function VideoEditor() {
|
||||
);
|
||||
setExportError(message);
|
||||
toast.error(message);
|
||||
mcpResult = { success: false, error: message };
|
||||
}
|
||||
} else {
|
||||
const message = buildExportDiagnosticMessage({
|
||||
@@ -1723,6 +1740,7 @@ export default function VideoEditor() {
|
||||
});
|
||||
setExportError(message);
|
||||
toast.error(message);
|
||||
mcpResult = { success: false, error: message };
|
||||
}
|
||||
} else {
|
||||
// MP4 Export
|
||||
@@ -1794,6 +1812,11 @@ export default function VideoEditor() {
|
||||
if (saveResult.success && saveResult.path) {
|
||||
setUnsavedExport(null);
|
||||
handleExportSaved("Video", saveResult.path);
|
||||
mcpResult = {
|
||||
success: true,
|
||||
path: saveResult.path,
|
||||
message: "Video exported successfully",
|
||||
};
|
||||
} else {
|
||||
setUnsavedExport({ arrayBuffer, fileName: targetFileName, format: "mp4" });
|
||||
const message = buildSaveDiagnosticMessage(
|
||||
@@ -1802,6 +1825,7 @@ export default function VideoEditor() {
|
||||
);
|
||||
setExportError(message);
|
||||
toast.error(message);
|
||||
mcpResult = { success: false, error: message };
|
||||
}
|
||||
} else {
|
||||
const message = buildExportDiagnosticMessage({
|
||||
@@ -1816,6 +1840,7 @@ export default function VideoEditor() {
|
||||
});
|
||||
setExportError(message);
|
||||
toast.error(message);
|
||||
mcpResult = { success: false, error: message };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1828,6 +1853,7 @@ export default function VideoEditor() {
|
||||
const message = t("errors.exportBackgroundLoadFailed", { url: error.displayUrl });
|
||||
setExportError(message);
|
||||
toast.error(message);
|
||||
mcpResult = { success: false, error: message };
|
||||
} else {
|
||||
const errorMessage = error instanceof Error ? error.message : "Unknown error";
|
||||
const message = buildExportDiagnosticMessage({
|
||||
@@ -1837,6 +1863,7 @@ export default function VideoEditor() {
|
||||
});
|
||||
setExportError(message);
|
||||
toast.error(t("errors.exportFailedWithError", { error: message }));
|
||||
mcpResult = { success: false, error: message };
|
||||
}
|
||||
} finally {
|
||||
setIsExporting(false);
|
||||
@@ -1846,6 +1873,7 @@ export default function VideoEditor() {
|
||||
setShowExportDialog(false);
|
||||
setExportProgress(null);
|
||||
}
|
||||
return mcpResult;
|
||||
},
|
||||
[
|
||||
videoPath,
|
||||
@@ -1948,6 +1976,48 @@ export default function VideoEditor() {
|
||||
handleExport,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
const unsubscribe = window.electronAPI?.onMcpControlRequest?.(async (request) => {
|
||||
if (request.action === "status") {
|
||||
return {
|
||||
success: true,
|
||||
recording: false,
|
||||
data: { videoPath, editorReady: Boolean(videoPlaybackRef.current?.video) },
|
||||
};
|
||||
}
|
||||
|
||||
if (request.action !== "export_video") {
|
||||
return {
|
||||
success: false,
|
||||
error: `The editor cannot handle MCP action: ${request.action}`,
|
||||
};
|
||||
}
|
||||
|
||||
const payload = (request.payload ?? {}) as McpExportVideoPayload;
|
||||
const requestedSettings = payload.settings ?? {};
|
||||
const settings: ExportSettings = {
|
||||
format: requestedSettings.format ?? "mp4",
|
||||
quality: requestedSettings.quality ?? exportQuality,
|
||||
gifConfig:
|
||||
requestedSettings.format === "gif" && requestedSettings.gifConfig
|
||||
? requestedSettings.gifConfig
|
||||
: undefined,
|
||||
};
|
||||
if (settings.format === "gif" && !settings.gifConfig) {
|
||||
return {
|
||||
success: false,
|
||||
error: "MCP GIF export requires gifConfig dimensions and frame settings.",
|
||||
};
|
||||
}
|
||||
|
||||
return handleExport(settings, { targetPath: payload.outputPath });
|
||||
});
|
||||
|
||||
return () => {
|
||||
unsubscribe?.();
|
||||
};
|
||||
}, [exportQuality, handleExport, videoPath]);
|
||||
|
||||
const handleCancelExport = useCallback(() => {
|
||||
if (exporterRef.current) {
|
||||
exporterRef.current.cancel();
|
||||
|
||||
@@ -65,7 +65,11 @@ describe("buildGuideVideoAnnotations", () => {
|
||||
startMs: 1200,
|
||||
content: "1. Click Settings.",
|
||||
});
|
||||
expect(annotations[0]?.endMs).toBe(3200);
|
||||
expect(annotations[0]?.position.x).toBeGreaterThan(20);
|
||||
expect(annotations[1]?.endMs).toBe(3200);
|
||||
expect(annotations[1]?.position.x).toBeGreaterThan((annotations[0]?.position.x ?? 0) + 34);
|
||||
expect(annotations[1]?.position.y).toBeCloseTo(30.5);
|
||||
expect(annotations[1]).toMatchObject({
|
||||
id: "guide-video-2",
|
||||
type: "magnifier",
|
||||
@@ -79,10 +83,13 @@ describe("buildGuideVideoAnnotations", () => {
|
||||
expect(annotations[2]).toMatchObject({
|
||||
id: "guide-video-3",
|
||||
type: "figure",
|
||||
endMs: 3200,
|
||||
figureData: {
|
||||
arrowDirection: "left",
|
||||
color: "#34B27B",
|
||||
},
|
||||
});
|
||||
expect(annotations[2]?.position.x).toBeGreaterThan(20);
|
||||
});
|
||||
|
||||
it("returns an empty list when no draft exists", () => {
|
||||
@@ -97,7 +104,7 @@ describe("buildGuideVideoAnnotations", () => {
|
||||
expect(annotations).toEqual([]);
|
||||
});
|
||||
|
||||
it("creates 0.3x speed regions for one second at each guide point", () => {
|
||||
it("creates 0.3x speed regions for two seconds at each guide point", () => {
|
||||
let id = 1;
|
||||
const speedRegions = buildGuideVideoSpeedRegions(createSession(), {
|
||||
nextId: () => `guide-speed-${id++}`,
|
||||
@@ -107,7 +114,7 @@ describe("buildGuideVideoAnnotations", () => {
|
||||
{
|
||||
id: "guide-speed-1",
|
||||
startMs: 1200,
|
||||
endMs: 2200,
|
||||
endMs: 3200,
|
||||
speed: 0.3,
|
||||
},
|
||||
]);
|
||||
|
||||
@@ -14,13 +14,14 @@ export interface BuildGuideVideoAnnotationsOptions {
|
||||
defaultDurationMs?: number;
|
||||
}
|
||||
|
||||
const DEFAULT_STEP_DURATION_MS = 3200;
|
||||
const DEFAULT_STEP_SLOW_MOTION_DURATION_MS = 1000;
|
||||
const DEFAULT_STEP_DURATION_MS = 2000;
|
||||
const DEFAULT_STEP_SLOW_MOTION_DURATION_MS = 2000;
|
||||
const DEFAULT_STEP_SLOW_MOTION_SPEED = 0.3;
|
||||
const CAPTION_WIDTH = 34;
|
||||
const CAPTION_HEIGHT = 13;
|
||||
const MAGNIFIER_SIZE = 18;
|
||||
const ARROW_SIZE = 10;
|
||||
const ANNOTATION_GAP = 2;
|
||||
|
||||
function clamp(value: number, min: number, max: number) {
|
||||
return Math.min(max, Math.max(min, value));
|
||||
@@ -58,15 +59,19 @@ function getCaptionPosition(candidate: GuideStepCandidate | undefined) {
|
||||
|
||||
function getArrowDirection(
|
||||
candidate: GuideStepCandidate | undefined,
|
||||
captionPosition: { x: number; y: number },
|
||||
originPosition: { x: number; y: number },
|
||||
originSize: { width: number; height: number } = {
|
||||
width: CAPTION_WIDTH,
|
||||
height: CAPTION_HEIGHT,
|
||||
},
|
||||
): ArrowDirection {
|
||||
const target = candidate?.position;
|
||||
if (!target) return "right";
|
||||
|
||||
const captionCenterX = captionPosition.x + CAPTION_WIDTH / 2;
|
||||
const captionCenterY = captionPosition.y + CAPTION_HEIGHT / 2;
|
||||
const dx = target.normalizedX * 100 - captionCenterX;
|
||||
const dy = target.normalizedY * 100 - captionCenterY;
|
||||
const originCenterX = originPosition.x + originSize.width / 2;
|
||||
const originCenterY = originPosition.y + originSize.height / 2;
|
||||
const dx = target.normalizedX * 100 - originCenterX;
|
||||
const dy = target.normalizedY * 100 - originCenterY;
|
||||
const horizontal = dx > 8 ? "right" : dx < -8 ? "left" : "";
|
||||
const vertical = dy > 8 ? "down" : dy < -8 ? "up" : "";
|
||||
|
||||
@@ -74,6 +79,40 @@ function getArrowDirection(
|
||||
return (horizontal || vertical || "right") as ArrowDirection;
|
||||
}
|
||||
|
||||
function getMagnifierPosition(captionPosition: { x: number; y: number }) {
|
||||
const canPlaceRight = captionPosition.x + CAPTION_WIDTH + ANNOTATION_GAP + MAGNIFIER_SIZE <= 98;
|
||||
const x = canPlaceRight
|
||||
? captionPosition.x + CAPTION_WIDTH + ANNOTATION_GAP
|
||||
: captionPosition.x - MAGNIFIER_SIZE - ANNOTATION_GAP;
|
||||
const y = captionPosition.y + (CAPTION_HEIGHT - MAGNIFIER_SIZE) / 2;
|
||||
|
||||
return {
|
||||
x: clamp(x, 2, 100 - MAGNIFIER_SIZE - 2),
|
||||
y: clamp(y, 2, 100 - MAGNIFIER_SIZE - 2),
|
||||
};
|
||||
}
|
||||
|
||||
function getArrowPosition(
|
||||
position: NonNullable<GuideStepCandidate["position"]>,
|
||||
originPosition: { x: number; y: number },
|
||||
originSize: { width: number; height: number },
|
||||
) {
|
||||
const targetX = position.normalizedX * 100;
|
||||
const targetY = position.normalizedY * 100;
|
||||
const originCenterX = originPosition.x + originSize.width / 2;
|
||||
const originCenterY = originPosition.y + originSize.height / 2;
|
||||
const distance = Math.hypot(targetX - originCenterX, targetY - originCenterY);
|
||||
const targetOffset = Math.min(18, Math.max(10, distance * 0.35));
|
||||
const ratio = distance > 0 ? Math.max(0, (distance - targetOffset) / distance) : 0;
|
||||
const arrowCenterX = originCenterX + (targetX - originCenterX) * ratio;
|
||||
const arrowCenterY = originCenterY + (targetY - originCenterY) * ratio;
|
||||
|
||||
return {
|
||||
x: clamp(arrowCenterX - ARROW_SIZE / 2, 0, 100 - ARROW_SIZE),
|
||||
y: clamp(arrowCenterY - ARROW_SIZE / 2, 0, 100 - ARROW_SIZE),
|
||||
};
|
||||
}
|
||||
|
||||
function buildCaption(step: GeneratedGuideStep) {
|
||||
const instruction = step.instruction.trim();
|
||||
const title = step.title.trim();
|
||||
@@ -101,7 +140,6 @@ export function buildGuideVideoAnnotations(
|
||||
const startMs = Math.max(0, Math.round(candidate?.timeMs ?? index * durationMs));
|
||||
const endMs = Math.max(startMs + 750, startMs + durationMs);
|
||||
const captionPosition = getCaptionPosition(candidate);
|
||||
const arrowDirection = getArrowDirection(candidate, captionPosition);
|
||||
|
||||
annotations.push({
|
||||
id: options.nextId(),
|
||||
@@ -124,24 +162,23 @@ export function buildGuideVideoAnnotations(
|
||||
});
|
||||
|
||||
if (candidate?.position) {
|
||||
const magnifierPosition = getMagnifierPosition(captionPosition);
|
||||
const arrowPosition = getArrowPosition(candidate.position, magnifierPosition, {
|
||||
width: MAGNIFIER_SIZE,
|
||||
height: MAGNIFIER_SIZE,
|
||||
});
|
||||
const arrowDirection = getArrowDirection(candidate, arrowPosition, {
|
||||
width: ARROW_SIZE,
|
||||
height: ARROW_SIZE,
|
||||
});
|
||||
|
||||
annotations.push({
|
||||
id: options.nextId(),
|
||||
startMs,
|
||||
endMs,
|
||||
type: "magnifier",
|
||||
content: buildCaption(step),
|
||||
position: {
|
||||
x: clamp(
|
||||
candidate.position.normalizedX * 100 - MAGNIFIER_SIZE / 2,
|
||||
0,
|
||||
100 - MAGNIFIER_SIZE,
|
||||
),
|
||||
y: clamp(
|
||||
candidate.position.normalizedY * 100 - MAGNIFIER_SIZE / 2,
|
||||
0,
|
||||
100 - MAGNIFIER_SIZE,
|
||||
),
|
||||
},
|
||||
position: magnifierPosition,
|
||||
size: { width: MAGNIFIER_SIZE, height: MAGNIFIER_SIZE },
|
||||
style: { ...DEFAULT_ANNOTATION_STYLE },
|
||||
zIndex: options.nextZIndex(),
|
||||
@@ -160,10 +197,7 @@ export function buildGuideVideoAnnotations(
|
||||
endMs,
|
||||
type: "figure",
|
||||
content: "",
|
||||
position: {
|
||||
x: clamp(candidate.position.normalizedX * 100 - ARROW_SIZE / 2, 0, 100 - ARROW_SIZE),
|
||||
y: clamp(candidate.position.normalizedY * 100 - ARROW_SIZE / 2, 0, 100 - ARROW_SIZE),
|
||||
},
|
||||
position: arrowPosition,
|
||||
size: { width: ARROW_SIZE, height: ARROW_SIZE },
|
||||
style: { ...DEFAULT_ANNOTATION_STYLE },
|
||||
zIndex: options.nextZIndex(),
|
||||
|
||||
@@ -50,6 +50,7 @@ const WEBCAM_TARGET_FRAME_RATE = 30;
|
||||
type UseScreenRecorderReturn = {
|
||||
recording: boolean;
|
||||
paused: boolean;
|
||||
countdownActive: boolean;
|
||||
elapsedSeconds: number;
|
||||
toggleRecording: () => void;
|
||||
togglePaused: () => void;
|
||||
@@ -1783,6 +1784,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
|
||||
return {
|
||||
recording,
|
||||
paused,
|
||||
countdownActive,
|
||||
elapsedSeconds,
|
||||
toggleRecording,
|
||||
togglePaused,
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import type { ExportSettings } from "@/lib/exporter";
|
||||
|
||||
export type McpControlAction =
|
||||
| "list_sources"
|
||||
| "record_video"
|
||||
| "stop_recording"
|
||||
| "export_video"
|
||||
| "status";
|
||||
|
||||
export interface McpControlRequest {
|
||||
id: string;
|
||||
action: McpControlAction;
|
||||
payload?: unknown;
|
||||
}
|
||||
|
||||
export interface McpRecordVideoPayload {
|
||||
guideMode?: boolean;
|
||||
sourceType?: "screen" | "window";
|
||||
sourceId?: string;
|
||||
sourceName?: string;
|
||||
displayIndex?: number;
|
||||
}
|
||||
|
||||
export interface McpStopRecordingPayload {
|
||||
discard?: boolean;
|
||||
}
|
||||
|
||||
export interface McpExportVideoPayload {
|
||||
outputPath?: string;
|
||||
settings?: Partial<ExportSettings>;
|
||||
}
|
||||
|
||||
export interface McpControlResult {
|
||||
success: boolean;
|
||||
message?: string;
|
||||
recording?: boolean;
|
||||
path?: string;
|
||||
url?: string;
|
||||
data?: unknown;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export function isMcpControlAction(value: unknown): value is McpControlAction {
|
||||
return (
|
||||
value === "list_sources" ||
|
||||
value === "record_video" ||
|
||||
value === "stop_recording" ||
|
||||
value === "export_video" ||
|
||||
value === "status"
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user