5 Commits

Author SHA1 Message Date
huanld 5912a462e4 Fix Gitea release asset publishing 2026-06-25 02:38:25 +07:00
huanld b225f38188 Add Gitea Windows release workflow 2026-06-25 02:32:13 +07:00
huanld 5272c0696a Document MCP screen recording
CI / Lint (push) Waiting to run
CI / Type Check (push) Waiting to run
CI / Test (push) Waiting to run
CI / Build (push) Waiting to run
2026-06-25 02:08:35 +07:00
huanld aae562f146 Add MCP recording controls
CI / Lint (push) Waiting to run
CI / Type Check (push) Waiting to run
CI / Test (push) Waiting to run
CI / Build (push) Waiting to run
Bump Nix package on release / bump (release) Waiting to run
Update Homebrew Cask / update-cask (release) Waiting to run
2026-06-25 01:55:42 +07:00
huanld 5069354df3 Adjust guide video annotation timing
CI / Lint (push) Has been cancelled
CI / Type Check (push) Has been cancelled
CI / Test (push) Has been cancelled
CI / Build (push) Has been cancelled
Bump Nix package on release / bump (release) Has been cancelled
Update Homebrew Cask / update-cask (release) Has been cancelled
2026-06-05 20:39:26 +07:00
15 changed files with 2112 additions and 63 deletions
+233
View File
@@ -0,0 +1,233 @@
name: Windows Release
on:
workflow_dispatch:
push:
tags:
- "v*"
permissions:
contents: write
jobs:
build-and-publish:
runs-on: windows-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: 22
cache: npm
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Install dependencies
run: npm ci
- name: Build Windows setup
run: npm run build:win
- name: Normalize release artifacts
id: artifacts
shell: pwsh
run: |
$version = node -p "require('./package.json').version"
$releaseDir = "release/$version"
$setupName = "Openscreen-Setup-$version.exe"
$blockmapName = "$setupName.blockmap"
$setupPath = Join-Path $releaseDir "Openscreen Setup $version.exe"
$blockmapPath = Join-Path $releaseDir "Openscreen Setup $version.exe.blockmap"
$normalizedSetupPath = Join-Path $releaseDir $setupName
$normalizedBlockmapPath = Join-Path $releaseDir $blockmapName
$latestPath = Join-Path $releaseDir "latest.yml"
if (!(Test-Path -LiteralPath $setupPath)) {
throw "Missing Windows setup file: $setupPath"
}
if (!(Test-Path -LiteralPath $blockmapPath)) {
throw "Missing Windows setup blockmap: $blockmapPath"
}
if (!(Test-Path -LiteralPath $latestPath)) {
throw "Missing electron-builder metadata: $latestPath"
}
Copy-Item -LiteralPath $setupPath -Destination $normalizedSetupPath -Force
Copy-Item -LiteralPath $blockmapPath -Destination $normalizedBlockmapPath -Force
(Get-Content -LiteralPath $latestPath -Raw) `
-replace "Openscreen Setup $version\.exe", $setupName |
Set-Content -LiteralPath $latestPath -Encoding utf8
$hash = Get-FileHash -LiteralPath $normalizedSetupPath -Algorithm SHA256
$sha256Path = "$normalizedSetupPath.sha256"
"$($hash.Hash.ToLowerInvariant()) $setupName" |
Set-Content -LiteralPath $sha256Path -Encoding ascii
"version=$version" >> $env:GITHUB_OUTPUT
"tag=v$version" >> $env:GITHUB_OUTPUT
"release_dir=$releaseDir" >> $env:GITHUB_OUTPUT
"setup_name=$setupName" >> $env:GITHUB_OUTPUT
"setup_path=$normalizedSetupPath" >> $env:GITHUB_OUTPUT
"blockmap_path=$normalizedBlockmapPath" >> $env:GITHUB_OUTPUT
"sha256_path=$sha256Path" >> $env:GITHUB_OUTPUT
"latest_path=$latestPath" >> $env:GITHUB_OUTPUT
- name: Upload workflow artifact
uses: actions/upload-artifact@v4
with:
name: openscreen-windows-${{ steps.artifacts.outputs.version }}
path: |
${{ steps.artifacts.outputs.setup_path }}
${{ steps.artifacts.outputs.blockmap_path }}
${{ steps.artifacts.outputs.sha256_path }}
${{ steps.artifacts.outputs.latest_path }}
retention-days: 30
- name: Ensure release tag exists
shell: pwsh
run: |
$tag = "${{ steps.artifacts.outputs.tag }}"
git config user.name "gitea-actions"
git config user.email "actions@gitea.local"
git fetch --tags
if (!(git tag --list $tag)) {
git tag -a $tag -m "Release $tag"
git push origin $tag
}
- name: Publish Gitea release assets
shell: pwsh
env:
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
run: |
if (!$env:GITEA_TOKEN) {
throw "Missing secrets.GITEA_TOKEN. Gitea Actions should provide this built-in token."
}
$apiBase = "${{ github.server_url }}/api/v1"
$repo = "${{ github.repository }}"
$tag = "${{ steps.artifacts.outputs.tag }}"
$version = "${{ steps.artifacts.outputs.version }}"
$setupName = "${{ steps.artifacts.outputs.setup_name }}"
$rawBase = "${{ github.server_url }}/${{ github.repository }}/raw/branch/release-assets%2F$tag"
$headers = @{
Authorization = "token $env:GITEA_TOKEN"
Accept = "application/json"
}
$releaseBody = @"
OpenScreen Windows setup $tag
Setup EXE:
$rawBase/$setupName
SHA256:
$rawBase/$setupName.sha256
Auto-update feed:
${{ github.server_url }}/${{ github.repository }}/raw/branch/release-assets%2Flatest/latest.yml
"@
try {
$release = Invoke-RestMethod `
-Method Get `
-Uri "$apiBase/repos/$repo/releases/tags/$tag" `
-Headers $headers
} catch {
$body = @{
tag_name = $tag
target_commitish = "${{ github.sha }}"
name = $tag
body = $releaseBody
draft = $false
prerelease = $false
} | ConvertTo-Json
$release = Invoke-RestMethod `
-Method Post `
-Uri "$apiBase/repos/$repo/releases" `
-Headers $headers `
-ContentType "application/json" `
-Body $body
}
$updateBody = @{
tag_name = $tag
target_commitish = "${{ github.sha }}"
name = "OpenScreen $version"
body = $releaseBody
draft = $false
prerelease = $false
} | ConvertTo-Json
$release = Invoke-RestMethod `
-Method Patch `
-Uri "$apiBase/repos/$repo/releases/$($release.id)" `
-Headers $headers `
-ContentType "application/json" `
-Body $updateBody
$assets = @(
"${{ steps.artifacts.outputs.blockmap_path }}",
"${{ steps.artifacts.outputs.sha256_path }}",
"${{ steps.artifacts.outputs.latest_path }}"
)
foreach ($assetPath in $assets) {
$assetName = Split-Path -Leaf $assetPath
$existing = @($release.assets | Where-Object { $_.name -eq $assetName })
foreach ($asset in $existing) {
Invoke-RestMethod `
-Method Delete `
-Uri "$apiBase/repos/$repo/releases/$($release.id)/assets/$($asset.id)" `
-Headers $headers | Out-Null
}
$uploadUri = "$apiBase/repos/$repo/releases/$($release.id)/assets?name=$([uri]::EscapeDataString($assetName))"
& curl.exe `
-sS `
-f `
-X POST `
-H "Authorization: token $env:GITEA_TOKEN" `
-H "Accept: application/json" `
-F "attachment=@$assetPath" `
$uploadUri | Out-Null
if ($LASTEXITCODE -ne 0) {
throw "Failed to upload release asset: $assetName"
}
}
- name: Publish generic updater branch
shell: pwsh
run: |
$version = "${{ steps.artifacts.outputs.version }}"
$tag = "${{ steps.artifacts.outputs.tag }}"
$branch = "release-assets/$tag"
$releaseDir = "${{ steps.artifacts.outputs.release_dir }}"
$setupName = "${{ steps.artifacts.outputs.setup_name }}"
$blockmapName = "$setupName.blockmap"
$sha256Name = "$setupName.sha256"
git config user.name "gitea-actions"
git config user.email "actions@gitea.local"
git checkout -B $branch
Copy-Item -LiteralPath (Join-Path $releaseDir $setupName) -Destination $setupName -Force
Copy-Item -LiteralPath (Join-Path $releaseDir $blockmapName) -Destination $blockmapName -Force
Copy-Item -LiteralPath (Join-Path $releaseDir $sha256Name) -Destination $sha256Name -Force
Copy-Item -LiteralPath (Join-Path $releaseDir "latest.yml") -Destination "latest.yml" -Force
git add latest.yml $setupName $blockmapName $sha256Name
if (git status --short) {
git commit -m "Publish $tag Windows release assets"
} else {
Write-Host "Release asset branch is already up to date."
}
git push origin $branch --force
git push origin "${branch}:release-assets/latest" --force
+281
View File
@@ -0,0 +1,281 @@
# MCP Screen Recording
OpenScreen exposes a local MCP server so an AI client can start a screen or window recording, stop it, export the loaded editor project, and receive a local file URL for the result.
## Architecture
The integration has two layers:
- `scripts/openscreen-mcp-server.mjs` is the MCP stdio server. MCP clients start this process and call its tools.
- `electron/mcpControlServer.ts` is a local HTTP control server inside the Electron app. It listens on `127.0.0.1:52347` by default and forwards MCP actions to the renderer through the preload bridge.
The renderer owns the actual recording and export behavior:
- `LaunchWindow` handles `list_sources`, `record_video`, `stop_recording`, and recording status.
- `VideoEditor` handles `export_video` and editor status.
- The control server adds a `file://` URL whenever a successful result includes a filesystem path.
OpenScreen must be running before MCP tools can control it. The HTTP control server is local-only and is not exposed on the network.
## Configuration
From a source checkout, configure an MCP client to run the server with Node:
```json
{
"mcpServers": {
"openscreen": {
"command": "node",
"args": ["D:\\Code\\OpenScreen\\scripts\\openscreen-mcp-server.mjs"],
"env": {
"OPENSCREEN_MCP_CONTROL_URL": "http://127.0.0.1:52347"
}
}
}
}
```
For local development, this is equivalent to:
```powershell
npm run mcp
```
Optional environment variables:
| Variable | Default | Purpose |
| --- | --- | --- |
| `OPENSCREEN_MCP_CONTROL_URL` | `http://127.0.0.1:52347` | URL used by the MCP stdio server to reach OpenScreen. |
| `OPENSCREEN_MCP_CONTROL_TOKEN` | unset | Shared bearer token. Set it on both the Electron app and MCP server to require authorization. |
| `OPENSCREEN_MCP_CONTROL_PORT` | `52347` | Port used by the Electron control server. Set it on the Electron app process. |
If `OPENSCREEN_MCP_CONTROL_TOKEN` is set, the MCP server sends:
```http
Authorization: Bearer <token>
```
## Tools
### `list_sources`
Lists available capture sources. Results include both screens and windows and can be used to choose a `sourceId`.
Input:
```json
{}
```
Typical result:
```json
{
"success": true,
"data": [
{
"id": "screen:0:0",
"name": "Entire Screen",
"type": "screen",
"displayIndex": 0,
"bounds": { "x": 0, "y": 0, "width": 1920, "height": 1080 }
},
{
"id": "window:123456:0",
"name": "Example App",
"type": "window"
}
]
}
```
### `record_video`
Starts recording. If no source is supplied, OpenScreen selects the current/default source. When `sourceType` is provided, OpenScreen can choose between a full screen and a window.
Input:
```json
{
"guideMode": false,
"sourceType": "screen",
"sourceId": "screen:0:0",
"sourceName": "Entire Screen",
"displayIndex": 0
}
```
Fields:
| Field | Type | Description |
| --- | --- | --- |
| `guideMode` | boolean | Enables Guide Mode for the recording. |
| `sourceType` | `"screen"` or `"window"` | Restricts source selection to a screen/display or an app window. |
| `sourceId` | string | Exact source ID returned by `list_sources`. This has highest priority. |
| `sourceName` | string | Exact or partial source/window name used when `sourceId` is omitted. |
| `displayIndex` | number | Zero-based display index for screen capture. |
Source selection priority:
1. Exact `sourceId`
2. `displayIndex`
3. Exact or partial `sourceName`
4. First matching screen
5. First available source
Example: record the primary screen.
```json
{
"sourceType": "screen",
"displayIndex": 0
}
```
Example: record a specific window.
```json
{
"sourceType": "window",
"sourceName": "Chrome"
}
```
### `stop_recording`
Stops the active recording. The saved video path and URL are returned when the recording is kept.
Input:
```json
{
"discard": false
}
```
Typical result:
```json
{
"success": true,
"path": "C:\\Users\\user\\AppData\\Roaming\\openscreen\\recordings\\recording-123.mp4",
"url": "file:///C:/Users/user/AppData/Roaming/openscreen/recordings/recording-123.mp4"
}
```
Use `"discard": true` to cancel and remove the recording instead of saving it.
### `export_video`
Exports the currently loaded editor project. If `outputPath` is omitted, OpenScreen writes to the user's Downloads folder and returns the generated path and URL.
Input:
```json
{
"outputPath": "C:\\Users\\user\\Downloads\\demo.mp4",
"format": "mp4",
"quality": "good"
}
```
Fields:
| Field | Type | Description |
| --- | --- | --- |
| `outputPath` | string | Absolute output path. Defaults to Downloads. |
| `format` | `"mp4"` or `"gif"` | Export format. Defaults to `mp4`. |
| `quality` | `"medium"`, `"good"`, or `"source"` | MP4 quality preset. Defaults to `good`. |
Typical result:
```json
{
"success": true,
"path": "C:\\Users\\user\\Downloads\\demo.mp4",
"url": "file:///C:/Users/user/Downloads/demo.mp4"
}
```
### `status`
Returns whether OpenScreen is currently recording. In the editor it also reports whether the editor is ready.
Input:
```json
{}
```
Typical result:
```json
{
"success": true,
"recording": false
}
```
## Recommended Workflow
1. Start OpenScreen.
2. Call `list_sources`.
3. Pick a source:
- Use `sourceType: "screen"` plus `displayIndex` for a monitor.
- Use `sourceType: "window"` plus `sourceId` or `sourceName` for an app window.
4. Call `record_video`.
5. Call `status` if the client needs to verify recording state.
6. Call `stop_recording`.
7. Optionally call `export_video` after OpenScreen opens the editor for the saved recording.
8. Use the returned `url` field as the exported video URL.
## Direct Control API
The MCP server is the supported integration surface, but the Electron app also exposes the local control endpoints used by the MCP server:
| Endpoint | Method | Action |
| --- | --- | --- |
| `/health` | `GET` | Returns whether the local OpenScreen control server is alive. |
| `/mcp/list_sources` | `POST` | Lists screen/window sources. |
| `/mcp/record_video` | `POST` | Starts recording. |
| `/mcp/stop_recording` | `POST` | Stops or discards recording. |
| `/mcp/export_video` | `POST` | Exports the current editor project. |
| `/mcp/status` | `POST` | Returns recording/editor status. |
PowerShell health check:
```powershell
Invoke-WebRequest -Uri "http://127.0.0.1:52347/health" -UseBasicParsing
```
Direct recording example:
```powershell
$body = @{
sourceType = "screen"
displayIndex = 0
} | ConvertTo-Json
Invoke-RestMethod `
-Method Post `
-Uri "http://127.0.0.1:52347/mcp/record_video" `
-ContentType "application/json" `
-Body $body
```
## Troubleshooting
| Symptom | Cause | Fix |
| --- | --- | --- |
| `OpenScreen window is not available` | Electron app is not running or not ready. | Start OpenScreen and wait for the window to load. |
| `Unauthorized` | Token mismatch. | Set the same `OPENSCREEN_MCP_CONTROL_TOKEN` for both OpenScreen and the MCP server. |
| `Unsupported MCP action` | Wrong endpoint/tool name. | Use one of the documented tool names. |
| Timeout while handling an action | Renderer did not respond within 120 seconds. | Bring OpenScreen to the foreground, check the selected source, and retry. |
| Export returns an error | No editor project is loaded. | Stop a recording first or open a recording in the editor before exporting. |
## Security Notes
- The control server binds to `127.0.0.1` only.
- Set `OPENSCREEN_MCP_CONTROL_TOKEN` when multiple local processes or users can reach the machine.
- Do not expose the control port through a tunnel or reverse proxy.
- Treat returned file URLs as local machine paths; they are not public web URLs.
+7
View File
@@ -365,6 +365,13 @@ interface Window {
showCountdownOverlay: (value: number, runId: number) => Promise<void>; showCountdownOverlay: (value: number, runId: number) => Promise<void>;
setCountdownOverlayValue: (value: number, runId: number) => Promise<void>; setCountdownOverlayValue: (value: number, runId: number) => Promise<void>;
hideCountdownOverlay: (runId: number) => Promise<void>; hideCountdownOverlay: (runId: number) => Promise<void>;
onMcpControlRequest: (
callback: (
request: import("../src/lib/mcpControl").McpControlRequest,
) =>
| Promise<import("../src/lib/mcpControl").McpControlResult>
| import("../src/lib/mcpControl").McpControlResult,
) => () => void;
onCountdownOverlayValue: (callback: (value: number | null) => void) => () => void; onCountdownOverlayValue: (callback: (value: number | null) => void) => () => void;
setMicrophoneExpanded: (expanded: boolean) => void; setMicrophoneExpanded: (expanded: boolean) => void;
setHasUnsavedChanges: (hasChanges: boolean) => void; setHasUnsavedChanges: (hasChanges: boolean) => void;
+12 -1
View File
@@ -13,6 +13,7 @@ import {
} from "electron"; } from "electron";
import { mainT, setMainLocale } from "./i18n"; import { mainT, setMainLocale } from "./i18n";
import { getSelectedDesktopSource, registerIpcHandlers } from "./ipc/handlers"; import { getSelectedDesktopSource, registerIpcHandlers } from "./ipc/handlers";
import { startMcpControlServer } from "./mcpControlServer";
import { initializeAutoUpdates } from "./updater"; import { initializeAutoUpdates } from "./updater";
import { import {
createCountdownOverlayWindow, createCountdownOverlayWindow,
@@ -519,7 +520,6 @@ app.whenReady().then(async () => {
initializeAutoUpdates(); initializeAutoUpdates();
// Ensure recordings directory exists // Ensure recordings directory exists
await ensureRecordingsDir(); await ensureRecordingsDir();
function switchToHudWrapper() { function switchToHudWrapper() {
if (mainWindow) { if (mainWindow) {
isForceClosing = true; isForceClosing = true;
@@ -530,6 +530,17 @@ app.whenReady().then(async () => {
showMainWindow(); showMainWindow();
} }
startMcpControlServer({
getMainWindow: () => mainWindow,
ensureWindow: (action) => {
if (action === "list_sources" || action === "record_video") {
switchToHudWrapper();
return;
}
showMainWindow();
},
});
registerIpcHandlers( registerIpcHandlers(
createEditorWindowWrapper, createEditorWindowWrapper,
createSourceSelectorWindowWrapper, createSourceSelectorWindowWrapper,
+179
View File
@@ -0,0 +1,179 @@
import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { app, type BrowserWindow, ipcMain } from "electron";
import type { McpControlAction, McpControlRequest, McpControlResult } from "../src/lib/mcpControl";
import { isMcpControlAction } from "../src/lib/mcpControl";
const DEFAULT_MCP_CONTROL_PORT = 52347;
const MCP_CONTROL_REQUEST_TIMEOUT_MS = 120_000;
type GetWindow = () => BrowserWindow | null;
interface PendingMcpRequest {
resolve: (result: McpControlResult) => void;
reject: (error: Error) => void;
timeout: NodeJS.Timeout;
}
interface StartMcpControlServerOptions {
getMainWindow: GetWindow;
ensureWindow: (action: McpControlAction) => void;
}
const pendingRequests = new Map<string, PendingMcpRequest>();
function sendJson(response: ServerResponse, statusCode: number, body: unknown) {
response.writeHead(statusCode, {
"content-type": "application/json; charset=utf-8",
});
response.end(JSON.stringify(body));
}
async function readJsonBody(request: IncomingMessage) {
const chunks: Buffer[] = [];
for await (const chunk of request) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
}
const text = Buffer.concat(chunks).toString("utf-8").trim();
return text ? JSON.parse(text) : {};
}
function withFileUrl(result: McpControlResult): McpControlResult {
if (!result.success || result.url || !result.path) {
return result;
}
return {
...result,
url: pathToFileURL(result.path).toString(),
};
}
function resolveDefaultExportPath(payload: unknown) {
if (
!payload ||
typeof payload !== "object" ||
("outputPath" in payload && typeof payload.outputPath === "string" && payload.outputPath)
) {
return payload;
}
const requested = payload as { outputPath?: unknown; settings?: { format?: unknown } };
const format = requested.settings?.format === "gif" ? "gif" : "mp4";
const fileName = `openscreen-export-${Date.now()}.${format}`;
return {
...requested,
outputPath: path.join(app.getPath("downloads"), fileName),
};
}
function dispatchRendererRequest(
window: BrowserWindow,
action: McpControlAction,
payload: unknown,
): Promise<McpControlResult> {
return new Promise((resolve, reject) => {
const id = `${Date.now()}-${Math.random().toString(16).slice(2)}`;
const timeout = setTimeout(() => {
pendingRequests.delete(id);
reject(new Error(`Timed out waiting for renderer to handle ${action}`));
}, MCP_CONTROL_REQUEST_TIMEOUT_MS);
pendingRequests.set(id, { resolve, reject, timeout });
const request: McpControlRequest = { id, action, payload };
window.webContents.send("mcp-control-request", request);
});
}
function registerMcpControlIpc() {
ipcMain.on(
"mcp-control-response",
(_event, response: { id?: unknown; result?: McpControlResult }) => {
if (typeof response.id !== "string") {
return;
}
const pending = pendingRequests.get(response.id);
if (!pending) {
return;
}
clearTimeout(pending.timeout);
pendingRequests.delete(response.id);
pending.resolve(
response.result ?? { success: false, error: "Renderer returned an empty result" },
);
},
);
}
export function startMcpControlServer({
getMainWindow,
ensureWindow,
}: StartMcpControlServerOptions): Server {
registerMcpControlIpc();
const token = process.env.OPENSCREEN_MCP_CONTROL_TOKEN;
const port = Number(process.env.OPENSCREEN_MCP_CONTROL_PORT) || DEFAULT_MCP_CONTROL_PORT;
const server = createServer(async (request, response) => {
try {
const url = new URL(request.url ?? "/", "http://127.0.0.1");
if (request.method === "GET" && url.pathname === "/health") {
sendJson(response, 200, { success: true, app: "OpenScreen" });
return;
}
if (request.method !== "POST" || !url.pathname.startsWith("/mcp/")) {
sendJson(response, 404, { success: false, error: "Not found" });
return;
}
if (token && request.headers.authorization !== `Bearer ${token}`) {
sendJson(response, 401, { success: false, error: "Unauthorized" });
return;
}
const action = url.pathname.slice("/mcp/".length);
if (!isMcpControlAction(action)) {
sendJson(response, 400, { success: false, error: `Unsupported MCP action: ${action}` });
return;
}
const body = await readJsonBody(request);
const payload = action === "export_video" ? resolveDefaultExportPath(body) : body;
ensureWindow(action);
const window = getMainWindow();
if (!window || window.isDestroyed()) {
sendJson(response, 503, { success: false, error: "OpenScreen window is not available" });
return;
}
if (window.webContents.isLoading()) {
await new Promise<void>((resolve) => {
window.webContents.once("did-finish-load", () => resolve());
});
}
const result = withFileUrl(await dispatchRendererRequest(window, action, payload));
sendJson(response, result.success ? 200 : 409, result);
} catch (error) {
sendJson(response, 500, {
success: false,
error: error instanceof Error ? error.message : String(error),
});
}
});
server.listen(port, "127.0.0.1", () => {
console.info(`[mcp-control] listening on http://127.0.0.1:${port}`);
});
server.on("error", (error) => {
console.error("[mcp-control] failed to start:", error);
});
app.once("before-quit", () => {
server.close();
});
return server;
}
+21
View File
@@ -12,6 +12,7 @@ import type {
SaveGuideInput, SaveGuideInput,
WriteGuideSnapshotInput, WriteGuideSnapshotInput,
} from "../src/guide/contracts"; } from "../src/guide/contracts";
import type { McpControlRequest, McpControlResult } from "../src/lib/mcpControl";
import type { NativeMacRecordingRequest } from "../src/lib/nativeMacRecording"; import type { NativeMacRecordingRequest } from "../src/lib/nativeMacRecording";
import type { NativeWindowsRecordingRequest } from "../src/lib/nativeWindowsRecording"; import type { NativeWindowsRecordingRequest } from "../src/lib/nativeWindowsRecording";
import type { RecordingSession, StoreRecordedSessionInput } from "../src/lib/recordingSession"; import type { RecordingSession, StoreRecordedSessionInput } from "../src/lib/recordingSession";
@@ -315,6 +316,26 @@ contextBridge.exposeInMainWorld("electronAPI", {
hideCountdownOverlay: (runId: number) => { hideCountdownOverlay: (runId: number) => {
return ipcRenderer.invoke("countdown-overlay-hide", runId); return ipcRenderer.invoke("countdown-overlay-hide", runId);
}, },
onMcpControlRequest: (
callback: (request: McpControlRequest) => Promise<McpControlResult> | McpControlResult,
) => {
const listener = async (_event: unknown, request: McpControlRequest) => {
try {
const result = await callback(request);
ipcRenderer.send("mcp-control-response", { id: request.id, result });
} catch (error) {
ipcRenderer.send("mcp-control-response", {
id: request.id,
result: {
success: false,
error: error instanceof Error ? error.message : String(error),
},
});
}
};
ipcRenderer.on("mcp-control-request", listener);
return () => ipcRenderer.removeListener("mcp-control-request", listener);
},
onCountdownOverlayValue: (callback: (value: number | null) => void) => { onCountdownOverlayValue: (callback: (value: number | null) => void) => {
const listener = (_event: unknown, value: number | null) => callback(value); const listener = (_event: unknown, value: number | null) => callback(value);
ipcRenderer.on("countdown-overlay-value", listener); ipcRenderer.on("countdown-overlay-value", listener);
+870 -24
View File
File diff suppressed because it is too large Load Diff
+3 -1
View File
@@ -1,7 +1,7 @@
{ {
"name": "openscreen", "name": "openscreen",
"private": true, "private": true,
"version": "1.4.11", "version": "1.4.13",
"type": "module", "type": "module",
"packageManager": "npm@10.9.4", "packageManager": "npm@10.9.4",
"engines": { "engines": {
@@ -20,6 +20,7 @@
"format": "biome format --write .", "format": "biome format --write .",
"i18n:check": "node scripts/i18n-check.mjs", "i18n:check": "node scripts/i18n-check.mjs",
"preview": "vite preview", "preview": "vite preview",
"mcp": "node scripts/openscreen-mcp-server.mjs",
"build:native:mac": "node scripts/build-macos-screencapturekit-helper.mjs", "build:native:mac": "node scripts/build-macos-screencapturekit-helper.mjs",
"build:mac": "npm run build:native:mac && tsc && vite build && electron-builder --mac --config electron-builder.json5", "build:mac": "npm run build:native:mac && tsc && vite build && electron-builder --mac --config electron-builder.json5",
"build:native:win": "node scripts/build-windows-wgc-helper.mjs", "build:native:win": "node scripts/build-windows-wgc-helper.mjs",
@@ -50,6 +51,7 @@
}, },
"dependencies": { "dependencies": {
"@fix-webm-duration/fix": "^1.0.1", "@fix-webm-duration/fix": "^1.0.1",
"@modelcontextprotocol/sdk": "^1.29.0",
"@pixi/filter-drop-shadow": "^5.2.0", "@pixi/filter-drop-shadow": "^5.2.0",
"@radix-ui/react-accordion": "^1.2.12", "@radix-ui/react-accordion": "^1.2.12",
"@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-dialog": "^1.1.15",
+133
View File
@@ -0,0 +1,133 @@
#!/usr/bin/env node
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
import { z } from "zod";
const controlUrl = process.env.OPENSCREEN_MCP_CONTROL_URL || "http://127.0.0.1:52347";
const token = process.env.OPENSCREEN_MCP_CONTROL_TOKEN;
async function callOpenScreen(action, payload = {}) {
const response = await fetch(`${controlUrl.replace(/\/$/, "")}/mcp/${action}`, {
method: "POST",
headers: {
"content-type": "application/json",
...(token ? { authorization: `Bearer ${token}` } : {}),
},
body: JSON.stringify(payload),
});
const result = await response.json().catch(() => ({
success: false,
error: `OpenScreen returned HTTP ${response.status}`,
}));
if (!response.ok || !result.success) {
throw new Error(
result.error || result.message || `OpenScreen returned HTTP ${response.status}`,
);
}
return result;
}
function textResult(result) {
return {
content: [
{
type: "text",
text: JSON.stringify(result, null, 2),
},
],
};
}
const server = new McpServer({
name: "openscreen",
version: "1.0.0",
});
server.registerTool(
"list_sources",
{
title: "List OpenScreen capture sources",
description: "List available screen and window sources that can be passed to record_video.",
inputSchema: {},
},
async () => textResult(await callOpenScreen("list_sources")),
);
server.registerTool(
"record_video",
{
title: "Start OpenScreen recording",
description:
"Start recording with a selected screen/window source, or the current/default source.",
inputSchema: {
guideMode: z.boolean().optional().describe("Enable Guide Mode for this recording."),
sourceType: z
.enum(["screen", "window"])
.optional()
.describe("Capture a screen/display or a window."),
sourceId: z
.string()
.optional()
.describe("Exact source id returned by list_sources, for example screen:0:0."),
sourceName: z
.string()
.optional()
.describe("Exact or partial source/window name to match when sourceId is not supplied."),
displayIndex: z
.number()
.int()
.nonnegative()
.optional()
.describe("Zero-based display index for screen capture."),
},
},
async (input) => textResult(await callOpenScreen("record_video", input)),
);
server.registerTool(
"stop_recording",
{
title: "Stop OpenScreen recording",
description: "Stop the active OpenScreen recording and return the saved video file URL.",
inputSchema: {
discard: z.boolean().optional().describe("Discard the recording instead of saving it."),
},
},
async (input) => textResult(await callOpenScreen("stop_recording", input)),
);
server.registerTool(
"export_video",
{
title: "Export OpenScreen video",
description:
"Export the currently loaded OpenScreen editor project and return the exported file URL.",
inputSchema: {
outputPath: z.string().optional().describe("Absolute output path. Defaults to Downloads."),
format: z.enum(["mp4", "gif"]).optional().describe("Export format. Defaults to mp4."),
quality: z.enum(["medium", "good", "source"]).optional().describe("MP4 quality preset."),
},
},
async ({ outputPath, format, quality }) =>
textResult(
await callOpenScreen("export_video", {
outputPath,
settings: {
format: format ?? "mp4",
quality: quality ?? "good",
},
}),
),
);
server.registerTool(
"status",
{
title: "Get OpenScreen status",
description: "Return whether OpenScreen is currently recording.",
inputSchema: {},
},
async () => textResult(await callOpenScreen("status")),
);
await server.connect(new StdioServerTransport());
+172
View File
@@ -22,6 +22,11 @@ import { RxDragHandleDots2 } from "react-icons/rx";
import { toast } from "sonner"; import { toast } from "sonner";
import { useI18n, useScopedT } from "@/contexts/I18nContext"; import { useI18n, useScopedT } from "@/contexts/I18nContext";
import { getAvailableLocales, getLocaleName } from "@/i18n/loader"; import { getAvailableLocales, getLocaleName } from "@/i18n/loader";
import type {
McpControlResult,
McpRecordVideoPayload,
McpStopRecordingPayload,
} from "@/lib/mcpControl";
import { nativeBridgeClient } from "@/native"; import { nativeBridgeClient } from "@/native";
import { useAudioLevelMeter } from "../../hooks/useAudioLevelMeter"; import { useAudioLevelMeter } from "../../hooks/useAudioLevelMeter";
import { useCameraDevices } from "../../hooks/useCameraDevices"; import { useCameraDevices } from "../../hooks/useCameraDevices";
@@ -96,6 +101,7 @@ export function LaunchWindow() {
const { const {
recording, recording,
paused, paused,
countdownActive,
elapsedSeconds, elapsedSeconds,
toggleRecording, toggleRecording,
togglePaused, togglePaused,
@@ -326,6 +332,16 @@ export function LaunchWindow() {
const [selectedSource, setSelectedSource] = useState("Screen"); const [selectedSource, setSelectedSource] = useState("Screen");
const [hasSelectedSource, setHasSelectedSource] = useState(false); const [hasSelectedSource, setHasSelectedSource] = useState(false);
const [, setRecordPointerDownCount] = useState(0); const [, setRecordPointerDownCount] = useState(0);
const recordingRef = useRef(recording);
const countdownActiveRef = useRef(countdownActive);
useEffect(() => {
recordingRef.current = recording;
}, [recording]);
useEffect(() => {
countdownActiveRef.current = countdownActive;
}, [countdownActive]);
useEffect(() => { useEffect(() => {
const checkSelectedSource = async () => { const checkSelectedSource = async () => {
@@ -376,6 +392,162 @@ export function LaunchWindow() {
await window.electronAPI.switchToEditor(); await window.electronAPI.switchToEditor();
}; };
const getMcpSources = useCallback(async () => {
const sources = await window.electronAPI.getSources({
types: ["screen", "window"],
thumbnailSize: { width: 0, height: 0 },
fetchWindowIcons: true,
});
return sources;
}, []);
const toMcpSourceSummary = useCallback((source: ProcessedDesktopSource) => {
return {
id: source.id,
name: source.name,
type: source.id.startsWith("window:") ? "window" : "screen",
displayId: source.displayId,
displayIndex: source.displayIndex ?? source.screenIndex,
displayLabel: source.displayLabel,
bounds: source.bounds,
};
}, []);
const selectMcpSource = useCallback(
async (payload: McpRecordVideoPayload = {}) => {
const sources = await getMcpSources();
const requestedType = payload.sourceType;
const sourceName = payload.sourceName?.trim().toLowerCase();
const typedSources = requestedType
? sources.filter((source) =>
requestedType === "window"
? source.id.startsWith("window:")
: source.id.startsWith("screen:"),
)
: sources;
const source =
(payload.sourceId
? typedSources.find((item) => item.id === payload.sourceId)
: undefined) ??
(typeof payload.displayIndex === "number"
? typedSources.find(
(item) =>
item.displayIndex === payload.displayIndex ||
item.screenIndex === payload.displayIndex,
)
: undefined) ??
(sourceName
? (typedSources.find((item) => item.name.toLowerCase() === sourceName) ??
typedSources.find((item) => item.name.toLowerCase().includes(sourceName)))
: undefined) ??
typedSources.find((item) => item.id.startsWith("screen:")) ??
typedSources[0] ??
sources.find((item) => item.id.startsWith("screen:")) ??
sources[0];
if (!source) {
return null;
}
const selected = await window.electronAPI.selectSource(source);
if (selected) {
setSelectedSource(selected.name);
setHasSelectedSource(true);
}
return selected;
},
[getMcpSources],
);
useEffect(() => {
const unsubscribe = window.electronAPI?.onMcpControlRequest?.(async (request) => {
if (request.action === "list_sources") {
const sources = await getMcpSources();
return {
success: true,
data: sources.map(toMcpSourceSummary),
};
}
if (request.action === "status") {
return {
success: true,
recording: recordingRef.current,
message: recordingRef.current ? "Recording is active" : "Recording is idle",
} satisfies McpControlResult;
}
if (request.action === "record_video") {
if (recordingRef.current) {
return { success: true, recording: true, message: "Recording is already active" };
}
const payload = (request.payload ?? {}) as McpRecordVideoPayload;
let sourceForRecording: ProcessedDesktopSource | null = null;
if (!hasSelectedSource || payload.sourceId || payload.sourceName || payload.sourceType) {
const selected = await selectMcpSource(payload);
if (!selected) {
return {
success: false,
error: "No screen or window source is available for recording.",
};
}
sourceForRecording = selected;
} else {
sourceForRecording = await window.electronAPI.getSelectedSource();
}
if (payload.guideMode === true) {
setGuideModeEnabled(true);
}
toggleRecording();
return {
success: true,
recording: true,
message: "Recording start requested",
data: sourceForRecording ? toMcpSourceSummary(sourceForRecording) : undefined,
};
}
if (request.action === "stop_recording") {
if (!recordingRef.current && !countdownActiveRef.current) {
return { success: false, recording: false, error: "No active recording to stop" };
}
const payload = (request.payload ?? {}) as McpStopRecordingPayload;
if (payload.discard === true) {
window.setTimeout(() => cancelRecording(), 0);
return { success: true, recording: false, message: "Recording discard requested" };
}
window.setTimeout(() => toggleRecording(), 0);
return {
success: true,
recording: false,
message: countdownActiveRef.current
? "Recording countdown cancel requested"
: "Recording stop requested",
};
}
return {
success: false,
error: `The HUD cannot handle MCP action: ${request.action}`,
};
});
return () => {
unsubscribe?.();
};
}, [
cancelRecording,
getMcpSources,
hasSelectedSource,
selectMcpSource,
setGuideModeEnabled,
toMcpSourceSummary,
toggleRecording,
]);
const sendHudOverlayHide = () => { const sendHudOverlayHide = () => {
if (window.electronAPI && window.electronAPI.hudOverlayHide) { if (window.electronAPI && window.electronAPI.hudOverlayHide) {
window.electronAPI.hudOverlayHide(); window.electronAPI.hudOverlayHide();
+81 -11
View File
@@ -34,6 +34,7 @@ import {
VideoExporter, VideoExporter,
} from "@/lib/exporter"; } from "@/lib/exporter";
import { computeFrameStepTime } from "@/lib/frameStep"; import { computeFrameStepTime } from "@/lib/frameStep";
import type { McpControlResult, McpExportVideoPayload } from "@/lib/mcpControl";
import type { CursorCaptureMode, ProjectMedia } from "@/lib/recordingSession"; import type { CursorCaptureMode, ProjectMedia } from "@/lib/recordingSession";
import { matchesShortcut } from "@/lib/shortcuts"; import { matchesShortcut } from "@/lib/shortcuts";
import { import {
@@ -1588,16 +1589,19 @@ export default function VideoEditor() {
}, [unsavedExport, handleExportSaved]); }, [unsavedExport, handleExportSaved]);
const handleExport = useCallback( const handleExport = useCallback(
async (settings: ExportSettings) => { async (
settings: ExportSettings,
options?: { targetPath?: string },
): Promise<McpControlResult> => {
if (!videoPath) { if (!videoPath) {
toast.error("No video loaded"); toast.error("No video loaded");
return; return { success: false, error: "No video loaded" };
} }
const video = videoPlaybackRef.current?.video; const video = videoPlaybackRef.current?.video;
if (!video) { if (!video) {
toast.error("Video not ready"); toast.error("Video not ready");
return; return { success: false, error: "Video not ready" };
} }
// Ask the user where to save BEFORE starting the export. This avoids the // Ask the user where to save BEFORE starting the export. This avoids the
@@ -1605,20 +1609,27 @@ export default function VideoEditor() {
// long-running export. // long-running export.
const isGifFormat = settings.format === "gif"; const isGifFormat = settings.format === "gif";
const targetFileName = `export-${Date.now()}.${isGifFormat ? "gif" : "mp4"}`; const targetFileName = `export-${Date.now()}.${isGifFormat ? "gif" : "mp4"}`;
const pickResult = await window.electronAPI.pickExportSavePath( let targetPath = options?.targetPath;
targetFileName, if (!targetPath) {
getExportFolder(), const pickResult = await window.electronAPI.pickExportSavePath(
); targetFileName,
if (pickResult.canceled || !pickResult.success || !pickResult.path) { getExportFolder(),
setShowExportDialog(false); );
return; if (pickResult.canceled || !pickResult.success || !pickResult.path) {
setShowExportDialog(false);
return { success: false, message: "Export canceled" };
}
targetPath = pickResult.path;
} }
const targetPath = pickResult.path;
setIsExporting(true); setIsExporting(true);
setExportProgress(null); setExportProgress(null);
setExportError(null); setExportError(null);
setExportedFilePath(null); setExportedFilePath(null);
let mcpResult: McpControlResult = {
success: false,
error: "Export did not complete",
};
try { try {
const wasPlaying = isPlaying; const wasPlaying = isPlaying;
@@ -1703,6 +1714,11 @@ export default function VideoEditor() {
if (saveResult.success && saveResult.path) { if (saveResult.success && saveResult.path) {
setUnsavedExport(null); setUnsavedExport(null);
handleExportSaved("GIF", saveResult.path); handleExportSaved("GIF", saveResult.path);
mcpResult = {
success: true,
path: saveResult.path,
message: "GIF exported successfully",
};
} else { } else {
setUnsavedExport({ arrayBuffer, fileName: targetFileName, format: "gif" }); setUnsavedExport({ arrayBuffer, fileName: targetFileName, format: "gif" });
const message = buildSaveDiagnosticMessage( const message = buildSaveDiagnosticMessage(
@@ -1711,6 +1727,7 @@ export default function VideoEditor() {
); );
setExportError(message); setExportError(message);
toast.error(message); toast.error(message);
mcpResult = { success: false, error: message };
} }
} else { } else {
const message = buildExportDiagnosticMessage({ const message = buildExportDiagnosticMessage({
@@ -1723,6 +1740,7 @@ export default function VideoEditor() {
}); });
setExportError(message); setExportError(message);
toast.error(message); toast.error(message);
mcpResult = { success: false, error: message };
} }
} else { } else {
// MP4 Export // MP4 Export
@@ -1794,6 +1812,11 @@ export default function VideoEditor() {
if (saveResult.success && saveResult.path) { if (saveResult.success && saveResult.path) {
setUnsavedExport(null); setUnsavedExport(null);
handleExportSaved("Video", saveResult.path); handleExportSaved("Video", saveResult.path);
mcpResult = {
success: true,
path: saveResult.path,
message: "Video exported successfully",
};
} else { } else {
setUnsavedExport({ arrayBuffer, fileName: targetFileName, format: "mp4" }); setUnsavedExport({ arrayBuffer, fileName: targetFileName, format: "mp4" });
const message = buildSaveDiagnosticMessage( const message = buildSaveDiagnosticMessage(
@@ -1802,6 +1825,7 @@ export default function VideoEditor() {
); );
setExportError(message); setExportError(message);
toast.error(message); toast.error(message);
mcpResult = { success: false, error: message };
} }
} else { } else {
const message = buildExportDiagnosticMessage({ const message = buildExportDiagnosticMessage({
@@ -1816,6 +1840,7 @@ export default function VideoEditor() {
}); });
setExportError(message); setExportError(message);
toast.error(message); toast.error(message);
mcpResult = { success: false, error: message };
} }
} }
@@ -1828,6 +1853,7 @@ export default function VideoEditor() {
const message = t("errors.exportBackgroundLoadFailed", { url: error.displayUrl }); const message = t("errors.exportBackgroundLoadFailed", { url: error.displayUrl });
setExportError(message); setExportError(message);
toast.error(message); toast.error(message);
mcpResult = { success: false, error: message };
} else { } else {
const errorMessage = error instanceof Error ? error.message : "Unknown error"; const errorMessage = error instanceof Error ? error.message : "Unknown error";
const message = buildExportDiagnosticMessage({ const message = buildExportDiagnosticMessage({
@@ -1837,6 +1863,7 @@ export default function VideoEditor() {
}); });
setExportError(message); setExportError(message);
toast.error(t("errors.exportFailedWithError", { error: message })); toast.error(t("errors.exportFailedWithError", { error: message }));
mcpResult = { success: false, error: message };
} }
} finally { } finally {
setIsExporting(false); setIsExporting(false);
@@ -1846,6 +1873,7 @@ export default function VideoEditor() {
setShowExportDialog(false); setShowExportDialog(false);
setExportProgress(null); setExportProgress(null);
} }
return mcpResult;
}, },
[ [
videoPath, videoPath,
@@ -1948,6 +1976,48 @@ export default function VideoEditor() {
handleExport, handleExport,
]); ]);
useEffect(() => {
const unsubscribe = window.electronAPI?.onMcpControlRequest?.(async (request) => {
if (request.action === "status") {
return {
success: true,
recording: false,
data: { videoPath, editorReady: Boolean(videoPlaybackRef.current?.video) },
};
}
if (request.action !== "export_video") {
return {
success: false,
error: `The editor cannot handle MCP action: ${request.action}`,
};
}
const payload = (request.payload ?? {}) as McpExportVideoPayload;
const requestedSettings = payload.settings ?? {};
const settings: ExportSettings = {
format: requestedSettings.format ?? "mp4",
quality: requestedSettings.quality ?? exportQuality,
gifConfig:
requestedSettings.format === "gif" && requestedSettings.gifConfig
? requestedSettings.gifConfig
: undefined,
};
if (settings.format === "gif" && !settings.gifConfig) {
return {
success: false,
error: "MCP GIF export requires gifConfig dimensions and frame settings.",
};
}
return handleExport(settings, { targetPath: payload.outputPath });
});
return () => {
unsubscribe?.();
};
}, [exportQuality, handleExport, videoPath]);
const handleCancelExport = useCallback(() => { const handleCancelExport = useCallback(() => {
if (exporterRef.current) { if (exporterRef.current) {
exporterRef.current.cancel(); exporterRef.current.cancel();
+9 -2
View File
@@ -65,7 +65,11 @@ describe("buildGuideVideoAnnotations", () => {
startMs: 1200, startMs: 1200,
content: "1. Click Settings.", content: "1. Click Settings.",
}); });
expect(annotations[0]?.endMs).toBe(3200);
expect(annotations[0]?.position.x).toBeGreaterThan(20); expect(annotations[0]?.position.x).toBeGreaterThan(20);
expect(annotations[1]?.endMs).toBe(3200);
expect(annotations[1]?.position.x).toBeGreaterThan((annotations[0]?.position.x ?? 0) + 34);
expect(annotations[1]?.position.y).toBeCloseTo(30.5);
expect(annotations[1]).toMatchObject({ expect(annotations[1]).toMatchObject({
id: "guide-video-2", id: "guide-video-2",
type: "magnifier", type: "magnifier",
@@ -79,10 +83,13 @@ describe("buildGuideVideoAnnotations", () => {
expect(annotations[2]).toMatchObject({ expect(annotations[2]).toMatchObject({
id: "guide-video-3", id: "guide-video-3",
type: "figure", type: "figure",
endMs: 3200,
figureData: { figureData: {
arrowDirection: "left",
color: "#34B27B", color: "#34B27B",
}, },
}); });
expect(annotations[2]?.position.x).toBeGreaterThan(20);
}); });
it("returns an empty list when no draft exists", () => { it("returns an empty list when no draft exists", () => {
@@ -97,7 +104,7 @@ describe("buildGuideVideoAnnotations", () => {
expect(annotations).toEqual([]); expect(annotations).toEqual([]);
}); });
it("creates 0.3x speed regions for one second at each guide point", () => { it("creates 0.3x speed regions for two seconds at each guide point", () => {
let id = 1; let id = 1;
const speedRegions = buildGuideVideoSpeedRegions(createSession(), { const speedRegions = buildGuideVideoSpeedRegions(createSession(), {
nextId: () => `guide-speed-${id++}`, nextId: () => `guide-speed-${id++}`,
@@ -107,7 +114,7 @@ describe("buildGuideVideoAnnotations", () => {
{ {
id: "guide-speed-1", id: "guide-speed-1",
startMs: 1200, startMs: 1200,
endMs: 2200, endMs: 3200,
speed: 0.3, speed: 0.3,
}, },
]); ]);
+58 -24
View File
@@ -14,13 +14,14 @@ export interface BuildGuideVideoAnnotationsOptions {
defaultDurationMs?: number; defaultDurationMs?: number;
} }
const DEFAULT_STEP_DURATION_MS = 3200; const DEFAULT_STEP_DURATION_MS = 2000;
const DEFAULT_STEP_SLOW_MOTION_DURATION_MS = 1000; const DEFAULT_STEP_SLOW_MOTION_DURATION_MS = 2000;
const DEFAULT_STEP_SLOW_MOTION_SPEED = 0.3; const DEFAULT_STEP_SLOW_MOTION_SPEED = 0.3;
const CAPTION_WIDTH = 34; const CAPTION_WIDTH = 34;
const CAPTION_HEIGHT = 13; const CAPTION_HEIGHT = 13;
const MAGNIFIER_SIZE = 18; const MAGNIFIER_SIZE = 18;
const ARROW_SIZE = 10; const ARROW_SIZE = 10;
const ANNOTATION_GAP = 2;
function clamp(value: number, min: number, max: number) { function clamp(value: number, min: number, max: number) {
return Math.min(max, Math.max(min, value)); return Math.min(max, Math.max(min, value));
@@ -58,15 +59,19 @@ function getCaptionPosition(candidate: GuideStepCandidate | undefined) {
function getArrowDirection( function getArrowDirection(
candidate: GuideStepCandidate | undefined, candidate: GuideStepCandidate | undefined,
captionPosition: { x: number; y: number }, originPosition: { x: number; y: number },
originSize: { width: number; height: number } = {
width: CAPTION_WIDTH,
height: CAPTION_HEIGHT,
},
): ArrowDirection { ): ArrowDirection {
const target = candidate?.position; const target = candidate?.position;
if (!target) return "right"; if (!target) return "right";
const captionCenterX = captionPosition.x + CAPTION_WIDTH / 2; const originCenterX = originPosition.x + originSize.width / 2;
const captionCenterY = captionPosition.y + CAPTION_HEIGHT / 2; const originCenterY = originPosition.y + originSize.height / 2;
const dx = target.normalizedX * 100 - captionCenterX; const dx = target.normalizedX * 100 - originCenterX;
const dy = target.normalizedY * 100 - captionCenterY; const dy = target.normalizedY * 100 - originCenterY;
const horizontal = dx > 8 ? "right" : dx < -8 ? "left" : ""; const horizontal = dx > 8 ? "right" : dx < -8 ? "left" : "";
const vertical = dy > 8 ? "down" : dy < -8 ? "up" : ""; const vertical = dy > 8 ? "down" : dy < -8 ? "up" : "";
@@ -74,6 +79,40 @@ function getArrowDirection(
return (horizontal || vertical || "right") as ArrowDirection; return (horizontal || vertical || "right") as ArrowDirection;
} }
function getMagnifierPosition(captionPosition: { x: number; y: number }) {
const canPlaceRight = captionPosition.x + CAPTION_WIDTH + ANNOTATION_GAP + MAGNIFIER_SIZE <= 98;
const x = canPlaceRight
? captionPosition.x + CAPTION_WIDTH + ANNOTATION_GAP
: captionPosition.x - MAGNIFIER_SIZE - ANNOTATION_GAP;
const y = captionPosition.y + (CAPTION_HEIGHT - MAGNIFIER_SIZE) / 2;
return {
x: clamp(x, 2, 100 - MAGNIFIER_SIZE - 2),
y: clamp(y, 2, 100 - MAGNIFIER_SIZE - 2),
};
}
function getArrowPosition(
position: NonNullable<GuideStepCandidate["position"]>,
originPosition: { x: number; y: number },
originSize: { width: number; height: number },
) {
const targetX = position.normalizedX * 100;
const targetY = position.normalizedY * 100;
const originCenterX = originPosition.x + originSize.width / 2;
const originCenterY = originPosition.y + originSize.height / 2;
const distance = Math.hypot(targetX - originCenterX, targetY - originCenterY);
const targetOffset = Math.min(18, Math.max(10, distance * 0.35));
const ratio = distance > 0 ? Math.max(0, (distance - targetOffset) / distance) : 0;
const arrowCenterX = originCenterX + (targetX - originCenterX) * ratio;
const arrowCenterY = originCenterY + (targetY - originCenterY) * ratio;
return {
x: clamp(arrowCenterX - ARROW_SIZE / 2, 0, 100 - ARROW_SIZE),
y: clamp(arrowCenterY - ARROW_SIZE / 2, 0, 100 - ARROW_SIZE),
};
}
function buildCaption(step: GeneratedGuideStep) { function buildCaption(step: GeneratedGuideStep) {
const instruction = step.instruction.trim(); const instruction = step.instruction.trim();
const title = step.title.trim(); const title = step.title.trim();
@@ -101,7 +140,6 @@ export function buildGuideVideoAnnotations(
const startMs = Math.max(0, Math.round(candidate?.timeMs ?? index * durationMs)); const startMs = Math.max(0, Math.round(candidate?.timeMs ?? index * durationMs));
const endMs = Math.max(startMs + 750, startMs + durationMs); const endMs = Math.max(startMs + 750, startMs + durationMs);
const captionPosition = getCaptionPosition(candidate); const captionPosition = getCaptionPosition(candidate);
const arrowDirection = getArrowDirection(candidate, captionPosition);
annotations.push({ annotations.push({
id: options.nextId(), id: options.nextId(),
@@ -124,24 +162,23 @@ export function buildGuideVideoAnnotations(
}); });
if (candidate?.position) { if (candidate?.position) {
const magnifierPosition = getMagnifierPosition(captionPosition);
const arrowPosition = getArrowPosition(candidate.position, magnifierPosition, {
width: MAGNIFIER_SIZE,
height: MAGNIFIER_SIZE,
});
const arrowDirection = getArrowDirection(candidate, arrowPosition, {
width: ARROW_SIZE,
height: ARROW_SIZE,
});
annotations.push({ annotations.push({
id: options.nextId(), id: options.nextId(),
startMs, startMs,
endMs, endMs,
type: "magnifier", type: "magnifier",
content: buildCaption(step), content: buildCaption(step),
position: { position: magnifierPosition,
x: clamp(
candidate.position.normalizedX * 100 - MAGNIFIER_SIZE / 2,
0,
100 - MAGNIFIER_SIZE,
),
y: clamp(
candidate.position.normalizedY * 100 - MAGNIFIER_SIZE / 2,
0,
100 - MAGNIFIER_SIZE,
),
},
size: { width: MAGNIFIER_SIZE, height: MAGNIFIER_SIZE }, size: { width: MAGNIFIER_SIZE, height: MAGNIFIER_SIZE },
style: { ...DEFAULT_ANNOTATION_STYLE }, style: { ...DEFAULT_ANNOTATION_STYLE },
zIndex: options.nextZIndex(), zIndex: options.nextZIndex(),
@@ -160,10 +197,7 @@ export function buildGuideVideoAnnotations(
endMs, endMs,
type: "figure", type: "figure",
content: "", content: "",
position: { position: arrowPosition,
x: clamp(candidate.position.normalizedX * 100 - ARROW_SIZE / 2, 0, 100 - ARROW_SIZE),
y: clamp(candidate.position.normalizedY * 100 - ARROW_SIZE / 2, 0, 100 - ARROW_SIZE),
},
size: { width: ARROW_SIZE, height: ARROW_SIZE }, size: { width: ARROW_SIZE, height: ARROW_SIZE },
style: { ...DEFAULT_ANNOTATION_STYLE }, style: { ...DEFAULT_ANNOTATION_STYLE },
zIndex: options.nextZIndex(), zIndex: options.nextZIndex(),
+2
View File
@@ -50,6 +50,7 @@ const WEBCAM_TARGET_FRAME_RATE = 30;
type UseScreenRecorderReturn = { type UseScreenRecorderReturn = {
recording: boolean; recording: boolean;
paused: boolean; paused: boolean;
countdownActive: boolean;
elapsedSeconds: number; elapsedSeconds: number;
toggleRecording: () => void; toggleRecording: () => void;
togglePaused: () => void; togglePaused: () => void;
@@ -1783,6 +1784,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
return { return {
recording, recording,
paused, paused,
countdownActive,
elapsedSeconds, elapsedSeconds,
toggleRecording, toggleRecording,
togglePaused, togglePaused,
+51
View File
@@ -0,0 +1,51 @@
import type { ExportSettings } from "@/lib/exporter";
export type McpControlAction =
| "list_sources"
| "record_video"
| "stop_recording"
| "export_video"
| "status";
export interface McpControlRequest {
id: string;
action: McpControlAction;
payload?: unknown;
}
export interface McpRecordVideoPayload {
guideMode?: boolean;
sourceType?: "screen" | "window";
sourceId?: string;
sourceName?: string;
displayIndex?: number;
}
export interface McpStopRecordingPayload {
discard?: boolean;
}
export interface McpExportVideoPayload {
outputPath?: string;
settings?: Partial<ExportSettings>;
}
export interface McpControlResult {
success: boolean;
message?: string;
recording?: boolean;
path?: string;
url?: string;
data?: unknown;
error?: string;
}
export function isMcpControlAction(value: unknown): value is McpControlAction {
return (
value === "list_sources" ||
value === "record_video" ||
value === "stop_recording" ||
value === "export_video" ||
value === "status"
);
}