huanld/openscreen
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

fix: record native cursor click events

Browse Source
This commit is contained in:
EtienneLescot
2026-05-05 21:49:40 +02:00
parent 3a32a140cc
commit e33d2205e6
4 changed files with 208 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts
+77 -3
View File
@@ -1,4 +1,4 @@
export function buildPowerShellCommand(sampleIntervalMs: number, windowHandle?: string | null) {
export function buildPowerShellScript(sampleIntervalMs: number, windowHandle?: string | null) {
const targetWindowHandle =
typeof windowHandle === "string" && /^(?:0x[0-9a-fA-F]+|\d+)$/.test(windowHandle)
? `'${windowHandle}'`
@@ -6,14 +6,32 @@ export function buildPowerShellCommand(sampleIntervalMs: number, windowHandle?:
const script = String.raw`
$ErrorActionPreference = 'Stop'
Add-Type -AssemblyName System.Drawing
Add-Type -AssemblyName System.Windows.Forms
$targetWindowHandle = ${targetWindowHandle}
$source = @"
using System;
using System.Diagnostics;
using System.Runtime.InteropServices;
public static class OpenScreenCursorInterop {
private const int WH_MOUSE_LL = 14;
private const int WM_LBUTTONDOWN = 0x0201;
private const int WM_LBUTTONUP = 0x0202;
private static readonly object MouseSync = new object();
private static int LeftDownCount = 0;
private static int LeftUpCount = 0;
private static IntPtr MouseHook = IntPtr.Zero;
private static LowLevelMouseProc MouseProcDelegate = MouseHookCallback;
public delegate IntPtr LowLevelMouseProc(int nCode, IntPtr wParam, IntPtr lParam);
public struct MouseButtonEvents {
public int LeftDownCount;
public int LeftUpCount;
}
[StructLayout(LayoutKind.Sequential)]
public struct POINT {
public int X;
@@ -46,6 +64,48 @@ public static class OpenScreenCursorInterop {
public int Bottom;
}
public static bool InstallMouseHook() {
if (MouseHook != IntPtr.Zero) {
return true;
}
using (Process process = Process.GetCurrentProcess())
using (ProcessModule module = process.MainModule) {
MouseHook = SetWindowsHookEx(WH_MOUSE_LL, MouseProcDelegate, GetModuleHandle(module.ModuleName), 0);
}
return MouseHook != IntPtr.Zero;
}
public static MouseButtonEvents ConsumeMouseButtonEvents() {
lock (MouseSync) {
MouseButtonEvents events = new MouseButtonEvents {
LeftDownCount = LeftDownCount,
LeftUpCount = LeftUpCount
};
LeftDownCount = 0;
LeftUpCount = 0;
return events;
}
}
private static IntPtr MouseHookCallback(int nCode, IntPtr wParam, IntPtr lParam) {
if (nCode >= 0) {
int message = wParam.ToInt32();
if (message == WM_LBUTTONDOWN || message == WM_LBUTTONUP) {
lock (MouseSync) {
if (message == WM_LBUTTONDOWN) {
LeftDownCount += 1;
} else {
LeftUpCount += 1;
}
}
}
}
return CallNextHookEx(MouseHook, nCode, wParam, lParam);
}
[DllImport("user32.dll", SetLastError = true)]
[return: MarshalAs(UnmanagedType.Bool)]
public static extern bool GetCursorInfo(ref CURSORINFO pci);
@@ -78,6 +138,15 @@ public static class OpenScreenCursorInterop {
[DllImport("gdi32.dll", SetLastError = true)]
[return: MarshalAs(UnmanagedType.Bool)]
public static extern bool DeleteObject(IntPtr hObject);
[DllImport("user32.dll", SetLastError = true)]
private static extern IntPtr SetWindowsHookEx(int idHook, LowLevelMouseProc lpfn, IntPtr hMod, uint dwThreadId);
[DllImport("user32.dll", SetLastError = true)]
private static extern IntPtr CallNextHookEx(IntPtr hhk, int nCode, IntPtr wParam, IntPtr lParam);
[DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)]
private static extern IntPtr GetModuleHandle(string lpModuleName);
}
"@
@@ -263,11 +332,14 @@ function Get-CursorAsset($cursorHandle, $cursorId) {
}
}
[OpenScreenCursorInterop]::InstallMouseHook() | Out-Null
[OpenScreenCursorInterop]::GetAsyncKeyState(0x01) | Out-Null
Write-JsonLine @{ type = 'ready'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() }
$lastCursorId = $null
while ($true) {
[System.Windows.Forms.Application]::DoEvents()
$mouseEvents = [OpenScreenCursorInterop]::ConsumeMouseButtonEvents()
$cursorInfo = New-Object OpenScreenCursorInterop+CURSORINFO
$cursorInfo.cbSize = [Runtime.InteropServices.Marshal]::SizeOf([type][OpenScreenCursorInterop+CURSORINFO])
@@ -282,7 +354,8 @@ while ($true) {
$cursorType = Get-StandardCursorType $cursorInfo.hCursor
$leftButtonState = [OpenScreenCursorInterop]::GetAsyncKeyState(0x01)
$leftButtonDown = ($leftButtonState -band 0x8000) -ne 0
$leftButtonPressed = ($leftButtonState -band 0x0001) -ne 0
$leftButtonPressed = ($mouseEvents.LeftDownCount -gt 0) -or (($leftButtonState -band 0x0001) -ne 0)
$leftButtonReleased = $mouseEvents.LeftUpCount -gt 0
$asset = $null
if ($visible -and $cursorId -and $cursorId -ne $lastCursorId) {
@@ -305,6 +378,7 @@ while ($true) {
cursorType = $cursorType
leftButtonDown = $leftButtonDown
leftButtonPressed = $leftButtonPressed
leftButtonReleased = $leftButtonReleased
bounds = Get-TargetBounds
asset = $asset
}
@@ -313,5 +387,5 @@ while ($true) {
}
`;
return Buffer.from(script, "utf16le").toString("base64");
return script;
}
electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts
+40 -5
View File
@@ -1,4 +1,8 @@
import { type ChildProcessByStdio, spawn } from "node:child_process";
import { randomUUID } from "node:crypto";
import { mkdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { Readable } from "node:stream";
import { screen } from "electron";
import { parseWindowHandleFromSourceId } from "../../../../src/lib/nativeWindowsRecording";
@@ -8,7 +12,7 @@ import type {
NativeCursorAsset,
} from "../../../../src/native/contracts";
import type { CursorRecordingSession } from "./session";
import { buildPowerShellCommand } from "./windowsNativeRecordingSession.script";
import { buildPowerShellScript } from "./windowsNativeRecordingSession.script";
import type {
WindowsCursorEvent,
WindowsNativeRecordingSessionOptions,
@@ -25,6 +29,7 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession {
private assets = new Map<string, NativeCursorAsset>();
private samples: CursorRecordingSample[] = [];
private process: ChildProcessByStdio<null, Readable, Readable> | null = null;
private helperScriptPath: string | null = null;
private lineBuffer = "";
private startTimeMs = 0;
private readyResolve: (() => void) | null = null;
@@ -45,10 +50,18 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession {
this.outOfBoundsSampleCount = 0;
this.previousLeftButtonDown = false;
const encodedCommand = buildPowerShellCommand(
const script = buildPowerShellScript(
this.options.sampleIntervalMs,
parseWindowHandleFromSourceId(this.options.sourceId),
);
const helperScriptDir = join(tmpdir(), "openscreen-cursor-native");
mkdirSync(helperScriptDir, { recursive: true });
const helperScriptPath = join(
helperScriptDir,
`cursor-sampler-${process.pid}-${Date.now()}-${randomUUID()}.ps1`,
);
writeFileSync(helperScriptPath, script, "utf8");
this.helperScriptPath = helperScriptPath;
const child = spawn(
"powershell.exe",
[
@@ -57,8 +70,8 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession {
"-NonInteractive",
"-ExecutionPolicy",
"Bypass",
"-EncodedCommand",
encodedCommand,
"-File",
helperScriptPath,
],
{
stdio: ["ignore", "pipe", "pipe"],
@@ -87,6 +100,7 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession {
console.error("[cursor-native]", message);
});
child.once("exit", (code, signal) => {
this.cleanupHelperScript(helperScriptPath);
this.logDiagnostic("exit", {
code,
signal,
@@ -99,6 +113,7 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession {
);
});
child.once("error", (error) => {
this.cleanupHelperScript(helperScriptPath);
this.logDiagnostic("process-error", { message: error.message });
this.rejectReady(error);
});
@@ -212,10 +227,11 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession {
normalizedX >= 0 && normalizedX <= 1 && normalizedY >= 0 && normalizedY <= 1;
const leftButtonDown = payload.leftButtonDown === true;
const leftButtonPressed = payload.leftButtonPressed === true;
const leftButtonReleased = payload.leftButtonReleased === true;
const interactionType =
leftButtonPressed || (leftButtonDown && !this.previousLeftButtonDown)
? "click"
: !leftButtonDown && this.previousLeftButtonDown
: leftButtonReleased || (!leftButtonDown && this.previousLeftButtonDown)
? "mouseup"
: "move";
this.previousLeftButtonDown = leftButtonDown;
@@ -287,6 +303,25 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession {
this.readyReject = null;
}
private cleanupHelperScript(scriptPath = this.helperScriptPath) {
if (!scriptPath) {
return;
}
try {
rmSync(scriptPath, { force: true });
} catch (error) {
this.logDiagnostic("script-cleanup-error", {
path: scriptPath,
message: error instanceof Error ? error.message : String(error),
});
} finally {
if (this.helperScriptPath === scriptPath) {
this.helperScriptPath = null;
}
}
}
private logDiagnostic(event: string, data: Record<string, unknown>) {
console.info(
"[cursor-native][win32]",
electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts
+1
View File
@@ -11,6 +11,7 @@ export interface WindowsCursorSampleEvent {
cursorType?: NativeCursorType | null;
leftButtonDown?: boolean;
leftButtonPressed?: boolean;
leftButtonReleased?: boolean;
bounds?: {
x: number;
y: number;
scripts/test-windows-native-cursor.mjs
+90 -12
View File
@@ -65,17 +65,14 @@ function runPowerShell(script) {
}
function spawnPowerShell(script, { onStdout, onStderr } = {}) {
const scriptPath = path.join(
os.tmpdir(),
`openscreen-powershell-${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2)}.ps1`,
);
fs.writeFileSync(scriptPath, script, "utf8");
const child = spawn(
"powershell.exe",
[
"-NoLogo",
"-NoProfile",
"-NonInteractive",
"-ExecutionPolicy",
"Bypass",
"-EncodedCommand",
encodePowerShell(script),
],
["-NoLogo", "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-File", scriptPath],
{ stdio: ["ignore", "pipe", "pipe"], windowsHide: true },
);
@@ -85,8 +82,15 @@ function spawnPowerShell(script, { onStdout, onStderr } = {}) {
child.stderr.on("data", (chunk) => onStderr?.(chunk));
const done = new Promise((resolve, reject) => {
child.once("error", reject);
const cleanup = () => {
fs.rmSync(scriptPath, { force: true });
};
child.once("error", (error) => {
cleanup();
reject(error);
});
child.once("exit", (code, signal) => {
cleanup();
if (code === 0 || child.killed) {
resolve({ code, signal });
return;
@@ -107,9 +111,26 @@ Add-Type -AssemblyName System.Windows.Forms
$source = @"
using System;
using System.Diagnostics;
using System.Runtime.InteropServices;
public static class OpenScreenCursorDiagnosticInterop {
private const int WH_MOUSE_LL = 14;
private const int WM_LBUTTONDOWN = 0x0201;
private const int WM_LBUTTONUP = 0x0202;
private static readonly object MouseSync = new object();
private static int LeftDownCount = 0;
private static int LeftUpCount = 0;
private static IntPtr MouseHook = IntPtr.Zero;
private static LowLevelMouseProc MouseProcDelegate = MouseHookCallback;
public delegate IntPtr LowLevelMouseProc(int nCode, IntPtr wParam, IntPtr lParam);
public struct MouseButtonEvents {
public int LeftDownCount;
public int LeftUpCount;
}
[StructLayout(LayoutKind.Sequential)]
public struct POINT {
public int X;
@@ -134,6 +155,48 @@ public static class OpenScreenCursorDiagnosticInterop {
public IntPtr hbmColor;
}
public static bool InstallMouseHook() {
if (MouseHook != IntPtr.Zero) {
return true;
}
using (Process process = Process.GetCurrentProcess())
using (ProcessModule module = process.MainModule) {
MouseHook = SetWindowsHookEx(WH_MOUSE_LL, MouseProcDelegate, GetModuleHandle(module.ModuleName), 0);
}
return MouseHook != IntPtr.Zero;
}
public static MouseButtonEvents ConsumeMouseButtonEvents() {
lock (MouseSync) {
MouseButtonEvents events = new MouseButtonEvents {
LeftDownCount = LeftDownCount,
LeftUpCount = LeftUpCount
};
LeftDownCount = 0;
LeftUpCount = 0;
return events;
}
}
private static IntPtr MouseHookCallback(int nCode, IntPtr wParam, IntPtr lParam) {
if (nCode >= 0) {
int message = wParam.ToInt32();
if (message == WM_LBUTTONDOWN || message == WM_LBUTTONUP) {
lock (MouseSync) {
if (message == WM_LBUTTONDOWN) {
LeftDownCount += 1;
} else {
LeftUpCount += 1;
}
}
}
}
return CallNextHookEx(MouseHook, nCode, wParam, lParam);
}
[DllImport("user32.dll", SetLastError = true)]
[return: MarshalAs(UnmanagedType.Bool)]
public static extern bool GetCursorInfo(ref CURSORINFO pci);
@@ -158,6 +221,15 @@ public static class OpenScreenCursorDiagnosticInterop {
[DllImport("gdi32.dll", SetLastError = true)]
[return: MarshalAs(UnmanagedType.Bool)]
public static extern bool DeleteObject(IntPtr hObject);
[DllImport("user32.dll", SetLastError = true)]
private static extern IntPtr SetWindowsHookEx(int idHook, LowLevelMouseProc lpfn, IntPtr hMod, uint dwThreadId);
[DllImport("user32.dll", SetLastError = true)]
private static extern IntPtr CallNextHookEx(IntPtr hhk, int nCode, IntPtr wParam, IntPtr lParam);
[DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)]
private static extern IntPtr GetModuleHandle(string lpModuleName);
}
"@
@@ -308,12 +380,15 @@ function Get-CursorAsset($cursorHandle, $cursorId) {
}
}
[OpenScreenCursorDiagnosticInterop]::InstallMouseHook() | Out-Null
[OpenScreenCursorDiagnosticInterop]::GetAsyncKeyState(0x01) | Out-Null
Write-JsonLine @{ type = 'ready'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() }
$lastCursorId = $null
$screenBounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds
while ($true) {
[System.Windows.Forms.Application]::DoEvents()
$mouseEvents = [OpenScreenCursorDiagnosticInterop]::ConsumeMouseButtonEvents()
$cursorInfo = New-Object OpenScreenCursorDiagnosticInterop+CURSORINFO
$cursorInfo.cbSize = [Runtime.InteropServices.Marshal]::SizeOf([type][OpenScreenCursorDiagnosticInterop+CURSORINFO])
@@ -328,7 +403,8 @@ while ($true) {
$cursorType = Get-StandardCursorType $cursorInfo.hCursor
$leftButtonState = [OpenScreenCursorDiagnosticInterop]::GetAsyncKeyState(0x01)
$leftButtonDown = ($leftButtonState -band 0x8000) -ne 0
$leftButtonPressed = ($leftButtonState -band 0x0001) -ne 0
$leftButtonPressed = ($mouseEvents.LeftDownCount -gt 0) -or (($leftButtonState -band 0x0001) -ne 0)
$leftButtonReleased = $mouseEvents.LeftUpCount -gt 0
$asset = $null
if ($visible -and $cursorId -and $cursorId -ne $lastCursorId) {
@@ -351,6 +427,7 @@ while ($true) {
cursorType = $cursorType
leftButtonDown = $leftButtonDown
leftButtonPressed = $leftButtonPressed
leftButtonReleased = $leftButtonReleased
bounds = @{
x = $screenBounds.Left
y = $screenBounds.Top
@@ -517,10 +594,11 @@ function toRecordingData(samples, assets) {
const leftButtonDown = sample.leftButtonDown === true;
const leftButtonPressed = sample.leftButtonPressed === true;
const leftButtonReleased = sample.leftButtonReleased === true;
const interactionType =
leftButtonPressed || (leftButtonDown && !previousLeftButtonDown)
? "click"
: !leftButtonDown && previousLeftButtonDown
: leftButtonReleased || (!leftButtonDown && previousLeftButtonDown)
? "mouseup"
: "move";
previousLeftButtonDown = leftButtonDown;