diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts index e192819..bed037a 100644 --- a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts @@ -1,4 +1,4 @@ -export function buildPowerShellCommand(sampleIntervalMs: number, windowHandle?: string | null) { +export function buildPowerShellScript(sampleIntervalMs: number, windowHandle?: string | null) { const targetWindowHandle = typeof windowHandle === "string" && /^(?:0x[0-9a-fA-F]+|\d+)$/.test(windowHandle) ? `'${windowHandle}'` @@ -6,14 +6,32 @@ export function buildPowerShellCommand(sampleIntervalMs: number, windowHandle?: const script = String.raw` $ErrorActionPreference = 'Stop' Add-Type -AssemblyName System.Drawing +Add-Type -AssemblyName System.Windows.Forms $targetWindowHandle = ${targetWindowHandle} $source = @" using System; +using System.Diagnostics; using System.Runtime.InteropServices; public static class OpenScreenCursorInterop { + private const int WH_MOUSE_LL = 14; + private const int WM_LBUTTONDOWN = 0x0201; + private const int WM_LBUTTONUP = 0x0202; + private static readonly object MouseSync = new object(); + private static int LeftDownCount = 0; + private static int LeftUpCount = 0; + private static IntPtr MouseHook = IntPtr.Zero; + private static LowLevelMouseProc MouseProcDelegate = MouseHookCallback; + + public delegate IntPtr LowLevelMouseProc(int nCode, IntPtr wParam, IntPtr lParam); + + public struct MouseButtonEvents { + public int LeftDownCount; + public int LeftUpCount; + } + [StructLayout(LayoutKind.Sequential)] public struct POINT { public int X; @@ -46,6 +64,48 @@ public static class OpenScreenCursorInterop { public int Bottom; } + public static bool InstallMouseHook() { + if (MouseHook != IntPtr.Zero) { + return true; + } + + using (Process process = Process.GetCurrentProcess()) + using (ProcessModule module = process.MainModule) { + MouseHook = SetWindowsHookEx(WH_MOUSE_LL, MouseProcDelegate, GetModuleHandle(module.ModuleName), 0); + } + + return MouseHook != IntPtr.Zero; + } + + public static MouseButtonEvents ConsumeMouseButtonEvents() { + lock (MouseSync) { + MouseButtonEvents events = new MouseButtonEvents { + LeftDownCount = LeftDownCount, + LeftUpCount = LeftUpCount + }; + LeftDownCount = 0; + LeftUpCount = 0; + return events; + } + } + + private static IntPtr MouseHookCallback(int nCode, IntPtr wParam, IntPtr lParam) { + if (nCode >= 0) { + int message = wParam.ToInt32(); + if (message == WM_LBUTTONDOWN || message == WM_LBUTTONUP) { + lock (MouseSync) { + if (message == WM_LBUTTONDOWN) { + LeftDownCount += 1; + } else { + LeftUpCount += 1; + } + } + } + } + + return CallNextHookEx(MouseHook, nCode, wParam, lParam); + } + [DllImport("user32.dll", SetLastError = true)] [return: MarshalAs(UnmanagedType.Bool)] public static extern bool GetCursorInfo(ref CURSORINFO pci); @@ -78,6 +138,15 @@ public static class OpenScreenCursorInterop { [DllImport("gdi32.dll", SetLastError = true)] [return: MarshalAs(UnmanagedType.Bool)] public static extern bool DeleteObject(IntPtr hObject); + + [DllImport("user32.dll", SetLastError = true)] + private static extern IntPtr SetWindowsHookEx(int idHook, LowLevelMouseProc lpfn, IntPtr hMod, uint dwThreadId); + + [DllImport("user32.dll", SetLastError = true)] + private static extern IntPtr CallNextHookEx(IntPtr hhk, int nCode, IntPtr wParam, IntPtr lParam); + + [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)] + private static extern IntPtr GetModuleHandle(string lpModuleName); } "@ @@ -263,11 +332,14 @@ function Get-CursorAsset($cursorHandle, $cursorId) { } } +[OpenScreenCursorInterop]::InstallMouseHook() | Out-Null [OpenScreenCursorInterop]::GetAsyncKeyState(0x01) | Out-Null Write-JsonLine @{ type = 'ready'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() } $lastCursorId = $null while ($true) { + [System.Windows.Forms.Application]::DoEvents() + $mouseEvents = [OpenScreenCursorInterop]::ConsumeMouseButtonEvents() $cursorInfo = New-Object OpenScreenCursorInterop+CURSORINFO $cursorInfo.cbSize = [Runtime.InteropServices.Marshal]::SizeOf([type][OpenScreenCursorInterop+CURSORINFO]) @@ -282,7 +354,8 @@ while ($true) { $cursorType = Get-StandardCursorType $cursorInfo.hCursor $leftButtonState = [OpenScreenCursorInterop]::GetAsyncKeyState(0x01) $leftButtonDown = ($leftButtonState -band 0x8000) -ne 0 - $leftButtonPressed = ($leftButtonState -band 0x0001) -ne 0 + $leftButtonPressed = ($mouseEvents.LeftDownCount -gt 0) -or (($leftButtonState -band 0x0001) -ne 0) + $leftButtonReleased = $mouseEvents.LeftUpCount -gt 0 $asset = $null if ($visible -and $cursorId -and $cursorId -ne $lastCursorId) { @@ -305,6 +378,7 @@ while ($true) { cursorType = $cursorType leftButtonDown = $leftButtonDown leftButtonPressed = $leftButtonPressed + leftButtonReleased = $leftButtonReleased bounds = Get-TargetBounds asset = $asset } @@ -313,5 +387,5 @@ while ($true) { } `; - return Buffer.from(script, "utf16le").toString("base64"); + return script; } diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts index a037cc4..59acf80 100644 --- a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts @@ -1,4 +1,8 @@ import { type ChildProcessByStdio, spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import type { Readable } from "node:stream"; import { screen } from "electron"; import { parseWindowHandleFromSourceId } from "../../../../src/lib/nativeWindowsRecording"; @@ -8,7 +12,7 @@ import type { NativeCursorAsset, } from "../../../../src/native/contracts"; import type { CursorRecordingSession } from "./session"; -import { buildPowerShellCommand } from "./windowsNativeRecordingSession.script"; +import { buildPowerShellScript } from "./windowsNativeRecordingSession.script"; import type { WindowsCursorEvent, WindowsNativeRecordingSessionOptions, @@ -25,6 +29,7 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession { private assets = new Map(); private samples: CursorRecordingSample[] = []; private process: ChildProcessByStdio | null = null; + private helperScriptPath: string | null = null; private lineBuffer = ""; private startTimeMs = 0; private readyResolve: (() => void) | null = null; @@ -45,10 +50,18 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession { this.outOfBoundsSampleCount = 0; this.previousLeftButtonDown = false; - const encodedCommand = buildPowerShellCommand( + const script = buildPowerShellScript( this.options.sampleIntervalMs, parseWindowHandleFromSourceId(this.options.sourceId), ); + const helperScriptDir = join(tmpdir(), "openscreen-cursor-native"); + mkdirSync(helperScriptDir, { recursive: true }); + const helperScriptPath = join( + helperScriptDir, + `cursor-sampler-${process.pid}-${Date.now()}-${randomUUID()}.ps1`, + ); + writeFileSync(helperScriptPath, script, "utf8"); + this.helperScriptPath = helperScriptPath; const child = spawn( "powershell.exe", [ @@ -57,8 +70,8 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession { "-NonInteractive", "-ExecutionPolicy", "Bypass", - "-EncodedCommand", - encodedCommand, + "-File", + helperScriptPath, ], { stdio: ["ignore", "pipe", "pipe"], @@ -87,6 +100,7 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession { console.error("[cursor-native]", message); }); child.once("exit", (code, signal) => { + this.cleanupHelperScript(helperScriptPath); this.logDiagnostic("exit", { code, signal, @@ -99,6 +113,7 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession { ); }); child.once("error", (error) => { + this.cleanupHelperScript(helperScriptPath); this.logDiagnostic("process-error", { message: error.message }); this.rejectReady(error); }); @@ -212,10 +227,11 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession { normalizedX >= 0 && normalizedX <= 1 && normalizedY >= 0 && normalizedY <= 1; const leftButtonDown = payload.leftButtonDown === true; const leftButtonPressed = payload.leftButtonPressed === true; + const leftButtonReleased = payload.leftButtonReleased === true; const interactionType = leftButtonPressed || (leftButtonDown && !this.previousLeftButtonDown) ? "click" - : !leftButtonDown && this.previousLeftButtonDown + : leftButtonReleased || (!leftButtonDown && this.previousLeftButtonDown) ? "mouseup" : "move"; this.previousLeftButtonDown = leftButtonDown; @@ -287,6 +303,25 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession { this.readyReject = null; } + private cleanupHelperScript(scriptPath = this.helperScriptPath) { + if (!scriptPath) { + return; + } + + try { + rmSync(scriptPath, { force: true }); + } catch (error) { + this.logDiagnostic("script-cleanup-error", { + path: scriptPath, + message: error instanceof Error ? error.message : String(error), + }); + } finally { + if (this.helperScriptPath === scriptPath) { + this.helperScriptPath = null; + } + } + } + private logDiagnostic(event: string, data: Record) { console.info( "[cursor-native][win32]", diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts index 2ee43c4..35a8594 100644 --- a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts @@ -11,6 +11,7 @@ export interface WindowsCursorSampleEvent { cursorType?: NativeCursorType | null; leftButtonDown?: boolean; leftButtonPressed?: boolean; + leftButtonReleased?: boolean; bounds?: { x: number; y: number; diff --git a/scripts/test-windows-native-cursor.mjs b/scripts/test-windows-native-cursor.mjs index ba99162..24f568e 100644 --- a/scripts/test-windows-native-cursor.mjs +++ b/scripts/test-windows-native-cursor.mjs @@ -65,17 +65,14 @@ function runPowerShell(script) { } function spawnPowerShell(script, { onStdout, onStderr } = {}) { + const scriptPath = path.join( + os.tmpdir(), + `openscreen-powershell-${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2)}.ps1`, + ); + fs.writeFileSync(scriptPath, script, "utf8"); const child = spawn( "powershell.exe", - [ - "-NoLogo", - "-NoProfile", - "-NonInteractive", - "-ExecutionPolicy", - "Bypass", - "-EncodedCommand", - encodePowerShell(script), - ], + ["-NoLogo", "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-File", scriptPath], { stdio: ["ignore", "pipe", "pipe"], windowsHide: true }, ); @@ -85,8 +82,15 @@ function spawnPowerShell(script, { onStdout, onStderr } = {}) { child.stderr.on("data", (chunk) => onStderr?.(chunk)); const done = new Promise((resolve, reject) => { - child.once("error", reject); + const cleanup = () => { + fs.rmSync(scriptPath, { force: true }); + }; + child.once("error", (error) => { + cleanup(); + reject(error); + }); child.once("exit", (code, signal) => { + cleanup(); if (code === 0 || child.killed) { resolve({ code, signal }); return; @@ -107,9 +111,26 @@ Add-Type -AssemblyName System.Windows.Forms $source = @" using System; +using System.Diagnostics; using System.Runtime.InteropServices; public static class OpenScreenCursorDiagnosticInterop { + private const int WH_MOUSE_LL = 14; + private const int WM_LBUTTONDOWN = 0x0201; + private const int WM_LBUTTONUP = 0x0202; + private static readonly object MouseSync = new object(); + private static int LeftDownCount = 0; + private static int LeftUpCount = 0; + private static IntPtr MouseHook = IntPtr.Zero; + private static LowLevelMouseProc MouseProcDelegate = MouseHookCallback; + + public delegate IntPtr LowLevelMouseProc(int nCode, IntPtr wParam, IntPtr lParam); + + public struct MouseButtonEvents { + public int LeftDownCount; + public int LeftUpCount; + } + [StructLayout(LayoutKind.Sequential)] public struct POINT { public int X; @@ -134,6 +155,48 @@ public static class OpenScreenCursorDiagnosticInterop { public IntPtr hbmColor; } + public static bool InstallMouseHook() { + if (MouseHook != IntPtr.Zero) { + return true; + } + + using (Process process = Process.GetCurrentProcess()) + using (ProcessModule module = process.MainModule) { + MouseHook = SetWindowsHookEx(WH_MOUSE_LL, MouseProcDelegate, GetModuleHandle(module.ModuleName), 0); + } + + return MouseHook != IntPtr.Zero; + } + + public static MouseButtonEvents ConsumeMouseButtonEvents() { + lock (MouseSync) { + MouseButtonEvents events = new MouseButtonEvents { + LeftDownCount = LeftDownCount, + LeftUpCount = LeftUpCount + }; + LeftDownCount = 0; + LeftUpCount = 0; + return events; + } + } + + private static IntPtr MouseHookCallback(int nCode, IntPtr wParam, IntPtr lParam) { + if (nCode >= 0) { + int message = wParam.ToInt32(); + if (message == WM_LBUTTONDOWN || message == WM_LBUTTONUP) { + lock (MouseSync) { + if (message == WM_LBUTTONDOWN) { + LeftDownCount += 1; + } else { + LeftUpCount += 1; + } + } + } + } + + return CallNextHookEx(MouseHook, nCode, wParam, lParam); + } + [DllImport("user32.dll", SetLastError = true)] [return: MarshalAs(UnmanagedType.Bool)] public static extern bool GetCursorInfo(ref CURSORINFO pci); @@ -158,6 +221,15 @@ public static class OpenScreenCursorDiagnosticInterop { [DllImport("gdi32.dll", SetLastError = true)] [return: MarshalAs(UnmanagedType.Bool)] public static extern bool DeleteObject(IntPtr hObject); + + [DllImport("user32.dll", SetLastError = true)] + private static extern IntPtr SetWindowsHookEx(int idHook, LowLevelMouseProc lpfn, IntPtr hMod, uint dwThreadId); + + [DllImport("user32.dll", SetLastError = true)] + private static extern IntPtr CallNextHookEx(IntPtr hhk, int nCode, IntPtr wParam, IntPtr lParam); + + [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)] + private static extern IntPtr GetModuleHandle(string lpModuleName); } "@ @@ -308,12 +380,15 @@ function Get-CursorAsset($cursorHandle, $cursorId) { } } +[OpenScreenCursorDiagnosticInterop]::InstallMouseHook() | Out-Null [OpenScreenCursorDiagnosticInterop]::GetAsyncKeyState(0x01) | Out-Null Write-JsonLine @{ type = 'ready'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() } $lastCursorId = $null $screenBounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds while ($true) { + [System.Windows.Forms.Application]::DoEvents() + $mouseEvents = [OpenScreenCursorDiagnosticInterop]::ConsumeMouseButtonEvents() $cursorInfo = New-Object OpenScreenCursorDiagnosticInterop+CURSORINFO $cursorInfo.cbSize = [Runtime.InteropServices.Marshal]::SizeOf([type][OpenScreenCursorDiagnosticInterop+CURSORINFO]) @@ -328,7 +403,8 @@ while ($true) { $cursorType = Get-StandardCursorType $cursorInfo.hCursor $leftButtonState = [OpenScreenCursorDiagnosticInterop]::GetAsyncKeyState(0x01) $leftButtonDown = ($leftButtonState -band 0x8000) -ne 0 - $leftButtonPressed = ($leftButtonState -band 0x0001) -ne 0 + $leftButtonPressed = ($mouseEvents.LeftDownCount -gt 0) -or (($leftButtonState -band 0x0001) -ne 0) + $leftButtonReleased = $mouseEvents.LeftUpCount -gt 0 $asset = $null if ($visible -and $cursorId -and $cursorId -ne $lastCursorId) { @@ -351,6 +427,7 @@ while ($true) { cursorType = $cursorType leftButtonDown = $leftButtonDown leftButtonPressed = $leftButtonPressed + leftButtonReleased = $leftButtonReleased bounds = @{ x = $screenBounds.Left y = $screenBounds.Top @@ -517,10 +594,11 @@ function toRecordingData(samples, assets) { const leftButtonDown = sample.leftButtonDown === true; const leftButtonPressed = sample.leftButtonPressed === true; + const leftButtonReleased = sample.leftButtonReleased === true; const interactionType = leftButtonPressed || (leftButtonDown && !previousLeftButtonDown) ? "click" - : !leftButtonDown && previousLeftButtonDown + : leftButtonReleased || (!leftButtonDown && previousLeftButtonDown) ? "mouseup" : "move"; previousLeftButtonDown = leftButtonDown;