From 92d2a41296d2b86198871e4bff2e68ccfed6bde0 Mon Sep 17 00:00:00 2001 From: FabLrc Date: Fri, 27 Feb 2026 00:24:27 +0100 Subject: [PATCH 1/7] fix: improve encoder queue management and adjust latency mode for better throughput --- src/lib/exporter/videoExporter.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index 937c4a3..c2f9417 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -125,8 +125,8 @@ export class VideoExporter { }); // Check encoder queue before encoding to keep it full - while (this.encodeQueue >= this.MAX_ENCODE_QUEUE && !this.cancelled) { - await new Promise(resolve => setTimeout(resolve, 0)); + while (this.encoder && this.encoder.encodeQueueSize >= this.MAX_ENCODE_QUEUE && !this.cancelled) { + await new Promise(resolve => setTimeout(resolve, 5)); } if (this.encoder && this.encoder.state === 'configured') { @@ -250,7 +250,7 @@ export class VideoExporter { height: this.config.height, bitrate: this.config.bitrate, framerate: this.config.frameRate, - latencyMode: 'realtime', + latencyMode: 'quality', // Changed from 'realtime' to 'quality' for better throughput bitrateMode: 'variable', hardwareAcceleration: 'prefer-hardware', }; From 5573c9f427efdc70e4a871c0cf88e593dc71685a Mon Sep 17 00:00:00 2001 From: Siddharth Date: Fri, 27 Feb 2026 21:04:31 -0800 Subject: [PATCH 2/7] rm testing files --- src/lib/exporter/gifExporter.test.ts | 474 --------------------------- src/lib/exporter/types.test.ts | 63 ---- 2 files changed, 537 deletions(-) delete mode 100644 src/lib/exporter/gifExporter.test.ts delete mode 100644 src/lib/exporter/types.test.ts diff --git a/src/lib/exporter/gifExporter.test.ts b/src/lib/exporter/gifExporter.test.ts deleted file mode 100644 index 0d3cbe7..0000000 --- a/src/lib/exporter/gifExporter.test.ts +++ /dev/null @@ -1,474 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import * as fc from 'fast-check'; -import { calculateOutputDimensions } from './gifExporter'; -import { GIF_SIZE_PRESETS, GifSizePreset } from './types'; - -/** - * Property 2: Loop Encoding Correctness - * - * *For any* GIF export configuration, when loop is enabled the output GIF SHALL - * have a loop count of 0 (infinite), and when loop is disabled the output GIF - * SHALL have a loop count of 1 (play once). - * - * **Validates: Requirements 3.2, 3.3** - * - * Feature: gif-export, Property 2: Loop Encoding Correctness - */ -describe('GIF Exporter', () => { - describe('Property 2: Loop Encoding Correctness', () => { - /** - * Test the loop configuration mapping logic. - * In gif.js: repeat=0 means infinite loop, repeat=1 means play once (no loop) - */ - it('should map loop=true to repeat=0 (infinite) and loop=false to repeat=1 (once)', () => { - fc.assert( - fc.property( - fc.boolean(), - (loopEnabled: boolean) => { - // This is the logic used in GifExporter constructor - const repeat = loopEnabled ? 0 : 1; - - if (loopEnabled) { - // When loop is enabled, repeat should be 0 (infinite loop) - expect(repeat).toBe(0); - } else { - // When loop is disabled, repeat should be 1 (play once) - expect(repeat).toBe(1); - } - } - ), - { numRuns: 100 } - ); - }); - - it('should always produce valid repeat values (0 or 1)', () => { - fc.assert( - fc.property( - fc.boolean(), - (loopEnabled: boolean) => { - const repeat = loopEnabled ? 0 : 1; - expect([0, 1]).toContain(repeat); - } - ), - { numRuns: 100 } - ); - }); - }); - - /** - * Property 4: Aspect Ratio Preservation - * - * *For any* source video with aspect ratio R and any size preset, the exported - * GIF SHALL have an aspect ratio within 0.01 of R. - * - * **Validates: Requirements 4.4** - * - * Feature: gif-export, Property 4: Aspect Ratio Preservation - */ - describe('Property 4: Aspect Ratio Preservation', () => { - const sizePresets: GifSizePreset[] = ['medium', 'large', 'original']; - - it('should preserve aspect ratio within 0.01 tolerance for all size presets', () => { - fc.assert( - fc.property( - fc.integer({ min: 100, max: 4000 }), // sourceWidth - fc.integer({ min: 100, max: 4000 }), // sourceHeight - fc.constantFrom(...sizePresets), - (sourceWidth: number, sourceHeight: number, sizePreset: GifSizePreset) => { - const originalAspectRatio = sourceWidth / sourceHeight; - - const { width, height } = calculateOutputDimensions( - sourceWidth, - sourceHeight, - sizePreset, - GIF_SIZE_PRESETS - ); - - const outputAspectRatio = width / height; - - // Aspect ratio should be preserved within 0.01 tolerance - // (small deviation allowed due to rounding to even numbers) - expect(Math.abs(originalAspectRatio - outputAspectRatio)).toBeLessThan(0.02); - } - ), - { numRuns: 100 } - ); - }); - - it('should return original dimensions when source is smaller than preset max height', () => { - fc.assert( - fc.property( - fc.integer({ min: 100, max: 400 }), // sourceWidth (small) - fc.integer({ min: 100, max: 400 }), // sourceHeight (small, less than 720p) - (sourceWidth: number, sourceHeight: number) => { - // For 'medium' preset with maxHeight 720, if source is smaller, use original - const { width, height } = calculateOutputDimensions( - sourceWidth, - sourceHeight, - 'medium', - GIF_SIZE_PRESETS - ); - - expect(width).toBe(sourceWidth); - expect(height).toBe(sourceHeight); - } - ), - { numRuns: 100 } - ); - }); - - it('should return original dimensions for "original" preset regardless of size', () => { - fc.assert( - fc.property( - fc.integer({ min: 100, max: 4000 }), - fc.integer({ min: 100, max: 4000 }), - (sourceWidth: number, sourceHeight: number) => { - const { width, height } = calculateOutputDimensions( - sourceWidth, - sourceHeight, - 'original', - GIF_SIZE_PRESETS - ); - - expect(width).toBe(sourceWidth); - expect(height).toBe(sourceHeight); - } - ), - { numRuns: 100 } - ); - }); - - it('should scale down to preset max height when source is larger', () => { - fc.assert( - fc.property( - fc.integer({ min: 1000, max: 4000 }), // sourceWidth (large) - fc.integer({ min: 800, max: 2000 }), // sourceHeight (larger than 720p) - (sourceWidth: number, sourceHeight: number) => { - // For 'medium' preset with maxHeight 720 - const { width, height } = calculateOutputDimensions( - sourceWidth, - sourceHeight, - 'medium', - GIF_SIZE_PRESETS - ); - - // Height should be at most 720 (or 722 due to even rounding) - expect(height).toBeLessThanOrEqual(722); - // Width should be scaled proportionally - expect(width).toBeLessThan(sourceWidth); - } - ), - { numRuns: 100 } - ); - }); - }); -}); - - -/** - * Property 3: Size Preset Resolution Mapping - * - * *For any* valid size preset and source video dimensions, the GIF_Exporter SHALL - * produce output with height matching the preset's max height (or source height if smaller), - * with width calculated to maintain aspect ratio. - * - * **Validates: Requirements 4.2** - * - * Feature: gif-export, Property 3: Size Preset Resolution Mapping - */ -describe('Property 3: Size Preset Resolution Mapping', () => { - it('should map size presets to correct max heights', () => { - fc.assert( - fc.property( - fc.integer({ min: 800, max: 4000 }), // sourceWidth (large enough to trigger scaling) - fc.integer({ min: 800, max: 2000 }), // sourceHeight (larger than all presets except original) - fc.constantFrom('medium', 'large') as fc.Arbitrary, - (sourceWidth: number, sourceHeight: number, sizePreset: GifSizePreset) => { - const { height } = calculateOutputDimensions( - sourceWidth, - sourceHeight, - sizePreset, - GIF_SIZE_PRESETS - ); - - const expectedMaxHeight = GIF_SIZE_PRESETS[sizePreset].maxHeight; - - // Height should be at or below the preset's max height - // (allowing +2 for even number rounding) - expect(height).toBeLessThanOrEqual(expectedMaxHeight + 2); - } - ), - { numRuns: 100 } - ); - }); - - it('should use source dimensions when smaller than preset', () => { - fc.assert( - fc.property( - fc.integer({ min: 100, max: 400 }), // sourceWidth - fc.integer({ min: 100, max: 400 }), // sourceHeight (smaller than 720p 'medium' preset) - fc.constantFrom('medium', 'large', 'original') as fc.Arbitrary, - (sourceWidth: number, sourceHeight: number, sizePreset: GifSizePreset) => { - const { width, height } = calculateOutputDimensions( - sourceWidth, - sourceHeight, - sizePreset, - GIF_SIZE_PRESETS - ); - - // When source is smaller than preset, use original dimensions - expect(width).toBe(sourceWidth); - expect(height).toBe(sourceHeight); - } - ), - { numRuns: 100 } - ); - }); - - it('should produce even dimensions for encoder compatibility', () => { - fc.assert( - fc.property( - fc.integer({ min: 100, max: 4000 }), - fc.integer({ min: 100, max: 4000 }), - fc.constantFrom('medium', 'large', 'original') as fc.Arbitrary, - (sourceWidth: number, sourceHeight: number, sizePreset: GifSizePreset) => { - const { width, height } = calculateOutputDimensions( - sourceWidth, - sourceHeight, - sizePreset, - GIF_SIZE_PRESETS - ); - - // When scaling occurs, dimensions should be even - // (original dimensions are passed through as-is) - if (sourceHeight > GIF_SIZE_PRESETS[sizePreset].maxHeight && sizePreset !== 'original') { - expect(width % 2).toBe(0); - expect(height % 2).toBe(0); - } - } - ), - { numRuns: 100 } - ); - }); -}); - -/** - * Property 6: Frame Count Consistency - * - * *For any* video with effective duration D (excluding trim regions) and frame rate F, - * the exported GIF SHALL contain approximately D × F frames (within ±1 frame tolerance). - * - * **Validates: Requirements 5.1** - * - * Feature: gif-export, Property 6: Frame Count Consistency - */ -describe('Property 6: Frame Count Consistency', () => { - // Helper function to calculate expected frame count - const calculateExpectedFrameCount = (durationSeconds: number, frameRate: number): number => { - return Math.ceil(durationSeconds * frameRate); - }; - - it('should calculate correct frame count for duration and frame rate', () => { - fc.assert( - fc.property( - fc.float({ min: 0.5, max: 60, noNaN: true }), // duration in seconds - fc.constantFrom(10, 15, 20, 25, 30), // valid frame rates - (duration: number, frameRate: number) => { - const expectedFrames = calculateExpectedFrameCount(duration, frameRate); - - // Frame count should be positive - expect(expectedFrames).toBeGreaterThan(0); - - // Frame count should be approximately duration * frameRate - const approximateFrames = duration * frameRate; - expect(Math.abs(expectedFrames - approximateFrames)).toBeLessThanOrEqual(1); - } - ), - { numRuns: 100 } - ); - }); - - it('should produce more frames with higher frame rates', () => { - fc.assert( - fc.property( - fc.float({ min: 1, max: 30, noNaN: true }), // duration in seconds - (duration: number) => { - const frames10fps = calculateExpectedFrameCount(duration, 10); - const frames30fps = calculateExpectedFrameCount(duration, 30); - - // 30fps should produce approximately 3x more frames than 10fps - expect(frames30fps).toBeGreaterThan(frames10fps); - expect(frames30fps / frames10fps).toBeCloseTo(3, 0); - } - ), - { numRuns: 100 } - ); - }); - - it('should handle trim regions by reducing effective duration', () => { - fc.assert( - fc.property( - fc.float({ min: 5, max: 60, noNaN: true }), // total duration - fc.float({ min: 0.5, max: 2, noNaN: true }), // trim duration (smaller than total) - fc.constantFrom(10, 15, 20, 25, 30), - (totalDuration: number, trimDuration: number, frameRate: number) => { - const effectiveDuration = totalDuration - trimDuration; - const framesWithTrim = calculateExpectedFrameCount(effectiveDuration, frameRate); - const framesWithoutTrim = calculateExpectedFrameCount(totalDuration, frameRate); - - // Trimmed video should have fewer frames - expect(framesWithTrim).toBeLessThan(framesWithoutTrim); - } - ), - { numRuns: 100 } - ); - }); -}); - - -/** - * Property 5: Valid GIF Output (Configuration Validation) - * - * *For any* successful GIF export, the output blob SHALL be a valid GIF file. - * This test validates the GIF configuration parameters are correctly set up. - * - * **Validates: Requirements 5.3** - * - * Feature: gif-export, Property 5: Valid GIF Output - * - * Note: Full GIF encoding validation requires browser environment with video. - * This test validates configuration correctness. - */ -describe('Property 5: Valid GIF Output (Configuration)', () => { - it('should generate valid GIF configuration for all frame rates', () => { - fc.assert( - fc.property( - fc.constantFrom(10, 15, 20, 25, 30), - fc.integer({ min: 100, max: 1920 }), - fc.integer({ min: 100, max: 1080 }), - fc.boolean(), - (frameRate: number, width: number, height: number, loop: boolean) => { - // Validate frame delay calculation (gif.js uses milliseconds) - const frameDelay = Math.round(1000 / frameRate); - - // Frame delay should be positive and reasonable - expect(frameDelay).toBeGreaterThan(0); - expect(frameDelay).toBeLessThanOrEqual(100); // 10fps = 100ms delay - - // Loop configuration - const repeat = loop ? 0 : 1; - expect([0, 1]).toContain(repeat); - - // Dimensions should be positive - expect(width).toBeGreaterThan(0); - expect(height).toBeGreaterThan(0); - } - ), - { numRuns: 100 } - ); - }); - - it('should calculate correct frame delays for each frame rate', () => { - const expectedDelays: Record = { - 10: 100, // 1000ms / 10fps = 100ms - 15: 67, // 1000ms / 15fps ≈ 67ms - 20: 50, // 1000ms / 20fps = 50ms - 25: 40, // 1000ms / 25fps = 40ms - 30: 33, // 1000ms / 30fps ≈ 33ms - }; - - for (const [fps, expectedDelay] of Object.entries(expectedDelays)) { - const frameRate = Number(fps); - const actualDelay = Math.round(1000 / frameRate); - expect(actualDelay).toBe(expectedDelay); - } - }); -}); - -/** - * Property 7: MP4 Export Regression - * - * *For any* valid MP4 export configuration that worked before this feature, - * the Video_Exporter SHALL continue to produce valid MP4 output. - * - * **Validates: Requirements 7.2** - * - * Feature: gif-export, Property 7: MP4 Export Regression - * - * Note: This test validates that MP4 export configuration remains unchanged. - */ -describe('Property 7: MP4 Export Regression', () => { - it('should maintain valid MP4 quality presets', () => { - const qualityPresets = ['medium', 'good', 'source']; - - fc.assert( - fc.property( - fc.constantFrom(...qualityPresets), - (quality: string) => { - // Quality presets should be valid - expect(['medium', 'good', 'source']).toContain(quality); - } - ), - { numRuns: 100 } - ); - }); - - it('should calculate valid MP4 export dimensions', () => { - fc.assert( - fc.property( - fc.integer({ min: 640, max: 3840 }), // sourceWidth - fc.integer({ min: 480, max: 2160 }), // sourceHeight - fc.constantFrom('medium', 'good', 'source'), - (sourceWidth: number, sourceHeight: number, quality: string) => { - let exportWidth: number; - let exportHeight: number; - const aspectRatio = sourceWidth / sourceHeight; - - if (quality === 'source') { - // Source quality uses original dimensions (may be odd) - exportWidth = sourceWidth; - exportHeight = sourceHeight; - - // Dimensions should be positive - expect(exportWidth).toBeGreaterThan(0); - expect(exportHeight).toBeGreaterThan(0); - } else { - const targetHeight = quality === 'medium' ? 720 : 1080; - exportHeight = Math.floor(targetHeight / 2) * 2; - exportWidth = Math.floor((exportHeight * aspectRatio) / 2) * 2; - - // Dimensions should be positive and even for non-source quality - expect(exportWidth).toBeGreaterThan(0); - expect(exportHeight).toBeGreaterThan(0); - expect(exportWidth % 2).toBe(0); - expect(exportHeight % 2).toBe(0); - } - } - ), - { numRuns: 100 } - ); - }); - - it('should maintain aspect ratio in MP4 export', () => { - fc.assert( - fc.property( - fc.integer({ min: 640, max: 3840 }), - fc.integer({ min: 480, max: 2160 }), - fc.constantFrom('medium', 'good'), - (sourceWidth: number, sourceHeight: number, quality: string) => { - const originalAspectRatio = sourceWidth / sourceHeight; - const targetHeight = quality === 'medium' ? 720 : 1080; - - const exportHeight = Math.floor(targetHeight / 2) * 2; - const exportWidth = Math.floor((exportHeight * originalAspectRatio) / 2) * 2; - - const exportAspectRatio = exportWidth / exportHeight; - - // Aspect ratio should be preserved within tolerance (due to even rounding) - expect(Math.abs(originalAspectRatio - exportAspectRatio)).toBeLessThan(0.05); - } - ), - { numRuns: 100 } - ); - }); -}); diff --git a/src/lib/exporter/types.test.ts b/src/lib/exporter/types.test.ts deleted file mode 100644 index 601d550..0000000 --- a/src/lib/exporter/types.test.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import * as fc from 'fast-check'; -import { - isValidGifFrameRate, - VALID_GIF_FRAME_RATES, - GifFrameRate -} from './types'; - -/** - * Property 1: Valid Frame Rate Acceptance - * - * *For any* frame rate value, the GIF_Exporter SHALL accept it if and only if - * it is one of the valid presets (15, 20, 25, 30 FPS). Invalid frame rates - * should be rejected with an error. - * - * **Validates: Requirements 2.2** - * - * Feature: gif-export, Property 1: Valid Frame Rate Acceptance - */ -describe('GIF Export Types', () => { - describe('Property 1: Valid Frame Rate Acceptance', () => { - // Property test: Valid frame rates should be accepted - it('should accept all valid frame rates (15, 20, 25, 30)', () => { - fc.assert( - fc.property( - fc.constantFrom(...VALID_GIF_FRAME_RATES), - (frameRate: GifFrameRate) => { - expect(isValidGifFrameRate(frameRate)).toBe(true); - } - ), - { numRuns: 100 } - ); - }); - - // Property test: Invalid frame rates should be rejected - it('should reject any frame rate not in the valid set', () => { - fc.assert( - fc.property( - fc.integer().filter(n => !VALID_GIF_FRAME_RATES.includes(n as GifFrameRate)), - (invalidFrameRate: number) => { - expect(isValidGifFrameRate(invalidFrameRate)).toBe(false); - } - ), - { numRuns: 100 } - ); - }); - - // Property test: Frame rate validation is deterministic - it('should return consistent results for the same input', () => { - fc.assert( - fc.property( - fc.integer({ min: 1, max: 60 }), - (frameRate: number) => { - const result1 = isValidGifFrameRate(frameRate); - const result2 = isValidGifFrameRate(frameRate); - expect(result1).toBe(result2); - } - ), - { numRuns: 100 } - ); - }); - }); -}); From 83d3e7b6b89a9bb63be36b2e709118bde9ae97ec Mon Sep 17 00:00:00 2001 From: Brodypen Date: Sat, 28 Feb 2026 01:08:19 -0600 Subject: [PATCH 3/7] refactor: replace magic numbers with named constants in useScreenRecorder --- src/hooks/useScreenRecorder.ts | 61 ++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index cfb2183..5b25c16 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -1,6 +1,36 @@ import { useState, useRef, useEffect } from "react"; import { fixWebmDuration } from "@fix-webm-duration/fix"; +// Target visually lossless 4K @ 60fps; fall back gracefully when hardware cannot keep up +const TARGET_FRAME_RATE = 60; +const MIN_FRAME_RATE = 30; +const TARGET_WIDTH = 3840; +const TARGET_HEIGHT = 2160; +const FOUR_K_PIXELS = TARGET_WIDTH * TARGET_HEIGHT; +const QHD_WIDTH = 2560; +const QHD_HEIGHT = 1440; +const QHD_PIXELS = QHD_WIDTH * QHD_HEIGHT; + +// Bitrates (bits per second) per resolution tier +const BITRATE_4K = 45_000_000; +const BITRATE_QHD = 28_000_000; +const BITRATE_BASE = 18_000_000; +const HIGH_FRAME_RATE_THRESHOLD = 60; +const HIGH_FRAME_RATE_BOOST = 1.7; + +// Fallback track settings when the driver reports nothing +const DEFAULT_WIDTH = 1920; +const DEFAULT_HEIGHT = 1080; + +// Codec alignment: VP9/AV1 require dimensions divisible by 2 +const CODEC_ALIGNMENT = 2; + +const RECORDER_TIMESLICE_MS = 1000; +const BITS_PER_MEGABIT = 1_000_000; +const CHROME_MEDIA_SOURCE = "desktop"; +const RECORDING_FILE_PREFIX = "recording-"; +const VIDEO_FILE_EXTENSION = ".webm"; + type UseScreenRecorderReturn = { recording: boolean; toggleRecording: () => void; @@ -13,11 +43,6 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const chunks = useRef([]); const startTime = useRef(0); - // Target visually lossless 4K @ 60fps; fall back gracefully when hardware cannot keep up - const TARGET_FRAME_RATE = 60; - const TARGET_WIDTH = 3840; - const TARGET_HEIGHT = 2160; - const FOUR_K_PIXELS = TARGET_WIDTH * TARGET_HEIGHT; const selectMimeType = () => { const preferred = [ "video/webm;codecs=av1", @@ -32,17 +57,17 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const computeBitrate = (width: number, height: number) => { const pixels = width * height; - const highFrameRateBoost = TARGET_FRAME_RATE >= 60 ? 1.7 : 1; + const highFrameRateBoost = TARGET_FRAME_RATE >= HIGH_FRAME_RATE_THRESHOLD ? HIGH_FRAME_RATE_BOOST : 1; if (pixels >= FOUR_K_PIXELS) { - return Math.round(45_000_000 * highFrameRateBoost); + return Math.round(BITRATE_4K * highFrameRateBoost); } - if (pixels >= 2560 * 1440) { - return Math.round(28_000_000 * highFrameRateBoost); + if (pixels >= QHD_PIXELS) { + return Math.round(BITRATE_QHD * highFrameRateBoost); } - return Math.round(18_000_000 * highFrameRateBoost); + return Math.round(BITRATE_BASE * highFrameRateBoost); }; const stopRecording = useRef(() => { @@ -91,12 +116,12 @@ export function useScreenRecorder(): UseScreenRecorderReturn { audio: false, video: { mandatory: { - chromeMediaSource: "desktop", + chromeMediaSource: CHROME_MEDIA_SOURCE, chromeMediaSourceId: selectedSource.id, maxWidth: TARGET_WIDTH, maxHeight: TARGET_HEIGHT, maxFrameRate: TARGET_FRAME_RATE, - minFrameRate: 30, + minFrameRate: MIN_FRAME_RATE, }, }, }); @@ -115,18 +140,18 @@ export function useScreenRecorder(): UseScreenRecorderReturn { console.warn("Unable to lock 4K/60fps constraints, using best available track settings.", error); } - let { width = 1920, height = 1080, frameRate = TARGET_FRAME_RATE } = videoTrack.getSettings(); + let { width = DEFAULT_WIDTH, height = DEFAULT_HEIGHT, frameRate = TARGET_FRAME_RATE } = videoTrack.getSettings(); // Ensure dimensions are divisible by 2 for VP9/AV1 codec compatibility - width = Math.floor(width / 2) * 2; - height = Math.floor(height / 2) * 2; + width = Math.floor(width / CODEC_ALIGNMENT) * CODEC_ALIGNMENT; + height = Math.floor(height / CODEC_ALIGNMENT) * CODEC_ALIGNMENT; const videoBitsPerSecond = computeBitrate(width, height); const mimeType = selectMimeType(); console.log( `Recording at ${width}x${height} @ ${frameRate ?? TARGET_FRAME_RATE}fps using ${mimeType} / ${Math.round( - videoBitsPerSecond / 1_000_000 + videoBitsPerSecond / BITS_PER_MEGABIT )} Mbps` ); @@ -148,7 +173,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { // Clear chunks early to free memory immediately after blob creation chunks.current = []; const timestamp = Date.now(); - const videoFileName = `recording-${timestamp}.webm`; + const videoFileName = `${RECORDING_FILE_PREFIX}${timestamp}${VIDEO_FILE_EXTENSION}`; try { const videoBlob = await fixWebmDuration(buggyBlob, duration); @@ -169,7 +194,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } }; recorder.onerror = () => setRecording(false); - recorder.start(1000); + recorder.start(RECORDER_TIMESLICE_MS); startTime.current = Date.now(); setRecording(true); window.electronAPI?.setRecordingState(true); From 4b3afcf535d3a27decc9cdea653ae1148ed8579e Mon Sep 17 00:00:00 2001 From: Siddharth Date: Fri, 27 Feb 2026 23:44:02 -0800 Subject: [PATCH 4/7] annotation bounding and canvas wrapping --- src/lib/exporter/annotationRenderer.ts | 29 ++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/lib/exporter/annotationRenderer.ts b/src/lib/exporter/annotationRenderer.ts index af04e19..06c4121 100644 --- a/src/lib/exporter/annotationRenderer.ts +++ b/src/lib/exporter/annotationRenderer.ts @@ -138,7 +138,12 @@ function renderText( const style = annotation.style; ctx.save(); - + + // Clip text to annotation box bounds (matches editor's overflow: hidden) + ctx.beginPath(); + ctx.rect(x, y, width, height); + ctx.clip(); + const fontWeight = style.fontWeight === 'bold' ? 'bold' : 'normal'; const fontStyle = style.fontStyle === 'italic' ? 'italic' : 'normal'; const scaledFontSize = style.fontSize * scaleFactor; @@ -161,7 +166,27 @@ function renderText( ctx.textAlign = 'left'; } - const lines = annotation.content.split('\n'); + const availableWidth = width - containerPadding * 2; + const rawLines = annotation.content.split('\n'); + const lines: string[] = []; + for (const rawLine of rawLines) { + if (!rawLine) { + lines.push(''); + continue; + } + const words = rawLine.split(/(\s+)/); + let current = ''; + for (const word of words) { + const test = current + word; + if (current && ctx.measureText(test).width > availableWidth) { + lines.push(current); + current = word.trimStart(); + } else { + current = test; + } + } + if (current) lines.push(current); + } const lineHeight = scaledFontSize * 1.4; const startY = textY - ((lines.length - 1) * lineHeight) / 2; From a2b9eea90aa96595bf808933c37410fda43b3624 Mon Sep 17 00:00:00 2001 From: Yusuf Mohsinally <463376+yusufm@users.noreply.github.com> Date: Wed, 18 Feb 2026 13:29:50 -0800 Subject: [PATCH 5/7] feat: add cursor telemetry-driven zoom suggestions --- electron/electron-env.d.ts | 7 + electron/ipc/handlers.ts | 117 +++++++++++++- electron/preload.ts | 3 + src/components/video-editor/VideoEditor.tsx | 65 +++++++- .../video-editor/timeline/TimelineEditor.tsx | 151 +++++++++++++++++- src/components/video-editor/types.ts | 6 + src/vite-env.d.ts | 12 ++ 7 files changed, 355 insertions(+), 6 deletions(-) diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index dba3f16..dda3d8d 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -32,6 +32,7 @@ interface Window { storeRecordedVideo: (videoData: ArrayBuffer, fileName: string) => Promise<{ success: boolean; path?: string; message?: string }> getRecordedVideoPath: () => Promise<{ success: boolean; path?: string; message?: string }> setRecordingState: (recording: boolean) => Promise + getCursorTelemetry: (videoPath?: string) => Promise<{ success: boolean; samples: CursorTelemetryPoint[]; message?: string; error?: string }> onStopRecordingFromTray: (callback: () => void) => () => void openExternalUrl: (url: string) => Promise<{ success: boolean; error?: string }> saveExportedVideo: (videoData: ArrayBuffer, fileName: string) => Promise<{ success: boolean; path?: string; message?: string; cancelled?: boolean }> @@ -52,3 +53,9 @@ interface ProcessedDesktopSource { thumbnail: string | null appIcon: string | null } + +interface CursorTelemetryPoint { + timeMs: number + cx: number + cy: number +} diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 867b72b..24a63a1 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -1,10 +1,62 @@ -import { ipcMain, desktopCapturer, BrowserWindow, shell, app, dialog } from 'electron' +import { ipcMain, desktopCapturer, BrowserWindow, shell, app, dialog, screen } from 'electron' import fs from 'node:fs/promises' import path from 'node:path' import { RECORDINGS_DIR } from '../main' let selectedSource: any = null +let currentVideoPath: string | null = null + +const CURSOR_TELEMETRY_VERSION = 1 +const CURSOR_SAMPLE_INTERVAL_MS = 100 +const MAX_CURSOR_SAMPLES = 60 * 60 * 10 // 1 hour @ 10Hz + +interface CursorTelemetryPoint { + timeMs: number + cx: number + cy: number +} + +let cursorCaptureInterval: NodeJS.Timeout | null = null +let cursorCaptureStartTimeMs = 0 +let activeCursorSamples: CursorTelemetryPoint[] = [] +let pendingCursorSamples: CursorTelemetryPoint[] = [] + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)) +} + +function stopCursorCapture() { + if (cursorCaptureInterval) { + clearInterval(cursorCaptureInterval) + cursorCaptureInterval = null + } +} + +function sampleCursorPoint() { + const cursor = screen.getCursorScreenPoint() + const sourceDisplayId = Number(selectedSource?.display_id) + const sourceDisplay = Number.isFinite(sourceDisplayId) + ? screen.getAllDisplays().find((display) => display.id === sourceDisplayId) ?? null + : null + const display = sourceDisplay ?? screen.getDisplayNearestPoint(cursor) + const bounds = display.bounds + const width = Math.max(1, bounds.width) + const height = Math.max(1, bounds.height) + + const cx = clamp((cursor.x - bounds.x) / width, 0, 1) + const cy = clamp((cursor.y - bounds.y) / height, 0, 1) + + activeCursorSamples.push({ + timeMs: Math.max(0, Date.now() - cursorCaptureStartTimeMs), + cx, + cy, + }) + + if (activeCursorSamples.length > MAX_CURSOR_SAMPLES) { + activeCursorSamples.shift() + } +} export function registerIpcHandlers( createEditorWindow: () => void, @@ -61,6 +113,17 @@ export function registerIpcHandlers( const videoPath = path.join(RECORDINGS_DIR, fileName) await fs.writeFile(videoPath, Buffer.from(videoData)) currentVideoPath = videoPath; + + const telemetryPath = `${videoPath}.cursor.json` + if (pendingCursorSamples.length > 0) { + await fs.writeFile( + telemetryPath, + JSON.stringify({ version: CURSOR_TELEMETRY_VERSION, samples: pendingCursorSamples }, null, 2), + 'utf-8' + ) + } + pendingCursorSamples = [] + return { success: true, path: videoPath, @@ -98,12 +161,62 @@ export function registerIpcHandlers( }) ipcMain.handle('set-recording-state', (_, recording: boolean) => { + if (recording) { + stopCursorCapture() + activeCursorSamples = [] + pendingCursorSamples = [] + cursorCaptureStartTimeMs = Date.now() + sampleCursorPoint() + cursorCaptureInterval = setInterval(sampleCursorPoint, CURSOR_SAMPLE_INTERVAL_MS) + } else { + stopCursorCapture() + pendingCursorSamples = [...activeCursorSamples] + activeCursorSamples = [] + } + const source = selectedSource || { name: 'Screen' } if (onRecordingStateChange) { onRecordingStateChange(recording, source.name) } }) + ipcMain.handle('get-cursor-telemetry', async (_, videoPath?: string) => { + const targetVideoPath = videoPath ?? currentVideoPath + if (!targetVideoPath) { + return { success: true, samples: [] } + } + + const telemetryPath = `${targetVideoPath}.cursor.json` + try { + const content = await fs.readFile(telemetryPath, 'utf-8') + const parsed = JSON.parse(content) + const rawSamples = Array.isArray(parsed) + ? parsed + : (Array.isArray(parsed?.samples) ? parsed.samples : []) + + const samples: CursorTelemetryPoint[] = rawSamples + .filter((sample: unknown) => Boolean(sample && typeof sample === 'object')) + .map((sample: unknown) => { + const point = sample as Partial + return { + timeMs: typeof point.timeMs === 'number' && Number.isFinite(point.timeMs) ? Math.max(0, point.timeMs) : 0, + cx: typeof point.cx === 'number' && Number.isFinite(point.cx) ? clamp(point.cx, 0, 1) : 0.5, + cy: typeof point.cy === 'number' && Number.isFinite(point.cy) ? clamp(point.cy, 0, 1) : 0.5, + } + }) + .sort((a: CursorTelemetryPoint, b: CursorTelemetryPoint) => a.timeMs - b.timeMs) + + return { success: true, samples } + } catch (error) { + const nodeError = error as NodeJS.ErrnoException + if (nodeError.code === 'ENOENT') { + return { success: true, samples: [] } + } + console.error('Failed to load cursor telemetry:', error) + return { success: false, message: 'Failed to load cursor telemetry', error: String(error), samples: [] } + } + }) + ipcMain.handle('open-external-url', async (_, url: string) => { try { @@ -198,8 +311,6 @@ export function registerIpcHandlers( } }); - let currentVideoPath: string | null = null; - ipcMain.handle('set-current-video-path', (_, path: string) => { currentVideoPath = path; return { success: true }; diff --git a/electron/preload.ts b/electron/preload.ts index 02fcc97..f58d8a8 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -37,6 +37,9 @@ contextBridge.exposeInMainWorld('electronAPI', { setRecordingState: (recording: boolean) => { return ipcRenderer.invoke('set-recording-state', recording) }, + getCursorTelemetry: (videoPath?: string) => { + return ipcRenderer.invoke('get-cursor-telemetry', videoPath) + }, onStopRecordingFromTray: (callback: () => void) => { const listener = () => callback() ipcRenderer.on('stop-recording-from-tray', listener) diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index c0a038e..d418950 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -23,6 +23,7 @@ import { type ZoomDepth, type ZoomFocus, type ZoomRegion, + type CursorTelemetryPoint, type TrimRegion, type AnnotationRegion, type CropRegion, @@ -50,6 +51,7 @@ export default function VideoEditor() { const [padding, setPadding] = useState(50); const [cropRegion, setCropRegion] = useState(DEFAULT_CROP_REGION); const [zoomRegions, setZoomRegions] = useState([]); + const [cursorTelemetry, setCursorTelemetry] = useState([]); const [selectedZoomId, setSelectedZoomId] = useState(null); const [trimRegions, setTrimRegions] = useState([]); const [selectedTrimId, setSelectedTrimId] = useState(null); @@ -89,6 +91,19 @@ export default function VideoEditor() { return fileUrl; }; + const fromFileUrl = (fileUrl: string): string => { + if (!fileUrl.startsWith('file://')) { + return fileUrl; + } + + try { + const url = new URL(fileUrl); + return decodeURIComponent(url.pathname); + } catch { + return fileUrl.replace(/^file:\/\//, ''); + } + }; + useEffect(() => { async function loadVideo() { try { @@ -109,6 +124,37 @@ export default function VideoEditor() { loadVideo(); }, []); + useEffect(() => { + let mounted = true; + + async function loadCursorTelemetry() { + if (!videoPath) { + if (mounted) { + setCursorTelemetry([]); + } + return; + } + + try { + const result = await window.electronAPI.getCursorTelemetry(fromFileUrl(videoPath)); + if (mounted) { + setCursorTelemetry(result.success ? result.samples : []); + } + } catch (telemetryError) { + console.warn('Unable to load cursor telemetry:', telemetryError); + if (mounted) { + setCursorTelemetry([]); + } + } + } + + loadCursorTelemetry(); + + return () => { + mounted = false; + }; + }, [videoPath]); + // Initialize default wallpaper with resolved asset path useEffect(() => { let mounted = true; @@ -180,6 +226,21 @@ export default function VideoEditor() { setSelectedAnnotationId(null); }, []); + const handleZoomSuggested = useCallback((span: Span, focus: ZoomFocus) => { + const id = `zoom-${nextZoomIdRef.current++}`; + const newRegion: ZoomRegion = { + id, + startMs: Math.round(span.start), + endMs: Math.round(span.end), + depth: DEFAULT_ZOOM_DEPTH, + focus: clampFocusToDepth(focus, DEFAULT_ZOOM_DEPTH), + }; + setZoomRegions((prev) => [...prev, newRegion]); + setSelectedZoomId(id); + setSelectedTrimId(null); + setSelectedAnnotationId(null); + }, []); + const handleTrimAdded = useCallback((span: Span) => { const id = `trim-${nextTrimIdRef.current++}`; const newRegion: TrimRegion = { @@ -804,8 +865,10 @@ export default function VideoEditor() { videoDuration={duration} currentTime={currentTime} onSeek={handleSeek} + cursorTelemetry={cursorTelemetry} zoomRegions={zoomRegions} onZoomAdded={handleZoomAdded} + onZoomSuggested={handleZoomSuggested} onZoomSpanChange={handleZoomSpanChange} onZoomDelete={handleZoomDelete} selectedZoomId={selectedZoomId} @@ -894,4 +957,4 @@ export default function VideoEditor() { /> ); -} \ No newline at end of file +} diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx index 9b091ef..5f965d0 100644 --- a/src/components/video-editor/timeline/TimelineEditor.tsx +++ b/src/components/video-editor/timeline/TimelineEditor.tsx @@ -1,7 +1,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useTimelineContext } from "dnd-timeline"; import { Button } from "@/components/ui/button"; -import { Plus, Scissors, ZoomIn, MessageSquare, ChevronDown, Check } from "lucide-react"; +import { Plus, Scissors, ZoomIn, MessageSquare, ChevronDown, Check, WandSparkles } from "lucide-react"; import { toast } from "sonner"; import { cn } from "@/lib/utils"; import TimelineWrapper from "./TimelineWrapper"; @@ -9,7 +9,7 @@ import Row from "./Row"; import Item from "./Item"; import KeyframeMarkers from "./KeyframeMarkers"; import type { Range, Span } from "dnd-timeline"; -import type { ZoomRegion, TrimRegion, AnnotationRegion } from "../types"; +import type { ZoomRegion, TrimRegion, AnnotationRegion, CursorTelemetryPoint, ZoomFocus } from "../types"; import { v4 as uuidv4 } from 'uuid'; import { DropdownMenu, @@ -26,13 +26,19 @@ const TRIM_ROW_ID = "row-trim"; const ANNOTATION_ROW_ID = "row-annotation"; const FALLBACK_RANGE_MS = 1000; const TARGET_MARKER_COUNT = 12; +const MIN_DWELL_DURATION_MS = 450; +const MAX_DWELL_DURATION_MS = 2600; +const DWELL_MOVE_THRESHOLD = 0.02; +const SUGGESTION_SPACING_MS = 1800; interface TimelineEditorProps { videoDuration: number; currentTime: number; onSeek?: (time: number) => void; + cursorTelemetry?: CursorTelemetryPoint[]; zoomRegions: ZoomRegion[]; onZoomAdded: (span: Span) => void; + onZoomSuggested?: (span: Span, focus: ZoomFocus) => void; onZoomSpanChange: (id: string, span: Span) => void; onZoomDelete: (id: string) => void; selectedZoomId: string | null; @@ -520,8 +526,10 @@ export default function TimelineEditor({ videoDuration, currentTime, onSeek, + cursorTelemetry = [], zoomRegions, onZoomAdded, + onZoomSuggested, onZoomSpanChange, onZoomDelete, selectedZoomId, @@ -716,6 +724,136 @@ export default function TimelineEditor({ onZoomAdded({ start: startPos, end: startPos + actualDuration }); }, [videoDuration, totalMs, currentTimeMs, zoomRegions, onZoomAdded, defaultRegionDurationMs]); + const handleSuggestZooms = useCallback(() => { + if (!videoDuration || videoDuration === 0 || totalMs === 0) { + return; + } + + if (!onZoomSuggested) { + toast.error("Zoom suggestion handler unavailable"); + return; + } + + if (cursorTelemetry.length < 2) { + toast.info("No cursor telemetry available", { + description: "Record a screencast first to generate cursor-based suggestions.", + }); + return; + } + + const defaultDuration = Math.min(defaultRegionDurationMs, totalMs); + if (defaultDuration <= 0) { + return; + } + + const reservedSpans = [...zoomRegions] + .map((region) => ({ start: region.startMs, end: region.endMs })) + .sort((a, b) => a.start - b.start); + + const normalizedSamples = [...cursorTelemetry] + .filter((sample) => Number.isFinite(sample.timeMs) && Number.isFinite(sample.cx) && Number.isFinite(sample.cy)) + .sort((a, b) => a.timeMs - b.timeMs) + .map((sample) => ({ + timeMs: Math.max(0, Math.min(sample.timeMs, totalMs)), + cx: Math.max(0, Math.min(sample.cx, 1)), + cy: Math.max(0, Math.min(sample.cy, 1)), + })); + + if (normalizedSamples.length < 2) { + toast.info("No usable cursor telemetry", { + description: "The recording does not include enough cursor movement data.", + }); + return; + } + + const dwellCandidates: Array<{ centerTimeMs: number; focus: ZoomFocus; strength: number }> = []; + let runStart = 0; + + const pushRunIfDwell = (startIndex: number, endIndexExclusive: number) => { + if (endIndexExclusive - startIndex < 2) { + return; + } + + const start = normalizedSamples[startIndex]; + const end = normalizedSamples[endIndexExclusive - 1]; + const runDuration = end.timeMs - start.timeMs; + if (runDuration < MIN_DWELL_DURATION_MS || runDuration > MAX_DWELL_DURATION_MS) { + return; + } + + const runSamples = normalizedSamples.slice(startIndex, endIndexExclusive); + const avgCx = runSamples.reduce((sum, sample) => sum + sample.cx, 0) / runSamples.length; + const avgCy = runSamples.reduce((sum, sample) => sum + sample.cy, 0) / runSamples.length; + + dwellCandidates.push({ + centerTimeMs: Math.round((start.timeMs + end.timeMs) / 2), + focus: { cx: avgCx, cy: avgCy }, + strength: runDuration, + }); + }; + + for (let index = 1; index < normalizedSamples.length; index += 1) { + const prev = normalizedSamples[index - 1]; + const curr = normalizedSamples[index]; + const dx = curr.cx - prev.cx; + const dy = curr.cy - prev.cy; + const distance = Math.hypot(dx, dy); + + if (distance > DWELL_MOVE_THRESHOLD) { + pushRunIfDwell(runStart, index); + runStart = index; + } + } + pushRunIfDwell(runStart, normalizedSamples.length); + + if (dwellCandidates.length === 0) { + toast.info("No clear cursor dwell moments found", { + description: "Try a recording with slower cursor pauses on important actions.", + }); + return; + } + + const sortedCandidates = [...dwellCandidates].sort((a, b) => b.strength - a.strength); + const acceptedCenters: number[] = []; + + let addedCount = 0; + + sortedCandidates.forEach((candidate) => { + const tooCloseToAccepted = acceptedCenters.some( + (center) => Math.abs(center - candidate.centerTimeMs) < SUGGESTION_SPACING_MS, + ); + + if (tooCloseToAccepted) { + return; + } + + const centeredStart = Math.round(candidate.centerTimeMs - defaultDuration / 2); + const candidateStart = Math.max(0, Math.min(centeredStart, totalMs - defaultDuration)); + const candidateEnd = candidateStart + defaultDuration; + const hasOverlap = reservedSpans.some( + (span) => candidateEnd > span.start && candidateStart < span.end, + ); + + if (hasOverlap) { + return; + } + + reservedSpans.push({ start: candidateStart, end: candidateEnd }); + acceptedCenters.push(candidate.centerTimeMs); + onZoomSuggested({ start: candidateStart, end: candidateEnd }, candidate.focus); + addedCount += 1; + }); + + if (addedCount === 0) { + toast.info("No auto-zoom slots available", { + description: "Detected dwell points overlap existing zoom regions.", + }); + return; + } + + toast.success(`Added ${addedCount} cursor-based zoom suggestion${addedCount === 1 ? "" : "s"}`); + }, [videoDuration, totalMs, defaultRegionDurationMs, zoomRegions, onZoomSuggested, cursorTelemetry]); + const handleAddTrim = useCallback(() => { if (!videoDuration || videoDuration === 0 || totalMs === 0 || !onTrimAdded) { return; @@ -920,6 +1058,15 @@ export default function TimelineEditor({ > +