huanld/openscreen
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

feat: apply native cursor visual effects

Browse Source
This commit is contained in:
EtienneLescot
2026-05-05 21:13:02 +02:00
parent ab3d38d90f
commit d0341580d6
11 changed files with 340 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/components/video-editor/VideoEditor.tsx
+9
View File
@@ -1507,6 +1507,9 @@ export default function VideoEditor() {
cropRegion,
cursorRecordingData,
cursorScale: showCursor ? cursorSize : 0,
cursorSmoothing,
cursorMotionBlur,
cursorClickBounce,
annotationRegions,
webcamLayoutPreset,
webcamMaskShape,
@@ -1650,6 +1653,9 @@ export default function VideoEditor() {
cropRegion,
cursorRecordingData,
cursorScale: showCursor ? cursorSize : 0,
cursorSmoothing,
cursorMotionBlur,
cursorClickBounce,
annotationRegions,
webcamLayoutPreset,
webcamMaskShape,
@@ -1744,6 +1750,9 @@ export default function VideoEditor() {
effectiveCursorHighlight,
showCursor,
cursorSize,
cursorSmoothing,
cursorMotionBlur,
cursorClickBounce,
t,
],
);
src/components/video-editor/VideoPlayback.tsx
+41 -4
View File
@@ -26,11 +26,19 @@ import {
type WebcamSizePreset,
} from "@/lib/compositeLayout";
import {
createNativeCursorMotionBlurState,
createNativeCursorSmoothingState,
getNativeCursorClickBounceProgress,
getNativeCursorClickBounceScale,
getNativeCursorMotionBlurPx,
hasNativeCursorRecordingData,
projectNativeCursorToLocal,
projectNativeCursorToStage,
resetNativeCursorMotionBlurState,
resetNativeCursorSmoothingState,
resolveInterpolatedNativeCursorFrame,
resolveNativeCursorRenderAsset,
smoothNativeCursorSample,
} from "@/lib/cursor/nativeCursor";
import { classifyWallpaper, DEFAULT_WALLPAPER, resolveImageWallpaperUrl } from "@/lib/wallpaper";
import { getCssClipPath } from "@/lib/webcamMaskShapes";
@@ -642,6 +650,8 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
useEffect(() => {
cursorRecordingDataRef.current = cursorRecordingData;
resetNativeCursorSmoothingState(nativeCursorSmoothingStateRef.current);
resetNativeCursorMotionBlurState(nativeCursorMotionBlurStateRef.current);
}, [cursorRecordingData]);
useEffect(() => {
@@ -1311,7 +1321,10 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
}
if (nativeCursorImage) {
nativeCursorImage.style.display = "none";
nativeCursorImage.style.filter = "none";
}
resetNativeCursorSmoothingState(nativeCursorSmoothingStateRef.current);
resetNativeCursorMotionBlurState(nativeCursorMotionBlurStateRef.current);
};
if (nativeCursorImage) {
if (hasNativeCursorRecordingRef.current && showCursorRef.current) {
@@ -1321,13 +1334,20 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
timeMs,
);
if (frame) {
const displaySample = smoothNativeCursorSample({
forceSnap: !isPlayingRef.current || isSeekingRef.current,
sample: frame.sample,
smoothing: cursorSmoothingRef.current,
state: nativeCursorSmoothingStateRef.current,
timeMs,
});
const cameraContainer = cameraContainerRef.current;
const videoContainer = videoContainerRef.current;
const cropRegionValue = cropRegionRef.current ?? { x: 0, y: 0, width: 1, height: 1 };
const projectedLocalPoint = projectNativeCursorToLocal({
cropRegion: cropRegionValue,
maskRect: baseMaskRef.current,
sample: frame.sample,
sample: displaySample,
});
const projectedStagePoint =
cameraContainer && videoContainer
@@ -1339,17 +1359,32 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
x: videoContainer.x,
y: videoContainer.y,
},
sample: frame.sample,
sample: displaySample,
})
: null;
if (projectedLocalPoint && projectedStagePoint) {
const renderAsset = resolveNativeCursorRenderAsset(
frame.asset,
window.devicePixelRatio || 1,
frame.sample,
displaySample,
);
const scale = Math.max(0, cursorSizeRef.current);
const bounceProgress = getNativeCursorClickBounceProgress(
cursorRecordingDataRef.current,
timeMs,
);
const scale =
Math.max(0, cursorSizeRef.current) *
getNativeCursorClickBounceScale(cursorClickBounceRef.current, bounceProgress);
const transformedScale = scale * Math.abs(cameraContainer?.scale.x || 1);
const blurPx =
!isPlayingRef.current || isSeekingRef.current
? 0
: getNativeCursorMotionBlurPx({
motionBlur: cursorMotionBlurRef.current,
point: projectedStagePoint,
state: nativeCursorMotionBlurStateRef.current,
timeMs,
});
if (nativeCursorImageIdRef.current !== renderAsset.id) {
nativeCursorImage.src = renderAsset.imageDataUrl;
nativeCursorImageIdRef.current = renderAsset.id;
@@ -1357,6 +1392,8 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
nativeCursorImage.style.display = "block";
nativeCursorImage.style.width = `${renderAsset.width * transformedScale}px`;
nativeCursorImage.style.height = `${renderAsset.height * transformedScale}px`;
nativeCursorImage.style.filter =
blurPx > 0 ? `blur(${blurPx.toFixed(2)}px)` : "none";
nativeCursorImage.style.transform = `translate3d(${
projectedStagePoint.x - renderAsset.hotspotX * transformedScale
}px, ${projectedStagePoint.y - renderAsset.hotspotY * transformedScale}px, 0)`;
src/lib/cursor/nativeCursor.ts
+174 -5
View File
@@ -28,6 +28,20 @@ export interface ActiveNativeCursorFrame {
sample: CursorRecordingSample;
}
export interface NativeCursorSmoothingState {
cx: number;
cy: number;
lastTimeMs: number | null;
initialized: boolean;
}
export interface NativeCursorMotionBlurState {
x: number;
y: number;
lastTimeMs: number | null;
initialized: boolean;
}
interface ProjectNativeCursorOptions {
cropRegion: CropRegion;
maskRect: { x: number; y: number; width: number; height: number };
@@ -43,6 +57,9 @@ function clamp(value: number, min: number, max: number) {
return Math.min(max, Math.max(min, value));
}
const NATIVE_CURSOR_CLICK_ANIMATION_MS = 140;
const NATIVE_CURSOR_MOTION_BLUR_MAX_PX = 6;
interface PrettyNativeCursorAsset {
imageDataUrl: string;
width: number;
@@ -167,17 +184,20 @@ const PRETTY_NATIVE_CURSOR_ASSETS: Partial<Record<NativeCursorType, PrettyNative
};
function resolveUntypedPrettyNativeCursorAsset(asset: NativeCursorAsset) {
if (asset.cursorType || asset.width < 24 || asset.width > 64 || asset.height < 24 || asset.height > 64) {
if (
asset.cursorType ||
asset.width < 24 ||
asset.width > 64 ||
asset.height < 24 ||
asset.height > 64
) {
return null;
}
const hotspotXNorm = asset.hotspotX / asset.width;
const hotspotYNorm = asset.hotspotY / asset.height;
const looksLikeChromiumGrabCursor =
hotspotXNorm >= 0.22 &&
hotspotXNorm <= 0.55 &&
hotspotYNorm >= 0.2 &&
hotspotYNorm <= 0.45;
hotspotXNorm >= 0.22 && hotspotXNorm <= 0.55 && hotspotYNorm >= 0.2 && hotspotYNorm <= 0.45;
return looksLikeChromiumGrabCursor ? (PRETTY_NATIVE_CURSOR_ASSETS["open-hand"] ?? null) : null;
}
@@ -193,6 +213,155 @@ export function hasNativeCursorRecordingData(
);
}
export function createNativeCursorSmoothingState(): NativeCursorSmoothingState {
return {
cx: 0,
cy: 0,
lastTimeMs: null,
initialized: false,
};
}
export function resetNativeCursorSmoothingState(state: NativeCursorSmoothingState) {
state.cx = 0;
state.cy = 0;
state.lastTimeMs = null;
state.initialized = false;
}
export function createNativeCursorMotionBlurState(): NativeCursorMotionBlurState {
return {
x: 0,
y: 0,
lastTimeMs: null,
initialized: false,
};
}
export function resetNativeCursorMotionBlurState(state: NativeCursorMotionBlurState) {
state.x = 0;
state.y = 0;
state.lastTimeMs = null;
state.initialized = false;
}
export function smoothNativeCursorSample({
forceSnap = false,
sample,
smoothing,
state,
timeMs,
}: {
forceSnap?: boolean;
sample: CursorRecordingSample;
smoothing: number;
state: NativeCursorSmoothingState;
timeMs: number;
}): CursorRecordingSample {
const clampedSmoothing = clamp(Number.isFinite(smoothing) ? smoothing : 0, 0, 0.98);
const previousTimeMs = state.lastTimeMs;
const shouldSnap =
forceSnap ||
clampedSmoothing <= 0 ||
!state.initialized ||
previousTimeMs === null ||
timeMs <= previousTimeMs;
if (shouldSnap) {
state.cx = sample.cx;
state.cy = sample.cy;
state.lastTimeMs = timeMs;
state.initialized = true;
return sample;
}
const frameCount = Math.max(1, (timeMs - previousTimeMs) / (1000 / 60));
const alpha = 1 - Math.pow(clampedSmoothing, frameCount);
state.cx += (sample.cx - state.cx) * alpha;
state.cy += (sample.cy - state.cy) * alpha;
state.lastTimeMs = timeMs;
return {
...sample,
cx: state.cx,
cy: state.cy,
};
}
export function getNativeCursorClickBounceProgress(
recordingData: CursorRecordingData | null | undefined,
timeMs: number,
) {
if (!hasNativeCursorRecordingData(recordingData)) {
return 0;
}
for (let index = recordingData.samples.length - 1; index >= 0; index -= 1) {
const sample = recordingData.samples[index];
if (sample.timeMs > timeMs) {
continue;
}
const ageMs = timeMs - sample.timeMs;
if (ageMs > NATIVE_CURSOR_CLICK_ANIMATION_MS) {
return 0;
}
if (sample.interactionType === "click") {
return 1 - ageMs / NATIVE_CURSOR_CLICK_ANIMATION_MS;
}
}
return 0;
}
export function getNativeCursorClickBounceScale(clickBounce: number, progress: number) {
if (progress <= 0 || clickBounce <= 0) {
return 1;
}
const bounceAmount = Math.sin(progress * Math.PI);
const amplitude = clamp(clickBounce, 0, 4) * 0.08;
return Math.max(0.72, 1 - bounceAmount * amplitude);
}
export function getNativeCursorMotionBlurPx({
motionBlur,
point,
state,
timeMs,
}: {
motionBlur: number;
point: { x: number; y: number };
state: NativeCursorMotionBlurState;
timeMs: number;
}) {
const clampedMotionBlur = clamp(Number.isFinite(motionBlur) ? motionBlur : 0, 0, 1);
const previousTimeMs = state.lastTimeMs;
const shouldSnap =
clampedMotionBlur <= 0 ||
!state.initialized ||
previousTimeMs === null ||
timeMs <= previousTimeMs;
if (shouldSnap) {
state.x = point.x;
state.y = point.y;
state.lastTimeMs = timeMs;
state.initialized = true;
return 0;
}
const deltaMs = Math.max(1, timeMs - previousTimeMs);
const distance = Math.hypot(point.x - state.x, point.y - state.y);
const speedPxPerSecond = (distance / deltaMs) * 1000;
state.x = point.x;
state.y = point.y;
state.lastTimeMs = timeMs;
return clamp(speedPxPerSecond * clampedMotionBlur * 0.004, 0, NATIVE_CURSOR_MOTION_BLUR_MAX_PX);
}
function getCroppedCursorPosition(sample: CursorRecordingSample, cropRegion: CropRegion) {
if (cropRegion.width <= 0 || cropRegion.height <= 0) {
return null;
src/lib/exporter/frameRenderer.ts
+43 -7
View File
@@ -57,9 +57,17 @@ import {
type StyledRenderRect,
} from "@/lib/compositeLayout";
import {
createNativeCursorMotionBlurState,
createNativeCursorSmoothingState,
getNativeCursorClickBounceProgress,
getNativeCursorClickBounceScale,
getNativeCursorMotionBlurPx,
projectNativeCursorToLocal,
resetNativeCursorMotionBlurState,
resetNativeCursorSmoothingState,
resolveInterpolatedNativeCursorFrame,
resolveNativeCursorRenderAsset,
smoothNativeCursorSample,
} from "@/lib/cursor/nativeCursor";
import { BackgroundLoadError, classifyWallpaper, resolveImageWallpaperUrl } from "@/lib/wallpaper";
import { drawCanvasClipPath } from "@/lib/webcamMaskShapes";
@@ -87,6 +95,9 @@ interface FrameRenderConfig {
cropRegion: CropRegion;
cursorRecordingData?: CursorRecordingData | null;
cursorScale?: number;
cursorSmoothing?: number;
cursorMotionBlur?: number;
cursorClickBounce?: number;
videoWidth: number;
videoHeight: number;
webcamSize?: Size | null;
@@ -151,6 +162,8 @@ export class FrameRenderer {
private layoutCache: LayoutCache | null = null;
private currentVideoTime = 0;
private motionBlurState: MotionBlurState = createMotionBlurState();
private nativeCursorSmoothingState = createNativeCursorSmoothingState();
private nativeCursorMotionBlurState = createNativeCursorMotionBlurState();
private smoothedAutoFocus: { cx: number; cy: number } | null = null;
private prevAnimationTimeMs: number | null = null;
private prevTargetProgress = 0;
@@ -561,6 +574,8 @@ export class FrameRenderer {
}
if ((this.config.cursorScale ?? 1) <= 0) {
resetNativeCursorSmoothingState(this.nativeCursorSmoothingState);
resetNativeCursorMotionBlurState(this.nativeCursorMotionBlurState);
return;
}
@@ -569,23 +584,28 @@ export class FrameRenderer {
timeMs,
);
if (!activeNativeCursor) {
resetNativeCursorSmoothingState(this.nativeCursorSmoothingState);
resetNativeCursorMotionBlurState(this.nativeCursorMotionBlurState);
return;
}
const displaySample = smoothNativeCursorSample({
sample: activeNativeCursor.sample,
smoothing: this.config.cursorSmoothing ?? 0,
state: this.nativeCursorSmoothingState,
timeMs,
});
const projectedPoint = projectNativeCursorToLocal({
cropRegion: this.config.cropRegion,
maskRect: this.layoutCache.maskRect,
sample: activeNativeCursor.sample,
sample: displaySample,
});
if (!projectedPoint) {
resetNativeCursorMotionBlurState(this.nativeCursorMotionBlurState);
return;
}
const renderAsset = resolveNativeCursorRenderAsset(
activeNativeCursor.asset,
1,
activeNativeCursor.sample,
);
const renderAsset = resolveNativeCursorRenderAsset(activeNativeCursor.asset, 1, displaySample);
let image: HTMLImageElement;
try {
image = await this.getCursorImage(renderAsset);
@@ -593,10 +613,25 @@ export class FrameRenderer {
this.warnOnce("native-cursor-image-load", "Failed to load native cursor asset", error);
return;
}
const scale = Math.max(0, this.config.cursorScale ?? 1);
const scale =
Math.max(0, this.config.cursorScale ?? 1) *
getNativeCursorClickBounceScale(
this.config.cursorClickBounce ?? 0,
getNativeCursorClickBounceProgress(this.config.cursorRecordingData, timeMs),
);
const appliedScale = this.animationState.appliedScale;
const canvasX = projectedPoint.x * appliedScale + this.animationState.x;
const canvasY = projectedPoint.y * appliedScale + this.animationState.y;
const blurPx = getNativeCursorMotionBlurPx({
motionBlur: this.config.cursorMotionBlur ?? 0,
point: { x: canvasX, y: canvasY },
state: this.nativeCursorMotionBlurState,
timeMs,
});
const previousFilter = this.foregroundCtx.filter;
if (blurPx > 0) {
this.foregroundCtx.filter = `blur(${blurPx.toFixed(2)}px)`;
}
this.foregroundCtx.drawImage(
image,
canvasX - renderAsset.hotspotX * scale * appliedScale,
@@ -604,6 +639,7 @@ export class FrameRenderer {
renderAsset.width * scale * appliedScale,
renderAsset.height * scale * appliedScale,
);
this.foregroundCtx.filter = previousFilter;
}
private async getCursorImage(asset: { id: string; imageDataUrl: string }) {
src/lib/exporter/gifExporter.ts
+7 -1
View File
@@ -9,9 +9,9 @@ import type {
ZoomRegion,
} from "@/components/video-editor/types";
import { BackgroundLoadError } from "@/lib/wallpaper";
import type { CursorRecordingData } from "@/native/contracts";
import { getPlatform } from "@/utils/platformUtils";
import { AsyncVideoFrameQueue } from "./asyncVideoFrameQueue";
import type { CursorRecordingData } from "@/native/contracts";
import { FrameRenderer } from "./frameRenderer";
import { StreamingVideoDecoder } from "./streamingDecoder";
import type {
@@ -50,6 +50,9 @@ interface GifExporterConfig {
webcamPosition?: { cx: number; cy: number } | null;
cursorRecordingData?: CursorRecordingData | null;
cursorScale?: number;
cursorSmoothing?: number;
cursorMotionBlur?: number;
cursorClickBounce?: number;
annotationRegions?: AnnotationRegion[];
previewWidth?: number;
previewHeight?: number;
@@ -156,6 +159,9 @@ export class GifExporter {
cropRegion: this.config.cropRegion,
cursorRecordingData: this.config.cursorRecordingData,
cursorScale: this.config.cursorScale,
cursorSmoothing: this.config.cursorSmoothing,
cursorMotionBlur: this.config.cursorMotionBlur,
cursorClickBounce: this.config.cursorClickBounce,
videoWidth: videoInfo.width,
videoHeight: videoInfo.height,
webcamSize: webcamInfo ? { width: webcamInfo.width, height: webcamInfo.height } : null,
src/lib/exporter/videoExporter.ts
+7 -1
View File
@@ -8,9 +8,9 @@ import type {
ZoomRegion,
} from "@/components/video-editor/types";
import { BackgroundLoadError } from "@/lib/wallpaper";
import type { CursorRecordingData } from "@/native/contracts";
import { getPlatform } from "@/utils/platformUtils";
import { AsyncVideoFrameQueue } from "./asyncVideoFrameQueue";
import type { CursorRecordingData } from "@/native/contracts";
import { AudioProcessor } from "./audioEncoder";
import { FrameRenderer } from "./frameRenderer";
import { VideoMuxer } from "./muxer";
@@ -41,6 +41,9 @@ interface VideoExporterConfig extends ExportConfig {
webcamPosition?: { cx: number; cy: number } | null;
cursorRecordingData?: CursorRecordingData | null;
cursorScale?: number;
cursorSmoothing?: number;
cursorMotionBlur?: number;
cursorClickBounce?: number;
annotationRegions?: AnnotationRegion[];
previewWidth?: number;
previewHeight?: number;
@@ -151,6 +154,9 @@ export class VideoExporter {
cropRegion: this.config.cropRegion,
cursorRecordingData: this.config.cursorRecordingData,
cursorScale: this.config.cursorScale,
cursorSmoothing: this.config.cursorSmoothing,
cursorMotionBlur: this.config.cursorMotionBlur,
cursorClickBounce: this.config.cursorClickBounce,
videoWidth: videoInfo.width,
videoHeight: videoInfo.height,
webcamSize: webcamInfo ? { width: webcamInfo.width, height: webcamInfo.height } : null,
src/native/contracts.ts
+1
View File
@@ -31,6 +31,7 @@ export interface CursorRecordingSample extends CursorTelemetryPoint {
assetId?: string | null;
visible?: boolean;
cursorType?: NativeCursorType | null;
interactionType?: "move" | "click" | "mouseup";
}
export interface NativeCursorAsset {