From b5c105b5d6a4cd960468fd1adf455fbba770b216 Mon Sep 17 00:00:00 2001 From: Sagar Dash <7631656+sagar290@users.noreply.github.com> Date: Mon, 18 May 2026 18:28:13 +0600 Subject: [PATCH 1/4] fix: respect native webcam orientation on Windows --- src/hooks/useScreenRecorder.ts | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 45aa7b3..1b9148e 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -40,8 +40,6 @@ const AUDIO_BITRATE_VOICE = 128_000; const AUDIO_BITRATE_SYSTEM = 192_000; const MIC_GAIN_BOOST = 1.4; -const WEBCAM_TARGET_WIDTH = 1280; -const WEBCAM_TARGET_HEIGHT = 720; const WEBCAM_TARGET_FRAME_RATE = 30; type UseScreenRecorderReturn = { @@ -247,13 +245,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn { video: webcamDeviceId ? { deviceId: { exact: webcamDeviceId }, - width: { ideal: WEBCAM_TARGET_WIDTH }, - height: { ideal: WEBCAM_TARGET_HEIGHT }, frameRate: { ideal: WEBCAM_TARGET_FRAME_RATE, max: WEBCAM_TARGET_FRAME_RATE }, } : { - width: { ideal: WEBCAM_TARGET_WIDTH }, - height: { ideal: WEBCAM_TARGET_HEIGHT }, frameRate: { ideal: WEBCAM_TARGET_FRAME_RATE, max: WEBCAM_TARGET_FRAME_RATE }, }, }); @@ -599,12 +593,12 @@ export function useScreenRecorder(): UseScreenRecorderReturn { if (availability.reason === "unsupported-os") { return false; } + if (availability.reason === "missing-helper") { + console.warn("Native Windows capture helper is not available; using browser capture."); + return false; + } - throw new Error( - availability.reason === "missing-helper" - ? "Native Windows capture helper is not available." - : (availability.error ?? "Native Windows capture is not available."), - ); + throw new Error(availability.error ?? "Native Windows capture is not available."); } if (!isCountdownRunActive(countdownRunToken)) { @@ -646,8 +640,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { enabled: webcamEnabled, deviceId: webcamDeviceId, deviceName: webcamDeviceName, - width: WEBCAM_TARGET_WIDTH, - height: WEBCAM_TARGET_HEIGHT, + width: 0, + height: 0, fps: WEBCAM_TARGET_FRAME_RATE, }, cursor: { From ef5855f1f44a2b77796649295e47c3f63aa269c5 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Fri, 22 May 2026 11:20:35 +0200 Subject: [PATCH 2/4] Fix native Windows webcam sidecar capture Record browser webcam sidecar when native Windows capture is active. Add native webcam sidecar output and DirectShow NV12/YUY2 fallback. Sample exported webcam frames by source timestamp. --- .../wgc-capture/src/dshow_webcam_capture.cpp | 158 ++++++++++++++--- .../wgc-capture/src/dshow_webcam_capture.h | 19 +- electron/native/wgc-capture/src/main.cpp | 80 +++++++-- .../native/wgc-capture/src/mf_encoder.cpp | 82 +++++++++ electron/native/wgc-capture/src/mf_encoder.h | 2 + .../native/wgc-capture/src/webcam_capture.cpp | 12 +- .../native/wgc-capture/src/webcam_capture.h | 3 +- scripts/test-windows-wgc-helper.mjs | 27 ++- src/hooks/useScreenRecorder.ts | 163 ++++++++++++------ src/lib/exporter/gifExporter.ts | 14 +- .../timestampedVideoFrameQueue.test.ts | 50 ++++++ .../exporter/timestampedVideoFrameQueue.ts | 105 +++++++++++ src/lib/exporter/videoExporter.ts | 14 +- 13 files changed, 618 insertions(+), 111 deletions(-) create mode 100644 src/lib/exporter/timestampedVideoFrameQueue.test.ts create mode 100644 src/lib/exporter/timestampedVideoFrameQueue.ts diff --git a/electron/native/wgc-capture/src/dshow_webcam_capture.cpp b/electron/native/wgc-capture/src/dshow_webcam_capture.cpp index 14cb888..436bdea 100644 --- a/electron/native/wgc-capture/src/dshow_webcam_capture.cpp +++ b/electron/native/wgc-capture/src/dshow_webcam_capture.cpp @@ -5,22 +5,18 @@ #include #include +#include #include #include +#include #include +#include namespace { const CLSID CLSID_SampleGrabberLocal = {0xC1F400A0, 0x3F08, 0x11D3, {0x9F, 0x0B, 0x00, 0x60, 0x08, 0x03, 0x9E, 0x37}}; const CLSID CLSID_NullRendererLocal = {0xC1F400A4, 0x3F08, 0x11D3, {0x9F, 0x0B, 0x00, 0x60, 0x08, 0x03, 0x9E, 0x37}}; -MIDL_INTERFACE("0579154A-2B53-4994-B0D0-E773148EFF85") -ISampleGrabberCB : public IUnknown { -public: - virtual HRESULT STDMETHODCALLTYPE SampleCB(double sampleTime, IMediaSample* sample) = 0; - virtual HRESULT STDMETHODCALLTYPE BufferCB(double sampleTime, BYTE* buffer, long bufferLength) = 0; -}; - MIDL_INTERFACE("6B652FFF-11FE-4FCE-92AD-0266B5D7C78F") ISampleGrabber : public IUnknown { public: @@ -30,7 +26,7 @@ public: virtual HRESULT STDMETHODCALLTYPE SetBufferSamples(BOOL bufferThem) = 0; virtual HRESULT STDMETHODCALLTYPE GetCurrentBuffer(long* bufferSize, long* buffer) = 0; virtual HRESULT STDMETHODCALLTYPE GetCurrentSample(IMediaSample** sample) = 0; - virtual HRESULT STDMETHODCALLTYPE SetCallback(ISampleGrabberCB* callback, long whichMethodToCallback) = 0; + virtual HRESULT STDMETHODCALLTYPE SetCallback(IUnknown* callback, long whichMethodToCallback) = 0; }; bool succeeded(HRESULT hr, const char* label) { @@ -43,6 +39,34 @@ bool succeeded(HRESULT hr, const char* label) { return false; } +std::string guidToString(const GUID& guid) { + if (guid == MEDIASUBTYPE_RGB32) { + return "RGB32"; + } + if (guid == MEDIASUBTYPE_YUY2) { + return "YUY2"; + } + if (guid == MEDIASUBTYPE_NV12) { + return "NV12"; + } + + std::ostringstream stream; + stream << std::hex << std::setfill('0') + << '{' << std::setw(8) << guid.Data1 + << '-' << std::setw(4) << guid.Data2 + << '-' << std::setw(4) << guid.Data3 + << '-'; + for (int index = 0; index < 2; index += 1) { + stream << std::setw(2) << static_cast(guid.Data4[index]); + } + stream << '-'; + for (int index = 2; index < 8; index += 1) { + stream << std::setw(2) << static_cast(guid.Data4[index]); + } + stream << '}'; + return stream.str(); +} + void freeMediaType(AM_MEDIA_TYPE& type) { if (type.cbFormat != 0) { CoTaskMemFree(type.pbFormat); @@ -55,6 +79,20 @@ void freeMediaType(AM_MEDIA_TYPE& type) { } } +BYTE clampToByte(int value) { + return static_cast(std::clamp(value, 0, 255)); +} + +std::array yuvToBgr(int y, int u, int v) { + const int c = y - 16; + const int d = u - 128; + const int e = v - 128; + const int blue = (298 * c + 516 * d + 128) >> 8; + const int green = (298 * c - 100 * d - 208 * e + 128) >> 8; + const int red = (298 * c + 409 * e + 128) >> 8; + return {clampToByte(blue), clampToByte(green), clampToByte(red)}; +} + } // namespace struct DirectShowWebcamCapture::Impl { @@ -137,9 +175,8 @@ bool DirectShowWebcamCapture::initialize( AM_MEDIA_TYPE requestedType{}; requestedType.majortype = MEDIATYPE_Video; - requestedType.subtype = MEDIASUBTYPE_RGB32; requestedType.formattype = FORMAT_VideoInfo; - if (!succeeded(impl_->sampleGrabber->SetMediaType(&requestedType), "SetMediaType(DirectShow RGB32)")) { + if (!succeeded(impl_->sampleGrabber->SetMediaType(&requestedType), "SetMediaType(DirectShow video)")) { return false; } @@ -170,17 +207,40 @@ bool DirectShowWebcamCapture::initialize( if (!succeeded(impl_->sampleGrabber->GetConnectedMediaType(&connectedType), "GetConnectedMediaType(DirectShow webcam)")) { return false; } + if (connectedType.subtype == MEDIASUBTYPE_YUY2) { + pixelFormat_ = PixelFormat::Yuy2; + } else if (connectedType.subtype == MEDIASUBTYPE_NV12) { + pixelFormat_ = PixelFormat::Nv12; + } else if (connectedType.subtype == MEDIASUBTYPE_RGB32) { + pixelFormat_ = PixelFormat::Bgra; + } else { + std::cerr << "ERROR: Unsupported DirectShow webcam media subtype " + << guidToString(connectedType.subtype) << std::endl; + freeMediaType(connectedType); + return false; + } if (connectedType.formattype == FORMAT_VideoInfo && connectedType.pbFormat) { const auto* videoInfo = reinterpret_cast(connectedType.pbFormat); width_ = std::abs(videoInfo->bmiHeader.biWidth); height_ = std::abs(videoInfo->bmiHeader.biHeight); - sourceTopDown_ = videoInfo->bmiHeader.biHeight < 0; + const int bitsPerPixel = videoInfo->bmiHeader.biBitCount > 0 ? videoInfo->bmiHeader.biBitCount : 16; + if (pixelFormat_ == PixelFormat::Nv12) { + sourceStride_ = ((width_ + 3) / 4) * 4; + } else { + sourceStride_ = ((width_ * bitsPerPixel + 31) / 32) * 4; + } + sourceTopDown_ = pixelFormat_ != PixelFormat::Bgra || videoInfo->bmiHeader.biHeight < 0; } + std::cerr << "INFO: DirectShow webcam connected subtype " << guidToString(connectedType.subtype) + << " " << width_ << "x" << height_ << " stride=" << sourceStride_ << std::endl; freeMediaType(connectedType); if (width_ <= 0 || height_ <= 0) { width_ = requestedWidth > 0 ? requestedWidth : 1280; height_ = requestedHeight > 0 ? requestedHeight : 720; } + if (sourceStride_ <= 0) { + sourceStride_ = pixelFormat_ == PixelFormat::Bgra ? width_ * 4 : ((width_ + 3) / 4) * 4; + } impl_->sampleGrabber->SetBufferSamples(TRUE); impl_->sampleGrabber->SetOneShot(FALSE); @@ -262,36 +322,88 @@ void DirectShowWebcamCapture::captureLoop() { } void DirectShowWebcamCapture::storeFrame(const BYTE* buffer, long length) { - const int stride = width_ * 4; - const int expectedLength = stride * height_; + const int destinationStride = width_ * 4; + const int sourceStride = sourceStride_ > 0 ? sourceStride_ : destinationStride; + const int expectedLength = pixelFormat_ == PixelFormat::Nv12 + ? sourceStride * height_ + sourceStride * ((height_ + 1) / 2) + : sourceStride * height_; if (!buffer || length < expectedLength || width_ <= 0 || height_ <= 0) { return; } - std::vector frame(static_cast(expectedLength)); + std::vector frame(static_cast(destinationStride * height_)); for (int y = 0; y < height_; y += 1) { const int sourceY = sourceTopDown_ ? y : height_ - 1 - y; - const BYTE* source = buffer + sourceY * stride; - BYTE* destination = frame.data() + y * stride; - std::copy(source, source + stride, destination); - for (int x = 0; x < width_; x += 1) { - destination[x * 4 + 3] = 255; + const BYTE* source = buffer + sourceY * sourceStride; + BYTE* destination = frame.data() + y * destinationStride; + if (pixelFormat_ == PixelFormat::Bgra) { + std::copy(source, source + destinationStride, destination); + for (int x = 0; x < width_; x += 1) { + destination[x * 4 + 3] = 255; + } + continue; + } + + if (pixelFormat_ == PixelFormat::Nv12) { + const BYTE* yPlane = buffer + sourceY * sourceStride; + const BYTE* uvPlane = buffer + sourceStride * height_ + (sourceY / 2) * sourceStride; + for (int x = 0; x < width_; x += 1) { + const int uvX = (x / 2) * 2; + const auto color = yuvToBgr(yPlane[x], uvPlane[uvX], uvPlane[uvX + 1]); + BYTE* pixel = destination + x * 4; + pixel[0] = color[0]; + pixel[1] = color[1]; + pixel[2] = color[2]; + pixel[3] = 255; + } + continue; + } + + for (int x = 0; x + 1 < width_; x += 2) { + const BYTE y0 = source[x * 2]; + const BYTE u = source[x * 2 + 1]; + const BYTE y1 = source[x * 2 + 2]; + const BYTE v = source[x * 2 + 3]; + const auto first = yuvToBgr(y0, u, v); + const auto second = yuvToBgr(y1, u, v); + BYTE* firstPixel = destination + x * 4; + BYTE* secondPixel = firstPixel + 4; + firstPixel[0] = first[0]; + firstPixel[1] = first[1]; + firstPixel[2] = first[2]; + firstPixel[3] = 255; + secondPixel[0] = second[0]; + secondPixel[1] = second[1]; + secondPixel[2] = second[2]; + secondPixel[3] = 255; + } + if (width_ % 2 == 1) { + const int x = width_ - 1; + const BYTE* pair = source + (x - 1) * 2; + const auto color = yuvToBgr(pair[2], pair[1], pair[3]); + BYTE* pixel = destination + x * 4; + pixel[0] = color[0]; + pixel[1] = color[1]; + pixel[2] = color[2]; + pixel[3] = 255; } } std::scoped_lock lock(frameMutex_); latestFrame_ = std::move(frame); + latestFrameSequence_ += 1; } -bool DirectShowWebcamCapture::copyLatestFrame(std::vector& destination, int& width, int& height) { +bool DirectShowWebcamCapture::copyLatestFrame(WebcamFrameSnapshot& destination) { std::scoped_lock lock(frameMutex_); if (latestFrame_.empty() || width_ <= 0 || height_ <= 0) { return false; } - destination = latestFrame_; - width = width_; - height = height_; + destination.data = latestFrame_; + destination.width = width_; + destination.height = height_; + destination.sequence = latestFrameSequence_; return true; } diff --git a/electron/native/wgc-capture/src/dshow_webcam_capture.h b/electron/native/wgc-capture/src/dshow_webcam_capture.h index 906da8f..3debcbe 100644 --- a/electron/native/wgc-capture/src/dshow_webcam_capture.h +++ b/electron/native/wgc-capture/src/dshow_webcam_capture.h @@ -3,11 +3,19 @@ #include #include +#include #include #include #include #include +struct WebcamFrameSnapshot { + std::vector data; + int width = 0; + int height = 0; + uint64_t sequence = 0; +}; + class DirectShowWebcamCapture { public: DirectShowWebcamCapture() = default; @@ -25,7 +33,7 @@ public: int requestedFps); bool start(); void stop(); - bool copyLatestFrame(std::vector& destination, int& width, int& height); + bool copyLatestFrame(WebcamFrameSnapshot& destination); int width() const; int height() const; @@ -34,6 +42,12 @@ public: void storeFrame(const BYTE* buffer, long length); private: + enum class PixelFormat { + Bgra, + Nv12, + Yuy2, + }; + struct Impl; void captureLoop(); @@ -42,9 +56,12 @@ private: std::atomic stopRequested_ = false; std::mutex frameMutex_; std::vector latestFrame_; + uint64_t latestFrameSequence_ = 0; int width_ = 0; int height_ = 0; int fps_ = 30; + int sourceStride_ = 0; bool sourceTopDown_ = false; + PixelFormat pixelFormat_ = PixelFormat::Bgra; std::wstring selectedDeviceName_; }; diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp index f8f56cd..7968d94 100644 --- a/electron/native/wgc-capture/src/main.cpp +++ b/electron/native/wgc-capture/src/main.cpp @@ -29,6 +29,7 @@ struct CaptureConfig { std::string sourceId; std::string windowHandle; std::string outputPath; + std::string webcamOutputPath; int fps = 60; int width = 0; int height = 0; @@ -311,6 +312,7 @@ bool parseConfig(const std::string& json, CaptureConfig& config) { config.webcamDeviceId = findString(json, "webcamDeviceId"); config.webcamDeviceName = findString(json, "webcamDeviceName"); config.webcamDirectShowClsid = findString(json, "webcamDirectShowClsid"); + config.webcamOutputPath = findString(json, "webcamPath"); config.webcamWidth = findInt(json, "webcamWidth", 0); config.webcamHeight = findInt(json, "webcamHeight", 0); config.webcamFps = findInt(json, "webcamFps", 0); @@ -389,6 +391,7 @@ int main(int argc, char* argv[]) { WebcamCapture webcamCapture; bool webcamActive = false; + bool writeSeparateWebcam = false; if (config.webcamEnabled) { if (!webcamCapture.initialize( utf8ToWide(config.webcamDeviceId), @@ -405,6 +408,7 @@ int main(int argc, char* argv[]) { << ",\"fps\":" << webcamCapture.fps() << ",\"deviceName\":\"" << jsonEscape(wideToUtf8(webcamCapture.selectedDeviceName())) << "\"}" << std::endl; + writeSeparateWebcam = !config.webcamOutputPath.empty(); } WasapiLoopbackCapture loopbackCapture; @@ -466,6 +470,24 @@ int main(int argc, char* argv[]) { return 1; } + MFEncoder webcamEncoder; + if (writeSeparateWebcam) { + const int webcamPixels = std::max(1, webcamCapture.width()) * std::max(1, webcamCapture.height()); + const int webcamBitrate = webcamPixels >= 1280 * 720 ? 8'000'000 : 4'000'000; + if (!webcamEncoder.initialize( + utf8ToWide(config.webcamOutputPath), + webcamCapture.width(), + webcamCapture.height(), + webcamCapture.fps(), + webcamBitrate, + session.device(), + session.context(), + nullptr)) { + std::cerr << "ERROR: Failed to initialize native webcam encoder" << std::endl; + return 1; + } + } + std::mutex mutex; std::condition_variable cv; std::atomic stopRequested = false; @@ -477,6 +499,7 @@ int main(int argc, char* argv[]) { std::vector latestWebcamFrame; int latestWebcamWidth = 0; int latestWebcamHeight = 0; + uint64_t latestWebcamSequence = 0; bool hasVisibleWebcamFrame = false; session.setFrameCallback([&](ID3D11Texture2D* texture, int64_t timestampHns) { @@ -509,20 +532,22 @@ int main(int argc, char* argv[]) { auto writeVideoFrames = [&]() { const auto startedAt = std::chrono::steady_clock::now(); uint64_t frameIndex = 0; + uint64_t lastWrittenWebcamSequence = 0; + uint64_t webcamOutputFrameIndex = 0; int64_t lastEncodedVideoTimestampHns = -1; while (!stopRequested && !encodeFailed) { { std::scoped_lock lock(mutex); if (webcamActive) { - std::vector candidateWebcamFrame; - int candidateWebcamWidth = 0; - int candidateWebcamHeight = 0; - if (webcamCapture.copyLatestFrame(candidateWebcamFrame, candidateWebcamWidth, candidateWebcamHeight) && - hasVisibleBgraContent(candidateWebcamFrame)) { - latestWebcamFrame = std::move(candidateWebcamFrame); - latestWebcamWidth = candidateWebcamWidth; - latestWebcamHeight = candidateWebcamHeight; + WebcamFrameSnapshot candidateWebcamFrame; + if (webcamCapture.copyLatestFrame(candidateWebcamFrame) && + candidateWebcamFrame.sequence != latestWebcamSequence && + hasVisibleBgraContent(candidateWebcamFrame.data)) { + latestWebcamFrame = std::move(candidateWebcamFrame.data); + latestWebcamWidth = candidateWebcamFrame.width; + latestWebcamHeight = candidateWebcamFrame.height; + latestWebcamSequence = candidateWebcamFrame.sequence; hasVisibleWebcamFrame = true; } } @@ -545,10 +570,23 @@ int main(int argc, char* argv[]) { frameTimestampHns = lastEncodedVideoTimestampHns + static_cast(10'000'000ULL / config.fps); } + if (writeSeparateWebcam && webcamFrame.data && + latestWebcamSequence != lastWrittenWebcamSequence) { + const int64_t webcamTimestampHns = static_cast( + (webcamOutputFrameIndex * 10'000'000ULL) / std::max(1, webcamCapture.fps())); + if (!webcamEncoder.writeBgraFrame(webcamFrame, webcamTimestampHns)) { + encodeFailed = true; + stopRequested = true; + cv.notify_all(); + return; + } + lastWrittenWebcamSequence = latestWebcamSequence; + webcamOutputFrameIndex += 1; + } if (latestFrameTexture && !encoder.writeFrame( latestFrameTexture.Get(), frameTimestampHns, - webcamFrame.data ? &webcamFrame : nullptr)) { + !writeSeparateWebcam && webcamFrame.data ? &webcamFrame : nullptr)) { encodeFailed = true; stopRequested = true; cv.notify_all(); @@ -659,14 +697,13 @@ int main(int argc, char* argv[]) { webcamActive = true; const auto webcamDeadline = std::chrono::steady_clock::now() + std::chrono::seconds(3); while (std::chrono::steady_clock::now() < webcamDeadline && !hasVisibleWebcamFrame) { - std::vector candidateWebcamFrame; - int candidateWebcamWidth = 0; - int candidateWebcamHeight = 0; - if (webcamCapture.copyLatestFrame(candidateWebcamFrame, candidateWebcamWidth, candidateWebcamHeight) && - hasVisibleBgraContent(candidateWebcamFrame)) { - latestWebcamFrame = std::move(candidateWebcamFrame); - latestWebcamWidth = candidateWebcamWidth; - latestWebcamHeight = candidateWebcamHeight; + WebcamFrameSnapshot candidateWebcamFrame; + if (webcamCapture.copyLatestFrame(candidateWebcamFrame) && + hasVisibleBgraContent(candidateWebcamFrame.data)) { + latestWebcamFrame = std::move(candidateWebcamFrame.data); + latestWebcamWidth = candidateWebcamFrame.width; + latestWebcamHeight = candidateWebcamFrame.height; + latestWebcamSequence = candidateWebcamFrame.sequence; hasVisibleWebcamFrame = true; break; } @@ -740,6 +777,9 @@ int main(int argc, char* argv[]) { { std::scoped_lock lock(mutex); encoder.finalize(); + if (writeSeparateWebcam) { + webcamEncoder.finalize(); + } } if (stdinThread.joinable()) { @@ -752,7 +792,11 @@ int main(int argc, char* argv[]) { } std::cout << "{\"event\":\"recording-stopped\",\"schemaVersion\":2,\"screenPath\":\"" - << jsonEscape(config.outputPath) << "\"}" << std::endl; + << jsonEscape(config.outputPath) << "\""; + if (writeSeparateWebcam) { + std::cout << ",\"webcamPath\":\"" << jsonEscape(config.webcamOutputPath) << "\""; + } + std::cout << "}" << std::endl; std::cout << "Recording stopped. Output path: " << config.outputPath << std::endl; return 0; } diff --git a/electron/native/wgc-capture/src/mf_encoder.cpp b/electron/native/wgc-capture/src/mf_encoder.cpp index b56386e..c04353a 100644 --- a/electron/native/wgc-capture/src/mf_encoder.cpp +++ b/electron/native/wgc-capture/src/mf_encoder.cpp @@ -254,6 +254,40 @@ bool MFEncoder::copyFrameToBuffer( return true; } +bool MFEncoder::copyBgraFrameToBuffer(const BgraFrameView& frame, BYTE* destination, DWORD destinationSize) { + if (!frame.data || frame.width <= 0 || frame.height <= 0) { + return false; + } + + const DWORD rowBytes = static_cast(width_ * 4); + const DWORD requiredBytes = rowBytes * static_cast(height_); + if (destinationSize < requiredBytes) { + std::cerr << "ERROR: Media Foundation webcam buffer is too small" << std::endl; + return false; + } + + if (frame.width == width_ && frame.height == height_) { + std::memcpy(destination, frame.data, requiredBytes); + return true; + } + + for (int y = 0; y < height_; y += 1) { + const int sourceY = static_cast((static_cast(y) * frame.height) / height_); + BYTE* destinationRow = destination + rowBytes * y; + for (int x = 0; x < width_; x += 1) { + const int sourceX = static_cast((static_cast(x) * frame.width) / width_); + const BYTE* source = frame.data + (sourceY * frame.width + sourceX) * 4; + BYTE* target = destinationRow + x * 4; + target[0] = source[0]; + target[1] = source[1]; + target[2] = source[2]; + target[3] = 255; + } + } + + return true; +} + bool MFEncoder::writeFrame(ID3D11Texture2D* texture, int64_t timestampHns, const BgraFrameView* webcamFrame) { std::scoped_lock writerLock(writerMutex_); if (!sinkWriter_ || finalized_) { @@ -302,6 +336,54 @@ bool MFEncoder::writeFrame(ID3D11Texture2D* texture, int64_t timestampHns, const return succeeded(sinkWriter_->WriteSample(videoStreamIndex_, sample.Get()), "WriteSample"); } +bool MFEncoder::writeBgraFrame(const BgraFrameView& frame, int64_t timestampHns) { + std::scoped_lock writerLock(writerMutex_); + if (!sinkWriter_ || finalized_) { + return false; + } + + if (firstTimestampHns_ < 0) { + firstTimestampHns_ = timestampHns; + } + + int64_t sampleTime = timestampHns - firstTimestampHns_; + if (sampleTime <= lastTimestampHns_) { + sampleTime = lastTimestampHns_ + (10'000'000LL / fps_); + } + const int64_t sampleDuration = 10'000'000LL / fps_; + lastTimestampHns_ = sampleTime; + + Microsoft::WRL::ComPtr buffer; + const DWORD frameBytes = static_cast(width_ * height_ * 4); + if (!succeeded(MFCreateMemoryBuffer(frameBytes, &buffer), "MFCreateMemoryBuffer(webcam)")) { + return false; + } + + BYTE* data = nullptr; + DWORD maxLength = 0; + DWORD currentLength = 0; + if (!succeeded(buffer->Lock(&data, &maxLength, ¤tLength), "IMFMediaBuffer::Lock(webcam)")) { + return false; + } + + const bool copied = copyBgraFrameToBuffer(frame, data, maxLength); + buffer->Unlock(); + if (!copied) { + return false; + } + buffer->SetCurrentLength(frameBytes); + + Microsoft::WRL::ComPtr sample; + if (!succeeded(MFCreateSample(&sample), "MFCreateSample(webcam)")) { + return false; + } + sample->AddBuffer(buffer.Get()); + sample->SetSampleTime(sampleTime); + sample->SetSampleDuration(sampleDuration); + + return succeeded(sinkWriter_->WriteSample(videoStreamIndex_, sample.Get()), "WriteSample(webcam)"); +} + bool MFEncoder::writeAudio(const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns) { std::scoped_lock writerLock(writerMutex_); if (!sinkWriter_ || finalized_ || !hasAudioStream_) { diff --git a/electron/native/wgc-capture/src/mf_encoder.h b/electron/native/wgc-capture/src/mf_encoder.h index a82a940..e7821e9 100644 --- a/electron/native/wgc-capture/src/mf_encoder.h +++ b/electron/native/wgc-capture/src/mf_encoder.h @@ -44,6 +44,7 @@ public: ID3D11DeviceContext* context, const AudioInputFormat* audioFormat = nullptr); bool writeFrame(ID3D11Texture2D* texture, int64_t timestampHns, const BgraFrameView* webcamFrame = nullptr); + bool writeBgraFrame(const BgraFrameView& frame, int64_t timestampHns); bool writeAudio(const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns); bool finalize(); @@ -54,6 +55,7 @@ private: BYTE* destination, DWORD destinationSize, const BgraFrameView* webcamFrame); + bool copyBgraFrameToBuffer(const BgraFrameView& frame, BYTE* destination, DWORD destinationSize); bool configureAudioStream(const AudioInputFormat& audioFormat); Microsoft::WRL::ComPtr sinkWriter_; diff --git a/electron/native/wgc-capture/src/webcam_capture.cpp b/electron/native/wgc-capture/src/webcam_capture.cpp index aff9fdb..783b854 100644 --- a/electron/native/wgc-capture/src/webcam_capture.cpp +++ b/electron/native/wgc-capture/src/webcam_capture.cpp @@ -365,6 +365,7 @@ void WebcamCapture::captureLoop() { if (currentLength >= expectedLength && expectedLength > 0) { std::scoped_lock lock(frameMutex_); latestFrame_.assign(data, data + expectedLength); + latestFrameSequence_ += 1; } buffer->Unlock(); @@ -373,18 +374,19 @@ void WebcamCapture::captureLoop() { CoUninitialize(); } -bool WebcamCapture::copyLatestFrame(std::vector& destination, int& width, int& height) { +bool WebcamCapture::copyLatestFrame(WebcamFrameSnapshot& destination) { if (usingDirectShow_) { - return directShowCapture_.copyLatestFrame(destination, width, height); + return directShowCapture_.copyLatestFrame(destination); } std::scoped_lock lock(frameMutex_); if (latestFrame_.empty() || width_ <= 0 || height_ <= 0) { return false; } - destination = latestFrame_; - width = width_; - height = height_; + destination.data = latestFrame_; + destination.width = width_; + destination.height = height_; + destination.sequence = latestFrameSequence_; return true; } diff --git a/electron/native/wgc-capture/src/webcam_capture.h b/electron/native/wgc-capture/src/webcam_capture.h index c539d02..5b61aa6 100644 --- a/electron/native/wgc-capture/src/webcam_capture.h +++ b/electron/native/wgc-capture/src/webcam_capture.h @@ -31,7 +31,7 @@ public: int requestedFps); bool start(); void stop(); - bool copyLatestFrame(std::vector& destination, int& width, int& height); + bool copyLatestFrame(WebcamFrameSnapshot& destination); int width() const; int height() const; @@ -50,6 +50,7 @@ private: std::atomic stopRequested_ = false; std::mutex frameMutex_; std::vector latestFrame_; + uint64_t latestFrameSequence_ = 0; int width_ = 0; int height_ = 0; int fps_ = 30; diff --git a/scripts/test-windows-wgc-helper.mjs b/scripts/test-windows-wgc-helper.mjs index 53cea19..5dd2dcc 100644 --- a/scripts/test-windows-wgc-helper.mjs +++ b/scripts/test-windows-wgc-helper.mjs @@ -230,6 +230,7 @@ const outputPath = path.join( os.tmpdir(), `openscreen-wgc-helper-${WITH_WEBCAM ? "webcam" : WITH_WINDOW ? "window" : WITH_SYSTEM_AUDIO || WITH_MICROPHONE ? "audio" : "video"}-${process.pid}-${Date.now()}-${randomUUID()}.mp4`, ); +const webcamOutputPath = WITH_WEBCAM ? outputPath.replace(/\.mp4$/i, "-webcam.mp4") : null; const fixtureWindow = WITH_WINDOW ? await startFixtureWindow() : null; @@ -263,7 +264,10 @@ const config = { webcamWidth: 640, webcamHeight: 360, webcamFps: 30, - outputs: { screenPath: outputPath }, + outputs: { + screenPath: outputPath, + ...(webcamOutputPath ? { webcamPath: webcamOutputPath } : {}), + }, }; let result; @@ -289,8 +293,13 @@ if (result.code !== 0) { if (!fs.existsSync(outputPath) || fs.statSync(outputPath).size === 0) { throw new Error(`WGC helper did not produce a video at ${outputPath}`); } +if (WITH_WEBCAM && (!fs.existsSync(webcamOutputPath) || fs.statSync(webcamOutputPath).size === 0)) { + throw new Error(`WGC helper did not produce a webcam video at ${webcamOutputPath}`); +} const streams = probeStreams(outputPath); +const webcamStreams = + webcamOutputPath && fs.existsSync(webcamOutputPath) ? probeStreams(webcamOutputPath) : []; const hasVideo = streams.some((stream) => stream.codec_type === "video"); const hasAudio = streams.some((stream) => stream.codec_type === "audio"); const webcamFormatLine = result.stdout @@ -318,6 +327,9 @@ const nativeMicrophoneDiagnostics = result.stderr if (!hasVideo) { throw new Error(`WGC helper output has no video stream: ${outputPath}`); } +if (WITH_WEBCAM && !webcamStreams.some((stream) => stream.codec_type === "video")) { + throw new Error(`WGC helper webcam output has no video stream: ${webcamOutputPath}`); +} if ( (CAPTURE_CURSOR && !cursorCapture) || (cursorCapture && @@ -342,13 +354,26 @@ console.log( { success: true, outputPath, + webcamOutputPath, bytes: fs.statSync(outputPath).size, + webcamBytes: + webcamOutputPath && fs.existsSync(webcamOutputPath) + ? fs.statSync(webcamOutputPath).size + : undefined, streams: streams.map((stream) => ({ index: stream.index, codecType: stream.codec_type, codecName: stream.codec_name, duration: stream.duration, })), + webcamStreams: webcamStreams.map((stream) => ({ + index: stream.index, + codecType: stream.codec_type, + codecName: stream.codec_name, + width: stream.width, + height: stream.height, + duration: stream.duration, + })), cursorCapture, selectedMicrophoneDeviceName: audioFormat?.microphoneDeviceName, selectedWebcamDeviceName: webcamFormat?.deviceName, diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 9941dc4..f64eec4 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -84,6 +84,7 @@ type RecorderHandle = { type NativeWindowsRecordingHandle = { recordingId: number; finalizing: boolean; + webcamRecorder: RecorderHandle | null; }; type NativeMacRecordingHandle = { @@ -422,58 +423,105 @@ export function useScreenRecorder(): UseScreenRecorderReturn { [cursorCaptureMode, teardownMedia], ); - const finalizeNativeWindowsRecording = useCallback(async (discard = false) => { - const activeNativeRecording = nativeWindowsRecording.current; - if (!activeNativeRecording || activeNativeRecording.finalizing) { - return false; - } - - activeNativeRecording.finalizing = true; - - const clearNativeRecordingState = () => { - nativeWindowsRecording.current = null; - setRecording(false); - setPaused(false); - setElapsedSeconds(0); - accumulatedDurationMs.current = 0; - segmentStartedAt.current = null; - }; - - try { - const result = await window.electronAPI.stopNativeWindowsRecording(discard); - if (discard || result.discarded) { - clearNativeRecordingState(); - return true; + const finalizeNativeWindowsRecording = useCallback( + async (discard = false) => { + const activeNativeRecording = nativeWindowsRecording.current; + if (!activeNativeRecording || activeNativeRecording.finalizing) { + return false; } - if (!result.success) { - console.error("Failed to stop native Windows recording:", result.error); - toast.error(result.error ?? "Failed to stop native Windows recording"); + + activeNativeRecording.finalizing = true; + const activeWebcamRecorder = activeNativeRecording.webcamRecorder; + const duration = Math.max(0, getRecordingDurationMs()); + if ( + activeWebcamRecorder?.recorder.state === "recording" || + activeWebcamRecorder?.recorder.state === "paused" + ) { + try { + activeWebcamRecorder.recorder.stop(); + } catch { + // Recorder may already be stopping. + } + } + if (activeWebcamRecorder && webcamRecorder.current === activeWebcamRecorder) { + webcamRecorder.current = null; + } + + const clearNativeRecordingState = () => { + nativeWindowsRecording.current = null; + setRecording(false); + setPaused(false); + setElapsedSeconds(0); + accumulatedDurationMs.current = 0; + segmentStartedAt.current = null; + }; + + try { + const result = await window.electronAPI.stopNativeWindowsRecording(discard); + if (discard || result.discarded) { + clearNativeRecordingState(); + return true; + } + if (!result.success) { + console.error("Failed to stop native Windows recording:", result.error); + toast.error(result.error ?? "Failed to stop native Windows recording"); + activeNativeRecording.finalizing = false; + return true; + } + + const nativeScreenPath = result.session?.screenVideoPath ?? result.path; + let storedSession = result.session; + if (activeWebcamRecorder && nativeScreenPath) { + const webcamBlob = await activeWebcamRecorder.recordedBlobPromise.catch(() => null); + const screenRead = await window.electronAPI.readBinaryFile(nativeScreenPath); + if (webcamBlob && webcamBlob.size > 0 && screenRead.success && screenRead.data) { + const fixedWebcamBlob = await fixWebmDuration(webcamBlob, duration); + const nativeScreenFileName = + nativeScreenPath.split(/[\\/]/).pop() ?? + `${RECORDING_FILE_PREFIX}${activeNativeRecording.recordingId}.mp4`; + const webcamFileName = `${RECORDING_FILE_PREFIX}${activeNativeRecording.recordingId}${WEBCAM_FILE_SUFFIX}${VIDEO_FILE_EXTENSION}`; + const stored = await window.electronAPI.storeRecordedSession({ + screen: { + videoData: screenRead.data, + fileName: nativeScreenFileName, + }, + webcam: { + videoData: await fixedWebcamBlob.arrayBuffer(), + fileName: webcamFileName, + }, + createdAt: activeNativeRecording.recordingId, + cursorCaptureMode, + }); + if (stored.success && stored.session) { + storedSession = stored.session; + } + } + } + + clearNativeRecordingState(); + if (storedSession) { + await window.electronAPI.setCurrentRecordingSession(storedSession); + } else if (result.path) { + await window.electronAPI.setCurrentVideoPath(result.path); + } + + await window.electronAPI.switchToEditor(); + return true; + } catch (error) { + console.error("Error saving native Windows recording:", error); + toast.error( + error instanceof Error ? error.message : "Failed to save native Windows recording", + ); activeNativeRecording.finalizing = false; return true; + } finally { + if (discardRecordingId.current === activeNativeRecording.recordingId) { + discardRecordingId.current = null; + } } - - clearNativeRecordingState(); - if (result.session) { - await window.electronAPI.setCurrentRecordingSession(result.session); - } else if (result.path) { - await window.electronAPI.setCurrentVideoPath(result.path); - } - - await window.electronAPI.switchToEditor(); - return true; - } catch (error) { - console.error("Error saving native Windows recording:", error); - toast.error( - error instanceof Error ? error.message : "Failed to save native Windows recording", - ); - activeNativeRecording.finalizing = false; - return true; - } finally { - if (discardRecordingId.current === activeNativeRecording.recordingId) { - discardRecordingId.current = null; - } - } - }, []); + }, + [cursorCaptureMode, getRecordingDurationMs], + ); const finalizeNativeMacRecording = useCallback( async (discard = false) => { @@ -747,7 +795,14 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const displayId = Number(selectedSource.display_id); const sourceType = selectedSource.id.startsWith("window:") ? "window" : "display"; const windowHandle = parseWindowHandleFromSourceId(selectedSource.id); - if (webcamEnabled) { + const browserWebcamRecorder = + webcamEnabled && webcamStream.current + ? createRecorderHandle(webcamStream.current, { + mimeType: selectMimeType(), + videoBitsPerSecond: BITRATE_BASE, + }) + : null; + if (webcamEnabled && !browserWebcamRecorder) { stopWebcamPreviewStream(); } const request: NativeWindowsRecordingRequest = { @@ -775,7 +830,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }, }, webcam: { - enabled: webcamEnabled, + enabled: webcamEnabled && !browserWebcamRecorder, deviceId: webcamDeviceId, deviceName: webcamDeviceName, width: WEBCAM_TARGET_WIDTH, @@ -788,6 +843,12 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }; const result = await window.electronAPI.startNativeWindowsRecording(request); if (!result.success || !result.recordingId) { + if ( + browserWebcamRecorder?.recorder.state === "recording" || + browserWebcamRecorder?.recorder.state === "paused" + ) { + browserWebcamRecorder.recorder.stop(); + } throw new Error(result.error ?? "Native Windows capture failed."); } @@ -795,7 +856,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn { nativeWindowsRecording.current = { recordingId: result.recordingId, finalizing: false, + webcamRecorder: browserWebcamRecorder, }; + webcamRecorder.current = browserWebcamRecorder; accumulatedDurationMs.current = 0; segmentStartedAt.current = Date.now(); allowAutoFinalize.current = true; diff --git a/src/lib/exporter/gifExporter.ts b/src/lib/exporter/gifExporter.ts index 0d2402f..7c0d2a6 100644 --- a/src/lib/exporter/gifExporter.ts +++ b/src/lib/exporter/gifExporter.ts @@ -11,9 +11,9 @@ import type { import { BackgroundLoadError } from "@/lib/wallpaper"; import type { CursorRecordingData } from "@/native/contracts"; import { getPlatform } from "@/utils/platformUtils"; -import { AsyncVideoFrameQueue } from "./asyncVideoFrameQueue"; import { FrameRenderer } from "./frameRenderer"; import { StreamingVideoDecoder } from "./streamingDecoder"; +import { TimestampedVideoFrameQueue } from "./timestampedVideoFrameQueue"; import type { ExportProgress, ExportResult, @@ -124,7 +124,7 @@ export class GifExporter { } async export(): Promise { - let webcamFrameQueue: AsyncVideoFrameQueue | null = null; + let webcamFrameQueue: TimestampedVideoFrameQueue | null = null; const warnings: string[] = []; const onWarning = (message: string) => warnings.push(message); @@ -216,7 +216,7 @@ export class GifExporter { console.log("[GifExporter] Using streaming decode (web-demuxer + VideoDecoder)"); let frameIndex = 0; - webcamFrameQueue = this.config.webcamVideoUrl ? new AsyncVideoFrameQueue() : null; + webcamFrameQueue = this.config.webcamVideoUrl ? new TimestampedVideoFrameQueue() : null; let stopWebcamDecode = false; let webcamDecodeError: Error | null = null; const webcamDecodePromise = @@ -228,7 +228,7 @@ export class GifExporter { this.config.frameRate, this.config.trimRegions, this.config.speedRegions, - async (webcamFrame) => { + async (webcamFrame, _exportTimestampUs, webcamSourceTimestampMs) => { while (queue.length >= 12 && !this.cancelled && !stopWebcamDecode) { await new Promise((resolve) => setTimeout(resolve, 2)); } @@ -236,7 +236,7 @@ export class GifExporter { webcamFrame.close(); return; } - queue.enqueue(webcamFrame); + queue.enqueue(webcamFrame, webcamSourceTimestampMs); }, onWarning, ) @@ -266,7 +266,9 @@ export class GifExporter { return; } - webcamFrame = webcamFrameQueue ? await webcamFrameQueue.dequeue() : null; + webcamFrame = webcamFrameQueue + ? await webcamFrameQueue.frameAt(sourceTimestampMs) + : null; const renderer = this.renderer; if (this.cancelled || !renderer) { return; diff --git a/src/lib/exporter/timestampedVideoFrameQueue.test.ts b/src/lib/exporter/timestampedVideoFrameQueue.test.ts new file mode 100644 index 0000000..24d625d --- /dev/null +++ b/src/lib/exporter/timestampedVideoFrameQueue.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it, vi } from "vitest"; +import { TimestampedVideoFrameQueue } from "./timestampedVideoFrameQueue"; + +class MockVideoFrame { + timestamp: number; + closed = false; + + constructor(source: MockVideoFrame | number) { + this.timestamp = typeof source === "number" ? source : source.timestamp; + } + + close() { + this.closed = true; + } +} + +describe("TimestampedVideoFrameQueue", () => { + it("samples the latest webcam frame at or before the requested source timestamp", async () => { + const originalVideoFrame = globalThis.VideoFrame; + vi.stubGlobal("VideoFrame", MockVideoFrame); + try { + const queue = new TimestampedVideoFrameQueue(); + const frame0 = new MockVideoFrame(0) as unknown as VideoFrame; + const frame33 = new MockVideoFrame(33_000) as unknown as VideoFrame; + const frame66 = new MockVideoFrame(66_000) as unknown as VideoFrame; + + queue.enqueue(frame0, 0); + queue.enqueue(frame33, 33); + queue.enqueue(frame66, 66); + + const sampled0 = await queue.frameAt(0); + const sampled20 = await queue.frameAt(20); + const sampled40 = await queue.frameAt(40); + const sampled80 = await queue.frameAt(80); + + expect(sampled0?.timestamp).toBe(0); + expect(sampled20?.timestamp).toBe(0); + expect(sampled40?.timestamp).toBe(33_000); + expect(sampled80?.timestamp).toBe(66_000); + + sampled0?.close(); + sampled20?.close(); + sampled40?.close(); + sampled80?.close(); + queue.destroy(); + } finally { + vi.stubGlobal("VideoFrame", originalVideoFrame); + } + }); +}); diff --git a/src/lib/exporter/timestampedVideoFrameQueue.ts b/src/lib/exporter/timestampedVideoFrameQueue.ts new file mode 100644 index 0000000..336b053 --- /dev/null +++ b/src/lib/exporter/timestampedVideoFrameQueue.ts @@ -0,0 +1,105 @@ +type TimestampedVideoFrame = { + frame: VideoFrame; + sourceTimestampMs: number; +}; + +type PendingConsumer = { + resolve: () => void; + reject: (error: Error) => void; +}; + +const TIMESTAMP_EPSILON_MS = 0.5; + +export class TimestampedVideoFrameQueue { + private frames: TimestampedVideoFrame[] = []; + private consumers: PendingConsumer[] = []; + private error: Error | null = null; + private closed = false; + private heldFrame: TimestampedVideoFrame | null = null; + + get length() { + return this.frames.length; + } + + enqueue(frame: VideoFrame, sourceTimestampMs: number) { + if (this.closed) { + frame.close(); + return; + } + + this.frames.push({ frame, sourceTimestampMs }); + const consumers = this.consumers.splice(0); + for (const consumer of consumers) { + consumer.resolve(); + } + } + + fail(error: Error) { + this.error = error; + this.closed = true; + const consumers = this.consumers.splice(0); + for (const consumer of consumers) { + consumer.reject(error); + } + this.closeOwnedFrames(); + } + + close() { + this.closed = true; + const consumers = this.consumers.splice(0); + for (const consumer of consumers) { + consumer.resolve(); + } + } + + async frameAt(sourceTimestampMs: number): Promise { + for (;;) { + if (this.error) { + throw this.error; + } + + const next = this.frames[0] ?? null; + if (next && next.sourceTimestampMs <= sourceTimestampMs + TIMESTAMP_EPSILON_MS) { + this.replaceHeldFrame(this.frames.shift() ?? null); + continue; + } + + if (this.heldFrame) { + return new VideoFrame(this.heldFrame.frame, { + timestamp: this.heldFrame.frame.timestamp, + }); + } + + if (next || this.closed) { + return null; + } + + await new Promise((resolve, reject) => { + this.consumers.push({ resolve, reject }); + }); + } + } + + destroy() { + this.close(); + this.closeOwnedFrames(); + } + + private replaceHeldFrame(frame: TimestampedVideoFrame | null) { + if (this.heldFrame) { + this.heldFrame.frame.close(); + } + this.heldFrame = frame; + } + + private closeOwnedFrames() { + if (this.heldFrame) { + this.heldFrame.frame.close(); + this.heldFrame = null; + } + for (const item of this.frames) { + item.frame.close(); + } + this.frames = []; + } +} diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index 346953b..35c3d55 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -10,11 +10,11 @@ import type { import { BackgroundLoadError } from "@/lib/wallpaper"; import type { CursorRecordingData } from "@/native/contracts"; import { getPlatform } from "@/utils/platformUtils"; -import { AsyncVideoFrameQueue } from "./asyncVideoFrameQueue"; import { AudioProcessor } from "./audioEncoder"; import { FrameRenderer } from "./frameRenderer"; import { VideoMuxer } from "./muxer"; import { StreamingVideoDecoder } from "./streamingDecoder"; +import { TimestampedVideoFrameQueue } from "./timestampedVideoFrameQueue"; import type { ExportConfig, ExportProgress, ExportResult } from "./types"; const ENCODER_STALL_TIMEOUT_MS = 15_000; @@ -195,7 +195,7 @@ export class VideoExporter { private async exportWithEncoderPreference( encoderPreference: HardwareAcceleration, ): Promise { - let webcamFrameQueue: AsyncVideoFrameQueue | null = null; + let webcamFrameQueue: TimestampedVideoFrameQueue | null = null; let stopWebcamDecode = false; let webcamDecodeError: Error | null = null; let webcamDecodePromise: Promise | null = null; @@ -290,7 +290,7 @@ export class VideoExporter { ? Math.min(this.MAX_ENCODE_QUEUE, 32) : this.MAX_ENCODE_QUEUE; - webcamFrameQueue = this.config.webcamVideoUrl ? new AsyncVideoFrameQueue() : null; + webcamFrameQueue = this.config.webcamVideoUrl ? new TimestampedVideoFrameQueue() : null; webcamDecodePromise = webcamDecoder && webcamFrameQueue ? (() => { @@ -300,7 +300,7 @@ export class VideoExporter { this.config.frameRate, this.config.trimRegions, this.config.speedRegions, - async (webcamFrame) => { + async (webcamFrame, _exportTimestampUs, webcamSourceTimestampMs) => { while (queue.length >= 12 && !this.cancelled && !stopWebcamDecode) { await new Promise((resolve) => setTimeout(resolve, 2)); } @@ -308,7 +308,7 @@ export class VideoExporter { webcamFrame.close(); return; } - queue.enqueue(webcamFrame); + queue.enqueue(webcamFrame, webcamSourceTimestampMs); }, onWarning, ) @@ -342,7 +342,9 @@ export class VideoExporter { } const timestamp = frameIndex * frameDuration; - webcamFrame = webcamFrameQueue ? await webcamFrameQueue.dequeue() : null; + webcamFrame = webcamFrameQueue + ? await webcamFrameQueue.frameAt(sourceTimestampMs) + : null; if (this.cancelled) { return; } From 10614c2950fd750faf87de1da1217071773572f3 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Fri, 22 May 2026 21:20:51 +0200 Subject: [PATCH 3/4] Address webcam sidecar review feedback --- .../wgc-capture/src/dshow_webcam_capture.cpp | 7 ++- .../native/wgc-capture/src/mf_encoder.cpp | 7 ++- src/hooks/useScreenRecorder.ts | 25 +++++++++++ .../timestampedVideoFrameQueue.test.ts | 45 ++++++++++++++++++- .../exporter/timestampedVideoFrameQueue.ts | 7 ++- 5 files changed, 86 insertions(+), 5 deletions(-) diff --git a/electron/native/wgc-capture/src/dshow_webcam_capture.cpp b/electron/native/wgc-capture/src/dshow_webcam_capture.cpp index 436bdea..7e3f8b7 100644 --- a/electron/native/wgc-capture/src/dshow_webcam_capture.cpp +++ b/electron/native/wgc-capture/src/dshow_webcam_capture.cpp @@ -379,8 +379,11 @@ void DirectShowWebcamCapture::storeFrame(const BYTE* buffer, long length) { } if (width_ % 2 == 1) { const int x = width_ - 1; - const BYTE* pair = source + (x - 1) * 2; - const auto color = yuvToBgr(pair[2], pair[1], pair[3]); + const int previousPairStart = ((x - 1) / 2) * 4; + const BYTE y = source[x * 2]; + const BYTE u = source[previousPairStart + 1]; + const BYTE v = source[previousPairStart + 3]; + const auto color = yuvToBgr(y, u, v); BYTE* pixel = destination + x * 4; pixel[0] = color[0]; pixel[1] = color[1]; diff --git a/electron/native/wgc-capture/src/mf_encoder.cpp b/electron/native/wgc-capture/src/mf_encoder.cpp index c04353a..18bc4cc 100644 --- a/electron/native/wgc-capture/src/mf_encoder.cpp +++ b/electron/native/wgc-capture/src/mf_encoder.cpp @@ -267,7 +267,12 @@ bool MFEncoder::copyBgraFrameToBuffer(const BgraFrameView& frame, BYTE* destinat } if (frame.width == width_ && frame.height == height_) { - std::memcpy(destination, frame.data, requiredBytes); + for (DWORD i = 0; i < requiredBytes; i += 4) { + destination[i] = frame.data[i]; + destination[i + 1] = frame.data[i + 1]; + destination[i + 2] = frame.data[i + 2]; + destination[i + 3] = 255; + } return true; } diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index f64eec4..cdd4088 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -764,6 +764,25 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const isCountdownRunActive = (runId?: number) => runId === undefined || countdownRunId.current === runId; + const waitForWebcamReady = async () => { + if (webcamReady.current) { + return; + } + + await new Promise((resolve) => { + const interval = setInterval(() => { + if (webcamReady.current) { + clearInterval(interval); + resolve(); + } + }, 50); + setTimeout(() => { + clearInterval(interval); + resolve(); + }, 5000); + }); + }; + const startNativeWindowsRecordingIfAvailable = async ( selectedSource: ProcessedDesktopSource, countdownRunToken?: number, @@ -795,6 +814,12 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const displayId = Number(selectedSource.display_id); const sourceType = selectedSource.id.startsWith("window:") ? "window" : "display"; const windowHandle = parseWindowHandleFromSourceId(selectedSource.id); + if (webcamEnabled) { + await waitForWebcamReady(); + if (!isCountdownRunActive(countdownRunToken)) { + return true; + } + } const browserWebcamRecorder = webcamEnabled && webcamStream.current ? createRecorderHandle(webcamStream.current, { diff --git a/src/lib/exporter/timestampedVideoFrameQueue.test.ts b/src/lib/exporter/timestampedVideoFrameQueue.test.ts index 24d625d..9645c98 100644 --- a/src/lib/exporter/timestampedVideoFrameQueue.test.ts +++ b/src/lib/exporter/timestampedVideoFrameQueue.test.ts @@ -14,6 +14,15 @@ class MockVideoFrame { } } +function restoreVideoFrame(originalVideoFrame: typeof globalThis.VideoFrame | undefined) { + if (originalVideoFrame === undefined) { + delete (globalThis as { VideoFrame?: typeof globalThis.VideoFrame }).VideoFrame; + return; + } + + vi.stubGlobal("VideoFrame", originalVideoFrame); +} + describe("TimestampedVideoFrameQueue", () => { it("samples the latest webcam frame at or before the requested source timestamp", async () => { const originalVideoFrame = globalThis.VideoFrame; @@ -27,6 +36,7 @@ describe("TimestampedVideoFrameQueue", () => { queue.enqueue(frame0, 0); queue.enqueue(frame33, 33); queue.enqueue(frame66, 66); + queue.close(); const sampled0 = await queue.frameAt(0); const sampled20 = await queue.frameAt(20); @@ -44,7 +54,40 @@ describe("TimestampedVideoFrameQueue", () => { sampled80?.close(); queue.destroy(); } finally { - vi.stubGlobal("VideoFrame", originalVideoFrame); + restoreVideoFrame(originalVideoFrame); + } + }); + + it("waits for a newer frame before falling back to the held frame while open", async () => { + const originalVideoFrame = globalThis.VideoFrame; + vi.stubGlobal("VideoFrame", MockVideoFrame); + try { + const queue = new TimestampedVideoFrameQueue(); + const frame0 = new MockVideoFrame(0) as unknown as VideoFrame; + const frame33 = new MockVideoFrame(33_000) as unknown as VideoFrame; + + queue.enqueue(frame0, 0); + const sampled0 = await queue.frameAt(0); + let resolved = false; + const pending = queue.frameAt(33).then((frame) => { + resolved = true; + return frame; + }); + + await Promise.resolve(); + expect(resolved).toBe(false); + + queue.enqueue(frame33, 33); + const sampled33 = await pending; + + expect(sampled0?.timestamp).toBe(0); + expect(sampled33?.timestamp).toBe(33_000); + + sampled0?.close(); + sampled33?.close(); + queue.destroy(); + } finally { + restoreVideoFrame(originalVideoFrame); } }); }); diff --git a/src/lib/exporter/timestampedVideoFrameQueue.ts b/src/lib/exporter/timestampedVideoFrameQueue.ts index 336b053..86c0fe9 100644 --- a/src/lib/exporter/timestampedVideoFrameQueue.ts +++ b/src/lib/exporter/timestampedVideoFrameQueue.ts @@ -64,7 +64,12 @@ export class TimestampedVideoFrameQueue { continue; } - if (this.heldFrame) { + if ( + this.heldFrame && + (next || + this.closed || + this.heldFrame.sourceTimestampMs >= sourceTimestampMs - TIMESTAMP_EPSILON_MS) + ) { return new VideoFrame(this.heldFrame.frame, { timestamp: this.heldFrame.frame.timestamp, }); From 0daf2295a37e73d5a99c3600de727736f01908e0 Mon Sep 17 00:00:00 2001 From: AjTheSpidey Date: Sat, 23 May 2026 03:39:26 +0800 Subject: [PATCH 4/4] fix: accept decimal custom speeds --- src/components/video-editor/SettingsPanel.tsx | 33 ++++++------ .../video-editor/customPlaybackSpeed.test.ts | 50 +++++++++++++++++++ .../video-editor/customPlaybackSpeed.ts | 37 ++++++++++++++ src/i18n/locales/it/settings.json | 3 +- 4 files changed, 106 insertions(+), 17 deletions(-) create mode 100644 src/components/video-editor/customPlaybackSpeed.test.ts create mode 100644 src/components/video-editor/customPlaybackSpeed.ts diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx index b0b46df..3bd8750 100644 --- a/src/components/video-editor/SettingsPanel.tsx +++ b/src/components/video-editor/SettingsPanel.tsx @@ -53,6 +53,7 @@ import ColorPicker from "../ui/color-picker"; import { AnnotationSettingsPanel } from "./AnnotationSettingsPanel"; import { BlurSettingsPanel } from "./BlurSettingsPanel"; import { CropControl } from "./CropControl"; +import { parseCustomPlaybackSpeedInput } from "./customPlaybackSpeed"; import { KeyboardShortcutsHelp } from "./KeyboardShortcutsHelp"; import type { AnnotationRegion, @@ -71,7 +72,6 @@ import type { } from "./types"; import { DEFAULT_WEBCAM_SIZE_PRESET, - MAX_PLAYBACK_SPEED, MAX_ZOOM_SCALE, MIN_ZOOM_SCALE, ROTATION_3D_PRESET_ORDER, @@ -90,37 +90,38 @@ function CustomSpeedInput({ onError: () => void; }) { const isPreset = SPEED_OPTIONS.some((o) => o.speed === value); - const [draft, setDraft] = useState(isPreset ? "" : String(Math.round(value))); + const [draft, setDraft] = useState(isPreset ? "" : String(value)); const [isFocused, setIsFocused] = useState(false); const prevValue = useRef(value); if (!isFocused && prevValue.current !== value) { prevValue.current = value; - setDraft(isPreset ? "" : String(Math.round(value))); + setDraft(isPreset ? "" : String(value)); } const handleChange = useCallback( (e: React.ChangeEvent) => { - const digits = e.target.value.replace(/\D/g, ""); - if (digits === "") { - setDraft(""); - return; - } - const num = Number(digits); - if (num > MAX_PLAYBACK_SPEED) { + const result = parseCustomPlaybackSpeedInput(e.target.value); + if (result.status === "too-fast") { onError(); return; } - setDraft(digits); - if (num >= 1) onChange(num); + + setDraft(result.draft); + if (result.status === "valid") { + onChange(result.speed); + } }, [onChange, onError], ); const handleBlur = useCallback(() => { setIsFocused(false); - if (!draft || Number(draft) < 1) { - setDraft(isPreset ? "" : String(Math.round(value))); + const result = parseCustomPlaybackSpeedInput(draft); + if (result.status === "valid") { + setDraft(String(result.speed)); + } else { + setDraft(isPreset ? "" : String(value)); } }, [draft, isPreset, value]); @@ -128,8 +129,8 @@ function CustomSpeedInput({
setIsFocused(true)} diff --git a/src/components/video-editor/customPlaybackSpeed.test.ts b/src/components/video-editor/customPlaybackSpeed.test.ts new file mode 100644 index 0000000..223175c --- /dev/null +++ b/src/components/video-editor/customPlaybackSpeed.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "vitest"; +import { parseCustomPlaybackSpeedInput } from "./customPlaybackSpeed"; + +describe("parseCustomPlaybackSpeedInput", () => { + it("accepts decimal playback speeds", () => { + expect(parseCustomPlaybackSpeedInput("1.1")).toEqual({ + status: "valid", + draft: "1.1", + speed: 1.1, + }); + }); + + it("keeps a single decimal point while typing", () => { + expect(parseCustomPlaybackSpeedInput("1.2.3")).toEqual({ + status: "valid", + draft: "1.23", + speed: 1.23, + }); + }); + + it("allows sub-1 custom speeds down to the editor minimum", () => { + expect(parseCustomPlaybackSpeedInput("0.1")).toEqual({ + status: "valid", + draft: "0.1", + speed: 0.1, + }); + }); + + it("rejects speeds below the editor minimum", () => { + expect(parseCustomPlaybackSpeedInput("0.09")).toEqual({ + status: "too-slow", + draft: "0.09", + }); + }); + + it("accepts comma decimal input by normalizing to a dot", () => { + expect(parseCustomPlaybackSpeedInput("1,1")).toEqual({ + status: "valid", + draft: "1.1", + speed: 1.1, + }); + }); + + it("rejects speeds above the editor maximum", () => { + expect(parseCustomPlaybackSpeedInput("16.1")).toEqual({ + status: "too-fast", + draft: "16.1", + }); + }); +}); diff --git a/src/components/video-editor/customPlaybackSpeed.ts b/src/components/video-editor/customPlaybackSpeed.ts new file mode 100644 index 0000000..64cc3e3 --- /dev/null +++ b/src/components/video-editor/customPlaybackSpeed.ts @@ -0,0 +1,37 @@ +import { + clampPlaybackSpeed, + MAX_PLAYBACK_SPEED, + MIN_PLAYBACK_SPEED, + type PlaybackSpeed, +} from "./types"; + +export type CustomPlaybackSpeedInputResult = + | { status: "empty"; draft: string } + | { status: "too-fast"; draft: string } + | { status: "too-slow"; draft: string } + | { status: "valid"; draft: string; speed: PlaybackSpeed }; + +export function parseCustomPlaybackSpeedInput(rawValue: string): CustomPlaybackSpeedInputResult { + const decimalDraft = rawValue.replace(/,/g, ".").replace(/[^\d.]/g, ""); + const [whole = "", ...fractionParts] = decimalDraft.split("."); + const draft = fractionParts.length > 0 ? `${whole}.${fractionParts.join("")}` : whole; + + if (draft === "" || draft === ".") { + return { status: "empty", draft }; + } + + const speed = Number(draft); + if (!Number.isFinite(speed)) { + return { status: "empty", draft }; + } + + if (speed > MAX_PLAYBACK_SPEED) { + return { status: "too-fast", draft }; + } + + if (speed < MIN_PLAYBACK_SPEED) { + return { status: "too-slow", draft }; + } + + return { status: "valid", draft, speed: clampPlaybackSpeed(speed) }; +} diff --git a/src/i18n/locales/it/settings.json b/src/i18n/locales/it/settings.json index e609d08..0515a76 100644 --- a/src/i18n/locales/it/settings.json +++ b/src/i18n/locales/it/settings.json @@ -104,8 +104,9 @@ "gifButton": "Esporta GIF", "chooseSaveLocation": "Scegli posizione di salvataggio" }, - "links": { + "support": { "reportBug": "Segnala bug", + "saveDiagnostics": "Salva dati diagnostici", "starOnGithub": "Metti stella su GitHub" }, "imageUpload": {