Merge remote-tracking branch 'origin/main' into codex/fix-windows-paused-recording

# Conflicts:
#	src/hooks/useScreenRecorder.ts
This commit is contained in:
Siddharth
2026-05-22 20:08:26 -07:00
17 changed files with 813 additions and 142 deletions
@@ -5,22 +5,18 @@
#include <wrl/client.h> #include <wrl/client.h>
#include <algorithm> #include <algorithm>
#include <array>
#include <chrono> #include <chrono>
#include <exception> #include <exception>
#include <iomanip>
#include <iostream> #include <iostream>
#include <sstream>
namespace { namespace {
const CLSID CLSID_SampleGrabberLocal = {0xC1F400A0, 0x3F08, 0x11D3, {0x9F, 0x0B, 0x00, 0x60, 0x08, 0x03, 0x9E, 0x37}}; const CLSID CLSID_SampleGrabberLocal = {0xC1F400A0, 0x3F08, 0x11D3, {0x9F, 0x0B, 0x00, 0x60, 0x08, 0x03, 0x9E, 0x37}};
const CLSID CLSID_NullRendererLocal = {0xC1F400A4, 0x3F08, 0x11D3, {0x9F, 0x0B, 0x00, 0x60, 0x08, 0x03, 0x9E, 0x37}}; const CLSID CLSID_NullRendererLocal = {0xC1F400A4, 0x3F08, 0x11D3, {0x9F, 0x0B, 0x00, 0x60, 0x08, 0x03, 0x9E, 0x37}};
MIDL_INTERFACE("0579154A-2B53-4994-B0D0-E773148EFF85")
ISampleGrabberCB : public IUnknown {
public:
virtual HRESULT STDMETHODCALLTYPE SampleCB(double sampleTime, IMediaSample* sample) = 0;
virtual HRESULT STDMETHODCALLTYPE BufferCB(double sampleTime, BYTE* buffer, long bufferLength) = 0;
};
MIDL_INTERFACE("6B652FFF-11FE-4FCE-92AD-0266B5D7C78F") MIDL_INTERFACE("6B652FFF-11FE-4FCE-92AD-0266B5D7C78F")
ISampleGrabber : public IUnknown { ISampleGrabber : public IUnknown {
public: public:
@@ -30,7 +26,7 @@ public:
virtual HRESULT STDMETHODCALLTYPE SetBufferSamples(BOOL bufferThem) = 0; virtual HRESULT STDMETHODCALLTYPE SetBufferSamples(BOOL bufferThem) = 0;
virtual HRESULT STDMETHODCALLTYPE GetCurrentBuffer(long* bufferSize, long* buffer) = 0; virtual HRESULT STDMETHODCALLTYPE GetCurrentBuffer(long* bufferSize, long* buffer) = 0;
virtual HRESULT STDMETHODCALLTYPE GetCurrentSample(IMediaSample** sample) = 0; virtual HRESULT STDMETHODCALLTYPE GetCurrentSample(IMediaSample** sample) = 0;
virtual HRESULT STDMETHODCALLTYPE SetCallback(ISampleGrabberCB* callback, long whichMethodToCallback) = 0; virtual HRESULT STDMETHODCALLTYPE SetCallback(IUnknown* callback, long whichMethodToCallback) = 0;
}; };
bool succeeded(HRESULT hr, const char* label) { bool succeeded(HRESULT hr, const char* label) {
@@ -43,6 +39,34 @@ bool succeeded(HRESULT hr, const char* label) {
return false; return false;
} }
std::string guidToString(const GUID& guid) {
if (guid == MEDIASUBTYPE_RGB32) {
return "RGB32";
}
if (guid == MEDIASUBTYPE_YUY2) {
return "YUY2";
}
if (guid == MEDIASUBTYPE_NV12) {
return "NV12";
}
std::ostringstream stream;
stream << std::hex << std::setfill('0')
<< '{' << std::setw(8) << guid.Data1
<< '-' << std::setw(4) << guid.Data2
<< '-' << std::setw(4) << guid.Data3
<< '-';
for (int index = 0; index < 2; index += 1) {
stream << std::setw(2) << static_cast<int>(guid.Data4[index]);
}
stream << '-';
for (int index = 2; index < 8; index += 1) {
stream << std::setw(2) << static_cast<int>(guid.Data4[index]);
}
stream << '}';
return stream.str();
}
void freeMediaType(AM_MEDIA_TYPE& type) { void freeMediaType(AM_MEDIA_TYPE& type) {
if (type.cbFormat != 0) { if (type.cbFormat != 0) {
CoTaskMemFree(type.pbFormat); CoTaskMemFree(type.pbFormat);
@@ -55,6 +79,20 @@ void freeMediaType(AM_MEDIA_TYPE& type) {
} }
} }
BYTE clampToByte(int value) {
return static_cast<BYTE>(std::clamp(value, 0, 255));
}
std::array<BYTE, 3> yuvToBgr(int y, int u, int v) {
const int c = y - 16;
const int d = u - 128;
const int e = v - 128;
const int blue = (298 * c + 516 * d + 128) >> 8;
const int green = (298 * c - 100 * d - 208 * e + 128) >> 8;
const int red = (298 * c + 409 * e + 128) >> 8;
return {clampToByte(blue), clampToByte(green), clampToByte(red)};
}
} // namespace } // namespace
struct DirectShowWebcamCapture::Impl { struct DirectShowWebcamCapture::Impl {
@@ -137,9 +175,8 @@ bool DirectShowWebcamCapture::initialize(
AM_MEDIA_TYPE requestedType{}; AM_MEDIA_TYPE requestedType{};
requestedType.majortype = MEDIATYPE_Video; requestedType.majortype = MEDIATYPE_Video;
requestedType.subtype = MEDIASUBTYPE_RGB32;
requestedType.formattype = FORMAT_VideoInfo; requestedType.formattype = FORMAT_VideoInfo;
if (!succeeded(impl_->sampleGrabber->SetMediaType(&requestedType), "SetMediaType(DirectShow RGB32)")) { if (!succeeded(impl_->sampleGrabber->SetMediaType(&requestedType), "SetMediaType(DirectShow video)")) {
return false; return false;
} }
@@ -170,17 +207,40 @@ bool DirectShowWebcamCapture::initialize(
if (!succeeded(impl_->sampleGrabber->GetConnectedMediaType(&connectedType), "GetConnectedMediaType(DirectShow webcam)")) { if (!succeeded(impl_->sampleGrabber->GetConnectedMediaType(&connectedType), "GetConnectedMediaType(DirectShow webcam)")) {
return false; return false;
} }
if (connectedType.subtype == MEDIASUBTYPE_YUY2) {
pixelFormat_ = PixelFormat::Yuy2;
} else if (connectedType.subtype == MEDIASUBTYPE_NV12) {
pixelFormat_ = PixelFormat::Nv12;
} else if (connectedType.subtype == MEDIASUBTYPE_RGB32) {
pixelFormat_ = PixelFormat::Bgra;
} else {
std::cerr << "ERROR: Unsupported DirectShow webcam media subtype "
<< guidToString(connectedType.subtype) << std::endl;
freeMediaType(connectedType);
return false;
}
if (connectedType.formattype == FORMAT_VideoInfo && connectedType.pbFormat) { if (connectedType.formattype == FORMAT_VideoInfo && connectedType.pbFormat) {
const auto* videoInfo = reinterpret_cast<VIDEOINFOHEADER*>(connectedType.pbFormat); const auto* videoInfo = reinterpret_cast<VIDEOINFOHEADER*>(connectedType.pbFormat);
width_ = std::abs(videoInfo->bmiHeader.biWidth); width_ = std::abs(videoInfo->bmiHeader.biWidth);
height_ = std::abs(videoInfo->bmiHeader.biHeight); height_ = std::abs(videoInfo->bmiHeader.biHeight);
sourceTopDown_ = videoInfo->bmiHeader.biHeight < 0; const int bitsPerPixel = videoInfo->bmiHeader.biBitCount > 0 ? videoInfo->bmiHeader.biBitCount : 16;
if (pixelFormat_ == PixelFormat::Nv12) {
sourceStride_ = ((width_ + 3) / 4) * 4;
} else {
sourceStride_ = ((width_ * bitsPerPixel + 31) / 32) * 4;
}
sourceTopDown_ = pixelFormat_ != PixelFormat::Bgra || videoInfo->bmiHeader.biHeight < 0;
} }
std::cerr << "INFO: DirectShow webcam connected subtype " << guidToString(connectedType.subtype)
<< " " << width_ << "x" << height_ << " stride=" << sourceStride_ << std::endl;
freeMediaType(connectedType); freeMediaType(connectedType);
if (width_ <= 0 || height_ <= 0) { if (width_ <= 0 || height_ <= 0) {
width_ = requestedWidth > 0 ? requestedWidth : 1280; width_ = requestedWidth > 0 ? requestedWidth : 1280;
height_ = requestedHeight > 0 ? requestedHeight : 720; height_ = requestedHeight > 0 ? requestedHeight : 720;
} }
if (sourceStride_ <= 0) {
sourceStride_ = pixelFormat_ == PixelFormat::Bgra ? width_ * 4 : ((width_ + 3) / 4) * 4;
}
impl_->sampleGrabber->SetBufferSamples(TRUE); impl_->sampleGrabber->SetBufferSamples(TRUE);
impl_->sampleGrabber->SetOneShot(FALSE); impl_->sampleGrabber->SetOneShot(FALSE);
@@ -262,36 +322,91 @@ void DirectShowWebcamCapture::captureLoop() {
} }
void DirectShowWebcamCapture::storeFrame(const BYTE* buffer, long length) { void DirectShowWebcamCapture::storeFrame(const BYTE* buffer, long length) {
const int stride = width_ * 4; const int destinationStride = width_ * 4;
const int expectedLength = stride * height_; const int sourceStride = sourceStride_ > 0 ? sourceStride_ : destinationStride;
const int expectedLength = pixelFormat_ == PixelFormat::Nv12
? sourceStride * height_ + sourceStride * ((height_ + 1) / 2)
: sourceStride * height_;
if (!buffer || length < expectedLength || width_ <= 0 || height_ <= 0) { if (!buffer || length < expectedLength || width_ <= 0 || height_ <= 0) {
return; return;
} }
std::vector<BYTE> frame(static_cast<size_t>(expectedLength)); std::vector<BYTE> frame(static_cast<size_t>(destinationStride * height_));
for (int y = 0; y < height_; y += 1) { for (int y = 0; y < height_; y += 1) {
const int sourceY = sourceTopDown_ ? y : height_ - 1 - y; const int sourceY = sourceTopDown_ ? y : height_ - 1 - y;
const BYTE* source = buffer + sourceY * stride; const BYTE* source = buffer + sourceY * sourceStride;
BYTE* destination = frame.data() + y * stride; BYTE* destination = frame.data() + y * destinationStride;
std::copy(source, source + stride, destination); if (pixelFormat_ == PixelFormat::Bgra) {
for (int x = 0; x < width_; x += 1) { std::copy(source, source + destinationStride, destination);
destination[x * 4 + 3] = 255; for (int x = 0; x < width_; x += 1) {
destination[x * 4 + 3] = 255;
}
continue;
}
if (pixelFormat_ == PixelFormat::Nv12) {
const BYTE* yPlane = buffer + sourceY * sourceStride;
const BYTE* uvPlane = buffer + sourceStride * height_ + (sourceY / 2) * sourceStride;
for (int x = 0; x < width_; x += 1) {
const int uvX = (x / 2) * 2;
const auto color = yuvToBgr(yPlane[x], uvPlane[uvX], uvPlane[uvX + 1]);
BYTE* pixel = destination + x * 4;
pixel[0] = color[0];
pixel[1] = color[1];
pixel[2] = color[2];
pixel[3] = 255;
}
continue;
}
for (int x = 0; x + 1 < width_; x += 2) {
const BYTE y0 = source[x * 2];
const BYTE u = source[x * 2 + 1];
const BYTE y1 = source[x * 2 + 2];
const BYTE v = source[x * 2 + 3];
const auto first = yuvToBgr(y0, u, v);
const auto second = yuvToBgr(y1, u, v);
BYTE* firstPixel = destination + x * 4;
BYTE* secondPixel = firstPixel + 4;
firstPixel[0] = first[0];
firstPixel[1] = first[1];
firstPixel[2] = first[2];
firstPixel[3] = 255;
secondPixel[0] = second[0];
secondPixel[1] = second[1];
secondPixel[2] = second[2];
secondPixel[3] = 255;
}
if (width_ % 2 == 1) {
const int x = width_ - 1;
const int previousPairStart = ((x - 1) / 2) * 4;
const BYTE y = source[x * 2];
const BYTE u = source[previousPairStart + 1];
const BYTE v = source[previousPairStart + 3];
const auto color = yuvToBgr(y, u, v);
BYTE* pixel = destination + x * 4;
pixel[0] = color[0];
pixel[1] = color[1];
pixel[2] = color[2];
pixel[3] = 255;
} }
} }
std::scoped_lock lock(frameMutex_); std::scoped_lock lock(frameMutex_);
latestFrame_ = std::move(frame); latestFrame_ = std::move(frame);
latestFrameSequence_ += 1;
} }
bool DirectShowWebcamCapture::copyLatestFrame(std::vector<BYTE>& destination, int& width, int& height) { bool DirectShowWebcamCapture::copyLatestFrame(WebcamFrameSnapshot& destination) {
std::scoped_lock lock(frameMutex_); std::scoped_lock lock(frameMutex_);
if (latestFrame_.empty() || width_ <= 0 || height_ <= 0) { if (latestFrame_.empty() || width_ <= 0 || height_ <= 0) {
return false; return false;
} }
destination = latestFrame_; destination.data = latestFrame_;
width = width_; destination.width = width_;
height = height_; destination.height = height_;
destination.sequence = latestFrameSequence_;
return true; return true;
} }
@@ -3,11 +3,19 @@
#include <Windows.h> #include <Windows.h>
#include <atomic> #include <atomic>
#include <cstdint>
#include <mutex> #include <mutex>
#include <string> #include <string>
#include <thread> #include <thread>
#include <vector> #include <vector>
struct WebcamFrameSnapshot {
std::vector<BYTE> data;
int width = 0;
int height = 0;
uint64_t sequence = 0;
};
class DirectShowWebcamCapture { class DirectShowWebcamCapture {
public: public:
DirectShowWebcamCapture() = default; DirectShowWebcamCapture() = default;
@@ -25,7 +33,7 @@ public:
int requestedFps); int requestedFps);
bool start(); bool start();
void stop(); void stop();
bool copyLatestFrame(std::vector<BYTE>& destination, int& width, int& height); bool copyLatestFrame(WebcamFrameSnapshot& destination);
int width() const; int width() const;
int height() const; int height() const;
@@ -34,6 +42,12 @@ public:
void storeFrame(const BYTE* buffer, long length); void storeFrame(const BYTE* buffer, long length);
private: private:
enum class PixelFormat {
Bgra,
Nv12,
Yuy2,
};
struct Impl; struct Impl;
void captureLoop(); void captureLoop();
@@ -42,9 +56,12 @@ private:
std::atomic<bool> stopRequested_ = false; std::atomic<bool> stopRequested_ = false;
std::mutex frameMutex_; std::mutex frameMutex_;
std::vector<BYTE> latestFrame_; std::vector<BYTE> latestFrame_;
uint64_t latestFrameSequence_ = 0;
int width_ = 0; int width_ = 0;
int height_ = 0; int height_ = 0;
int fps_ = 30; int fps_ = 30;
int sourceStride_ = 0;
bool sourceTopDown_ = false; bool sourceTopDown_ = false;
PixelFormat pixelFormat_ = PixelFormat::Bgra;
std::wstring selectedDeviceName_; std::wstring selectedDeviceName_;
}; };
+62 -18
View File
@@ -30,6 +30,7 @@ struct CaptureConfig {
std::string sourceId; std::string sourceId;
std::string windowHandle; std::string windowHandle;
std::string outputPath; std::string outputPath;
std::string webcamOutputPath;
int fps = 60; int fps = 60;
int width = 0; int width = 0;
int height = 0; int height = 0;
@@ -343,6 +344,7 @@ bool parseConfig(const std::string& json, CaptureConfig& config) {
config.webcamDeviceId = findString(json, "webcamDeviceId"); config.webcamDeviceId = findString(json, "webcamDeviceId");
config.webcamDeviceName = findString(json, "webcamDeviceName"); config.webcamDeviceName = findString(json, "webcamDeviceName");
config.webcamDirectShowClsid = findString(json, "webcamDirectShowClsid"); config.webcamDirectShowClsid = findString(json, "webcamDirectShowClsid");
config.webcamOutputPath = findString(json, "webcamPath");
config.webcamWidth = findInt(json, "webcamWidth", 0); config.webcamWidth = findInt(json, "webcamWidth", 0);
config.webcamHeight = findInt(json, "webcamHeight", 0); config.webcamHeight = findInt(json, "webcamHeight", 0);
config.webcamFps = findInt(json, "webcamFps", 0); config.webcamFps = findInt(json, "webcamFps", 0);
@@ -435,6 +437,7 @@ int main(int argc, char* argv[]) {
WebcamCapture webcamCapture; WebcamCapture webcamCapture;
bool webcamActive = false; bool webcamActive = false;
bool writeSeparateWebcam = false;
if (config.webcamEnabled) { if (config.webcamEnabled) {
if (!webcamCapture.initialize( if (!webcamCapture.initialize(
utf8ToWide(config.webcamDeviceId), utf8ToWide(config.webcamDeviceId),
@@ -451,6 +454,7 @@ int main(int argc, char* argv[]) {
<< ",\"fps\":" << webcamCapture.fps() << ",\"fps\":" << webcamCapture.fps()
<< ",\"deviceName\":\"" << jsonEscape(wideToUtf8(webcamCapture.selectedDeviceName())) << ",\"deviceName\":\"" << jsonEscape(wideToUtf8(webcamCapture.selectedDeviceName()))
<< "\"}" << std::endl; << "\"}" << std::endl;
writeSeparateWebcam = !config.webcamOutputPath.empty();
} }
WasapiLoopbackCapture loopbackCapture; WasapiLoopbackCapture loopbackCapture;
@@ -512,6 +516,24 @@ int main(int argc, char* argv[]) {
return 1; return 1;
} }
MFEncoder webcamEncoder;
if (writeSeparateWebcam) {
const int webcamPixels = std::max(1, webcamCapture.width()) * std::max(1, webcamCapture.height());
const int webcamBitrate = webcamPixels >= 1280 * 720 ? 8'000'000 : 4'000'000;
if (!webcamEncoder.initialize(
utf8ToWide(config.webcamOutputPath),
webcamCapture.width(),
webcamCapture.height(),
webcamCapture.fps(),
webcamBitrate,
session.device(),
session.context(),
nullptr)) {
std::cerr << "ERROR: Failed to initialize native webcam encoder" << std::endl;
return 1;
}
}
std::mutex mutex; std::mutex mutex;
CaptureControl control; CaptureControl control;
std::atomic<bool> firstFrameWritten = false; std::atomic<bool> firstFrameWritten = false;
@@ -522,6 +544,7 @@ int main(int argc, char* argv[]) {
std::vector<BYTE> latestWebcamFrame; std::vector<BYTE> latestWebcamFrame;
int latestWebcamWidth = 0; int latestWebcamWidth = 0;
int latestWebcamHeight = 0; int latestWebcamHeight = 0;
uint64_t latestWebcamSequence = 0;
bool hasVisibleWebcamFrame = false; bool hasVisibleWebcamFrame = false;
session.setFrameCallback([&](ID3D11Texture2D* texture, int64_t timestampHns) { session.setFrameCallback([&](ID3D11Texture2D* texture, int64_t timestampHns) {
@@ -555,6 +578,8 @@ int main(int argc, char* argv[]) {
const auto frameDuration = std::chrono::duration_cast<std::chrono::steady_clock::duration>( const auto frameDuration = std::chrono::duration_cast<std::chrono::steady_clock::duration>(
std::chrono::duration<double>(1.0 / config.fps)); std::chrono::duration<double>(1.0 / config.fps));
uint64_t frameIndex = 0; uint64_t frameIndex = 0;
uint64_t lastWrittenWebcamSequence = 0;
uint64_t webcamOutputFrameIndex = 0;
int64_t lastEncodedVideoTimestampHns = -1; int64_t lastEncodedVideoTimestampHns = -1;
while (!control.stopRequested && !encodeFailed) { while (!control.stopRequested && !encodeFailed) {
@@ -569,14 +594,14 @@ int main(int argc, char* argv[]) {
break; break;
} }
if (webcamActive) { if (webcamActive) {
std::vector<BYTE> candidateWebcamFrame; WebcamFrameSnapshot candidateWebcamFrame;
int candidateWebcamWidth = 0; if (webcamCapture.copyLatestFrame(candidateWebcamFrame) &&
int candidateWebcamHeight = 0; candidateWebcamFrame.sequence != latestWebcamSequence &&
if (webcamCapture.copyLatestFrame(candidateWebcamFrame, candidateWebcamWidth, candidateWebcamHeight) && hasVisibleBgraContent(candidateWebcamFrame.data)) {
hasVisibleBgraContent(candidateWebcamFrame)) { latestWebcamFrame = std::move(candidateWebcamFrame.data);
latestWebcamFrame = std::move(candidateWebcamFrame); latestWebcamWidth = candidateWebcamFrame.width;
latestWebcamWidth = candidateWebcamWidth; latestWebcamHeight = candidateWebcamFrame.height;
latestWebcamHeight = candidateWebcamHeight; latestWebcamSequence = candidateWebcamFrame.sequence;
hasVisibleWebcamFrame = true; hasVisibleWebcamFrame = true;
} }
} }
@@ -601,10 +626,23 @@ int main(int argc, char* argv[]) {
frameTimestampHns = frameTimestampHns =
lastEncodedVideoTimestampHns + static_cast<int64_t>(10'000'000ULL / config.fps); lastEncodedVideoTimestampHns + static_cast<int64_t>(10'000'000ULL / config.fps);
} }
if (writeSeparateWebcam && webcamFrame.data &&
latestWebcamSequence != lastWrittenWebcamSequence) {
const int64_t webcamTimestampHns = static_cast<int64_t>(
(webcamOutputFrameIndex * 10'000'000ULL) / std::max(1, webcamCapture.fps()));
if (!webcamEncoder.writeBgraFrame(webcamFrame, webcamTimestampHns)) {
encodeFailed = true;
stopRequested = true;
cv.notify_all();
return;
}
lastWrittenWebcamSequence = latestWebcamSequence;
webcamOutputFrameIndex += 1;
}
if (latestFrameTexture && !encoder.writeFrame( if (latestFrameTexture && !encoder.writeFrame(
latestFrameTexture.Get(), latestFrameTexture.Get(),
frameTimestampHns, frameTimestampHns,
webcamFrame.data ? &webcamFrame : nullptr)) { !writeSeparateWebcam && webcamFrame.data ? &webcamFrame : nullptr)) {
encodeFailed = true; encodeFailed = true;
control.stopRequested = true; control.stopRequested = true;
control.cv.notify_all(); control.cv.notify_all();
@@ -712,14 +750,13 @@ int main(int argc, char* argv[]) {
webcamActive = true; webcamActive = true;
const auto webcamDeadline = std::chrono::steady_clock::now() + std::chrono::seconds(3); const auto webcamDeadline = std::chrono::steady_clock::now() + std::chrono::seconds(3);
while (std::chrono::steady_clock::now() < webcamDeadline && !hasVisibleWebcamFrame) { while (std::chrono::steady_clock::now() < webcamDeadline && !hasVisibleWebcamFrame) {
std::vector<BYTE> candidateWebcamFrame; WebcamFrameSnapshot candidateWebcamFrame;
int candidateWebcamWidth = 0; if (webcamCapture.copyLatestFrame(candidateWebcamFrame) &&
int candidateWebcamHeight = 0; hasVisibleBgraContent(candidateWebcamFrame.data)) {
if (webcamCapture.copyLatestFrame(candidateWebcamFrame, candidateWebcamWidth, candidateWebcamHeight) && latestWebcamFrame = std::move(candidateWebcamFrame.data);
hasVisibleBgraContent(candidateWebcamFrame)) { latestWebcamWidth = candidateWebcamFrame.width;
latestWebcamFrame = std::move(candidateWebcamFrame); latestWebcamHeight = candidateWebcamFrame.height;
latestWebcamWidth = candidateWebcamWidth; latestWebcamSequence = candidateWebcamFrame.sequence;
latestWebcamHeight = candidateWebcamHeight;
hasVisibleWebcamFrame = true; hasVisibleWebcamFrame = true;
break; break;
} }
@@ -797,6 +834,9 @@ int main(int argc, char* argv[]) {
{ {
std::scoped_lock lock(mutex); std::scoped_lock lock(mutex);
encoder.finalize(); encoder.finalize();
if (writeSeparateWebcam) {
webcamEncoder.finalize();
}
} }
if (stdinThread.joinable()) { if (stdinThread.joinable()) {
@@ -809,7 +849,11 @@ int main(int argc, char* argv[]) {
} }
std::cout << "{\"event\":\"recording-stopped\",\"schemaVersion\":2,\"screenPath\":\"" std::cout << "{\"event\":\"recording-stopped\",\"schemaVersion\":2,\"screenPath\":\""
<< jsonEscape(config.outputPath) << "\"}" << std::endl; << jsonEscape(config.outputPath) << "\"";
if (writeSeparateWebcam) {
std::cout << ",\"webcamPath\":\"" << jsonEscape(config.webcamOutputPath) << "\"";
}
std::cout << "}" << std::endl;
std::cout << "Recording stopped. Output path: " << config.outputPath << std::endl; std::cout << "Recording stopped. Output path: " << config.outputPath << std::endl;
return 0; return 0;
} }
@@ -254,6 +254,45 @@ bool MFEncoder::copyFrameToBuffer(
return true; return true;
} }
bool MFEncoder::copyBgraFrameToBuffer(const BgraFrameView& frame, BYTE* destination, DWORD destinationSize) {
if (!frame.data || frame.width <= 0 || frame.height <= 0) {
return false;
}
const DWORD rowBytes = static_cast<DWORD>(width_ * 4);
const DWORD requiredBytes = rowBytes * static_cast<DWORD>(height_);
if (destinationSize < requiredBytes) {
std::cerr << "ERROR: Media Foundation webcam buffer is too small" << std::endl;
return false;
}
if (frame.width == width_ && frame.height == height_) {
for (DWORD i = 0; i < requiredBytes; i += 4) {
destination[i] = frame.data[i];
destination[i + 1] = frame.data[i + 1];
destination[i + 2] = frame.data[i + 2];
destination[i + 3] = 255;
}
return true;
}
for (int y = 0; y < height_; y += 1) {
const int sourceY = static_cast<int>((static_cast<int64_t>(y) * frame.height) / height_);
BYTE* destinationRow = destination + rowBytes * y;
for (int x = 0; x < width_; x += 1) {
const int sourceX = static_cast<int>((static_cast<int64_t>(x) * frame.width) / width_);
const BYTE* source = frame.data + (sourceY * frame.width + sourceX) * 4;
BYTE* target = destinationRow + x * 4;
target[0] = source[0];
target[1] = source[1];
target[2] = source[2];
target[3] = 255;
}
}
return true;
}
bool MFEncoder::writeFrame(ID3D11Texture2D* texture, int64_t timestampHns, const BgraFrameView* webcamFrame) { bool MFEncoder::writeFrame(ID3D11Texture2D* texture, int64_t timestampHns, const BgraFrameView* webcamFrame) {
std::scoped_lock writerLock(writerMutex_); std::scoped_lock writerLock(writerMutex_);
if (!sinkWriter_ || finalized_) { if (!sinkWriter_ || finalized_) {
@@ -302,6 +341,54 @@ bool MFEncoder::writeFrame(ID3D11Texture2D* texture, int64_t timestampHns, const
return succeeded(sinkWriter_->WriteSample(videoStreamIndex_, sample.Get()), "WriteSample"); return succeeded(sinkWriter_->WriteSample(videoStreamIndex_, sample.Get()), "WriteSample");
} }
bool MFEncoder::writeBgraFrame(const BgraFrameView& frame, int64_t timestampHns) {
std::scoped_lock writerLock(writerMutex_);
if (!sinkWriter_ || finalized_) {
return false;
}
if (firstTimestampHns_ < 0) {
firstTimestampHns_ = timestampHns;
}
int64_t sampleTime = timestampHns - firstTimestampHns_;
if (sampleTime <= lastTimestampHns_) {
sampleTime = lastTimestampHns_ + (10'000'000LL / fps_);
}
const int64_t sampleDuration = 10'000'000LL / fps_;
lastTimestampHns_ = sampleTime;
Microsoft::WRL::ComPtr<IMFMediaBuffer> buffer;
const DWORD frameBytes = static_cast<DWORD>(width_ * height_ * 4);
if (!succeeded(MFCreateMemoryBuffer(frameBytes, &buffer), "MFCreateMemoryBuffer(webcam)")) {
return false;
}
BYTE* data = nullptr;
DWORD maxLength = 0;
DWORD currentLength = 0;
if (!succeeded(buffer->Lock(&data, &maxLength, &currentLength), "IMFMediaBuffer::Lock(webcam)")) {
return false;
}
const bool copied = copyBgraFrameToBuffer(frame, data, maxLength);
buffer->Unlock();
if (!copied) {
return false;
}
buffer->SetCurrentLength(frameBytes);
Microsoft::WRL::ComPtr<IMFSample> sample;
if (!succeeded(MFCreateSample(&sample), "MFCreateSample(webcam)")) {
return false;
}
sample->AddBuffer(buffer.Get());
sample->SetSampleTime(sampleTime);
sample->SetSampleDuration(sampleDuration);
return succeeded(sinkWriter_->WriteSample(videoStreamIndex_, sample.Get()), "WriteSample(webcam)");
}
bool MFEncoder::writeAudio(const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns) { bool MFEncoder::writeAudio(const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns) {
std::scoped_lock writerLock(writerMutex_); std::scoped_lock writerLock(writerMutex_);
if (!sinkWriter_ || finalized_ || !hasAudioStream_) { if (!sinkWriter_ || finalized_ || !hasAudioStream_) {
@@ -44,6 +44,7 @@ public:
ID3D11DeviceContext* context, ID3D11DeviceContext* context,
const AudioInputFormat* audioFormat = nullptr); const AudioInputFormat* audioFormat = nullptr);
bool writeFrame(ID3D11Texture2D* texture, int64_t timestampHns, const BgraFrameView* webcamFrame = nullptr); bool writeFrame(ID3D11Texture2D* texture, int64_t timestampHns, const BgraFrameView* webcamFrame = nullptr);
bool writeBgraFrame(const BgraFrameView& frame, int64_t timestampHns);
bool writeAudio(const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns); bool writeAudio(const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns);
bool finalize(); bool finalize();
@@ -54,6 +55,7 @@ private:
BYTE* destination, BYTE* destination,
DWORD destinationSize, DWORD destinationSize,
const BgraFrameView* webcamFrame); const BgraFrameView* webcamFrame);
bool copyBgraFrameToBuffer(const BgraFrameView& frame, BYTE* destination, DWORD destinationSize);
bool configureAudioStream(const AudioInputFormat& audioFormat); bool configureAudioStream(const AudioInputFormat& audioFormat);
Microsoft::WRL::ComPtr<IMFSinkWriter> sinkWriter_; Microsoft::WRL::ComPtr<IMFSinkWriter> sinkWriter_;
@@ -365,6 +365,7 @@ void WebcamCapture::captureLoop() {
if (currentLength >= expectedLength && expectedLength > 0) { if (currentLength >= expectedLength && expectedLength > 0) {
std::scoped_lock lock(frameMutex_); std::scoped_lock lock(frameMutex_);
latestFrame_.assign(data, data + expectedLength); latestFrame_.assign(data, data + expectedLength);
latestFrameSequence_ += 1;
} }
buffer->Unlock(); buffer->Unlock();
@@ -373,18 +374,19 @@ void WebcamCapture::captureLoop() {
CoUninitialize(); CoUninitialize();
} }
bool WebcamCapture::copyLatestFrame(std::vector<BYTE>& destination, int& width, int& height) { bool WebcamCapture::copyLatestFrame(WebcamFrameSnapshot& destination) {
if (usingDirectShow_) { if (usingDirectShow_) {
return directShowCapture_.copyLatestFrame(destination, width, height); return directShowCapture_.copyLatestFrame(destination);
} }
std::scoped_lock lock(frameMutex_); std::scoped_lock lock(frameMutex_);
if (latestFrame_.empty() || width_ <= 0 || height_ <= 0) { if (latestFrame_.empty() || width_ <= 0 || height_ <= 0) {
return false; return false;
} }
destination = latestFrame_; destination.data = latestFrame_;
width = width_; destination.width = width_;
height = height_; destination.height = height_;
destination.sequence = latestFrameSequence_;
return true; return true;
} }
@@ -31,7 +31,7 @@ public:
int requestedFps); int requestedFps);
bool start(); bool start();
void stop(); void stop();
bool copyLatestFrame(std::vector<BYTE>& destination, int& width, int& height); bool copyLatestFrame(WebcamFrameSnapshot& destination);
int width() const; int width() const;
int height() const; int height() const;
@@ -50,6 +50,7 @@ private:
std::atomic<bool> stopRequested_ = false; std::atomic<bool> stopRequested_ = false;
std::mutex frameMutex_; std::mutex frameMutex_;
std::vector<BYTE> latestFrame_; std::vector<BYTE> latestFrame_;
uint64_t latestFrameSequence_ = 0;
int width_ = 0; int width_ = 0;
int height_ = 0; int height_ = 0;
int fps_ = 30; int fps_ = 30;
+26 -1
View File
@@ -230,6 +230,7 @@ const outputPath = path.join(
os.tmpdir(), os.tmpdir(),
`openscreen-wgc-helper-${WITH_WEBCAM ? "webcam" : WITH_WINDOW ? "window" : WITH_SYSTEM_AUDIO || WITH_MICROPHONE ? "audio" : "video"}-${process.pid}-${Date.now()}-${randomUUID()}.mp4`, `openscreen-wgc-helper-${WITH_WEBCAM ? "webcam" : WITH_WINDOW ? "window" : WITH_SYSTEM_AUDIO || WITH_MICROPHONE ? "audio" : "video"}-${process.pid}-${Date.now()}-${randomUUID()}.mp4`,
); );
const webcamOutputPath = WITH_WEBCAM ? outputPath.replace(/\.mp4$/i, "-webcam.mp4") : null;
const fixtureWindow = WITH_WINDOW ? await startFixtureWindow() : null; const fixtureWindow = WITH_WINDOW ? await startFixtureWindow() : null;
@@ -263,7 +264,10 @@ const config = {
webcamWidth: 640, webcamWidth: 640,
webcamHeight: 360, webcamHeight: 360,
webcamFps: 30, webcamFps: 30,
outputs: { screenPath: outputPath }, outputs: {
screenPath: outputPath,
...(webcamOutputPath ? { webcamPath: webcamOutputPath } : {}),
},
}; };
let result; let result;
@@ -289,8 +293,13 @@ if (result.code !== 0) {
if (!fs.existsSync(outputPath) || fs.statSync(outputPath).size === 0) { if (!fs.existsSync(outputPath) || fs.statSync(outputPath).size === 0) {
throw new Error(`WGC helper did not produce a video at ${outputPath}`); throw new Error(`WGC helper did not produce a video at ${outputPath}`);
} }
if (WITH_WEBCAM && (!fs.existsSync(webcamOutputPath) || fs.statSync(webcamOutputPath).size === 0)) {
throw new Error(`WGC helper did not produce a webcam video at ${webcamOutputPath}`);
}
const streams = probeStreams(outputPath); const streams = probeStreams(outputPath);
const webcamStreams =
webcamOutputPath && fs.existsSync(webcamOutputPath) ? probeStreams(webcamOutputPath) : [];
const hasVideo = streams.some((stream) => stream.codec_type === "video"); const hasVideo = streams.some((stream) => stream.codec_type === "video");
const hasAudio = streams.some((stream) => stream.codec_type === "audio"); const hasAudio = streams.some((stream) => stream.codec_type === "audio");
const webcamFormatLine = result.stdout const webcamFormatLine = result.stdout
@@ -318,6 +327,9 @@ const nativeMicrophoneDiagnostics = result.stderr
if (!hasVideo) { if (!hasVideo) {
throw new Error(`WGC helper output has no video stream: ${outputPath}`); throw new Error(`WGC helper output has no video stream: ${outputPath}`);
} }
if (WITH_WEBCAM && !webcamStreams.some((stream) => stream.codec_type === "video")) {
throw new Error(`WGC helper webcam output has no video stream: ${webcamOutputPath}`);
}
if ( if (
(CAPTURE_CURSOR && !cursorCapture) || (CAPTURE_CURSOR && !cursorCapture) ||
(cursorCapture && (cursorCapture &&
@@ -342,13 +354,26 @@ console.log(
{ {
success: true, success: true,
outputPath, outputPath,
webcamOutputPath,
bytes: fs.statSync(outputPath).size, bytes: fs.statSync(outputPath).size,
webcamBytes:
webcamOutputPath && fs.existsSync(webcamOutputPath)
? fs.statSync(webcamOutputPath).size
: undefined,
streams: streams.map((stream) => ({ streams: streams.map((stream) => ({
index: stream.index, index: stream.index,
codecType: stream.codec_type, codecType: stream.codec_type,
codecName: stream.codec_name, codecName: stream.codec_name,
duration: stream.duration, duration: stream.duration,
})), })),
webcamStreams: webcamStreams.map((stream) => ({
index: stream.index,
codecType: stream.codec_type,
codecName: stream.codec_name,
width: stream.width,
height: stream.height,
duration: stream.duration,
})),
cursorCapture, cursorCapture,
selectedMicrophoneDeviceName: audioFormat?.microphoneDeviceName, selectedMicrophoneDeviceName: audioFormat?.microphoneDeviceName,
selectedWebcamDeviceName: webcamFormat?.deviceName, selectedWebcamDeviceName: webcamFormat?.deviceName,
+17 -16
View File
@@ -53,6 +53,7 @@ import ColorPicker from "../ui/color-picker";
import { AnnotationSettingsPanel } from "./AnnotationSettingsPanel"; import { AnnotationSettingsPanel } from "./AnnotationSettingsPanel";
import { BlurSettingsPanel } from "./BlurSettingsPanel"; import { BlurSettingsPanel } from "./BlurSettingsPanel";
import { CropControl } from "./CropControl"; import { CropControl } from "./CropControl";
import { parseCustomPlaybackSpeedInput } from "./customPlaybackSpeed";
import { KeyboardShortcutsHelp } from "./KeyboardShortcutsHelp"; import { KeyboardShortcutsHelp } from "./KeyboardShortcutsHelp";
import type { import type {
AnnotationRegion, AnnotationRegion,
@@ -71,7 +72,6 @@ import type {
} from "./types"; } from "./types";
import { import {
DEFAULT_WEBCAM_SIZE_PRESET, DEFAULT_WEBCAM_SIZE_PRESET,
MAX_PLAYBACK_SPEED,
MAX_ZOOM_SCALE, MAX_ZOOM_SCALE,
MIN_ZOOM_SCALE, MIN_ZOOM_SCALE,
ROTATION_3D_PRESET_ORDER, ROTATION_3D_PRESET_ORDER,
@@ -90,37 +90,38 @@ function CustomSpeedInput({
onError: () => void; onError: () => void;
}) { }) {
const isPreset = SPEED_OPTIONS.some((o) => o.speed === value); const isPreset = SPEED_OPTIONS.some((o) => o.speed === value);
const [draft, setDraft] = useState(isPreset ? "" : String(Math.round(value))); const [draft, setDraft] = useState(isPreset ? "" : String(value));
const [isFocused, setIsFocused] = useState(false); const [isFocused, setIsFocused] = useState(false);
const prevValue = useRef(value); const prevValue = useRef(value);
if (!isFocused && prevValue.current !== value) { if (!isFocused && prevValue.current !== value) {
prevValue.current = value; prevValue.current = value;
setDraft(isPreset ? "" : String(Math.round(value))); setDraft(isPreset ? "" : String(value));
} }
const handleChange = useCallback( const handleChange = useCallback(
(e: React.ChangeEvent<HTMLInputElement>) => { (e: React.ChangeEvent<HTMLInputElement>) => {
const digits = e.target.value.replace(/\D/g, ""); const result = parseCustomPlaybackSpeedInput(e.target.value);
if (digits === "") { if (result.status === "too-fast") {
setDraft("");
return;
}
const num = Number(digits);
if (num > MAX_PLAYBACK_SPEED) {
onError(); onError();
return; return;
} }
setDraft(digits);
if (num >= 1) onChange(num); setDraft(result.draft);
if (result.status === "valid") {
onChange(result.speed);
}
}, },
[onChange, onError], [onChange, onError],
); );
const handleBlur = useCallback(() => { const handleBlur = useCallback(() => {
setIsFocused(false); setIsFocused(false);
if (!draft || Number(draft) < 1) { const result = parseCustomPlaybackSpeedInput(draft);
setDraft(isPreset ? "" : String(Math.round(value))); if (result.status === "valid") {
setDraft(String(result.speed));
} else {
setDraft(isPreset ? "" : String(value));
} }
}, [draft, isPreset, value]); }, [draft, isPreset, value]);
@@ -128,8 +129,8 @@ function CustomSpeedInput({
<div className="flex items-center gap-1"> <div className="flex items-center gap-1">
<input <input
type="text" type="text"
inputMode="numeric" inputMode="decimal"
pattern="[0-9]*" pattern="[0-9]*[.]?[0-9]*"
placeholder="--" placeholder="--"
value={draft} value={draft}
onFocus={() => setIsFocused(true)} onFocus={() => setIsFocused(true)}
@@ -0,0 +1,50 @@
import { describe, expect, it } from "vitest";
import { parseCustomPlaybackSpeedInput } from "./customPlaybackSpeed";
describe("parseCustomPlaybackSpeedInput", () => {
it("accepts decimal playback speeds", () => {
expect(parseCustomPlaybackSpeedInput("1.1")).toEqual({
status: "valid",
draft: "1.1",
speed: 1.1,
});
});
it("keeps a single decimal point while typing", () => {
expect(parseCustomPlaybackSpeedInput("1.2.3")).toEqual({
status: "valid",
draft: "1.23",
speed: 1.23,
});
});
it("allows sub-1 custom speeds down to the editor minimum", () => {
expect(parseCustomPlaybackSpeedInput("0.1")).toEqual({
status: "valid",
draft: "0.1",
speed: 0.1,
});
});
it("rejects speeds below the editor minimum", () => {
expect(parseCustomPlaybackSpeedInput("0.09")).toEqual({
status: "too-slow",
draft: "0.09",
});
});
it("accepts comma decimal input by normalizing to a dot", () => {
expect(parseCustomPlaybackSpeedInput("1,1")).toEqual({
status: "valid",
draft: "1.1",
speed: 1.1,
});
});
it("rejects speeds above the editor maximum", () => {
expect(parseCustomPlaybackSpeedInput("16.1")).toEqual({
status: "too-fast",
draft: "16.1",
});
});
});
@@ -0,0 +1,37 @@
import {
clampPlaybackSpeed,
MAX_PLAYBACK_SPEED,
MIN_PLAYBACK_SPEED,
type PlaybackSpeed,
} from "./types";
export type CustomPlaybackSpeedInputResult =
| { status: "empty"; draft: string }
| { status: "too-fast"; draft: string }
| { status: "too-slow"; draft: string }
| { status: "valid"; draft: string; speed: PlaybackSpeed };
export function parseCustomPlaybackSpeedInput(rawValue: string): CustomPlaybackSpeedInputResult {
const decimalDraft = rawValue.replace(/,/g, ".").replace(/[^\d.]/g, "");
const [whole = "", ...fractionParts] = decimalDraft.split(".");
const draft = fractionParts.length > 0 ? `${whole}.${fractionParts.join("")}` : whole;
if (draft === "" || draft === ".") {
return { status: "empty", draft };
}
const speed = Number(draft);
if (!Number.isFinite(speed)) {
return { status: "empty", draft };
}
if (speed > MAX_PLAYBACK_SPEED) {
return { status: "too-fast", draft };
}
if (speed < MIN_PLAYBACK_SPEED) {
return { status: "too-slow", draft };
}
return { status: "valid", draft, speed: clampPlaybackSpeed(speed) };
}
+146 -64
View File
@@ -45,8 +45,6 @@ const AUDIO_BITRATE_VOICE = 128_000;
const AUDIO_BITRATE_SYSTEM = 192_000; const AUDIO_BITRATE_SYSTEM = 192_000;
const MIC_GAIN_BOOST = 1.4; const MIC_GAIN_BOOST = 1.4;
const WEBCAM_TARGET_WIDTH = 1280;
const WEBCAM_TARGET_HEIGHT = 720;
const WEBCAM_TARGET_FRAME_RATE = 30; const WEBCAM_TARGET_FRAME_RATE = 30;
type UseScreenRecorderReturn = { type UseScreenRecorderReturn = {
@@ -85,6 +83,7 @@ type NativeWindowsRecordingHandle = {
recordingId: number; recordingId: number;
finalizing: boolean; finalizing: boolean;
paused: boolean; paused: boolean;
webcamRecorder: RecorderHandle | null;
}; };
type NativeMacRecordingHandle = { type NativeMacRecordingHandle = {
@@ -268,13 +267,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
video: webcamDeviceId video: webcamDeviceId
? { ? {
deviceId: { exact: webcamDeviceId }, deviceId: { exact: webcamDeviceId },
width: { ideal: WEBCAM_TARGET_WIDTH },
height: { ideal: WEBCAM_TARGET_HEIGHT },
frameRate: { ideal: WEBCAM_TARGET_FRAME_RATE, max: WEBCAM_TARGET_FRAME_RATE }, frameRate: { ideal: WEBCAM_TARGET_FRAME_RATE, max: WEBCAM_TARGET_FRAME_RATE },
} }
: { : {
width: { ideal: WEBCAM_TARGET_WIDTH },
height: { ideal: WEBCAM_TARGET_HEIGHT },
frameRate: { ideal: WEBCAM_TARGET_FRAME_RATE, max: WEBCAM_TARGET_FRAME_RATE }, frameRate: { ideal: WEBCAM_TARGET_FRAME_RATE, max: WEBCAM_TARGET_FRAME_RATE },
}, },
}); });
@@ -423,58 +418,105 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
[cursorCaptureMode, teardownMedia], [cursorCaptureMode, teardownMedia],
); );
const finalizeNativeWindowsRecording = useCallback(async (discard = false) => { const finalizeNativeWindowsRecording = useCallback(
const activeNativeRecording = nativeWindowsRecording.current; async (discard = false) => {
if (!activeNativeRecording || activeNativeRecording.finalizing) { const activeNativeRecording = nativeWindowsRecording.current;
return false; if (!activeNativeRecording || activeNativeRecording.finalizing) {
} return false;
activeNativeRecording.finalizing = true;
const clearNativeRecordingState = () => {
nativeWindowsRecording.current = null;
setRecording(false);
setPaused(false);
setElapsedSeconds(0);
accumulatedDurationMs.current = 0;
segmentStartedAt.current = null;
};
try {
const result = await window.electronAPI.stopNativeWindowsRecording(discard);
if (discard || result.discarded) {
clearNativeRecordingState();
return true;
} }
if (!result.success) {
console.error("Failed to stop native Windows recording:", result.error); activeNativeRecording.finalizing = true;
toast.error(result.error ?? "Failed to stop native Windows recording"); const activeWebcamRecorder = activeNativeRecording.webcamRecorder;
const duration = Math.max(0, getRecordingDurationMs());
if (
activeWebcamRecorder?.recorder.state === "recording" ||
activeWebcamRecorder?.recorder.state === "paused"
) {
try {
activeWebcamRecorder.recorder.stop();
} catch {
// Recorder may already be stopping.
}
}
if (activeWebcamRecorder && webcamRecorder.current === activeWebcamRecorder) {
webcamRecorder.current = null;
}
const clearNativeRecordingState = () => {
nativeWindowsRecording.current = null;
setRecording(false);
setPaused(false);
setElapsedSeconds(0);
accumulatedDurationMs.current = 0;
segmentStartedAt.current = null;
};
try {
const result = await window.electronAPI.stopNativeWindowsRecording(discard);
if (discard || result.discarded) {
clearNativeRecordingState();
return true;
}
if (!result.success) {
console.error("Failed to stop native Windows recording:", result.error);
toast.error(result.error ?? "Failed to stop native Windows recording");
activeNativeRecording.finalizing = false;
return true;
}
const nativeScreenPath = result.session?.screenVideoPath ?? result.path;
let storedSession = result.session;
if (activeWebcamRecorder && nativeScreenPath) {
const webcamBlob = await activeWebcamRecorder.recordedBlobPromise.catch(() => null);
const screenRead = await window.electronAPI.readBinaryFile(nativeScreenPath);
if (webcamBlob && webcamBlob.size > 0 && screenRead.success && screenRead.data) {
const fixedWebcamBlob = await fixWebmDuration(webcamBlob, duration);
const nativeScreenFileName =
nativeScreenPath.split(/[\\/]/).pop() ??
`${RECORDING_FILE_PREFIX}${activeNativeRecording.recordingId}.mp4`;
const webcamFileName = `${RECORDING_FILE_PREFIX}${activeNativeRecording.recordingId}${WEBCAM_FILE_SUFFIX}${VIDEO_FILE_EXTENSION}`;
const stored = await window.electronAPI.storeRecordedSession({
screen: {
videoData: screenRead.data,
fileName: nativeScreenFileName,
},
webcam: {
videoData: await fixedWebcamBlob.arrayBuffer(),
fileName: webcamFileName,
},
createdAt: activeNativeRecording.recordingId,
cursorCaptureMode,
});
if (stored.success && stored.session) {
storedSession = stored.session;
}
}
}
clearNativeRecordingState();
if (storedSession) {
await window.electronAPI.setCurrentRecordingSession(storedSession);
} else if (result.path) {
await window.electronAPI.setCurrentVideoPath(result.path);
}
await window.electronAPI.switchToEditor();
return true;
} catch (error) {
console.error("Error saving native Windows recording:", error);
toast.error(
error instanceof Error ? error.message : "Failed to save native Windows recording",
);
activeNativeRecording.finalizing = false; activeNativeRecording.finalizing = false;
return true; return true;
} finally {
if (discardRecordingId.current === activeNativeRecording.recordingId) {
discardRecordingId.current = null;
}
} }
},
clearNativeRecordingState(); [cursorCaptureMode, getRecordingDurationMs],
if (result.session) { );
await window.electronAPI.setCurrentRecordingSession(result.session);
} else if (result.path) {
await window.electronAPI.setCurrentVideoPath(result.path);
}
await window.electronAPI.switchToEditor();
return true;
} catch (error) {
console.error("Error saving native Windows recording:", error);
toast.error(
error instanceof Error ? error.message : "Failed to save native Windows recording",
);
activeNativeRecording.finalizing = false;
return true;
} finally {
if (discardRecordingId.current === activeNativeRecording.recordingId) {
discardRecordingId.current = null;
}
}
}, []);
const finalizeNativeMacRecording = useCallback( const finalizeNativeMacRecording = useCallback(
async (discard = false) => { async (discard = false) => {
@@ -717,6 +759,25 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
const isCountdownRunActive = (runId?: number) => const isCountdownRunActive = (runId?: number) =>
runId === undefined || countdownRunId.current === runId; runId === undefined || countdownRunId.current === runId;
const waitForWebcamReady = async () => {
if (webcamReady.current) {
return;
}
await new Promise<void>((resolve) => {
const interval = setInterval(() => {
if (webcamReady.current) {
clearInterval(interval);
resolve();
}
}, 50);
setTimeout(() => {
clearInterval(interval);
resolve();
}, 5000);
});
};
const startNativeWindowsRecordingIfAvailable = async ( const startNativeWindowsRecordingIfAvailable = async (
selectedSource: ProcessedDesktopSource, selectedSource: ProcessedDesktopSource,
countdownRunToken?: number, countdownRunToken?: number,
@@ -732,12 +793,12 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
if (availability.reason === "unsupported-os") { if (availability.reason === "unsupported-os") {
return false; return false;
} }
if (availability.reason === "missing-helper") {
console.warn("Native Windows capture helper is not available; using browser capture.");
return false;
}
throw new Error( throw new Error(availability.error ?? "Native Windows capture is not available.");
availability.reason === "missing-helper"
? "Native Windows capture helper is not available."
: (availability.error ?? "Native Windows capture is not available."),
);
} }
if (!isCountdownRunActive(countdownRunToken)) { if (!isCountdownRunActive(countdownRunToken)) {
@@ -749,6 +810,19 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
const sourceType = selectedSource.id.startsWith("window:") ? "window" : "display"; const sourceType = selectedSource.id.startsWith("window:") ? "window" : "display";
const windowHandle = parseWindowHandleFromSourceId(selectedSource.id); const windowHandle = parseWindowHandleFromSourceId(selectedSource.id);
if (webcamEnabled) { if (webcamEnabled) {
await waitForWebcamReady();
if (!isCountdownRunActive(countdownRunToken)) {
return true;
}
}
const browserWebcamRecorder =
webcamEnabled && webcamStream.current
? createRecorderHandle(webcamStream.current, {
mimeType: selectMimeType(),
videoBitsPerSecond: BITRATE_BASE,
})
: null;
if (webcamEnabled && !browserWebcamRecorder) {
stopWebcamPreviewStream(); stopWebcamPreviewStream();
} }
const request: NativeWindowsRecordingRequest = { const request: NativeWindowsRecordingRequest = {
@@ -776,11 +850,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
}, },
}, },
webcam: { webcam: {
enabled: webcamEnabled, enabled: webcamEnabled && !browserWebcamRecorder,
deviceId: webcamDeviceId, deviceId: webcamDeviceId,
deviceName: webcamDeviceName, deviceName: webcamDeviceName,
width: WEBCAM_TARGET_WIDTH, width: 0,
height: WEBCAM_TARGET_HEIGHT, height: 0,
fps: WEBCAM_TARGET_FRAME_RATE, fps: WEBCAM_TARGET_FRAME_RATE,
}, },
cursor: { cursor: {
@@ -789,6 +863,12 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
}; };
const result = await window.electronAPI.startNativeWindowsRecording(request); const result = await window.electronAPI.startNativeWindowsRecording(request);
if (!result.success || !result.recordingId) { if (!result.success || !result.recordingId) {
if (
browserWebcamRecorder?.recorder.state === "recording" ||
browserWebcamRecorder?.recorder.state === "paused"
) {
browserWebcamRecorder.recorder.stop();
}
throw new Error(result.error ?? "Native Windows capture failed."); throw new Error(result.error ?? "Native Windows capture failed.");
} }
@@ -797,7 +877,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
recordingId: result.recordingId, recordingId: result.recordingId,
finalizing: false, finalizing: false,
paused: false, paused: false,
webcamRecorder: browserWebcamRecorder,
}; };
webcamRecorder.current = browserWebcamRecorder;
accumulatedDurationMs.current = 0; accumulatedDurationMs.current = 0;
segmentStartedAt.current = Date.now(); segmentStartedAt.current = Date.now();
allowAutoFinalize.current = true; allowAutoFinalize.current = true;
@@ -909,8 +991,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
enabled: webcamEnabled, enabled: webcamEnabled,
deviceId: webcamDeviceId, deviceId: webcamDeviceId,
deviceName: webcamDeviceName, deviceName: webcamDeviceName,
width: WEBCAM_TARGET_WIDTH, width: 0,
height: WEBCAM_TARGET_HEIGHT, height: 0,
fps: WEBCAM_TARGET_FRAME_RATE, fps: WEBCAM_TARGET_FRAME_RATE,
}, },
cursor: { cursor: {
+2 -1
View File
@@ -104,8 +104,9 @@
"gifButton": "Esporta GIF", "gifButton": "Esporta GIF",
"chooseSaveLocation": "Scegli posizione di salvataggio" "chooseSaveLocation": "Scegli posizione di salvataggio"
}, },
"links": { "support": {
"reportBug": "Segnala bug", "reportBug": "Segnala bug",
"saveDiagnostics": "Salva dati diagnostici",
"starOnGithub": "Metti stella su GitHub" "starOnGithub": "Metti stella su GitHub"
}, },
"imageUpload": { "imageUpload": {
+8 -6
View File
@@ -11,9 +11,9 @@ import type {
import { BackgroundLoadError } from "@/lib/wallpaper"; import { BackgroundLoadError } from "@/lib/wallpaper";
import type { CursorRecordingData } from "@/native/contracts"; import type { CursorRecordingData } from "@/native/contracts";
import { getPlatform } from "@/utils/platformUtils"; import { getPlatform } from "@/utils/platformUtils";
import { AsyncVideoFrameQueue } from "./asyncVideoFrameQueue";
import { FrameRenderer } from "./frameRenderer"; import { FrameRenderer } from "./frameRenderer";
import { StreamingVideoDecoder } from "./streamingDecoder"; import { StreamingVideoDecoder } from "./streamingDecoder";
import { TimestampedVideoFrameQueue } from "./timestampedVideoFrameQueue";
import type { import type {
ExportProgress, ExportProgress,
ExportResult, ExportResult,
@@ -124,7 +124,7 @@ export class GifExporter {
} }
async export(): Promise<ExportResult> { async export(): Promise<ExportResult> {
let webcamFrameQueue: AsyncVideoFrameQueue | null = null; let webcamFrameQueue: TimestampedVideoFrameQueue | null = null;
const warnings: string[] = []; const warnings: string[] = [];
const onWarning = (message: string) => warnings.push(message); const onWarning = (message: string) => warnings.push(message);
@@ -216,7 +216,7 @@ export class GifExporter {
console.log("[GifExporter] Using streaming decode (web-demuxer + VideoDecoder)"); console.log("[GifExporter] Using streaming decode (web-demuxer + VideoDecoder)");
let frameIndex = 0; let frameIndex = 0;
webcamFrameQueue = this.config.webcamVideoUrl ? new AsyncVideoFrameQueue() : null; webcamFrameQueue = this.config.webcamVideoUrl ? new TimestampedVideoFrameQueue() : null;
let stopWebcamDecode = false; let stopWebcamDecode = false;
let webcamDecodeError: Error | null = null; let webcamDecodeError: Error | null = null;
const webcamDecodePromise = const webcamDecodePromise =
@@ -228,7 +228,7 @@ export class GifExporter {
this.config.frameRate, this.config.frameRate,
this.config.trimRegions, this.config.trimRegions,
this.config.speedRegions, this.config.speedRegions,
async (webcamFrame) => { async (webcamFrame, _exportTimestampUs, webcamSourceTimestampMs) => {
while (queue.length >= 12 && !this.cancelled && !stopWebcamDecode) { while (queue.length >= 12 && !this.cancelled && !stopWebcamDecode) {
await new Promise((resolve) => setTimeout(resolve, 2)); await new Promise((resolve) => setTimeout(resolve, 2));
} }
@@ -236,7 +236,7 @@ export class GifExporter {
webcamFrame.close(); webcamFrame.close();
return; return;
} }
queue.enqueue(webcamFrame); queue.enqueue(webcamFrame, webcamSourceTimestampMs);
}, },
onWarning, onWarning,
) )
@@ -266,7 +266,9 @@ export class GifExporter {
return; return;
} }
webcamFrame = webcamFrameQueue ? await webcamFrameQueue.dequeue() : null; webcamFrame = webcamFrameQueue
? await webcamFrameQueue.frameAt(sourceTimestampMs)
: null;
const renderer = this.renderer; const renderer = this.renderer;
if (this.cancelled || !renderer) { if (this.cancelled || !renderer) {
return; return;
@@ -0,0 +1,93 @@
import { describe, expect, it, vi } from "vitest";
import { TimestampedVideoFrameQueue } from "./timestampedVideoFrameQueue";
class MockVideoFrame {
timestamp: number;
closed = false;
constructor(source: MockVideoFrame | number) {
this.timestamp = typeof source === "number" ? source : source.timestamp;
}
close() {
this.closed = true;
}
}
function restoreVideoFrame(originalVideoFrame: typeof globalThis.VideoFrame | undefined) {
if (originalVideoFrame === undefined) {
delete (globalThis as { VideoFrame?: typeof globalThis.VideoFrame }).VideoFrame;
return;
}
vi.stubGlobal("VideoFrame", originalVideoFrame);
}
describe("TimestampedVideoFrameQueue", () => {
it("samples the latest webcam frame at or before the requested source timestamp", async () => {
const originalVideoFrame = globalThis.VideoFrame;
vi.stubGlobal("VideoFrame", MockVideoFrame);
try {
const queue = new TimestampedVideoFrameQueue();
const frame0 = new MockVideoFrame(0) as unknown as VideoFrame;
const frame33 = new MockVideoFrame(33_000) as unknown as VideoFrame;
const frame66 = new MockVideoFrame(66_000) as unknown as VideoFrame;
queue.enqueue(frame0, 0);
queue.enqueue(frame33, 33);
queue.enqueue(frame66, 66);
queue.close();
const sampled0 = await queue.frameAt(0);
const sampled20 = await queue.frameAt(20);
const sampled40 = await queue.frameAt(40);
const sampled80 = await queue.frameAt(80);
expect(sampled0?.timestamp).toBe(0);
expect(sampled20?.timestamp).toBe(0);
expect(sampled40?.timestamp).toBe(33_000);
expect(sampled80?.timestamp).toBe(66_000);
sampled0?.close();
sampled20?.close();
sampled40?.close();
sampled80?.close();
queue.destroy();
} finally {
restoreVideoFrame(originalVideoFrame);
}
});
it("waits for a newer frame before falling back to the held frame while open", async () => {
const originalVideoFrame = globalThis.VideoFrame;
vi.stubGlobal("VideoFrame", MockVideoFrame);
try {
const queue = new TimestampedVideoFrameQueue();
const frame0 = new MockVideoFrame(0) as unknown as VideoFrame;
const frame33 = new MockVideoFrame(33_000) as unknown as VideoFrame;
queue.enqueue(frame0, 0);
const sampled0 = await queue.frameAt(0);
let resolved = false;
const pending = queue.frameAt(33).then((frame) => {
resolved = true;
return frame;
});
await Promise.resolve();
expect(resolved).toBe(false);
queue.enqueue(frame33, 33);
const sampled33 = await pending;
expect(sampled0?.timestamp).toBe(0);
expect(sampled33?.timestamp).toBe(33_000);
sampled0?.close();
sampled33?.close();
queue.destroy();
} finally {
restoreVideoFrame(originalVideoFrame);
}
});
});
@@ -0,0 +1,110 @@
type TimestampedVideoFrame = {
frame: VideoFrame;
sourceTimestampMs: number;
};
type PendingConsumer = {
resolve: () => void;
reject: (error: Error) => void;
};
const TIMESTAMP_EPSILON_MS = 0.5;
export class TimestampedVideoFrameQueue {
private frames: TimestampedVideoFrame[] = [];
private consumers: PendingConsumer[] = [];
private error: Error | null = null;
private closed = false;
private heldFrame: TimestampedVideoFrame | null = null;
get length() {
return this.frames.length;
}
enqueue(frame: VideoFrame, sourceTimestampMs: number) {
if (this.closed) {
frame.close();
return;
}
this.frames.push({ frame, sourceTimestampMs });
const consumers = this.consumers.splice(0);
for (const consumer of consumers) {
consumer.resolve();
}
}
fail(error: Error) {
this.error = error;
this.closed = true;
const consumers = this.consumers.splice(0);
for (const consumer of consumers) {
consumer.reject(error);
}
this.closeOwnedFrames();
}
close() {
this.closed = true;
const consumers = this.consumers.splice(0);
for (const consumer of consumers) {
consumer.resolve();
}
}
async frameAt(sourceTimestampMs: number): Promise<VideoFrame | null> {
for (;;) {
if (this.error) {
throw this.error;
}
const next = this.frames[0] ?? null;
if (next && next.sourceTimestampMs <= sourceTimestampMs + TIMESTAMP_EPSILON_MS) {
this.replaceHeldFrame(this.frames.shift() ?? null);
continue;
}
if (
this.heldFrame &&
(next ||
this.closed ||
this.heldFrame.sourceTimestampMs >= sourceTimestampMs - TIMESTAMP_EPSILON_MS)
) {
return new VideoFrame(this.heldFrame.frame, {
timestamp: this.heldFrame.frame.timestamp,
});
}
if (next || this.closed) {
return null;
}
await new Promise<void>((resolve, reject) => {
this.consumers.push({ resolve, reject });
});
}
}
destroy() {
this.close();
this.closeOwnedFrames();
}
private replaceHeldFrame(frame: TimestampedVideoFrame | null) {
if (this.heldFrame) {
this.heldFrame.frame.close();
}
this.heldFrame = frame;
}
private closeOwnedFrames() {
if (this.heldFrame) {
this.heldFrame.frame.close();
this.heldFrame = null;
}
for (const item of this.frames) {
item.frame.close();
}
this.frames = [];
}
}
+8 -6
View File
@@ -10,11 +10,11 @@ import type {
import { BackgroundLoadError } from "@/lib/wallpaper"; import { BackgroundLoadError } from "@/lib/wallpaper";
import type { CursorRecordingData } from "@/native/contracts"; import type { CursorRecordingData } from "@/native/contracts";
import { getPlatform } from "@/utils/platformUtils"; import { getPlatform } from "@/utils/platformUtils";
import { AsyncVideoFrameQueue } from "./asyncVideoFrameQueue";
import { AudioProcessor } from "./audioEncoder"; import { AudioProcessor } from "./audioEncoder";
import { FrameRenderer } from "./frameRenderer"; import { FrameRenderer } from "./frameRenderer";
import { VideoMuxer } from "./muxer"; import { VideoMuxer } from "./muxer";
import { StreamingVideoDecoder } from "./streamingDecoder"; import { StreamingVideoDecoder } from "./streamingDecoder";
import { TimestampedVideoFrameQueue } from "./timestampedVideoFrameQueue";
import type { ExportConfig, ExportProgress, ExportResult } from "./types"; import type { ExportConfig, ExportProgress, ExportResult } from "./types";
const ENCODER_STALL_TIMEOUT_MS = 15_000; const ENCODER_STALL_TIMEOUT_MS = 15_000;
@@ -195,7 +195,7 @@ export class VideoExporter {
private async exportWithEncoderPreference( private async exportWithEncoderPreference(
encoderPreference: HardwareAcceleration, encoderPreference: HardwareAcceleration,
): Promise<ExportResult> { ): Promise<ExportResult> {
let webcamFrameQueue: AsyncVideoFrameQueue | null = null; let webcamFrameQueue: TimestampedVideoFrameQueue | null = null;
let stopWebcamDecode = false; let stopWebcamDecode = false;
let webcamDecodeError: Error | null = null; let webcamDecodeError: Error | null = null;
let webcamDecodePromise: Promise<void> | null = null; let webcamDecodePromise: Promise<void> | null = null;
@@ -290,7 +290,7 @@ export class VideoExporter {
? Math.min(this.MAX_ENCODE_QUEUE, 32) ? Math.min(this.MAX_ENCODE_QUEUE, 32)
: this.MAX_ENCODE_QUEUE; : this.MAX_ENCODE_QUEUE;
webcamFrameQueue = this.config.webcamVideoUrl ? new AsyncVideoFrameQueue() : null; webcamFrameQueue = this.config.webcamVideoUrl ? new TimestampedVideoFrameQueue() : null;
webcamDecodePromise = webcamDecodePromise =
webcamDecoder && webcamFrameQueue webcamDecoder && webcamFrameQueue
? (() => { ? (() => {
@@ -300,7 +300,7 @@ export class VideoExporter {
this.config.frameRate, this.config.frameRate,
this.config.trimRegions, this.config.trimRegions,
this.config.speedRegions, this.config.speedRegions,
async (webcamFrame) => { async (webcamFrame, _exportTimestampUs, webcamSourceTimestampMs) => {
while (queue.length >= 12 && !this.cancelled && !stopWebcamDecode) { while (queue.length >= 12 && !this.cancelled && !stopWebcamDecode) {
await new Promise((resolve) => setTimeout(resolve, 2)); await new Promise((resolve) => setTimeout(resolve, 2));
} }
@@ -308,7 +308,7 @@ export class VideoExporter {
webcamFrame.close(); webcamFrame.close();
return; return;
} }
queue.enqueue(webcamFrame); queue.enqueue(webcamFrame, webcamSourceTimestampMs);
}, },
onWarning, onWarning,
) )
@@ -342,7 +342,9 @@ export class VideoExporter {
} }
const timestamp = frameIndex * frameDuration; const timestamp = frameIndex * frameDuration;
webcamFrame = webcamFrameQueue ? await webcamFrameQueue.dequeue() : null; webcamFrame = webcamFrameQueue
? await webcamFrameQueue.frameAt(sourceTimestampMs)
: null;
if (this.cancelled) { if (this.cancelled) {
return; return;
} }