Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ playwright-report/

# Vitest browser mode screenshots
__screenshots__/
.vitest-attachments/

# shell files
/shell.sh
Expand Down
57 changes: 46 additions & 11 deletions electron/native/wgc-capture/src/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -606,9 +606,14 @@ int main(int argc, char* argv[]) {
int64_t lastEncodedVideoTimestampHns = -1;

while (!control.stopRequested && !encodeFailed) {
Microsoft::WRL::ComPtr<IMFSample> videoSample;
Microsoft::WRL::ComPtr<IMFSample> webcamSample;
bool hasVideoSample = false;
bool hasWebcamSample = false;

{
std::unique_lock lock(mutex);
control.cv.wait(lock, [&] {
control.cv.wait_for(lock, std::chrono::milliseconds(100), [&] {
return control.stopRequested.load() ||
encodeFailed.load() ||
(!control.paused.load() && latestFrameTexture);
Expand Down Expand Up @@ -653,29 +658,59 @@ int main(int argc, char* argv[]) {
latestWebcamSequence != lastWrittenWebcamSequence) {
const int64_t webcamTimestampHns = static_cast<int64_t>(
(webcamOutputFrameIndex * 10'000'000ULL) / std::max(1, webcamCapture.fps()));
if (!webcamEncoder.writeBgraFrame(webcamFrame, webcamTimestampHns)) {
hasWebcamSample = webcamEncoder.captureBgraSample(webcamFrame, webcamTimestampHns, webcamSample);
if (!hasWebcamSample) {
encodeFailed = true;
control.stopRequested = true;
control.cv.notify_all();
return;
break;
}
lastWrittenWebcamSequence = latestWebcamSequence;
webcamOutputFrameIndex += 1;
}
if (latestFrameTexture && !encoder.writeFrame(
if (latestFrameTexture) {
// captureVideoSample performs the GPU readback
// (CopyResource/Map) from latestFrameTexture, which must
// stay serialized (via `mutex`) against the WGC
// frame-arrival callback above, which writes new data
// into the same texture on another thread.
hasVideoSample = encoder.captureVideoSample(
latestFrameTexture.Get(),
frameTimestampHns,
!writeSeparateWebcam && webcamFrame.data ? &webcamFrame : nullptr)) {
encodeFailed = true;
control.stopRequested = true;
control.cv.notify_all();
return;
}
if (latestFrameTexture) {
!writeSeparateWebcam && webcamFrame.data ? &webcamFrame : nullptr,
videoSample);
if (!hasVideoSample) {
encodeFailed = true;
control.stopRequested = true;
control.cv.notify_all();
break;
}
lastEncodedVideoTimestampHns = frameTimestampHns;
}
}

// Submit the captured samples to their sink writers OUTSIDE
// `mutex`. IMFSinkWriter::WriteSample runs the H.264 encode
// synchronously and can be slow (especially the software encoder
// fallback used when preferSoftwareEncoder is set). Holding
// `mutex` across it would block the main thread's stop-wait
// (which locks the same mutex to check control.stopRequested)
// for as long as this thread keeps re-acquiring the lock faster
// than the main thread can, hanging the helper indefinitely
// after a stop request (issue #115).
if (hasWebcamSample && !webcamEncoder.submitVideoSample(webcamSample.Get())) {
encodeFailed = true;
control.stopRequested = true;
control.cv.notify_all();
break;
}
if (hasVideoSample && !encoder.submitVideoSample(videoSample.Get())) {
encodeFailed = true;
control.stopRequested = true;
control.cv.notify_all();
break;
}

frameIndex += 1;
std::this_thread::sleep_for(frameDuration);
}
Expand Down
98 changes: 70 additions & 28 deletions electron/native/wgc-capture/src/mf_encoder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -603,23 +603,38 @@ bool MFEncoder::copyBgraFrameToBuffer(const BgraFrameView& frame, BYTE* destinat
return true;
}

bool MFEncoder::writeFrame(ID3D11Texture2D* texture, int64_t timestampHns, const BgraFrameView* webcamFrame) {
std::scoped_lock writerLock(writerMutex_);
if (!sinkWriter_ || finalized_) {
return false;
}
bool MFEncoder::captureVideoSample(
ID3D11Texture2D* texture,
int64_t timestampHns,
const BgraFrameView* webcamFrame,
Microsoft::WRL::ComPtr<IMFSample>& outSample) {
outSample.Reset();

if (firstTimestampHns_ < 0) {
firstTimestampHns_ = timestampHns;
}
const int64_t sampleDuration = 10'000'000LL / fps_;
int64_t sampleTime = 0;
{
std::scoped_lock writerLock(writerMutex_);
if (!sinkWriter_ || finalized_) {
return false;
}

int64_t sampleTime = timestampHns - firstTimestampHns_;
if (sampleTime <= lastTimestampHns_) {
sampleTime = lastTimestampHns_ + (10'000'000LL / fps_);
if (firstTimestampHns_ < 0) {
firstTimestampHns_ = timestampHns;
}

sampleTime = timestampHns - firstTimestampHns_;
if (sampleTime <= lastTimestampHns_) {
sampleTime = lastTimestampHns_ + sampleDuration;
}
lastTimestampHns_ = sampleTime;
}
const int64_t sampleDuration = 10'000'000LL / fps_;
lastTimestampHns_ = sampleTime;

// The GPU readback below (copyFrameToBuffer -> CopyResource/Map on
// `texture`) is not internally synchronized here. Callers must hold their
// own lock around this call that also serializes against whatever thread
// writes new frame data into `texture` (see main.cpp's writeVideoFrames,
// which holds the shared frame-state mutex across this call but not
// across submitVideoSample).
Microsoft::WRL::ComPtr<IMFMediaBuffer> buffer;
const DWORD frameBytes = static_cast<DWORD>(width_ * height_ * 4);
if (!succeeded(MFCreateMemoryBuffer(frameBytes, &buffer), "MFCreateMemoryBuffer")) {
Expand Down Expand Up @@ -648,25 +663,34 @@ bool MFEncoder::writeFrame(ID3D11Texture2D* texture, int64_t timestampHns, const
sample->SetSampleTime(sampleTime);
sample->SetSampleDuration(sampleDuration);

return succeeded(sinkWriter_->WriteSample(videoStreamIndex_, sample.Get()), "WriteSample");
outSample = sample;
return true;
}

bool MFEncoder::writeBgraFrame(const BgraFrameView& frame, int64_t timestampHns) {
std::scoped_lock writerLock(writerMutex_);
if (!sinkWriter_ || finalized_) {
return false;
}
bool MFEncoder::captureBgraSample(
const BgraFrameView& frame,
int64_t timestampHns,
Microsoft::WRL::ComPtr<IMFSample>& outSample) {
outSample.Reset();

if (firstTimestampHns_ < 0) {
firstTimestampHns_ = timestampHns;
}
const int64_t sampleDuration = 10'000'000LL / fps_;
int64_t sampleTime = 0;
{
std::scoped_lock writerLock(writerMutex_);
if (!sinkWriter_ || finalized_) {
return false;
}

int64_t sampleTime = timestampHns - firstTimestampHns_;
if (sampleTime <= lastTimestampHns_) {
sampleTime = lastTimestampHns_ + (10'000'000LL / fps_);
if (firstTimestampHns_ < 0) {
firstTimestampHns_ = timestampHns;
}

sampleTime = timestampHns - firstTimestampHns_;
if (sampleTime <= lastTimestampHns_) {
sampleTime = lastTimestampHns_ + sampleDuration;
}
lastTimestampHns_ = sampleTime;
}
const int64_t sampleDuration = 10'000'000LL / fps_;
lastTimestampHns_ = sampleTime;

Microsoft::WRL::ComPtr<IMFMediaBuffer> buffer;
const DWORD frameBytes = static_cast<DWORD>(width_ * height_ * 4);
Expand Down Expand Up @@ -696,7 +720,25 @@ bool MFEncoder::writeBgraFrame(const BgraFrameView& frame, int64_t timestampHns)
sample->SetSampleTime(sampleTime);
sample->SetSampleDuration(sampleDuration);

return succeeded(sinkWriter_->WriteSample(videoStreamIndex_, sample.Get()), "WriteSample(webcam)");
outSample = sample;
return true;
}

bool MFEncoder::submitVideoSample(IMFSample* sample) {
if (!sample) {
return false;
}

// This is the potentially slow, blocking step (especially with the
// software H.264 encoder fallback): IMFSinkWriter::WriteSample runs the
// encode synchronously on the calling thread. Callers must NOT hold any
// lock shared with a thread that needs to make timely progress (e.g. a
// stop-request check) across this call.
std::scoped_lock writerLock(writerMutex_);
if (!sinkWriter_ || finalized_) {
return false;
}
return succeeded(sinkWriter_->WriteSample(videoStreamIndex_, sample), "WriteSample");
}

bool MFEncoder::writeAudio(const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns) {
Expand Down
21 changes: 19 additions & 2 deletions electron/native/wgc-capture/src/mf_encoder.h
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,25 @@ class MFEncoder {
ID3D11DeviceContext* context,
const AudioInputFormat* audioFormat = nullptr,
MFEncoderOptions options = {});
bool writeFrame(ID3D11Texture2D* texture, int64_t timestampHns, const BgraFrameView* webcamFrame = nullptr);
bool writeBgraFrame(const BgraFrameView& frame, int64_t timestampHns);
// Capturing a video/webcam sample (GPU readback + IMFSample creation) is
// split from submitting it to the sink writer (IMFSinkWriter::WriteSample)
// so callers that hold an external lock across the GPU-touching capture
// step (to serialize against a producer thread writing into the same
// texture) are not forced to also hold that lock across the potentially
// slow, blocking WriteSample call. See main.cpp's writeVideoFrames for why
// this split exists: holding the shared frame-state mutex across
// WriteSample let the software H.264 encoder path starve the main
// thread's stop-request check indefinitely (issue #115).
bool captureVideoSample(
ID3D11Texture2D* texture,
int64_t timestampHns,
const BgraFrameView* webcamFrame,
Microsoft::WRL::ComPtr<IMFSample>& outSample);
bool captureBgraSample(
const BgraFrameView& frame,
int64_t timestampHns,
Microsoft::WRL::ComPtr<IMFSample>& outSample);
bool submitVideoSample(IMFSample* sample);
bool writeAudio(const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns);
bool finalize();
const char* videoEncoderSelection() const;
Expand Down
30 changes: 26 additions & 4 deletions electron/native/wgc-capture/src/wgc_session.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,28 @@ int64_t timeSpanToHns(wf::TimeSpan const& value) {
return value.count();
}

// H.264 encoding (and the RGB32->NV12 conversion feeding it) requires even
// frame dimensions. Monitor resolutions are always even in practice, so
// CreateForMonitor items never hit this. Windows, however, frequently have
// odd client-area dimensions (arbitrary drag-resize, DPI rounding), and
// GraphicsCaptureItem::Size() reports the window's *actual* size verbatim.
// If we requested a Direct3D11CaptureFramePool sized to that odd value while
// the rest of the pipeline (main.cpp's bitrate calc, MFEncoder) rounds down
// to even, the frame pool's real DXGI textures end up one pixel wider/taller
// than the staging texture the encoder allocates. ID3D11DeviceContext::
// CopyResource silently no-ops on a size mismatch (it only emits a debug-
// layer warning), so the staging texture never receives pixel data and the
// output is solid black for the entire recording -- or, if the mismatch
// trips up the video MFT's input negotiation, SetInputMediaType fails
// outright. Rounding up to the nearest even size here, and using that
// rounded size (not the raw item size) for both the frame pool and
// `captureWidth()`/`captureHeight()`, keeps every consumer of this session
// looking at the exact same dimensions as the real captured texture.
int roundUpToEven(int value) {
const int clamped = std::max(2, value);
return (clamped % 2 == 0) ? clamped : clamped + 1;
}

} // namespace

WgcSession::~WgcSession() {
Expand Down Expand Up @@ -135,8 +157,8 @@ bool WgcSession::createCaptureItem(HWND window) {

item_ = item;
const auto size = item_.Size();
width_ = static_cast<int>(size.Width);
height_ = static_cast<int>(size.Height);
width_ = roundUpToEven(static_cast<int>(size.Width));
height_ = roundUpToEven(static_cast<int>(size.Height));
return width_ > 0 && height_ > 0;
}

Expand Down Expand Up @@ -199,7 +221,7 @@ bool WgcSession::initialize(HMONITOR monitor, int fps, bool captureCursor) {
winrtDevice_,
wgdx::DirectXPixelFormat::B8G8R8A8UIntNormalized,
2,
item_.Size());
winrt::Windows::Graphics::SizeInt32{width_, height_});
session_ = framePool_.CreateCaptureSession(item_);

if (!applySessionOptions(captureCursor)) {
Expand All @@ -223,7 +245,7 @@ bool WgcSession::initialize(HWND window, int fps, bool captureCursor) {
winrtDevice_,
wgdx::DirectXPixelFormat::B8G8R8A8UIntNormalized,
2,
item_.Size());
winrt::Windows::Graphics::SizeInt32{width_, height_});
session_ = framePool_.CreateCaptureSession(item_);

if (!applySessionOptions(captureCursor)) {
Expand Down
20 changes: 18 additions & 2 deletions src/components/video-editor/VideoEditor.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -1004,6 +1004,16 @@ export default function VideoEditor() {
video.currentTime = time;
}

// Reads the live playhead position directly off the <video> element, bypassing
// `currentTime` React state entirely. The timeline's playhead subscribes to this
// via its own rAF loop so it stays in lockstep with actual playback regardless of
// how long this (large) component's own re-render takes — see issue #111.
const getPlaybackTimeMs = useCallback(() => {
const video = videoPlaybackRef.current?.video;
if (video) return video.currentTime * 1000;
return currentTimeRef.current * 1000;
}, []);

const handleSelectZoom = useCallback((id: string | null) => {
setSelectedZoomId(id);
if (id) {
Expand Down Expand Up @@ -1136,6 +1146,11 @@ export default function VideoEditor() {

// Builds fresh "auto" zoom regions from cursor telemetry without overlapping
// existing ones. Used by both the on-load auto-suggest pass and the wand toggle.
// These regions always follow the cursor for their whole span (focusMode "auto") —
// that's the entire point of an auto-placed zoom: it should pan to track the cursor
// as it moves, not freeze at the dwell point that triggered the suggestion. This is
// independent of the global "Auto Focus All" toggle, which only affects the default
// for manually-drawn zoom regions.
const buildAutoZoomRegions = useCallback(
(existingRegions: ZoomRegion[]): ZoomRegion[] => {
const totalMs = Math.round(duration * 1000);
Expand All @@ -1152,11 +1167,11 @@ export default function VideoEditor() {
depth: DEFAULT_ZOOM_DEPTH,
customScale: ZOOM_DEPTH_SCALES[DEFAULT_ZOOM_DEPTH],
focus: clampFocusToDepth(suggestion.focus, DEFAULT_ZOOM_DEPTH),
focusMode: autoFocusAll ? ("auto" as const) : undefined,
focusMode: "auto" as const,
source: "auto" as const,
}));
},
[cursorTelemetry, duration, autoFocusAll],
[cursorTelemetry, duration],
);

// Auto-suggest zooms once per fresh recording (no existing zooms, telemetry
Expand Down Expand Up @@ -3213,6 +3228,7 @@ export default function VideoEditor() {
<TimelineEditor
videoDuration={duration}
currentTime={currentTime}
getPlaybackTimeMs={getPlaybackTimeMs}
onSeek={handleSeek}
zoomRegions={zoomRegions}
onZoomAdded={handleZoomAdded}
Expand Down
Loading
Loading