From 2f4303bd558abf9aa7b871dedf5d3e0df8d40568 Mon Sep 17 00:00:00 2001 From: WilliamSolari <118758182+B077AS@users.noreply.github.com> Date: Sat, 29 Aug 2026 10:44:41 +0200 Subject: [PATCH 1/2] fix: define RTC_ENABLE_WIN_WGC for the JNI wrapper (Windows ABI mismatch) Windows screen/window capture has been completely broken since the branch-heads/7339 -> 7977 (m140 -> m152) bump: zero frames captured, and the "screens" enumeration returned application windows. Root cause is a DesktopCaptureOptions ABI mismatch between webrtc.lib and the JNI wrapper. gn builds the lib with its default rtc_enable_win_wgc=true (webrtc.gni: rtc_enable_win_wgc = is_win), so the lib-side class contains the RTC_ENABLE_WIN_WGC-guarded members; the wrapper compiled without the define sees a smaller class (64 vs 80 bytes on 7977) with different member offsets for everything after the WGC block. DesktopCaptureOptions::CreateDefault() and the copy constructor are defined out-of-line in the lib, so `auto options = CreateDefault()` in the wrapper writes an 80-byte object into a 64-byte stack slot: 16 bytes of adjacent stack are zero-smashed on every capturer construction. That deterministic smash is what zeroed jni::DesktopCapturer's spilled screenCapturer argument (every ScreenCapturer.initialize() logged screenCapturer=0 and constructed the window capturer instead - the "Screens tab lists applications" bug), and general field-offset skew after the WGC block broke frame delivery entirely. The mismatch already existed on 7339, but the guarded block was then just 5 bools at the tail-adjacent position: an 8-byte zero overflow and skewed reads of cosmetic flags (disable_effects_), which happened to be survivable - which is why 0.14.0 worked. 7977 grew the block (7 bools + LUID) and added std::optional env_ after it, making the corruption fatal. Fix: define RTC_ENABLE_WIN_WGC for the wrapper so both sides agree on the layout. This is the same ABI bug class already fixed on Linux with WEBRTC_USE_X11 (#236) and WEBRTC_USE_PIPEWIRE (#261). All allow_wgc_* options default to false, so capturer selection behavior is unchanged (DirectX/GDI as before). --- .../src/main/cpp/dependencies/webrtc/CMakeLists.txt | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/webrtc-jni/src/main/cpp/dependencies/webrtc/CMakeLists.txt b/webrtc-jni/src/main/cpp/dependencies/webrtc/CMakeLists.txt index 02151ab5..d1341a4e 100644 --- a/webrtc-jni/src/main/cpp/dependencies/webrtc/CMakeLists.txt +++ b/webrtc-jni/src/main/cpp/dependencies/webrtc/CMakeLists.txt @@ -228,7 +228,15 @@ elseif(LINUX) target_compile_definitions(${PROJECT_NAME} PUBLIC WEBRTC_LINUX WEBRTC_POSIX WEBRTC_USE_H264 WEBRTC_USE_X11 WEBRTC_USE_PIPEWIRE WEBRTC_USE_GIO) target_link_libraries(${PROJECT_NAME} X11 Xext Xfixes Xdamage Xtst Xrandr Xcomposite dbus-1 ${GIO_LIBRARIES} ${GBM_LIBRARIES} ${DRM_LIBRARIES}) elseif(WIN32) - target_compile_definitions(${PROJECT_NAME} PUBLIC WEBRTC_WIN WEBRTC_USE_H264 NOMINMAX WIN32_LEAN_AND_MEAN NDEBUG) + # RTC_ENABLE_WIN_WGC must match the gn build: webrtc.lib is compiled with gn's + # default rtc_enable_win_wgc=true (webrtc.gni: "rtc_enable_win_wgc = is_win"), + # and DesktopCaptureOptions has members guarded by that define. Compiling the + # JNI wrapper without it changes the class layout and size (64 vs 80 bytes as + # of branch-heads/7977) — the out-of-line CreateDefault()/copy constructor in + # the lib then overflow the wrapper's smaller stack slot and every member + # after the guarded block is read at wrong offsets, silently breaking desktop + # capture (same ABI bug WEBRTC_USE_X11 fixed on Linux). + target_compile_definitions(${PROJECT_NAME} PUBLIC WEBRTC_WIN WEBRTC_USE_H264 NOMINMAX WIN32_LEAN_AND_MEAN NDEBUG RTC_ENABLE_WIN_WGC) target_link_libraries(${PROJECT_NAME} D3D11 DXGI user32 gdi32 iphlpapi dmoguids msdmo secur32 strmiids winmm wmcodecdspuuid ws2_32) endif() From 334429f5a49e5a285b617fe4deba7f9ddd3263cd Mon Sep 17 00:00:00 2001 From: WilliamSolari <118758182+B077AS@users.noreply.github.com> Date: Sat, 29 Aug 2026 11:03:44 +0200 Subject: [PATCH 2/2] fix: pass full frame height as libyuv src_height (broken window capture) With the RTC_ENABLE_WIN_WGC ABI fix in place, screen capture works but window (application) capture still delivered no frames: neither thumbnails (DesktopCaptureCallback) nor the live share (VideoTrackDesktopSource). Both files feed libyuv::ConvertToI420 with the cropped output height (i420Buffer/buffer->height() == crop_h) in the src_height slot. libyuv bounds-checks crop_y + crop_height <= src_height, so the conversion fails with -1 whenever crop_y > 0. The Windows border-crop branch sets crop_y = -top_left().y() for frames that start above the screen origin, which is the case for every MAXIMIZED window (its frame sits at -SM_CXPADDEDBORDER). Screen frames never enter that branch at all (exact stride, `fullscreen == true`), which is why only window capture appeared broken. Confirmed at runtime on a diagnostic build (Komm, Windows 11): size=1928x1043 crop_y=7 crop_w=1920 crop_h=1032 Failed to convert desktop frame to I420, libyuv result=-1 Also propagate non-SUCCESS capture results to the Java callback (with a null frame) instead of silently returning, so callers don't have to burn a timeout to notice a failed capture. --- .../cpp/src/media/video/VideoTrackDesktopSource.cpp | 8 +++++++- .../media/video/desktop/DesktopCaptureCallback.cpp | 13 ++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp b/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp index 2b93a3aa..75a35f02 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp @@ -194,7 +194,13 @@ namespace jni buffer->MutableDataU(), buffer->StrideU(), buffer->MutableDataV(), buffer->StrideV(), crop_x, crop_y, - frame->stride() / webrtc::DesktopFrame::kBytesPerPixel, buffer->height(), crop_w, crop_h, + // (src_width, src_height) must describe the FULL source frame, not the cropped + // output. Passing buffer->height() (== crop_h) here made libyuv's internal + // bounds check (crop_y + crop_height <= src_height) fail with -1 whenever + // crop_y > 0 — i.e. for every maximized window, whose frame sits at + // top_left().y() == -border. Screen frames never hit this branch (exact stride, + // fullscreen == true), which is why only window capture appeared broken. + frame->stride() / webrtc::DesktopFrame::kBytesPerPixel, height, crop_w, crop_h, libyuv::kRotate0, libyuv::FOURCC_ARGB); diff --git a/webrtc-jni/src/main/cpp/src/media/video/desktop/DesktopCaptureCallback.cpp b/webrtc-jni/src/main/cpp/src/media/video/desktop/DesktopCaptureCallback.cpp index 46b8d6d2..ac29ddb5 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/desktop/DesktopCaptureCallback.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/desktop/DesktopCaptureCallback.cpp @@ -44,6 +44,11 @@ namespace jni JNIEnv * env = AttachCurrentThread(); if (result != webrtc::DesktopCapturer::Result::SUCCESS) { + // Propagate the failure to Java instead of silently dropping it — + // callers waiting for a frame otherwise have to rely on timeouts. + auto jerror = JavaEnums::toJava(env, result); + env->CallVoidMethod(callback, javaClass->onCaptureResult, jerror.get(), nullptr); + ExceptionCheck(env); return; } @@ -83,7 +88,13 @@ namespace jni i420Buffer->MutableDataU(), i420Buffer->StrideU(), i420Buffer->MutableDataV(), i420Buffer->StrideV(), crop_x, crop_y, - frame->stride() / webrtc::DesktopFrame::kBytesPerPixel, i420Buffer->height(), crop_w, crop_h, + // (src_width, src_height) must describe the FULL source frame, not the cropped + // output. Passing i420Buffer->height() (== crop_h) here made libyuv's internal + // bounds check (crop_y + crop_height <= src_height) fail with -1 whenever + // crop_y > 0 — i.e. for every maximized window, whose frame sits at + // top_left().y() == -border. Screen frames never hit this branch (exact stride, + // fullscreen == true), which is why only window capture appeared broken. + frame->stride() / webrtc::DesktopFrame::kBytesPerPixel, height, crop_w, crop_h, libyuv::kRotate0, libyuv::FOURCC_ARGB);