diff --git a/AGENTS.md b/AGENTS.md index 4bd3a7dac..028be56a0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,7 +117,7 @@ This section is the *mechanics*. **What to actually run is [`technical-documenta **The HUD widget** (recording controller) - **It is invisible in screenshots by default.** The HUD (and the Notes window) call `setContentProtection(true)` so the recording controls never end up baked into a recording — the same `SetWindowDisplayAffinity` that WGC honours also hides them from *your* screenshots. The window is there, and clicks land, but you are aiming blind at a rectangle you cannot see. Set **`OPENSCREEN_DISABLE_CONTENT_PROTECTION=1`** in the app's environment to turn it off for a session; every skipped window logs a warning. Unset it before recording anything real, or the HUD ends up in the video. -- **On macOS 26+ content protection is auto-disabled, so the HUD *is* visible and screenshottable with no flag.** That OS never displays a content-protected window at all — not just absent from captures, but never painted, leaving a tray icon, a live renderer and nothing on screen (confirmed on macOS 26.5 / Electron 41.2.1). `applyContentProtection` therefore skips the call there and logs a warning per window; the trade-off is that the HUD can appear in recordings on that OS until the ScreenCaptureKit helper excludes our own windows via `SCContentFilter(excludingWindows:)`, which it currently passes as `[]`. `OPENSCREEN_FORCE_CONTENT_PROTECTION=1` re-enables it to re-test against a future Electron. +- **On macOS 26+ content protection is auto-disabled, so the HUD *is* visible and screenshottable with no flag.** That OS never displays a content-protected window at all — not just absent from captures, but never painted, leaving a tray icon, a live renderer and nothing on screen (confirmed on macOS 26.5 / Electron 41.2.1). `applyContentProtection` therefore skips the call there and logs a warning per window. The native ScreenCaptureKit helper still keeps the HUD and an open Notes window out of full-display recordings through `SCContentFilter(excludingWindows:)`; ordinary OpenScreen windows remain recordable. `OPENSCREEN_FORCE_CONTENT_PROTECTION=1` re-enables Electron's protection to re-test against a future Electron. - The HUD is what opens the editor (clapper icon, tooltip *Open Studio*), so without that flag a whole slice of the app is unreachable from automation: killing the app to redeploy a native addon leaves you unable to reopen a project. - Frameless, transparent, always-on-top, `skipTaskbar`, centered at the **bottom of the primary display** (`createHudOverlayWindow`, 820×560 at construction, then resized to fit its content — measured 904×698 with the bar at the bottom and mostly empty reserve above it). It is **click-through** (`setIgnoreMouseEvents(ignore)`): moving the real cursor over an interactive control makes that region clickable and shows its tooltip, so `mouse_move` → screenshot → `left_click` works; a blind click on empty HUD area passes through to the desktop. - **Only a real OS mouse move reaches the HUD — on macOS as much as on Windows.** While the window is input-transparent Chromium delivers it no pointer events at all, so the main process samples the OS cursor instead: the `hud-overlay-cursor` poll in `electron/windows.ts` reads `screen.getCursorScreenPoint()` while the HUD is click-through and pushes the window-relative point to the renderer, which hit-tests it with `elementFromPoint(…).closest("[data-hud-interactive='true']")`. One path, both platforms — there is no platform branch. **Linux is the exception** (`!enabled && !isLinuxHud` in `LaunchWindow.tsx`, where the call is a no-op), so it is the one platform where a blind click on the HUD simply lands. What the poll keys off is the OS cursor's position *relative to the window*, so a resize or re-anchor that slides the bar under a motionless pointer produces a fresh sample too. What it can never key off is synthesised input: Playwright's `.click()`, `javascript_tool`-dispatched pointer events and everything like them move no pointer at all, so they never put one on a control. They arrive *below* the OS hit-test, fire the DOM handler, and look like they worked — while the click-through path was never exercised at all. `tests/e2e/windows-native-checklist.spec.ts` does click HUD test IDs and stays green for exactly that reason; it proves renderer wiring, not reachability, and a macOS spec written the same way would prove no more. Use computer-use (`mouse_move` → `left_click`), and never conclude from a passing injected click that a user could have clicked it. (Until #385 the lift was Electron's `{ forward: true }` — a global `WH_MOUSE_LL` hook on Windows — which Windows can revoke without telling the app, leaving the HUD painted and permanently dead. The poll replaced it. The rule for you is unchanged, because both mechanisms key off the real cursor.) diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 958ef6d96..58b8d5210 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -20,7 +20,10 @@ import { type NativeLinuxRecordingRequest, portalCursorMode, } from "../../src/lib/nativeLinuxRecording"; -import type { NativeMacRecordingRequest } from "../../src/lib/nativeMacRecording"; +import { + collectMacCaptureExcludedWindowIds, + type NativeMacRecordingRequest, +} from "../../src/lib/nativeMacRecording"; import type { NativeWindowsRecordingRequest } from "../../src/lib/nativeWindowsRecording"; import { type CursorCaptureMode, @@ -2657,10 +2660,19 @@ export function registerIpcHandlers( null) : getSelectedDisplay(); const bounds = request.source.bounds ?? sourceDisplay?.bounds ?? getSelectedSourceBounds(); + const captureExcludedWindowSourceIds: string[] = []; + if (request.source.type === "display") { + for (const window of [getMainWindow(), getNotesWindow()]) { + if (window && !window.isDestroyed()) { + captureExcludedWindowSourceIds.push(window.getMediaSourceId()); + } + } + } const config: NativeMacRecordingRequest = { ...request, schemaVersion: 1, recordingId, + excludedWindowIds: collectMacCaptureExcludedWindowIds(captureExcludedWindowSourceIds), source: { ...request.source, bounds, @@ -2688,6 +2700,7 @@ export function registerIpcHandlers( console.info("[native-sck] starting macOS capture", { helperPath, source: config.source, + excludedWindowIds: config.excludedWindowIds, audio: config.audio, webcam: config.webcam, cursor: config.cursor, diff --git a/electron/native/README.md b/electron/native/README.md index 8ff2e3cdd..75f76ed45 100644 --- a/electron/native/README.md +++ b/electron/native/README.md @@ -25,7 +25,7 @@ npm run build:native:mac On non-macOS hosts this command exits successfully and does not affect Windows/Linux development. On macOS it builds the Swift package at `electron/native/screencapturekit`, writes the development binaries to `electron/native/screencapturekit/build`, and copies redistributable binaries to `electron/native/bin/darwin-${arch}`. -The current helper implementation supports display/window ScreenCaptureKit video capture, cursor exclusion through `SCStreamConfiguration.showsCursor`, H.264 encoding, MP4 muxing (with `AVAssetWriter.movieFragmentInterval` at 1s, so a helper that dies before `finishWriting()` still leaves a readable file — same reasoning as the Windows fragmented sink below), and ScreenCaptureKit system audio. It also attempts native ScreenCaptureKit microphone capture when the running macOS version exposes that capability. Webcam recording currently stays as an Electron sidecar and is attached to the same recording session after the native screen capture stops. +The current helper implementation supports display/window ScreenCaptureKit video capture, control-window exclusion through the request's optional `excludedWindowIds` and `SCContentFilter(display:excludingWindows:)`, cursor exclusion through `SCStreamConfiguration.showsCursor`, H.264 encoding, MP4 muxing (with `AVAssetWriter.movieFragmentInterval` at 1s, so a helper that dies before `finishWriting()` still leaves a readable file — same reasoning as the Windows fragmented sink below), and ScreenCaptureKit system audio. Electron supplies only the HUD and an open Notes window as exclusions for full-display capture, so ordinary OpenScreen windows remain recordable. The helper also attempts native ScreenCaptureKit microphone capture when the running macOS version exposes that capability. Webcam recording currently stays as an Electron sidecar and is attached to the same recording session after the native screen capture stops. Electron exposes `is-native-mac-capture-available` for capability probing. It resolves the same helper locations listed above and reports `missing-helper` until a Swift helper binary is present. When available, macOS recording routes screen/window capture through the native helper so editable cursor recordings do not bake the system cursor into the video. Cursor positions are sampled in Electron; when the cursor helper is available and Accessibility is granted, samples are also tagged with link/text cursor hints such as `pointer`. diff --git a/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/CaptureWindowExclusion.swift b/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/CaptureWindowExclusion.swift new file mode 100644 index 000000000..bce68f223 --- /dev/null +++ b/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/CaptureWindowExclusion.swift @@ -0,0 +1,13 @@ +/// Resolves native window IDs against the same ScreenCaptureKit snapshot used to +/// construct the stream filter. The order is stable for useful diagnostics, and +/// duplicate requests cannot produce duplicate filter entries. +public func resolveCaptureExcludedWindowIDs( + requestedWindowIDs: [UInt32], + availableWindowIDs: [UInt32] +) -> [UInt32] { + let available = Set(availableWindowIDs) + var seen = Set() + return requestedWindowIDs.filter { windowID in + available.contains(windowID) && seen.insert(windowID).inserted + } +} diff --git a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift index 5add8074d..dadbd8f4c 100644 --- a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift +++ b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift @@ -65,6 +65,7 @@ struct RecordingRequest: Decodable { let schemaVersion: Int? let recordingId: Int? + let excludedWindowIds: [UInt32]? let source: Source let video: Video let audio: Audio @@ -387,7 +388,22 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { guard let display = content.displays.first(where: { $0.displayID == displayId }) else { throw HelperError.sourceNotFound("No ScreenCaptureKit display found for id \(displayId).") } - let filter = SCContentFilter(display: display, excludingWindows: []) + let requestedWindowIDs = request.excludedWindowIds ?? [] + let resolvedWindowIDs = resolveCaptureExcludedWindowIDs( + requestedWindowIDs: requestedWindowIDs, + availableWindowIDs: content.windows.map(\.windowID) + ) + let resolvedWindowIDSet = Set(resolvedWindowIDs) + let excludedWindows = content.windows.filter { + resolvedWindowIDSet.contains($0.windowID) + } + let filter = SCContentFilter(display: display, excludingWindows: excludedWindows) + emit([ + "event": "capture-window-exclusion", + "requestedWindowIds": requestedWindowIDs, + "resolvedWindowIds": resolvedWindowIDs, + "excludedWindowCount": excludedWindows.count, + ]) let size = captureSize( for: filter, fallbackPointSize: display.frame.size, diff --git a/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/CaptureWindowExclusionTests.swift b/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/CaptureWindowExclusionTests.swift new file mode 100644 index 000000000..ab5630804 --- /dev/null +++ b/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/CaptureWindowExclusionTests.swift @@ -0,0 +1,34 @@ +import XCTest +@testable import OpenScreenCaptureCore + +final class CaptureWindowExclusionTests: XCTestCase { + func testResolvesUniqueRequestedWindowsInRequestOrder() { + XCTAssertEqual( + resolveCaptureExcludedWindowIDs( + requestedWindowIDs: [42, 7, 42, 99], + availableWindowIDs: [99, 42, 7, 500] + ), + [42, 7, 99] + ) + } + + func testIgnoresWindowsUnavailableToScreenCaptureKit() { + XCTAssertEqual( + resolveCaptureExcludedWindowIDs( + requestedWindowIDs: [42, 404, 7], + availableWindowIDs: [7, 42] + ), + [42, 7] + ) + } + + func testLegacyRequestWithoutExclusionsRemainsEmpty() { + XCTAssertEqual( + resolveCaptureExcludedWindowIDs( + requestedWindowIDs: [], + availableWindowIDs: [42] + ), + [] + ) + } +} diff --git a/electron/windows.ts b/electron/windows.ts index 4b5ceb7fe..af60a8ad6 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -46,12 +46,9 @@ const CONTENT_PROTECTION_FORCED = process.env["OPENSCREEN_FORCE_CONTENT_PROTECTI * older macOS — where it may well work — would be a privacy regression made on * no evidence. * - * NOTE: this leaves the HUD capturable on macOS 26. Apple already made that - * partly true regardless — ScreenCaptureKit ignores `sharingType`, so any - * SCK-based recorder (including *ours*, see - * `electron/native/screencapturekit/`) captures these windows anyway. The - * durable fix is to exclude our own windows via `SCContentFilter`'s - * `excludingWindows:`, which that helper currently passes as `[]`. + * ScreenCaptureKit ignores `sharingType`, so the native recorder independently + * excludes the HUD and Notes windows by their native IDs. This call remains the + * Windows protection and a second line of defence on older macOS releases. */ const CONTENT_PROTECTION_BREAKS_DISPLAY = (() => { if (process.platform !== "darwin") return false; diff --git a/src/lib/nativeMacRecording.test.ts b/src/lib/nativeMacRecording.test.ts index fce88f6a7..6a25d2526 100644 --- a/src/lib/nativeMacRecording.test.ts +++ b/src/lib/nativeMacRecording.test.ts @@ -1,7 +1,35 @@ import { describe, expect, it } from "vitest"; -import { parseMacDisplayIdFromSourceId, parseMacWindowIdFromSourceId } from "./nativeMacRecording"; +import { + collectMacCaptureExcludedWindowIds, + parseMacDisplayIdFromSourceId, + parseMacWindowIdFromSourceId, +} from "./nativeMacRecording"; describe("nativeMacRecording source parsing", () => { + it("collects unique native window ids for ScreenCaptureKit exclusion", () => { + expect( + collectMacCaptureExcludedWindowIds([ + "window:42:0", + "screen:1:0", + "window:7:0", + "window:42:0", + "window:not-a-number:0", + null, + ]), + ).toEqual([42, 7]); + expect(collectMacCaptureExcludedWindowIds([])).toEqual([]); + }); + + it("rejects window ids outside the ScreenCaptureKit UInt32 range", () => { + expect( + collectMacCaptureExcludedWindowIds([ + "window:0:0", + "window:4294967295:0", + "window:4294967296:0", + ]), + ).toEqual([4294967295]); + }); + it("parses Electron window source ids into ScreenCaptureKit window ids", () => { expect(parseMacWindowIdFromSourceId("window:12345:0")).toBe(12345); expect(parseMacWindowIdFromSourceId("window:987")).toBe(987); diff --git a/src/lib/nativeMacRecording.ts b/src/lib/nativeMacRecording.ts index e5137c3de..ff48674a3 100644 --- a/src/lib/nativeMacRecording.ts +++ b/src/lib/nativeMacRecording.ts @@ -6,6 +6,8 @@ export type NativeMacSourceType = "display" | "window"; export type NativeMacRecordingRequest = { schemaVersion: 1; recordingId?: number; + /** Main-process injected native BrowserWindow IDs excluded from display capture. */ + excludedWindowIds?: number[]; source: { type: NativeMacSourceType; sourceId: string; @@ -91,6 +93,29 @@ export type NativeMacRecordingStartResult = { error?: string; }; +export function collectMacCaptureExcludedWindowIds( + mediaSourceIds: ReadonlyArray, +) { + const windowIds: number[] = []; + const seen = new Set(); + for (const sourceId of mediaSourceIds) { + const windowId = parseMacWindowIdFromSourceId(sourceId); + if ( + windowId === null || + !Number.isSafeInteger(windowId) || + windowId <= 0 || + windowId > 0xffff_ffff || + seen.has(windowId) + ) { + continue; + } + + seen.add(windowId); + windowIds.push(windowId); + } + return windowIds; +} + export function parseMacWindowIdFromSourceId(sourceId?: string | null) { if (!sourceId?.startsWith("window:")) { return null;