Skip to content

Commit 4c35f9b

Browse files
janicduplessisthymikee
authored andcommitted
fix(macos-helper): skip the window-sharing indicator when choosing the front window
While a ScreenCaptureKit stream shares an app window, macOS adds a small normal-layer window with a WindowSharingSessionButton in front of it. The native backend took it for the app's front window, so scroll refused with no-scroll-bar under its center and screenshot captured the indicator. Refs callstack#3213
1 parent 630226b commit 4c35f9b

3 files changed

Lines changed: 54 additions & 6 deletions

File tree

‎apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift‎

Lines changed: 41 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -655,8 +655,13 @@ struct OnScreenWindow {
655655
/// Only normal-layer windows qualify; app-modal alerts at level 8 are excluded.
656656
let frontWindowLayer = 0
657657

658-
/// The pid's visible, nonempty normal-layer windows, front to back.
659-
func onScreenWindows(pid: pid_t, windowInfo: [[String: Any]]) -> [OnScreenWindow] {
658+
/// The pid's visible, nonempty normal-layer windows, front to back, other than the windows whose
659+
/// frames are excluded.
660+
func onScreenWindows(
661+
pid: pid_t,
662+
windowInfo: [[String: Any]],
663+
excluding excludedFrames: [CGRect] = []
664+
) -> [OnScreenWindow] {
660665
windowInfo.compactMap { entry in
661666
guard (entry[kCGWindowOwnerPID as String] as? Int32) == pid,
662667
(entry[kCGWindowLayer as String] as? Int) == frontWindowLayer,
@@ -665,7 +670,8 @@ func onScreenWindows(pid: pid_t, windowInfo: [[String: Any]]) -> [OnScreenWindow
665670
let number = entry[kCGWindowNumber as String] as? Int,
666671
let boundsDict = entry[kCGWindowBounds as String] as? NSDictionary,
667672
let bounds = CGRect(dictionaryRepresentation: boundsDict as CFDictionary),
668-
bounds.width > 0, bounds.height > 0
673+
bounds.width > 0, bounds.height > 0,
674+
!excludedFrames.contains(where: { framesMatch($0, bounds) })
669675
else {
670676
return nil
671677
}
@@ -682,8 +688,16 @@ func appWindowInfo() -> [[String: Any]]? {
682688
/// The app's front on-screen window: frontmost among its eligible windows, or nil when it has
683689
/// none. A window list that cannot be read is a refusal, not an empty list: the caller must not
684690
/// report "no window" for a WindowServer it could not ask.
685-
func frontWindow(pid: pid_t, windowInfo: [[String: Any]]? = appWindowInfo()) throws -> OnScreenWindow? {
686-
onScreenWindows(pid: pid, windowInfo: try requireWindowInfo(windowInfo)).first
691+
func frontWindow(
692+
pid: pid_t,
693+
windowInfo: [[String: Any]]? = appWindowInfo(),
694+
sharingIndicatorFrames: [CGRect]? = nil
695+
) throws -> OnScreenWindow? {
696+
onScreenWindows(
697+
pid: pid,
698+
windowInfo: try requireWindowInfo(windowInfo),
699+
excluding: sharingIndicatorFrames ?? windowSharingIndicatorFrames(pid: pid)
700+
).first
687701
}
688702

689703
func requireWindowInfo(_ windowInfo: [[String: Any]]? = appWindowInfo()) throws -> [[String: Any]] {
@@ -699,6 +713,28 @@ func requireWindowInfo(_ windowInfo: [[String: Any]]? = appWindowInfo()) throws
699713
return windowInfo
700714
}
701715

716+
/// While a ScreenCaptureKit stream shares one of an app's windows, macOS 27 adds a 66 x 20
717+
/// normal-layer window titled "Window" in front of it. Its only child is a button titled
718+
/// `WindowSharingSessionButton`. That window is a system indicator, not the app's front window.
719+
private let windowSharingIndicatorButtonTitle = "WindowSharingSessionButton"
720+
721+
func windowSharingIndicatorFrames(pid: pid_t) -> [CGRect] {
722+
windows(of: AXUIElementCreateApplication(pid)).compactMap { window in
723+
let isIndicator = children(of: window).contains { child in
724+
role(of: child) == "AXButton"
725+
&& stringAttribute(child, attribute: kAXTitleAttribute as String)
726+
== windowSharingIndicatorButtonTitle
727+
}
728+
guard isIndicator, let rect = rectAttribute(window) else { return nil }
729+
return CGRect(x: rect.x, y: rect.y, width: rect.width, height: rect.height)
730+
}
731+
}
732+
733+
private func framesMatch(_ a: CGRect, _ b: CGRect) -> Bool {
734+
abs(a.minX - b.minX) < 1 && abs(a.minY - b.minY) < 1 && abs(a.width - b.width) < 1
735+
&& abs(a.height - b.height) < 1
736+
}
737+
702738
func frontWindowFrame(pid: pid_t, windowInfo: [[String: Any]]? = appWindowInfo()) throws -> CGRect? {
703739
try frontWindow(pid: pid, windowInfo: windowInfo)?.bounds
704740
}

‎apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,18 @@ final class AppWindowScreenshotTests: XCTestCase {
4343
XCTAssertEqual(try frontWindowFrame(pid: 42, windowInfo: windows), front?.bounds)
4444
}
4545

46+
/// A shared window gets a small system indicator window in front of it; capture, scroll and the
47+
/// click fallback must keep using the shared window rather than the indicator.
48+
func testSkipsTheWindowSharingIndicatorInFrontOfTheSharedWindow() throws {
49+
let windows = [windowInfo(id: 9, width: 66, height: 20), windowInfo(id: 8, width: 900, height: 700)]
50+
let indicator = CGRect(x: 10, y: 10, width: 66, height: 20)
51+
let front = try frontWindow(pid: 42, windowInfo: windows, sharingIndicatorFrames: [indicator])
52+
53+
XCTAssertEqual(front?.number, 8)
54+
XCTAssertEqual(
55+
try frontWindow(pid: 42, windowInfo: windows, sharingIndicatorFrames: [])?.number, 9)
56+
}
57+
4658
func testRefusesWhenOnlyInvisibleOrEmptyWindowsRemain() throws {
4759
let windows = [
4860
windowInfo(id: 1, onScreen: false),

‎website/docs/docs/commands.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -363,7 +363,7 @@ With the native backend, each app you open on the `app` surface is a device of i
363363
364364
Copies of one app need distinct bundle ids. Screenshots run one at a time across the Mac. Use one agent-device version per claims directory.
365365
366-
- `click`, `press`, and `fill` use accessibility actions on the element at the target point, which must be inside one of the app's on-screen windows, not its menu bar. `click --button secondary` opens the context menu of the nearest element at the point that offers one (`AXShowMenu`); the open menu shows in the next `snapshot`, and `click` on one of its items picks it. It fails with `UNSUPPORTED_OPERATION` (`helperReason: no-context-menu`) when no element there offers a menu (`no-accessible-target` when no element is at the point). It fails with `COMMAND_FAILED` and `reason: context-menu-not-observed` when the app's window list was readable but showed no new menu window, and with `reason: window-list-unreadable` when the window list could not be read before the action or by any read during the wait for the menu. `type` inserts at the focused control or falls back to key events. `scroll` moves the scroll bar under the front window's center. Electron and other Chromium-based desktop apps expose their web content in the snapshot: the first `snapshot` turns the app's Chromium accessibility tree on, and a snapshot warns when that tree did not populate. `screenshot` captures only the app's front window, even when it is covered. The front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, and a sheet is captured as its own window. If that window cannot be captured, `screenshot` fails; it never captures another window.
366+
- `click`, `press`, and `fill` use accessibility actions on the element at the target point, which must be inside one of the app's on-screen windows, not its menu bar. `click --button secondary` opens the context menu of the nearest element at the point that offers one (`AXShowMenu`); the open menu shows in the next `snapshot`, and `click` on one of its items picks it. It fails with `UNSUPPORTED_OPERATION` (`helperReason: no-context-menu`) when no element there offers a menu (`no-accessible-target` when no element is at the point). It fails with `COMMAND_FAILED` and `reason: context-menu-not-observed` when the app's window list was readable but showed no new menu window, and with `reason: window-list-unreadable` when the window list could not be read before the action or by any read during the wait for the menu. `type` inserts at the focused control or falls back to key events. `scroll` moves the scroll bar under the front window's center. Electron and other Chromium-based desktop apps expose their web content in the snapshot: the first `snapshot` turns the app's Chromium accessibility tree on, and a snapshot warns when that tree did not populate. `screenshot` captures only the app's front window, even when it is covered. The front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, as is the small indicator window macOS adds in front of a window that a screen-sharing stream captures, and a sheet is captured as its own window. If that window cannot be captured, `screenshot` fails; it never captures another window.
367367
- `record`, `prepare`, `back`, `longpress`, `--hold-ms`, `--double-tap`, gestures, and clicks on an element with no accessibility action fail with `UNSUPPORTED_OPERATION` (`reason: unsupported-device-backend`). Use the default XCTest backend for those.
368368
369369
```bash

0 commit comments

Comments
 (0)