Skip to content

Commit 99fb0b9

Browse files
thymikeeclaude
andcommitted
fix(macos-helper): refuse a missed hit test as no-accessible-target
A secondary click whose hit test finds no element now refuses with no-accessible-target, as press does, instead of no-context-menu. The walk for the menu owner also stops at an open menu or the menu bar, so a secondary click on an open menu's item does not climb to the row and reopen its menu. The walk is now a pure function over the hit chain, with unit rows for a row menu, a Button menu, a plain button, an open menu item, and a missed hit test. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent e8688c3 commit 99fb0b9

4 files changed

Lines changed: 68 additions & 16 deletions

File tree

‎apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift‎

Lines changed: 35 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -223,9 +223,22 @@ func showMenuInBackground(
223223
) throws -> BackgroundMenuResponse {
224224
try requireAppWindowPoint(point, app: app)
225225
cursor?.move(to: point)
226-
guard let owner = elementAtPoint(in: app, point: point).flatMap(menuOwner) else {
227-
throw refusal(
228-
.noContextMenu, "no accessibility element at the point offers a context menu", app: app)
226+
let hitChain = sequence(state: elementAtPoint(in: app, point: point)) { current in
227+
defer { current = current.flatMap { elementAttribute($0, attribute: kAXParentAttribute as String) } }
228+
return current
229+
}.lazy.map { element in
230+
(
231+
element: element, role: role(of: element),
232+
offersMenu: actionNames(of: element).contains(showMenuAction as String)
233+
)
234+
}
235+
let owner: AXUIElement
236+
switch contextMenuOwner(upFrom: hitChain) {
237+
case .success(let element): owner = element
238+
case .failure(.noAccessibleTarget):
239+
throw refusal(.noAccessibleTarget, "no accessibility element at the point", app: app)
240+
case .failure(let reason):
241+
throw refusal(reason, "no accessibility element at the point offers a context menu", app: app)
229242
}
230243
if isInMenuBar(owner) {
231244
throw refusal(.noAccessibleTarget, "the point is on the app's menu bar", app: app)
@@ -270,19 +283,26 @@ func popUpMenuWindowNumbers(pid: pid_t, windowInfo: [[String: Any]]) -> Set<Int>
270283

271284
private let showMenuAction = "AXShowMenu" as CFString
272285

286+
extension BackgroundRefusal: Error {}
287+
288+
/// An open menu's items sit under the element that opened it, so the walk stops at a menu too.
289+
private let contextMenuOwnerBoundaryRoles: Set<String> = [
290+
"AXWindow", "AXApplication", "AXMenu", "AXMenuBar",
291+
]
292+
273293
/// AppKit offers a secondary click to the hit view and then to each superview in turn, so the
274-
/// nearest ancestor with a menu owns it.
275-
private func menuOwner(from hit: AXUIElement) -> AXUIElement? {
276-
var current: AXUIElement? = hit
277-
var depth = 0
278-
while let element = current, depth < 32 {
279-
let elementRole = role(of: element)
280-
if elementRole == "AXWindow" || elementRole == "AXApplication" { return nil }
281-
if actionNames(of: element).contains(showMenuAction as String) { return element }
282-
current = elementAttribute(element, attribute: kAXParentAttribute as String)
283-
depth += 1
284-
}
285-
return nil
294+
/// nearest ancestor with a menu owns it. The chain runs from the hit element up; an empty chain
295+
/// is a hit test that found nothing.
296+
func contextMenuOwner<Element>(
297+
upFrom hitChain: some Sequence<(element: Element, role: String, offersMenu: Bool)>
298+
) -> Result<Element, BackgroundRefusal> {
299+
var hitAny = false
300+
for link in hitChain.prefix(32) {
301+
hitAny = true
302+
if contextMenuOwnerBoundaryRoles.contains(link.role) { break }
303+
if link.offersMenu { return .success(link.element) }
304+
}
305+
return .failure(hitAny ? .noContextMenu : .noAccessibleTarget)
286306
}
287307

288308
/// A window list that never became readable is a refusal, not an unobserved menu.

‎apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,37 @@ final class BackgroundInteractionTests: XCTestCase {
123123
XCTAssertEqual(popUpMenuWindowNumbers(pid: 42, windowInfo: windowInfo), [2])
124124
}
125125

126+
/// A secondary click opens the menu of the nearest element up the hit chain that offers one,
127+
/// and stops at the window or at an open menu, whose items sit under the element that opened it.
128+
func testContextMenuOwnerIsTheNearestAncestorWithAMenuInsideTheWindow() {
129+
func chain(_ links: (String, String, Bool)...) -> [(element: String, role: String, offersMenu: Bool)] {
130+
links.map { (element: $0.0, role: $0.1, offersMenu: $0.2) }
131+
}
132+
let rowMenu = chain(
133+
("label", "AXStaticText", false), ("row", "AXCell", true), ("list", "AXOutline", false),
134+
("window", "AXWindow", false))
135+
let buttonMenu = chain(("button", "AXMenuButton", true), ("window", "AXWindow", true))
136+
let plainButton = chain(
137+
("button", "AXButton", false), ("window", "AXWindow", true), ("app", "AXApplication", true))
138+
let openMenuItem = chain(
139+
("rename", "AXMenuItem", false), ("menu", "AXMenu", false), ("row", "AXCell", true))
140+
let menuBarItem = chain(
141+
("file", "AXMenuBarItem", false), ("bar", "AXMenuBar", false), ("app", "AXApplication", true))
142+
143+
XCTAssertEqual(contextMenuOwner(upFrom: rowMenu), .success("row"))
144+
XCTAssertEqual(contextMenuOwner(upFrom: buttonMenu), .success("button"))
145+
XCTAssertEqual(contextMenuOwner(upFrom: plainButton), .failure(.noContextMenu))
146+
XCTAssertEqual(contextMenuOwner(upFrom: openMenuItem), .failure(.noContextMenu))
147+
XCTAssertEqual(contextMenuOwner(upFrom: menuBarItem), .failure(.noContextMenu))
148+
}
149+
150+
/// A hit test that found nothing is no target at the point, not a target without a menu.
151+
func testNoHitIsNoAccessibleTargetRatherThanNoContextMenu() {
152+
XCTAssertEqual(
153+
contextMenuOwner(upFrom: [] as [(element: String, role: String, offersMenu: Bool)]),
154+
.failure(.noAccessibleTarget))
155+
}
156+
126157
/// Chromium marks wrapper groups pressable; only a control role may win the press outright.
127158
func testWrapperGroupsAreNotPressableControls() {
128159
XCTAssertTrue(isPressableControlRole("AXButton"))

‎docs/adr/0031-macos-native-app-backend.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ Pointer actions have no event fallback until an event path verifies its outcome.
8989
**Secondary click.**
9090
The helper performs `AXShowMenu` on the nearest element up the hit chain (at most 32 levels) that offers it.
9191
Unlike a press, the walk crosses containers, which can own a menu, as AppKit consults each superview.
92+
The walk stops at the window and at an open menu, so a secondary click on an open menu's item does not reopen its owner's menu.
9293
Parity with a real click outside SwiftUI is unmeasured; an `NSTableView` menu may report no clicked row.
9394
In the measured SwiftUI app, the app tracks the open menu and does not answer the action (`kAXErrorCannotComplete`).
9495
So the reply is abandoned after 0.5 s, and a new app-owned `kCGPopUpMenuWindowLevel` window within 1 s decides success.

‎website/docs/docs/commands.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -356,7 +356,7 @@ With the native backend, each app you open on the `app` surface is a device of i
356356
357357
Copies of one app need distinct bundle ids. Screenshots run one at a time across the Mac. Use one agent-device version per claims directory.
358358
359-
- `click`, `press`, and `fill` use accessibility actions on the element at the target point, which must be inside one of the app's on-screen windows, not its menu bar. `click --button secondary` opens the context menu of the nearest element at the point that offers one (`AXShowMenu`); the open menu shows in the next `snapshot`, and `click` on one of its items picks it. It fails with `UNSUPPORTED_OPERATION` (`helperReason: no-context-menu`) when no element there offers a menu, and with `COMMAND_FAILED` (`reason: context-menu-not-observed`) when no menu window opened. `type` inserts at the focused control or falls back to key events. `scroll` moves the scroll bar under the front window's center. Electron and other Chromium-based desktop apps expose their web content in the snapshot: the first `snapshot` turns the app's Chromium accessibility tree on, and a snapshot warns when that tree did not populate. `screenshot` captures only the app's front window, even when it is covered. The front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, and a sheet is captured as its own window. If that window cannot be captured, `screenshot` fails; it never captures another window.
359+
- `click`, `press`, and `fill` use accessibility actions on the element at the target point, which must be inside one of the app's on-screen windows, not its menu bar. `click --button secondary` opens the context menu of the nearest element at the point that offers one (`AXShowMenu`); the open menu shows in the next `snapshot`, and `click` on one of its items picks it. It fails with `UNSUPPORTED_OPERATION` (`helperReason: no-context-menu`) when no element there offers a menu (`no-accessible-target` when no element is at the point), and with `COMMAND_FAILED` (`reason: context-menu-not-observed`) when no menu window opened. `type` inserts at the focused control or falls back to key events. `scroll` moves the scroll bar under the front window's center. Electron and other Chromium-based desktop apps expose their web content in the snapshot: the first `snapshot` turns the app's Chromium accessibility tree on, and a snapshot warns when that tree did not populate. `screenshot` captures only the app's front window, even when it is covered. The front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, and a sheet is captured as its own window. If that window cannot be captured, `screenshot` fails; it never captures another window.
360360
- `record`, `prepare`, `back`, `longpress`, `--hold-ms`, `--double-tap`, gestures, and clicks on an element with no accessibility action fail with `UNSUPPORTED_OPERATION` (`reason: unsupported-device-backend`). Use the default XCTest backend for those.
361361
362362
```bash

0 commit comments

Comments
 (0)