Skip to content

Commit 70acf88

Browse files
committed
fix(macos): distinguish unreadable window lists from missing windows
Refuse screenshot and scroll with window-list-unreadable when the shared window-list read fails, and preserve a retry hint through the host helper client. Review: #3761 (comment)
1 parent 5e16f81 commit 70acf88

7 files changed

Lines changed: 110 additions & 26 deletions

File tree

‎apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) thro
1010
throw HelperError.commandFailed("app window screenshots require macOS 14 or newer")
1111
}
1212
let pid = app.processIdentifier
13-
let frontWindow = frontWindow(pid: pid)
13+
let frontWindow = try frontWindow(pid: pid)
1414
let frontWindowID = frontWindow?.number
1515
let content = try awaitShareableContent()
1616
let candidates = content.windows.filter { $0.owningApplication?.processID == pid }

‎apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift‎

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ func pressInBackground(
9797
let point = CGPoint(x: request.x, y: request.y)
9898
try requireAppWindowPoint(point, app: app)
9999
cursor?.move(to: point)
100-
let target = resolvePressTarget(app: app, point: point)
100+
let target = try resolvePressTarget(app: app, point: point)
101101
if let target, isInMenuBar(target.element) {
102102
throw refusal(.noAccessibleTarget, "the point is on the app's menu bar", app: app)
103103
}
@@ -172,7 +172,7 @@ func fillInBackground(
172172
cursor?.move(to: point)
173173
let hit = elementAtPoint(in: app, point: point)
174174
let chain = hit.map(pressSearchChain) ?? []
175-
let fallback = actionWindow(app: app, hit: hit).flatMap { window in
175+
let fallback = try actionWindow(app: app, hit: hit).flatMap { window in
176176
smallestElement(in: window, containing: point, where: isTextInput)
177177
}
178178
if let input = chain.first(where: isTextInput) ?? fallback, isInMenuBar(input) {
@@ -206,7 +206,7 @@ func scrollInBackground(
206206
guard isVertical || direction == "left" || direction == "right" else {
207207
throw HelperError.invalidArgs("scroll requires --direction <up|down|left|right>")
208208
}
209-
guard let frame = frontWindowFrame(pid: app.processIdentifier) else {
209+
guard let frame = try frontWindowFrame(pid: app.processIdentifier) else {
210210
throw HelperError.commandFailed(
211211
"scroll could not resolve an on-screen window",
212212
details: ["reason": "window-not-found", "bundleId": app.bundleIdentifier ?? ""]
@@ -276,12 +276,12 @@ private enum PressTarget {
276276

277277
/// What a click at a point means. The app's own hit test answers first; web content often answers
278278
/// with a wrapper group, so the snapshot geometry decides when the hit names no control.
279-
private func resolvePressTarget(app: NSRunningApplication, point: CGPoint) -> PressTarget? {
279+
private func resolvePressTarget(app: NSRunningApplication, point: CGPoint) throws -> PressTarget? {
280280
let hit = elementAtPoint(in: app, point: point)
281281
let chain = hit.map(pressSearchChain) ?? []
282282
if let input = chain.first(where: isTextInput) { return .focus(input) }
283283
if let control = chain.first(where: isPressableControl) { return .press(control) }
284-
if let found = actionWindow(app: app, hit: hit).flatMap({
284+
if let found = try actionWindow(app: app, hit: hit).flatMap({
285285
smallestElement(in: $0, containing: point, where: { isTextInput($0) || isPressableControl($0) })
286286
}) {
287287
return isTextInput(found) ? .focus(found) : .press(found)
@@ -361,12 +361,12 @@ private func pressSearchChain(from hit: AXUIElement) -> [AXUIElement] {
361361
/// The one window a point action may act in: the window the app's hit test landed in, else the
362362
/// app's front on-screen window. A fallback target never comes from a window behind it, and the
363363
/// response names the window so the host can check it.
364-
private func actionWindow(app: NSRunningApplication, hit: AXUIElement?) -> AXUIElement? {
364+
private func actionWindow(app: NSRunningApplication, hit: AXUIElement?) throws -> AXUIElement? {
365365
if let hit {
366366
if role(of: hit) == "AXWindow" { return hit }
367367
if let window = elementAttribute(hit, attribute: kAXWindowAttribute as String) { return window }
368368
}
369-
guard let front = frontWindowFrame(pid: app.processIdentifier) else { return nil }
369+
guard let front = try frontWindowFrame(pid: app.processIdentifier) else { return nil }
370370
return windows(of: AXUIElementCreateApplication(app.processIdentifier)).first { window in
371371
guard let rect = rectAttribute(window) else { return false }
372372
return abs(rect.x - front.minX) < 1 && abs(rect.y - front.minY) < 1
@@ -512,12 +512,22 @@ func appWindowInfo() -> [[String: Any]]? {
512512
as? [[String: Any]]
513513
}
514514

515-
func frontWindow(pid: pid_t, windowInfo: [[String: Any]] = appWindowInfo() ?? []) -> OnScreenWindow? {
516-
onScreenWindows(pid: pid, windowInfo: windowInfo).first
515+
enum WindowListRefusal: String, CaseIterable {
516+
case unreadable = "window-list-unreadable"
517517
}
518518

519-
func frontWindowFrame(pid: pid_t, windowInfo: [[String: Any]] = appWindowInfo() ?? []) -> CGRect? {
520-
frontWindow(pid: pid, windowInfo: windowInfo)?.bounds
519+
func frontWindow(pid: pid_t, windowInfo: [[String: Any]]? = appWindowInfo()) throws -> OnScreenWindow? {
520+
guard let windowInfo else {
521+
throw HelperError.commandFailed(
522+
"the app's on-screen windows could not be read",
523+
details: ["reason": WindowListRefusal.unreadable.rawValue]
524+
)
525+
}
526+
return onScreenWindows(pid: pid, windowInfo: windowInfo).first
527+
}
528+
529+
func frontWindowFrame(pid: pid_t, windowInfo: [[String: Any]]? = appWindowInfo()) throws -> CGRect? {
530+
try frontWindow(pid: pid, windowInfo: windowInfo)?.bounds
521531
}
522532

523533
private let returnVirtualKey: CGKeyCode = 36

‎apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift‎

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ private struct Candidate: Equatable {
1010

1111
/// An app screenshot captures the front window or refuses; it never substitutes another window.
1212
final class AppWindowScreenshotTests: XCTestCase {
13-
func testCaptureIdentityAndScrollAndClickFallbackFrameSelectTheSameEligibleFrontWindow() {
13+
func testCaptureIdentityAndScrollAndClickFallbackFrameSelectTheSameEligibleFrontWindow() throws {
1414
let windows = [
1515
windowInfo(id: 1, pid: 99),
1616
windowInfo(id: 2, layer: 1),
@@ -22,45 +22,62 @@ final class AppWindowScreenshotTests: XCTestCase {
2222
windowInfo(id: 8),
2323
]
2424

25-
let frontID = frontWindow(pid: 42, windowInfo: windows)?.number
25+
let frontID = try frontWindow(pid: 42, windowInfo: windows)?.number
2626
let selected = frontCaptureWindow([7, 8], frontWindowID: frontID, sharingState: 1, windowID: { $0 })
2727

2828
// Scroll and the click actionWindow fallback both consume frontWindowFrame.
2929
XCTAssertEqual(
30-
frontWindowFrame(pid: 42, windowInfo: windows),
30+
try frontWindowFrame(pid: 42, windowInfo: windows),
3131
CGRect(x: 10, y: 10, width: 20, height: 20))
3232
XCTAssertEqual(frontID, 7)
3333
XCTAssertEqual(try selected.get(), 7)
3434
}
3535

3636
func testSkipsAppModalAlertInFrontOfNormalLayerWindow() throws {
3737
let windows = [windowInfo(id: 8, layer: 8), windowInfo(id: 3)]
38-
let front = frontWindow(pid: 42, windowInfo: windows)
38+
let front = try frontWindow(pid: 42, windowInfo: windows)
3939
let selected = frontCaptureWindow(
4040
[8, 3], frontWindowID: front?.number, sharingState: front?.sharingState, windowID: { $0 })
4141

4242
XCTAssertEqual(front?.number, 3)
4343
XCTAssertEqual(try selected.get(), 3)
44-
XCTAssertEqual(frontWindowFrame(pid: 42, windowInfo: windows), front?.bounds)
44+
XCTAssertEqual(try frontWindowFrame(pid: 42, windowInfo: windows), front?.bounds)
4545
}
4646

47-
func testRefusesWhenOnlyInvisibleOrEmptyWindowsRemain() {
47+
func testRefusesWhenOnlyInvisibleOrEmptyWindowsRemain() throws {
4848
let windows = [
4949
windowInfo(id: 1, onScreen: false),
5050
windowInfo(id: 2, alpha: 0),
5151
windowInfo(id: 3, width: 0),
5252
windowInfo(id: 4, height: 0),
5353
]
54-
let frontID = frontWindow(pid: 42, windowInfo: windows)?.number
54+
let frontID = try frontWindow(pid: 42, windowInfo: windows)?.number
5555
let selected = frontCaptureWindow([5], frontWindowID: frontID, sharingState: 1, windowID: { $0 })
5656

57-
XCTAssertNil(frontWindowFrame(pid: 42, windowInfo: windows))
57+
XCTAssertNil(try frontWindowFrame(pid: 42, windowInfo: windows))
5858
XCTAssertNil(frontID)
5959
XCTAssertThrowsError(try selected.get()) { error in
6060
XCTAssertEqual(error as? AppWindowScreenshotRefusal, .windowNotFound)
6161
}
6262
}
6363

64+
func testUnreadableWindowListRefusesBeforeCaptureOrScrollCanReportNoWindow() {
65+
XCTAssertThrowsError(try frontWindow(pid: 42, windowInfo: nil)) { error in
66+
guard case HelperError.commandFailed(_, let details) = error else {
67+
return XCTFail("expected a typed window-list refusal, got \(error)")
68+
}
69+
XCTAssertEqual(details["reason"], "window-list-unreadable")
70+
}
71+
XCTAssertThrowsError(try frontWindowFrame(pid: 42, windowInfo: nil)) { error in
72+
guard case HelperError.commandFailed(_, let details) = error else {
73+
return XCTFail("expected a typed window-list refusal, got \(error)")
74+
}
75+
XCTAssertEqual(details["reason"], "window-list-unreadable")
76+
}
77+
XCTAssertNil(try frontWindow(pid: 42, windowInfo: []))
78+
XCTAssertNil(try frontWindowFrame(pid: 42, windowInfo: []))
79+
}
80+
6481
func testSelectsTheFrontWindowEvenWhenAnotherIsLarger() {
6582
let candidates = [Candidate(id: 7, area: 10_000), Candidate(id: 3, area: 100)]
6683

@@ -79,9 +96,9 @@ final class AppWindowScreenshotTests: XCTestCase {
7996
}
8097
}
8198

82-
func testCaptureExclusionSurvivesWindowSelectionAndNeverSubstitutesAnotherWindow() {
99+
func testCaptureExclusionSurvivesWindowSelectionAndNeverSubstitutesAnotherWindow() throws {
83100
let windows = [windowInfo(id: 3, sharingState: 0), windowInfo(id: 7)]
84-
let front = frontWindow(pid: 42, windowInfo: windows)
101+
let front = try frontWindow(pid: 42, windowInfo: windows)
85102
XCTAssertEqual(front?.number, 3)
86103
XCTAssertEqual(front?.sharingState, 0)
87104

@@ -95,7 +112,7 @@ final class AppWindowScreenshotTests: XCTestCase {
95112

96113
var entry = windowInfo(id: 3)
97114
entry.removeValue(forKey: kCGWindowSharingState as String)
98-
let unknown = frontWindow(pid: 42, windowInfo: [entry])
115+
let unknown = try frontWindow(pid: 42, windowInfo: [entry])
99116
let selected = frontCaptureWindow(
100117
[3], frontWindowID: unknown?.number, sharingState: unknown?.sharingState, windowID: { $0 })
101118
XCTAssertEqual(try selected.get(), 3)

‎apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ private struct HelperOutcomesFixture: Decodable {
2727
let refusalReasons: [String]
2828
let deliveryMechanisms: [String]
2929
let screenshotRefusalReasons: [String]
30+
let windowListRefusalReasons: [String]
3031
}
3132

3233
/// `#filePath` is `<repo>/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/<file>`.
@@ -42,6 +43,7 @@ final class BackgroundInteractionTests: XCTestCase {
4243
from: Data(
4344
contentsOf: repoRoot.appendingPathComponent("contracts/fixtures/macos-native-helper-outcomes.json"))
4445
)
46+
XCTAssertEqual(WindowListRefusal.allCases.map(\.rawValue), fixture.windowListRefusalReasons)
4547
XCTAssertEqual(BackgroundRefusal.allCases.map(\.rawValue), fixture.refusalReasons)
4648
XCTAssertEqual(BackgroundDeliveryMechanism.allCases.map(\.rawValue), fixture.deliveryMechanisms)
4749
XCTAssertEqual(

‎contracts/fixtures/macos-native-helper-outcomes.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"description": "Vocabulary the macOS helper reports for app-surface actions. Swift: BackgroundRefusal, BackgroundDeliveryMechanism, and AppWindowScreenshotRefusal in apple/macos-helper. TypeScript: the macOS helper client in packages/platform-apple/src/os/macos/helper.ts.",
2+
"description": "Vocabulary the macOS helper reports for app-surface actions. Swift: BackgroundRefusal, BackgroundDeliveryMechanism, WindowListRefusal, and AppWindowScreenshotRefusal in apple/macos-helper. TypeScript: the macOS helper client in packages/platform-apple/src/os/macos/helper.ts.",
33
"refusalReasons": [
44
"background-pointer-gesture",
55
"no-accessible-target",
@@ -19,5 +19,6 @@
1919
"window-not-found",
2020
"front-window-not-shareable",
2121
"front-window-capture-excluded"
22-
]
22+
],
23+
"windowListRefusalReasons": ["window-list-unreadable"]
2324
}

‎packages/platform-apple/src/os/macos/helper.test.ts‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,14 @@ import { readFileSync } from 'node:fs';
77
import {
88
MACOS_DELIVERY_MECHANISMS,
99
MACOS_HELPER_REFUSAL_REASONS,
10+
MACOS_WINDOW_LIST_REFUSAL_REASONS,
1011
MACOS_SCREENSHOT_REFUSAL_REASONS,
1112
macOsClickScheduleMs,
1213
runMacOsPressAction,
1314
runMacOsReadTextAction,
1415
runMacOsScreenshotAction,
1516
runMacOsSnapshotAction,
17+
runMacOsScrollAction,
1618
} from './helper.ts';
1719

1820
const desktop = macOsHelperSurface('desktop', 'xctest')!;
@@ -312,6 +314,51 @@ test('helper entry points accept only an owner-routed surface', () => {
312314
assert.equal(typeof widenedCalls, 'function');
313315
});
314316

317+
for (const action of ['screenshot', 'scroll'] as const) {
318+
for (const reason of ['window-list-unreadable', undefined]) {
319+
test(`${action} maps an unreadable window list by reason, not error text (${reason})`, async () => {
320+
const provider = createLocalAppleToolProvider({
321+
macosHelper: {
322+
run: async () => ({
323+
exitCode: 1,
324+
stdout: JSON.stringify({
325+
ok: false,
326+
error: {
327+
message: "the app's on-screen windows could not be read",
328+
details: { reason },
329+
},
330+
}),
331+
stderr: '',
332+
}),
333+
},
334+
});
335+
await assert.rejects(
336+
withAppleToolProvider(provider, async () =>
337+
action === 'screenshot'
338+
? runMacOsScreenshotAction('/tmp/out.png', {
339+
surface: nativeApp,
340+
bundleId: 'com.apple.TextEdit',
341+
})
342+
: runMacOsScrollAction('down', { bundleId: 'com.apple.TextEdit' }),
343+
),
344+
(error: unknown) => {
345+
assert.ok(error instanceof AppError);
346+
assert.equal(error.code, 'COMMAND_FAILED');
347+
assert.equal(error.details?.reason, reason);
348+
if (reason) {
349+
assert.match(String(error.details?.hint), /window list could not be read/);
350+
assert.match(String(error.details?.hint), /Retry the command/);
351+
assert.doesNotMatch(String(error.details?.hint), /Open or unminimize|XCTest/);
352+
} else {
353+
assert.equal(error.details?.hint, undefined);
354+
}
355+
return true;
356+
},
357+
);
358+
});
359+
}
360+
}
361+
315362
test('the helper vocabulary matches the table the Swift helper is held to', () => {
316363
const fixture = JSON.parse(
317364
readFileSync(
@@ -325,7 +372,9 @@ test('the helper vocabulary matches the table the Swift helper is held to', () =
325372
refusalReasons: string[];
326373
deliveryMechanisms: string[];
327374
screenshotRefusalReasons: string[];
375+
windowListRefusalReasons: string[];
328376
};
377+
assert.deepEqual([...MACOS_WINDOW_LIST_REFUSAL_REASONS], fixture.windowListRefusalReasons);
329378
assert.deepEqual([...MACOS_HELPER_REFUSAL_REASONS], fixture.refusalReasons);
330379
assert.deepEqual([...MACOS_DELIVERY_MECHANISMS], fixture.deliveryMechanisms);
331380
assert.deepEqual([...MACOS_SCREENSHOT_REFUSAL_REASONS], fixture.screenshotRefusalReasons);

‎packages/platform-apple/src/os/macos/helper.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,7 @@ const MACOS_SCREENSHOT_REFUSAL_HINTS: Record<MacOsScreenshotRefusalReason, strin
160160
'front-window-capture-excluded':
161161
"The app excludes the front window from screen capture. Run `snapshot` to read the window's UI tree; retrying the excluded window will not help.",
162162
};
163+
export const MACOS_WINDOW_LIST_REFUSAL_REASONS = ['window-list-unreadable'] as const;
163164
export const MACOS_DELIVERY_MECHANISMS = [
164165
'ax-press',
165166
'ax-focus',
@@ -372,6 +373,7 @@ async function runMacOsHelper<T extends Record<string, unknown>>(
372373
parsed && !parsed.ok
373374
? (parsed.error?.message ?? `macOS helper exited with code ${result.exitCode}`)
374375
: stdout || result.stderr.trim() || `macOS helper exited with code ${result.exitCode}`;
376+
const details = parsed && !parsed.ok ? parsed.error?.details : undefined;
375377
// exec-guard-allow: the message is already built from the helper's JSON error
376378
// envelope (or its output); a stderr excerpt would only degrade it, and the
377379
// throw is reachable at exit 0 when the envelope reports ok=false.
@@ -381,7 +383,10 @@ async function runMacOsHelper<T extends Record<string, unknown>>(
381383
stdout: result.stdout,
382384
stderr: result.stderr,
383385
exitCode: result.exitCode,
384-
...(parsed && !parsed.ok ? parsed.error?.details : {}),
386+
...details,
387+
...(details?.reason === MACOS_WINDOW_LIST_REFUSAL_REASONS[0]
388+
? { hint: "The app's on-screen window list could not be read. Retry the command." }
389+
: {}),
385390
});
386391
}
387392

0 commit comments

Comments
 (0)