Skip to content

Commit 6d40fbc

Browse files
committed
refactor(apple): move macOS refusal hints into the helper
The helper owns each refusal's reason and remedy, so its details envelope carries the hint and the TypeScript client needs no lookup table, wrapper, or special case. Collapse the one-case WindowListRefusal enum and the screenshot Result mapping into the throw sites that already own them, and keep the front-window selection as one plain function. Press, fill, and their window fallback use the non-throwing path; requireAppWindowPoint has already refused an unreadable list there. The front-window help sentence states the layer-0 rule and the alert exclusion once.
1 parent 70acf88 commit 6d40fbc

9 files changed

Lines changed: 152 additions & 213 deletions

File tree

‎apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift‎

Lines changed: 48 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -10,18 +10,15 @@ func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) thro
1010
throw HelperError.commandFailed("app window screenshots require macOS 14 or newer")
1111
}
1212
let pid = app.processIdentifier
13-
let frontWindow = try frontWindow(pid: pid)
14-
let frontWindowID = frontWindow?.number
13+
let front = try frontWindow(pid: pid)
1514
let content = try awaitShareableContent()
1615
let candidates = content.windows.filter { $0.owningApplication?.processID == pid }
1716
let window: SCWindow
18-
let selection = frontCaptureWindow(
19-
candidates, frontWindowID: frontWindowID, sharingState: frontWindow?.sharingState, windowID: { Int($0.windowID) })
20-
switch selection {
21-
case .success(let front):
22-
window = front
17+
switch frontShareableWindow(candidates, front: front, windowID: { Int($0.windowID) }) {
18+
case .success(let shareable):
19+
window = shareable
2320
case .failure(let refusal):
24-
throw screenshotRefusal(refusal, app: app, frontWindowID: frontWindowID)
21+
throw refusal.error(bundleId: app.bundleIdentifier)
2522
}
2623

2724
let filter = SCContentFilter(desktopIndependentWindow: window)
@@ -50,52 +47,56 @@ func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) thro
5047
try writePNG(capturedImage, to: outPath)
5148
}
5249

53-
/// Why an app screenshot refuses. The capture is the app's front window or nothing: another
54-
/// window of the app would show state the user is not looking at. Pinned with the host by
55-
/// `contracts/fixtures/macos-native-helper-outcomes.json`.
56-
enum AppWindowScreenshotRefusal: String, CaseIterable, Error {
57-
case windowNotFound = "window-not-found"
58-
case frontWindowNotShareable = "front-window-not-shareable"
59-
case frontWindowCaptureExcluded = "front-window-capture-excluded"
50+
/// Why an app screenshot refuses, and the error the host reports for it: reason, remedy, and the
51+
/// front window it refused. The capture is the app's front window or nothing; another window of
52+
/// the app would show state the user is not looking at.
53+
enum AppWindowScreenshotRefusal: Error {
54+
case windowNotFound
55+
case frontWindowNotShareable(frontWindowId: Int)
56+
case frontWindowCaptureExcluded(frontWindowId: Int)
57+
58+
func error(bundleId: String?) -> HelperError {
59+
var details = ["bundleId": bundleId ?? ""]
60+
let message: String
61+
switch self {
62+
case .windowNotFound:
63+
details["reason"] = "window-not-found"
64+
details["hint"] =
65+
"Open or unminimize a window of the app on the current Space, then retry the screenshot."
66+
message = "screenshot could not find an on-screen window for the app"
67+
case .frontWindowCaptureExcluded(let frontWindowId):
68+
details["reason"] = "front-window-capture-excluded"
69+
details["frontWindowId"] = String(frontWindowId)
70+
details["hint"] =
71+
"The app excludes the front window from screen capture. Run `snapshot` to read the window's UI tree; retrying the excluded window will not help."
72+
message =
73+
"screenshot could not capture the app's front window: the app excludes it from capture"
74+
case .frontWindowNotShareable(let frontWindowId):
75+
details["reason"] = "front-window-not-shareable"
76+
details["frontWindowId"] = String(frontWindowId)
77+
details["hint"] =
78+
"The front window is unavailable to screen capture. If it is opening or closing, retry after it settles."
79+
message =
80+
"screenshot could not capture the app's front window: ScreenCaptureKit does not offer it"
81+
}
82+
return .commandFailed(message, details: details)
83+
}
6084
}
6185

62-
/// The shareable candidate whose WindowServer ID is the app's front window.
63-
func frontCaptureWindow<Window>(
86+
/// The shareable candidate that is the app's front window, or the reason the screenshot refuses.
87+
func frontShareableWindow<Window>(
6488
_ candidates: [Window],
65-
frontWindowID: Int?,
66-
sharingState: Int?,
89+
front: OnScreenWindow?,
6790
windowID: (Window) -> Int
6891
) -> Result<Window, AppWindowScreenshotRefusal> {
69-
guard let frontWindowID else { return .failure(.windowNotFound) }
70-
guard sharingState != 0 else { return .failure(.frontWindowCaptureExcluded) }
71-
guard let front = candidates.first(where: { windowID($0) == frontWindowID }) else {
72-
return .failure(.frontWindowNotShareable)
92+
guard let front else { return .failure(.windowNotFound) }
93+
guard front.sharingState != 0 else {
94+
return .failure(.frontWindowCaptureExcluded(frontWindowId: front.number))
7395
}
74-
return .success(front)
75-
}
76-
77-
private func screenshotRefusal(
78-
_ refusal: AppWindowScreenshotRefusal,
79-
app: NSRunningApplication,
80-
frontWindowID: Int?
81-
) -> HelperError {
82-
var details = ["reason": refusal.rawValue, "bundleId": app.bundleIdentifier ?? ""]
83-
switch refusal {
84-
case .windowNotFound:
85-
return .commandFailed("screenshot could not find an on-screen window for the app", details: details)
86-
case .frontWindowCaptureExcluded:
87-
details["frontWindowId"] = frontWindowID.map(String.init) ?? ""
88-
return .commandFailed(
89-
"screenshot could not capture the app's front window: the app excludes it from capture",
90-
details: details
91-
)
92-
case .frontWindowNotShareable:
93-
details["frontWindowId"] = frontWindowID.map(String.init) ?? ""
94-
return .commandFailed(
95-
"screenshot could not capture the app's front window: ScreenCaptureKit does not offer it",
96-
details: details
97-
)
96+
guard let shareable = candidates.first(where: { windowID($0) == front.number }) else {
97+
return .failure(.frontWindowNotShareable(frontWindowId: front.number))
9898
}
99+
return .success(shareable)
99100
}
100101

101102
@available(macOS 14.0, *)

‎apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ func pressInBackground(
9797
let point = CGPoint(x: request.x, y: request.y)
9898
try requireAppWindowPoint(point, app: app)
9999
cursor?.move(to: point)
100-
let target = try resolvePressTarget(app: app, point: point)
100+
let target = resolvePressTarget(app: app, point: point)
101101
if let target, isInMenuBar(target.element) {
102102
throw refusal(.noAccessibleTarget, "the point is on the app's menu bar", app: app)
103103
}
@@ -172,7 +172,7 @@ func fillInBackground(
172172
cursor?.move(to: point)
173173
let hit = elementAtPoint(in: app, point: point)
174174
let chain = hit.map(pressSearchChain) ?? []
175-
let fallback = try actionWindow(app: app, hit: hit).flatMap { window in
175+
let fallback = actionWindow(app: app, hit: hit).flatMap { window in
176176
smallestElement(in: window, containing: point, where: isTextInput)
177177
}
178178
if let input = chain.first(where: isTextInput) ?? fallback, isInMenuBar(input) {
@@ -276,12 +276,12 @@ private enum PressTarget {
276276

277277
/// What a click at a point means. The app's own hit test answers first; web content often answers
278278
/// with a wrapper group, so the snapshot geometry decides when the hit names no control.
279-
private func resolvePressTarget(app: NSRunningApplication, point: CGPoint) throws -> PressTarget? {
279+
private func resolvePressTarget(app: NSRunningApplication, point: CGPoint) -> PressTarget? {
280280
let hit = elementAtPoint(in: app, point: point)
281281
let chain = hit.map(pressSearchChain) ?? []
282282
if let input = chain.first(where: isTextInput) { return .focus(input) }
283283
if let control = chain.first(where: isPressableControl) { return .press(control) }
284-
if let found = try actionWindow(app: app, hit: hit).flatMap({
284+
if let found = actionWindow(app: app, hit: hit).flatMap({
285285
smallestElement(in: $0, containing: point, where: { isTextInput($0) || isPressableControl($0) })
286286
}) {
287287
return isTextInput(found) ? .focus(found) : .press(found)
@@ -360,13 +360,14 @@ private func pressSearchChain(from hit: AXUIElement) -> [AXUIElement] {
360360

361361
/// The one window a point action may act in: the window the app's hit test landed in, else the
362362
/// app's front on-screen window. A fallback target never comes from a window behind it, and the
363-
/// response names the window so the host can check it.
364-
private func actionWindow(app: NSRunningApplication, hit: AXUIElement?) throws -> AXUIElement? {
363+
/// response names the window so the host can check it. The caller's `requireAppWindowPoint` has
364+
/// already read the window list, so a list that turned unreadable here is treated as no window.
365+
private func actionWindow(app: NSRunningApplication, hit: AXUIElement?) -> AXUIElement? {
365366
if let hit {
366367
if role(of: hit) == "AXWindow" { return hit }
367368
if let window = elementAttribute(hit, attribute: kAXWindowAttribute as String) { return window }
368369
}
369-
guard let front = try frontWindowFrame(pid: app.processIdentifier) else { return nil }
370+
guard let front = try? frontWindowFrame(pid: app.processIdentifier) else { return nil }
370371
return windows(of: AXUIElementCreateApplication(app.processIdentifier)).first { window in
371372
guard let rect = rectAttribute(window) else { return false }
372373
return abs(rect.x - front.minX) < 1 && abs(rect.y - front.minY) < 1
@@ -512,15 +513,17 @@ func appWindowInfo() -> [[String: Any]]? {
512513
as? [[String: Any]]
513514
}
514515

515-
enum WindowListRefusal: String, CaseIterable {
516-
case unreadable = "window-list-unreadable"
517-
}
518-
516+
/// The app's front on-screen window: frontmost among its eligible windows, or nil when it has
517+
/// none. A window list that cannot be read is a refusal, not an empty list: the caller must not
518+
/// report "no window" for a WindowServer it could not ask.
519519
func frontWindow(pid: pid_t, windowInfo: [[String: Any]]? = appWindowInfo()) throws -> OnScreenWindow? {
520520
guard let windowInfo else {
521521
throw HelperError.commandFailed(
522522
"the app's on-screen windows could not be read",
523-
details: ["reason": WindowListRefusal.unreadable.rawValue]
523+
details: [
524+
"reason": "window-list-unreadable",
525+
"hint": "The app's on-screen window list could not be read. Retry the command.",
526+
]
524527
)
525528
}
526529
return onScreenWindows(pid: pid, windowInfo: windowInfo).first

‎apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift‎

Lines changed: 72 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -22,22 +22,21 @@ final class AppWindowScreenshotTests: XCTestCase {
2222
windowInfo(id: 8),
2323
]
2424

25-
let frontID = try frontWindow(pid: 42, windowInfo: windows)?.number
26-
let selected = frontCaptureWindow([7, 8], frontWindowID: frontID, sharingState: 1, windowID: { $0 })
25+
let front = try frontWindow(pid: 42, windowInfo: windows)
26+
let selected = frontShareableWindow([7, 8], front: front, windowID: { $0 })
2727

2828
// Scroll and the click actionWindow fallback both consume frontWindowFrame.
2929
XCTAssertEqual(
3030
try frontWindowFrame(pid: 42, windowInfo: windows),
3131
CGRect(x: 10, y: 10, width: 20, height: 20))
32-
XCTAssertEqual(frontID, 7)
32+
XCTAssertEqual(front?.number, 7)
3333
XCTAssertEqual(try selected.get(), 7)
3434
}
3535

3636
func testSkipsAppModalAlertInFrontOfNormalLayerWindow() throws {
3737
let windows = [windowInfo(id: 8, layer: 8), windowInfo(id: 3)]
3838
let front = try frontWindow(pid: 42, windowInfo: windows)
39-
let selected = frontCaptureWindow(
40-
[8, 3], frontWindowID: front?.number, sharingState: front?.sharingState, windowID: { $0 })
39+
let selected = frontShareableWindow([8, 3], front: front, windowID: { $0 })
4140

4241
XCTAssertEqual(front?.number, 3)
4342
XCTAssertEqual(try selected.get(), 3)
@@ -51,28 +50,22 @@ final class AppWindowScreenshotTests: XCTestCase {
5150
windowInfo(id: 3, width: 0),
5251
windowInfo(id: 4, height: 0),
5352
]
54-
let frontID = try frontWindow(pid: 42, windowInfo: windows)?.number
55-
let selected = frontCaptureWindow([5], frontWindowID: frontID, sharingState: 1, windowID: { $0 })
53+
let front = try frontWindow(pid: 42, windowInfo: windows)
54+
let selected = frontShareableWindow([5], front: front, windowID: { $0 })
5655

5756
XCTAssertNil(try frontWindowFrame(pid: 42, windowInfo: windows))
58-
XCTAssertNil(frontID)
57+
XCTAssertNil(front)
5958
XCTAssertThrowsError(try selected.get()) { error in
60-
XCTAssertEqual(error as? AppWindowScreenshotRefusal, .windowNotFound)
59+
assertRefusal(error, reason: "window-not-found", hint: "Open or unminimize")
6160
}
6261
}
6362

6463
func testUnreadableWindowListRefusesBeforeCaptureOrScrollCanReportNoWindow() {
6564
XCTAssertThrowsError(try frontWindow(pid: 42, windowInfo: nil)) { error in
66-
guard case HelperError.commandFailed(_, let details) = error else {
67-
return XCTFail("expected a typed window-list refusal, got \(error)")
68-
}
69-
XCTAssertEqual(details["reason"], "window-list-unreadable")
65+
assertUnreadableList(error)
7066
}
7167
XCTAssertThrowsError(try frontWindowFrame(pid: 42, windowInfo: nil)) { error in
72-
guard case HelperError.commandFailed(_, let details) = error else {
73-
return XCTFail("expected a typed window-list refusal, got \(error)")
74-
}
75-
XCTAssertEqual(details["reason"], "window-list-unreadable")
68+
assertUnreadableList(error)
7669
}
7770
XCTAssertNil(try frontWindow(pid: 42, windowInfo: []))
7871
XCTAssertNil(try frontWindowFrame(pid: 42, windowInfo: []))
@@ -81,18 +74,22 @@ final class AppWindowScreenshotTests: XCTestCase {
8174
func testSelectsTheFrontWindowEvenWhenAnotherIsLarger() {
8275
let candidates = [Candidate(id: 7, area: 10_000), Candidate(id: 3, area: 100)]
8376

84-
let selected = frontCaptureWindow(candidates, frontWindowID: 3, sharingState: 1, windowID: \.id)
77+
let selected = frontShareableWindow(candidates, front: onScreenWindow(id: 3), windowID: \.id)
8578

8679
XCTAssertEqual(try selected.get(), Candidate(id: 3, area: 100))
8780
}
8881

8982
func testRefusesWhenTheFrontWindowIsNotShareable() {
9083
let candidates = [Candidate(id: 7, area: 10_000), Candidate(id: 8, area: 100)]
9184

92-
let selected = frontCaptureWindow(candidates, frontWindowID: 3, sharingState: 1, windowID: \.id)
85+
let selected = frontShareableWindow(candidates, front: onScreenWindow(id: 3), windowID: \.id)
9386

9487
XCTAssertThrowsError(try selected.get()) { error in
95-
XCTAssertEqual(error as? AppWindowScreenshotRefusal, .frontWindowNotShareable)
88+
assertRefusal(
89+
error,
90+
reason: "front-window-not-shareable",
91+
hint: "retry after it settles",
92+
frontWindowId: "3")
9693
}
9794
}
9895

@@ -103,31 +100,78 @@ final class AppWindowScreenshotTests: XCTestCase {
103100
XCTAssertEqual(front?.sharingState, 0)
104101

105102
for candidates in [[3, 7], [7]] {
106-
let selected = frontCaptureWindow(
107-
candidates, frontWindowID: front?.number, sharingState: front?.sharingState, windowID: { $0 })
103+
let selected = frontShareableWindow(candidates, front: front, windowID: { $0 })
108104
XCTAssertThrowsError(try selected.get()) { error in
109-
XCTAssertEqual(error as? AppWindowScreenshotRefusal, .frontWindowCaptureExcluded)
105+
assertRefusal(
106+
error,
107+
reason: "front-window-capture-excluded",
108+
hint: "Run `snapshot` to read the window",
109+
frontWindowId: "3")
110110
}
111111
}
112112

113113
var entry = windowInfo(id: 3)
114114
entry.removeValue(forKey: kCGWindowSharingState as String)
115115
let unknown = try frontWindow(pid: 42, windowInfo: [entry])
116-
let selected = frontCaptureWindow(
117-
[3], frontWindowID: unknown?.number, sharingState: unknown?.sharingState, windowID: { $0 })
116+
let selected = frontShareableWindow([3], front: unknown, windowID: { $0 })
118117
XCTAssertEqual(try selected.get(), 3)
119118
}
120119

121120
func testRefusesWhenTheAppHasNoOnScreenWindow() {
122-
let selected = frontCaptureWindow(
123-
[Candidate(id: 7, area: 10_000)], frontWindowID: nil, sharingState: nil, windowID: \.id)
121+
let selected = frontShareableWindow(
122+
[Candidate(id: 7, area: 10_000)], front: nil, windowID: \.id)
124123

125124
XCTAssertThrowsError(try selected.get()) { error in
126-
XCTAssertEqual(error as? AppWindowScreenshotRefusal, .windowNotFound)
125+
assertRefusal(error, reason: "window-not-found", hint: "Open or unminimize")
127126
}
128127
}
129128
}
130129

130+
/// The refusal carries the reason and the remedy the agent runs next, alongside the front window
131+
/// it refused.
132+
private func assertRefusal(
133+
_ error: Error,
134+
reason: String,
135+
hint: String,
136+
frontWindowId: String? = nil,
137+
file: StaticString = #filePath,
138+
line: UInt = #line
139+
) {
140+
guard let refusal = error as? AppWindowScreenshotRefusal else {
141+
return XCTFail("expected an AppWindowScreenshotRefusal, got \(error)", file: file, line: line)
142+
}
143+
guard
144+
case HelperError.commandFailed(_, let details) = refusal.error(
145+
bundleId: "com.apple.TextEdit")
146+
else {
147+
return XCTFail("expected a commandFailed refusal error", file: file, line: line)
148+
}
149+
XCTAssertEqual(details["reason"], reason, file: file, line: line)
150+
XCTAssertEqual(details["bundleId"], "com.apple.TextEdit", file: file, line: line)
151+
XCTAssertEqual(details["frontWindowId"], frontWindowId, file: file, line: line)
152+
XCTAssertTrue(
153+
(details["hint"] ?? "").contains(hint),
154+
"hint \(details["hint"] ?? "<nil>") lacks \(hint)",
155+
file: file, line: line)
156+
}
157+
158+
private func assertUnreadableList(
159+
_ error: Error, file: StaticString = #filePath, line: UInt = #line
160+
) {
161+
guard case HelperError.commandFailed(_, let details) = error else {
162+
return XCTFail("expected a typed window-list refusal, got \(error)", file: file, line: line)
163+
}
164+
XCTAssertEqual(details["reason"], "window-list-unreadable", file: file, line: line)
165+
XCTAssertEqual(
166+
details["hint"],
167+
"The app's on-screen window list could not be read. Retry the command.",
168+
file: file, line: line)
169+
}
170+
171+
private func onScreenWindow(id: Int) -> OnScreenWindow {
172+
OnScreenWindow(number: id, bounds: .zero, sharingState: 1)
173+
}
174+
131175
private func windowInfo(
132176
id: Int, pid: Int32 = 42, layer: Int = 0, onScreen: Bool = true,
133177
alpha: Double = 1, sharingState: Int = 1, width: Double = 100, height: Double = 100

‎apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,6 @@ private struct ScrollGestureFixture: Decodable {
2626
private struct HelperOutcomesFixture: Decodable {
2727
let refusalReasons: [String]
2828
let deliveryMechanisms: [String]
29-
let screenshotRefusalReasons: [String]
30-
let windowListRefusalReasons: [String]
3129
}
3230

3331
/// `#filePath` is `<repo>/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/<file>`.
@@ -43,11 +41,8 @@ final class BackgroundInteractionTests: XCTestCase {
4341
from: Data(
4442
contentsOf: repoRoot.appendingPathComponent("contracts/fixtures/macos-native-helper-outcomes.json"))
4543
)
46-
XCTAssertEqual(WindowListRefusal.allCases.map(\.rawValue), fixture.windowListRefusalReasons)
4744
XCTAssertEqual(BackgroundRefusal.allCases.map(\.rawValue), fixture.refusalReasons)
4845
XCTAssertEqual(BackgroundDeliveryMechanism.allCases.map(\.rawValue), fixture.deliveryMechanisms)
49-
XCTAssertEqual(
50-
AppWindowScreenshotRefusal.allCases.map(\.rawValue), fixture.screenshotRefusalReasons)
5146
}
5247

5348
/// The native backend scrolls as far as the runner does: its travel agrees with every case of

0 commit comments

Comments
 (0)