Skip to content

Commit 9c5740b

Browse files
committed
fix(apple): confirm Device Hub identity before choosing its window
1 parent a734856 commit 9c5740b

7 files changed

Lines changed: 125 additions & 81 deletions

File tree

‎apple/macos-helper/Sources/AgentDeviceMacOSDeviceHub/DeviceHubWindowInventory.swift‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,15 @@ public struct DeviceHubWindowInventory<Window> {
3434
}
3535
}
3636

37+
public func identifiedWindow<Row>(
38+
using identify: (DeviceHubWindowCandidate<Window>) -> Row?
39+
) -> (candidate: DeviceHubWindowCandidate<Window>, row: Row)? {
40+
for candidate in candidates {
41+
if let row = identify(candidate) { return (candidate, row) }
42+
}
43+
return nil
44+
}
45+
3746
public var failureReason: DeviceHubWindowFailureReason {
3847
failures.isEmpty ? .missing : .readFailed
3948
}

‎apple/macos-helper/Sources/AgentDeviceMacOSHelper/DeviceHubPose.swift‎

Lines changed: 15 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,13 @@ func handleDeviceHub(arguments: [String]) throws -> any Encodable {
5151
}
5252
let discovery = try discoverDeviceHubWindow(udid: udid, deviceName: deviceName)
5353
let window = discovery.window
54-
let appElement = AXUIElementCreateApplication(discovery.processID)
55-
let selected = try selectDevice(udid: udid, deviceName: deviceName, in: window, appElement: appElement)
54+
defer {
55+
if discovery.revealedSidebar {
56+
_ = setDeviceHubSidebar(visible: false, window: window,
57+
deadline: ProcessInfo.processInfo.systemUptime + 0.5)
58+
}
59+
}
60+
let selected = try selectDevice(row: discovery.row, deviceName: deviceName, in: window)
5661
let windowTitle = stringAttribute(window, attribute: kAXTitleAttribute as String) ?? ""
5762
guard let button = awaitPoseButton(in: window, description: control) else {
5863
throw HelperError.commandFailed(
@@ -123,30 +128,13 @@ func sendDeviceHubReopenEvent(to pid: pid_t, timeout: TimeInterval) throws {
123128
/// simulators sharing a name cannot confuse. A window already titled with the device still gets
124129
/// the selection, because the title alone cannot tell such twins apart.
125130
private func selectDevice(
126-
udid: String,
131+
row: AXUIElement,
127132
deviceName: String,
128-
in window: AXUIElement,
129-
appElement: AXUIElement
133+
in window: AXUIElement
130134
) throws -> Bool {
131135
let deadline = ProcessInfo.processInfo.systemUptime + deviceHubSettleDeadline
132-
var shownSidebar = false
133-
var row = deviceHubDeviceRow(udid: udid, in: window, deadline: deadline)
134-
if row == nil, showSidebar(appElement: appElement) {
135-
shownSidebar = true
136-
while row == nil, ProcessInfo.processInfo.systemUptime < deadline {
137-
Thread.sleep(forTimeInterval: deviceHubPoll)
138-
row = deviceHubDeviceRow(udid: udid, in: window, deadline: deadline)
139-
}
140-
}
141-
defer {
142-
if shownSidebar { _ = pressMenuItem(appElement: appElement, menu: "View", item: "Hide Sidebar") }
143-
}
144-
guard let row else {
145-
throw HelperError.commandFailed(
146-
"Device Hub lists no device \(udid) in its sidebar",
147-
details: ["reason": "device-hub-device-missing", "udid": udid, "deviceName": deviceName]
148-
)
149-
}
136+
AXUIElementSetMessagingTimeout(row, 0.25)
137+
AXUIElementSetMessagingTimeout(window, 0.25)
150138
let status = AXUIElementSetAttributeValue(row, kAXSelectedAttribute as CFString, kCFBooleanTrue)
151139
guard status == .success else {
152140
throw HelperError.commandFailed(
@@ -170,7 +158,7 @@ private func selectDevice(
170158
}
171159

172160
func deviceHubDeviceRow(udid: String, in window: AXUIElement, deadline: TimeInterval) -> AXUIElement? {
173-
guard let label = findElement(root: window, depth: 0, deadline: deadline, where: {
161+
guard let label = findDeviceHubElement(root: window, depth: 0, deadline: deadline, where: {
174162
stringAttribute($0, attribute: "AXIdentifier") == deviceHubDeviceRowIdentifier(udid: udid)
175163
}) else {
176164
return nil
@@ -186,26 +174,6 @@ func deviceHubDeviceRow(udid: String, in window: AXUIElement, deadline: TimeInte
186174
return nil
187175
}
188176

189-
private func showSidebar(appElement: AXUIElement) -> Bool {
190-
return pressMenuItem(appElement: appElement, menu: "View", item: "Show Sidebar")
191-
}
192-
193-
private func pressMenuItem(appElement: AXUIElement, menu: String, item: String) -> Bool {
194-
guard let menuBar = elementAttribute(appElement, attribute: kAXMenuBarAttribute as String) else {
195-
return false
196-
}
197-
for menuBarItem in children(of: menuBar)
198-
where stringAttribute(menuBarItem, attribute: kAXTitleAttribute as String) == menu {
199-
for submenu in children(of: menuBarItem) {
200-
for menuItem in children(of: submenu)
201-
where stringAttribute(menuItem, attribute: kAXTitleAttribute as String) == item {
202-
return AXUIElementPerformAction(menuItem, kAXPressAction as CFString) == .success
203-
}
204-
}
205-
}
206-
return false
207-
}
208-
209177
/// The action bar is rebuilt for the device the window shows, so right after a sidebar selection
210178
/// the window already carries the new title while the pose controls are still being laid out.
211179
/// The controls are therefore awaited, not looked up once.
@@ -224,13 +192,13 @@ private func awaitPoseButton(in window: AXUIElement, description: String) -> AXU
224192
/// name; the simulated screen inside the same window is an iOS content group whose own buttons
225193
/// carry app labels, never these three.
226194
private func poseButton(in window: AXUIElement, description: String, deadline: TimeInterval) -> AXUIElement? {
227-
return findElement(root: window, depth: 0, deadline: deadline) {
195+
return findDeviceHubElement(root: window, depth: 0, deadline: deadline) {
228196
stringAttribute($0, attribute: kAXRoleAttribute as String) == "AXButton"
229197
&& stringAttribute($0, attribute: kAXDescriptionAttribute as String) == description
230198
}
231199
}
232200

233-
private func findElement(
201+
func findDeviceHubElement(
234202
root: AXUIElement,
235203
depth: Int,
236204
deadline: TimeInterval,
@@ -251,7 +219,7 @@ private func findElement(
251219
if stringAttribute(child, attribute: kAXSubroleAttribute as String) == "iOSContentGroup" {
252220
continue
253221
}
254-
if let nested = findElement(root: child, depth: depth + 1, deadline: deadline, where: matches) {
222+
if let nested = findDeviceHubElement(root: child, depth: depth + 1, deadline: deadline, where: matches) {
255223
return nested
256224
}
257225
}
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
import AgentDeviceMacOSDeviceHub
2+
import ApplicationServices
3+
import Foundation
4+
5+
struct IdentifiedDeviceHubRow {
6+
let row: AXUIElement
7+
let revealedSidebar: Bool
8+
}
9+
10+
func identifyDeviceHubRow(
11+
udid: String,
12+
candidate: DeviceHubWindowCandidate<AXUIElement>,
13+
deadline: TimeInterval
14+
) -> IdentifiedDeviceHubRow? {
15+
let now = ProcessInfo.processInfo.systemUptime
16+
let visibleRowDeadline = now + max(0, deadline - now) / 3
17+
if let row = deviceHubDeviceRow(udid: udid, in: candidate.window, deadline: visibleRowDeadline) {
18+
return IdentifiedDeviceHubRow(row: row, revealedSidebar: false)
19+
}
20+
// Keep time to undo a sidebar reveal on a candidate that does not contain the device.
21+
let searchDeadline = deadline - 0.25
22+
guard setDeviceHubSidebar(visible: true, window: candidate.window,
23+
deadline: searchDeadline) else { return nil }
24+
var found: AXUIElement?
25+
defer {
26+
if found == nil {
27+
_ = setDeviceHubSidebar(visible: false, window: candidate.window,
28+
deadline: deadline)
29+
}
30+
}
31+
repeat {
32+
found = deviceHubDeviceRow(udid: udid, in: candidate.window, deadline: searchDeadline)
33+
if let found { return IdentifiedDeviceHubRow(row: found, revealedSidebar: true) }
34+
let remaining = searchDeadline - ProcessInfo.processInfo.systemUptime
35+
if remaining <= 0 { return nil }
36+
Thread.sleep(forTimeInterval: min(0.05, remaining))
37+
} while ProcessInfo.processInfo.systemUptime < searchDeadline
38+
return nil
39+
}
40+
41+
func setDeviceHubSidebar(
42+
visible: Bool, window: AXUIElement, deadline: TimeInterval
43+
) -> Bool {
44+
let description = visible ? "Show Sidebar" : "Hide Sidebar"
45+
guard let button = findDeviceHubElement(root: window, depth: 0, deadline: deadline, where: {
46+
stringAttribute($0, attribute: kAXRoleAttribute as String) == "AXButton"
47+
&& stringAttribute($0, attribute: kAXDescriptionAttribute as String) == description
48+
}) else { return false }
49+
let remaining = deadline - ProcessInfo.processInfo.systemUptime
50+
guard remaining > 0 else { return false }
51+
AXUIElementSetMessagingTimeout(button, Float(min(0.1, remaining)))
52+
return AXUIElementPerformAction(button, kAXPressAction as CFString) == .success
53+
}

‎apple/macos-helper/Sources/AgentDeviceMacOSHelper/DeviceHubWindowDiscovery.swift‎

Lines changed: 19 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ struct DiscoveredDeviceHubWindow {
77
let window: AXUIElement
88
let processID: pid_t
99
let reopened: Bool
10+
let row: AXUIElement
11+
let revealedSidebar: Bool
1012
}
1113

1214
func discoverDeviceHubWindow(udid: String, deviceName: String) throws -> DiscoveredDeviceHubWindow {
@@ -40,11 +42,22 @@ func discoverDeviceHubWindow(udid: String, deviceName: String) throws -> Discove
4042
))
4143
}
4244
}
43-
if let candidate = preferredDeviceHubWindow(in: inventory, udid: udid, deviceName: deviceName, deadline: deadline) {
45+
var remainingCandidates = inventory.candidates.count
46+
let identified = inventory.identifiedWindow { candidate in
47+
let now = ProcessInfo.processInfo.systemUptime
48+
// Reserve half of the remaining budget for reopen and subsequent discovery passes.
49+
let candidateDeadline = now + max(0, deadline - now) / Double(max(1, remainingCandidates)) / 2
50+
remainingCandidates -= 1
51+
return identifyDeviceHubRow(udid: udid, candidate: candidate, deadline: candidateDeadline)
52+
}
53+
if let identified {
54+
let candidate = identified.candidate
4455
return DiscoveredDeviceHubWindow(
4556
window: candidate.window,
4657
processID: candidate.processID,
47-
reopened: reopened.contains(candidate.processID)
58+
reopened: reopened.contains(candidate.processID),
59+
row: identified.row.row,
60+
revealedSidebar: identified.row.revealedSidebar
4861
)
4962
}
5063
for pid in inventory.emptyProcessIDs where !reopened.contains(pid) && reopenFailures[pid] == nil {
@@ -66,12 +79,14 @@ func discoverDeviceHubWindow(udid: String, deviceName: String) throws -> Discove
6679
}
6780
let statuses = inventory.failures.sorted { $0.key < $1.key }
6881
.map { "\($0.key):\($0.value)" }.joined(separator: ",")
82+
let deviceMissing = !inventory.candidates.isEmpty && inventory.failures.isEmpty
6983
throw HelperError.commandFailed(
70-
inventory.failures.isEmpty
84+
deviceMissing ? "Device Hub lists no device \(udid) in its accessible windows" : inventory.failures.isEmpty
7185
? "Device Hub reported no accessible device windows"
7286
: "Could not read Device Hub's accessible windows",
7387
details: [
74-
"reason": inventory.failureReason.rawValue,
88+
"reason": deviceMissing ? "device-hub-device-missing" : inventory.failureReason.rawValue,
89+
"udid": udid,
7590
"deviceName": deviceName,
7691
"processIDs": processIDs.map(String.init).joined(separator: ","),
7792
"axWindowReadStatuses": statuses,
@@ -80,34 +95,6 @@ func discoverDeviceHubWindow(udid: String, deviceName: String) throws -> Discove
8095
)
8196
}
8297

83-
private func preferredDeviceHubWindow(
84-
in inventory: DeviceHubWindowInventory<AXUIElement>,
85-
udid: String,
86-
deviceName: String,
87-
deadline: TimeInterval
88-
) -> DeviceHubWindowCandidate<AXUIElement>? {
89-
var identified: DeviceHubWindowCandidate<AXUIElement>?
90-
for (index, candidate) in inventory.candidates.enumerated() {
91-
let now = ProcessInfo.processInfo.systemUptime
92-
guard now < deadline else { break }
93-
let candidateDeadline = now + (deadline - now) / Double(inventory.candidates.count - index)
94-
guard let row = deviceHubDeviceRow(udid: udid, in: candidate.window, deadline: candidateDeadline) else { continue }
95-
var selected: CFTypeRef?
96-
if AXUIElementCopyAttributeValue(row, kAXSelectedAttribute as CFString, &selected) == .success,
97-
(selected as? Bool) == true {
98-
return candidate
99-
}
100-
if identified == nil { identified = candidate }
101-
}
102-
return identified ?? inventory.candidates.first { candidate in
103-
guard ProcessInfo.processInfo.systemUptime < deadline else { return false }
104-
return deviceHubWindowShows(
105-
deviceName: deviceName,
106-
title: stringAttribute(candidate.window, attribute: kAXTitleAttribute as String)
107-
)
108-
} ?? inventory.candidates.first
109-
}
110-
11198
private func deviceHubProcessIdentifiers() -> [pid_t] {
11299
let byteCount = proc_listpids(UInt32(PROC_ALL_PIDS), 0, nil, 0)
113100
guard byteCount > 0 else { return [] }

‎apple/macos-helper/Tests/AgentDeviceMacOSDeviceHubTests/DeviceHubWindowInventoryTests.swift‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,29 @@ final class DeviceHubWindowInventoryTests: XCTestCase {
3838
XCTAssertEqual(inventory.candidates.map(\.processID), [10, 10, 20])
3939
}
4040

41+
func testUnrelatedWindowDoesNotStopIdentificationInAnotherProcess() {
42+
var inventory = DeviceHubWindowInventory<String>()
43+
inventory.record(processID: 10, read: .available(["same-name-wrong-device"]))
44+
inventory.record(processID: 20, read: .available(["target-window"]))
45+
var visited: [String] = []
46+
let identified = inventory.identifiedWindow { candidate -> String? in
47+
visited.append(candidate.window)
48+
return candidate.processID == 20 ? "TableRow.Device.target" : nil
49+
}
50+
XCTAssertEqual(visited, ["same-name-wrong-device", "target-window"])
51+
XCTAssertEqual(identified?.candidate.processID, 20)
52+
XCTAssertEqual(identified?.row, "TableRow.Device.target")
53+
}
54+
55+
func testUnrelatedWindowLeavesEmptyProcessEligibleForReopen() {
56+
var inventory = DeviceHubWindowInventory<String>()
57+
inventory.record(processID: 10, read: .available(["unrelated-window"]))
58+
inventory.record(processID: 20, read: .available([]))
59+
let identified = inventory.identifiedWindow { _ -> String? in nil }
60+
XCTAssertNil(identified)
61+
XCTAssertEqual(inventory.emptyProcessIDs, [20])
62+
}
63+
4164
func testSecondaryAndSpanningWindowsUseGlobalDisplayBounds() {
4265
let displays: [(id: UInt32, bounds: CGRect)] = [
4366
(1, CGRect(x: 0, y: 0, width: 1512, height: 982)),

‎docs/adr/0025-foldable-apple-panels.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,11 @@ retry budget. These are application-wide windows; discovery does not click throu
120120
move windows to the main display. A failed accessibility read retains its AX status and returns
121121
`device-hub-window-read-failed`, rather than masquerading as an empty window list. Only successful
122122
empty reads trigger a reopen. A matching UDID sidebar row is required before pressing, including
123-
when a window title already matches. Diagnostics record the chosen process, window bounds, and
123+
when a window title already matches. An unrelated window does not stop attempts on other
124+
windows or empty processes. A hidden sidebar is revealed through that window’s own toolbar button;
125+
application-wide menus cannot redirect the action to another window. Sidebar reads and
126+
restoration use bounded AX calls. Discovery passes the proven row
127+
to selection instead of searching twice. Diagnostics record the chosen process, window bounds, and
124128
intersecting host display IDs; negative display origins and spanning windows are supported.
125129

126130
## Pose is derived from panel power, and official control does not exist

‎website/docs/docs/commands.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ agent-device fold open
9292
- `action-button` does not activate or relaunch the session's app, and it takes no `--settle`: pressing a hardware button is not a navigation, so the app stays where it was.
9393
- `action-button` reports that the press was dispatched, not what the system did with it. Simulators run no Shortcuts and no App Intents, so what a press triggers can only be verified on a physical iPhone; on a Simulator the command proves the press was accepted and that the session app was not brought forward.
9494
- `fold <closed|half-open|open>` puts a foldable iPhone simulator (iPhone Duo) into a hinge pose. No official host API sets a pose (ADR 0025), so the command presses the pose control in the Xcode Device Hub window through macOS accessibility, then reads the hinge angle back with `devicectl device motion hinge-angle` and reports the pose only when that reading agrees: `closed` is 0°, `open` is 180°, and `half-open` is any angle between them (Device Hub's Book preset, 130° on iOS 27.1), reported once the hinge stops moving or when the read budget ends while it still reads half-open. The response names the panel the device now lights and its point size, because a pose change moves the app to a different coordinate space: re-snapshot afterwards, and never carry refs or coordinates across a `fold`.
95-
- `fold` is simulator-only and needs Xcode 27.1 or newer with Device Hub. The host needs Accessibility permission (`settings permission grant accessibility --platform macos`), and Device Hub must list the simulator; the command reopens Device Hub's window when it shows none, selects the device through the sidebar row keyed by its UDID, and restores the sidebar afterwards. A single-panel simulator such as an iPhone 17 fails with `UNSUPPORTED_OPERATION`; Android, web, Linux, HarmonyOS, Vega, physical devices, and the tvOS, macOS, and visionOS leaves refuse it.
95+
- `fold` is simulator-only and needs Xcode 27.1 or newer with Device Hub. The host needs Accessibility permission (`settings permission grant accessibility --platform macos`), and Device Hub must list the simulator; the command reopens Device Hub's window when it shows none, selects the device through the sidebar row keyed by its UDID, and attempts to restore any sidebar it revealed afterwards within a bounded cleanup budget. Sidebar actions use the selected window’s toolbar button. A single-panel simulator such as an iPhone 17 fails with `UNSUPPORTED_OPERATION`; Android, web, Linux, HarmonyOS, Vega, physical devices, and the tvOS, macOS, and visionOS leaves refuse it.
9696
- `fold` discovers windows across running Device Hub processes without requiring main-display placement. Failed accessibility reads report `device-hub-window-read-failed` with AX status and process/display diagnostics; they do not count as an empty window list. Successful pose-control diagnostics include the selected process, window bounds, and intersecting host display IDs.
9797
- `fold` costs one bounded hinge stream per read, and devicectl's smallest stream is five seconds: `closed` and `open` take about ten seconds, `half-open` about sixteen, because the hinge animates and the command waits for it to stop. A refusal never names the pose that was asked for: only a hinge whose last reading is some other pose fails, with `COMMAND_FAILED` and `reason: fold-pose-unverified`, naming the angle CoreDevice still reports.
9898
- `action-button` is not a cheap command to loop. On an iPhone 17 Pro Simulator the press itself spent about five seconds inside XCUITest, while `home` and `app-switcher` on the same session took under two seconds each.

0 commit comments

Comments
 (0)