Skip to content

Commit 4d6f523

Browse files
committed
fix(ios): scope collapsed-tab candidates to the captured lit display
1 parent 294c7c6 commit 4d6f523

9 files changed

Lines changed: 160 additions & 42 deletions

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -93,11 +93,11 @@ extension RunnerTests {
9393

9494
/// The roots of every element read: the app, or the windows proven on the lit display. A walk that
9595
/// cannot read the list, or that records an issue (#2995), throws `APP_SCREEN_UNRESOLVED`.
96-
func elementSearchRoots(app: XCUIApplication) throws -> [XCUIElement] {
97-
guard let litPanel = requestLitPanel else { return [app] }
96+
func elementSearchRoots(root: XCUIElement, litDisplayID: UInt?) throws -> [XCUIElement] {
97+
guard let litDisplayID else { return [root] }
9898
var windows: [XCUIElement]?
9999
let walk = holdingRecordedIssues {
100-
windows = RunnerCopyLitPanelWindows(app, litPanel)?.compactMap { $0 as? XCUIElement }
100+
windows = RunnerCopyLitPanelWindows(root, litDisplayID)?.compactMap { $0 as? XCUIElement }
101101
#if AGENT_DEVICE_RUNNER_UNIT_TESTS
102102
if litPanelWindowWalkIssueForTesting {
103103
record(XCTIssue(type: .assertionFailure, compactDescription: "Window vanished during the lit-panel walk"))
@@ -118,6 +118,14 @@ extension RunnerTests {
118118
}
119119

120120
func elementQueries(app: XCUIApplication, matching type: XCUIElement.ElementType) throws -> [XCUIElementQuery] {
121-
try elementSearchRoots(app: app).map { $0.descendants(matching: type) }
121+
try elementQueries(root: app, matching: type, litDisplayID: requestLitPanel?.displayID)
122+
}
123+
124+
func elementQueries(
125+
root: XCUIElement,
126+
matching type: XCUIElement.ElementType,
127+
litDisplayID: UInt?
128+
) throws -> [XCUIElementQuery] {
129+
try elementSearchRoots(root: root, litDisplayID: litDisplayID).map { $0.descendants(matching: type) }
122130
}
123131
}

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -193,11 +193,11 @@ extension RunnerTests {
193193
) throws -> (acquisition: SnapshotAcquisition, outcome: SnapshotTierOutcome) {
194194
var outcome = SnapshotTierOutcome.completed
195195
var cachedCollapsedTabCandidates: [CollapsedTabCandidate]?
196-
func collapsedTabCandidatesOnce() -> [CollapsedTabCandidate] {
196+
func collapsedTabCandidatesOnce() throws -> [CollapsedTabCandidate] {
197197
if let cachedCollapsedTabCandidates {
198198
return cachedCollapsedTabCandidates
199199
}
200-
let candidates = collapsedTabCandidates(context: context, processingDeadline: processingDeadline)
200+
let candidates = try collapsedTabCandidates(context: context, processingDeadline: processingDeadline)
201201
if candidates == nil { outcome = .deadlineExhausted }
202202
cachedCollapsedTabCandidates = candidates ?? []
203203
return candidates ?? []
@@ -217,7 +217,7 @@ extension RunnerTests {
217217
let rootChildren = try Self.litPanelRootChildren(context.rootSnapshot, surface: context.litPanelSurface)
218218
var nodes: [RawAXNode] = [root]
219219
if Self.canDescendAtRawDepth(0, hint: hint) {
220-
appendCollapsedTabFallbackNodes(
220+
try appendCollapsedTabFallbackNodes(
221221
to: &nodes,
222222
containerSnapshot: context.rootSnapshot,
223223
resolveCandidates: collapsedTabCandidatesOnce,
@@ -274,7 +274,7 @@ extension RunnerTests {
274274
throw regularSnapshotTooLargeFailure(nodeCount: nodes.count)
275275
}
276276
if Self.canDescendAtRawDepth(depth, hint: hint) {
277-
appendCollapsedTabFallbackNodes(
277+
try appendCollapsedTabFallbackNodes(
278278
to: &nodes,
279279
containerSnapshot: snapshot,
280280
resolveCandidates: collapsedTabCandidatesOnce,

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift‎

Lines changed: 29 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -299,11 +299,11 @@ extension RunnerTests {
299299
func appendCollapsedTabFallbackNodes(
300300
to nodes: inout [RawAXNode],
301301
containerSnapshot: XCUIElementSnapshot,
302-
resolveCandidates: () -> [CollapsedTabCandidate],
302+
resolveCandidates: () throws -> [CollapsedTabCandidate],
303303
depth: Int,
304304
parentIndex: Int
305-
) {
306-
let fallbackNodes = collapsedTabFallbackNodes(
305+
) throws {
306+
let fallbackNodes = try collapsedTabFallbackNodes(
307307
for: containerSnapshot,
308308
resolveCandidates: resolveCandidates,
309309
startingIndex: nodes.count,
@@ -336,35 +336,41 @@ extension RunnerTests {
336336
let selected: Bool
337337
}
338338

339-
/// The live elements any collapsed tab container in this capture may adopt, read in one XCTest
340-
/// query: `collapsedTabCandidatePredicate` copies each match's attributes while XCTest evaluates
341-
/// it, so no match costs a further round trip. The query starts only while the time left before
342-
/// `processingDeadline` covers what this capture's root tree read took; nil means it did not start.
339+
/// Live controls collapsed tab containers may adopt, queried under the capture's proven-lit roots.
340+
/// The predicate copies attributes during each query without further round trips. Queries start
341+
/// only while the remaining slice covers the root read's duration; nil means a query could not start.
343342
@MainActor
344343
func collapsedTabCandidates(
345344
context: SnapshotTraversalContext,
346345
processingDeadline: Date
347-
) -> [CollapsedTabCandidate]? {
346+
) throws -> [CollapsedTabCandidate]? {
348347
let containerFrames = Self.snapshotDescendants(of: context.rootSnapshot)
349348
.filter(isCollapsedTabContainer)
350349
.map(\.frame)
351350
var candidates: [CollapsedTabCandidate] = []
352351
let predicate = Self.collapsedTabCandidatePredicate(containerFrames: containerFrames) { attributes in
353352
candidates.append(self.collapsedTabCandidate(attributes))
354353
}
355-
guard
356-
Self.xctestQueryCanStart(
357-
deadline: processingDeadline,
358-
now: Date(),
359-
expectedCost: context.rootReadDuration
360-
)
361-
else {
362-
NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_TAB_FALLBACK_QUERY_NOT_STARTED")
363-
return nil
354+
func canStartQuery() -> Bool {
355+
guard Self.xctestQueryCanStart(deadline: processingDeadline, now: Date(), expectedCost: context.rootReadDuration) else {
356+
NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_TAB_FALLBACK_QUERY_NOT_STARTED")
357+
return false
358+
}
359+
return true
360+
}
361+
guard canStartQuery() else { return nil }
362+
let queries = try elementQueries(
363+
root: context.queryRoot,
364+
matching: .any,
365+
litDisplayID: context.litPanelSurface?.displayID
366+
)
367+
var matches: [XCUIElement] = []
368+
for query in queries {
369+
guard canStartQuery() else { return nil }
370+
matches += snapshotElementsQuery {
371+
query.matching(predicate).allElementsBoundByIndex
372+
}.elements
364373
}
365-
let matches = snapshotElementsQuery {
366-
context.queryRoot.descendants(matching: .any).matching(predicate).allElementsBoundByIndex
367-
}.elements
368374
return matches.isEmpty ? [] : candidates
369375
}
370376

@@ -434,18 +440,18 @@ extension RunnerTests {
434440

435441
private func collapsedTabFallbackNodes(
436442
for containerSnapshot: XCUIElementSnapshot,
437-
resolveCandidates: () -> [CollapsedTabCandidate],
443+
resolveCandidates: () throws -> [CollapsedTabCandidate],
438444
startingIndex: Int,
439445
depth: Int,
440446
parentIndex: Int
441-
) -> [RawAXNode] {
447+
) throws -> [RawAXNode] {
442448
guard isCollapsedTabContainer(containerSnapshot) else { return [] }
443449
// Candidates are read from live elements, which XCTest reports in the app's own space, and a
444450
// collapsed tab container sits under the app's own window. `SnapshotGeometrySpace.normalized` keys
445451
// rotation on ancestry, so these nodes never enter a declared native space and need no special
446452
// case here; `isCollapsedTabCandidate` compares reported frames to each other.
447453
let containerFrame = containerSnapshot.frame
448-
let candidates = resolveCandidates().compactMap { candidate in
454+
let candidates = try resolveCandidates().compactMap { candidate in
449455
collapsedTabCandidateNode(
450456
candidate: candidate,
451457
containerSnapshot: containerSnapshot,

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ FOUNDATION_EXPORT BOOL RunnerResolveApplicationScreen(
8181
// Every application window that reports `litPanel`'s display, in list order, uncapped so element
8282
// search sees all of them; the caller holds the walk's XCTest issues. A window that is gone or names
8383
// no display is left out. Nil when the window list itself cannot be read.
84-
FOUNDATION_EXPORT NSArray * _Nullable RunnerCopyLitPanelWindows(id application, RunnerLitPanel litPanel);
84+
FOUNDATION_EXPORT NSArray * _Nullable RunnerCopyLitPanelWindows(id application, NSUInteger litDisplayID);
8585

8686
// Reads the display ID off an already-resolved window, keeping a synthesized record's display and
8787
// the caller's booked reference frame on the same window. The frame is still read before the

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -316,7 +316,7 @@ RunnerApplicationScreenFailure RunnerResolveLitPanelWindow(
316316
return outcome;
317317
}
318318

319-
NSArray *RunnerCopyLitPanelWindows(id application, RunnerLitPanel litPanel) {
319+
NSArray *RunnerCopyLitPanelWindows(id application, NSUInteger litDisplayID) {
320320
NSArray *windows = RunnerCopyWindowList(application);
321321
if (windows == nil) {
322322
return nil;
@@ -328,7 +328,7 @@ RunnerApplicationScreenFailure RunnerResolveLitPanelWindow(
328328
NSUInteger displayID = 0;
329329
if (RunnerReadLitPanelCandidate(candidate, &frame, &screen, &displayID) ==
330330
RunnerApplicationScreenFailureNone &&
331-
displayID == litPanel.displayID) {
331+
displayID == litDisplayID) {
332332
[lit addObject:candidate];
333333
}
334334
}

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift‎

Lines changed: 105 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@ import XCTest
55
import ObjectiveC.runtime
66

77
private final class LitPanelBookingScreen: NSObject {
8-
@objc let displayID: UInt = 3
8+
@objc let displayID: UInt
9+
10+
init(_ displayID: UInt = 3) { self.displayID = displayID }
911
}
1012

1113
@MainActor
@@ -15,6 +17,7 @@ private final class LitPanelBookingProbe: NSObject {
1517
static var windowListReads = 0
1618
static var maxWindowListReads: Int?
1719
static var window: XCUIElement?
20+
static var darkWindow: XCUIElement?
1821
static var coordinate: XCUICoordinate?
1922
static var windowFrame = CGRect.zero
2023
static var orientation = RunnerInterfaceOrientation.landscapeLeft
@@ -31,8 +34,8 @@ private final class LitPanelBookingProbe: NSObject {
3134
case "allElementsBoundByIndex":
3235
Self.windowListReads += 1
3336
if let max = Self.maxWindowListReads, Self.windowListReads > max { return [] }
34-
return [Self.window!]
35-
case "screen": return LitPanelBookingScreen()
37+
return [Self.window!, Self.darkWindow].compactMap { $0 }
38+
case "screen": return LitPanelBookingScreen(self === Self.darkWindow ? 1 : 3)
3639
case "exists": return true
3740
case "frame": return NSValue(cgRect: Self.windowFrame)
3841
default: break
@@ -114,6 +117,80 @@ extension RunnerTests {
114117
}
115118
}
116119

120+
@MainActor
121+
func testCollapsedTabCandidatesExcludeOverlappingDarkWindowsAndKeepTheCaptureScope() throws {
122+
try withLitPanelBookingProbe { window in
123+
let darkWindow = XCUIApplication(bundleIdentifier: "com.apple.springboard").windows.firstMatch
124+
LitPanelBookingProbe.darkWindow = darkWindow
125+
LitPanelBookingProbe.windowFrame = CGRect(x: 0, y: 0, width: 669, height: 951)
126+
setRequestLitPanel(from: try runnerCommandFixture(
127+
#"{"command":"snapshot","litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"#
128+
))
129+
XCTAssertNil(bookRequestLitPanelSurface(app: app))
130+
let container = ElementAttributesFixture(.tabBar, CGRect(x: 0, y: 800, width: 669, height: 31))
131+
let context = SnapshotTraversalContext(
132+
queryRoot: app,
133+
rootSnapshot: ElementAttributesFixture(.application, LitPanelBookingProbe.windowFrame, children: [
134+
ElementAttributesFixture(.window, LitPanelBookingProbe.windowFrame, children: [container], displayID: 3),
135+
]),
136+
rootReadDuration: 0,
137+
viewport: .missing(reason: .notProvided),
138+
keyboardBand: nil,
139+
litPanelSurface: try XCTUnwrap(litPanelState.surface)
140+
)
141+
let lit = ElementAttributesFixture(.button, CGRect(x: 30, y: 800, width: 100, height: 30), label: "Lit")
142+
let second = ElementAttributesFixture(.button, CGRect(x: 170, y: 800, width: 100, height: 30), label: "Second")
143+
let dark = ElementAttributesFixture(.button, lit.frame, label: "Dark")
144+
let appQuery = app.descendants(matching: .any)
145+
let litQuery = window.descendants(matching: .any)
146+
let darkQuery = darkWindow.descendants(matching: .any)
147+
var predicates: [ObjectIdentifier: NSPredicate] = [:]
148+
var restores: [() -> Void] = []
149+
defer { restores.reversed().forEach { $0() } }
150+
func swap(_ type: AnyClass, _ name: String, _ block: Any) throws {
151+
let method = try XCTUnwrap(class_getInstanceMethod(type, NSSelectorFromString(name)))
152+
let replacement = imp_implementationWithBlock(block)
153+
let original = method_setImplementation(method, replacement)
154+
restores.append {
155+
method_setImplementation(method, original)
156+
imp_removeBlock(replacement)
157+
}
158+
}
159+
try swap(XCUIElement.self, "descendantsMatchingType:", { (root: AnyObject, _: UInt) -> XCUIElementQuery in
160+
root === window ? litQuery : root === darkWindow ? darkQuery : appQuery
161+
} as @convention(block) (AnyObject, UInt) -> XCUIElementQuery)
162+
try swap(XCUIElementQuery.self, "matchingPredicate:", { (query: AnyObject, predicate: NSPredicate) -> XCUIElementQuery in
163+
predicates[ObjectIdentifier(query)] = predicate
164+
return query as! XCUIElementQuery
165+
} as @convention(block) (AnyObject, NSPredicate) -> XCUIElementQuery)
166+
try swap(XCUIElementQuery.self, "allElementsBoundByIndex", { (query: AnyObject) -> NSArray in
167+
let attributes = query === litQuery ? [lit, second] : query === darkQuery ? [dark] : [lit, second, dark]
168+
let matches = attributes.filter { predicates[ObjectIdentifier(query)]?.evaluate(with: $0) == true }
169+
return matches.isEmpty ? [] : [window]
170+
} as @convention(block) (AnyObject) -> NSArray)
171+
let candidates = try XCTUnwrap(try collapsedTabCandidates(context: context, processingDeadline: .distantFuture))
172+
XCTAssertEqual(candidates.map(\.label), ["Lit", "Second"], "an overlapping control on a dark window must not enter the capture")
173+
let hint = CaptureHint(projection: .regular, depth: nil, regularPresentedDepth: nil, interactiveOnly: false, customActions: false)
174+
let capture = try recursiveTreeSnapshotAcquisition(context: context, hint: hint, processingDeadline: .distantFuture)
175+
let labels = capture.acquisition.nodes.compactMap(\.label)
176+
XCTAssertTrue(labels.contains("Lit"))
177+
XCTAssertTrue(labels.contains("Second"))
178+
XCTAssertFalse(labels.contains("Dark"), "the serialized capture must exclude the overlapping dark control")
179+
endRequestLitPanel(try XCTUnwrap(litPanelState.requestID))
180+
let pinned = try XCTUnwrap(try collapsedTabCandidates(context: context, processingDeadline: .distantFuture))
181+
XCTAssertEqual(pinned.map(\.label), ["Lit", "Second"], "the capture's display scope survives a cleared current request")
182+
var unscoped = context
183+
unscoped.litPanelSurface = nil
184+
let all = try XCTUnwrap(try collapsedTabCandidates(context: unscoped, processingDeadline: .distantFuture))
185+
XCTAssertEqual(all.map(\.label), ["Lit", "Second", "Dark"], "an unscoped capture keeps application-wide candidates")
186+
litPanelWindowWalkIssueForTesting = true
187+
defer { litPanelWindowWalkIssueForTesting = false }
188+
XCTAssertThrowsError(try collapsedTabCandidates(context: context, processingDeadline: .distantFuture)) {
189+
XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .unresolvedScreen)
190+
}
191+
}
192+
}
193+
117194
@MainActor
118195
func testLitPanelBookingUsesBundleIdentityAcrossProxiesAndRequests() throws {
119196
try withLitPanelBookingProbe { window in
@@ -232,6 +309,7 @@ extension RunnerTests {
232309
var originals: [(Method, IMP)] = []
233310
defer {
234311
for (method, original) in originals.reversed() { method_setImplementation(method, original) }
312+
LitPanelBookingProbe.darkWindow = nil
235313
LitPanelBookingProbe.window = nil
236314
LitPanelBookingProbe.coordinate = nil
237315
LitPanelBookingProbe.anchor = nil
@@ -519,6 +597,30 @@ extension RunnerTests {
519597
}
520598
}
521599

600+
@MainActor
601+
func testCollapsedTabSnapshotReadsProvenLitLiveWindows() throws {
602+
app.launchArguments = ["--agent-device-collapsed-tab-bar", "--agent-device-wide-slider"]
603+
let (panel, _) = launchAppOnTheLitPanel()
604+
defer { tearDownLitPanelApp() }
605+
setRequestLitPanel(from: try runnerCommandFixture(#"{"command":"snapshot",\#(panel)}"#))
606+
XCTAssertNil(bookRequestLitPanelSurface(app: app))
607+
let hint = CaptureHint(projection: .regular, depth: nil, regularPresentedDepth: nil, interactiveOnly: false, customActions: false)
608+
let context = try XCTUnwrap(try makeSnapshotTraversalContext(
609+
app: app,
610+
hint: hint,
611+
litPanelSurface: try XCTUnwrap(litPanelState.surface)
612+
))
613+
let container = try XCTUnwrap(Self.snapshotDescendants(of: context.rootSnapshot).first { $0.elementType == .tabBar })
614+
XCTAssertTrue(container.children.isEmpty, "the live fixture must reach collapsed-tab recovery")
615+
let candidates = try XCTUnwrap(try collapsedTabCandidates(context: context, processingDeadline: .distantFuture))
616+
XCTAssertTrue(candidates.contains { $0.label == "Left" })
617+
XCTAssertTrue(candidates.contains { $0.label == "Right" })
618+
let capture = try recursiveTreeSnapshotAcquisition(context: context, hint: hint, processingDeadline: .distantFuture)
619+
let labels = capture.acquisition.nodes.compactMap(\.label)
620+
XCTAssertTrue(labels.contains("Left"))
621+
XCTAssertTrue(labels.contains("Right"))
622+
}
623+
522624
/// A refused pin is a request refusal: it never reaches the capture plan, so it does not
523625
/// invalidate the cached target the next command relies on.
524626
@MainActor

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotAcquisitionTests.swift‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,15 @@ final class ElementAttributesFixture: NSObject, XCUIElementSnapshot {
1010
let label: String
1111
let children: [XCUIElementSnapshot]
1212
let identifier: String
13+
@objc let displayID: NSNumber?
1314

14-
init(_ elementType: XCUIElement.ElementType, _ frame: CGRect, label: String = "", identifier: String = "", children: [XCUIElementSnapshot] = []) {
15+
init(_ elementType: XCUIElement.ElementType, _ frame: CGRect, label: String = "", identifier: String = "", children: [XCUIElementSnapshot] = [], displayID: NSNumber? = nil) {
1516
self.elementType = elementType
1617
self.frame = frame
1718
self.label = label
1819
self.children = children
1920
self.identifier = identifier
21+
self.displayID = displayID
2022
}
2123

2224
var value: Any? { nil }

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -365,7 +365,7 @@ extension RunnerTests {
365365
DisplayWindowFixture(frame: Self.outerStatusBar, displayID: 1)
366366
} + [later]
367367
let lit = RunnerCopyLitPanelWindows(
368-
DisplayApplicationFixture(windows: WindowListFixture(windows)), Self.innerPanel)
368+
DisplayApplicationFixture(windows: WindowListFixture(windows)), Self.innerPanel.displayID)
369369
XCTAssertEqual(lit?.count, 2)
370370
XCTAssertTrue((lit?.last as AnyObject) === later)
371371
}
@@ -380,13 +380,13 @@ extension RunnerTests {
380380
DisplayWindowFixture(frame: Self.innerHome, displayID: 3, present: false), dock,
381381
]
382382
let lit = RunnerCopyLitPanelWindows(
383-
DisplayApplicationFixture(windows: WindowListFixture(windows)), Self.innerPanel)
383+
DisplayApplicationFixture(windows: WindowListFixture(windows)), Self.innerPanel.displayID)
384384
XCTAssertEqual(lit?.count, 2)
385385
XCTAssertTrue((lit?.first as AnyObject) === home)
386386
XCTAssertTrue((lit?.last as AnyObject) === dock)
387387
XCTAssertNil(
388388
RunnerCopyLitPanelWindows(
389-
DisplayApplicationFixture(windows: UnanswerableWindowsFixture()), Self.innerPanel),
389+
DisplayApplicationFixture(windows: UnanswerableWindowsFixture()), Self.innerPanel.displayID),
390390
"an unreadable window list is not an empty lit display"
391391
)
392392
}

0 commit comments

Comments
 (0)