Skip to content

Commit 0f44265

Browse files
committed
test(ios): run the shared AX recovery fixture through the runner and characterize its depth memory
Replay every recovery case of contracts/fixtures/ios-ax-recovery-conformance.json through the runner's real private AX bridge, depth ladder, frontier extension, and completeness verdict, asserting the delivered tree's canonical signature and node count alongside outcome and request accounting. To make that possible RunnerAXSnapshotBridge gains a resolved client/target capture seam and the ladder loop becomes a platform-neutral function; behaviour is unchanged. Add the fixture's hint cases as the runner's accepted-depth memory characterization: each step is a real ladder capture against a client that rejects above the accepted rung, bounded by node budget when the step is not complete and rejected at every depth when it fails, whose observed rejections, accepted rung, and boundedness are asserted before the runner's own learning step records or skips it.
1 parent d9a3c2a commit 0f44265

7 files changed

Lines changed: 970 additions & 50 deletions

File tree

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,18 @@ FOUNDATION_EXPORT NSString *const RunnerAXSnapshotCustomActionsBlockedKey;
4949
customActionLimit:(NSInteger)customActionLimit
5050
deadline:(nullable NSDate *)deadline;
5151

52+
/// The capture behind `snapshotTreeForApplication:`, for an AX client and target
53+
/// application element that are already resolved. Every request, extension, and
54+
/// custom-action decision happens here; the application entry point only resolves
55+
/// the two identities.
56+
+ (NSDictionary<NSString *, id> *)snapshotTreeWithClient:(id)axClient
57+
target:(id)target
58+
maxDepth:(NSInteger)maxDepth
59+
maxNodes:(NSInteger)maxNodes
60+
deepExtensionCallLimit:(NSInteger)deepExtensionCallLimit
61+
customActionLimit:(NSInteger)customActionLimit
62+
deadline:(nullable NSDate *)deadline;
63+
5264
/// Names of the element's UIAccessibilityCustomActions, or nil when it has
5365
/// none.
5466
///

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,27 @@ @implementation RunnerAXSnapshotBridge
104104
if (nil == target) {
105105
return [self failure:@"Could not match active AX application for XCTest application"];
106106
}
107+
return [self snapshotTreeWithClient:axClient
108+
target:target
109+
maxDepth:maxDepth
110+
maxNodes:maxNodes
111+
deepExtensionCallLimit:deepExtensionCallLimit
112+
customActionLimit:customActionLimit
113+
deadline:deadline];
114+
} @catch (NSException *exception) {
115+
return [self failure:exception.reason ?: exception.name ?: @"AX snapshot bridge exception"];
116+
}
117+
}
107118

119+
+ (NSDictionary<NSString *, id> *)snapshotTreeWithClient:(id)axClient
120+
target:(id)target
121+
maxDepth:(NSInteger)maxDepth
122+
maxNodes:(NSInteger)maxNodes
123+
deepExtensionCallLimit:(NSInteger)deepExtensionCallLimit
124+
customActionLimit:(NSInteger)customActionLimit
125+
deadline:(nullable NSDate *)deadline
126+
{
127+
@try {
108128
NSArray *attributes = [self snapshotAttributes];
109129
NSError *error = nil;
110130
id root = [self requestSnapshotFromClient:axClient

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift‎

Lines changed: 76 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ extension RunnerTests {
88
/// depth-capped Bluesky-class tree resolves in 1-3 chained requests (~100-300ms
99
/// each); the bound exists so a pathological tree cannot stack requests past
1010
/// the capture-plan deadline, which is also enforced per call.
11-
private static let privateAXDeepExtensionCallLimit = 8
11+
static let privateAXDeepExtensionCallLimit = 8
1212

1313
/// Upper bound on per-element custom-action reads per capture. Each is its own
1414
/// AX round trip (~100ms on an idle simulator), so this caps the opt-in cost
@@ -67,6 +67,65 @@ extension RunnerTests {
6767
read: read, candidates: candidates, truncated: truncated, blocked: blocked)
6868
}
6969

70+
struct PrivateAXLadderOutcome {
71+
let response: [String: Any]
72+
let effectiveDepth: Int
73+
let deadlineSpent: Bool
74+
let lastError: String
75+
76+
var succeeded: Bool { response["ok"] as? Bool == true }
77+
}
78+
79+
/// Walks the ladder rungs until one capture succeeds. The first rung always runs (the plan
80+
/// gated entry on its own budget); later rungs stop when the capture-plan deadline is spent so
81+
/// ladder retries can never stack past the runner's main-thread watchdog (#1105).
82+
static func privateAXLadderCapture(
83+
attemptDepths: [Int],
84+
deadline: Date,
85+
capture: (Int) -> [String: Any]
86+
) -> PrivateAXLadderOutcome {
87+
var response: [String: Any] = [:]
88+
var effectiveDepth = attemptDepths.first ?? 0
89+
var lastError = "unknown private AX snapshot failure"
90+
for depth in attemptDepths {
91+
if depth != attemptDepths.first, Date() >= deadline {
92+
NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_BUDGET_EXHAUSTED depth=%ld", depth)
93+
return PrivateAXLadderOutcome(
94+
response: response, effectiveDepth: effectiveDepth, deadlineSpent: true,
95+
lastError: lastError)
96+
}
97+
response = capture(depth)
98+
if response["ok"] as? Bool == true {
99+
effectiveDepth = depth
100+
break
101+
}
102+
lastError = response["error"] as? String ?? lastError
103+
NSLog(
104+
"AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_DEPTH_RETRY depth=%ld error=%@",
105+
depth,
106+
lastError
107+
)
108+
}
109+
return PrivateAXLadderOutcome(
110+
response: response, effectiveDepth: effectiveDepth, deadlineSpent: false, lastError: lastError)
111+
}
112+
113+
/// Only a capture that actually descended records memory: a first-rung success on a
114+
/// remembered depth deliberately does NOT refresh the TTL, so expiry re-probes the full
115+
/// requested depth once per window instead of capping this screen class forever.
116+
func recordPrivateAXAcceptedDepth(
117+
exactDepthRequested: Bool,
118+
effectiveDepth: Int,
119+
attemptDepths: [Int]
120+
) {
121+
guard !exactDepthRequested, effectiveDepth != attemptDepths.first else { return }
122+
rememberPrivateAXAcceptedDepth(
123+
bundleId: currentBundleId,
124+
processIdentifier: currentAppProcessIdentifier,
125+
depth: effectiveDepth
126+
)
127+
}
128+
70129
func rememberPrivateAXAcceptedDepth(bundleId: String?, processIdentifier: Int?, depth: Int) {
71130
// No PID means no way to notice a relaunch later; record nothing rather than risk serving
72131
// a stale shallow rung to a fresh process.
@@ -125,54 +184,32 @@ extension RunnerTests {
125184
requestedDepth: requestedDepth,
126185
rememberedDepth: rememberedDepth
127186
)
128-
var response: [String: Any] = [:]
129-
var effectiveDepth = requestedDepth
130-
var lastError = "unknown private AX snapshot failure"
131-
for depth in attemptDepths {
132-
// The first rung always runs (the plan gated entry on its own budget); later rungs
133-
// stop when the capture-plan deadline is spent so ladder retries can never stack
134-
// past the runner's main-thread watchdog (#1105).
135-
if depth != attemptDepths.first, Date() >= deadline {
136-
NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_BUDGET_EXHAUSTED depth=%ld", depth)
137-
break
138-
}
139-
// Declared residue (#1797): the bridge caps the tree at 5000 nodes while serializing,
140-
// BEFORE either projection exists, so a raw capture of a huge screen is bounded rather
141-
// than failing the way the tree backend's own raw cap does. The cap is disclosed as
142-
// `truncated`, and it applied to the acquired tree before this projection split too.
143-
response = RunnerAXSnapshotBridge.snapshotTree(
187+
// Declared residue (#1797): the bridge caps the tree at 5000 nodes while serializing,
188+
// BEFORE either projection exists, so a raw capture of a huge screen is bounded rather
189+
// than failing the way the tree backend's own raw cap does. The cap is disclosed as
190+
// `truncated`, and it applied to the acquired tree before this projection split too.
191+
let ladder = Self.privateAXLadderCapture(attemptDepths: attemptDepths, deadline: deadline) {
192+
depth in
193+
RunnerAXSnapshotBridge.snapshotTree(
144194
for: app,
145195
maxDepth: depth,
146196
maxNodes: Self.privateAXSnapshotMaxNodes,
147197
deepExtensionCallLimit: exactDepthRequested ? 0 : Self.privateAXDeepExtensionCallLimit,
148198
customActionLimit: hint.customActions ? Self.privateAXCustomActionLimit : 0,
149199
deadline: deadline
150200
)
151-
if response["ok"] as? Bool == true {
152-
effectiveDepth = depth
153-
break
154-
}
155-
lastError = response["error"] as? String ?? lastError
156-
NSLog(
157-
"AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_DEPTH_RETRY depth=%ld error=%@",
158-
depth,
159-
lastError
160-
)
161201
}
162-
guard response["ok"] as? Bool == true else {
163-
NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_FAILED=%@", lastError)
202+
let response = ladder.response
203+
let effectiveDepth = ladder.effectiveDepth
204+
guard ladder.succeeded else {
205+
NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_FAILED=%@", ladder.lastError)
164206
return nil
165207
}
166-
// Only a capture that actually descended records memory: a first-rung success on a
167-
// remembered depth deliberately does NOT refresh the TTL, so expiry re-probes the full
168-
// requested depth once per window instead of capping this screen class forever.
169-
if !exactDepthRequested, effectiveDepth != attemptDepths.first {
170-
rememberPrivateAXAcceptedDepth(
171-
bundleId: currentBundleId,
172-
processIdentifier: currentAppProcessIdentifier,
173-
depth: effectiveDepth
174-
)
175-
}
208+
recordPrivateAXAcceptedDepth(
209+
exactDepthRequested: exactDepthRequested,
210+
effectiveDepth: effectiveDepth,
211+
attemptDepths: attemptDepths
212+
)
176213
guard let root = response["root"] as? [String: Any] else {
177214
NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_FAILED=missing root")
178215
return nil

0 commit comments

Comments
 (0)