Skip to content

Commit 4695528

Browse files
committed
refactor(ios): establish snapshot presentation seam
Introduce RawAXNode and PresentedNode so acquisition backends can no longer construct the wire-facing snapshot shape directly. Preserve current output while #1797 moves semantics behind the seam. Non-vacuity: setting PresentedNode.label to nil made testSnapshotPresentationPreservesCurrentWireShape execute once and fail on the missing label field; restoring the production mapping made the same focused XCTest pass.
1 parent 4bba404 commit 4695528

12 files changed

Lines changed: 251 additions & 124 deletions

‎.github/workflows/ios.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@ jobs:
116116
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain \
117117
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrainForSnapshotRaw \
118118
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotTraversalIdentityPreservesSameOriginNodesWithDifferentBounds \
119+
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotPresentationPreservesCurrentWireShape \
119120
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testFlatSnapshotProjectionMatchesElementReverseScrollCapture \
120121
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXDepthLimitedRequiresEveryFrontierResolved \
121122
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testDeepExtensionCountsMissedFrontiers \

‎CONTEXT.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,11 @@ task touches:
215215

216216
### Snapshots & capture
217217

218+
- Raw AX node: backend-owned iOS acquisition value before the snapshot presentation boundary. During
219+
the #1797 migration it temporarily carries existing derived fields so the seam can land without a
220+
wire-behavior change.
221+
- Presented node: wire-facing iOS snapshot value constructed only by `SnapshotPresentation`; response
222+
payload assembly accepts this type rather than backend-owned raw values.
218223
- Snapshot capture plan: per-strategy ordered chain of iOS snapshot capture backends (recursive tree,
219224
query sweep, private AX) run by one plan runner under a shared wall-clock budget; recovery ordering
220225
is declared data, never a per-call-site branch.

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,10 @@ extension RunnerTests {
203203
deepExtension?[RunnerAXSnapshotDeepExtensionNodesAddedKey] as? Int ?? 0
204204
)
205205
return SnapshotBackendCapture(
206-
payload: DataPayload(nodes: nodes, truncated: (response["truncated"] as? Bool) == true),
206+
payload: DataPayload(
207+
presenting: nodes,
208+
truncated: (response["truncated"] as? Bool) == true
209+
),
207210
effectiveDepth: depthLimited ? effectiveDepth : nil,
208211
customActions: Self.privateAXCustomActionCoverage(
209212
response[RunnerAXSnapshotCustomActionsKey]

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift‎

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -416,22 +416,24 @@ extension RunnerTests {
416416
return try! JSONDecoder().decode(Command.self, from: Data(json.utf8))
417417
}
418418

419-
private func runnerJournalNode() -> SnapshotNode {
420-
SnapshotNode(
421-
index: 0,
422-
type: "button",
423-
label: "Continue",
424-
identifier: "continue",
425-
value: nil,
426-
rect: SnapshotRect(x: 10, y: 20, width: 100, height: 44),
427-
enabled: true,
428-
focused: nil,
429-
selected: nil,
430-
hittable: true,
431-
depth: 0,
432-
parentIndex: nil,
433-
hiddenContentAbove: nil,
434-
hiddenContentBelow: nil
419+
private func runnerJournalNode() -> PresentedNode {
420+
SnapshotPresentation.singleElementRead(
421+
RawAXNode(
422+
index: 0,
423+
type: "button",
424+
label: "Continue",
425+
identifier: "continue",
426+
value: nil,
427+
rect: SnapshotRect(x: 10, y: 20, width: 100, height: 44),
428+
enabled: true,
429+
focused: nil,
430+
selected: nil,
431+
hittable: true,
432+
depth: 0,
433+
parentIndex: nil,
434+
hiddenContentAbove: nil,
435+
hiddenContentBelow: nil
436+
)
435437
)
436438
}
437439

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift‎

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -229,21 +229,23 @@ extension RunnerTests {
229229
let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
230230
let valueText = String(describing: element.value ?? "")
231231
.trimmingCharacters(in: .whitespacesAndNewlines)
232-
let node = SnapshotNode(
233-
index: 0,
234-
type: elementTypeName(element.elementType),
235-
label: label.isEmpty ? nil : label,
236-
identifier: identifier.isEmpty ? nil : identifier,
237-
value: valueText.isEmpty ? nil : valueText,
238-
rect: snapshotRect(from: element.frame),
239-
enabled: element.isEnabled,
240-
focused: nil,
241-
selected: element.isSelected ? true : nil,
242-
hittable: element.isHittable,
243-
depth: 0,
244-
parentIndex: nil,
245-
hiddenContentAbove: nil,
246-
hiddenContentBelow: nil
232+
let node = SnapshotPresentation.singleElementRead(
233+
RawAXNode(
234+
index: 0,
235+
type: elementTypeName(element.elementType),
236+
label: label.isEmpty ? nil : label,
237+
identifier: identifier.isEmpty ? nil : identifier,
238+
value: valueText.isEmpty ? nil : valueText,
239+
rect: snapshotRect(from: element.frame),
240+
enabled: element.isEnabled,
241+
focused: nil,
242+
selected: element.isSelected ? true : nil,
243+
hittable: element.isHittable,
244+
depth: 0,
245+
parentIndex: nil,
246+
hiddenContentAbove: nil,
247+
hiddenContentBelow: nil
248+
)
247249
)
248250
return Response(
249251
ok: true,

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift‎

Lines changed: 2 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,7 @@ struct DataPayload: Codable {
235235
let text: String?
236236
let found: Bool?
237237
let items: [String]?
238-
let nodes: [SnapshotNode]?
238+
let nodes: [PresentedNode]?
239239
let truncated: Bool?
240240
let snapshotQuality: SnapshotQuality?
241241
let gestureStartUptimeMs: Double?
@@ -277,7 +277,7 @@ struct DataPayload: Codable {
277277
text: String? = nil,
278278
found: Bool? = nil,
279279
items: [String]? = nil,
280-
nodes: [SnapshotNode]? = nil,
280+
nodes: [PresentedNode]? = nil,
281281
truncated: Bool? = nil,
282282
snapshotQuality: SnapshotQuality? = nil,
283283
gestureStartUptimeMs: Double? = nil,
@@ -375,26 +375,6 @@ struct SnapshotRect: Codable {
375375
let height: Double
376376
}
377377

378-
struct SnapshotNode: Codable {
379-
let index: Int
380-
let type: String
381-
let label: String?
382-
let identifier: String?
383-
let value: String?
384-
let rect: SnapshotRect
385-
let enabled: Bool
386-
let focused: Bool?
387-
let selected: Bool?
388-
let hittable: Bool
389-
let depth: Int
390-
let parentIndex: Int?
391-
let hiddenContentAbove: Bool?
392-
let hiddenContentBelow: Bool?
393-
/// Names of the element's UIAccessibilityCustomActions, present only when the
394-
/// capture asked for them and the element carries some.
395-
var actions: [String]? = nil
396-
}
397-
398378
struct SnapshotOptions {
399379
let interactiveOnly: Bool
400380
let depth: Int?

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,17 @@ extension RunnerTests {
44
private static let privateAXProjectionTolerance: CGFloat = 1
55

66
func privateAXPresentation(rawRoot: [String: Any], options: SnapshotOptions, viewport: CGRect)
7-
-> [SnapshotNode]
7+
-> [RawAXNode]
88
{
9-
var nodes: [SnapshotNode] = []
9+
var nodes: [RawAXNode] = []
1010
var hints: [Int: (above: Bool, below: Bool)] = [:]
1111
appendPrivateAXNode(rawRoot, to: &nodes, hints: &hints, options: options, viewport: viewport,
1212
depth: 0, parentIndex: nil, insideMatchedScope: false, scrollContext: nil,
1313
projectionCursor: .root)
1414
return applyHiddenContentHints(hints, to: nodes)
1515
}
1616

17-
private func appendPrivateAXNode(_ raw: [String: Any], to nodes: inout [SnapshotNode],
17+
private func appendPrivateAXNode(_ raw: [String: Any], to nodes: inout [RawAXNode],
1818
hints: inout [Int: (above: Bool, below: Bool)], options: SnapshotOptions, viewport: CGRect,
1919
depth: Int, parentIndex: Int?, insideMatchedScope: Bool,
2020
scrollContext: (index: Int, rect: CGRect)?, projectionCursor: FlatSnapshotProjectionCursor)
@@ -65,7 +65,7 @@ extension RunnerTests {
6565
if include {
6666
currentIndex = nodes.count
6767
let typeName = elementType.map(elementTypeName) ?? "Element(\(rawType))"
68-
nodes.append(SnapshotNode(index: nodes.count, type: typeName,
68+
nodes.append(RawAXNode(index: nodes.count, type: typeName,
6969
label: label.isEmpty ? nil : label, identifier: identifier.isEmpty ? nil : identifier,
7070
value: value.isEmpty ? nil : value, rect: snapshotRect(from: rect), enabled: enabled,
7171
focused: privateAXPresentationBool(raw["focused"]) == true ? true : nil,

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift‎

Lines changed: 27 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ extension RunnerTests {
147147
return result.elements
148148
}
149149

150-
var nodes: [SnapshotNode] = []
150+
var nodes: [RawAXNode] = []
151151
var hiddenContentHintsByNodeIndex: [Int: (above: Bool, below: Bool)] = [:]
152152
let rootEvaluation = evaluateSnapshot(context.rootSnapshot, in: context)
153153
nodes.append(
@@ -316,7 +316,7 @@ extension RunnerTests {
316316
}
317317

318318
return DataPayload(
319-
nodes: applyHiddenContentHints(hiddenContentHintsByNodeIndex, to: nodes),
319+
presenting: applyHiddenContentHints(hiddenContentHintsByNodeIndex, to: nodes),
320320
truncated: false
321321
)
322322
}
@@ -422,7 +422,7 @@ extension RunnerTests {
422422
context: SnapshotTraversalContext,
423423
options: SnapshotOptions
424424
) throws -> DataPayload {
425-
var nodes: [SnapshotNode] = []
425+
var nodes: [RawAXNode] = []
426426

427427
func walk(_ snapshot: XCUIElementSnapshot, depth: Int, parentIndex: Int?) throws {
428428
if let limit = options.depth, depth > limit { return }
@@ -460,19 +460,19 @@ extension RunnerTests {
460460
}
461461

462462
try walk(context.rootSnapshot, depth: 0, parentIndex: nil)
463-
return DataPayload(nodes: nodes, truncated: false)
463+
return DataPayload(presenting: nodes, truncated: false)
464464
}
465465

466466
func snapshotFlatInteractive(
467467
app: XCUIApplication,
468468
options: SnapshotOptions,
469469
planDeadline: Date = .distantFuture
470470
) -> DataPayload {
471-
var nodes: [SnapshotNode] = [
471+
var nodes: [RawAXNode] = [
472472
interactiveRootNode(rect: .zero)
473473
]
474474
if options.depth == 0 {
475-
return DataPayload(nodes: nodes, truncated: false)
475+
return DataPayload(presenting: nodes, truncated: false)
476476
}
477477

478478
// Bounded by both its own sweep budget and the umbrella capture-plan deadline, so a
@@ -483,7 +483,7 @@ extension RunnerTests {
483483
let deadline = min(sweepDeadline, planDeadline)
484484
let viewport = safeSnapshotViewport(app: app)
485485
var seen = Set<String>()
486-
var candidates: [SnapshotNode] = []
486+
var candidates: [RawAXNode] = []
487487
let flatElements = flatInteractiveElements(app: app, deadline: deadline)
488488
var truncated = flatElements.truncated
489489
for element in flatElements.elements {
@@ -525,7 +525,7 @@ extension RunnerTests {
525525
nodes[0] = interactiveRootNode(rect: rootRect)
526526
for candidate in candidates {
527527
nodes.append(
528-
SnapshotNode(
528+
RawAXNode(
529529
index: nodes.count,
530530
type: candidate.type,
531531
label: candidate.label,
@@ -543,7 +543,7 @@ extension RunnerTests {
543543
)
544544
)
545545
}
546-
return DataPayload(nodes: nodes, truncated: truncated)
546+
return DataPayload(presenting: nodes, truncated: truncated)
547547
}
548548

549549
func snapshotAccessibilityUnavailable(failure: SnapshotCaptureFailure) -> DataPayload {
@@ -588,7 +588,7 @@ extension RunnerTests {
588588
) -> DataPayload {
589589
return DataPayload(
590590
message: message,
591-
nodes: [interactiveRootNode(rect: .zero)],
591+
nodes: SnapshotPresentation.preservingCurrentSemantics([interactiveRootNode(rect: .zero)]),
592592
truncated: true,
593593
snapshotQuality: snapshotQuality,
594594
runnerFatal: runnerFatal,
@@ -830,8 +830,8 @@ extension RunnerTests {
830830
}
831831
#endif
832832

833-
private func interactiveRootNode(rect: CGRect) -> SnapshotNode {
834-
SnapshotNode(
833+
private func interactiveRootNode(rect: CGRect) -> RawAXNode {
834+
RawAXNode(
835835
index: 0,
836836
type: "Application",
837837
label: nil,
@@ -849,7 +849,7 @@ extension RunnerTests {
849849
)
850850
}
851851

852-
private func interactiveRootFrame(for candidates: [SnapshotNode]) -> CGRect {
852+
private func interactiveRootFrame(for candidates: [RawAXNode]) -> CGRect {
853853
guard !candidates.isEmpty else {
854854
return .zero
855855
}
@@ -1120,8 +1120,8 @@ extension RunnerTests {
11201120
depth: Int,
11211121
index: Int,
11221122
parentIndex: Int?
1123-
) -> SnapshotNode {
1124-
return SnapshotNode(
1123+
) -> RawAXNode {
1124+
return RawAXNode(
11251125
index: index,
11261126
type: elementTypeName(snapshot.elementType),
11271127
label: evaluation.label.isEmpty ? nil : evaluation.label,
@@ -1241,7 +1241,7 @@ extension RunnerTests {
12411241
}
12421242

12431243
private func appendCollapsedTabFallbackNodes(
1244-
to nodes: inout [SnapshotNode],
1244+
to nodes: inout [RawAXNode],
12451245
containerSnapshot: XCUIElementSnapshot,
12461246
resolveElements: () -> [XCUIElement],
12471247
depth: Int,
@@ -1306,14 +1306,14 @@ extension RunnerTests {
13061306

13071307
func applyHiddenContentHints(
13081308
_ hints: [Int: (above: Bool, below: Bool)],
1309-
to nodes: [SnapshotNode]
1310-
) -> [SnapshotNode] {
1309+
to nodes: [RawAXNode]
1310+
) -> [RawAXNode] {
13111311
if hints.isEmpty { return nodes }
13121312
return nodes.map { node in
13131313
guard let hint = hints[node.index] else { return node }
13141314
let hiddenContentAbove: Bool? = (node.hiddenContentAbove == true || hint.above) ? true : nil
13151315
let hiddenContentBelow: Bool? = (node.hiddenContentBelow == true || hint.below) ? true : nil
1316-
return SnapshotNode(
1316+
return RawAXNode(
13171317
index: node.index,
13181318
type: node.type,
13191319
label: node.label,
@@ -1339,7 +1339,7 @@ extension RunnerTests {
13391339
startingIndex: Int,
13401340
depth: Int,
13411341
parentIndex: Int
1342-
) -> [SnapshotNode] {
1342+
) -> [RawAXNode] {
13431343
if !containerSnapshot.children.isEmpty { return [] }
13441344
guard shouldExpandCollapsedTabContainer(containerSnapshot) else { return [] }
13451345
let containerFrame = containerSnapshot.frame
@@ -1378,7 +1378,7 @@ extension RunnerTests {
13781378
if uniqueCandidates.count < 2 { return [] }
13791379

13801380
return uniqueCandidates.enumerated().map { offset, node in
1381-
SnapshotNode(
1381+
RawAXNode(
13821382
index: startingIndex + offset,
13831383
type: node.type,
13841384
label: node.label,
@@ -1401,8 +1401,8 @@ extension RunnerTests {
14011401
element: XCUIElement,
14021402
containerSnapshot: XCUIElementSnapshot,
14031403
containerFrame: CGRect
1404-
) -> SnapshotNode? {
1405-
var node: SnapshotNode?
1404+
) -> RawAXNode? {
1405+
var node: RawAXNode?
14061406
let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
14071407
if !element.exists { return }
14081408
let elementType = element.elementType
@@ -1430,7 +1430,7 @@ extension RunnerTests {
14301430
return
14311431
}
14321432

1433-
node = SnapshotNode(
1433+
node = RawAXNode(
14341434
index: 0,
14351435
type: elementTypeName(elementType),
14361436
label: label.isEmpty ? nil : label,
@@ -1568,8 +1568,8 @@ extension RunnerTests {
15681568
parentIndex: Int?,
15691569
viewport: CGRect,
15701570
options: SnapshotOptions
1571-
) -> SnapshotNode? {
1572-
var node: SnapshotNode?
1571+
) -> RawAXNode? {
1572+
var node: RawAXNode?
15731573
let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
15741574
if !element.exists { return }
15751575
let frame = element.frame
@@ -1601,7 +1601,7 @@ extension RunnerTests {
16011601
return
16021602
}
16031603

1604-
node = SnapshotNode(
1604+
node = RawAXNode(
16051605
index: index,
16061606
type: elementTypeName(elementType),
16071607
label: label.isEmpty ? nil : label,

0 commit comments

Comments
 (0)