Skip to content

Commit ae023d5

Browse files
committed
refactor(ios): normalize snapshot geometry in one pass after acquisition (#2661)
Move the coordinate-rotation table and SnapshotGeometrySpace into the snapshot-presentation package and add SnapshotGeometrySpace.normalized(nodes:viewport:interfaceOrientation:). Acquisition now serializes reported frames only; captureWithBackend settles every node into the app's orientation space in one ancestry-keyed pass instead of threading geometrySpace/parentIsWindow through the recursive, raw, private-AX, and query-sweep walkers. Drop the fold-consulting regular-depth frontier. It read reported (un-normalized) rects mid-walk to increment presented depth, so a turned keyboard band reported in the device's native space could be pruned before the single pass ever moved it back into the app's viewport. A regular --depth is now a presentation cut only: the walk's bounds are raw traversal depth (nil for a regular capture) plus a runaway node cap, and SnapshotPresentation applies the visibility fold and the presented-depth cut on the normalized array. Reclassify the iOS tree backend's regular-depth capability from presented-frontier to presentation-cut across the Swift declaration, the cross-runtime parity fixture, and the TypeScript registry, and remove the now-unused Swift enum case (Android's helper keeps presented-frontier). Update ADR 0004 and the ios.yml PR-lane test list.
1 parent 321aef2 commit ae023d5

15 files changed

Lines changed: 458 additions & 411 deletions

‎.github/workflows/ios.yml‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -203,9 +203,10 @@ jobs:
203203
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotPresentationOwnsBackendNeutralEligibility \
204204
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotPresentationOwnsScopeAndRelativeDepth \
205205
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testPresentationRefusesAnAcquisitionCapturedForTheOtherProjection \
206-
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testCaptureHintIsTheOnlyAcquisitionViewOfARequest \
207-
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularDepthFrontierSurvivesStructuralWrapperCollapse \
208-
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularDepthFrontierKeepsVisibleIndependentChildPastClippedParent \
206+
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testCaptureHintIsTheOnlyAcquisitionViewOfARequest \
207+
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularDepthCutsPresentationNotAcquisition \
208+
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testVisibilityFoldKeepsIndependentChildPastClippedParent \
209+
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularDepthKeepsTurnedKeyboardSubtreeAfterOneNormalizationPass \
209210
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularFoldClipsScrollOverflowReparentsAndBooksHints \
210211
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularFoldKeepsWindowCarriersButNeverHittableOutsideClip \
211212
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularFoldDropsSubPixelContentlessDecorationOnEveryBackend \

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift‎

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -220,9 +220,7 @@ extension RunnerTests {
220220
let viewport = geometry.viewport
221221
let nodes = privateAXAcquisition(
222222
rawRoot: root,
223-
hint: hint,
224-
viewport: viewport,
225-
interfaceOrientation: geometry.interfaceOrientation
223+
hint: hint
226224
)
227225
// Serialization-level emptiness only: an acquired-but-fully-clipped tree is presentation's
228226
// verdict now, surfaced by the plan's sparse classifier on the presented payload (#1797).
@@ -256,7 +254,8 @@ extension RunnerTests {
256254
customActions: Self.privateAXCustomActionCoverage(
257255
response[RunnerAXSnapshotCustomActionsKey]
258256
),
259-
viewport: viewport
257+
viewport: viewport,
258+
interfaceOrientation: geometry.interfaceOrientation
260259
)
261260
#else
262261
return nil
@@ -697,9 +696,7 @@ extension RunnerTests {
697696
rawRoot: tree,
698697
hint: CaptureHint(
699698
projection: .regular, depth: nil, regularPresentedDepth: nil,
700-
interactiveOnly: false, customActions: false),
701-
viewport: CGRect(x: 0, y: 0, width: 390, height: 844),
702-
interfaceOrientation: RunnerInterfaceOrientation.portrait
699+
interactiveOnly: false, customActions: false)
703700
)
704701

705702
let card = nodes.first { $0.label == "feedItem-by-whiskers.test" }
@@ -732,9 +729,7 @@ extension RunnerTests {
732729
scope: "homeScreen",
733730
raw: false
734731
)
735-
),
736-
viewport: .infinite,
737-
interfaceOrientation: RunnerInterfaceOrientation.portrait
732+
)
738733
)
739734

740735
let labels = nodes.compactMap { $0.label ?? $0.identifier }
@@ -812,9 +807,11 @@ extension RunnerTests {
812807
let hint = CaptureHint(
813808
projection: .regular, depth: nil, regularPresentedDepth: nil,
814809
interactiveOnly: true, customActions: false)
815-
let acquired = privateAXAcquisition(
816-
rawRoot: tree, hint: hint, viewport: viewport,
817-
interfaceOrientation: RunnerInterfaceOrientation.portrait)
810+
let acquired = SnapshotGeometrySpace.normalized(
811+
nodes: privateAXAcquisition(rawRoot: tree, hint: hint),
812+
viewport: viewport,
813+
interfaceOrientation: RunnerInterfaceOrientation.portrait
814+
)
818815
// Acquisition serializes the drawer too; the shared fold is what hides it (#1797).
819816
XCTAssertTrue(acquired.compactMap(\.label).contains("Admin settings"))
820817

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift‎

Lines changed: 37 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -22,51 +22,26 @@ extension RunnerTests {
2222
/// depth -- membership, the clip fold, scroll hints, and collapsed depth are
2323
/// `SnapshotPresentation`'s alone (#1797). The traversal-depth cut is the backend's one
2424
/// narrowing, complete for raw (raw depth *is* traversal depth) and a declared residue for
25-
/// regular; `hittable` is the shared geometric fact the fold recomputes for effective geometry.
25+
/// regular; the frame carried here is the one the platform reported, and
26+
/// `SnapshotGeometrySpace.normalized` turns it once, after this walk, before presentation (#2661).
2627
func privateAXAcquisition(
2728
rawRoot: [String: Any],
28-
hint: CaptureHint,
29-
viewport: CGRect,
30-
interfaceOrientation: Int
29+
hint: CaptureHint
3130
) -> [RawAXNode] {
3231
var nodes: [RawAXNode] = []
33-
appendPrivateAXNode(rawRoot, to: &nodes, hint: hint, viewport: viewport,
34-
interfaceOrientation: interfaceOrientation, geometrySpace: .appOrientation,
35-
parentIsWindow: false, depth: 0, parentIndex: nil)
32+
appendPrivateAXNode(rawRoot, to: &nodes, hint: hint, depth: 0, parentIndex: nil)
3633
return nodes
3734
}
3835

3936
private func appendPrivateAXNode(_ raw: [String: Any], to nodes: inout [RawAXNode],
40-
hint: CaptureHint, viewport: CGRect, interfaceOrientation: Int,
41-
geometrySpace: SnapshotGeometrySpace, parentIsWindow: Bool, depth: Int, parentIndex: Int?)
37+
hint: CaptureHint, depth: Int, parentIndex: Int?)
4238
{
4339
if let limit = hint.rawTraversalDepth, depth > limit { return }
4440
let fields = privateAXFields(raw)
45-
let nodeSpace = SnapshotGeometrySpace.space(
46-
reportedBySurfaceHost: SnapshotGeometrySpace.isSurfaceHost(
47-
isWindow: isWindowElement(fields.elementType),
48-
parentIsWindow: parentIsWindow
49-
),
50-
reportedFrame: fields.rect,
51-
inheritedFrom: geometrySpace,
52-
appFrame: viewport,
53-
interfaceOrientation: interfaceOrientation
54-
)
5541
let index = nodes.count
56-
nodes.append(
57-
privateAXNode(
58-
fields,
59-
index: index,
60-
depth: depth,
61-
parentIndex: parentIndex,
62-
viewport: viewport,
63-
geometrySpace: nodeSpace
64-
)
65-
)
42+
nodes.append(privateAXNode(fields, index: index, depth: depth, parentIndex: parentIndex))
6643
for child in fields.children {
67-
appendPrivateAXNode(child, to: &nodes, hint: hint, viewport: viewport,
68-
interfaceOrientation: interfaceOrientation, geometrySpace: nodeSpace,
69-
parentIsWindow: isWindowElement(fields.elementType), depth: depth + 1, parentIndex: index)
44+
appendPrivateAXNode(child, to: &nodes, hint: hint, depth: depth + 1, parentIndex: index)
7045
}
7146
}
7247

@@ -87,22 +62,17 @@ extension RunnerTests {
8762
)
8863
}
8964

90-
private func privateAXNode(_ fields: PrivateAXFields, index: Int, depth: Int, parentIndex: Int?,
91-
viewport: CGRect, geometrySpace: SnapshotGeometrySpace) -> RawAXNode
65+
private func privateAXNode(_ fields: PrivateAXFields, index: Int, depth: Int,
66+
parentIndex: Int?) -> RawAXNode
9267
{
93-
let frame = geometrySpace.orientedFrame(of: fields.rect)
9468
return RawAXNode(index: index,
9569
type: fields.elementType.map(elementTypeName) ?? "Element(\(fields.rawType))",
9670
label: fields.label.isEmpty ? nil : fields.label,
9771
identifier: fields.identifier.isEmpty ? nil : fields.identifier,
9872
value: fields.value.isEmpty ? nil : fields.value,
99-
rect: snapshotRect(from: frame), enabled: fields.enabled,
73+
rect: snapshotRect(from: fields.rect), enabled: fields.enabled,
10074
focused: fields.focused, selected: fields.selected,
101-
hittable: parentIndex != nil && SnapshotGeometry.isGeometricallyActionable(
102-
enabled: fields.enabled,
103-
frame: frame,
104-
viewport: viewport
105-
),
75+
hittable: false,
10676
depth: depth, parentIndex: parentIndex, hiddenContentAbove: nil, hiddenContentBelow: nil,
10777
actions: fields.actions)
10878
}
@@ -145,8 +115,9 @@ extension RunnerTests {
145115
"frame": frame(16, 900, 360, 44)]]]]]
146116
}
147117

148-
/// Acquire with the private-AX serializer, then present through the shared regular fold --
149-
/// the production route for this backend since the fold moved into presentation (#1797).
118+
/// Acquire with the private-AX serializer, run the one normalization pass the production capture
119+
/// plan runs (`captureWithBackend`), then present through the shared regular fold -- the production
120+
/// route for this backend since the fold moved into presentation (#1797, #2661).
150121
fileprivate func privateAXRegularPresentation(
151122
rawRoot: [String: Any],
152123
viewport: CGRect,
@@ -155,21 +126,36 @@ extension RunnerTests {
155126
let hint = CaptureHint(
156127
projection: .regular, depth: nil, regularPresentedDepth: nil,
157128
interactiveOnly: interactiveOnly, customActions: false)
158-
let acquired = privateAXAcquisition(
129+
let nodes = privateAXNormalizedAcquisition(
159130
rawRoot: rawRoot, hint: hint, viewport: viewport,
160131
interfaceOrientation: RunnerInterfaceOrientation.portrait)
161132
return try SnapshotPresentation.presentRegular(
162133
SnapshotAcquisition(
163-
hint: hint, nodes: acquired, truncated: false, effectiveDepth: nil, viewport: viewport),
134+
hint: hint, nodes: nodes, truncated: false, effectiveDepth: nil, viewport: viewport,
135+
interfaceOrientation: RunnerInterfaceOrientation.portrait),
164136
options: PresentationOptions(
165137
interactiveOnly: interactiveOnly, depth: nil, scope: nil, raw: false),
166138
policy: .cursorProjected
167139
).nodes
168140
}
169141

170-
/// The walk's space declaration has to reach the node it publishes: this asserts the rotated rect of
171-
/// a key under a turned surface host, and an untouched sibling under the app's own window, so a walk
172-
/// that drops the space it computed fails here.
142+
/// Acquire, then normalize once -- the exact pair `captureWithBackend` runs for this backend.
143+
fileprivate func privateAXNormalizedAcquisition(
144+
rawRoot: [String: Any],
145+
hint: CaptureHint,
146+
viewport: CGRect,
147+
interfaceOrientation: Int
148+
) -> [RawAXNode] {
149+
SnapshotGeometrySpace.normalized(
150+
nodes: privateAXAcquisition(rawRoot: rawRoot, hint: hint),
151+
viewport: viewport,
152+
interfaceOrientation: interfaceOrientation
153+
)
154+
}
155+
156+
/// The one normalization pass has to reach the node it publishes: this asserts the rotated rect of
157+
/// a key under a turned surface host, and an untouched sibling under the app's own window, so a
158+
/// pass that drops the space a subtree declared fails here.
173159
func testPrivateAXAcquisitionPublishesATurnedSurfaceHostInAppOrientationSpace() {
174160
let frame = Self.privateAXFrame
175161
let appWindow: [String: Any] = [
@@ -195,7 +181,7 @@ extension RunnerTests {
195181
let hint = CaptureHint(
196182
projection: .raw, depth: nil, regularPresentedDepth: nil,
197183
interactiveOnly: false, customActions: false)
198-
let nodes = privateAXAcquisition(
184+
let nodes = privateAXNormalizedAcquisition(
199185
rawRoot: [
200186
"type": Int(XCUIElement.ElementType.application.rawValue),
201187
"label": "Element", "frame": frame(0, 0, 874, 402),
@@ -236,7 +222,7 @@ extension RunnerTests {
236222
let root = Self.privateAXScrolledFixture
237223
let regular = try privateAXRegularPresentation(rawRoot: root, viewport: viewport,
238224
interactiveOnly: true)
239-
let raw = privateAXAcquisition(rawRoot: root,
225+
let raw = privateAXNormalizedAcquisition(rawRoot: root,
240226
hint: CaptureHint(
241227
projection: .raw, depth: nil, regularPresentedDepth: nil,
242228
interactiveOnly: false, customActions: false),
@@ -265,7 +251,7 @@ extension RunnerTests {
265251
/// Raw depth is traversal depth, so a raw `--depth` request is the one narrowing this backend
266252
/// can prove complete.
267253
func testPrivateAXRawProjectionAppliesRequestedTraversalDepth() {
268-
let raw = privateAXAcquisition(rawRoot: Self.privateAXScrolledFixture,
254+
let raw = privateAXNormalizedAcquisition(rawRoot: Self.privateAXScrolledFixture,
269255
hint: CaptureHint(
270256
projection: .raw, depth: 2, regularPresentedDepth: nil,
271257
interactiveOnly: false, customActions: false),

0 commit comments

Comments
 (0)