Skip to content

Commit dccb40f

Browse files
committed
refactor(ios): normalize snapshot geometry in one pass after acquisition (#2661)
Move the coordinate-rotation table and SnapshotGeometrySpace into the snapshot-presentation package and add SnapshotGeometrySpace.normalized(nodes:viewport:interfaceOrientation:). Acquisition now serializes reported frames only; captureWithBackend settles every node into the app's orientation space in one ancestry-keyed pass instead of threading geometrySpace/parentIsWindow through the recursive, raw, private-AX, and query-sweep walkers. The regular presented-depth frontier stays an acquisition bound (it bounds walk work and the scroll hints a presented boundary container publishes); only the per-node coordinate space was removed. Portrait capture and the #2612 landscape keyboard band are unchanged.
1 parent 91652a8 commit dccb40f

10 files changed

Lines changed: 674 additions & 442 deletions

File tree

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerCoordinateSpace.swift‎

Lines changed: 31 additions & 241 deletions
Original file line numberDiff line numberDiff line change
@@ -1,219 +1,11 @@
11
import XCTest
2+
import AgentDeviceSnapshotPresentation
23

3-
// Where a piece of geometry lives, and how it gets where the caller means it.
4-
//
5-
// DISPATCH: synthesized events skip XCTest's orientation handling, so a point planned in an app's
6-
// interface space has to be rotated into the device's native (portrait-up) space before it is
7-
// performed. Without that, a landscape tap lands somewhere else.
8-
//
9-
// CAPTURE: iOS hosts some system surfaces in that native space even while the app is rotated, so
10-
// their whole subtree arrives quarter-turned. Measured on iPhone 17 Pro (iOS 26.2) with the system
11-
// keyboard up in landscape: the app viewport is 874 x 402, `UIRemoteKeyboardWindow` reports its own
12-
// box as 402 x 874, a key arrives 45 x 72 at x 154, and the key plane as 202 x 724 — a strip down
13-
// the left edge — while the screenshot draws a 724 x 204 band docked at y 198, which is exactly
14-
// what the live `app.keyboards` query measures. Rules that read the reported numbers refused app
15-
// content the keyboard was nowhere near and let a tap through into a key (#2612, #2589).
16-
//
17-
// Both directions live in ONE table on purpose: capture rotates back with the exact inverse of what
18-
// dispatch rotates forward, so a performed tap and a reported rect can never disagree about which
19-
// pixel is which. The round-trip test below is that proof, not a comment asking for care.
20-
21-
/// The UIInterfaceOrientation raw values the rotation table switches on. XCTest exposes the app's
22-
/// interface orientation as an integer, and 0 (unknown) deliberately rotates nothing.
23-
enum RunnerInterfaceOrientation {
24-
static let unknown = 0
25-
static let portrait = 1
26-
static let portraitUpsideDown = 2
27-
static let landscapeRight = 3
28-
static let landscapeLeft = 4
29-
}
30-
31-
/// The quarter-turn between an app's interface space and the device's native (portrait-up) space.
32-
enum CoordinateSpaceRotation {
33-
/// An interface-space point, in the native space synthesized events are performed in.
34-
static func native(
35-
point: CGPoint,
36-
in frame: CGRect,
37-
interfaceOrientation: Int
38-
) -> CGPoint {
39-
let localX = Double(point.x) - Double(frame.minX)
40-
let localY = Double(point.y) - Double(frame.minY)
41-
let width = Double(frame.width)
42-
let height = Double(frame.height)
43-
switch interfaceOrientation {
44-
case RunnerInterfaceOrientation.landscapeRight:
45-
return CGPoint(x: height - localY, y: localX)
46-
case RunnerInterfaceOrientation.landscapeLeft:
47-
return CGPoint(x: localY, y: width - localX)
48-
case RunnerInterfaceOrientation.portraitUpsideDown:
49-
return CGPoint(x: width - localX, y: height - localY)
50-
default: // portrait, or an orientation the platform did not name
51-
return CGPoint(x: localX, y: localY)
52-
}
53-
}
54-
55-
/// An interface-space translation vector, in the same native space as `native(point:)`.
56-
static func native(vector: CGVector, interfaceOrientation: Int) -> CGVector {
57-
switch interfaceOrientation {
58-
case RunnerInterfaceOrientation.landscapeRight:
59-
return CGVector(dx: -vector.dy, dy: vector.dx)
60-
case RunnerInterfaceOrientation.landscapeLeft:
61-
return CGVector(dx: vector.dy, dy: -vector.dx)
62-
case RunnerInterfaceOrientation.portraitUpsideDown:
63-
return CGVector(dx: -vector.dx, dy: -vector.dy)
64-
default: // portrait, or an orientation the platform did not name
65-
return vector
66-
}
67-
}
68-
69-
/// The inverse of `native(point:)`: a native-space point, in the app's interface space.
70-
static func oriented(
71-
point: CGPoint,
72-
in frame: CGRect,
73-
interfaceOrientation: Int
74-
) -> CGPoint {
75-
let width = Double(frame.width)
76-
let height = Double(frame.height)
77-
let localX: Double
78-
let localY: Double
79-
switch interfaceOrientation {
80-
case RunnerInterfaceOrientation.landscapeRight:
81-
localX = Double(point.y)
82-
localY = height - Double(point.x)
83-
case RunnerInterfaceOrientation.landscapeLeft:
84-
localX = width - Double(point.y)
85-
localY = Double(point.x)
86-
case RunnerInterfaceOrientation.portraitUpsideDown:
87-
localX = width - Double(point.x)
88-
localY = height - Double(point.y)
89-
default: // portrait, or an orientation the platform did not name
90-
localX = Double(point.x)
91-
localY = Double(point.y)
92-
}
93-
return CGPoint(x: localX + Double(frame.minX), y: localY + Double(frame.minY))
94-
}
95-
96-
/// The inverse of `native(point:)` for a rect. A quarter turn swaps the axes, so opposite corners
97-
/// of the reported rect become opposite corners of the result; ordering them keeps the origin
98-
/// top-left, which is what every consumer of a published rect reads.
99-
static func oriented(
100-
rect: CGRect,
101-
in frame: CGRect,
102-
interfaceOrientation: Int
103-
) -> CGRect {
104-
let leading = oriented(
105-
point: CGPoint(x: rect.minX, y: rect.minY),
106-
in: frame,
107-
interfaceOrientation: interfaceOrientation
108-
)
109-
let trailing = oriented(
110-
point: CGPoint(x: rect.maxX, y: rect.maxY),
111-
in: frame,
112-
interfaceOrientation: interfaceOrientation
113-
)
114-
return CGRect(
115-
x: min(leading.x, trailing.x),
116-
y: min(leading.y, trailing.y),
117-
width: abs(trailing.x - leading.x),
118-
height: abs(trailing.y - leading.y)
119-
)
120-
}
121-
}
122-
123-
/// The space one captured subtree reports its rects in.
124-
///
125-
/// RULE: the capture publishes every rect in the app's orientation space. A surface host declares the
126-
/// space of its own subtree — its box is either the app's box, or the app's box turned through a
127-
/// quarter, which is how a surface hosted in the device's native space announces itself — and anything
128-
/// below it inherits that space. Nothing is declared, and every rect stays exactly as the platform
129-
/// reported it, when the app frame is unusable, when the interface orientation names no quarter turn
130-
/// (portrait, or an orientation the platform did not name), or when the app frame is square. Geometry
131-
/// the capture cannot place is not a claim about where it is, and consumers already fail open on it the
132-
/// way they do on any missing platform fact.
133-
///
134-
/// A half turn is a different claim: an upside-down app's own box is already the native box, so no
135-
/// reported window distinguishes the two spaces and nothing is turned back. That is the same limit the
136-
/// box-only rule accepts for a square app frame, and it is why the consumer rules in
137-
/// `tap-keyboard-occlusion.ts` stay rather than becoming an assertion here.
138-
enum SnapshotGeometrySpace: Equatable {
139-
/** The app's own orientation space: the one every consumer of a published tree reads. */
140-
case appOrientation
141-
/** The device's native (portrait-up) space, with the way back to the app's space attached. */
142-
case deviceNative(appFrame: CGRect, interfaceOrientation: Int)
143-
144-
/// How far a window's side lengths may miss the app's swapped side lengths and still be that
145-
/// quarter turn. Measured captures match to the point; this absorbs float representation only.
146-
static let quarterTurnTolerance: Double = 1
147-
148-
/// The reported rect, in the space the capture publishes.
149-
func orientedFrame(of reportedFrame: CGRect) -> CGRect {
150-
switch self {
151-
case .appOrientation:
152-
return reportedFrame
153-
case .deviceNative(let appFrame, let interfaceOrientation):
154-
return CoordinateSpaceRotation.oriented(
155-
rect: reportedFrame,
156-
in: appFrame,
157-
interfaceOrientation: interfaceOrientation
158-
)
159-
}
160-
}
161-
162-
/// The space one node's subtree reports in.
163-
///
164-
/// Only a surface host may declare a space: the window itself, or the surface the window hands its
165-
/// content to. XCTest reports `UIRemoteKeyboardWindow` with the app's own box and its child with
166-
/// the turned one, so the window alone is not where the turn shows up, while a deep node claiming a
167-
/// turned box is content whose reported bounds happen to be large and is left in the inherited
168-
/// space. A host that reports anything other than a turned box declares the app's space for its
169-
/// subtree, which is how a tree with no rotated surface stays exactly as reported.
170-
static func space(
171-
reportedBySurfaceHost isSurfaceHost: Bool,
172-
reportedFrame: CGRect,
173-
inheritedFrom inherited: SnapshotGeometrySpace,
174-
appFrame: CGRect,
175-
interfaceOrientation: Int
176-
) -> SnapshotGeometrySpace {
177-
guard isSurfaceHost else { return inherited }
178-
guard namesQuarterTurn(interfaceOrientation) else { return .appOrientation }
179-
guard isQuarterTurned(reportedFrame, relativeTo: appFrame) else { return .appOrientation }
180-
return .deviceNative(appFrame: appFrame, interfaceOrientation: interfaceOrientation)
181-
}
182-
183-
/// Whether this orientation is a turn the reported box can be measured against. Portrait and an
184-
/// unnamed orientation leave the device's native space upright, so nothing under them is turned.
185-
static func namesQuarterTurn(_ interfaceOrientation: Int) -> Bool {
186-
interfaceOrientation == RunnerInterfaceOrientation.landscapeLeft
187-
|| interfaceOrientation == RunnerInterfaceOrientation.landscapeRight
188-
}
189-
190-
/// Whether this node, or the window above it, is where a hosted surface's box appears.
191-
static func isSurfaceHost(
192-
elementType: XCUIElement.ElementType?,
193-
parentIsWindow: Bool
194-
) -> Bool {
195-
elementType == .application || elementType == .window || parentIsWindow
196-
}
197-
198-
/// A window that reports the app's two side lengths on the other axes is hosted in the device's
199-
/// native space. A square app frame cannot be told from its own quarter turn, so it is left alone
200-
/// rather than guessed at.
201-
private static func isQuarterTurned(_ frame: CGRect, relativeTo appFrame: CGRect) -> Bool {
202-
guard isPlottable(frame), isPlottable(appFrame),
203-
abs(appFrame.width - appFrame.height) > quarterTurnTolerance
204-
else {
205-
return false
206-
}
207-
return abs(frame.width - appFrame.height) <= quarterTurnTolerance
208-
&& abs(frame.height - appFrame.width) <= quarterTurnTolerance
209-
}
210-
211-
private static func isPlottable(_ frame: CGRect) -> Bool {
212-
frame.origin.x.isFinite && frame.origin.y.isFinite
213-
&& frame.width.isFinite && frame.height.isFinite
214-
&& frame.width > 0 && frame.height > 0
215-
}
216-
}
4+
// The rotation table and the geometry-space rule live in the shared presentation package
5+
// (`SnapshotCoordinateSpace.swift`), which is where the capture's one normalization pass
6+
// (`SnapshotGeometrySpace.normalized`) needs them and where both directions of the turn stay in one
7+
// place. The runner reads them through `import AgentDeviceSnapshotPresentation`; these are the
8+
// runner-side unit tests for that shared rule.
2179

21810
#if AGENT_DEVICE_RUNNER_UNIT_TESTS
21911
private struct WindowCoordinateSpaceFixture: Decodable {
@@ -429,7 +221,7 @@ extension RunnerTests {
429221
XCTAssertEqual(
430222
SnapshotGeometrySpace.space(
431223
reportedBySurfaceHost: SnapshotGeometrySpace.isSurfaceHost(
432-
elementType: XCUIElement.ElementType(rawValue: 20),
224+
type: "Key",
433225
parentIsWindow: false
434226
),
435227
reportedFrame: CGRect(x: 154, y: 77, width: 45, height: 72),
@@ -496,19 +288,7 @@ extension RunnerTests {
496288
/// which window declares the native space cannot depend on which way the app is turned; the
497289
/// rotation rows carry their own orientation.
498290
func testWindowCoordinateSpaceMatchesGoldenParityTable() throws {
499-
let fixtureURL = URL(fileURLWithPath: #filePath)
500-
.deletingLastPathComponent() // AgentDeviceRunnerUITests
501-
.deletingLastPathComponent() // AgentDeviceRunner
502-
.deletingLastPathComponent() // runner
503-
.deletingLastPathComponent() // apple
504-
.deletingLastPathComponent() // repo root
505-
.appendingPathComponent("contracts")
506-
.appendingPathComponent("fixtures")
507-
.appendingPathComponent("window-coordinate-space.json")
508-
let fixture = try JSONDecoder().decode(
509-
WindowCoordinateSpaceFixture.self,
510-
from: Data(contentsOf: fixtureURL)
511-
)
291+
let fixture = try Self.decodeWindowCoordinateSpaceFixture()
512292
XCTAssertFalse(fixture.quarterTurnCases.isEmpty, "parity table must not be empty")
513293
XCTAssertFalse(fixture.rotationCases.isEmpty, "parity table must not be empty")
514294
XCTAssertEqual(
@@ -549,6 +329,25 @@ extension RunnerTests {
549329
)
550330
}
551331
}
332+
333+
fileprivate static func windowCoordinateSpaceFixtureURL() -> URL {
334+
URL(fileURLWithPath: #filePath)
335+
.deletingLastPathComponent() // AgentDeviceRunnerUITests
336+
.deletingLastPathComponent() // AgentDeviceRunner
337+
.deletingLastPathComponent() // runner
338+
.deletingLastPathComponent() // apple
339+
.deletingLastPathComponent() // repo root
340+
.appendingPathComponent("contracts")
341+
.appendingPathComponent("fixtures")
342+
.appendingPathComponent("window-coordinate-space.json")
343+
}
344+
345+
fileprivate static func decodeWindowCoordinateSpaceFixture() throws -> WindowCoordinateSpaceFixture {
346+
try JSONDecoder().decode(
347+
WindowCoordinateSpaceFixture.self,
348+
from: Data(contentsOf: windowCoordinateSpaceFixtureURL())
349+
)
350+
}
552351
}
553352
#endif
554353

@@ -561,19 +360,10 @@ extension RunnerTests {
561360
let landscape = RunnerInterfaceOrientation.landscapeRight
562361

563362
// XCTest gives `UIRemoteKeyboardWindow` the app's own box and its child the turned one, so the
564-
// surface below a window is a host too.
565-
XCTAssertTrue(
566-
SnapshotGeometrySpace.isSurfaceHost(
567-
elementType: XCUIElement.ElementType(rawValue: 21),
568-
parentIsWindow: true
569-
)
570-
);
571-
XCTAssertFalse(
572-
SnapshotGeometrySpace.isSurfaceHost(
573-
elementType: XCUIElement.ElementType(rawValue: 21),
574-
parentIsWindow: false
575-
)
576-
);
363+
// surface below a window is a host too. A non-window type is a host only through its window parent.
364+
XCTAssertTrue(SnapshotGeometrySpace.isSurfaceHost(type: "Element(21)", parentIsWindow: true));
365+
XCTAssertFalse(SnapshotGeometrySpace.isSurfaceHost(type: "Element(21)", parentIsWindow: false));
366+
XCTAssertTrue(SnapshotGeometrySpace.isSurfaceHost(type: "Window", parentIsWindow: false));
577367

578368
let windowSpace = SnapshotGeometrySpace.space(
579369
reportedBySurfaceHost: true,

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import XCTest
2+
import AgentDeviceSnapshotPresentation
23

34
// The scroll viewport rule the runner shares with the TS runtime (#2500).
45
//

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift‎

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -220,9 +220,7 @@ extension RunnerTests {
220220
let viewport = geometry.viewport
221221
let nodes = privateAXAcquisition(
222222
rawRoot: root,
223-
hint: hint,
224-
viewport: viewport,
225-
interfaceOrientation: geometry.interfaceOrientation
223+
hint: hint
226224
)
227225
// Serialization-level emptiness only: an acquired-but-fully-clipped tree is presentation's
228226
// verdict now, surfaced by the plan's sparse classifier on the presented payload (#1797).
@@ -256,7 +254,8 @@ extension RunnerTests {
256254
customActions: Self.privateAXCustomActionCoverage(
257255
response[RunnerAXSnapshotCustomActionsKey]
258256
),
259-
viewport: viewport
257+
viewport: viewport,
258+
interfaceOrientation: geometry.interfaceOrientation
260259
)
261260
#else
262261
return nil
@@ -697,9 +696,7 @@ extension RunnerTests {
697696
rawRoot: tree,
698697
hint: CaptureHint(
699698
projection: .regular, depth: nil, regularPresentedDepth: nil,
700-
interactiveOnly: false, customActions: false),
701-
viewport: CGRect(x: 0, y: 0, width: 390, height: 844),
702-
interfaceOrientation: RunnerInterfaceOrientation.portrait
699+
interactiveOnly: false, customActions: false)
703700
)
704701

705702
let card = nodes.first { $0.label == "feedItem-by-whiskers.test" }
@@ -732,9 +729,7 @@ extension RunnerTests {
732729
scope: "homeScreen",
733730
raw: false
734731
)
735-
),
736-
viewport: .infinite,
737-
interfaceOrientation: RunnerInterfaceOrientation.portrait
732+
)
738733
)
739734

740735
let labels = nodes.compactMap { $0.label ?? $0.identifier }
@@ -812,9 +807,11 @@ extension RunnerTests {
812807
let hint = CaptureHint(
813808
projection: .regular, depth: nil, regularPresentedDepth: nil,
814809
interactiveOnly: true, customActions: false)
815-
let acquired = privateAXAcquisition(
816-
rawRoot: tree, hint: hint, viewport: viewport,
817-
interfaceOrientation: RunnerInterfaceOrientation.portrait)
810+
let acquired = SnapshotGeometrySpace.normalized(
811+
nodes: privateAXAcquisition(rawRoot: tree, hint: hint),
812+
viewport: viewport,
813+
interfaceOrientation: RunnerInterfaceOrientation.portrait
814+
)
818815
// Acquisition serializes the drawer too; the shared fold is what hides it (#1797).
819816
XCTAssertTrue(acquired.compactMap(\.label).contains("Admin settings"))
820817

0 commit comments

Comments
 (0)