Skip to content

Commit a7370b2

Browse files
committed
fix(ios): publish a windowed iPad app's capture in screen space (#3899)
An iPad window that does not fill the screen reports its own screen box while its subtree arrives anchored at its own top-left. The capture published a screen-space viewport around window-relative rects: the fold clipped content to (w - x) x (h - y), a rect named no screenshot position, and a synthesized tap landed at the point minus the window origin. SnapshotContentTranslation reads the one vector from a carrier's (0,0)-anchored copy of the viewport box to the viewport itself, and SnapshotGeometrySpace.normalized translates the subtree it declares in the same pass that turns a quarter-turned subtree. CoordinateSpaceRotation now turns axes only; the digitizer origin stays where the display put it. Replayed from contracts/fixtures/window-coordinate-space.json in both languages. Refs #3899
1 parent b08b5e7 commit a7370b2

8 files changed

Lines changed: 600 additions & 49 deletions

File tree

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -662,6 +662,133 @@ extension RunnerTests {
662662
XCTAssertNotNil(keyQ, "`q` key must be presented under --depth \(options.depth ?? -1)")
663663
}
664664

665+
/// #3899: an iPad window that does not fill the screen publishes screen space. The measured shape is
666+
/// iPad Pro 11-inch (M5, iOS 27.0) with Windowed Apps on and a Calendar window at (224,207,386x682):
667+
/// the app's own window arrives as a (0,0)-anchored copy of that box and its subtree with it, so the
668+
/// Add button reports (335,14,37,36) for a control drawn at (559,221). The one normalization pass
669+
/// reads the declaration and publishes the subtree at its place on the screen, which is what lets the
670+
/// fold keep every node instead of clipping each one to (w − x) × (h − y) and refusing the capture.
671+
func testAWindowedSubtreeIsPublishedInScreenSpaceAndKeepsEveryNode() throws {
672+
let windowOnScreen = CGRect(x: 224, y: 207, width: 386, height: 682)
673+
func node(
674+
_ index: Int, _ type: String, _ label: String, _ rect: CGRect, _ parent: Int?, _ depth: Int
675+
) -> RawAXNode {
676+
RawAXNode(
677+
index: index, type: type, label: label, identifier: nil, value: nil,
678+
rect: SnapshotRect(x: rect.minX, y: rect.minY, width: rect.width, height: rect.height),
679+
enabled: true, focused: nil, selected: nil, hittable: false,
680+
depth: depth, parentIndex: parent, hiddenContentAbove: nil, hiddenContentBelow: nil
681+
)
682+
}
683+
let reportedRoot = CGRect(x: 0, y: 0, width: 386, height: 682)
684+
let acquired = [
685+
node(0, "Application", "Calendar", reportedRoot, nil, 0),
686+
node(1, "Window", "Window", reportedRoot, 0, 1),
687+
node(2, "Other", "Toolbar", reportedRoot, 1, 2),
688+
node(3, "Button", "Add", CGRect(x: 335, y: 14, width: 37, height: 36), 2, 3),
689+
node(4, "Button", "Today", CGRect(x: 8, y: 583, width: 90, height: 38), 2, 3),
690+
// A SpringBoard overlay under the app root reports the screen's own box: its subtree is already
691+
// in screen space and must not move.
692+
node(5, "Window", "Alert", CGRect(x: 0, y: 0, width: 834, height: 1210), 0, 1),
693+
node(6, "Button", "Discard Changes", CGRect(x: 300, y: 500, width: 240, height: 44), 5, 2),
694+
]
695+
let normalized = SnapshotGeometrySpace.normalized(
696+
nodes: acquired,
697+
viewport: .reported(
698+
box: windowOnScreen,
699+
interfaceOrientation: RunnerInterfaceOrientation.portrait
700+
)
701+
)
702+
let options = PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false)
703+
let capture = try XCTUnwrap(try SnapshotPresentation.present(
704+
SnapshotAcquisition(
705+
hint: SnapshotPresentation.captureHint(for: options),
706+
nodes: normalized,
707+
truncated: false,
708+
effectiveDepth: nil,
709+
viewport: .reported(box: windowOnScreen)
710+
),
711+
options: options
712+
))
713+
714+
// One space: the root answers with the window's screen box, and the button with the position the
715+
// screenshot draws.
716+
XCTAssertEqual(capture.nodes.first?.rect, SnapshotRect(windowOnScreen))
717+
let add = try XCTUnwrap(capture.nodes.first { $0.label == "Add" })
718+
XCTAssertEqual(add.rect, SnapshotRect(x: 559, y: 221, width: 37, height: 36))
719+
XCTAssertEqual(add.hittable, true)
720+
// Nothing is dropped for being "outside the clip": the whole acquired tree is still there. The one
721+
// clipped node is the system overlay that really is larger than the app's window, which is the
722+
// fold's normal answer to oversized content — not the mixed-space clip that used to publish
723+
// (w − x) × (h − y) boxes and refuse this capture outright.
724+
XCTAssertEqual(Set(capture.nodes.map(\.label)), Set(acquired.map(\.label)))
725+
let clipNote = capture.qualityNotes.first { $0.code == .regularGeometryClipped }
726+
XCTAssertEqual(clipNote?.droppedNodeCount, 0, "no node is dropped for the window's origin")
727+
XCTAssertEqual(clipNote?.clippedNodeCount, 1, "only the overlay larger than the window is clipped")
728+
XCTAssertEqual(
729+
capture.nodes.first { $0.label == "Alert" }?.rect,
730+
SnapshotRect(x: 224, y: 207, width: 386, height: 682),
731+
"the overlay is clipped to the window's visible box, not to (w − x) × (h − y) of it"
732+
)
733+
// The system window and its button stay where the platform reported them.
734+
let discard = try XCTUnwrap(capture.nodes.first { $0.label == "Discard Changes" })
735+
XCTAssertEqual(discard.rect, SnapshotRect(x: 300, y: 500, width: 240, height: 44))
736+
}
737+
738+
/// #3899 keeps a full-screen capture byte-identical: the same tree reported from the screen origin is
739+
/// published exactly as reported, with no translation read from its zero-origin viewport.
740+
func testAFullScreenCapturePublishesWhatThePlatformReported() throws {
741+
let screen = CGRect(x: 0, y: 0, width: 834, height: 1210)
742+
func node(
743+
_ index: Int, _ type: String, _ label: String, _ rect: CGRect, _ parent: Int?, _ depth: Int
744+
) -> RawAXNode {
745+
RawAXNode(
746+
index: index, type: type, label: label, identifier: nil, value: nil,
747+
rect: SnapshotRect(x: rect.minX, y: rect.minY, width: rect.width, height: rect.height),
748+
enabled: true, focused: nil, selected: nil, hittable: false,
749+
depth: depth, parentIndex: parent, hiddenContentAbove: nil, hiddenContentBelow: nil
750+
)
751+
}
752+
let acquired = [
753+
node(0, "Application", "Calendar", screen, nil, 0),
754+
node(1, "Window", "Window", screen, 0, 1),
755+
node(2, "Button", "Add", CGRect(x: 335, y: 14, width: 37, height: 36), 1, 2),
756+
]
757+
let normalized = SnapshotGeometrySpace.normalized(
758+
nodes: acquired,
759+
viewport: .reported(
760+
box: screen,
761+
interfaceOrientation: RunnerInterfaceOrientation.portrait
762+
)
763+
)
764+
XCTAssertEqual(normalized.map(\.rect), acquired.map(\.rect))
765+
XCTAssertEqual(
766+
SnapshotContentTranslation.offset(reportedBox: screen, viewportBox: screen),
767+
nil,
768+
"a viewport at the screen origin carries no translation"
769+
)
770+
}
771+
772+
/// #3899 keeps ADR 0025 foldable panel taps: a panel reports its own box from its own display's
773+
/// origin, so no translation is read and the panel's geometry stays exactly as measured.
774+
func testAFoldablePanelReportsNoContentTranslation() {
775+
let panel = CGRect(x: 0, y: 0, width: 951, height: 669)
776+
XCTAssertNil(
777+
SnapshotContentTranslation.offset(reportedBox: panel, viewportBox: panel),
778+
"a panel box anchored at its display's origin names no translation"
779+
)
780+
// A digitizer point on the panel is passed through, so the fold's display selection stays the only
781+
// window-dependent choice the tap path makes.
782+
XCTAssertEqual(
783+
CoordinateSpaceRotation.native(
784+
point: CGPoint(x: 476, y: 226),
785+
in: panel,
786+
interfaceOrientation: RunnerInterfaceOrientation.portrait
787+
),
788+
CGPoint(x: 476, y: 226)
789+
)
790+
}
791+
665792
/// #2891: a capture whose viewport read failed publishes no `hittable` for a node whose answer is
666793
/// containment, while a root and a disabled node stay a declared `false`. Pinned on the
667794
/// encoded objects rather than on `capture.nodes`, because the promise is about the wire: a test

‎apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift‎

Lines changed: 88 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@ import CoreGraphics
44
// Rotation between an app's interface space and the device's native (portrait-up) space, shared by
55
// synthesized dispatch (forward) and capture (inverse). Rationale and measurements: ADR 0004
66
// "the coordinate space of a captured subtree" and contracts/fixtures/window-coordinate-space.json.
7+
//
8+
// Rotation moves axes only. It never moves an origin: a placed window's offset from the screen
9+
// belongs to `SnapshotContentTranslation` (#3899), which is the one rule that turns a reported box
10+
// into its screen-space box. Keeping the two apart is what lets a capture publish one space.
711

812
public enum RunnerInterfaceOrientation {
913
public static let unknown = 0
@@ -14,18 +18,18 @@ public enum RunnerInterfaceOrientation {
1418
}
1519

1620
public enum CoordinateSpaceRotation {
17-
// Dispatch rotates a point about the reference frame's top-left. The origin is localized away
18-
// and never added back: an XCUIApplication session observes only its own scene surface, so its
19-
// windows report (0,0)-anchored frames (measured on iPhone landscape both ways and on every
20-
// system window in window-coordinate-space.json), and displacement between panels or panes is
21-
// carried by the record's displayID, not by an origin.
21+
// Dispatch turns a point's axes and never moves its origin: the turn says which axis a box runs
22+
// along, not where that box sits (#3899). The digitizer's origin is the display's own, which is
23+
// where a windowed app's screen-space points are already measured from, so a placed window's
24+
// offset from the screen belongs to `SnapshotContentTranslation` — the one rule that reads an
25+
// offset — and not to this function. The `frame` argument names the side lengths the turn swaps.
2226
public static func native(
2327
point: CGPoint,
2428
in frame: CGRect,
2529
interfaceOrientation: Int
2630
) -> CGPoint {
27-
let localX = Double(point.x) - Double(frame.minX)
28-
let localY = Double(point.y) - Double(frame.minY)
31+
let localX = Double(point.x)
32+
let localY = Double(point.y)
2933
let width = Double(frame.width)
3034
let height = Double(frame.height)
3135
switch interfaceOrientation {
@@ -60,23 +64,23 @@ public enum CoordinateSpaceRotation {
6064
) -> CGPoint {
6165
let width = Double(frame.width)
6266
let height = Double(frame.height)
63-
let localX: Double
64-
let localY: Double
67+
let x: Double
68+
let y: Double
6569
switch interfaceOrientation {
6670
case RunnerInterfaceOrientation.landscapeRight:
67-
localX = Double(point.y)
68-
localY = height - Double(point.x)
71+
x = Double(point.y)
72+
y = height - Double(point.x)
6973
case RunnerInterfaceOrientation.landscapeLeft:
70-
localX = width - Double(point.y)
71-
localY = Double(point.x)
74+
x = width - Double(point.y)
75+
y = Double(point.x)
7276
case RunnerInterfaceOrientation.portraitUpsideDown:
73-
localX = width - Double(point.x)
74-
localY = height - Double(point.y)
77+
x = width - Double(point.x)
78+
y = height - Double(point.y)
7579
default:
76-
localX = Double(point.x)
77-
localY = Double(point.y)
80+
x = Double(point.x)
81+
y = Double(point.y)
7882
}
79-
return CGPoint(x: localX + Double(frame.minX), y: localY + Double(frame.minY))
83+
return CGPoint(x: x, y: y)
8084
}
8185

8286
public static func oriented(
@@ -103,9 +107,57 @@ public enum CoordinateSpaceRotation {
103107
}
104108
}
105109

110+
/// Whether a carrier's own box declares that its subtree reports in the carrier's content space
111+
/// rather than in the space the viewport names (#3899).
112+
///
113+
/// An iPad window that does not fill the screen (Windowed Apps, Stage Manager) reports its own frame
114+
/// at its place on the screen while its subtree arrives anchored at its own top-left: measured on
115+
/// iPad Pro 11-inch (M5, iOS 27.0) with a Calendar window whose `frame` is `(224,207,386,682)`, the
116+
/// application root reports `(0,0,386,682)` and its Add button `(335,14,37,36)` at screen
117+
/// `(559,221)`. A full-screen window, an iPhone, and each foldable panel report origins at zero on
118+
/// both sides, so the rule reads zero translation there and every published rect is unchanged.
119+
///
120+
/// The signature is the carrier's box as a `(0,0)`-anchored copy of the viewport box: the side
121+
/// lengths match within the tolerance and the viewport — the app's screen-anchored box — starts
122+
/// somewhere else on the screen. Any other carrier box names no space: it is not the app's own
123+
/// window, so its numbers say nothing about where the app's subtree starts.
124+
public enum SnapshotContentTranslation {
125+
/// The vector from a carrier's reported box to the screen space it is placed at, or `nil` when the
126+
/// carrier's box declares no translation. One box offset from one viewport box, decided from the
127+
/// same tolerance the quarter-turn rule reads; replayed against
128+
/// `contracts/fixtures/window-coordinate-space.json` in `windowContentCases`.
129+
public static func offset(reportedBox: CGRect, viewportBox: CGRect) -> CGSize? {
130+
guard SnapshotGeometry.isPositiveFinite(reportedBox),
131+
SnapshotGeometry.isPositiveFinite(viewportBox)
132+
else {
133+
return nil
134+
}
135+
guard abs(reportedBox.minX) <= SnapshotGeometrySpace.quarterTurnTolerance,
136+
abs(reportedBox.minY) <= SnapshotGeometrySpace.quarterTurnTolerance
137+
else {
138+
return nil
139+
}
140+
guard abs(viewportBox.minX) > SnapshotGeometrySpace.quarterTurnTolerance
141+
|| abs(viewportBox.minY) > SnapshotGeometrySpace.quarterTurnTolerance
142+
else {
143+
return nil
144+
}
145+
guard abs(reportedBox.width - viewportBox.width) <= SnapshotGeometrySpace.quarterTurnTolerance,
146+
abs(reportedBox.height - viewportBox.height) <= SnapshotGeometrySpace.quarterTurnTolerance
147+
else {
148+
return nil
149+
}
150+
return CGSize(width: viewportBox.minX, height: viewportBox.minY)
151+
}
152+
}
153+
106154
public enum SnapshotGeometrySpace: Equatable {
107155
case appOrientation
108156
case deviceNative(appFrame: SnapshotViewport.Box, interfaceOrientation: Int)
157+
/// The subtree was reported in a placed window's own content space and is published translated
158+
/// into the viewport's screen space (`SnapshotContentTranslation`, #3899). The offset is the
159+
/// vector from the window's reported box to the viewport box it sits at on screen.
160+
case windowContent(offset: CGSize)
109161

110162
public static let quarterTurnTolerance: Double = 1
111163

@@ -119,24 +171,36 @@ public enum SnapshotGeometrySpace: Equatable {
119171
in: appFrame.rect,
120172
interfaceOrientation: interfaceOrientation
121173
)
174+
case .windowContent(let offset):
175+
return reportedFrame.offsetBy(dx: offset.width, dy: offset.height)
122176
}
123177
}
124178

125-
/// Only a `.reported` viewport carries an orientation, so only it can anchor a rotation.
179+
/// A carrier declares its own subtree's space from its own box. A `.reported` viewport is the
180+
/// screen-anchored fact both declarations read against: a quarter turn needs its orientation, and
181+
/// a window-content offset needs its origin. A `.derived` viewport carries no screen anchor, so a
182+
/// subtree under it publishes what the platform reported rather than a guessed space.
126183
public static func space(
127184
reportedBySurfaceHost isSurfaceHost: Bool,
128185
reportedFrame: CGRect,
129186
inheritedFrom inherited: SnapshotGeometrySpace,
130187
viewport: SnapshotViewport
131188
) -> SnapshotGeometrySpace {
132189
guard isSurfaceHost else { return inherited }
133-
guard case .reported(let appFrame, let interfaceOrientation) = viewport,
134-
namesQuarterTurn(interfaceOrientation),
135-
isQuarterTurned(reportedFrame, relativeTo: appFrame.rect)
136-
else {
190+
guard case .reported(let appFrame, let interfaceOrientation) = viewport else {
137191
return .appOrientation
138192
}
139-
return .deviceNative(appFrame: appFrame, interfaceOrientation: interfaceOrientation)
193+
if namesQuarterTurn(interfaceOrientation),
194+
isQuarterTurned(reportedFrame, relativeTo: appFrame.rect) {
195+
return .deviceNative(appFrame: appFrame, interfaceOrientation: interfaceOrientation)
196+
}
197+
if let offset = SnapshotContentTranslation.offset(
198+
reportedBox: reportedFrame,
199+
viewportBox: appFrame.rect
200+
) {
201+
return .windowContent(offset: offset)
202+
}
203+
return .appOrientation
140204
}
141205

142206
public static func namesQuarterTurn(_ interfaceOrientation: Int) -> Bool {

0 commit comments

Comments
 (0)