Skip to content

Commit a8dbd01

Browse files
committed
fix(ios): translate only snapshot frames and pin the unmeasured landscape case [ #3899 ]
1 parent 7fa3590 commit a8dbd01

5 files changed

Lines changed: 62 additions & 17 deletions

File tree

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,7 @@ extension RunnerTests {
188188
/// compared against screen-space probes, so the center enters the target translated.
189189
func withResolvedInput(
190190
_ input: ResolvedTextEntryInput?,
191-
translation: CGSize? = nil
191+
translation: CGSize?
192192
) -> TextEntryTarget {
193193
guard let input else {
194194
return self
@@ -270,7 +270,7 @@ extension RunnerTests {
270270

271271
init(
272272
target: TextEntryTarget,
273-
translation: CGSize? = nil,
273+
translation: CGSize?,
274274
resolve: @escaping (TextEntryTarget) -> ResolvedTextEntryInput?
275275
) {
276276
self.translation = translation

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift‎

Lines changed: 44 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -135,9 +135,10 @@ extension RunnerTests {
135135
let target = input?.element
136136
let requestedPoint = CGPoint(x: x, y: y)
137137
// The input's frame arrives from a snapshot, which reports a placed window's content in the
138-
// window's own space; the tap below is a screen point (#3899).
139-
if let read = input, !read.attributes.frame.isEmpty {
140-
let frame = snapshotFrameOnScreen(read.attributes.frame, translation: liveContentTranslation(app: app))
138+
// window's own space; the tap below is a screen point (#3899). An input read live (no
139+
// snapshot) already names screen space, so only the snapshot frame is translated here.
140+
if let snapshotFrame = input?.snapshot?.frame, !snapshotFrame.isEmpty {
141+
let frame = snapshotFrameOnScreen(snapshotFrame, translation: liveContentTranslation(app: app))
141142
_ = tapAt(app: app, x: frame.midX, y: frame.midY)
142143
} else {
143144
_ = tapAt(app: app, x: x, y: y)
@@ -146,7 +147,11 @@ extension RunnerTests {
146147
// belong to a previous field. With a concrete target we type through XCUIElement.typeText,
147148
// so after tapping it the iOS readiness timeout cannot prove anything stronger.
148149
if keyboardVisibleBeforeTap, let target {
149-
return focusedTextEntryTarget(target, requestedPoint: requestedPoint)
150+
return focusedTextEntryTarget(
151+
target,
152+
requestedPoint: requestedPoint,
153+
translation: liveContentTranslation(app: app)
154+
)
150155
}
151156
let stabilized = stabilizeTextInputBeforeTyping(
152157
app: app,
@@ -160,7 +165,12 @@ extension RunnerTests {
160165
stabilized.polls
161166
)
162167
if case .inputReportsKeyboardFocus(let read) = stabilized.evidence {
163-
return focusedTextEntryTarget(read.element, read: read, requestedPoint: requestedPoint)
168+
return focusedTextEntryTarget(
169+
read.element,
170+
read: read,
171+
requestedPoint: requestedPoint,
172+
translation: liveContentTranslation(app: app)
173+
)
164174
}
165175
let readyTarget = TextEntryTarget(
166176
element: stabilized.element ?? target,
@@ -170,26 +180,44 @@ extension RunnerTests {
170180
let element = stabilized.focusConfirmed
171181
? readyTarget.element
172182
: (waitForTextEntryReadiness(app: app, target: readyTarget) ?? readyTarget.element)
173-
return focusedTextEntryTarget(element, requestedPoint: requestedPoint)
183+
return focusedTextEntryTarget(
184+
element,
185+
requestedPoint: requestedPoint,
186+
translation: liveContentTranslation(app: app)
187+
)
174188
}
175189

176190
/// The target the focus step hands to text entry, with the read its refresh point came from.
177-
private func focusedTextEntryTarget(_ element: XCUIElement?, requestedPoint: CGPoint) -> TextEntryTarget {
191+
private func focusedTextEntryTarget(
192+
_ element: XCUIElement?,
193+
requestedPoint: CGPoint,
194+
translation: CGSize?
195+
) -> TextEntryTarget {
178196
guard let element else {
179197
return TextEntryTarget(element: nil, refreshPoint: requestedPoint, prefersFocusedElement: false)
180198
}
181-
return focusedTextEntryTarget(element, read: ResolvedTextEntryInput(reading: element), requestedPoint: requestedPoint)
199+
return focusedTextEntryTarget(
200+
element,
201+
read: ResolvedTextEntryInput(reading: element),
202+
requestedPoint: requestedPoint,
203+
translation: translation
204+
)
182205
}
183206

184207
/// `read` is `element` read after the focus tap, or nil when the read found no element.
185208
private func focusedTextEntryTarget(
186209
_ element: XCUIElement,
187210
read: ResolvedTextEntryInput?,
188-
requestedPoint: CGPoint
211+
requestedPoint: CGPoint,
212+
translation: CGSize?
189213
) -> TextEntryTarget {
190214
return TextEntryTarget(
191215
element: element,
192-
refreshPoint: textEntryRefreshPoint(frame: read?.snapshot?.frame, fallbackTo: requestedPoint),
216+
refreshPoint: textEntryRefreshPoint(
217+
frame: read?.snapshot?.frame,
218+
fallbackTo: requestedPoint,
219+
translation: translation
220+
),
193221
prefersFocusedElement: false,
194222
postFocusRead: read?.snapshot == nil ? nil : read
195223
)
@@ -206,14 +234,18 @@ extension RunnerTests {
206234
/// typing into that other field.
207235
private func textEntryRefreshPoint(
208236
frame: CGRect?,
209-
fallbackTo requestedPoint: CGPoint
237+
fallbackTo requestedPoint: CGPoint,
238+
translation: CGSize?
210239
) -> CGPoint? {
211240
guard let frame else {
212241
return nil
213242
}
214243
guard !frame.isEmpty else {
215244
return requestedPoint
216245
}
217-
return CGPoint(x: frame.midX, y: frame.midY)
246+
// The read's frame names the window's content space on a placed window; a refresh point is
247+
// compared against screen-space probes, so the center enters the target translated (#3899).
248+
let onScreen = snapshotFrameOnScreen(frame, translation: translation)
249+
return CGPoint(x: onScreen.midX, y: onScreen.midY)
218250
}
219251
}

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,15 @@ extension RunnerTests {
170170
if shouldStop() { break }
171171
guard let candidate = ResolvedTextEntryInput(readingListed: element) else { continue }
172172
let attributes = candidate.attributes
173-
let frame = snapshotFrameOnScreen(attributes.frame, translation: translation)
173+
// Only the snapshot names the window's content space; a candidate read live (no snapshot)
174+
// already reports screen space, and translating it again would move the candidate off the
175+
// point it is being matched against (#3899).
176+
let frame: CGRect
177+
if let snapshot = candidate.snapshot {
178+
frame = snapshotFrameOnScreen(snapshot.frame, translation: translation)
179+
} else {
180+
frame = attributes.frame
181+
}
174182
if isCoordinateTextInputCandidate(enabled: attributes.isEnabled, frame: frame, point: point) {
175183
inputs.append(candidate)
176184
}

‎apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ final class CoordinateSpaceTests: XCTestCase {
7878
),
7979
CGPoint(x: 288, y: 1030),
8080
"a frame offset from the screen never moves the turned point (#3899)"
81-
);
81+
)
8282
}
8383

8484
func testNativeSynthesizedVectorRotatesByInterfaceOrientation() {
@@ -581,7 +581,7 @@ final class CoordinateSpaceTests: XCTestCase {
581581
)
582582
).map(\.rect),
583583
acquired.map(\.rect)
584-
);
584+
)
585585

586586
// A windowed app, measured shape: the app's own window declares the translation and carries the
587587
// subtree; the keyboard window and the SpringBoard overlay report the screen's own box, so their

‎docs/adr/0004-ios-snapshot-backend-strategy.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -663,6 +663,11 @@ The measured dispatch spaces for a placed window are both screen-anchored: the s
663663
The live `XCUIElement.frame` property reports screen space on a placed window, as the selector-tap path proved on the device: it measured the `Add` button at (641,321) for a window at (324,307).
664664
The one live read that does not is `XCUIElementSnapshot.frame`, which reports the window's content space like the capture channel does.
665665
So the snapshot capture converts in `SnapshotGeometrySpace.normalized`, and the text-input probe converts its snapshot reads through the same `SnapshotContentTranslation` rule read from the two live boxes (`app.frame` against the resolved window's frame); every other live read needs no conversion.
666+
Where a probe read has no snapshot and falls back to the live element, its frame is already screen space and is not translated.
667+
`app.frame` unions transformed subtrees, so content pushed off a placed window (a closed drawer at negative x) can stop the two live boxes matching and read no vector; the capture still converts from the snapshot root box, so a probe may then read window-local frames where the capture published screen space. A mid-command window move has the same effect on a vector read at command start.
668+
Both are reads of a moving target, not a second space: a re-opened command reads the current vector.
669+
The host bridge keeps refusing a windowed capture (`app-window-not-full-screen`) rather than converting one, so the TypeScript twin of the rule runs today only in the parity replay.
670+
Portrait is the only orientation measured for a placed window; the quarter-turn table's windowed row pins the arithmetic (a frame origin never travels through the turn), not a device measurement, and landscape under Windowed Apps stays unverified until measured.
666671

667672
## Amendment: modal containment in the presentation cut
668673

0 commit comments

Comments
 (0)