Skip to content

Commit 6bb1ff8

Browse files
committed
fix(ios): read a placed window's snapshot frames at their screen place (#3899)
XCUIElement.frame reports a placed window's content in screen space, but XCUIElementSnapshot.frame reports it in the window's own content space, so the text-input probe saw two spaces on an iPad Windowed Apps window and refused a fill whose point was on the screen. The probe, the focus tap, and the typing reader now read the same SnapshotContentTranslation vector the capture reads, from the two live boxes that name it. Refs #3899
1 parent 8b0427f commit 6bb1ff8

5 files changed

Lines changed: 64 additions & 11 deletions

File tree

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -182,11 +182,18 @@ extension RunnerTests {
182182

183183
/// The target moved to the input a resolution read: its element, a refresh point at its center,
184184
/// and its identity while the target is unbound.
185-
func withResolvedInput(_ input: ResolvedTextEntryInput?) -> TextEntryTarget {
185+
///
186+
/// `translation` is the live snapshot-frame vector for the reading session (#3899): the snapshot
187+
/// frame of a placed window's input names the window's content space, while a refresh point is
188+
/// compared against screen-space probes, so the center enters the target translated.
189+
func withResolvedInput(
190+
_ input: ResolvedTextEntryInput?,
191+
translation: CGSize? = nil
192+
) -> TextEntryTarget {
186193
guard let input else {
187194
return self
188195
}
189-
let frame = input.snapshot?.frame ?? .zero
196+
let frame = snapshotFrameOnScreen(input.snapshot?.frame ?? .zero, translation: translation)
190197
return TextEntryTarget(
191198
element: input.element,
192199
refreshPoint: frame.isEmpty ? refreshPoint : CGPoint(x: frame.midX, y: frame.midY),
@@ -259,9 +266,15 @@ extension RunnerTests {
259266
private(set) var target: TextEntryTarget
260267
private var lastRead: ResolvedTextEntryInput?
261268
private let resolve: (TextEntryTarget) -> ResolvedTextEntryInput?
269+
private let translation: CGSize?
262270

263-
init(target: TextEntryTarget, resolve: @escaping (TextEntryTarget) -> ResolvedTextEntryInput?) {
264-
self.target = target.withResolvedInput(target.postFocusRead)
271+
init(
272+
target: TextEntryTarget,
273+
translation: CGSize? = nil,
274+
resolve: @escaping (TextEntryTarget) -> ResolvedTextEntryInput?
275+
) {
276+
self.translation = translation
277+
self.target = target.withResolvedInput(target.postFocusRead, translation: translation)
265278
lastRead = target.postFocusRead
266279
self.resolve = resolve
267280
}
@@ -271,7 +284,7 @@ extension RunnerTests {
271284
return lastRead
272285
}
273286
let read = resolve(target)
274-
target = target.withResolvedInput(read)
287+
target = target.withResolvedInput(read, translation: translation)
275288
lastRead = read
276289
return read
277290
}

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,10 @@ extension RunnerTests {
134134
let input = textInputAt(app: app, x: x, y: y)
135135
let target = input?.element
136136
let requestedPoint = CGPoint(x: x, y: y)
137-
if let frame = input?.attributes.frame, !frame.isEmpty {
137+
// The input's frame arrives from a snapshot, which reports a placed window's content in the
138+
// window's own space; the tap below is a screen point (#3899).
139+
if let read = input, !read.attributes.frame.isEmpty {
140+
let frame = snapshotFrameOnScreen(read.attributes.frame, translation: liveContentTranslation(app: app))
138141
_ = tapAt(app: app, x: frame.midX, y: frame.midY)
139142
} else {
140143
_ = tapAt(app: app, x: x, y: y)

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import XCTest
2+
import AgentDeviceSnapshotPresentation
23

34
final class TextInputProbeIssues {
45
let thread = Thread.current
@@ -153,6 +154,7 @@ extension RunnerTests {
153154
shouldStop: () -> Bool
154155
) -> [ResolvedTextEntryInput] {
155156
var inputs: [ResolvedTextEntryInput] = []
157+
let translation = liveContentTranslation(app: app)
156158
for query in [app.textFields, app.secureTextFields, app.searchFields, app.textViews] {
157159
if shouldStop() { break }
158160
let candidates = query.allElementsBoundByIndex
@@ -168,12 +170,36 @@ extension RunnerTests {
168170
if shouldStop() { break }
169171
guard let candidate = ResolvedTextEntryInput(readingListed: element) else { continue }
170172
let attributes = candidate.attributes
171-
if isCoordinateTextInputCandidate(enabled: attributes.isEnabled, frame: attributes.frame, point: point) {
173+
let frame = snapshotFrameOnScreen(attributes.frame, translation: translation)
174+
if isCoordinateTextInputCandidate(enabled: attributes.isEnabled, frame: frame, point: point) {
172175
inputs.append(candidate)
173176
}
174177
}
175178
}
176179
guard !shouldStop() else { return [] }
177180
return inputs.sorted { smallestElementFirst($0.attributes, $1.attributes) }
178181
}
182+
183+
/// The vector that carries this session's snapshot-read app-content frames to their place on the
184+
/// screen, or nil when the app fills the screen (#3899).
185+
///
186+
/// The live `XCUIElement.frame` property already reports a placed window's content in screen space,
187+
/// but `XCUIElementSnapshot.frame` reports it in the window's own content space, so the same probe
188+
/// sees two spaces on an iPad Windowed Apps window. The signature and the vector are the capture's
189+
/// rule (`SnapshotContentTranslation`, ADR 0004) read from the two live boxes that name it: the
190+
/// application's `frame` is a `(0,0)`-anchored copy of the box the resolved window reports at its
191+
/// place on the screen. Any other pair of boxes declares no vector, so full-screen, iPhone, and
192+
/// each foldable panel keep their frames exactly as read.
193+
func liveContentTranslation(app: XCUIApplication) -> CGSize? {
194+
safely("LIVE_CONTENT_TRANSLATION", nil) {
195+
guard let resolved = resolveRunnerWindowElement(app: app) else { return nil }
196+
return SnapshotContentTranslation.offset(reportedBox: app.frame, viewportBox: resolved.frame)
197+
}
198+
}
199+
}
200+
201+
/// `frame` as the screen names it, given `liveContentTranslation`'s vector.
202+
func snapshotFrameOnScreen(_ frame: CGRect, translation: CGSize?) -> CGRect {
203+
guard let translation else { return frame }
204+
return frame.offsetBy(dx: translation.width, dy: translation.height)
179205
}

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,13 @@ extension RunnerTests {
1616
) -> TextEntryResult {
1717
let totalStartedAt = Date()
1818
let resolve = { (target: TextEntryTarget) in self.resolveTextEntryInput(app: app, target: target) }
19-
var reader = TextEntryReader(target: target, resolve: resolve)
19+
// Reads inside the typing loop answer from snapshots, which report a placed window's content in
20+
// the window's own space; the reader's refresh points are screen points (#3899).
21+
var reader = TextEntryReader(
22+
target: target,
23+
translation: liveContentTranslation(app: app),
24+
resolve: resolve
25+
)
2026
guard !text.isEmpty else {
2127
// Replacing a field WITH the empty string is the clear-field primitive (#2063): the typing
2228
// is vacuous but the clear is not. Returning here unconditionally left the old value in
@@ -74,7 +80,11 @@ extension RunnerTests {
7480
case .completed(let result):
7581
return result
7682
case .fallback(let fallbackTarget):
77-
reader = TextEntryReader(target: fallbackTarget, resolve: resolve)
83+
reader = TextEntryReader(
84+
target: fallbackTarget,
85+
translation: liveContentTranslation(app: app),
86+
resolve: resolve
87+
)
7888
}
7989
}
8090
let shouldUseSynthesizedFirstResponderType = Self.shouldUseSynthesizedFirstResponderType(

‎docs/adr/0004-ios-snapshot-backend-strategy.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -660,8 +660,9 @@ Translation is the one rule that reads an origin, so it is the only place the fi
660660
Capture is half of the promise; dispatch is the other half.
661661
The measured dispatch spaces for a placed window are both screen-anchored: the synthesized path lands a point at its screen position, and XCTest's own `XCUICoordinate` on the app anchors at the screen's origin too.
662662
`interactionCoordinate` therefore passes the caller's screen point through, and `resolveRunnerWindow` keeps naming the window a foldable's tap resolves its display on (ADR 0025).
663-
The live `XCUIElement.frame` reads that the tap path already compares — the selector match, the input probe, the tree's sweep candidates — report screen space on a placed window, as the ref-tap path proved on the device.
664-
Only the snapshot channel arrives window-relative, and only that channel converts.
663+
The live `XCUIElement.frame` property reports screen space on a placed window, as the selector-tap path proved on the device: it measured the `Add` button at (641,321) for a window at (324,307).
664+
The one live read that does not is `XCUIElementSnapshot.frame`, which reports the window's content space like the capture channel does.
665+
So the snapshot capture converts in `SnapshotGeometrySpace.normalized`, and the text-input probe converts its snapshot reads through the same `SnapshotContentTranslation` rule read from the two live boxes (`app.frame` against the resolved window's frame); every other live read needs no conversion.
665666

666667
## Amendment: modal containment in the presentation cut
667668

0 commit comments

Comments
 (0)