Skip to content

Commit 009b65c

Browse files
committed
refactor(ios): make ResolvedTextEntryInput own the probe-side screen frame (#3899)
The readiness retap re-read the input as a bare XCUIElementSnapshot and tapped its raw center through tapAt, which takes screen points, so on a windowed iPad the retap landed outside the window. The live window lookup also ran whole-app queries four to six times per fill on every iOS device. Every snapshot frame that becomes a dispatched point now passes through the one content translation, held on ResolvedTextEntryInput.frameOnScreen. The vector is read once per command and stored on TextEntryTarget.contentTranslation; the probe matching, focus tap, refresh point, caret tap and readiness retap all read that one property. The threaded translation: parameters and the per-read lookups are gone, and the once-per-command keyboard read in the typing command is reverted to the base behavior with the retap fixed.
1 parent 0e4753f commit 009b65c

19 files changed

Lines changed: 364 additions & 169 deletions

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,14 @@ extension RunnerTests {
155155
)
156156
let textInput: XCUIElement?
157157
if !xCTestTextInputProbeSkipped {
158-
textInput = coordinateTapTextInputAt(app: activeApp, x: x, y: y)
158+
// The lookup compares the point, which is a screen point, with candidate frames; on a
159+
// placed window the snapshot frames need the content vector (#3899).
160+
textInput = coordinateTapTextInputAt(
161+
app: activeApp,
162+
x: x,
163+
y: y,
164+
contentTranslation: liveContentTranslation(app: activeApp)
165+
)
159166
} else {
160167
textInput = nil
161168
// Clear before dispatch so a throwing gesture cannot retain the preceding field's witness.

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -352,7 +352,13 @@ extension RunnerTests {
352352

353353
func readTextAt(app: XCUIApplication, x: Double, y: Double) -> String? {
354354
let point = CGPoint(x: x, y: y)
355-
let textInputCandidates = textInputCandidatesAt(app: app, point: point).map(\.element)
355+
// The point is a screen point; candidate snapshot frames of a placed window need the content
356+
// vector to be compared with it (#3899).
357+
let textInputCandidates = textInputCandidatesAt(
358+
app: app,
359+
point: point,
360+
contentTranslation: liveContentTranslation(app: app)
361+
).map(\.element)
356362
for element in textInputCandidates where prefersExpandedTextRead(element) {
357363
if let text = readableText(for: element) {
358364
return text
@@ -617,12 +623,15 @@ extension RunnerTests {
617623
#if !os(tvOS)
618624
private func interactionCoordinate(app: XCUIApplication, x: Double, y: Double) -> XCUICoordinate {
619625
#if os(iOS)
620-
// Coordinate taps, double taps, long presses, and drags anchor at the app origin, not the
621-
// resolved window: on iOS the snapshot hands back app-space points, and the first qualifying
622-
// window is not always the one under the finger — a SpringBoard `alert accept` can live in an
623-
// alert window while the wallpaper or status-bar window qualifies first, so a window-relative
624-
// anchor misses the button. Reference frames and the synthesized display ID still resolve
625-
// through `resolveRunnerWindow`, so a foldable tap keeps its panel and stays on its display.
626+
// A caller's point is a screen point: coordinate taps, double taps, long presses, and drags
627+
// anchor at the application's origin, not the resolved window, so the point is dispatched as
628+
// given. That holds because `app.coordinate(withNormalizedOffset: .zero)` sits at the origin of
629+
// the space `app.frame` names, and for a windowed app the measured `app.frame` starts at the
630+
// screen's origin. The first qualifying window is not always the one under the finger — a
631+
// SpringBoard `alert accept` can live in an alert window while the wallpaper or status-bar
632+
// window qualifies first, so a window-relative anchor misses the button. Reference frames and
633+
// the synthesized display ID still resolve through `resolveRunnerWindow`, so a foldable tap
634+
// keeps its panel and stays on its display (#3899, ADR 0025).
626635
let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
627636
return origin.withOffset(CGVector(dx: x, dy: y))
628637
#else

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift‎

Lines changed: 89 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,11 @@ extension RunnerTests {
149149
let element: XCUIElement?
150150
let refreshPoint: CGPoint?
151151
let prefersFocusedElement: Bool
152+
/// The live snapshot-frame vector for the reading session, read once when this target was built
153+
/// (#3899). Every snapshot frame the target's reads produce is compared with screen points, so
154+
/// every such read carries this vector (`ResolvedTextEntryInput.frameOnScreen`). Nil on a device
155+
/// whose app fills the screen, where a snapshot frame is already a screen frame.
156+
let contentTranslation: CGSize?
152157
let fromTapWitness: Bool
153158
/// The input the first resolved element was. Once bound, resolution refuses any other input.
154159
let boundIdentity: TextEntryInputIdentity?
@@ -166,6 +171,7 @@ extension RunnerTests {
166171
element: XCUIElement?,
167172
refreshPoint: CGPoint?,
168173
prefersFocusedElement: Bool,
174+
contentTranslation: CGSize? = nil,
169175
fromTapWitness: Bool = false,
170176
boundIdentity: TextEntryInputIdentity? = nil,
171177
inputAtRefreshPoint: TextInputAtPoint? = nil,
@@ -174,30 +180,34 @@ extension RunnerTests {
174180
self.element = element
175181
self.refreshPoint = refreshPoint
176182
self.prefersFocusedElement = prefersFocusedElement
183+
self.contentTranslation = contentTranslation
177184
self.fromTapWitness = fromTapWitness
178185
self.boundIdentity = boundIdentity ?? inputAtRefreshPoint?.identity
179186
self.inputAtRefreshPoint = inputAtRefreshPoint
180187
self.postFocusRead = postFocusRead
181188
}
182189

183-
/// The target moved to the input a resolution read: its element, a refresh point at its center,
184-
/// and its identity while the target is unbound.
185-
///
186-
/// `translation` is the live snapshot-frame vector for the reading session (#3899): the snapshot
187-
/// frame of a placed window's input names the window's content space, while a refresh point is
188-
/// compared against screen-space probes, so the center enters the target translated.
189-
func withResolvedInput(
190-
_ input: ResolvedTextEntryInput?,
191-
translation: CGSize?
192-
) -> TextEntryTarget {
190+
/// The target moved to the input a resolution read: its element and identity, and — when the
191+
/// read carries a snapshot — a refresh point at that frame's center on the screen (`frameOnScreen`
192+
/// plus this target's content vector, #3899). A read with no snapshot keeps the old refresh
193+
/// point: its center would come from the live `element.frame`, the read a post-focus handle
194+
/// cannot answer without recording a failure (#3060).
195+
func withResolvedInput(_ input: ResolvedTextEntryInput?) -> TextEntryTarget {
193196
guard let input else {
194197
return self
195198
}
196-
let frame = snapshotFrameOnScreen(input.snapshot?.frame ?? .zero, translation: translation)
199+
let nextRefreshPoint: CGPoint?
200+
if let snapshot = input.snapshot {
201+
let frame = input.frameOnScreen
202+
nextRefreshPoint = frame.isEmpty ? refreshPoint : CGPoint(x: frame.midX, y: frame.midY)
203+
} else {
204+
nextRefreshPoint = refreshPoint
205+
}
197206
return TextEntryTarget(
198207
element: input.element,
199-
refreshPoint: frame.isEmpty ? refreshPoint : CGPoint(x: frame.midX, y: frame.midY),
208+
refreshPoint: nextRefreshPoint,
200209
prefersFocusedElement: prefersFocusedElement,
210+
contentTranslation: contentTranslation,
201211
fromTapWitness: fromTapWitness,
202212
boundIdentity: boundIdentity ?? input.identity,
203213
inputAtRefreshPoint: inputAtRefreshPoint
@@ -211,20 +221,31 @@ extension RunnerTests {
211221
/// Pro, iOS 27.0). Where a live read of an element that stopped answering its query, such as a
212222
/// removed input or a focused Flutter password field (#3060), records an XCTest failure that ends
213223
/// the runner session, the snapshot only throws.
224+
///
225+
/// This type owns the probe-side space of #3899 (ADR 0004). `XCUIElementSnapshot.frame` reports a
226+
/// placed window's content in the window's own space, while every point this read feeds — a refresh
227+
/// point, a focus tap, a probe match, a readiness retap — is a screen point. So the read carries
228+
/// the content vector it was taken under, and `frameOnScreen` is the only frame consumers use: a
229+
/// snapshot frame plus the vector, or the live frame when there is no snapshot. Raw frames
230+
/// (`snapshot.frame`) are compared only with each other.
214231
struct ResolvedTextEntryInput {
215232
let element: XCUIElement
216233
/// Nil when the element exists but could not be snapshotted.
217234
let snapshot: XCUIElementSnapshot?
235+
/// The live snapshot-frame vector read once for the command that made this read (#3899). Nil
236+
/// when the app fills the screen, where a snapshot frame is already a screen frame.
237+
let contentTranslation: CGSize?
218238

219-
private init(element: XCUIElement, snapshot: XCUIElementSnapshot?) {
239+
private init(element: XCUIElement, snapshot: XCUIElementSnapshot?, contentTranslation: CGSize?) {
220240
self.element = element
221241
self.snapshot = snapshot
242+
self.contentTranslation = contentTranslation
222243
}
223244

224245
/// Reads a handle that may name an input the app has removed; nil when it names none or the read
225246
/// raised. The snapshot of a missing element throws only after XCTest waits about two seconds for
226247
/// a match, so existence is read first.
227-
init?(reading element: XCUIElement) {
248+
init?(reading element: XCUIElement, contentTranslation: CGSize? = nil) {
228249
var exists = false
229250
var snapshot: XCUIElementSnapshot?
230251
if let exception = RunnerObjCExceptionCatcher.catchException({
@@ -235,15 +256,15 @@ extension RunnerTests {
235256
return nil
236257
}
237258
guard exists else { return nil }
238-
self.init(element: element, snapshot: snapshot)
259+
self.init(element: element, snapshot: snapshot, contentTranslation: contentTranslation)
239260
}
240261

241262
/// Reads a handle its query has just listed, inside the caller's exception containment, so the
242263
/// snapshot is read first and existence only when the snapshot fails; nil when it names no element.
243-
init?(readingListed element: XCUIElement) {
264+
init?(readingListed element: XCUIElement, contentTranslation: CGSize? = nil) {
244265
let snapshot = try? element.snapshot()
245266
guard snapshot != nil || element.exists else { return nil }
246-
self.init(element: element, snapshot: snapshot)
267+
self.init(element: element, snapshot: snapshot, contentTranslation: contentTranslation)
247268
}
248269

249270
var identity: TextEntryInputIdentity? {
@@ -256,25 +277,37 @@ extension RunnerTests {
256277
}
257278
return element
258279
}
280+
281+
/// The input's box in screen space: the snapshot frame carried by `contentTranslation` to its
282+
/// place on the screen, or the live frame when there is no snapshot — the live
283+
/// `XCUIElement.frame` already reports a placed window's content in screen space (#3899).
284+
var frameOnScreen: CGRect {
285+
guard let snapshot else {
286+
return element.frame
287+
}
288+
guard let contentTranslation else {
289+
return snapshot.frame
290+
}
291+
return snapshot.frame.offsetBy(dx: contentTranslation.width, dy: contentTranslation.height)
292+
}
259293
}
260294

261295
/// A text entry target and its input as last read. `input()` hands the last read out again until
262296
/// `dispatch` runs, and resolves the target again after that, so a bound target refuses an input
263297
/// that took its place. Each resolution moves the target to the input it read. Every post and every
264298
/// pause between reads goes through `dispatch`; the pause a typing plan takes itself follows a post.
299+
/// The reader hands the target's content vector to every read it resolves, so one vector rules the
300+
/// command (#3899).
265301
struct TextEntryReader {
266302
private(set) var target: TextEntryTarget
267303
private var lastRead: ResolvedTextEntryInput?
268304
private let resolve: (TextEntryTarget) -> ResolvedTextEntryInput?
269-
private let translation: CGSize?
270305

271306
init(
272307
target: TextEntryTarget,
273-
translation: CGSize?,
274308
resolve: @escaping (TextEntryTarget) -> ResolvedTextEntryInput?
275309
) {
276-
self.translation = translation
277-
self.target = target.withResolvedInput(target.postFocusRead, translation: translation)
310+
self.target = target.withResolvedInput(target.postFocusRead)
278311
lastRead = target.postFocusRead
279312
self.resolve = resolve
280313
}
@@ -284,7 +317,7 @@ extension RunnerTests {
284317
return lastRead
285318
}
286319
let read = resolve(target)
287-
target = target.withResolvedInput(read, translation: translation)
320+
target = target.withResolvedInput(read)
288321
lastRead = read
289322
return read
290323
}
@@ -507,7 +540,10 @@ extension RunnerTests {
507540
return resolveUnboundTextEntryInput(app: app, target: target)
508541
}
509542
func boundInput(_ candidate: XCUIElement) -> ResolvedTextEntryInput? {
510-
guard let input = ResolvedTextEntryInput(reading: candidate), input.identity == boundIdentity else { return nil }
543+
guard let input = ResolvedTextEntryInput(
544+
reading: candidate,
545+
contentTranslation: target.contentTranslation
546+
), input.identity == boundIdentity else { return nil }
511547
return input
512548
}
513549
if target.prefersFocusedElement, let focused = focusedTextInput(app: app), let input = boundInput(focused) {
@@ -516,12 +552,17 @@ extension RunnerTests {
516552
if let element = target.element, let input = boundInput(element) {
517553
return input
518554
}
519-
if let input = target.inputAtRefreshPoint, let read = textInputIfStillResolves(input) {
555+
if let input = target.inputAtRefreshPoint,
556+
let read = textInputIfStillResolves(input, contentTranslation: target.contentTranslation) {
520557
return read
521558
}
522559
if target.inputAtRefreshPoint == nil,
523560
let refreshPoint = target.refreshPoint,
524-
case .matches(let candidates) = probeTextInputs(app: app, point: refreshPoint),
561+
case .matches(let candidates) = probeTextInputs(
562+
app: app,
563+
point: refreshPoint,
564+
contentTranslation: target.contentTranslation
565+
),
525566
let match = candidates.first(where: { $0.identity == boundIdentity }) {
526567
return match
527568
}
@@ -556,7 +597,11 @@ extension RunnerTests {
556597
guard let refreshPoint = target.refreshPoint else {
557598
return true
558599
}
559-
switch probeTextInputs(app: app, point: refreshPoint) {
600+
switch probeTextInputs(
601+
app: app,
602+
point: refreshPoint,
603+
contentTranslation: target.contentTranslation
604+
) {
560605
case .absent:
561606
return true
562607
case .matches(let candidates):
@@ -590,17 +635,25 @@ extension RunnerTests {
590635

591636
/// Resolution for a target not bound to an input yet: the first element that exists.
592637
private func resolveUnboundTextEntryInput(app: XCUIApplication, target: TextEntryTarget) -> ResolvedTextEntryInput? {
593-
if target.prefersFocusedElement, let focused = focusedTextInput(app: app).flatMap(ResolvedTextEntryInput.init(reading:)) {
638+
let translation = target.contentTranslation
639+
if target.prefersFocusedElement,
640+
let focused = focusedTextInput(app: app).flatMap({ ResolvedTextEntryInput(reading: $0, contentTranslation: translation) }) {
594641
return focused
595642
}
596-
if let element = target.element, let input = ResolvedTextEntryInput(reading: element) {
643+
if let element = target.element,
644+
let input = ResolvedTextEntryInput(reading: element, contentTranslation: translation) {
597645
return input
598646
}
599647
if let refreshPoint = target.refreshPoint,
600-
let refreshed = textInputAt(app: app, x: refreshPoint.x, y: refreshPoint.y) {
648+
let refreshed = textInputAt(
649+
app: app,
650+
x: refreshPoint.x,
651+
y: refreshPoint.y,
652+
contentTranslation: translation
653+
) {
601654
return refreshed
602655
}
603-
return focusedTextInput(app: app).flatMap(ResolvedTextEntryInput.init(reading:))
656+
return focusedTextInput(app: app).flatMap { ResolvedTextEntryInput(reading: $0, contentTranslation: translation) }
604657
}
605658

606659
private func moveCaretToEnd(app: XCUIApplication, input: ResolvedTextEntryInput) {
@@ -619,12 +672,14 @@ extension RunnerTests {
619672
// zero-size one (how a departed handle reads), there is no point to dispatch — an edge-tap
620673
// from a frame the field moved after records a failure or navigates away, the shape
621674
// `clearTextInput`'s comment refuses — so degrade to the caller's point-free delete burst.
622-
// The snapshot frame names the window's own content space on a placed window, so it enters
623-
// `tapAt` translated (#3899).
624-
guard let read = input.snapshot?.frame, !read.isEmpty else {
675+
// The read's snapshot frame names the window's own content space on a placed window, so the
676+
// point enters `tapAt` through `frameOnScreen`, which carries the read's content vector (#3899).
677+
// A read with no snapshot gets no point at all: `frameOnScreen`'s fallback is the live
678+
// `element.frame` read this function exists to avoid.
679+
guard input.snapshot?.frame.isEmpty == false else {
625680
return
626681
}
627-
let frame = snapshotFrameOnScreen(read, translation: liveContentTranslation(app: app))
682+
let frame = input.frameOnScreen
628683
_ = tapAt(
629684
app: app,
630685
x: frame.minX + max(2, frame.width - 4),

0 commit comments

Comments
 (0)