Skip to content

Commit bef8706

Browse files
thymikeeclaude
andcommitted
fix(ios): fall back when any point-lookup position moved
The point lookup validated a handle only for a candidate at the point, after the point filter. An input of another type that left the app after an earlier position's read moved a later input into a position already read, so the input at the point was never read: the next position read an input elsewhere, which the filter dropped, and the last position no longer existed and was skipped. The lookup returned no input without falling back, so a resolved-coordinate fill failed with NO_TEXT_INPUT_AT_POINT and a coordinate tap could lose its typing witness. Every position now must still exist and every handle must read as its position's input, at the point or not; otherwise the lookup runs the per-type queries. A position's existence is read only when its snapshot fails, so a lookup reads each candidate twice, as before for the input at the point. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 6711463 commit bef8706

2 files changed

Lines changed: 56 additions & 25 deletions

File tree

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift‎

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -189,31 +189,36 @@ extension RunnerTests {
189189
var inputs: [ResolvedTextEntryInput] = []
190190
for (position, element) in candidates.enumerated() {
191191
if shouldStop() { break }
192-
guard element.exists else { continue }
192+
// Each candidate re-runs the shared query, so an input that left the app since an earlier
193+
// candidate's read moves a later input into a position already read, and the per-type count
194+
// no longer names the input a position reads. Every position must still exist and every
195+
// handle must read as its position's input, at the point or not, or the lookup can miss an
196+
// input. A missing position's snapshot throws only after XCTest waits for a match, which only
197+
// a lookup that falls back pays, so existence is read only when the snapshot fails.
193198
let snapshot = try? element.snapshot()
199+
guard snapshot != nil || element.exists else {
200+
return queryTextInputsByType(app: app, point: point, shouldStop: shouldStop)
201+
}
194202
let elementType = snapshot?.elementType ?? element.elementType
195203
let typeIndex = indexByType[elementType, default: 0]
196204
indexByType[elementType] = typeIndex + 1
205+
let handle = app.descendants(matching: elementType).element(boundBy: typeIndex)
206+
let candidate: ResolvedTextEntryInput
207+
if let snapshot {
208+
guard let handleSnapshot = try? handle.snapshot(), Self.isSameTextInput(handleSnapshot, as: snapshot) else {
209+
return queryTextInputsByType(app: app, point: point, shouldStop: shouldStop)
210+
}
211+
candidate = ResolvedTextEntryInput(element: handle, snapshot: handleSnapshot)
212+
} else {
213+
candidate = ResolvedTextEntryInput(element: handle, snapshot: nil)
214+
}
197215
#if AGENT_DEVICE_RUNNER_UNIT_TESTS
198216
textInputCandidateReadForTesting?(position)
199217
#endif
200-
let handle = app.descendants(matching: elementType).element(boundBy: typeIndex)
201-
let candidate = ResolvedTextEntryInput(element: handle, snapshot: snapshot)
202218
let attributes = candidate.attributes
203-
guard isCoordinateTextInputCandidate(enabled: attributes.isEnabled, frame: attributes.frame, point: point) else {
204-
continue
205-
}
206-
guard let snapshot else {
219+
if isCoordinateTextInputCandidate(enabled: attributes.isEnabled, frame: attributes.frame, point: point) {
207220
inputs.append(candidate)
208-
continue
209-
}
210-
// Each candidate re-runs the shared query, so an input that left the app since an earlier
211-
// candidate's read shifts the positions after it, and the per-type count no longer names the
212-
// input this candidate read. The handle must read as that input, or no position can be trusted.
213-
guard let handleSnapshot = try? handle.snapshot(), Self.isSameTextInput(handleSnapshot, as: snapshot) else {
214-
return queryTextInputsByType(app: app, point: point, shouldStop: shouldStop)
215221
}
216-
inputs.append(ResolvedTextEntryInput(element: handle, snapshot: handleSnapshot))
217222
}
218223
return inputs.sorted { smallestElementFirst($0.attributes, $1.attributes) }
219224
}

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift‎

Lines changed: 36 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -85,38 +85,64 @@ extension RunnerTests {
8585
XCTAssertFalse(didRecordXCTestFailure(since: failures))
8686
}
8787

88-
// The lookup reads its candidates through one query for every text input type. The text field
89-
// ahead of both search fields in that query leaves the app after the lookup read it, which moves
90-
// the second search field into the first one's position. The input the lookup hands out for the
91-
// point must still be the second search field, whose snapshot it read, and not the first.
88+
// The lookup reads its candidates through one query for every text input type, in the order text
89+
// field, first search field, second search field. Each case removes the text field after the lookup
90+
// reads one position, which moves each search field into the position before it, one already read.
91+
92+
// The second search field moves into the first one's position, the one the lookup reads next. The
93+
// input the lookup hands out must be the second search field, whose snapshot it read.
9294
@MainActor
9395
func testTextInputLookupKeepsItsInputWhenAnEarlierInputOfAnotherTypeLeaves() throws {
96+
try assertMixedTextInputLookup(finds: "mixed-search-b", removingTextFieldAfterPosition: 0)
97+
}
98+
99+
// The first search field moves into the position already read. The position read next holds the
100+
// second search field, which is not at the point.
101+
@MainActor
102+
func testTextInputLookupFindsTheFirstSearchFieldWhenTheTextFieldLeavesAfterTheFirstRead() throws {
103+
try assertMixedTextInputLookup(finds: "mixed-search-a", removingTextFieldAfterPosition: 0)
104+
}
105+
106+
// The second search field moves into the position already read. The position read next no longer
107+
// exists.
108+
@MainActor
109+
func testTextInputLookupFindsTheSecondSearchFieldWhenTheTextFieldLeavesAfterTheSecondRead() throws {
110+
try assertMixedTextInputLookup(finds: "mixed-search-b", removingTextFieldAfterPosition: 1)
111+
}
112+
113+
@MainActor
114+
private func assertMixedTextInputLookup(
115+
finds identifier: String,
116+
removingTextFieldAfterPosition removalPosition: Int
117+
) throws {
94118
app.launchArguments = ["--agent-device-mixed-text-inputs"]
95119
app.launch()
96120
defer {
97121
textInputCandidateReadForTesting = nil
98122
app.terminate()
99123
}
100124
let field = app.textFields["mixed-text"]
101-
let second = app.searchFields["mixed-search-b"]
102-
XCTAssertTrue(second.waitForFixtureExistence(timeout: appExistenceTimeout))
125+
let target = app.searchFields[identifier]
126+
XCTAssertTrue(target.waitForFixtureExistence(timeout: appExistenceTimeout))
103127
XCTAssertTrue(field.exists)
104-
let frame = second.frame
128+
let frame = target.frame
105129
let removeField = app.buttons["mixed-remove-text"]
106130
var removals = 0
107131
textInputCandidateReadForTesting = { position in
108-
guard position == 0, removals == 0 else { return }
132+
guard position == removalPosition, removals == 0 else { return }
109133
removals += 1
110134
removeField.tap()
111135
}
136+
let failures = currentXCTestFailureCount()
112137

113138
let input = try XCTUnwrap(textInputAt(app: app, x: frame.midX, y: frame.midY))
114139
textInputCandidateReadForTesting = nil
115140

141+
XCTAssertFalse(didRecordXCTestFailure(since: failures))
116142
XCTAssertEqual(removals, 1)
117143
XCTAssertFalse(field.exists)
118-
XCTAssertEqual(input.snapshot?.identifier, "mixed-search-b")
119-
XCTAssertEqual(input.element.identifier, "mixed-search-b")
144+
XCTAssertEqual(input.snapshot?.identifier, identifier)
145+
XCTAssertEqual(input.element.identifier, identifier)
120146
XCTAssertEqual(input.element.frame, frame)
121147
}
122148

0 commit comments

Comments
 (0)