Skip to content

Commit 6711463

Browse files
thymikeeclaude
andcommitted
fix(ios): keep each fill read tied to the input it named
Two reuses from the one-snapshot change could pair a read with the wrong input: - The point lookup counted per-type positions across candidates read from a shared query that each re-ran it. An input that left the app between two candidate reads shifted the later positions, so a handle could name another input than the snapshot it carried. Each handed-out handle now reads as the candidate's identity and frame; on a mismatch the lookup runs the per-type queries, whose handles read through their own snapshots. - The warmup kept its read across the pause it takes for an unreadable value or an expired deadline, so the next post skipped the identity check and could type into a successor. Only a read that no pause follows is reused now. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 8894c98 commit 6711463

6 files changed

Lines changed: 217 additions & 23 deletions

File tree

‎apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m‎

Lines changed: 69 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,8 @@ @interface AgentDeviceRunnerViewController : UIViewController
109109
@property(nonatomic, assign) NSTimeInterval textEntryBurstMinGap;
110110
@property(nonatomic, assign) NSTimeInterval textEntryAcknowledgeWindowSeconds;
111111
@property(nonatomic, strong) NSLayoutConstraint *textEntryFieldTop;
112+
@property(nonatomic, weak) UITextField *textEntryField;
113+
@property(nonatomic, weak) UITextField *mixedTextInputField;
112114
@property(nonatomic, assign) BOOL alertFixtureStarted;
113115
@property(nonatomic, strong) NSTimer *alertActivationBusyBackstop;
114116
@property(nonatomic, strong) NSTimer *alertBannerRepost;
@@ -545,20 +547,38 @@ - (void)agentDeviceTextEntryDidChange:(UITextField *)textField {
545547
}
546548
if ((codeComplete && [arguments containsObject:@"--agent-device-text-entry-auto-submit"]) ||
547549
(textField.text.length > 0 && [arguments containsObject:@"--agent-device-text-entry-replace-after-input"])) {
548-
UITextField *nextScreenField = [[UITextField alloc] initWithFrame:textField.frame];
549-
if (![arguments containsObject:@"--agent-device-text-entry-unnamed-input"]) {
550-
nextScreenField.accessibilityIdentifier = @"agent-device-auto-submit-next-screen-input";
551-
}
552-
nextScreenField.borderStyle = UITextBorderStyleRoundedRect;
553-
[textField.superview addSubview:nextScreenField];
554-
[textField removeFromSuperview];
555-
[nextScreenField becomeFirstResponder];
550+
[self replaceTextEntryFieldWithNextScreenInput:textField];
556551
} else if (codeComplete &&
557552
[arguments containsObject:@"--agent-device-text-entry-auto-submit-without-successor"]) {
558553
[textField removeFromSuperview];
559554
}
560555
}
561556

557+
- (void)replaceTextEntryFieldWithNextScreenInput:(UITextField *)textField {
558+
UITextField *nextScreenField = [[UITextField alloc] initWithFrame:textField.frame];
559+
if (![NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-unnamed-input"]) {
560+
nextScreenField.accessibilityIdentifier = @"agent-device-auto-submit-next-screen-input";
561+
}
562+
nextScreenField.borderStyle = UITextBorderStyleRoundedRect;
563+
[textField.superview addSubview:nextScreenField];
564+
[textField removeFromSuperview];
565+
[nextScreenField becomeFirstResponder];
566+
}
567+
568+
// The same navigation as `replace-after-input`, landing when a test presses the button instead of on
569+
// an edit, so the test decides which runner step it lands between.
570+
- (void)agentDeviceTextEntryReplaceDidPress:(UIButton *)sender {
571+
(void)sender;
572+
if (self.textEntryField.superview != nil) {
573+
[self replaceTextEntryFieldWithNextScreenInput:self.textEntryField];
574+
}
575+
}
576+
577+
- (void)agentDeviceMixedTextInputRemoveFieldDidPress:(UIButton *)sender {
578+
(void)sender;
579+
[self.mixedTextInputField removeFromSuperview];
580+
}
581+
562582
// Stops answering the element-type query that found it, the moment it takes focus, under
563583
// `--agent-device-text-entry-unqueryable-on-focus`. #3060's Flutter password field reaches this shape
564584
// because its two accessibility channels disagree once focused — `TextField` through the legacy
@@ -689,6 +709,31 @@ - (void)viewDidLoad {
689709
}
690710
}
691711

712+
// A text field ahead of two search fields among all text inputs. Removing the text field moves the
713+
// second search field into the first one's position among all text inputs, but not among search
714+
// fields. A search bar's field reads as a search field; a bare `UISearchTextField` reads as a text
715+
// field.
716+
if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-mixed-text-inputs"]) {
717+
UITextField *field = [[UITextField alloc] initWithFrame:CGRectMake(40, 120, 260, 44)];
718+
field.accessibilityIdentifier = @"mixed-text";
719+
field.borderStyle = UITextBorderStyleRoundedRect;
720+
[self.view addSubview:field];
721+
self.mixedTextInputField = field;
722+
for (NSUInteger index = 0; index < 2; index++) {
723+
UISearchBar *search = [[UISearchBar alloc] initWithFrame:CGRectMake(40, 180 + index * 60, 260, 44)];
724+
search.searchTextField.accessibilityIdentifier = @[@"mixed-search-a", @"mixed-search-b"][index];
725+
[self.view addSubview:search];
726+
}
727+
UIButton *removeField = [UIButton buttonWithType:UIButtonTypeSystem];
728+
removeField.frame = CGRectMake(40, 320, 260, 44);
729+
removeField.accessibilityIdentifier = @"mixed-remove-text";
730+
[removeField setTitle:@"Remove text field" forState:UIControlStateNormal];
731+
[removeField addTarget:self
732+
action:@selector(agentDeviceMixedTextInputRemoveFieldDidPress:)
733+
forControlEvents:UIControlEventTouchUpInside];
734+
[self.view addSubview:removeField];
735+
}
736+
692737
if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-regression"]) {
693738
BOOL digitCountValue =
694739
[NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-digit-count-value"];
@@ -738,6 +783,7 @@ - (void)viewDidLoad {
738783
forControlEvents:UIControlEventEditingChanged];
739784
textField.translatesAutoresizingMaskIntoConstraints = NO;
740785
[self.view addSubview:textField];
786+
self.textEntryField = textField;
741787
self.textEntryFieldTop = [textField.topAnchor constraintEqualToAnchor:label.bottomAnchor
742788
constant:AgentDeviceTextEntryFieldTopInset];
743789
[NSLayoutConstraint activateConstraints:@[
@@ -800,6 +846,21 @@ - (void)viewDidLoad {
800846
]];
801847
[self updateTapWitnessStatus];
802848
}
849+
if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-replace-on-request"]) {
850+
// Above the title, so the software keyboard never covers it.
851+
UIButton *replace = [UIButton buttonWithType:UIButtonTypeSystem];
852+
replace.accessibilityIdentifier = @"agent-device-text-entry-replace";
853+
[replace setTitle:@"Replace input" forState:UIControlStateNormal];
854+
replace.translatesAutoresizingMaskIntoConstraints = NO;
855+
[replace addTarget:self
856+
action:@selector(agentDeviceTextEntryReplaceDidPress:)
857+
forControlEvents:UIControlEventTouchUpInside];
858+
[self.view addSubview:replace];
859+
[NSLayoutConstraint activateConstraints:@[
860+
[replace.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor],
861+
[replace.bottomAnchor constraintEqualToAnchor:label.topAnchor constant:-24],
862+
]];
863+
}
803864
if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-unqueryable-on-focus"]) {
804865
self.textEntryFocusWitness = [[UILabel alloc] init];
805866
self.textEntryFocusWitness.accessibilityIdentifier = @"agent-device-text-entry-focus";

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift‎

Lines changed: 55 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -186,20 +186,64 @@ extension RunnerTests {
186186
// Each handle binds by index within its own element type, so an input of another type that
187187
// appears ahead of it later does not take its place.
188188
var indexByType: [XCUIElement.ElementType: Int] = [:]
189-
return candidates.compactMap { element -> ResolvedTextEntryInput? in
190-
guard !shouldStop(), element.exists else { return nil }
189+
var inputs: [ResolvedTextEntryInput] = []
190+
for (position, element) in candidates.enumerated() {
191+
if shouldStop() { break }
192+
guard element.exists else { continue }
191193
let snapshot = try? element.snapshot()
192194
let elementType = snapshot?.elementType ?? element.elementType
193195
let typeIndex = indexByType[elementType, default: 0]
194196
indexByType[elementType] = typeIndex + 1
195-
let input = ResolvedTextEntryInput(
196-
element: app.descendants(matching: elementType).element(boundBy: typeIndex),
197-
snapshot: snapshot
198-
)
199-
let attributes = input.attributes
200-
return isCoordinateTextInputCandidate(enabled: attributes.isEnabled, frame: attributes.frame, point: point)
201-
? input
202-
: nil
203-
}.sorted { smallestElementFirst($0.attributes, $1.attributes) }
197+
#if AGENT_DEVICE_RUNNER_UNIT_TESTS
198+
textInputCandidateReadForTesting?(position)
199+
#endif
200+
let handle = app.descendants(matching: elementType).element(boundBy: typeIndex)
201+
let candidate = ResolvedTextEntryInput(element: handle, snapshot: snapshot)
202+
let attributes = candidate.attributes
203+
guard isCoordinateTextInputCandidate(enabled: attributes.isEnabled, frame: attributes.frame, point: point) else {
204+
continue
205+
}
206+
guard let snapshot else {
207+
inputs.append(candidate)
208+
continue
209+
}
210+
// Each candidate re-runs the shared query, so an input that left the app since an earlier
211+
// candidate's read shifts the positions after it, and the per-type count no longer names the
212+
// input this candidate read. The handle must read as that input, or no position can be trusted.
213+
guard let handleSnapshot = try? handle.snapshot(), Self.isSameTextInput(handleSnapshot, as: snapshot) else {
214+
return queryTextInputsByType(app: app, point: point, shouldStop: shouldStop)
215+
}
216+
inputs.append(ResolvedTextEntryInput(element: handle, snapshot: handleSnapshot))
217+
}
218+
return inputs.sorted { smallestElementFirst($0.attributes, $1.attributes) }
219+
}
220+
221+
/// `queryTextInputs` with one query per element type, whose handles each read through their own
222+
/// snapshot, for a lookup whose shared query changed while it read the candidates.
223+
private func queryTextInputsByType(
224+
app: XCUIApplication,
225+
point: CGPoint,
226+
shouldStop: () -> Bool
227+
) -> [ResolvedTextEntryInput] {
228+
NSLog("AGENT_DEVICE_RUNNER_TEXT_INPUT_LOOKUP_BY_TYPE reason=inputs-changed-during-lookup")
229+
var inputs: [ResolvedTextEntryInput] = []
230+
for query in [app.textFields, app.secureTextFields, app.searchFields, app.textViews] {
231+
for element in query.allElementsBoundByIndex {
232+
guard !shouldStop() else { return [] }
233+
guard element.exists else { continue }
234+
let input = ResolvedTextEntryInput(element: element, snapshot: try? element.snapshot())
235+
let attributes = input.attributes
236+
if isCoordinateTextInputCandidate(enabled: attributes.isEnabled, frame: attributes.frame, point: point) {
237+
inputs.append(input)
238+
}
239+
}
240+
}
241+
return inputs.sorted { smallestElementFirst($0.attributes, $1.attributes) }
242+
}
243+
244+
private static func isSameTextInput(_ read: XCUIElementSnapshot, as candidate: XCUIElementSnapshot) -> Bool {
245+
read.elementType == candidate.elementType
246+
&& read.identifier == candidate.identifier
247+
&& read.frame == candidate.frame
204248
}
205249
}

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,13 @@ extension RunnerTests {
247247
)
248248
}
249249

250+
func pauseAfterWarmupRead() {
251+
#if AGENT_DEVICE_RUNNER_UNIT_TESTS
252+
textEntryWarmupPauseForTesting?()
253+
#endif
254+
sleepFor(TextEntryTiming.pollInterval)
255+
}
256+
250257
func readBackWarmupCharacter(_ peeledCharacter: String) {
251258
// A route with an element reads the peeled character back and waits up to `warmupValueTimeout`
252259
// for the app to accept it. The route that had to be budgeted has no element, so its expected
@@ -265,13 +272,18 @@ extension RunnerTests {
265272
// Each read refreshes the target from the field as the character left it.
266273
let input = resolveTextEntryInput(app: app, target: activeTarget)
267274
activeTarget = activeTarget.withResolvedInput(input)
268-
undispatchedRead = input
269275
let observed = textValueRead(for: input?.attributes)
270-
if !observed.warmupReadIsPending(expectedText: warmupExpectedText) {
271-
if observed == .unreadable { sleepFor(TextEntryTiming.pollInterval) }
276+
guard observed.warmupReadIsPending(expectedText: warmupExpectedText) else {
277+
if observed == .unreadable {
278+
pauseAfterWarmupRead()
279+
} else {
280+
// The next post follows this read directly, so it can type into it. After a pause the app
281+
// may have replaced the input, so the next post resolves the target again.
282+
undispatchedRead = input
283+
}
272284
return
273285
}
274-
sleepFor(TextEntryTiming.pollInterval)
286+
pauseAfterWarmupRead()
275287
}
276288
}
277289

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,12 @@ final class RunnerTests: XCTestCase {
225225
// Runs on the waiting thread after `runMainThreadWork`'s wait timed out and before it takes the
226226
// lock that decides between finished and abandoned, so a test can finish the work in that window.
227227
var mainThreadWorkTimedOutForTesting: (@Sendable () -> Void)?
228+
// Runs after the point lookup reads the candidate at a position of its shared query, so a test can
229+
// change the app's inputs between two candidate reads.
230+
var textInputCandidateReadForTesting: ((Int) -> Void)?
231+
// Runs before a fill pauses after reading its warmup character back, so a test can change the app's
232+
// inputs in that pause.
233+
var textEntryWarmupPauseForTesting: (() -> Void)?
228234
#endif
229235
// Observability for the record(_:) suppression below: how many AX-broken-screen snapshot
230236
// issues this session muted, so wedge investigations see the volume without grepping logs.

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,41 @@ extension RunnerTests {
8585
XCTAssertFalse(didRecordXCTestFailure(since: failures))
8686
}
8787

88+
// The lookup reads its candidates through one query for every text input type. The text field
89+
// ahead of both search fields in that query leaves the app after the lookup read it, which moves
90+
// the second search field into the first one's position. The input the lookup hands out for the
91+
// point must still be the second search field, whose snapshot it read, and not the first.
92+
@MainActor
93+
func testTextInputLookupKeepsItsInputWhenAnEarlierInputOfAnotherTypeLeaves() throws {
94+
app.launchArguments = ["--agent-device-mixed-text-inputs"]
95+
app.launch()
96+
defer {
97+
textInputCandidateReadForTesting = nil
98+
app.terminate()
99+
}
100+
let field = app.textFields["mixed-text"]
101+
let second = app.searchFields["mixed-search-b"]
102+
XCTAssertTrue(second.waitForFixtureExistence(timeout: appExistenceTimeout))
103+
XCTAssertTrue(field.exists)
104+
let frame = second.frame
105+
let removeField = app.buttons["mixed-remove-text"]
106+
var removals = 0
107+
textInputCandidateReadForTesting = { position in
108+
guard position == 0, removals == 0 else { return }
109+
removals += 1
110+
removeField.tap()
111+
}
112+
113+
let input = try XCTUnwrap(textInputAt(app: app, x: frame.midX, y: frame.midY))
114+
textInputCandidateReadForTesting = nil
115+
116+
XCTAssertEqual(removals, 1)
117+
XCTAssertFalse(field.exists)
118+
XCTAssertEqual(input.snapshot?.identifier, "mixed-search-b")
119+
XCTAssertEqual(input.element.identifier, "mixed-search-b")
120+
XCTAssertEqual(input.element.frame, frame)
121+
}
122+
88123
func testTextInputProbeContainmentExcludesRequiredReadsAndLaterIssues() {
89124
app.launchArguments = ["--agent-device-text-entry-regression"]
90125
app.launch()

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -276,6 +276,42 @@ extension RunnerTests {
276276
XCTAssertEqual(editableTextValue(for: nextScreenField, treatingPlaceholderAsEmpty: true), "")
277277
}
278278

279+
// The navigation lands while the fill pauses after reading its first masked character back, which
280+
// it does because a masked value cannot show whether the character arrived. The read before the
281+
// pause named the code field, so the next post must resolve the field again and refuse the
282+
// successor rather than type the rest of the code into it.
283+
@MainActor
284+
func testFillFailsWhenAppReplacesMaskedInputWhileWarmupPauses() throws {
285+
let textField = try launchRemovableInputFixture(
286+
"--agent-device-text-entry-masked-value",
287+
"--agent-device-text-entry-replace-on-request"
288+
)
289+
let replace = app.buttons["agent-device-text-entry-replace"]
290+
XCTAssertTrue(replace.exists)
291+
var replacements = 0
292+
textEntryWarmupPauseForTesting = {
293+
guard replacements == 0 else { return }
294+
replacements += 1
295+
replace.tap()
296+
}
297+
defer { textEntryWarmupPauseForTesting = nil }
298+
299+
let failureCountBefore = currentXCTestFailureCount()
300+
let response = executeTypeCommand(
301+
activeApp: app,
302+
command: try fillCommandFixture(commandId: "fill-masked-replaced-in-pause", text: "123456", at: textField)
303+
)
304+
305+
XCTAssertEqual(replacements, 1)
306+
XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore))
307+
XCTAssertFalse(response.ok)
308+
XCTAssertEqual(response.error?.code, "TEXT_INPUT_NOT_FOCUSED")
309+
XCTAssertFalse(textField.exists)
310+
let nextScreenField = app.textFields["agent-device-auto-submit-next-screen-input"]
311+
XCTAssertTrue(nextScreenField.exists)
312+
XCTAssertEqual(editableTextValue(for: nextScreenField, treatingPlaceholderAsEmpty: true), "")
313+
}
314+
279315
// Neither input has an identifier, so the successor carries the same identity as the code field.
280316
// The runner cannot tell them apart and must not clear and retype the successor: it reports the
281317
// mismatch it read instead of repairing.

0 commit comments

Comments
 (0)