Skip to content

Commit 2d34505

Browse files
thymikeeclaude
andcommitted
refactor(ios): look up text inputs through per-type queries only
The shared elementType IN query saved three queries per point lookup but needed index rebinding, a validation snapshot per candidate, and a fallback for every shift race. Per-type handles never shift across types, so the lookup now reads each type's query and snapshots each candidate once, reading existence only when the snapshot fails. Bluesky and test-app fills measured no slower. An unbound target that resolves through its refresh point now keeps the snapshot the point lookup read instead of reading the input again. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent bef8706 commit 2d34505

6 files changed

Lines changed: 80 additions & 186 deletions

File tree

‎apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m‎

Lines changed: 0 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,6 @@ @interface AgentDeviceRunnerViewController : UIViewController
110110
@property(nonatomic, assign) NSTimeInterval textEntryAcknowledgeWindowSeconds;
111111
@property(nonatomic, strong) NSLayoutConstraint *textEntryFieldTop;
112112
@property(nonatomic, weak) UITextField *textEntryField;
113-
@property(nonatomic, weak) UITextField *mixedTextInputField;
114113
@property(nonatomic, assign) BOOL alertFixtureStarted;
115114
@property(nonatomic, strong) NSTimer *alertActivationBusyBackstop;
116115
@property(nonatomic, strong) NSTimer *alertBannerRepost;
@@ -574,11 +573,6 @@ - (void)agentDeviceTextEntryReplaceDidPress:(UIButton *)sender {
574573
}
575574
}
576575

577-
- (void)agentDeviceMixedTextInputRemoveFieldDidPress:(UIButton *)sender {
578-
(void)sender;
579-
[self.mixedTextInputField removeFromSuperview];
580-
}
581-
582576
// Stops answering the element-type query that found it, the moment it takes focus, under
583577
// `--agent-device-text-entry-unqueryable-on-focus`. #3060's Flutter password field reaches this shape
584578
// because its two accessibility channels disagree once focused — `TextField` through the legacy
@@ -709,31 +703,6 @@ - (void)viewDidLoad {
709703
}
710704
}
711705

712-
// A text field ahead of two search fields among all text inputs. Removing the text field moves the
713-
// second search field into the first one's position among all text inputs, but not among search
714-
// fields. A search bar's field reads as a search field; a bare `UISearchTextField` reads as a text
715-
// field.
716-
if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-mixed-text-inputs"]) {
717-
UITextField *field = [[UITextField alloc] initWithFrame:CGRectMake(40, 120, 260, 44)];
718-
field.accessibilityIdentifier = @"mixed-text";
719-
field.borderStyle = UITextBorderStyleRoundedRect;
720-
[self.view addSubview:field];
721-
self.mixedTextInputField = field;
722-
for (NSUInteger index = 0; index < 2; index++) {
723-
UISearchBar *search = [[UISearchBar alloc] initWithFrame:CGRectMake(40, 180 + index * 60, 260, 44)];
724-
search.searchTextField.accessibilityIdentifier = @[@"mixed-search-a", @"mixed-search-b"][index];
725-
[self.view addSubview:search];
726-
}
727-
UIButton *removeField = [UIButton buttonWithType:UIButtonTypeSystem];
728-
removeField.frame = CGRectMake(40, 320, 260, 44);
729-
removeField.accessibilityIdentifier = @"mixed-remove-text";
730-
[removeField setTitle:@"Remove text field" forState:UIControlStateNormal];
731-
[removeField addTarget:self
732-
action:@selector(agentDeviceMixedTextInputRemoveFieldDidPress:)
733-
forControlEvents:UIControlEventTouchUpInside];
734-
[self.view addSubview:removeField];
735-
}
736-
737706
if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-regression"]) {
738707
BOOL digitCountValue =
739708
[NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-digit-count-value"];

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift‎

Lines changed: 32 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -332,18 +332,23 @@ extension RunnerTests {
332332
/// that took the bound one's place.
333333
func resolveTextEntryElement(app: XCUIApplication, target: TextEntryTarget) -> XCUIElement? {
334334
guard target.boundIdentity != nil else {
335-
return resolveUnboundTextEntryElement(app: app, target: target)
335+
return resolveUnboundTextEntry(app: app, target: target)?.element
336336
}
337337
return resolveTextEntryInput(app: app, target: target)?.element
338338
}
339339

340340
/// `resolveTextEntryElement` with the snapshot the resolution read. A bound target's resolution
341-
/// already snapshots the input to check its identity. An unbound target's resolution takes one
342-
/// snapshot after it finds the input.
341+
/// already snapshots the input to check its identity. An unbound target's resolution keeps the
342+
/// snapshot its point lookup read, and otherwise takes one snapshot after it finds the input.
343343
func resolveTextEntryInput(app: XCUIApplication, target: TextEntryTarget) -> ResolvedTextEntryInput? {
344344
guard let boundIdentity = target.boundIdentity else {
345-
return resolveUnboundTextEntryElement(app: app, target: target).map { element in
346-
ResolvedTextEntryInput(element: element, snapshot: snapshotTextEntryInput(element).snapshot)
345+
switch resolveUnboundTextEntry(app: app, target: target) {
346+
case .lookup(let input)?:
347+
return input
348+
case .element(let element)?:
349+
return ResolvedTextEntryInput(element: element, snapshot: snapshotTextEntryInput(element).snapshot)
350+
case nil:
351+
return nil
347352
}
348353
}
349354
func boundInput(_ candidate: XCUIElement) -> ResolvedTextEntryInput? {
@@ -457,26 +462,42 @@ extension RunnerTests {
457462
return exception == nil ? (probe, snapshot) : (.unavailable, nil)
458463
}
459464

465+
/// What an unbound target's resolution found: a handle it has not read, or the input its point
466+
/// lookup found, with the snapshot the lookup read.
467+
private enum UnboundTextEntryResolution {
468+
case element(XCUIElement)
469+
case lookup(ResolvedTextEntryInput)
470+
471+
var element: XCUIElement {
472+
switch self {
473+
case .element(let element):
474+
return element
475+
case .lookup(let input):
476+
return input.element
477+
}
478+
}
479+
}
480+
460481
/// Resolution for a target not bound to an input yet: the first element that exists.
461-
private func resolveUnboundTextEntryElement(app: XCUIApplication, target: TextEntryTarget) -> XCUIElement? {
482+
private func resolveUnboundTextEntry(app: XCUIApplication, target: TextEntryTarget) -> UnboundTextEntryResolution? {
462483
if target.prefersFocusedElement {
463484
if let focused = focusedTextInput(app: app) {
464-
return focused
485+
return .element(focused)
465486
}
466487
if let element = target.element, element.exists {
467-
return element
488+
return .element(element)
468489
}
469490
} else {
470491
if let element = target.element, element.exists {
471-
return element
492+
return .element(element)
472493
}
473494
}
474495
if let refreshPoint = target.refreshPoint,
475496
let refreshed = textInputAt(app: app, x: refreshPoint.x, y: refreshPoint.y) {
476-
return refreshed.element
497+
return .lookup(refreshed)
477498
}
478499
if let focused = focusedTextInput(app: app) {
479-
return focused
500+
return .element(focused)
480501
}
481502
return nil
482503
}

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift‎

Lines changed: 19 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -157,98 +157,38 @@ extension RunnerTests {
157157
return true
158158
}
159159

160-
private static let textInputElementTypeRawValues: [UInt] = [
161-
XCUIElement.ElementType.textField,
162-
.secureTextField,
163-
.searchField,
164-
.textView,
165-
].map(\.rawValue)
166-
167160
private func queryTextInputs(
168161
app: XCUIApplication,
169162
point: CGPoint,
170163
shouldStop: () -> Bool
171164
) -> [ResolvedTextEntryInput] {
172-
// One query for every text input type: each query snapshots the whole application.
173-
let candidates = app.descendants(matching: .any)
174-
.matching(NSPredicate(format: "elementType IN %@", Self.textInputElementTypeRawValues))
175-
.allElementsBoundByIndex
176-
#if AGENT_DEVICE_RUNNER_UNIT_TESTS
177-
if let issue = textInputProbeIssueForTesting {
178-
textInputProbeIssueForTesting = nil
179-
record(issue)
180-
}
181-
#endif
182-
guard !shouldStop() else { return [] }
183-
// One snapshot per candidate answers enablement, frame, and the ordering below: each attribute
184-
// read from the live element runs the candidate's query again. A candidate that cannot be
185-
// snapshotted is read live, so its failure still reaches the caller's issue handling.
186-
// Each handle binds by index within its own element type, so an input of another type that
187-
// appears ahead of it later does not take its place.
188-
var indexByType: [XCUIElement.ElementType: Int] = [:]
189165
var inputs: [ResolvedTextEntryInput] = []
190-
for (position, element) in candidates.enumerated() {
166+
for query in [app.textFields, app.secureTextFields, app.searchFields, app.textViews] {
191167
if shouldStop() { break }
192-
// Each candidate re-runs the shared query, so an input that left the app since an earlier
193-
// candidate's read moves a later input into a position already read, and the per-type count
194-
// no longer names the input a position reads. Every position must still exist and every
195-
// handle must read as its position's input, at the point or not, or the lookup can miss an
196-
// input. A missing position's snapshot throws only after XCTest waits for a match, which only
197-
// a lookup that falls back pays, so existence is read only when the snapshot fails.
198-
let snapshot = try? element.snapshot()
199-
guard snapshot != nil || element.exists else {
200-
return queryTextInputsByType(app: app, point: point, shouldStop: shouldStop)
201-
}
202-
let elementType = snapshot?.elementType ?? element.elementType
203-
let typeIndex = indexByType[elementType, default: 0]
204-
indexByType[elementType] = typeIndex + 1
205-
let handle = app.descendants(matching: elementType).element(boundBy: typeIndex)
206-
let candidate: ResolvedTextEntryInput
207-
if let snapshot {
208-
guard let handleSnapshot = try? handle.snapshot(), Self.isSameTextInput(handleSnapshot, as: snapshot) else {
209-
return queryTextInputsByType(app: app, point: point, shouldStop: shouldStop)
210-
}
211-
candidate = ResolvedTextEntryInput(element: handle, snapshot: handleSnapshot)
212-
} else {
213-
candidate = ResolvedTextEntryInput(element: handle, snapshot: nil)
214-
}
168+
let candidates = query.allElementsBoundByIndex
215169
#if AGENT_DEVICE_RUNNER_UNIT_TESTS
216-
textInputCandidateReadForTesting?(position)
217-
#endif
218-
let attributes = candidate.attributes
219-
if isCoordinateTextInputCandidate(enabled: attributes.isEnabled, frame: attributes.frame, point: point) {
220-
inputs.append(candidate)
170+
if let issue = textInputProbeIssueForTesting {
171+
textInputProbeIssueForTesting = nil
172+
record(issue)
221173
}
222-
}
223-
return inputs.sorted { smallestElementFirst($0.attributes, $1.attributes) }
224-
}
225-
226-
/// `queryTextInputs` with one query per element type, whose handles each read through their own
227-
/// snapshot, for a lookup whose shared query changed while it read the candidates.
228-
private func queryTextInputsByType(
229-
app: XCUIApplication,
230-
point: CGPoint,
231-
shouldStop: () -> Bool
232-
) -> [ResolvedTextEntryInput] {
233-
NSLog("AGENT_DEVICE_RUNNER_TEXT_INPUT_LOOKUP_BY_TYPE reason=inputs-changed-during-lookup")
234-
var inputs: [ResolvedTextEntryInput] = []
235-
for query in [app.textFields, app.secureTextFields, app.searchFields, app.textViews] {
236-
for element in query.allElementsBoundByIndex {
237-
guard !shouldStop() else { return [] }
238-
guard element.exists else { continue }
239-
let input = ResolvedTextEntryInput(element: element, snapshot: try? element.snapshot())
240-
let attributes = input.attributes
174+
#endif
175+
// One snapshot per candidate answers enablement, frame, and the ordering below: each attribute
176+
// read from the live element runs the candidate's query again. A candidate that cannot be
177+
// snapshotted is read live, so its failure still reaches the caller's issue handling. A missing
178+
// candidate's snapshot throws only after XCTest waits for a match, so existence is read only
179+
// when the snapshot fails.
180+
for element in candidates {
181+
if shouldStop() { break }
182+
let snapshot = try? element.snapshot()
183+
guard snapshot != nil || element.exists else { continue }
184+
let candidate = ResolvedTextEntryInput(element: element, snapshot: snapshot)
185+
let attributes = candidate.attributes
241186
if isCoordinateTextInputCandidate(enabled: attributes.isEnabled, frame: attributes.frame, point: point) {
242-
inputs.append(input)
187+
inputs.append(candidate)
243188
}
244189
}
245190
}
191+
guard !shouldStop() else { return [] }
246192
return inputs.sorted { smallestElementFirst($0.attributes, $1.attributes) }
247193
}
248-
249-
private static func isSameTextInput(_ read: XCUIElementSnapshot, as candidate: XCUIElementSnapshot) -> Bool {
250-
read.elementType == candidate.elementType
251-
&& read.identifier == candidate.identifier
252-
&& read.frame == candidate.frame
253-
}
254194
}

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -225,9 +225,6 @@ final class RunnerTests: XCTestCase {
225225
// Runs on the waiting thread after `runMainThreadWork`'s wait timed out and before it takes the
226226
// lock that decides between finished and abandoned, so a test can finish the work in that window.
227227
var mainThreadWorkTimedOutForTesting: (@Sendable () -> Void)?
228-
// Runs after the point lookup reads the candidate at a position of its shared query, so a test can
229-
// change the app's inputs between two candidate reads.
230-
var textInputCandidateReadForTesting: ((Int) -> Void)?
231228
// Runs before a fill pauses after reading its warmup character back, so a test can change the app's
232229
// inputs in that pause.
233230
var textEntryWarmupPauseForTesting: (() -> Void)?

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ extension RunnerTests {
2525
/// Bluesky's React Native Search screen (iPhone 17 Pro, iOS 27.0), so a fill's cost there is its
2626
/// query count. The fixture field counts how often XCTest reads its value, which it does once per
2727
/// query. Its value is masked, so the count does not depend on how many read-back polls fit in a
28-
/// stability window. This fill made 16 queries when each resolution's snapshot served the reads
28+
/// stability window. This fill made 19 queries when each resolution's snapshot served the reads
2929
/// after it, and 35 when every attribute was read from the live element.
3030
@MainActor
3131
func testFillReadsEachResolvedFieldThroughOneSnapshot() throws {
@@ -62,5 +62,33 @@ extension RunnerTests {
6262
XCTAssertLessThanOrEqual(readsAfter - readsBefore, 24)
6363
}
6464

65+
/// An unbound target without a live element resolves through its refresh point's lookup, which
66+
/// has already snapshotted the input it finds. The resolution reads the app no more often than the
67+
/// lookup alone does.
68+
@MainActor
69+
func testUnboundResolutionReadsItsInputThroughThePointLookup() throws {
70+
app.launchArguments = ["--agent-device-text-entry-regression", "--agent-device-text-entry-counts-value-reads"]
71+
app.launch()
72+
defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() }
73+
let field = app.textFields["agent-device-hardware-keyboard-input"]
74+
XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout))
75+
let valueReads = app.staticTexts["agent-device-text-entry-value-reads"]
76+
let frame = field.frame
77+
let point = CGPoint(x: frame.midX, y: frame.midY)
78+
func countingReads<T>(_ body: () -> T) throws -> (T, Int) {
79+
let before = try XCTUnwrap(Int(valueReads.label))
80+
let value = body()
81+
return (value, try XCTUnwrap(Int(valueReads.label)) - before)
82+
}
83+
let target = TextEntryTarget(element: nil, refreshPoint: point, prefersFocusedElement: false)
84+
85+
let (lookup, lookupReads) = try countingReads { textInputAt(app: app, x: point.x, y: point.y) }
86+
let (input, resolutionReads) = try countingReads { resolveTextEntryInput(app: app, target: target) }
87+
88+
XCTAssertEqual(lookup?.snapshot?.identifier, "agent-device-hardware-keyboard-input")
89+
XCTAssertEqual(input?.snapshot?.identifier, "agent-device-hardware-keyboard-input")
90+
XCTAssertEqual(resolutionReads, lookupReads)
91+
}
92+
6593
#endif
6694
}

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift‎

Lines changed: 0 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -85,67 +85,6 @@ extension RunnerTests {
8585
XCTAssertFalse(didRecordXCTestFailure(since: failures))
8686
}
8787

88-
// The lookup reads its candidates through one query for every text input type, in the order text
89-
// field, first search field, second search field. Each case removes the text field after the lookup
90-
// reads one position, which moves each search field into the position before it, one already read.
91-
92-
// The second search field moves into the first one's position, the one the lookup reads next. The
93-
// input the lookup hands out must be the second search field, whose snapshot it read.
94-
@MainActor
95-
func testTextInputLookupKeepsItsInputWhenAnEarlierInputOfAnotherTypeLeaves() throws {
96-
try assertMixedTextInputLookup(finds: "mixed-search-b", removingTextFieldAfterPosition: 0)
97-
}
98-
99-
// The first search field moves into the position already read. The position read next holds the
100-
// second search field, which is not at the point.
101-
@MainActor
102-
func testTextInputLookupFindsTheFirstSearchFieldWhenTheTextFieldLeavesAfterTheFirstRead() throws {
103-
try assertMixedTextInputLookup(finds: "mixed-search-a", removingTextFieldAfterPosition: 0)
104-
}
105-
106-
// The second search field moves into the position already read. The position read next no longer
107-
// exists.
108-
@MainActor
109-
func testTextInputLookupFindsTheSecondSearchFieldWhenTheTextFieldLeavesAfterTheSecondRead() throws {
110-
try assertMixedTextInputLookup(finds: "mixed-search-b", removingTextFieldAfterPosition: 1)
111-
}
112-
113-
@MainActor
114-
private func assertMixedTextInputLookup(
115-
finds identifier: String,
116-
removingTextFieldAfterPosition removalPosition: Int
117-
) throws {
118-
app.launchArguments = ["--agent-device-mixed-text-inputs"]
119-
app.launch()
120-
defer {
121-
textInputCandidateReadForTesting = nil
122-
app.terminate()
123-
}
124-
let field = app.textFields["mixed-text"]
125-
let target = app.searchFields[identifier]
126-
XCTAssertTrue(target.waitForFixtureExistence(timeout: appExistenceTimeout))
127-
XCTAssertTrue(field.exists)
128-
let frame = target.frame
129-
let removeField = app.buttons["mixed-remove-text"]
130-
var removals = 0
131-
textInputCandidateReadForTesting = { position in
132-
guard position == removalPosition, removals == 0 else { return }
133-
removals += 1
134-
removeField.tap()
135-
}
136-
let failures = currentXCTestFailureCount()
137-
138-
let input = try XCTUnwrap(textInputAt(app: app, x: frame.midX, y: frame.midY))
139-
textInputCandidateReadForTesting = nil
140-
141-
XCTAssertFalse(didRecordXCTestFailure(since: failures))
142-
XCTAssertEqual(removals, 1)
143-
XCTAssertFalse(field.exists)
144-
XCTAssertEqual(input.snapshot?.identifier, identifier)
145-
XCTAssertEqual(input.element.identifier, identifier)
146-
XCTAssertEqual(input.element.frame, frame)
147-
}
148-
14988
func testTextInputProbeContainmentExcludesRequiredReadsAndLaterIssues() {
15089
app.launchArguments = ["--agent-device-text-entry-regression"]
15190
app.launch()

0 commit comments

Comments
 (0)