Skip to content

Commit f3d2619

Browse files
thymikeeclaude
andauthored
perf(ios): confirm fill focus from the tapped input before the keyboard (#3906)
* perf(ios): confirm fill focus from the tapped input before the keyboard After a focus tap with no keyboard up, stabilization asked for the keyboard first. While a software keyboard is presented, that query blocks and then finds none: 0.68 s on an iPhone 17 Pro simulator and 0.57-3.77 s on CI, before the app-wide focus query confirmed focus anyway and the refresh point read the input again. Read the tapped input first and accept its own hasKeyboardFocus, which is stronger evidence than the keyboard's arrival; that read's frame serves the refresh point. The keyboard and the app-wide focus query stay as fallbacks. The runner log names the evidence that confirmed focus. Co-Authored-By: Claude Opus 5.5 <[email protected]> * perf(ios): carry the focus-confirming read in its evidence The tapped input's read rides in TextEntryFocusEvidence.inputReportsKeyboardFocus, so the evidence and the post-focus read cannot disagree. One read per poll serves both the input's own focus check and the app-wide focus query's expected snapshot. The keyboard-raising fill test skips before it asserts and pins the fixture's value-read count at 18 (17 measured, 19 when the entry resolves the input again). Device timings move from comments to the PR. Co-Authored-By: Claude Opus 5.5 <[email protected]> * test(ios): name the reads the keyboard-raising fill's pin catches Removing the focus step's hand-off and dropping the read's seeding both raise the fixture count from 17 to 19. Co-Authored-By: Claude Opus 5.5 <[email protected]> --------- Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent d217300 commit f3d2619

5 files changed

Lines changed: 152 additions & 24 deletions

File tree

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -513,6 +513,11 @@ extension RunnerTests {
513513
return focusBool(snapshot as! NSObject)
514514
}
515515

516+
/// `hasKeyboardFocus` alone: whether the snapshot's element is the one a keyboard types into.
517+
func snapshotHasKeyboardFocus(_ snapshot: XCUIElementSnapshot) -> Bool {
518+
return kvcBool(snapshot as! NSObject, forKey: "hasKeyboardFocus")
519+
}
520+
516521
/// Either focus is focus. `hasFocus` is the focus engine's answer (tvOS, keyboard navigation);
517522
/// the field a software keyboard is typing into holds `hasKeyboardFocus` instead, which the
518523
/// text-entry readiness check already consults. Both an element and its snapshot answer the two

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -311,9 +311,38 @@ extension RunnerTests {
311311
boundIdentity?.isDistinguishable ?? true
312312
}
313313

314+
/// What confirmed that a focus tap's input took focus.
315+
enum TextEntryFocusEvidence {
316+
/// tvOS moves focus with the focus engine, so there is nothing to wait for.
317+
case focusEngine
318+
/// The app-wide focus query named a text input (not iOS).
319+
case focusedInput
320+
/// The tapped input's own read reports keyboard focus. That read is the focus step's post-focus
321+
/// read.
322+
case inputReportsKeyboardFocus(ResolvedTextEntryInput)
323+
/// A software keyboard appeared after the tap, so some input took focus.
324+
case keyboardAppeared
325+
/// The app-wide keyboard-focus query answered with the tapped input.
326+
case focusQueryMatchesInput
327+
328+
/// The name `runner.log` records.
329+
var name: String {
330+
switch self {
331+
case .focusEngine: return "focus-engine"
332+
case .focusedInput: return "focused-input"
333+
case .inputReportsKeyboardFocus: return "input-reports-keyboard-focus"
334+
case .keyboardAppeared: return "keyboard-appeared"
335+
case .focusQueryMatchesInput: return "focus-query-matches-input"
336+
}
337+
}
338+
}
339+
314340
struct TextEntryStabilization {
315341
let element: XCUIElement?
316-
let focusConfirmed: Bool
342+
/// Nil when nothing confirmed focus before the focus timeout.
343+
let evidence: TextEntryFocusEvidence?
344+
345+
var focusConfirmed: Bool { evidence != nil }
317346
}
318347

319348
struct TextEntryTapWitness {

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift‎

Lines changed: 36 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -64,27 +64,38 @@ extension RunnerTests {
6464
keyboardVisibleBeforeTap: Bool? = nil
6565
) -> TextEntryStabilization {
6666
#if os(tvOS)
67-
return TextEntryStabilization(element: target, focusConfirmed: true)
67+
return TextEntryStabilization(element: target, evidence: .focusEngine)
6868
#else
6969
let keyboardVisibleAtEntry = keyboardVisibleBeforeTap ?? isKeyboardVisible(app: app)
7070
let deadline = Date().addingTimeInterval(TextEntryTiming.focusTimeout)
7171
while Date() < deadline {
7272
if let focused = focusedTextInput(app: app) {
73-
return TextEntryStabilization(element: focused, focusConfirmed: true)
73+
return TextEntryStabilization(element: focused, evidence: .focusedInput)
7474
}
75-
// focusedTextInput is intentionally nil on iOS; treat the keyboard transitioning to
76-
// visible after our tap as the focus-moved signal. Don't fast-path when it was already up.
75+
#if os(iOS)
76+
// The tapped input's own read comes before the keyboard. While a software keyboard is
77+
// presented, the keyboard query blocks and then finds none. An input that reports keyboard
78+
// focus is stronger evidence than the keyboard's arrival, which shows only that some input
79+
// took focus.
80+
let read = target.flatMap(ResolvedTextEntryInput.init(reading:))
81+
if let read, let snapshot = read.snapshot, snapshotHasKeyboardFocus(snapshot) {
82+
return TextEntryStabilization(element: read.element, evidence: .inputReportsKeyboardFocus(read))
83+
}
84+
#endif
85+
// focusedTextInput is intentionally nil on iOS; the keyboard transitioning to visible after
86+
// our tap is also a focus-moved signal. Don't fast-path when it was already up.
7787
if keyboardBecameVisible(app: app, wasVisibleAtEntry: keyboardVisibleAtEntry) {
78-
return TextEntryStabilization(element: target, focusConfirmed: true)
88+
return TextEntryStabilization(element: target, evidence: .keyboardAppeared)
7989
}
8090
#if os(iOS)
81-
if let target, keyboardFocusConfirmed(app: app, element: target) {
82-
return TextEntryStabilization(element: target, focusConfirmed: true)
91+
// The read came before the keyboard query, so it is not handed on as the post-focus read.
92+
if let read, keyboardFocusConfirmed(app: app, read: read) {
93+
return TextEntryStabilization(element: target, evidence: .focusQueryMatchesInput)
8394
}
8495
#endif
8596
sleepFor(TextEntryTiming.pollInterval)
8697
}
87-
return TextEntryStabilization(element: target, focusConfirmed: false)
98+
return TextEntryStabilization(element: target, evidence: nil)
8899
#endif
89100
}
90101

@@ -95,10 +106,10 @@ extension RunnerTests {
95106
if !softwareKeyboardVisible, let rememberedTarget = rememberedTextEntryTarget() {
96107
return rememberedTarget
97108
}
98-
// Bare `type` targets the current first responder. On iOS we intentionally do not trust
99-
// `hasKeyboardFocus`, but an already-visible software keyboard is sufficient evidence that
100-
// app.typeText has a receiver; waiting the full readiness timeout cannot prove a stronger
101-
// target because there is no selector/coordinate focus move to validate.
109+
// Bare `type` targets the current first responder. On iOS we intentionally do not let the
110+
// app-wide `hasKeyboardFocus` query pick it, but an already-visible software keyboard is
111+
// sufficient evidence that app.typeText has a receiver; waiting the full readiness timeout
112+
// cannot prove a stronger target because there is no selector/coordinate focus move to validate.
102113
if softwareKeyboardVisible {
103114
return TextEntryTarget(
104115
element: focusedTextInput(app: app),
@@ -137,6 +148,10 @@ extension RunnerTests {
137148
target: target,
138149
keyboardVisibleBeforeTap: keyboardVisibleBeforeTap
139150
)
151+
NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_FOCUS evidence=%@", stabilized.evidence?.name ?? "none")
152+
if case .inputReportsKeyboardFocus(let read) = stabilized.evidence {
153+
return focusedTextEntryTarget(read.element, read: read, requestedPoint: requestedPoint)
154+
}
140155
let readyTarget = TextEntryTarget(
141156
element: stabilized.element ?? target,
142157
refreshPoint: requestedPoint,
@@ -153,7 +168,15 @@ extension RunnerTests {
153168
guard let element else {
154169
return TextEntryTarget(element: nil, refreshPoint: requestedPoint, prefersFocusedElement: false)
155170
}
156-
let read = ResolvedTextEntryInput(reading: element)
171+
return focusedTextEntryTarget(element, read: ResolvedTextEntryInput(reading: element), requestedPoint: requestedPoint)
172+
}
173+
174+
/// `read` is `element` read after the focus tap, or nil when the read found no element.
175+
private func focusedTextEntryTarget(
176+
_ element: XCUIElement,
177+
read: ResolvedTextEntryInput?,
178+
requestedPoint: CGPoint
179+
) -> TextEntryTarget {
157180
return TextEntryTarget(
158181
element: element,
159182
refreshPoint: textEntryRefreshPoint(frame: read?.snapshot?.frame, fallbackTo: requestedPoint),

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift‎

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@ extension RunnerTests {
44
func focusedTextInput(app: XCUIApplication) -> XCUIElement? {
55
#if os(iOS)
66
// iOS focus predicates can return stale or misleading text-input matches
7-
// under XCUITest, so text entry readiness is driven by tap/keyboard state.
7+
// under XCUITest, so they never pick the text-entry target. Readiness corroborates the
8+
// input a tap or coordinate already chose.
89
return nil
910
#else
1011
return safely("FOCUSED_INPUT_QUERY") {
@@ -128,13 +129,22 @@ extension RunnerTests {
128129
/// fallback at all.
129130
func keyboardFocusConfirmed(app: XCUIApplication, element: XCUIElement) -> Bool {
130131
#if os(iOS)
132+
return ResolvedTextEntryInput(reading: element).map { keyboardFocusConfirmed(app: app, read: $0) } ?? false
133+
#else
134+
return false
135+
#endif
136+
}
137+
138+
#if os(iOS)
139+
/// `keyboardFocusConfirmed(app:element:)` against a read of the element already taken.
140+
func keyboardFocusConfirmed(app: XCUIApplication, read: ResolvedTextEntryInput) -> Bool {
141+
// An element that no longer resolves reads as identifier "" and frame `.zero`, which would
142+
// match any focused element that also reports an empty frame. Require a real frame first,
143+
// so a dead handle cannot corroborate anything.
144+
guard let expected = read.snapshot, !expected.frame.isEmpty else {
145+
return false
146+
}
131147
return safely("TEXT_ENTRY_FOCUS_CONFIRMED", false) {
132-
// An element that no longer resolves reads as identifier "" and frame `.zero`, which would
133-
// match any focused element that also reports an empty frame. Require a real frame first,
134-
// so a dead handle cannot corroborate anything.
135-
guard element.exists, let expected = try? element.snapshot(), !expected.frame.isEmpty else {
136-
return false
137-
}
138148
let focused = app
139149
.descendants(matching: .any)
140150
.matching(NSPredicate(format: "hasKeyboardFocus == 1"))
@@ -144,10 +154,8 @@ extension RunnerTests {
144154
}
145155
return observed.identifier == expected.identifier && observed.frame == expected.frame
146156
}
147-
#else
148-
return false
149-
#endif
150157
}
158+
#endif
151159

152160
private func keyboardElementExists(app: XCUIApplication) -> Bool {
153161
#if os(iOS)

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,5 +90,68 @@ extension RunnerTests {
9090
XCTAssertEqual(resolutionReads, lookupReads)
9191
}
9292

93+
/// After a focus tap raises the software keyboard, stabilization takes focus from the tapped
94+
/// input's own read before it asks for the keyboard. While the keyboard is presented, the keyboard
95+
/// query blocks and then finds none. The keyboard is already up here, so asking it first confirms
96+
/// focus from the keyboard's arrival instead.
97+
@MainActor
98+
func testFocusStabilizationAsksTheTappedInputBeforeTheKeyboard() throws {
99+
app.launchArguments = ["--agent-device-text-entry-regression", "--agent-device-text-entry-soft-keyboard"]
100+
app.launch()
101+
defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() }
102+
XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout))
103+
let field = app.textFields["agent-device-hardware-keyboard-input"]
104+
XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout))
105+
field.tap()
106+
try XCTSkipUnless(
107+
app.keyboards.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout),
108+
"software keyboard is down: this simulator cannot show the keyboard a focus tap raises"
109+
)
110+
111+
let stabilized = stabilizeTextInputBeforeTyping(app: app, target: field, keyboardVisibleBeforeTap: false)
112+
113+
guard case .inputReportsKeyboardFocus(let read) = stabilized.evidence else {
114+
return XCTFail("focus confirmed by \(stabilized.evidence?.name ?? "nothing"), not by the tapped input")
115+
}
116+
XCTAssertEqual(read.snapshot?.frame, field.frame)
117+
}
118+
119+
/// A fill whose focus tap raises the software keyboard hands the read that confirmed focus to the
120+
/// entry as its first read. The fixture counts queries as in
121+
/// `testFillReadsEachResolvedFieldThroughOneSnapshot`: this fill made 17, and 19 when the focus step
122+
/// or the entry read the input again. The one read of headroom covers a keyboard that arrives
123+
/// before the field reports focus.
124+
@MainActor
125+
func testFillThatRaisesTheSoftwareKeyboardReusesTheReadThatConfirmedFocus() throws {
126+
app.launchArguments = [
127+
"--agent-device-text-entry-regression",
128+
"--agent-device-text-entry-soft-keyboard",
129+
"--agent-device-text-entry-counts-value-reads",
130+
]
131+
app.launch()
132+
defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() }
133+
XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout))
134+
let field = app.textFields["agent-device-hardware-keyboard-input"]
135+
XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout))
136+
XCTAssertFalse(isKeyboardVisible(app: app))
137+
let valueReads = app.staticTexts["agent-device-text-entry-value-reads"]
138+
let frame = field.frame
139+
let fill = try runnerCommandFixture(
140+
#"{"command":"type","commandId":"fill-raises-keyboard","text":"hello","textEntryMode":"replace","x":\#(frame.midX),"y":\#(frame.midY)}"#
141+
)
142+
143+
let readsBefore = try XCTUnwrap(Int(valueReads.label))
144+
let response = executeTypeCommand(activeApp: app, command: fill)
145+
let readsAfter = try XCTUnwrap(Int(valueReads.label))
146+
147+
try XCTSkipUnless(
148+
app.keyboards.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout),
149+
"software keyboard is down: this simulator did not raise the keyboard the focus tap asked for"
150+
)
151+
XCTAssertTrue(response.ok, String(describing: response.error))
152+
XCTAssertEqual(response.data?.textEntryRoute, "xctest-element")
153+
XCTAssertEqual(valueReads.value as? String, "hello")
154+
XCTAssertLessThanOrEqual(readsAfter - readsBefore, 18)
155+
}
93156
#endif
94157
}

0 commit comments

Comments
 (0)