Skip to content

Commit fcc403f

Browse files
thymikeeclaude
andcommitted
perf(ios): confirm fill focus from the tapped input before the keyboard
After a focus tap with no keyboard up, stabilization asked for the keyboard first. While a software keyboard is presented, that query blocks and then finds none: 0.68 s on an iPhone 17 Pro simulator and 0.57-3.77 s on CI, before the app-wide focus query confirmed focus anyway and the refresh point read the input again. Read the tapped input first and accept its own hasKeyboardFocus, which is stronger evidence than the keyboard's arrival; that read's frame serves the refresh point. The keyboard and the app-wide focus query stay as fallbacks. The runner log names the evidence that confirmed focus. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent e3ed027 commit fcc403f

5 files changed

Lines changed: 130 additions & 14 deletions

File tree

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -513,6 +513,11 @@ extension RunnerTests {
513513
return focusBool(snapshot as! NSObject)
514514
}
515515

516+
/// `hasKeyboardFocus` alone: whether the snapshot's element is the one a keyboard types into.
517+
func snapshotHasKeyboardFocus(_ snapshot: XCUIElementSnapshot) -> Bool {
518+
return kvcBool(snapshot as! NSObject, forKey: "hasKeyboardFocus")
519+
}
520+
516521
/// Either focus is focus. `hasFocus` is the focus engine's answer (tvOS, keyboard navigation);
517522
/// the field a software keyboard is typing into holds `hasKeyboardFocus` instead, which the
518523
/// text-entry readiness check already consults. Both an element and its snapshot answer the two

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -226,9 +226,28 @@ extension RunnerTests {
226226
boundIdentity?.isDistinguishable ?? true
227227
}
228228

229+
/// What confirmed that a focus tap's input took focus.
230+
enum TextEntryFocusEvidence: String {
231+
/// tvOS moves focus with the focus engine, so there is nothing to wait for.
232+
case focusEngine = "focus-engine"
233+
/// The app-wide focus query named a text input (not iOS).
234+
case focusedInput = "focused-input"
235+
/// The tapped input's own snapshot reports keyboard focus.
236+
case inputReportsKeyboardFocus = "input-reports-keyboard-focus"
237+
/// A software keyboard appeared after the tap, so some input took focus.
238+
case keyboardAppeared = "keyboard-appeared"
239+
/// The app-wide keyboard-focus query answered with the tapped input.
240+
case focusQueryMatchesInput = "focus-query-matches-input"
241+
}
242+
229243
struct TextEntryStabilization {
230244
let element: XCUIElement?
231-
let focusConfirmed: Bool
245+
/// Nil when nothing confirmed focus before the focus timeout.
246+
let evidence: TextEntryFocusEvidence?
247+
/// `element`'s frame from the read that confirmed focus, when the input's own read did.
248+
var inputFrame: CGRect? = nil
249+
250+
var focusConfirmed: Bool { evidence != nil }
232251
}
233252

234253
struct TextEntryTapWitness {

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift‎

Lines changed: 36 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -64,27 +64,37 @@ extension RunnerTests {
6464
keyboardVisibleBeforeTap: Bool? = nil
6565
) -> TextEntryStabilization {
6666
#if os(tvOS)
67-
return TextEntryStabilization(element: target, focusConfirmed: true)
67+
return TextEntryStabilization(element: target, evidence: .focusEngine)
6868
#else
6969
let keyboardVisibleAtEntry = keyboardVisibleBeforeTap ?? isKeyboardVisible(app: app)
7070
let deadline = Date().addingTimeInterval(TextEntryTiming.focusTimeout)
7171
while Date() < deadline {
7272
if let focused = focusedTextInput(app: app) {
73-
return TextEntryStabilization(element: focused, focusConfirmed: true)
73+
return TextEntryStabilization(element: focused, evidence: .focusedInput)
7474
}
75-
// focusedTextInput is intentionally nil on iOS; treat the keyboard transitioning to
76-
// visible after our tap as the focus-moved signal. Don't fast-path when it was already up.
75+
#if os(iOS)
76+
// The tapped input's own read comes before the keyboard. While a software keyboard is
77+
// presented, the keyboard query blocks and then finds none: after a fill's focus tap it took
78+
// 0.68 s on an iPhone 17 Pro simulator (iOS 27.0) and 0.57–3.77 s on CI. An input that
79+
// reports keyboard focus is stronger evidence than the keyboard's arrival, which shows only
80+
// that some input took focus.
81+
if let target, let frame = keyboardFocusedInputFrame(target) {
82+
return TextEntryStabilization(element: target, evidence: .inputReportsKeyboardFocus, inputFrame: frame)
83+
}
84+
#endif
85+
// focusedTextInput is intentionally nil on iOS; the keyboard transitioning to visible after
86+
// our tap is also a focus-moved signal. Don't fast-path when it was already up.
7787
if keyboardBecameVisible(app: app, wasVisibleAtEntry: keyboardVisibleAtEntry) {
78-
return TextEntryStabilization(element: target, focusConfirmed: true)
88+
return TextEntryStabilization(element: target, evidence: .keyboardAppeared)
7989
}
8090
#if os(iOS)
8191
if let target, keyboardFocusConfirmed(app: app, element: target) {
82-
return TextEntryStabilization(element: target, focusConfirmed: true)
92+
return TextEntryStabilization(element: target, evidence: .focusQueryMatchesInput)
8393
}
8494
#endif
8595
sleepFor(TextEntryTiming.pollInterval)
8696
}
87-
return TextEntryStabilization(element: target, focusConfirmed: false)
97+
return TextEntryStabilization(element: target, evidence: nil)
8898
#endif
8999
}
90100

@@ -95,10 +105,10 @@ extension RunnerTests {
95105
if !softwareKeyboardVisible, let rememberedTarget = rememberedTextEntryTarget() {
96106
return rememberedTarget
97107
}
98-
// Bare `type` targets the current first responder. On iOS we intentionally do not trust
99-
// `hasKeyboardFocus`, but an already-visible software keyboard is sufficient evidence that
100-
// app.typeText has a receiver; waiting the full readiness timeout cannot prove a stronger
101-
// target because there is no selector/coordinate focus move to validate.
108+
// Bare `type` targets the current first responder. On iOS we intentionally do not let the
109+
// app-wide `hasKeyboardFocus` query pick it, but an already-visible software keyboard is
110+
// sufficient evidence that app.typeText has a receiver; waiting the full readiness timeout
111+
// cannot prove a stronger target because there is no selector/coordinate focus move to validate.
102112
if softwareKeyboardVisible {
103113
return TextEntryTarget(
104114
element: focusedTextInput(app: app),
@@ -145,6 +155,15 @@ extension RunnerTests {
145155
target: target,
146156
keyboardVisibleBeforeTap: keyboardVisibleBeforeTap
147157
)
158+
NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_FOCUS evidence=%@", stabilized.evidence?.rawValue ?? "none")
159+
if let frame = stabilized.inputFrame {
160+
// The read that confirmed focus came after the tap, so its frame serves the refresh point.
161+
return TextEntryTarget(
162+
element: stabilized.element,
163+
refreshPoint: textEntryRefreshPoint(frame: frame, fallbackTo: requestedPoint),
164+
prefersFocusedElement: false
165+
)
166+
}
148167
let readyTarget = TextEntryTarget(
149168
element: stabilized.element ?? target,
150169
refreshPoint: requestedPoint,
@@ -176,7 +195,12 @@ extension RunnerTests {
176195
guard let element else {
177196
return requestedPoint
178197
}
179-
guard let frame = textEntrySnapshotFrame(element) else {
198+
return textEntryRefreshPoint(frame: textEntrySnapshotFrame(element), fallbackTo: requestedPoint)
199+
}
200+
201+
/// `textEntryRefreshPoint(for:)` from a frame already read after the focus tap.
202+
private func textEntryRefreshPoint(frame: CGRect?, fallbackTo requestedPoint: CGPoint?) -> CGPoint? {
203+
guard let frame else {
180204
return nil
181205
}
182206
guard !frame.isEmpty else {

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@ extension RunnerTests {
44
func focusedTextInput(app: XCUIApplication) -> XCUIElement? {
55
#if os(iOS)
66
// iOS focus predicates can return stale or misleading text-input matches
7-
// under XCUITest, so text entry readiness is driven by tap/keyboard state.
7+
// under XCUITest, so they never pick the text-entry target. Readiness corroborates the
8+
// input a tap or coordinate already chose.
89
return nil
910
#else
1011
return safely("FOCUSED_INPUT_QUERY") {
@@ -149,6 +150,22 @@ extension RunnerTests {
149150
#endif
150151
}
151152

153+
/// `element`'s frame when its own snapshot reports keyboard focus, or nil. Read through
154+
/// `snapshot()`, which records nothing, after `exists`, so a field that stopped answering its query
155+
/// on focus (#3060) neither records an XCTest failure nor pays the snapshot's wait for a match.
156+
func keyboardFocusedInputFrame(_ element: XCUIElement) -> CGRect? {
157+
#if os(iOS)
158+
return safely("TEXT_ENTRY_INPUT_FOCUS") {
159+
guard element.exists, let snapshot = try? element.snapshot(), snapshotHasKeyboardFocus(snapshot) else {
160+
return nil
161+
}
162+
return snapshot.frame
163+
}
164+
#else
165+
return nil
166+
#endif
167+
}
168+
152169
private func keyboardElementExists(app: XCUIApplication) -> Bool {
153170
#if os(iOS)
154171
return safely("KEYBOARD_EXISTS", false) { app.keyboards.firstMatch.exists }

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,5 +21,56 @@ extension RunnerTests {
2121
XCTAssertLessThan(elapsed, TextEntryTiming.readinessTimeout)
2222
}
2323

24+
/// After a focus tap raises the software keyboard, stabilization takes focus from the tapped
25+
/// input's own read before it asks for the keyboard. While the keyboard is presented, the keyboard
26+
/// query blocks and then finds none: after a fill's focus tap it took 0.68 s on the test app
27+
/// (iPhone 17 Pro, iOS 27.0) and 0.57–3.77 s on CI. The keyboard is already up here, so asking it
28+
/// first confirms focus from the keyboard's arrival instead.
29+
@MainActor
30+
func testFocusStabilizationAsksTheTappedInputBeforeTheKeyboard() throws {
31+
app.launchArguments = ["--agent-device-text-entry-regression", "--agent-device-text-entry-soft-keyboard"]
32+
app.launch()
33+
defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() }
34+
XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout))
35+
let field = app.textFields["agent-device-hardware-keyboard-input"]
36+
XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout))
37+
field.tap()
38+
try XCTSkipUnless(
39+
app.keyboards.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout),
40+
"software keyboard is down: this simulator cannot show the keyboard a focus tap raises"
41+
)
42+
43+
let stabilized = stabilizeTextInputBeforeTyping(app: app, target: field, keyboardVisibleBeforeTap: false)
44+
45+
XCTAssertEqual(stabilized.evidence, .inputReportsKeyboardFocus)
46+
XCTAssertEqual(stabilized.inputFrame, field.frame)
47+
}
48+
49+
/// A fill whose focus tap raises the software keyboard types and reads its text back when focus is
50+
/// confirmed from the input rather than from the keyboard's arrival.
51+
@MainActor
52+
func testFillThatRaisesTheSoftwareKeyboardTypesAndReadsBack() throws {
53+
app.launchArguments = ["--agent-device-text-entry-regression", "--agent-device-text-entry-soft-keyboard"]
54+
app.launch()
55+
defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() }
56+
XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout))
57+
let field = app.textFields["agent-device-hardware-keyboard-input"]
58+
XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout))
59+
XCTAssertFalse(isKeyboardVisible(app: app))
60+
let frame = field.frame
61+
let fill = try runnerCommandFixture(
62+
#"{"command":"type","commandId":"fill-raises-keyboard","text":"hello","textEntryMode":"replace","x":\#(frame.midX),"y":\#(frame.midY)}"#
63+
)
64+
65+
let response = executeTypeCommand(activeApp: app, command: fill)
66+
67+
XCTAssertTrue(response.ok, String(describing: response.error))
68+
XCTAssertEqual(response.data?.textEntryRoute, "xctest-element")
69+
XCTAssertEqual(field.value as? String, "hello")
70+
try XCTSkipUnless(
71+
app.keyboards.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout),
72+
"software keyboard is down: this simulator did not raise the keyboard the focus tap asked for"
73+
)
74+
}
2475
#endif
2576
}

0 commit comments

Comments
 (0)