Skip to content

Commit c9575ad

Browse files
committed
fix(ios): wait for hidden-keyboard synthesized text to commit before responding
The synthesized-first-responder bare-type route returned ok as soon as the private XCTest event record was posted, while the target app was still committing characters. On slow CI simulators the trailing characters landed after the response, so agents (and the smoke test) observed a truncated field value through the public type path. After dispatch, poll the tapped element until its value reaches textBefore + typedText, exit immediately when the app transforms the input (formatter, mid-text caret, autocomplete), and if progress stalls as a strict prefix, re-synthesize the missing tail once. Submit-suffixed text keeps the old immediate return so a repair can never double-submit. Validated on a booted iPhone 17 Pro simulator: 5/5 passes of testBareTypeUsesTappedInputWhenSoftwareKeyboardIsHidden under full-core CPU load, with the commit wait absorbing up to ~390ms of post-dispatch lag that the previous code ignored (uniform ~494ms dispatch-only before); the tail repair never had to fire, consistent with commit lag rather than true drops.
1 parent a158434 commit c9575ad

4 files changed

Lines changed: 167 additions & 0 deletions

File tree

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,106 @@ extension RunnerTests {
159159
repairMode == .none && fromTapWitness && !softwareKeyboardVisible
160160
}
161161

162+
enum SynthesizedTextCommitProgress: Equatable {
163+
case committed
164+
case pending
165+
case diverged
166+
}
167+
168+
// The private synthesize call returns once the event record is posted, not once the target
169+
// app has committed the characters, so intermediate reads walk prefix-by-prefix toward the
170+
// expected value. Anything off that prefix path means the app transformed the input
171+
// (formatter, mid-text caret, autocomplete) and the runner must not second-guess it.
172+
static func synthesizedTextCommitProgress(
173+
observedText: String?,
174+
expectedText: String
175+
) -> SynthesizedTextCommitProgress {
176+
guard let observedText else {
177+
return .diverged
178+
}
179+
if observedText == expectedText {
180+
return .committed
181+
}
182+
return expectedText.hasPrefix(observedText) ? .pending : .diverged
183+
}
184+
185+
static func synthesizedTextCommitRepairTail(
186+
observedText: String,
187+
expectedText: String
188+
) -> String? {
189+
guard expectedText.hasPrefix(observedText), observedText.count < expectedText.count else {
190+
return nil
191+
}
192+
let tail = String(expectedText.dropFirst(observedText.count))
193+
guard !tail.contains("\n"), !tail.contains("\r") else {
194+
return nil
195+
}
196+
return tail
197+
}
198+
199+
/// Blocks until the synthesized bare-type text is observable in the target field, so `type`
200+
/// cannot report ok while trailing characters are still uncommitted (or dropped) on a slow
201+
/// simulator. Exits fast when the app transforms the input; re-synthesizes the missing tail
202+
/// once if commit progress stalls as a strict prefix of the expected value.
203+
func awaitSynthesizedFirstResponderCommit(
204+
app: XCUIApplication,
205+
target: TextEntryTarget,
206+
textBefore: String?,
207+
typedText: String,
208+
synthesizer: any TextEntrySynthesizing
209+
) {
210+
guard let textBefore, !typedText.contains("\n"), !typedText.contains("\r") else {
211+
return
212+
}
213+
let expectedText = textBefore + typedText
214+
var repaired = false
215+
var lastObservedText: String?
216+
var lastChangeAt = Date()
217+
let deadline = Date().addingTimeInterval(TextEntryTiming.synthesizedCommitTimeout)
218+
while Date() < deadline {
219+
guard let observedText = editableTextValue(
220+
for: resolveTextEntryElement(app: app, target: target),
221+
treatingPlaceholderAsEmpty: true
222+
) else {
223+
return
224+
}
225+
switch Self.synthesizedTextCommitProgress(observedText: observedText, expectedText: expectedText) {
226+
case .committed, .diverged:
227+
return
228+
case .pending:
229+
break
230+
}
231+
if lastObservedText != observedText {
232+
lastObservedText = observedText
233+
lastChangeAt = Date()
234+
} else if Date().timeIntervalSince(lastChangeAt) >= TextEntryTiming.synthesizedCommitQuietWindow {
235+
guard !repaired,
236+
let tail = Self.synthesizedTextCommitRepairTail(
237+
observedText: observedText,
238+
expectedText: expectedText
239+
)
240+
else {
241+
return
242+
}
243+
NSLog(
244+
"AGENT_DEVICE_RUNNER_REPAIR_TEXT_ENTRY route=synthesized-first-responder expectedLength=%d observedLength=%d",
245+
expectedText.count,
246+
observedText.count
247+
)
248+
guard case .continueTyping = synthesizer.enterText(
249+
app: app,
250+
text: tail,
251+
replacingExistingText: false
252+
) else {
253+
return
254+
}
255+
repaired = true
256+
lastChangeAt = Date()
257+
}
258+
sleepFor(TextEntryTiming.pollInterval)
259+
}
260+
}
261+
162262
static func shouldUseResolvedCoordinateTextEntryRoute(
163263
repairMode: TextTypingRepairMode,
164264
hasX: Bool,

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ extension RunnerTests {
4040
static let pollInterval: TimeInterval = 0.02
4141
static let warmupValueTimeout: TimeInterval = 0.4
4242
static let verificationStabilityWindow: TimeInterval = 0.2
43+
static let synthesizedCommitTimeout: TimeInterval = 3.0
44+
static let synthesizedCommitQuietWindow: TimeInterval = 0.6
4345
}
4446

4547
struct TextEntryResult {

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryPolicyTests.swift‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,56 @@ extension RunnerTests {
6161
}
6262
}
6363

64+
func testSynthesizedTextCommitProgressWalksExpectedPrefixOnly() {
65+
let expected = "hardware-keyboard"
66+
XCTAssertEqual(
67+
Self.synthesizedTextCommitProgress(observedText: "hardware-keyboard", expectedText: expected),
68+
.committed
69+
)
70+
XCTAssertEqual(
71+
Self.synthesizedTextCommitProgress(observedText: "", expectedText: expected),
72+
.pending
73+
)
74+
XCTAssertEqual(
75+
Self.synthesizedTextCommitProgress(observedText: "hardware-keyboa", expectedText: expected),
76+
.pending
77+
)
78+
// Transformed input (formatter, mid-text caret, autocomplete) must stop the wait.
79+
XCTAssertEqual(
80+
Self.synthesizedTextCommitProgress(observedText: "hardwarX", expectedText: expected),
81+
.diverged
82+
)
83+
XCTAssertEqual(
84+
Self.synthesizedTextCommitProgress(observedText: "hardware-keyboards", expectedText: expected),
85+
.diverged
86+
)
87+
XCTAssertEqual(
88+
Self.synthesizedTextCommitProgress(observedText: nil, expectedText: expected),
89+
.diverged
90+
)
91+
}
92+
93+
func testSynthesizedTextCommitRepairTailOnlyForStrictPrefixWithoutSubmitKeys() {
94+
XCTAssertEqual(
95+
Self.synthesizedTextCommitRepairTail(observedText: "hardware-keyboa", expectedText: "hardware-keyboard"),
96+
"rd"
97+
)
98+
XCTAssertEqual(
99+
Self.synthesizedTextCommitRepairTail(observedText: "", expectedText: "abc"),
100+
"abc"
101+
)
102+
XCTAssertNil(
103+
Self.synthesizedTextCommitRepairTail(observedText: "hardware-keyboard", expectedText: "hardware-keyboard")
104+
)
105+
XCTAssertNil(
106+
Self.synthesizedTextCommitRepairTail(observedText: "hardwarX", expectedText: "hardware-keyboard")
107+
)
108+
// Never re-synthesize a tail that would submit.
109+
XCTAssertNil(
110+
Self.synthesizedTextCommitRepairTail(observedText: "ab", expectedText: "abc\n")
111+
)
112+
}
113+
64114
#if os(iOS)
65115
func testSynthesizedTextEntryFallsBackOnlyWhenPrivateSynthesisIsUnavailable() {
66116
XCTAssertEqual(

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,8 +111,23 @@ extension RunnerTests {
111111
{
112112
textEntryRoute = "synthesized-first-responder"
113113
NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=synthesized-first-responder")
114+
let textBefore = editableTextValue(for: currentTarget, treatingPlaceholderAsEmpty: true)
114115
switch synthesizer.enterText(app: app, text: value, replacingExistingText: false) {
115116
case .continueTyping:
117+
// No refresh point: like the tap-witness target itself, the commit wait must observe
118+
// only the element the tap selected, never rediscover a different field.
119+
awaitSynthesizedFirstResponderCommit(
120+
app: app,
121+
target: TextEntryTarget(
122+
element: currentTarget,
123+
refreshPoint: nil,
124+
prefersFocusedElement: false,
125+
fromTapWitness: true
126+
),
127+
textBefore: textBefore,
128+
typedText: value,
129+
synthesizer: synthesizer
130+
)
116131
return (currentTarget, true, nil)
117132
case .fallback:
118133
return (nil, false, .synthesisUnavailable)

0 commit comments

Comments
 (0)