Skip to content

Commit 8a3931a

Browse files
committed
docs(evidence): record rejected faster fill clearing
1 parent 6a47e61 commit 8a3931a

2 files changed

Lines changed: 184 additions & 0 deletions

File tree

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
{
2+
"sourceCommit": "6a47e6177bf91be8ce1896b5dc789d1fe1ba9e4b",
3+
"upstream": "https://github.com/tester-army/e2e/pull/1236",
4+
"app": "dev.e2e.benchmark",
5+
"device": "BC54AC11-CD86-4C10-8C4E-5101C773FFEB",
6+
"os": "iOS 27.0",
7+
"cliVersion": "0.21.25",
8+
"baseline": {
9+
"samples": [
10+
{
11+
"kind": "secure-replace",
12+
"ms": 3206.373416999999,
13+
"ok": true
14+
},
15+
{
16+
"kind": "ordinary-replace",
17+
"ms": 3818.350292000003,
18+
"ok": true
19+
},
20+
{
21+
"kind": "secure-replace",
22+
"ms": 3225.345207999999,
23+
"ok": true
24+
},
25+
{
26+
"kind": "ordinary-replace",
27+
"ms": 3774.5687499999985,
28+
"ok": true
29+
},
30+
{
31+
"kind": "secure-replace",
32+
"ms": 3218.5184169999993,
33+
"ok": true
34+
},
35+
{
36+
"kind": "ordinary-replace",
37+
"ms": 3638.0006250000006,
38+
"ok": true
39+
},
40+
{
41+
"kind": "secure-replace",
42+
"ms": 3219.4461249999877,
43+
"ok": true
44+
},
45+
{
46+
"kind": "ordinary-replace",
47+
"ms": 3995.6581250000017,
48+
"ok": true
49+
}
50+
],
51+
"nativePasswordPhases": [
52+
{
53+
"commandId": "runner-b82cbf9c-4242-4e9a-a3de-c167caec3ae3",
54+
"phasesMs": {
55+
"focus": 554.8,
56+
"initial-resolve": 71.7,
57+
"clear": 1098.6,
58+
"type-first": 318.6,
59+
"warmup": 66.7,
60+
"type-remaining": 439.2,
61+
"verify": 46.8,
62+
"total": 2096.0
63+
}
64+
},
65+
{
66+
"commandId": "runner-1882b089-67a8-44f7-ae83-cbb6f3810599",
67+
"phasesMs": {
68+
"focus": 569.7,
69+
"initial-resolve": 67.4,
70+
"clear": 1092.0,
71+
"type-first": 309.9,
72+
"warmup": 63.7,
73+
"type-remaining": 456.6,
74+
"verify": 45.4,
75+
"total": 2081.0
76+
}
77+
},
78+
{
79+
"commandId": "runner-192297dc-c264-4059-9118-a786f6837cfd",
80+
"phasesMs": {
81+
"focus": 544.9,
82+
"initial-resolve": 64.1,
83+
"clear": 1100.5,
84+
"type-first": 307.6,
85+
"warmup": 79.1,
86+
"type-remaining": 472.1,
87+
"verify": 45.9,
88+
"total": 2119.7
89+
}
90+
},
91+
{
92+
"commandId": "runner-8b702125-ca4f-4b6f-a8e7-cf23438f6c14",
93+
"phasesMs": {
94+
"focus": 544.6,
95+
"initial-resolve": 67.2,
96+
"clear": 1096.8,
97+
"type-first": 318.8,
98+
"warmup": 68.9,
99+
"type-remaining": 449.1,
100+
"verify": 45.5,
101+
"total": 2098.2
102+
}
103+
}
104+
],
105+
"nativeTotalExcludesFocus": true
106+
},
107+
"rejectedCandidate": {
108+
"samples": [
109+
{
110+
"kind": "secure-replace",
111+
"ms": 2553.417874999999,
112+
"ok": false
113+
}
114+
],
115+
"nativePasswordPhases": [
116+
{
117+
"commandId": "runner-10f9f1f5-956d-41a4-b180-cd776eac14d7",
118+
"phasesMs": {
119+
"focus": 562.7,
120+
"initial-resolve": 62.3,
121+
"clear": 709.9,
122+
"type-first": 307.9,
123+
"warmup": 72.6,
124+
"type-remaining": 450.8,
125+
"verify": 47.3,
126+
"total": 1696.8
127+
}
128+
}
129+
],
130+
"passwordLoginVerified": false,
131+
"failure": "The next snapshot routed to com.apple.SafariViewService and failed with invalid-viewport; the surface-selection cause is unproven. Timer is provisional, not a successful fill speed claim."
132+
},
133+
"rejectedHarnessRuns": {
134+
"baseline": "interactive-only snapshot omitted success-message",
135+
"selectall": "retained baseline runner reused; not candidate evidence"
136+
},
137+
"guardTest": {
138+
"newTest": "testExperimentalClearMustRemoveTheWholeMaskedValueWhenSelectAllIsIgnored",
139+
"otherTests": [
140+
"testFillFailsWhenAppReplacesInputBeforeTextIsDelivered",
141+
"testFillFailsWhenAppRemovesInputBeforeTextIsDelivered"
142+
],
143+
"candidate": {
144+
"passedTests": 2,
145+
"failedTests": 1,
146+
"skippedTests": 0,
147+
"testFailures": [
148+
{
149+
"failureText": "XCTAssertFalse failed - fill reported success while the app retained stale text",
150+
"targetName": "AgentDeviceRunnerUITests",
151+
"testIdentifier": 1,
152+
"testIdentifierString": "RunnerTests/testExperimentalClearMustRemoveTheWholeMaskedValueWhenSelectAllIsIgnored()",
153+
"testIdentifierURL": "test://com.apple.xcode/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests/testExperimentalClearMustRemoveTheWholeMaskedValueWhenSelectAllIsIgnored",
154+
"testName": "testExperimentalClearMustRemoveTheWholeMaskedValueWhenSelectAllIsIgnored()"
155+
}
156+
]
157+
},
158+
"control": {
159+
"passedTests": 3,
160+
"failedTests": 0,
161+
"skippedTests": 0
162+
},
163+
"candidateExit": 65,
164+
"controlExit": 0,
165+
"controlRuntimeMatchesCommit": true
166+
},
167+
"candidateAndProbePatch": "diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m\nindex 03e77eac81..4329ddb1aa 100644\n--- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m\n+++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m\n@@ -69,6 +69,13 @@ @interface AgentDeviceMaskedTextField : UITextField\n \n @implementation AgentDeviceMaskedTextField\n \n+- (void)selectAll:(id)sender {\n+ if ([NSProcessInfo.processInfo.arguments containsObject:@\"--agent-device-ignore-select-all\"]) {\n+ return;\n+ }\n+ [super selectAll:sender];\n+}\n+\n - (NSString *)accessibilityValue {\n NSUInteger length = self.text.length;\n if (length == 0) {\n@@ -468,7 +475,10 @@ static NSTimeInterval AgentDeviceTextEntryAcknowledgeWindow(void) {\n \n - (void)agentDeviceTextEntryDidChange:(UITextField *)textField {\n if (self.maskedDeliveryStatus != nil) {\n- self.maskedDeliveryStatus.text = [textField.text hasSuffix:@\"not-a-secret-123\"] ? @\"delivered\" : @\"pending\";\n+ BOOL exact = [NSProcessInfo.processInfo.arguments containsObject:@\"--agent-device-exact-clear-probe\"];\n+ BOOL delivered = exact ? [textField.text isEqualToString:@\"not-a-secret-123\"]\n+ : [textField.text hasSuffix:@\"not-a-secret-123\"];\n+ self.maskedDeliveryStatus.text = delivered ? @\"delivered\" : @\"pending\";\n }\n // A field whose app owns its value, the way a controlled React Native `TextInput` does. A burst\n // typed faster than the app renders loses the characters that arrived while a render was in\ndiff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift\nindex 02b936bda3..870622a333 100644\n--- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift\n+++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift\n@@ -254,6 +254,11 @@ extension RunnerTests {\n existing.isEmpty {\n return\n }\n+#if os(iOS)\n+ element.typeKey(\"a\", modifierFlags: .command)\n+ element.typeText(XCUIKeyboardKey.delete.rawValue)\n+ return\n+#endif\n #if !os(tvOS)\n moveCaretToEnd(app: app, element: element)\n #endif\ndiff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift\nindex ef7475e197..df0201e020 100644\n--- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift\n+++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift\n@@ -4,6 +4,35 @@ import XCTest\n // `textEntryMode: \"append\"`, and the bare submit key with no mode.\n extension RunnerTests {\n #if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS)\n+ @MainActor\n+ func testExperimentalClearMustRemoveTheWholeMaskedValueWhenSelectAllIsIgnored() throws {\n+ app.launchArguments = [\n+ \"--agent-device-text-entry-regression\",\n+ \"--agent-device-text-entry-masked-value\",\n+ \"--agent-device-text-entry-prefilled\",\n+ \"--agent-device-ignore-select-all\",\n+ \"--agent-device-exact-clear-probe\",\n+ ]\n+ app.launch()\n+ defer { invalidateCachedTarget(reason: \"unit_test_cleanup\"); app.terminate() }\n+ XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout))\n+ let field = app.textFields[\"agent-device-hardware-keyboard-input\"]\n+ XCTAssertTrue(field.waitForExistence(timeout: appExistenceTimeout))\n+ try tapHardwareKeyboardInput(commandId: \"tap-select-all-probe\")\n+ let response = executeTypeCommand(\n+ activeApp: app,\n+ command: try runnerCommandFixture(\n+ #\"{\"command\":\"type\",\"commandId\":\"fill-select-all-probe\",\"text\":\"not-a-secret-123\",\"textEntryMode\":\"replace\"}\"#\n+ )\n+ )\n+ let delivered = app.staticTexts[\"agent-device-masked-delivery\"]\n+ let exact = XCTNSPredicateExpectation(predicate: NSPredicate(format: \"label == %@\", \"delivered\"), object: delivered)\n+ let committed = XCTWaiter.wait(for: [exact], timeout: 2) == .completed\n+ XCTAssertFalse(response.ok && !committed, \"fill reported success while the app retained stale text\")\n+ XCTAssertTrue(response.ok, String(describing: response.error))\n+ XCTAssertTrue(committed, \"the app did not receive exactly the replacement\")\n+ }\n+\n @MainActor\n func testTypeWithoutResolvedInputReturnsTypedFailureBeforeDispatchingText() throws {\n let command = try runnerCommandFixture(\n",
168+
"reproduction": {
169+
"setup": "Create a disposable checkout at sourceCommit; run pnpm install --frozen-lockfile && pnpm build. Apply candidateAndProbePatch. Build iOS with AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS=1 and a distinct AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH. For control, remove only the five-line Select All runtime change; keep the probe fixture/test patch unchanged. Build into another DerivedData directory. Run sequentially with no competing device processes.",
170+
"testCommand": "xcodebuild test-without-building -xctestrun <derived>/Build/Products/AgentDeviceRunner_AgentDeviceRunnerUITests_iphonesimulator27.1-arm64.xctestrun -destination platform=iOS\\ Simulator,id=BC54AC11-CD86-4C10-8C4E-5101C773FFEB -parallel-testing-enabled NO -only-testing:AgentDeviceRunnerUITests/RunnerTests/testExperimentalClearMustRemoveTheWholeMaskedValueWhenSelectAllIsIgnored -only-testing:AgentDeviceRunnerUITests/RunnerTests/testFillFailsWhenAppReplacesInputBeforeTextIsDelivered -only-testing:AgentDeviceRunnerUITests/RunnerTests/testFillFailsWhenAppRemovesInputBeforeTextIsDelivered"
171+
}
172+
}

‎docs/evidence/ios-text-daemon-latency-2026-10-19.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,18 @@ The first text baseline was interrupted by this investigation's cleanup and cont
7171

7272
This follow-up does not rerun the full upstream 42-case suite or test a physical device. It does not establish a general speed lead over the Swift engine. The earlier cross-engine report remains the comparison for the full suite and screenshots. Field clearing still costs about 1.1–1.2 seconds in these samples; it is the larger remaining text-entry cost.
7373

74+
## Follow-up: can replacement reach two seconds?
75+
76+
The next experiment replaced the caret edge-tap and delete burst with public, element-scoped XCTest calls: `typeKey("a", modifierFlags: .command)` followed by one delete. The warmup split, binding, and final read-back stayed unchanged. The candidate ran only in a disposable checkout. No further runtime optimization was accepted.
77+
78+
This follow-up used T3's installed `0.21.25` CLI, with explicit runner artifact paths, on the same upstream app and simulator. It therefore has a separate baseline from the SDK measurements above. Four baseline password replacements had a median of 3,219.0 ms, and all four reached successful login. Their median native clearing phase was 1,097.7 ms. A confirmed fresh candidate runner returned one replacement in 2,553.4 ms, with a 709.9 ms clearing phase. The next snapshot was routed to `com.apple.SafariViewService` and failed with an `invalid-viewport` timeout, so login was not verified. The evidence does not establish why that surface was selected. That timer is provisional and does not establish a successful-fill speed improvement.
79+
80+
The native safety comparison rejected the shortcut independently of that capture failure. A masked, prefilled fixture ignored Select All and exposed a separate app-owned exact-match indicator. The candidate returned success while that indicator showed that stale text remained. The candidate passed the existing two early-navigation regressions but failed the new whole-value replacement case: **2 passed, 1 failed**. The same fixture, test selection, and environment with the current clearing implementation passed **3 of 3**. This control's runtime source matched commit `6a47e6177bf91be8ce1896b5dc789d1fe1ba9e4b` byte-for-byte.
81+
82+
The [samples, phase timings, complete candidate/probe patch, and reproduction command](ios-fill-clear-latency-2026-10-19.json) preserve this result. An initial harness attempt omitted noninteractive success text; another candidate attempt reused the retained baseline runner. Both are excluded. The valid candidate was confirmed by its fresh xcodebuild artifact path and logged Command-A dispatches. The simulator was restarted between artifacts and shut down after the experiments.
83+
84+
There is no demonstrated safe two-second replacement here. Select All does not prove that a masked input was cleared, and final literal verification cannot supply that proof. The next design work is an identity-bound clear operation with observable completion, plus consolidating duplicate native reads within each observation. Keep the existing opaque-field clearing path until that evidence exists. Removing the first-character guard or globally increasing typing speed does not address these failures.
85+
7486
## Validation and reproduction
7587

7688
Base: `8f3eb3d2efde3654a560fddb39127d556e068352`, with the preserved implementation patch. Device: iPhone 17 Pro, iOS 27.0, `BC54AC11-CD86-4C10-8C4E-5101C773FFEB`. Host: M4 Pro, Xcode 27.1, Node 26.11.1. The live runs used the worktree patch; the PR records the published validation commit.

0 commit comments

Comments
 (0)