|
| 1 | +{ |
| 2 | + "sourceCommit": "6a47e6177bf91be8ce1896b5dc789d1fe1ba9e4b", |
| 3 | + "upstream": "https://github.com/tester-army/e2e/pull/1236", |
| 4 | + "app": "dev.e2e.benchmark", |
| 5 | + "device": "BC54AC11-CD86-4C10-8C4E-5101C773FFEB", |
| 6 | + "os": "iOS 27.0", |
| 7 | + "cliVersion": "0.21.25", |
| 8 | + "baseline": { |
| 9 | + "samples": [ |
| 10 | + { |
| 11 | + "kind": "secure-replace", |
| 12 | + "ms": 3206.373416999999, |
| 13 | + "ok": true |
| 14 | + }, |
| 15 | + { |
| 16 | + "kind": "ordinary-replace", |
| 17 | + "ms": 3818.350292000003, |
| 18 | + "ok": true |
| 19 | + }, |
| 20 | + { |
| 21 | + "kind": "secure-replace", |
| 22 | + "ms": 3225.345207999999, |
| 23 | + "ok": true |
| 24 | + }, |
| 25 | + { |
| 26 | + "kind": "ordinary-replace", |
| 27 | + "ms": 3774.5687499999985, |
| 28 | + "ok": true |
| 29 | + }, |
| 30 | + { |
| 31 | + "kind": "secure-replace", |
| 32 | + "ms": 3218.5184169999993, |
| 33 | + "ok": true |
| 34 | + }, |
| 35 | + { |
| 36 | + "kind": "ordinary-replace", |
| 37 | + "ms": 3638.0006250000006, |
| 38 | + "ok": true |
| 39 | + }, |
| 40 | + { |
| 41 | + "kind": "secure-replace", |
| 42 | + "ms": 3219.4461249999877, |
| 43 | + "ok": true |
| 44 | + }, |
| 45 | + { |
| 46 | + "kind": "ordinary-replace", |
| 47 | + "ms": 3995.6581250000017, |
| 48 | + "ok": true |
| 49 | + } |
| 50 | + ], |
| 51 | + "nativePasswordPhases": [ |
| 52 | + { |
| 53 | + "commandId": "runner-b82cbf9c-4242-4e9a-a3de-c167caec3ae3", |
| 54 | + "phasesMs": { |
| 55 | + "focus": 554.8, |
| 56 | + "initial-resolve": 71.7, |
| 57 | + "clear": 1098.6, |
| 58 | + "type-first": 318.6, |
| 59 | + "warmup": 66.7, |
| 60 | + "type-remaining": 439.2, |
| 61 | + "verify": 46.8, |
| 62 | + "total": 2096.0 |
| 63 | + } |
| 64 | + }, |
| 65 | + { |
| 66 | + "commandId": "runner-1882b089-67a8-44f7-ae83-cbb6f3810599", |
| 67 | + "phasesMs": { |
| 68 | + "focus": 569.7, |
| 69 | + "initial-resolve": 67.4, |
| 70 | + "clear": 1092.0, |
| 71 | + "type-first": 309.9, |
| 72 | + "warmup": 63.7, |
| 73 | + "type-remaining": 456.6, |
| 74 | + "verify": 45.4, |
| 75 | + "total": 2081.0 |
| 76 | + } |
| 77 | + }, |
| 78 | + { |
| 79 | + "commandId": "runner-192297dc-c264-4059-9118-a786f6837cfd", |
| 80 | + "phasesMs": { |
| 81 | + "focus": 544.9, |
| 82 | + "initial-resolve": 64.1, |
| 83 | + "clear": 1100.5, |
| 84 | + "type-first": 307.6, |
| 85 | + "warmup": 79.1, |
| 86 | + "type-remaining": 472.1, |
| 87 | + "verify": 45.9, |
| 88 | + "total": 2119.7 |
| 89 | + } |
| 90 | + }, |
| 91 | + { |
| 92 | + "commandId": "runner-8b702125-ca4f-4b6f-a8e7-cf23438f6c14", |
| 93 | + "phasesMs": { |
| 94 | + "focus": 544.6, |
| 95 | + "initial-resolve": 67.2, |
| 96 | + "clear": 1096.8, |
| 97 | + "type-first": 318.8, |
| 98 | + "warmup": 68.9, |
| 99 | + "type-remaining": 449.1, |
| 100 | + "verify": 45.5, |
| 101 | + "total": 2098.2 |
| 102 | + } |
| 103 | + } |
| 104 | + ], |
| 105 | + "nativeTotalExcludesFocus": true |
| 106 | + }, |
| 107 | + "rejectedCandidate": { |
| 108 | + "samples": [ |
| 109 | + { |
| 110 | + "kind": "secure-replace", |
| 111 | + "ms": 2553.417874999999, |
| 112 | + "ok": false |
| 113 | + } |
| 114 | + ], |
| 115 | + "nativePasswordPhases": [ |
| 116 | + { |
| 117 | + "commandId": "runner-10f9f1f5-956d-41a4-b180-cd776eac14d7", |
| 118 | + "phasesMs": { |
| 119 | + "focus": 562.7, |
| 120 | + "initial-resolve": 62.3, |
| 121 | + "clear": 709.9, |
| 122 | + "type-first": 307.9, |
| 123 | + "warmup": 72.6, |
| 124 | + "type-remaining": 450.8, |
| 125 | + "verify": 47.3, |
| 126 | + "total": 1696.8 |
| 127 | + } |
| 128 | + } |
| 129 | + ], |
| 130 | + "passwordLoginVerified": false, |
| 131 | + "failure": "The next snapshot routed to com.apple.SafariViewService and failed with invalid-viewport; the surface-selection cause is unproven. Timer is provisional, not a successful fill speed claim." |
| 132 | + }, |
| 133 | + "rejectedHarnessRuns": { |
| 134 | + "baseline": "interactive-only snapshot omitted success-message", |
| 135 | + "selectall": "retained baseline runner reused; not candidate evidence" |
| 136 | + }, |
| 137 | + "guardTest": { |
| 138 | + "newTest": "testExperimentalClearMustRemoveTheWholeMaskedValueWhenSelectAllIsIgnored", |
| 139 | + "otherTests": [ |
| 140 | + "testFillFailsWhenAppReplacesInputBeforeTextIsDelivered", |
| 141 | + "testFillFailsWhenAppRemovesInputBeforeTextIsDelivered" |
| 142 | + ], |
| 143 | + "candidate": { |
| 144 | + "passedTests": 2, |
| 145 | + "failedTests": 1, |
| 146 | + "skippedTests": 0, |
| 147 | + "testFailures": [ |
| 148 | + { |
| 149 | + "failureText": "XCTAssertFalse failed - fill reported success while the app retained stale text", |
| 150 | + "targetName": "AgentDeviceRunnerUITests", |
| 151 | + "testIdentifier": 1, |
| 152 | + "testIdentifierString": "RunnerTests/testExperimentalClearMustRemoveTheWholeMaskedValueWhenSelectAllIsIgnored()", |
| 153 | + "testIdentifierURL": "test://com.apple.xcode/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests/testExperimentalClearMustRemoveTheWholeMaskedValueWhenSelectAllIsIgnored", |
| 154 | + "testName": "testExperimentalClearMustRemoveTheWholeMaskedValueWhenSelectAllIsIgnored()" |
| 155 | + } |
| 156 | + ] |
| 157 | + }, |
| 158 | + "control": { |
| 159 | + "passedTests": 3, |
| 160 | + "failedTests": 0, |
| 161 | + "skippedTests": 0 |
| 162 | + }, |
| 163 | + "candidateExit": 65, |
| 164 | + "controlExit": 0, |
| 165 | + "controlRuntimeMatchesCommit": true |
| 166 | + }, |
| 167 | + "candidateAndProbePatch": "diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m\nindex 03e77eac81..4329ddb1aa 100644\n--- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m\n+++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m\n@@ -69,6 +69,13 @@ @interface AgentDeviceMaskedTextField : UITextField\n \n @implementation AgentDeviceMaskedTextField\n \n+- (void)selectAll:(id)sender {\n+ if ([NSProcessInfo.processInfo.arguments containsObject:@\"--agent-device-ignore-select-all\"]) {\n+ return;\n+ }\n+ [super selectAll:sender];\n+}\n+\n - (NSString *)accessibilityValue {\n NSUInteger length = self.text.length;\n if (length == 0) {\n@@ -468,7 +475,10 @@ static NSTimeInterval AgentDeviceTextEntryAcknowledgeWindow(void) {\n \n - (void)agentDeviceTextEntryDidChange:(UITextField *)textField {\n if (self.maskedDeliveryStatus != nil) {\n- self.maskedDeliveryStatus.text = [textField.text hasSuffix:@\"not-a-secret-123\"] ? @\"delivered\" : @\"pending\";\n+ BOOL exact = [NSProcessInfo.processInfo.arguments containsObject:@\"--agent-device-exact-clear-probe\"];\n+ BOOL delivered = exact ? [textField.text isEqualToString:@\"not-a-secret-123\"]\n+ : [textField.text hasSuffix:@\"not-a-secret-123\"];\n+ self.maskedDeliveryStatus.text = delivered ? @\"delivered\" : @\"pending\";\n }\n // A field whose app owns its value, the way a controlled React Native `TextInput` does. A burst\n // typed faster than the app renders loses the characters that arrived while a render was in\ndiff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift\nindex 02b936bda3..870622a333 100644\n--- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift\n+++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift\n@@ -254,6 +254,11 @@ extension RunnerTests {\n existing.isEmpty {\n return\n }\n+#if os(iOS)\n+ element.typeKey(\"a\", modifierFlags: .command)\n+ element.typeText(XCUIKeyboardKey.delete.rawValue)\n+ return\n+#endif\n #if !os(tvOS)\n moveCaretToEnd(app: app, element: element)\n #endif\ndiff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift\nindex ef7475e197..df0201e020 100644\n--- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift\n+++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift\n@@ -4,6 +4,35 @@ import XCTest\n // `textEntryMode: \"append\"`, and the bare submit key with no mode.\n extension RunnerTests {\n #if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS)\n+ @MainActor\n+ func testExperimentalClearMustRemoveTheWholeMaskedValueWhenSelectAllIsIgnored() throws {\n+ app.launchArguments = [\n+ \"--agent-device-text-entry-regression\",\n+ \"--agent-device-text-entry-masked-value\",\n+ \"--agent-device-text-entry-prefilled\",\n+ \"--agent-device-ignore-select-all\",\n+ \"--agent-device-exact-clear-probe\",\n+ ]\n+ app.launch()\n+ defer { invalidateCachedTarget(reason: \"unit_test_cleanup\"); app.terminate() }\n+ XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout))\n+ let field = app.textFields[\"agent-device-hardware-keyboard-input\"]\n+ XCTAssertTrue(field.waitForExistence(timeout: appExistenceTimeout))\n+ try tapHardwareKeyboardInput(commandId: \"tap-select-all-probe\")\n+ let response = executeTypeCommand(\n+ activeApp: app,\n+ command: try runnerCommandFixture(\n+ #\"{\"command\":\"type\",\"commandId\":\"fill-select-all-probe\",\"text\":\"not-a-secret-123\",\"textEntryMode\":\"replace\"}\"#\n+ )\n+ )\n+ let delivered = app.staticTexts[\"agent-device-masked-delivery\"]\n+ let exact = XCTNSPredicateExpectation(predicate: NSPredicate(format: \"label == %@\", \"delivered\"), object: delivered)\n+ let committed = XCTWaiter.wait(for: [exact], timeout: 2) == .completed\n+ XCTAssertFalse(response.ok && !committed, \"fill reported success while the app retained stale text\")\n+ XCTAssertTrue(response.ok, String(describing: response.error))\n+ XCTAssertTrue(committed, \"the app did not receive exactly the replacement\")\n+ }\n+\n @MainActor\n func testTypeWithoutResolvedInputReturnsTypedFailureBeforeDispatchingText() throws {\n let command = try runnerCommandFixture(\n", |
| 168 | + "reproduction": { |
| 169 | + "setup": "Create a disposable checkout at sourceCommit; run pnpm install --frozen-lockfile && pnpm build. Apply candidateAndProbePatch. Build iOS with AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS=1 and a distinct AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH. For control, remove only the five-line Select All runtime change; keep the probe fixture/test patch unchanged. Build into another DerivedData directory. Run sequentially with no competing device processes.", |
| 170 | + "testCommand": "xcodebuild test-without-building -xctestrun <derived>/Build/Products/AgentDeviceRunner_AgentDeviceRunnerUITests_iphonesimulator27.1-arm64.xctestrun -destination platform=iOS\\ Simulator,id=BC54AC11-CD86-4C10-8C4E-5101C773FFEB -parallel-testing-enabled NO -only-testing:AgentDeviceRunnerUITests/RunnerTests/testExperimentalClearMustRemoveTheWholeMaskedValueWhenSelectAllIsIgnored -only-testing:AgentDeviceRunnerUITests/RunnerTests/testFillFailsWhenAppReplacesInputBeforeTextIsDelivered -only-testing:AgentDeviceRunnerUITests/RunnerTests/testFillFailsWhenAppRemovesInputBeforeTextIsDelivered" |
| 171 | + } |
| 172 | +} |
0 commit comments