Skip to content

Commit e5bfde3

Browse files
authored
diagnose(1874): instrument the synthesized commit wait and add a dispatchable stall loop (#1941)
* diagnose(1874): instrument synthesized commit wait and add stall loop workflow * diagnose(1874): fix empty-array expansion under set -u; raise default iterations * diagnose(1874): add arm64 matrix leg to isolate the Rosetta factor * ci: build the iOS runner for the native arm64 slice A generic simulator destination leaves the active arch undefined; Xcode 26.6 defaults it to x86_64, running the whole runner under Rosetta on arm64 hosts. Pin ARCHS=arm64 across every lane that builds the iOS runner and bump the derived-data cache suffixes. Measured ~30% faster commits on identical CI hardware; delivery-throttle episodes still occur but start from a lower base. * diagnose(1874): keep commit-wait cadence evidence value-free The per-poll trace logged the observed field's contents (prefix(40)) on the shipped type path; that value is user content and runner.log persists. Log lengths and the expected-prefix walk instead, allowlist every string-interpolating NSLog format in the module behind a source-scan guard, and pin commonPrefixLength in the host-lane policy tests. * diagnose(1874): narrow the log-format match for typecheck * diagnose(1874): route cadence evidence through a typed value-free boundary logCommitCadence accepts Int lengths and a timestamp only, so observed field contents are unrepresentable at the poll call site; its emitted line is pinned by a sentinel-secret test in the host-lane policy tests. The source guard becomes structural — boundary present, poll path logs through it, no raw NSLog in the observe closure — instead of parsing Swift format strings. #1874 is reopened as the removal-tracking thread for this temporary instrumentation.
1 parent cb65d6c commit e5bfde3

10 files changed

Lines changed: 303 additions & 8 deletions

File tree

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
name: 1874 Diagnose
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
iterations:
7+
description: 'Loop iterations per job'
8+
required: false
9+
default: '25'
10+
push:
11+
branches:
12+
- diagnose/1874-commit-stall
13+
14+
permissions:
15+
contents: read
16+
17+
concurrency:
18+
group: ${{ github.workflow }}-${{ github.ref }}
19+
cancel-in-progress: true
20+
21+
jobs:
22+
stall-loop:
23+
name: ${{ matrix.arch }} ${{ matrix.mode }} stall loop
24+
runs-on: macos-26
25+
timeout-minutes: 90
26+
strategy:
27+
fail-fast: false
28+
matrix:
29+
mode: [isolated, pair]
30+
arch: [x86_64, arm64]
31+
env:
32+
IOS_RUNTIME_VERSION: '26.2'
33+
AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived
34+
AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS: '1'
35+
AGENT_DEVICE_XCUITEST_ARCHS: ${{ matrix.arch }}
36+
steps:
37+
- uses: actions/checkout@v4
38+
39+
- name: Setup toolchain
40+
uses: ./.github/actions/setup-node-pnpm
41+
42+
- name: Restore and build iOS XCTest runner
43+
uses: ./.github/actions/setup-apple-runner-build
44+
with:
45+
derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }}
46+
cache-key-prefix: ios-runner-prebuilt
47+
cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}-1874-${{ matrix.arch }}
48+
gate: swift-runner-ios
49+
xcuitest-platform: ios
50+
xcuitest-destination: generic/platform=iOS Simulator
51+
52+
- name: Boot iOS test simulator
53+
id: ios-simulator
54+
uses: ./.github/actions/boot-ios-test-simulator
55+
with:
56+
runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
57+
preferred-device-name: iPhone 17 Pro
58+
59+
- name: Stall loop
60+
run: |
61+
set -euo pipefail
62+
XCTESTRUN_PATH="$(find "$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Build/Products" -maxdepth 1 -name '*.xctestrun' -print -quit)"
63+
test -n "$XCTESTRUN_PATH"
64+
UDID="${{ steps.ios-simulator.outputs.simulator-udid }}"
65+
ITER="${{ github.event.inputs.iterations || '25' }}"
66+
67+
if [ "${{ matrix.mode }}" = "pair" ]; then
68+
EXTRA_ONLY_TESTING=(-only-testing:AgentDeviceRunnerUITests/RunnerTests/testBareDelayedTypeFailsWhenTappedInputDisappearsMidCommand)
69+
else
70+
EXTRA_ONLY_TESTING=()
71+
fi
72+
73+
sys_prof="$(system_profiler SPHardwareDataType 2>/dev/null | grep -E 'Chip|Cores|Memory' || true)"
74+
echo "HOST: $sys_prof" | tee stall-summary.txt
75+
76+
pass=0; fail=0
77+
for i in $(seq 1 "$ITER"); do
78+
LOG=".tmp/stall-run-$i.log"
79+
set +e
80+
xcodebuild test-without-building \
81+
-xctestrun "$XCTESTRUN_PATH" \
82+
-destination "platform=iOS Simulator,id=$UDID" \
83+
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testBareTypeUsesTappedInputWhenSoftwareKeyboardIsHidden \
84+
${EXTRA_ONLY_TESTING[@]+"${EXTRA_ONLY_TESTING[@]}"} \
85+
> "$LOG" 2>&1
86+
rc=$?
87+
set -e
88+
if grep -q "testBareTypeUsesTappedInputWhenSoftwareKeyboardIsHidden]' passed" "$LOG"; then
89+
pass=$((pass+1)); verdict=pass
90+
else
91+
fail=$((fail+1)); verdict=STALL
92+
fi
93+
phase="$(grep -o 'phase=type-all durationMs=[0-9.]*' "$LOG" | tail -1)"
94+
polls="$(grep -c 'DEBUG-1874] poll' "$LOG" || true)"
95+
echo "iter=$i verdict=$verdict rc=$rc $phase polls=$polls" | tee -a stall-summary.txt
96+
if [ "$verdict" = "STALL" ]; then
97+
grep 'DEBUG-1874' "$LOG" | head -40 | tee -a stall-summary.txt
98+
cp "$LOG" ".tmp/stall-failure-$i.log"
99+
fi
100+
done
101+
102+
echo "RESULT arch=${{ matrix.arch }} mode=${{ matrix.mode }}: $pass passed, $fail stalled of $ITER" | tee -a stall-summary.txt
103+
104+
- name: Upload stall evidence
105+
if: always()
106+
uses: actions/upload-artifact@v4
107+
with:
108+
name: stall-evidence-${{ matrix.arch }}-${{ matrix.mode }}
109+
path: |
110+
stall-summary.txt
111+
.tmp/stall-failure-*.log
112+
if-no-files-found: ignore

‎.github/workflows/conformance-differential.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ jobs:
4545
timeout-minutes: 90
4646
env:
4747
IOS_RUNTIME_VERSION: '26.2'
48+
AGENT_DEVICE_XCUITEST_ARCHS: 'arm64'
4849
AGENT_DEVICE_STATE_DIR: ${{ github.workspace }}/.tmp/agent-device-state
4950
AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived
5051
steps:
@@ -59,7 +60,7 @@ jobs:
5960
with:
6061
derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }}
6162
cache-key-prefix: ios-runner-prebuilt
62-
cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}
63+
cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}-arm64
6364
gate: swift-runner-ios
6465
xcuitest-platform: ios
6566
xcuitest-destination: generic/platform=iOS Simulator

‎.github/workflows/ios.yml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,11 @@ jobs:
7474
AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived
7575
AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS: '420000'
7676
AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS: '1'
77+
# A generic simulator destination leaves the active arch undefined and Xcode 26.6
78+
# defaults it to x86_64, which runs the whole runner under Rosetta on arm64 hosts.
79+
# Pin the native slice: measured ~30% faster commits on identical CI hardware
80+
# (#1874 diagnosis, PR #1941).
81+
AGENT_DEVICE_XCUITEST_ARCHS: 'arm64'
7782
AGENT_DEVICE_IOS_APP_EVENT_URL_TEMPLATE: agent-device-test-app:///automation?event={event}&payload={payload}
7883
steps:
7984
- name: Checkout
@@ -113,7 +118,7 @@ jobs:
113118
with:
114119
derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }}
115120
cache-key-prefix: ios-runner-prebuilt
116-
cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}
121+
cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}-arm64
117122
gate: swift-runner-ios
118123
xcuitest-platform: ios
119124
xcuitest-destination: generic/platform=iOS Simulator

‎.github/workflows/perf-nightly.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ jobs:
3232
timeout-minutes: 80
3333
env:
3434
IOS_RUNTIME_VERSION: '26.2'
35+
AGENT_DEVICE_XCUITEST_ARCHS: 'arm64'
3536
AGENT_DEVICE_STATE_DIR: ${{ github.workspace }}/.tmp/agent-device-state
3637
AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived
3738
AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS: '420000'
@@ -47,7 +48,7 @@ jobs:
4748
with:
4849
derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }}
4950
cache-key-prefix: ios-runner-prebuilt
50-
cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}
51+
cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}-arm64
5152
gate: swift-runner-ios
5253
xcuitest-platform: ios
5354
xcuitest-destination: generic/platform=iOS Simulator

‎.github/workflows/replays-manual.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ jobs:
9191
timeout-minutes: 80
9292
env:
9393
IOS_RUNTIME_VERSION: '26.2'
94+
AGENT_DEVICE_XCUITEST_ARCHS: 'arm64'
9495
AGENT_DEVICE_STATE_DIR: ${{ github.workspace }}/.tmp/agent-device-state
9596
AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived
9697
AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS: '420000'
@@ -111,7 +112,7 @@ jobs:
111112
with:
112113
derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }}
113114
cache-key-prefix: ios-runner-prebuilt
114-
cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}
115+
cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}-arm64
115116
gate: swift-runner-ios
116117
xcuitest-platform: ios
117118
xcuitest-destination: generic/platform=iOS Simulator

‎.github/workflows/xctest-nightly.yml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,8 +78,10 @@ jobs:
7878
# methods exist at all. Dropping it would build a different variant (cache miss) whose
7979
# bundle contains no tests, and `xcodebuild` reports running zero tests as success — the
8080
# "silent 0-test green" this lane would then be. The test-count assertion below is the
81-
# backstop for exactly that.
81+
# backstop for exactly that. AGENT_DEVICE_XCUITEST_ARCHS rides the cache key too: dropping
82+
# it silently reverts the runner to Xcode's x86_64 default under Rosetta.
8283
IOS_RUNTIME_VERSION: '26.2'
84+
AGENT_DEVICE_XCUITEST_ARCHS: 'arm64'
8385
AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS: '1'
8486
AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived
8587
RESULT_BUNDLE_PATH: ${{ github.workspace }}/.tmp/xctest-nightly/RunnerTests.xcresult
@@ -103,7 +105,7 @@ jobs:
103105
with:
104106
derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }}
105107
cache-key-prefix: ios-runner-prebuilt
106-
cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}
108+
cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}-arm64
107109
gate: swift-runner-ios
108110
xcuitest-platform: ios
109111
xcuitest-destination: generic/platform=iOS Simulator

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift‎

Lines changed: 65 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,14 @@ extension RunnerTests {
4040
replacingExistingText: Bool
4141
) -> SynthesizedTextEntryAction {
4242
#if os(iOS)
43+
let postStartedAt = Date()
4344
let result = replacingExistingText
4445
? RunnerSynthesizedTextEntry.replaceText(withApplication: app, text: text)
4546
: RunnerSynthesizedTextEntry.synthesizeText(withApplication: app, text: text)
47+
NSLog(
48+
"[DEBUG-1874] synthesize posted %d chars status=%d tookMs=%.0f",
49+
text.count, result.status.rawValue, postStartedAt.timeIntervalSinceNow * -1000
50+
)
4651
return Self.action(status: result.status, message: result.message)
4752
#else
4853
return .fallback
@@ -184,6 +189,47 @@ extension RunnerTests {
184189
return expectedText.hasPrefix(observedText) ? .pending : .diverged
185190
}
186191

192+
/// Length of the shared prefix of two strings. Feeds value-free commit-wait logging: the
193+
/// expected-prefix walk over time distinguishes throttled delivery (grows slowly) from a
194+
/// wedged pipeline (freezes) without ever logging the field's contents.
195+
static func commonPrefixLength(_ lhs: String, _ rhs: String) -> Int {
196+
var length = 0
197+
for (l, r) in zip(lhs, rhs) {
198+
if l != r { break }
199+
length += 1
200+
}
201+
return length
202+
}
203+
204+
/// The emitted cadence line, as a pure function so its output is assertable. Only lengths and
205+
/// a timestamp are representable here; there is no String parameter, so observed field
206+
/// contents cannot reach runner.log through this boundary whatever they contain.
207+
static func commitCadenceLogLine(
208+
elapsedMs: Int,
209+
observedLen: Int,
210+
expectedPrefixLen: Int
211+
) -> String {
212+
"[DEBUG-1874] poll t=\(elapsedMs)ms observedLen=\(observedLen) expectedPrefixLen=\(expectedPrefixLen)"
213+
}
214+
215+
/// The typed boundary for commit-wait cadence evidence. The poll path must log through this
216+
/// function and never through a raw NSLog: every parameter is an Int, so the polled value's
217+
/// contents are unrepresentable at the call site.
218+
static func logCommitCadence(
219+
elapsedMs: Int,
220+
observedLen: Int,
221+
expectedPrefixLen: Int
222+
) {
223+
NSLog(
224+
"%@",
225+
commitCadenceLogLine(
226+
elapsedMs: elapsedMs,
227+
observedLen: observedLen,
228+
expectedPrefixLen: expectedPrefixLen
229+
)
230+
)
231+
}
232+
187233
/// How the commit wait ended. Distinct from `SynthesizedTextCommitProgress`, which classifies a
188234
/// single observation: this is the whole wait's verdict, and it exists so the deadline can be
189235
/// told apart from success. The wait used to return `Void`, which made an expired deadline
@@ -265,20 +311,37 @@ extension RunnerTests {
265311
let expectedText = textBefore + typedText
266312
let placeholder = resolveTextEntryElement(app: app, target: target)?.placeholderValue
267313
let deadline = Date().addingTimeInterval(TextEntryTiming.synthesizedCommitTimeout)
268-
return Self.awaitSynthesizedCommitOutcome(
314+
let waitStartedAt = Date()
315+
NSLog("[DEBUG-1874] wait start expectedLen=%ld", expectedText.count)
316+
let outcome = Self.awaitSynthesizedCommitOutcome(
269317
expectedText: expectedText,
270318
placeholder: placeholder,
271319
isExpired: { Date() >= deadline },
272320
observe: {
273-
editableTextValue(
321+
let observedText = editableTextValue(
274322
for: resolveTextEntryElement(app: app, target: target),
275323
treatingPlaceholderAsEmpty: true
276324
)
325+
// Cadence evidence stays value-free: the polled value is user content typed through
326+
// `type` and must never reach runner.log. Lengths and the expected-prefix walk are
327+
// enough to distinguish throttling (prefix grows slowly) from a wedge (it freezes).
328+
Self.logCommitCadence(
329+
elapsedMs: Int(waitStartedAt.timeIntervalSinceNow * -1000),
330+
observedLen: observedText?.count ?? -1,
331+
expectedPrefixLen: observedText.map { Self.commonPrefixLength($0, expectedText) } ?? -1
332+
)
333+
return observedText
277334
},
278335
// XCUI resolution shares the automation channel with the in-flight synthesized event.
279336
// Sparse reads let the target consume that event instead of continuously interrupting it.
280337
waitForNextObservation: { sleepFor(TextEntryTiming.synthesizedCommitPollInterval) }
281338
)
339+
NSLog(
340+
"[DEBUG-1874] wait outcome=%@ elapsedMs=%.0f",
341+
String(describing: outcome),
342+
waitStartedAt.timeIntervalSinceNow * -1000
343+
)
344+
return outcome
282345
}
283346

284347
static func shouldUseResolvedCoordinateTextEntryRoute(

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -308,6 +308,31 @@ extension RunnerTests {
308308
XCTAssertFalse(result.repaired)
309309
XCTAssertEqual(result.textEntryRoute, "synthesized-first-responder-replacement")
310310
}
311+
312+
func testCommonPrefixLengthWalksTheExpectedPrefixOnly() {
313+
XCTAssertEqual(Self.commonPrefixLength("hardware-keyboard", "hardware-keyboard"), 17)
314+
XCTAssertEqual(Self.commonPrefixLength("h", "hardware-keyboard"), 1)
315+
XCTAssertEqual(Self.commonPrefixLength("ha", "hardware-keyboard"), 2)
316+
XCTAssertEqual(Self.commonPrefixLength("", "hardware-keyboard"), 0)
317+
// Divergence stops the count: the app transformed the input, and the walk must not
318+
// resume matching after the first differing character.
319+
XCTAssertEqual(Self.commonPrefixLength("hx", "hardware-keyboard"), 1)
320+
XCTAssertEqual(Self.commonPrefixLength("hardware-keyboarx", "hardware-keyboard"), 15)
321+
}
322+
323+
func testCommitCadenceLogLineEmitsLengthsOnlyNeverContents() {
324+
// Sentinel secret: even when the polled field holds credential-shaped content, the only
325+
// channel into runner.log is this line, and its inputs are lengths. The exact-equality
326+
// assert fails if any content-bearing parameter or interpolation is ever added.
327+
let secret = "hunter2-typed-credential"
328+
let line = Self.commitCadenceLogLine(
329+
elapsedMs: 42,
330+
observedLen: secret.count,
331+
expectedPrefixLen: 8
332+
)
333+
XCTAssertEqual(line, "[DEBUG-1874] poll t=42ms observedLen=24 expectedPrefixLen=8")
334+
XCTAssertFalse(line.contains(secret))
335+
}
311336
#endif
312337
#endif
313338
}

‎scripts/build-xcuitest-apple.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,14 @@ if is_truthy "${AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS:-}"; then
138138
SWIFT_FLAGS="$SWIFT_FLAGS -D AGENT_DEVICE_RUNNER_UNIT_TESTS"
139139
fi
140140

141+
# Optional arch override. A generic simulator destination leaves the active arch
142+
# undefined; Xcode versions differ on the default (26.6 picks x86_64, which runs
143+
# under Rosetta on arm64 hosts). Set AGENT_DEVICE_XCUITEST_ARCHS=arm64 to pin it.
144+
ARCH_BUILD_SETTINGS=""
145+
if [ -n "${AGENT_DEVICE_XCUITEST_ARCHS:-}" ]; then
146+
ARCH_BUILD_SETTINGS="ARCHS=$AGENT_DEVICE_XCUITEST_ARCHS"
147+
fi
148+
141149
node --experimental-strip-types scripts/swift-toolchain-tmpdir.ts xcodebuild build-for-testing \
142150
-project "$PROJECT_PATH" \
143151
-scheme "$SCHEME" \
@@ -154,6 +162,7 @@ node --experimental-strip-types scripts/swift-toolchain-tmpdir.ts xcodebuild bui
154162
-IDEPackageSupportDisablePluginExecutionSandbox=1 \
155163
ENABLE_USER_SCRIPT_SANDBOXING=NO \
156164
OTHER_SWIFT_FLAGS="$SWIFT_FLAGS" \
165+
$ARCH_BUILD_SETTINGS \
157166
$SIGNING_BUILD_SETTINGS
158167

159168
node --experimental-strip-types scripts/patch-xcuitest-runner-icon.ts "$DERIVED_PATH"

0 commit comments

Comments
 (0)