Skip to content

Commit a5e49c8

Browse files
committed
ci(1874): declare the diagnose lane and read its iterations honestly
The loop that #1874 is investigated with could not tell the truth about itself. It classified every non-`passed` iteration as a stall, which after #2035 gave the looped test an XCTSkipIf meant an environment flip would report a 100% stall rate; it captured cadence only for failures, though an absorbed episode now passes; and it read its logs with shell pipelines whose exit status means "did this match", so an iteration that legitimately matched nothing killed the job before it could be summarized. scripts/diagnose-1874-iteration.ts reads one iteration: xcodebuild's own verdict, the `type-all` duration, and the cadence worth keeping. A nonzero exit outranks a green measured test — in `pair` mode the neighbour or the runner can fail while the measured test passes — and a run that produced no verdict is named as ours rather than counted as a stall. The workflow gains the #1781 lane declaration it never had, including a kill criterion tied to #2080, the investigation it serves. Closes #1874. Both filed symptoms are resolved. `smoke:form-input` was root-caused and fixed in #2035: the fixture's placeholder was identical to the value every suite filled, so `fill` could never be verified on the penalized route — deterministic, not a flake, and only visible under load because that route is gated on a penalized XCTest channel. The targeted XCTest is mitigated by the progress-aware commit budget, with 200 consecutive green loop iterations across two dispatches. The issue's remaining question — why the input pipeline throttles — is answered by the second dispatch, and the premise was wrong: it does not. Posting 17 characters took 484 ms and the commit was observed on the first poll, inside an iteration whose `type-all` measured 14334 ms. The ~12.6 s went to accessibility round-trips before any character was posted, which is #1105's path, not the input pipeline's.
1 parent 77b5be8 commit a5e49c8

4 files changed

Lines changed: 266 additions & 16 deletions

File tree

‎.github/workflows/1874-diagnose.yml‎

Lines changed: 31 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,21 @@
11
name: 1874 Diagnose
22

3+
# Reproduces the #1874 text-entry commit episode on CI hardware: loops the test the issue records,
4+
# on both arches, with and without the neighbour test that shares the simulator.
5+
#
6+
# Under the lane rule in #1781:
7+
#
8+
# - Catches: iOS text-entry commit episodes. The commit wait absorbs an episode that keeps making
9+
# progress, so one surfaces as a slow pass rather than a failure; this loop records the
10+
# per-iteration cadence either way.
11+
# - Evidence: the 2026-08-21 dispatch passed 50/50 and still caught an episode (`type-all`
12+
# 2487 ms across 5 polls) that no PR lane would report; it does not reproduce locally.
13+
# - Cost: manual dispatch and the `diagnose/1874-commit-stall` branch only — never a PR or `main`.
14+
# One dispatch is four macOS slots, each a runner build plus `iterations` iterations of
15+
# simulator time.
16+
# - Kill criterion: delete this file when #2080 closes; it is that investigation's reproduction
17+
# path and nothing else reads its artifacts.
18+
319
on:
420
workflow_dispatch:
521
inputs:
@@ -63,6 +79,8 @@ jobs:
6379
test -n "$XCTESTRUN_PATH"
6480
UDID="${{ steps.ios-simulator.outputs.simulator-udid }}"
6581
ITER="${{ github.event.inputs.iterations || '25' }}"
82+
TEST_NAME=testBareTypeUsesTappedInputWhenSoftwareKeyboardIsHidden
83+
mkdir -p .tmp
6684
6785
if [ "${{ matrix.mode }}" = "pair" ]; then
6886
EXTRA_ONLY_TESTING=(-only-testing:AgentDeviceRunnerUITests/RunnerTests/testBareDelayedTypeFailsWhenTappedInputDisappearsMidCommand)
@@ -73,33 +91,30 @@ jobs:
7391
sys_prof="$(system_profiler SPHardwareDataType 2>/dev/null | grep -E 'Chip|Cores|Memory' || true)"
7492
echo "HOST: $sys_prof" | tee stall-summary.txt
7593
76-
pass=0; fail=0
94+
pass=0; skipped=0; failed=0; noresult=0; runfailed=0
7795
for i in $(seq 1 "$ITER"); do
7896
LOG=".tmp/stall-run-$i.log"
7997
set +e
8098
xcodebuild test-without-building \
8199
-xctestrun "$XCTESTRUN_PATH" \
82100
-destination "platform=iOS Simulator,id=$UDID" \
83-
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testBareTypeUsesTappedInputWhenSoftwareKeyboardIsHidden \
101+
-only-testing:"AgentDeviceRunnerUITests/RunnerTests/$TEST_NAME" \
84102
${EXTRA_ONLY_TESTING[@]+"${EXTRA_ONLY_TESTING[@]}"} \
85103
> "$LOG" 2>&1
86104
rc=$?
87105
set -e
88-
if grep -q "testBareTypeUsesTappedInputWhenSoftwareKeyboardIsHidden]' passed" "$LOG"; then
89-
pass=$((pass+1)); verdict=pass
90-
else
91-
fail=$((fail+1)); verdict=STALL
92-
fi
93-
phase="$(grep -o 'phase=type-all durationMs=[0-9.]*' "$LOG" | tail -1)"
94-
polls="$(grep -c 'DEBUG-1874] poll' "$LOG" || true)"
95-
echo "iter=$i verdict=$verdict rc=$rc $phase polls=$polls" | tee -a stall-summary.txt
96-
if [ "$verdict" = "STALL" ]; then
97-
grep 'DEBUG-1874' "$LOG" | head -40 | tee -a stall-summary.txt
98-
cp "$LOG" ".tmp/stall-failure-$i.log"
99-
fi
106+
verdict="$(node --experimental-strip-types scripts/diagnose-1874-iteration.ts \
107+
"$LOG" "$TEST_NAME" "$i" "$rc")"
108+
case "$verdict" in
109+
passed) pass=$((pass+1)) ;;
110+
skipped) skipped=$((skipped+1)) ;;
111+
no-result) noresult=$((noresult+1)) ;;
112+
run-failed) runfailed=$((runfailed+1)) ;;
113+
*) failed=$((failed+1)) ;;
114+
esac
100115
done
101116
102-
echo "RESULT arch=${{ matrix.arch }} mode=${{ matrix.mode }}: $pass passed, $fail stalled of $ITER" | tee -a stall-summary.txt
117+
echo "RESULT arch=${{ matrix.arch }} mode=${{ matrix.mode }}: $pass passed, $skipped skipped, $failed failed, $noresult no-result, $runfailed run-failed of $ITER" | tee -a stall-summary.txt
103118
104119
- name: Upload stall evidence
105120
if: always()
@@ -109,5 +124,5 @@ jobs:
109124
include-hidden-files: true
110125
path: |
111126
stall-summary.txt
112-
.tmp/stall-failure-*.log
127+
.tmp/stall-evidence-*.log
113128
if-no-files-found: ignore
Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
import { execFileSync } from 'node:child_process';
2+
import fs from 'node:fs';
3+
import os from 'node:os';
4+
import path from 'node:path';
5+
import { fileURLToPath } from 'node:url';
6+
import { expect, test } from 'vitest';
7+
import { readIteration, type IterationReport } from '../diagnose-1874-iteration.ts';
8+
9+
const script = path.join(
10+
path.dirname(fileURLToPath(import.meta.url)),
11+
'../diagnose-1874-iteration.ts',
12+
);
13+
const NAME = 'testBareTypeUsesTappedInputWhenSoftwareKeyboardIsHidden';
14+
const POLL = '[DEBUG-1874] poll t=1ms observedLen=0 expectedPrefixLen=0';
15+
const CADENCE = [
16+
'[DEBUG-1874] wait start expectedLen=17',
17+
POLL,
18+
'[DEBUG-1874] wait outcome=settled',
19+
];
20+
21+
const verdictLine = (test: string, word: string) =>
22+
`Test Case '-[X.RunnerTests ${test}]' ${word} (1.0 seconds).`;
23+
const phases = (typeAllMs: string) =>
24+
[
25+
`phase=focus durationMs=644.6`,
26+
`phase=type-all durationMs=${typeAllMs}`,
27+
`phase=total durationMs=7379.7`,
28+
].map((phase) => `AGENT_DEVICE_RUNNER_TEXT_ENTRY_PHASE commandId=c ${phase} chars=17`);
29+
30+
const SHAPES: readonly (Partial<IterationReport> & { name: string; log: string[]; rc?: number })[] =
31+
[
32+
{
33+
name: 'healthy pass',
34+
log: [verdictLine(NAME, 'passed'), ...phases('796.1'), ...CADENCE],
35+
verdict: 'passed',
36+
keepEvidence: false,
37+
lines: ['iter=1 verdict=passed rc=0 durationMs=796.1 polls=1'],
38+
},
39+
{
40+
name: 'slow pass — an absorbed episode, which is the evidence #1874 wants',
41+
log: [verdictLine(NAME, 'passed'), ...phases('14146.4'), ...CADENCE],
42+
verdict: 'passed',
43+
keepEvidence: true,
44+
lines: ['iter=1 verdict=passed rc=0 durationMs=14146.4 polls=1', ...CADENCE],
45+
},
46+
{
47+
name: 'a pass a tenth of a millisecond over the slow threshold is still slow',
48+
log: [verdictLine(NAME, 'passed'), ...phases('2000.1'), ...CADENCE],
49+
verdict: 'passed',
50+
keepEvidence: true,
51+
},
52+
{
53+
name: 'skipped — an environment flip, not a stall',
54+
log: [verdictLine(NAME, 'skipped')],
55+
verdict: 'skipped',
56+
keepEvidence: true,
57+
lines: ['iter=1 verdict=skipped rc=0 durationMs=0 polls=0'],
58+
},
59+
{
60+
name: 'no verdict at all — ours to own, not the product’s',
61+
log: ['Executed 0 tests, with 0 failures (0 unexpected) in 0.000 seconds'],
62+
verdict: 'no-result',
63+
keepEvidence: true,
64+
},
65+
{
66+
name: 'a verdict word this loop does not model is not a stall either',
67+
log: [verdictLine(NAME, 'errored')],
68+
verdict: 'no-result',
69+
},
70+
{
71+
name: 'pair mode — the neighbour test logs first',
72+
log: [
73+
verdictLine('testBareDelayedTypeFailsWhenTappedInputDisappearsMidCommand', 'passed'),
74+
'AGENT_DEVICE_RUNNER_TEXT_ENTRY_PHASE commandId=n phase=type-all durationMs=5000.0 chars=17',
75+
verdictLine(NAME, 'passed'),
76+
...phases('812.0'),
77+
],
78+
verdict: 'passed',
79+
keepEvidence: false,
80+
},
81+
{
82+
name: 'restarted — the final word is the outcome',
83+
log: [verdictLine(NAME, 'failed'), ...phases('900.0'), verdictLine(NAME, 'passed')],
84+
verdict: 'passed',
85+
keepEvidence: false,
86+
},
87+
{
88+
name: 'measured test passed, xcodebuild did not',
89+
log: [verdictLine(NAME, 'passed'), ...phases('796.1'), ...CADENCE],
90+
rc: 65,
91+
verdict: 'run-failed',
92+
keepEvidence: true,
93+
},
94+
{
95+
name: 'louder than the cadence cap',
96+
log: [verdictLine(NAME, 'failed'), ...Array.from({ length: 45 }, () => POLL)],
97+
verdict: 'failed',
98+
keepEvidence: true,
99+
lines: [
100+
'iter=1 verdict=failed rc=0 durationMs=0 polls=45',
101+
...Array.from({ length: 40 }, () => POLL),
102+
'… 5 more DEBUG-1874 lines (full log in the artifact)',
103+
],
104+
},
105+
];
106+
107+
test.each(SHAPES)('reads a $name iteration', (shape) => {
108+
const report = readIteration(shape.log.join('\n'), NAME, 1, shape.rc ?? 0);
109+
expect(report.verdict).toBe(shape.verdict);
110+
if (shape.keepEvidence !== undefined) expect(report.keepEvidence).toBe(shape.keepEvidence);
111+
if (shape.lines) expect(report.lines).toEqual(shape.lines);
112+
});
113+
114+
test('the test name is matched literally, not as a pattern', () => {
115+
const log = verdictLine('testAXB', 'passed');
116+
expect(readIteration(log, 'testA.B', 1, 0).verdict).toBe('no-result');
117+
expect(readIteration(log, 'testAXB', 1, 0).verdict).toBe('passed');
118+
});
119+
120+
test.each([
121+
{ label: 'slow pass', rc: 0, verdict: 'passed' },
122+
{ label: 'nonzero exit over a green measured test', rc: 65, verdict: 'run-failed' },
123+
])(
124+
'the entry point writes, keeps evidence and prints the verdict for a $label',
125+
({ rc, verdict }) => {
126+
const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'diagnose-1874-'));
127+
try {
128+
fs.mkdirSync(path.join(cwd, '.tmp'));
129+
const log = path.join(cwd, 'iteration.log');
130+
fs.writeFileSync(
131+
log,
132+
[verdictLine(NAME, 'passed'), ...phases('14146.4'), ...CADENCE].join('\n'),
133+
);
134+
135+
const printed = execFileSync(
136+
process.execPath,
137+
['--experimental-strip-types', script, log, NAME, '7', `${rc}`],
138+
{ cwd, encoding: 'utf8' },
139+
);
140+
141+
expect(printed).toBe(verdict);
142+
expect(fs.readFileSync(path.join(cwd, 'stall-summary.txt'), 'utf8')).toBe(
143+
[`iter=7 verdict=${verdict} rc=${rc} durationMs=14146.4 polls=1`, ...CADENCE, ''].join(
144+
'\n',
145+
),
146+
);
147+
expect(fs.existsSync(path.join(cwd, '.tmp', 'stall-evidence-7.log'))).toBe(true);
148+
} finally {
149+
fs.rmSync(cwd, { recursive: true, force: true });
150+
}
151+
},
152+
);

‎scripts/diagnose-1874-iteration.ts‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
// Reads one iteration of the #1874 diagnose loop (.github/workflows/1874-diagnose.yml).
2+
3+
import fs from 'node:fs';
4+
import path from 'node:path';
5+
import { pathToFileURL } from 'node:url';
6+
7+
const SLOW_PASS_MS = 2000;
8+
const CADENCE_LIMIT = 40;
9+
const SUMMARY = 'stall-summary.txt';
10+
const EVIDENCE_DIR = '.tmp';
11+
12+
type MeasuredVerdict = 'passed' | 'skipped' | 'failed';
13+
14+
export type IterationReport = {
15+
readonly verdict: MeasuredVerdict | 'no-result' | 'run-failed';
16+
readonly keepEvidence: boolean;
17+
readonly lines: readonly string[];
18+
};
19+
20+
export function readIteration(
21+
log: string,
22+
testName: string,
23+
iteration: number,
24+
rc: number,
25+
): IterationReport {
26+
const lines = log.split('\n');
27+
const marker = `${testName}]' `;
28+
const word = lines
29+
.filter((line) => line.includes(marker))
30+
.map((line) => line.slice(line.indexOf(marker) + marker.length).split(' ')[0])
31+
.at(-1);
32+
const measured = isMeasuredVerdict(word) ? word : 'no-result';
33+
const exitContradictsMeasured = rc !== 0 && measured !== 'failed' && measured !== 'no-result';
34+
const verdict = exitContradictsMeasured ? 'run-failed' : measured;
35+
const durationMs = Number(
36+
lines
37+
.flatMap((line) => /phase=type-all durationMs=(\d+(?:\.\d+)?)/.exec(line)?.[1] ?? [])
38+
.at(-1) ?? 0,
39+
);
40+
const cadence = lines.filter((line) => line.includes('[DEBUG-1874]'));
41+
const keepEvidence = verdict !== 'passed' || durationMs > SLOW_PASS_MS;
42+
43+
const polls = cadence.filter((line) => line.includes('] poll')).length;
44+
const summary = `iter=${iteration} verdict=${verdict} rc=${rc} durationMs=${durationMs} polls=${polls}`;
45+
const dropped = cadence.length - CADENCE_LIMIT;
46+
return {
47+
verdict,
48+
keepEvidence,
49+
lines: !keepEvidence
50+
? [summary]
51+
: [
52+
summary,
53+
...cadence.slice(0, CADENCE_LIMIT),
54+
...(dropped > 0 ? [`… ${dropped} more DEBUG-1874 lines (full log in the artifact)`] : []),
55+
],
56+
};
57+
}
58+
59+
function isMeasuredVerdict(word: string | undefined): word is MeasuredVerdict {
60+
return word === 'passed' || word === 'skipped' || word === 'failed';
61+
}
62+
63+
function main(): number {
64+
const [logPath, testName, iteration, rc] = process.argv.slice(2);
65+
if (!logPath || !testName || !iteration || !rc) {
66+
throw new Error('usage: diagnose-1874-iteration.ts <log> <testName> <iteration> <rc>');
67+
}
68+
const report = readIteration(
69+
fs.readFileSync(logPath, 'utf8'),
70+
testName,
71+
Number(iteration),
72+
Number(rc),
73+
);
74+
fs.appendFileSync(SUMMARY, `${report.lines.join('\n')}\n`);
75+
if (report.keepEvidence) {
76+
fs.copyFileSync(logPath, path.join(EVIDENCE_DIR, `stall-evidence-${iteration}.log`));
77+
}
78+
process.stdout.write(report.verdict);
79+
return 0;
80+
}
81+
82+
if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) process.exit(main());

‎vitest.config.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,7 @@ export default defineConfig({
153153
// Parses CI configuration only, so this action guard needs no device or subprocess lane.
154154
'test/ci/upload-agent-device-artifacts.test.ts',
155155
'test/ci/upload-artifact-hidden-paths.test.ts',
156+
'scripts/__tests__/diagnose-1874-iteration.test.ts',
156157
// The size reporter is preserved across a base checkout; its entrypoint and imported
157158
// modules must move as one directory or the Bundle Size lane fails before measuring.
158159
'test/ci/size-workflow.test.ts',

0 commit comments

Comments
 (0)