Skip to content

Commit 25ad971

Browse files
committed
chore(gates): pin the iOS snapshot convergence final corpus
`PUBLISHED_EVIDENCE` becomes `PUBLISHED_CORPORA`: the pinned evidence is now two corpora measured at two revisions on two runtimes, each carrying its own tag and evidence commit, and `PUBLISHED_EVIDENCE` derives from them. A missing file is now fetched with a hint naming the corpus that actually holds it, instead of a single tag that is wrong for one of them. Disjointness, naming, and the tag and commit each corpus is cited by are tests rather than comments. Also stops exporting the setup admission's internals, which the dead-code audit rightly flags.
1 parent bef7844 commit 25ad971

4 files changed

Lines changed: 156 additions & 38 deletions

File tree

‎scripts/ios-snapshot-benchmark/evidence.test.ts‎

Lines changed: 36 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { afterEach, test } from 'vitest';
66
import {
77
DEFAULT_EVIDENCE_DIR,
88
EVIDENCE_FIXTURE_PATH,
9+
PUBLISHED_CORPORA,
910
PUBLISHED_EVIDENCE,
1011
checkEvidenceCorpus,
1112
fetchEvidenceCommand,
@@ -80,13 +81,32 @@ test('the evidence README cites every published hash and the fetch recipe', () =
8081
assert.ok(readme.includes(fetchEvidenceCommand()));
8182
});
8283

83-
function oneOfThreePublishedFiles(): EvidenceFile {
84-
const [file, sha256] = Object.entries(PUBLISHED_EVIDENCE)[0]!;
84+
test('each published corpus is disjoint, named for its revision, and cited by tag and commit', () => {
85+
const readme = fs.readFileSync(path.join(DEFAULT_EVIDENCE_DIR, 'README.md'), 'utf8');
86+
const seen = new Set<string>();
87+
for (const corpus of PUBLISHED_CORPORA) {
88+
assert.match(corpus.tag, /^refs\/tags\/evidence\/ios-snapshot\/[0-9a-f]{9,}$/);
89+
assert.match(corpus.commit, /^[0-9a-f]{40}$/);
90+
assert.ok(readme.includes(corpus.tag), `README does not cite ${corpus.tag}`);
91+
assert.ok(readme.includes(corpus.commit), `README does not cite ${corpus.commit}`);
92+
for (const file of Object.keys(corpus.files)) {
93+
assert.ok(!seen.has(file), `${file} is published by two corpora`);
94+
seen.add(file);
95+
assert.ok(
96+
file.endsWith(`-${corpus.tag.split('/').pop()}.json`),
97+
`${file} is not named for ${corpus.tag}`,
98+
);
99+
}
100+
}
101+
});
102+
103+
function onePublishedFile(): EvidenceFile {
104+
const [file, sha256] = Object.entries(PUBLISHED_CORPORA[0]!.files)[0]!;
85105
return {
86106
file,
87107
sha256,
88108
published: 'match',
89-
revision: '71fb2483f30d90e615e949601c836aeebbf450c5',
109+
revision: PUBLISHED_CORPORA[0]!.revision,
90110
status: 'completed',
91111
cells: 2,
92112
errors: [],
@@ -95,15 +115,23 @@ function oneOfThreePublishedFiles(): EvidenceFile {
95115

96116
test('the default evidence directory must hold the complete published corpus, named files and all', () => {
97117
const dir = temporaryEvidenceDir({});
98-
assert.throws(
99-
() => checkEvidenceCorpus(dir, [oneOfThreePublishedFiles()], true),
100-
/is missing published evidence file\(s\): ios-snapshot-warm-relaunch-local-71fb2483f\.json, ios-snapshot-proxy-71fb2483f\.json/,
101-
);
118+
const present = onePublishedFile();
119+
let message = '';
120+
try {
121+
checkEvidenceCorpus(dir, [present], true);
122+
} catch (error) {
123+
message = error instanceof Error ? error.message : String(error);
124+
}
125+
assert.match(message, /is missing published evidence file\(s\)/);
126+
for (const file of Object.keys(PUBLISHED_EVIDENCE)) {
127+
if (file === present.file) continue;
128+
assert.ok(message.includes(file), `the error omits ${file}`);
129+
}
102130
});
103131

104132
test('an explicit --evidence-dir stays permissive: a partial corpus does not fail completeness', () => {
105133
const dir = temporaryEvidenceDir({});
106-
assert.doesNotThrow(() => checkEvidenceCorpus(dir, [oneOfThreePublishedFiles()], false));
134+
assert.doesNotThrow(() => checkEvidenceCorpus(dir, [onePublishedFile()], false));
107135
});
108136

109137
test('fetched evidence under the in-tree directory matches the published corpus', (context) => {

‎scripts/ios-snapshot-benchmark/evidence.ts‎

Lines changed: 83 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -6,25 +6,63 @@ import { validateRawResult } from './schema.ts';
66
import type { BenchmarkResult } from './types.ts';
77

88
/**
9-
* The evidence/ios-snapshot branch tip is mutable; this annotated tag and the full commit SHA
10-
* below are the durable, immutable ref the corpus is pinned to. Re-tag (a new suffix, a new
11-
* commit) if the corpus is ever re-measured — never move this tag.
9+
* Each corpus is pinned by an annotated tag on the mutable `evidence/ios-snapshot` branch tip plus
10+
* the full evidence commit below. Re-tag (a new suffix, a new commit) if a corpus is ever
11+
* re-measured — never move these tags.
1212
*/
13-
const EVIDENCE_TAG = 'refs/tags/evidence/ios-snapshot/71fb2483f';
14-
const EVIDENCE_COMMIT = '2d4baf461aa8897d49c6d4683cd16d8f43588ae8';
15-
export const DEFAULT_EVIDENCE_DIR = path.join(import.meta.dirname, 'evidence');
16-
export const EVIDENCE_FIXTURE_PATH = path.join(import.meta.dirname, 'evidence-fixture.v1.json');
13+
export type PublishedCorpus = {
14+
/** Revision the leg measured, as a full SHA. */
15+
revision: string;
16+
/** Durable ref the corpus is fetched from. */
17+
tag: string;
18+
/** Evidence-branch commit holding the files. */
19+
commit: string;
20+
/** Corpus file name -> sha256. */
21+
files: Readonly<Record<string, string>>;
22+
};
23+
24+
const BASELINE_CORPUS: PublishedCorpus = {
25+
revision: '71fb2483f30d90e615e949601c836aeebbf450c5',
26+
tag: 'refs/tags/evidence/ios-snapshot/71fb2483f',
27+
commit: '2d4baf461aa8897d49c6d4683cd16d8f43588ae8',
28+
files: {
29+
'ios-snapshot-cold-local-71fb2483f.json':
30+
'532a83247bfbf8ee47039f80ac429f067c84679e92c781768c1044da1ae6e9bf',
31+
'ios-snapshot-warm-relaunch-local-71fb2483f.json':
32+
'6d299e8baec69662dca2c1ad8f1348e4361d5afaa781080e9a6b9b3dac362cbf',
33+
'ios-snapshot-proxy-71fb2483f.json':
34+
'b11b7a07be9e4dcf003f3af66943682a6733c6f21f5f43d3d9e88b3fb37b51a7',
35+
},
36+
};
1737

18-
/** Raw results published on the evidence branch, keyed by file name; measured at 71fb2483f. */
19-
export const PUBLISHED_EVIDENCE: Readonly<Record<string, string>> = {
20-
'ios-snapshot-cold-local-71fb2483f.json':
21-
'532a83247bfbf8ee47039f80ac429f067c84679e92c781768c1044da1ae6e9bf',
22-
'ios-snapshot-warm-relaunch-local-71fb2483f.json':
23-
'6d299e8baec69662dca2c1ad8f1348e4361d5afaa781080e9a6b9b3dac362cbf',
24-
'ios-snapshot-proxy-71fb2483f.json':
25-
'b11b7a07be9e4dcf003f3af66943682a6733c6f21f5f43d3d9e88b3fb37b51a7',
38+
const CONVERGENCE_FINAL_CORPUS: PublishedCorpus = {
39+
revision: '7c434b575837e3291c51315bf9bb8b54c8ce7568',
40+
tag: 'refs/tags/evidence/ios-snapshot/7c434b575',
41+
commit: '96d4951c19fbe009ba19d192a9774668edcc3f56',
42+
files: {
43+
'ios-snapshot-cold-local-7c434b575.json':
44+
'4663897ee5104569ad54e2ac803c216b284280c1c70fd82a8b2cf7b675d8a8bd',
45+
'ios-snapshot-first-interaction-local-7c434b575.json':
46+
'87c686336f5581e3f18111e160cf7b733cd726b41e79ed6d8e5b53e2ab40c3fb',
47+
'ios-snapshot-warm-relaunch-local-7c434b575.json':
48+
'd49df3c3c943178f016a2b958449b257c6d46a44c8ffdf8fab75c1634ae1ebce',
49+
'ios-snapshot-proxy-7c434b575.json':
50+
'5b5353831851f3a0f60e19d6bfd0cf50db47c3c4db52a283024c10bb17e71573',
51+
},
2652
};
2753

54+
export const PUBLISHED_CORPORA: readonly PublishedCorpus[] = [
55+
BASELINE_CORPUS,
56+
CONVERGENCE_FINAL_CORPUS,
57+
];
58+
59+
/** Raw results published on the evidence branch, keyed by file name across every corpus. */
60+
export const PUBLISHED_EVIDENCE: Readonly<Record<string, string>> = Object.fromEntries(
61+
PUBLISHED_CORPORA.flatMap((corpus) => Object.entries(corpus.files)),
62+
);
63+
export const DEFAULT_EVIDENCE_DIR = path.join(import.meta.dirname, 'evidence');
64+
export const EVIDENCE_FIXTURE_PATH = path.join(import.meta.dirname, 'evidence-fixture.v1.json');
65+
2866
export type EvidenceFile = {
2967
file: string;
3068
sha256: string;
@@ -35,9 +73,31 @@ export type EvidenceFile = {
3573
errors: string[];
3674
};
3775

38-
export function fetchEvidenceCommand(file = '<file>'): string {
76+
function corpusFetchCommand(corpus: PublishedCorpus, file: string): string {
3977
const destination = path.posix.join('scripts/ios-snapshot-benchmark/evidence', file);
40-
return `git fetch origin ${EVIDENCE_TAG} && git show ${EVIDENCE_COMMIT}:${file} > ${destination}`;
78+
return `git show ${corpus.commit}:${file} > ${destination}`;
79+
}
80+
81+
/**
82+
* The fetch recipe for one corpus file. An unknown name falls back to the newest corpus, which is
83+
* what a generic hint should recommend.
84+
*/
85+
export function fetchEvidenceCommand(file = '<file>'): string {
86+
const corpus =
87+
PUBLISHED_CORPORA.find((candidate) => file in candidate.files) ?? PUBLISHED_CORPORA.at(-1)!;
88+
return `git fetch origin ${corpus.tag} && ${corpusFetchCommand(corpus, file)}`;
89+
}
90+
91+
/** One shell line per corpus that holds any of `files`, so a hint never names the wrong commit. */
92+
function fetchEvidenceCommands(files: readonly string[]): string {
93+
const lines: string[] = [];
94+
for (const corpus of PUBLISHED_CORPORA) {
95+
const missing = files.filter((file) => file in corpus.files);
96+
if (missing.length === 0) continue;
97+
const fetches = missing.map((file) => corpusFetchCommand(corpus, file));
98+
lines.push(`git fetch origin ${corpus.tag} && ${fetches.join(' && ')}`);
99+
}
100+
return lines.join('\n');
41101
}
42102

43103
export function listEvidenceFiles(dir: string): string[] {
@@ -133,8 +193,8 @@ export function checkEvidenceCorpus(dir: string, files: EvidenceFile[], isDefaul
133193
const missing = missingPublishedEvidence(files);
134194
if (missing.length > 0) {
135195
throw new Error(
136-
`${dir} is missing published evidence file(s): ${missing.join(', ')}. Fetch them with: ` +
137-
`${fetchEvidenceCommand()}`,
196+
`${dir} is missing published evidence file(s): ${missing.join(', ')}.\n` +
197+
`Fetch them with:\n${fetchEvidenceCommands(missing)}`,
138198
);
139199
}
140200
}
@@ -144,7 +204,10 @@ function runEvidenceReport(argv: string[]): void {
144204
const { dir, isDefault } = readEvidenceDirOption(argv);
145205
const files = readEvidenceDir(dir);
146206
if (files.length === 0) {
147-
throw new Error(`${dir} holds no evidence; fetch it with: ${fetchEvidenceCommand()}`);
207+
throw new Error(
208+
`${dir} holds no evidence; fetch it with:\n` +
209+
`${fetchEvidenceCommands(Object.keys(PUBLISHED_EVIDENCE))}`,
210+
);
148211
}
149212
process.stdout.write(renderEvidenceReport(dir, files));
150213
checkEvidenceCorpus(dir, files, isDefault);

‎scripts/ios-snapshot-benchmark/evidence/README.md‎

Lines changed: 35 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,45 @@
11
# iOS snapshot benchmark evidence
22

3-
The raw `pnpm bench:ios-snapshot` results measured at commit
4-
`71fb2483f30d90e615e949601c836aeebbf450c5` on `bench-golden-v2` (iPhone 17 Pro, iOS 27.0) live on
5-
the orphan branch `evidence/ios-snapshot`, not in this tree. The branch tip is mutable; the durable
6-
ref is the annotated tag `evidence/ios-snapshot/71fb2483f`, pinned to commit
7-
`2d4baf461aa8897d49c6d4683cd16d8f43588ae8` — fetch and read from the tag and that full SHA, never
8-
from the branch tip. Only Markdown summaries are kept here. The published hashes are declared in
9-
[`../evidence.ts`](../evidence.ts) as `PUBLISHED_EVIDENCE`:
3+
The raw `pnpm bench:ios-snapshot` results live on the orphan branch `evidence/ios-snapshot`, not in
4+
this tree. The branch tip is mutable; the durable refs are the annotated tags and full commits
5+
below — fetch from those, never from the branch tip. Only Markdown summaries are kept here. The
6+
published hashes are declared in [`../evidence.ts`](../evidence.ts) as `PUBLISHED_CORPORA`, and
7+
[`../evidence.ts`](../evidence.ts) derives `PUBLISHED_EVIDENCE` from them.
8+
9+
## Baseline corpus — `71fb2483f`
10+
11+
Measured at commit `71fb2483f30d90e615e949601c836aeebbf450c5` on `bench-golden-v2` (iPhone 17 Pro,
12+
iOS 27.0). Tag `refs/tags/evidence/ios-snapshot/71fb2483f`, commit
13+
`2d4baf461aa8897d49c6d4683cd16d8f43588ae8`.
1014

1115
| File | sha256 |
1216
| --- | --- |
1317
| `ios-snapshot-cold-local-71fb2483f.json` | `532a83247bfbf8ee47039f80ac429f067c84679e92c781768c1044da1ae6e9bf` |
1418
| `ios-snapshot-warm-relaunch-local-71fb2483f.json` | `6d299e8baec69662dca2c1ad8f1348e4361d5afaa781080e9a6b9b3dac362cbf` |
1519
| `ios-snapshot-proxy-71fb2483f.json` | `b11b7a07be9e4dcf003f3af66943682a6733c6f21f5f43d3d9e88b3fb37b51a7` |
1620

21+
## Convergence final corpus — `7c434b575`
22+
23+
Measured at commit `7c434b575837e3291c51315bf9bb8b54c8ce7568` on `bench-2188-final` (iPhone 17 Pro,
24+
iOS 26.2). Tag `refs/tags/evidence/ios-snapshot/7c434b575`, commit
25+
`96d4951c19fbe009ba19d192a9774668edcc3f56`. This is the corpus that closed the evidence sweep
26+
gating #2188 on #2199; [`../../../docs/evidence/ios-snapshot-convergence-final-2026-09-21.md`](../../../docs/evidence/ios-snapshot-convergence-final-2026-09-21.md)
27+
reads it against the baseline, including the target, runtime, host-load, app, and harness
28+
deviations between the two corpora.
29+
30+
| File | sha256 |
31+
| --- | --- |
32+
| `ios-snapshot-cold-local-7c434b575.json` | `4663897ee5104569ad54e2ac803c216b284280c1c70fd82a8b2cf7b675d8a8bd` |
33+
| `ios-snapshot-first-interaction-local-7c434b575.json` | `87c686336f5581e3f18111e160cf7b733cd726b41e79ed6d8e5b53e2ab40c3fb` |
34+
| `ios-snapshot-warm-relaunch-local-7c434b575.json` | `d49df3c3c943178f016a2b958449b257c6d46a44c8ffdf8fab75c1634ae1ebce` |
35+
| `ios-snapshot-proxy-7c434b575.json` | `5b5353831851f3a0f60e19d6bfd0cf50db47c3c4db52a283024c10bb17e71573` |
36+
1737
## Fetch
1838

1939
One file, from the repository root:
2040

2141
```sh
22-
git fetch origin refs/tags/evidence/ios-snapshot/71fb2483f && git show 2d4baf461aa8897d49c6d4683cd16d8f43588ae8:<file> > scripts/ios-snapshot-benchmark/evidence/<file>
42+
git fetch origin refs/tags/evidence/ios-snapshot/7c434b575 && git show 96d4951c19fbe009ba19d192a9774668edcc3f56:<file> > scripts/ios-snapshot-benchmark/evidence/<file>
2343
```
2444

2545
The whole corpus, then a schema and hash check:
@@ -31,6 +51,13 @@ for f in ios-snapshot-cold-local-71fb2483f.json \
3151
ios-snapshot-proxy-71fb2483f.json; do
3252
git show 2d4baf461aa8897d49c6d4683cd16d8f43588ae8:$f > scripts/ios-snapshot-benchmark/evidence/$f
3353
done
54+
git fetch origin refs/tags/evidence/ios-snapshot/7c434b575
55+
for f in ios-snapshot-cold-local-7c434b575.json \
56+
ios-snapshot-first-interaction-local-7c434b575.json \
57+
ios-snapshot-warm-relaunch-local-7c434b575.json \
58+
ios-snapshot-proxy-7c434b575.json; do
59+
git show 96d4951c19fbe009ba19d192a9774668edcc3f56:$f > scripts/ios-snapshot-benchmark/evidence/$f
60+
done
3461
pnpm bench:ios-snapshot:evidence
3562
```
3663

‎scripts/ios-snapshot-benchmark/fixture-admission.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ export type FixtureAnchorPhase = 'opened' | 'prepared' | 'sample';
1010
* is waited out instead of stopping the run. The budget must stay far below the operation timeout
1111
* it precedes; a fixture that never exposes its anchor still stops the run.
1212
*/
13-
export const FIXTURE_ANCHOR_ADMISSION_BUDGET_MS = 30_000;
13+
const FIXTURE_ANCHOR_ADMISSION_BUDGET_MS = 30_000;
1414
const FIXTURE_ANCHOR_POLL_INTERVAL_MS = 500;
1515

1616
export type FixtureOperationResult = {
@@ -77,7 +77,7 @@ async function observeFixtureAnchor(
7777
}
7878
}
7979

80-
export function hasFixtureAnchor(
80+
function hasFixtureAnchor(
8181
payload: unknown,
8282
fixture: ScreenFixture,
8383
phase: FixtureAnchorPhase,

0 commit comments

Comments
 (0)