Skip to content

Commit 2e80e27

Browse files
test: lock replay-compat provenance kind by corpus area and verify it in CI
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
1 parent 9e5a56f commit 2e80e27

8 files changed

Lines changed: 161 additions & 18 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,26 @@ jobs:
216216
- name: Check for production-unused exports
217217
run: pnpm check:production-exports
218218

219+
replay-compat-provenance:
220+
# The frozen replay-compat corpus (#1417) claims each entry was published by
221+
# a released tag. Only a full-history checkout can re-derive that claim, so
222+
# this job exists separately from the shallow-clone-safe unit lane.
223+
name: Replay-Compat Provenance
224+
runs-on: ubuntu-latest
225+
timeout-minutes: 10
226+
steps:
227+
- name: Checkout
228+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
229+
with:
230+
fetch-depth: 0
231+
fetch-tags: true
232+
233+
- name: Setup toolchain
234+
uses: ./.github/actions/setup-node-pnpm
235+
236+
- name: Verify corpus entries against their released blobs
237+
run: pnpm check:replay-compat
238+
219239
coverage:
220240
# Runs the full unit + provider-integration suites under coverage with
221241
# thresholds, so a separate unit-tests job would rerun the same tests.

‎scripts/check-affected/checks.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,15 @@ export const CHECK_CATALOG: readonly CheckSpec[] = [
152152
ciJobs: [],
153153
localRunnable: true,
154154
},
155+
{
156+
id: 'replay-compat',
157+
label: 'Replay-compat corpus provenance (released blobs)',
158+
kind: { type: 'script', script: 'check:replay-compat' },
159+
// Needs full history and tags, so it runs in its own fetch-depth: 0 job
160+
// rather than inside the shallow-clone-safe unit lane.
161+
ciJobs: ['Replay-Compat Provenance'],
162+
localRunnable: true,
163+
},
155164
];
156165

157166
export function getCheckSpec(id: CheckId): CheckSpec {

‎scripts/check-affected/model.test.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,10 +114,17 @@ test('a non-.ts fixture under an owned root fails open (format alone is not owne
114114
assert.equal(result.failOpenReasons[0]?.rule, 'ambiguous-path');
115115
});
116116

117-
test('a frozen replay-compat corpus script selects the unit lane that asserts it', () => {
117+
test('a frozen replay-compat corpus script selects the unit lane and the provenance verifier', () => {
118118
const result = plan(['test/replay-compat/scripts/examples/gesture-lab.v0.16.8.ad']);
119119
assert.equal(result.failOpen, false);
120120
assert.ok(result.checks.includes('unit'));
121+
assert.ok(result.checks.includes('replay-compat'));
122+
});
123+
124+
test('a replay-compat manifest edit selects the provenance verifier', () => {
125+
const result = plan(['test/replay-compat/manifest.ts']);
126+
assert.equal(result.failOpen, false);
127+
assert.ok(result.checks.includes('replay-compat'));
121128
});
122129

123130
test('skills guidance change selects format + skillgym, not docs-only', () => {
@@ -181,6 +188,7 @@ test('every catalog command resolves against package scripts', () => {
181188
'build:macos-helper': 'x',
182189
'test:smoke:web': 'x',
183190
'test:skillgym': 'x',
191+
'check:replay-compat': 'x',
184192
};
185193
for (const spec of CHECK_CATALOG) {
186194
const command = resolveCommand(spec, scripts, 'origin/main');

‎scripts/check-affected/model.ts‎

Lines changed: 30 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,8 @@ export type CheckId =
4040
| 'android-helpers'
4141
| 'macos-helper'
4242
| 'web-smoke'
43-
| 'skillgym';
43+
| 'skillgym'
44+
| 'replay-compat';
4445

4546
// The complete local check universe. A fail-open plan selects all of these;
4647
// keep it in sync with the catalog in checks.ts (asserted by the self-test).
@@ -63,6 +64,7 @@ export const ALL_CHECKS: readonly CheckId[] = [
6364
'macos-helper',
6465
'web-smoke',
6566
'skillgym',
67+
'replay-compat',
6668
];
6769

6870
export type SelectionReason = {
@@ -228,20 +230,33 @@ const nodeIntegrationOwnership: OwnershipRule = ({ file }) =>
228230
? [reason('integration-node', file, 'node-integration', 'node --test integration smoke')]
229231
: [];
230232

231-
// The frozen replay-compat corpus (#1417) is `.ad` fixture data whose only
232-
// consumer is `test/replay-compat/corpus.test.ts` in the unit lane, so a corpus
233-
// addition has a derivable owner instead of failing open on its extension.
234-
const replayCompatOwnership: OwnershipRule = ({ file }) =>
235-
file.startsWith('test/replay-compat/') && file.endsWith('.ad')
236-
? [
237-
reason(
238-
'unit',
239-
file,
240-
'own:replay-compat',
241-
'frozen replay-compat corpus is asserted by the unit-lane corpus test',
242-
),
243-
]
244-
: [];
233+
// The frozen replay-compat corpus (#1417). `.ad` fixture data would otherwise
234+
// fail open on its extension: its only consumer is the unit-lane corpus test.
235+
// Any corpus change — script or manifest — also runs the history-backed
236+
// provenance verifier, which is the only gate that can prove an entry's blob
237+
// really came from the release tag it names.
238+
const replayCompatOwnership: OwnershipRule = ({ file }) => {
239+
if (!file.startsWith('test/replay-compat/')) return [];
240+
const selections = [
241+
reason(
242+
'replay-compat',
243+
file,
244+
'own:replay-compat-provenance',
245+
'corpus provenance is re-derived from released git blobs',
246+
),
247+
];
248+
if (file.endsWith('.ad')) {
249+
selections.push(
250+
reason(
251+
'unit',
252+
file,
253+
'own:replay-compat',
254+
'frozen replay-compat corpus is asserted by the unit-lane corpus test',
255+
),
256+
);
257+
}
258+
return selections;
259+
};
245260

246261
// SkillGym validates skill guidance (`skills/`) and owns its harness
247262
// (`test/skillgym/`); the Testing Matrix in docs/agents/testing.md routes

‎scripts/check-affected/run.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ const ALL_SCRIPTS: Record<string, string> = {
101101
'test:integration:node': 'x',
102102
'test:integration:progress:check': 'x',
103103
'test:skillgym': 'x',
104+
'check:replay-compat': 'x',
104105
};
105106

106107
const ARGS = { base: 'origin/main', head: 'HEAD', json: false, run: true };

‎scripts/check-replay-compat-provenance.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,11 @@
1111

1212
import fs from 'node:fs';
1313
import path from 'node:path';
14-
import { REPLAY_COMPAT_CORPUS, REPLAY_COMPAT_RELEASED_TAGS } from '../test/replay-compat/manifest.ts';
14+
import {
15+
REPLAY_COMPAT_CORPUS,
16+
REPLAY_COMPAT_RELEASED_TAGS,
17+
} from '../test/replay-compat/manifest.ts';
18+
import { findProvenanceKindViolations } from '../test/replay-compat/provenance-rules.ts';
1519
import { runCmdSync } from '../src/utils/exec.ts';
1620

1721
const repoRoot = path.resolve(import.meta.dirname, '..');
@@ -27,7 +31,9 @@ if (git(['rev-parse', '--is-shallow-repository']) === 'true') {
2731
);
2832
}
2933

30-
const failures: string[] = [];
34+
// An entry that escaped `mined` would also escape the history check below, so
35+
// the area rule is enforced here as well as in the unit lane.
36+
const failures: string[] = [...findProvenanceKindViolations(REPLAY_COMPAT_CORPUS)];
3137

3238
const existingTags = new Set(git(['tag', '--list']).split('\n').filter(Boolean));
3339
if (existingTags.size === 0) {

‎test/replay-compat/corpus.test.ts‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import {
1111
type ReplayCompatCoverage,
1212
type ReplayCompatEntry,
1313
} from './manifest.ts';
14+
import { findProvenanceKindViolations, requiredProvenanceKind } from './provenance-rules.ts';
1415

1516
const CORPUS_DIR = fileURLToPath(new URL('.', import.meta.url));
1617

@@ -86,6 +87,33 @@ describe('replay-compat corpus', () => {
8687
},
8788
);
8889

90+
// Reclassifying a mined entry as `derived` would let edited bytes be re-pinned
91+
// by digest inside this same manifest and skipped by the historical verifier,
92+
// so the corpus area — not the manifest line — fixes the kind.
93+
test('provenance kind is fixed by corpus area', () => {
94+
expect(findProvenanceKindViolations(REPLAY_COMPAT_CORPUS)).toEqual([]);
95+
for (const entry of REPLAY_COMPAT_CORPUS) {
96+
expect(entry.provenance.kind, entry.id).toBe(requiredProvenanceKind(entry.file));
97+
}
98+
});
99+
100+
test('a mined entry relabelled as derived is rejected', () => {
101+
const mined = REPLAY_COMPAT_CORPUS.find((entry) => entry.provenance.kind === 'mined');
102+
expect(mined).toBeDefined();
103+
const reclassified: ReplayCompatEntry = {
104+
...(mined as ReplayCompatEntry),
105+
provenance: { kind: 'derived', from: 'hand-written', sha256: 'f'.repeat(64) },
106+
};
107+
expect(findProvenanceKindViolations([reclassified])).toHaveLength(1);
108+
expect(findProvenanceKindViolations([reclassified])[0]).toContain('must stay "mined"');
109+
});
110+
111+
test('a corpus area with no declared provenance kind is rejected', () => {
112+
expect(() => requiredProvenanceKind('scripts/handwritten/whatever.v0.20.0.ad')).toThrow(
113+
/no declared provenance kind/,
114+
);
115+
});
116+
89117
test('every corpus script is claimed by exactly one manifest entry', () => {
90118
const claimed = REPLAY_COMPAT_CORPUS.map((entry) => entry.file);
91119
expect([...claimed].sort()).toEqual(listCorpusScriptFiles().sort());
Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/**
2+
* Which provenance kind each corpus area is allowed to use (#1417).
3+
*
4+
* The kind decides how strongly an entry is pinned: `mined` bytes are re-derived
5+
* from git history by `pnpm check:replay-compat`, `derived` bytes are only
6+
* digest-pinned inside the same editable manifest. Without this rule an entry
7+
* could be edited, relabelled `derived`, given a matching digest, and quietly
8+
* escape the historical verifier. So the area — not the manifest line — decides:
9+
* anything mined out of the replay suites must stay `mined` forever.
10+
*/
11+
12+
import type { ReplayCompatEntry, ReplayCompatProvenance } from './manifest.ts';
13+
14+
const CORPUS_AREA_PROVENANCE_KIND: Readonly<Record<string, ReplayCompatProvenance['kind']>> = {
15+
// Mined from the git history of `test/integration/replays`.
16+
integration: 'mined',
17+
// Mined from the git history of `examples/test-app/replays`.
18+
examples: 'mined',
19+
// Composed from a release's own grammar/docs: no historical blob exists.
20+
docs: 'derived',
21+
};
22+
23+
function corpusArea(file: string): string {
24+
const area = file.split('/')[1];
25+
if (!area || !file.startsWith('scripts/')) {
26+
throw new Error(`Corpus file "${file}" must live under scripts/<area>/.`);
27+
}
28+
return area;
29+
}
30+
31+
export function requiredProvenanceKind(file: string): ReplayCompatProvenance['kind'] {
32+
const area = corpusArea(file);
33+
const kind = CORPUS_AREA_PROVENANCE_KIND[area];
34+
if (!kind) {
35+
throw new Error(
36+
`Corpus area "${area}" has no declared provenance kind. Add it to CORPUS_AREA_PROVENANCE_KIND ` +
37+
`in test/replay-compat/provenance-rules.ts before adding entries under it.`,
38+
);
39+
}
40+
return kind;
41+
}
42+
43+
/** Human-readable violations, so the same rule can fail a test or a CI script. */
44+
export function findProvenanceKindViolations(
45+
entries: readonly ReplayCompatEntry[],
46+
): readonly string[] {
47+
return entries.flatMap((entry) => {
48+
const required = requiredProvenanceKind(entry.file);
49+
if (entry.provenance.kind === required) return [];
50+
return [
51+
`${entry.id}: entries under scripts/${corpusArea(entry.file)}/ must stay "${required}", ` +
52+
`but this one is "${entry.provenance.kind}". Reclassifying a mined script drops it out of ` +
53+
`pnpm check:replay-compat; keep the released blob id instead.`,
54+
];
55+
});
56+
}

0 commit comments

Comments
 (0)