Skip to content

Commit 9e5a56f

Browse files
test: pin replay-compat corpus bytes to released blobs and assert via parseReplayInput
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
1 parent fadefcc commit 9e5a56f

6 files changed

Lines changed: 433 additions & 83 deletions

File tree

‎docs/agents/testing.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ The mapping it encodes, for when you need to run a gate directly or reason about
1919
| CLI help/guidance (`src/cli/parser/cli-help.ts`, `src/cli-schema/`) | `pnpm exec vitest run src/cli/parser/__tests__ src/cli-schema/command-schema-guards.test.ts scripts/__tests__` — the `scripts/__tests__` gates enforce help-topic benchmark coverage and pin the bench's quoted CLI samples to the real renderers |
2020
| Help benchmark cases (`scripts/help-conformance-*.mjs`) | `pnpm exec vitest run scripts/__tests__` (deterministic gates); model-backed: `pnpm bench:help-conformance` (paid LLM calls, local only) |
2121
| SkillGym prompts/assertions | `pnpm test:skillgym:case <case-id>` (broad: `pnpm test:skillgym`, filter with `-- --tag fixture-smoke` or `-- --tag skill-guidance`) — agentic routing + local-help-consumption proof only; command-planning knowledge checks belong in the help bench |
22-
| `.ad` grammar (`src/replay/script.ts`, gesture arity, replay vars) | `pnpm exec vitest run --project unit-core test/replay-compat` — the frozen replay-compat corpus asserts which released script surfaces still parse; a flipped verdict is edited in `test/replay-compat/manifest.ts`, never in the script |
22+
| `.ad` grammar (`src/replay/script.ts`, gesture arity, replay vars) | `pnpm exec vitest run --project unit-core test/replay-compat` — the frozen replay-compat corpus asserts which released script surfaces still parse; a flipped verdict is edited in `test/replay-compat/manifest.ts`, never in the script. Adding or re-pinning a corpus entry also runs `pnpm check:replay-compat`, which re-derives each entry from its release tag in git history |
2323
| Anything in `src/`, `test/`, `skills/` | `pnpm format` |
2424

2525
Two traps worth naming:

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@
122122
"depgraph:test": "node --experimental-strip-types --test scripts/depgraph/model.test.ts",
123123
"check:production-exports": "fallow dead-code --config fallow-production-exports.json --production --unused-exports --fail-on-issues",
124124
"check:bundle-owner-files": "node --experimental-strip-types scripts/check-bundle-owner-files.ts",
125+
"check:replay-compat": "node --experimental-strip-types scripts/check-replay-compat-provenance.ts",
125126
"check:freerange": "fr",
126127
"check:quick": "pnpm lint && pnpm typecheck",
127128
"sync:mcp-metadata": "node scripts/sync-mcp-metadata.mjs",
Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
/**
2+
* Re-derives the frozen replay-compat corpus (#1417) from git history: every
3+
* mined entry must still be the exact blob a RELEASED tag published at the path
4+
* it claims, and every tag the corpus cites must be a tag that was actually cut.
5+
*
6+
* The unit gate (`test/replay-compat/corpus.test.ts`) hashes the checked-in
7+
* bytes against the same object ids, which catches a rewritten script; only this
8+
* check can say those object ids came from a release rather than from whoever
9+
* wrote the manifest line.
10+
*/
11+
12+
import fs from 'node:fs';
13+
import path from 'node:path';
14+
import { REPLAY_COMPAT_CORPUS, REPLAY_COMPAT_RELEASED_TAGS } from '../test/replay-compat/manifest.ts';
15+
import { runCmdSync } from '../src/utils/exec.ts';
16+
17+
const repoRoot = path.resolve(import.meta.dirname, '..');
18+
const corpusRoot = path.join(repoRoot, 'test', 'replay-compat');
19+
20+
function git(args: string[]): string {
21+
return runCmdSync('git', args, { cwd: repoRoot, allowFailure: true }).stdout.trim();
22+
}
23+
24+
if (git(['rev-parse', '--is-shallow-repository']) === 'true') {
25+
throw new Error(
26+
'Corpus provenance needs full history and tags. Run `git fetch --unshallow --tags` first.',
27+
);
28+
}
29+
30+
const failures: string[] = [];
31+
32+
const existingTags = new Set(git(['tag', '--list']).split('\n').filter(Boolean));
33+
if (existingTags.size === 0) {
34+
throw new Error('No release tags found. Run `git fetch --tags` first.');
35+
}
36+
for (const tag of REPLAY_COMPAT_RELEASED_TAGS) {
37+
if (!existingTags.has(tag)) {
38+
failures.push(`${tag} is cited as a released version but is not a tag in this repository`);
39+
}
40+
}
41+
42+
for (const entry of REPLAY_COMPAT_CORPUS) {
43+
if (!REPLAY_COMPAT_RELEASED_TAGS.includes(entry.recordedBy)) {
44+
failures.push(`${entry.id}: recordedBy ${entry.recordedBy} is not in REPLAY_COMPAT_RELEASED_TAGS`);
45+
continue;
46+
}
47+
if (entry.provenance.kind !== 'mined') continue;
48+
49+
const { path: sourcePath, blob } = entry.provenance;
50+
const historical = git(['rev-parse', `${entry.recordedBy}:${sourcePath}`]);
51+
if (historical !== blob) {
52+
failures.push(
53+
`${entry.id}: ${entry.recordedBy}:${sourcePath} is ${historical || '<missing>'}, manifest pins ${blob}`,
54+
);
55+
continue;
56+
}
57+
const frozen = fs.readFileSync(path.join(corpusRoot, entry.file));
58+
const released = runCmdSync('git', ['cat-file', 'blob', blob], {
59+
cwd: repoRoot,
60+
binaryStdout: true,
61+
}).stdoutBuffer;
62+
if (!released?.equals(frozen)) {
63+
failures.push(`${entry.id}: ${entry.file} no longer matches the released blob ${blob}`);
64+
}
65+
}
66+
67+
if (failures.length > 0) {
68+
throw new Error(
69+
`Replay-compat corpus provenance is broken (see test/replay-compat/README.md):\n${failures
70+
.map((failure) => `- ${failure}`)
71+
.join('\n')}`,
72+
);
73+
}
74+
75+
const mined = REPLAY_COMPAT_CORPUS.filter((entry) => entry.provenance.kind === 'mined').length;
76+
process.stdout.write(
77+
`Verified ${mined} mined replay-compat scripts against their released blobs across ${REPLAY_COMPAT_RELEASED_TAGS.length} tags (${REPLAY_COMPAT_CORPUS.length - mined} derived entries are digest-pinned).\n`,
78+
);

‎test/replay-compat/README.md‎

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,17 @@
33
`.ad` scripts as **released** versions of agent-device wrote them, each paired with the verdict
44
today's parser owes it: `parses`, or `fails` with an error code and the migration hint substring.
55

6-
`corpus.test.ts` runs in `unit-core` and asserts every verdict, so a grammar change cannot quietly
7-
alter which historical script surface still loads. #1417.
6+
`corpus.test.ts` runs in `unit-core` and asserts every verdict through `parseReplayInput` — the same
7+
composition `replay`/`test` use — so a grammar change cannot quietly alter which historical script
8+
surface still loads. #1417.
89

910
## The freeze rule
1011

1112
**Never edit a script under `scripts/` to make a parser change pass.**
1213

1314
A corpus script is a recording that already exists in the wild; editing it deletes the evidence the
14-
gate is here to keep. When a grammar change flips a verdict:
15+
gate is here to keep, and the pinned identities below make an edited script fail rather than pass.
16+
When a grammar change flips a verdict:
1517

1618
1. Leave the script byte-identical.
1719
2. Update its `verdict` in `manifest.ts` in the same PR.
@@ -31,8 +33,21 @@ enough to put in front of whoever recorded the script.
3133
(`${VAR}` parameterization, reserved/ordering `env` refusals, `wait stable` and landmark waits,
3234
`target-v1` annotations, retired `gesture rotate` velocity and `gesture swipe` durationMs).
3335

34-
Provenance lives in `manifest.ts`: `recordedBy` is the released tag whose grammar produced the
35-
surface, and `source` is where it was mined from at that tag.
36+
## How "frozen" and "released" are enforced, not asserted
37+
38+
Provenance lives in `manifest.ts`. `recordedBy` is the released tag whose grammar produced the
39+
surface, and `provenance` pins the bytes:
40+
41+
- `mined` entries carry the **git object id of the historical blob**. `corpus.test.ts` hashes the
42+
checked-in bytes and must reproduce that id, so a rewritten script cannot be made green by editing
43+
the manifest to match — the id is only obtainable from the released content.
44+
- `derived` entries (`scripts/docs/`) have no historical blob to point at, so their bytes are pinned
45+
by SHA-256.
46+
47+
`pnpm check:replay-compat` closes the other half: it re-derives every mined id from git history
48+
(`git rev-parse <recordedBy>:<path>`) and checks every cited tag against `git tag --list`, so an
49+
entry cannot claim a version that was never cut. It needs full history and tags, so it is a
50+
maintainer/reviewer check rather than part of the shallow-clone-safe unit lane.
3651

3752
**Released surfaces only.** A grammar state that only ever existed between commits is not compat
3853
surface — `git tag --contains <commit>` decides, and unreleased shapes stay out (AGENTS.md,
@@ -42,8 +57,10 @@ surface — `git tag --contains <commit>` decides, and unreleased shapes stay ou
4257

4358
1. Find the surface at a release tag (`git show <tag>:<path>`, or the released grammar/docs that
4459
emitted it) and copy it verbatim into `scripts/<area>/<name>.<tag>.ad`.
45-
2. Add a `manifest.ts` entry with `recordedBy`, `source`, `covers`, and the observed verdict.
46-
3. Run `pnpm exec vitest run --project unit-core test/replay-compat`.
60+
2. Add a `manifest.ts` entry with `recordedBy`, `provenance`, `covers`, and the observed verdict. For
61+
a mined entry the blob id is `git rev-parse <tag>:<path>`; for a derived one it is
62+
`shasum -a 256 <file>`. Add the tag to `REPLAY_COMPAT_RELEASED_TAGS` if it is new.
63+
3. Run `pnpm exec vitest run --project unit-core test/replay-compat` and `pnpm check:replay-compat`.
4764

4865
Add an entry when a change retires, renames, or narrows a `.ad` form — the corpus is the record of
4966
what that costs someone with a saved recording.

‎test/replay-compat/corpus.test.ts‎

Lines changed: 33 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
import { describe, expect, test } from 'vitest';
2+
import { createHash } from 'node:crypto';
23
import { readdirSync, readFileSync } from 'node:fs';
34
import { join } from 'node:path';
45
import { fileURLToPath } from 'node:url';
6+
import { parseReplayInput } from '../../src/compat/replay-input.ts';
57
import { AppError } from '../../src/kernel/errors.ts';
6-
import { parseReplayScriptDetailed, readReplayScriptMetadata } from '../../src/replay/script.ts';
78
import {
89
REPLAY_COMPAT_CORPUS,
10+
REPLAY_COMPAT_RELEASED_TAGS,
911
type ReplayCompatCoverage,
1012
type ReplayCompatEntry,
1113
} from './manifest.ts';
@@ -25,14 +27,13 @@ const REQUIRED_COVERAGE: ReplayCompatCoverage[] = [
2527
'wait-landmark',
2628
];
2729

28-
function readCorpusScript(entry: ReplayCompatEntry): string {
29-
return readFileSync(join(CORPUS_DIR, entry.file), 'utf8');
30+
function readCorpusScript(entry: ReplayCompatEntry): Buffer {
31+
return readFileSync(join(CORPUS_DIR, entry.file));
3032
}
3133

32-
/** The whole parse surface a `.ad` file meets before replay executes anything. */
33-
function parseCorpusScript(script: string): void {
34-
readReplayScriptMetadata(script);
35-
parseReplayScriptDetailed(script);
34+
/** Git's own object id for a blob, so a mined entry's bytes carry their history. */
35+
function gitBlobId(bytes: Buffer): string {
36+
return createHash('sha1').update(`blob ${bytes.length}\0`).update(bytes).digest('hex');
3637
}
3738

3839
function listCorpusScriptFiles(): string[] {
@@ -45,17 +46,21 @@ function listCorpusScriptFiles(): string[] {
4546
}
4647

4748
describe('replay-compat corpus', () => {
49+
// The verdict is asserted through `parseReplayInput` — the same composition
50+
// (and the same ordering of the action parse and the metadata read) that
51+
// `replay`/`test` hit — so a multi-fault script reports the code and hint
52+
// production really surfaces, and moving the boundary breaks this gate.
4853
test.each(REPLAY_COMPAT_CORPUS.map((entry) => [entry.id, entry] as const))('%s', (_id, entry) => {
49-
const script = readCorpusScript(entry);
54+
const script = readCorpusScript(entry).toString('utf8');
5055
if (entry.verdict.kind === 'parses') {
5156
expect(() => {
52-
parseCorpusScript(script);
57+
parseReplayInput(script, undefined);
5358
}).not.toThrow();
5459
return;
5560
}
5661
let thrown: unknown;
5762
try {
58-
parseCorpusScript(script);
63+
parseReplayInput(script, undefined);
5964
} catch (error) {
6065
thrown = error;
6166
}
@@ -65,18 +70,33 @@ describe('replay-compat corpus', () => {
6570
expect(error.message).toContain(entry.verdict.hint);
6671
});
6772

73+
// The freeze rule needs teeth: a mined entry's bytes must still hash to the
74+
// git object id of the blob it was mined from, so "update the script until the
75+
// parser agrees" cannot pass as a corpus update. `pnpm check:replay-compat`
76+
// re-derives the same ids from history, which this hash alone cannot do.
77+
test.each(REPLAY_COMPAT_CORPUS.map((entry) => [entry.id, entry] as const))(
78+
'%s is byte-identical to its pinned source',
79+
(_id, entry) => {
80+
const bytes = readCorpusScript(entry);
81+
if (entry.provenance.kind === 'mined') {
82+
expect(gitBlobId(bytes)).toBe(entry.provenance.blob);
83+
return;
84+
}
85+
expect(createHash('sha256').update(bytes).digest('hex')).toBe(entry.provenance.sha256);
86+
},
87+
);
88+
6889
test('every corpus script is claimed by exactly one manifest entry', () => {
6990
const claimed = REPLAY_COMPAT_CORPUS.map((entry) => entry.file);
7091
expect([...claimed].sort()).toEqual(listCorpusScriptFiles().sort());
7192
expect(new Set(claimed).size).toBe(claimed.length);
7293
});
7394

74-
test('entry ids are unique and name the producing released version', () => {
95+
test('entry ids are unique and name a released producing version', () => {
7596
const ids = REPLAY_COMPAT_CORPUS.map((entry) => entry.id);
7697
expect(new Set(ids).size).toBe(ids.length);
7798
for (const entry of REPLAY_COMPAT_CORPUS) {
78-
expect(entry.recordedBy, entry.id).toMatch(/^v\d+\.\d+\.\d+$/);
79-
expect(entry.source, entry.id).toContain(entry.recordedBy);
99+
expect(REPLAY_COMPAT_RELEASED_TAGS, entry.id).toContain(entry.recordedBy);
80100
expect(entry.file, entry.id).toContain(`.${entry.recordedBy}.ad`);
81101
}
82102
});

0 commit comments

Comments
 (0)