Skip to content

Commit 615f51f

Browse files
committed
test(web): async-gen regression gate against a recorded baseline
1,500 generated scenarios (seed 3289, ~15 s) run with the web suite; each case's safety and liveness laws are compared to baseline.json. A case that newly breaks a law fails the suite (a change broke a neighbouring shape); a recorded failure that no longer reproduces is reported, and ASYNC_GEN_UPDATE=1 records it. The baseline holds today's 20 failing cases (#4362-#4367 shapes). Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_0117BoWKJDzwaZPgaGWE181b
1 parent c4d5391 commit 615f51f

2 files changed

Lines changed: 132 additions & 0 deletions

File tree

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
{
2+
"seed": 3289,
3+
"cases": 1500,
4+
"failing": {
5+
"112": [
6+
"safety/undefined-value"
7+
],
8+
"296": [
9+
"safety/undefined-value"
10+
],
11+
"310": [
12+
"safety/undefined-value"
13+
],
14+
"349": [
15+
"safety/undefined-value"
16+
],
17+
"465": [
18+
"liveness/final-screen"
19+
],
20+
"662": [
21+
"liveness/final-screen"
22+
],
23+
"752": [
24+
"safety/undefined-value"
25+
],
26+
"766": [
27+
"liveness/final-screen"
28+
],
29+
"858": [
30+
"safety/unhandled-rejection"
31+
],
32+
"864": [
33+
"safety/undefined-value"
34+
],
35+
"1046": [
36+
"liveness/final-screen"
37+
],
38+
"1091": [
39+
"liveness/final-screen"
40+
],
41+
"1188": [
42+
"liveness/final-screen"
43+
],
44+
"1222": [
45+
"liveness/final-screen"
46+
],
47+
"1237": [
48+
"liveness/final-screen"
49+
],
50+
"1258": [
51+
"liveness/final-screen"
52+
],
53+
"1270": [
54+
"safety/undefined-value"
55+
],
56+
"1307": [
57+
"safety/undefined-value"
58+
],
59+
"1383": [
60+
"liveness/final-screen"
61+
],
62+
"1455": [
63+
"safety/undefined-value"
64+
]
65+
}
66+
}
Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
/**
2+
* @vitest-environment jsdom
3+
*
4+
* The async-gen regression gate: a fixed sample of generated scenarios runs
5+
* with every suite, and each case's safety and liveness laws are compared to
6+
* `baseline.json`, the laws that case broke when the baseline was recorded.
7+
*
8+
* - A case breaking a law it did not break before fails the suite: a change
9+
* broke a neighbouring shape (the fix-on-fix pattern).
10+
* - A case that no longer breaks a recorded law is reported, not failed; run
11+
* with ASYNC_GEN_UPDATE=1 to record the improvement.
12+
*
13+
* ASYNC_GEN_UPDATE=1 npx vitest run test/async-gen/regression.spec.ts
14+
*/
15+
import { readFileSync, writeFileSync } from "node:fs";
16+
import { resolve } from "node:path";
17+
import { expect, test } from "vitest";
18+
import fc from "fast-check";
19+
import { runScenario } from "./run.js";
20+
import { describeScenario, scenarioArb } from "./scenario.js";
21+
22+
const SEED = 3289;
23+
const CASES = 1500;
24+
// vitest runs from the package root (jsdom gives import.meta.url no file scheme).
25+
const BASELINE = resolve(process.cwd(), "test/async-gen/baseline.json");
26+
27+
type Baseline = { seed: number; cases: number; failing: Record<string, string[]> };
28+
29+
test(
30+
`async-gen regression gate: seed ${SEED}, ${CASES} cases against baseline.json`,
31+
{ timeout: 120_000 },
32+
async () => {
33+
let baseline: Baseline = { seed: SEED, cases: CASES, failing: {} };
34+
try {
35+
baseline = JSON.parse(readFileSync(BASELINE, "utf8"));
36+
} catch {}
37+
const failing: Record<string, string[]> = {};
38+
const regressions: string[] = [];
39+
const improvements: string[] = [];
40+
const scenarios = fc.sample(scenarioArb(), { seed: SEED, numRuns: CASES });
41+
for (let i = 0; i < scenarios.length; i++) {
42+
const r = await runScenario(scenarios[i]);
43+
const laws = [...new Set(r.findings.map(f => `${f.oracle}/${f.law}`))].sort();
44+
if (laws.length) failing[i] = laws;
45+
const was = new Set(baseline.failing[i] ?? []);
46+
for (const law of laws)
47+
if (!was.has(law))
48+
regressions.push(
49+
`#${i} ${law}\n ${describeScenario(scenarios[i])}\n ${r.findings.find(f => `${f.oracle}/${f.law}` === law)!.detail.slice(0, 200)}`
50+
);
51+
for (const law of was) if (!laws.includes(law)) improvements.push(`#${i} ${law}`);
52+
}
53+
if (process.env.ASYNC_GEN_UPDATE) {
54+
writeFileSync(
55+
BASELINE,
56+
JSON.stringify({ seed: SEED, cases: CASES, failing }, null, 2) + "\n"
57+
);
58+
return;
59+
}
60+
if (improvements.length)
61+
process.stdout.write(
62+
`async-gen: ${improvements.length} recorded failures no longer reproduce (ASYNC_GEN_UPDATE=1 to record):\n ${improvements.join("\n ")}\n`
63+
);
64+
expect(regressions, "cases that newly break a law (see baseline.json)").toEqual([]);
65+
}
66+
);

0 commit comments

Comments
 (0)