|
| 1 | +/** |
| 2 | + * @vitest-environment jsdom |
| 3 | + * |
| 4 | + * The async app generator's campaign. Off by default; run with |
| 5 | + * |
| 6 | + * ASYNC_GEN=1 [ASYNC_GEN_SEED=3289] [ASYNC_GEN_CASES=200] |
| 7 | + * [ASYNC_GEN_IGNORE=final-screen,console-error] [ASYNC_GEN_MODE=survey|shrink] |
| 8 | + * npx vitest run test/async-gen |
| 9 | + * |
| 10 | + * ASYNC_GEN_RECORD / ASYNC_GEN_DIFF (survey mode): see differential.ts. |
| 11 | + * |
| 12 | + * survey runs every case and tallies findings by law and signature; shrink |
| 13 | + * stops on the first finding outside ASYNC_GEN_IGNORE and prints the reduced |
| 14 | + * scenario as JSON for replay.spec.ts. |
| 15 | + */ |
| 16 | +import { describe, expect, test } from "vitest"; |
| 17 | +import fc from "fast-check"; |
| 18 | +import { recordOrDiff, type Trace } from "./differential.js"; |
| 19 | +import { minimize } from "./minimize.js"; |
| 20 | +import { runScenario, type Finding, type Result, type Variant } from "./run.js"; |
| 21 | +import { describeScenario, scenarioArb, type Scenario } from "./scenario.js"; |
| 22 | + |
| 23 | +const FUZZ = !!process.env.ASYNC_GEN; |
| 24 | +const SEED = Number(process.env.ASYNC_GEN_SEED ?? 3289); |
| 25 | +const CASES = Number(process.env.ASYNC_GEN_CASES ?? 200); |
| 26 | +const IGNORE = new Set((process.env.ASYNC_GEN_IGNORE ?? "").split(",").filter(Boolean)); |
| 27 | +const MODE = process.env.ASYNC_GEN_MODE ?? "survey"; |
| 28 | + |
| 29 | +const PAIRS = (process.env.ASYNC_GEN_PAIRS ?? "").split(",").filter(Boolean); |
| 30 | + |
| 31 | +const out = (s: string) => process.stdout.write(s + "\n"); |
| 32 | + |
| 33 | +/** The equivalence pairs: two runs of one scenario that must show the same |
| 34 | + * screen at every settle point (or, `finalOnly`, once everything settled). */ |
| 35 | +const PAIR_DEFS: Record< |
| 36 | + string, |
| 37 | + { a: Variant; b: Variant; finalOnly?: boolean; quietOnly?: boolean } |
| 38 | +> = { |
| 39 | + E1: { a: { grouped: true }, b: { split: true } }, |
| 40 | + E2: { a: {}, b: { identity: true } }, |
| 41 | + E2e: { a: {}, b: { inertErrored: true } }, |
| 42 | + E3: { a: {}, b: { store: true } }, |
| 43 | + E6: { a: {}, b: { autoFlush: true }, finalOnly: true }, |
| 44 | + E7: { a: { owned: false }, b: { owned: true } }, |
| 45 | + // The reader seat (carve audit): a reader born before a write vs one born |
| 46 | + // in the flush that applies it, compared where nothing is in flight (a |
| 47 | + // fresh reader of a pending node may show a fallback the old one holds); |
| 48 | + // `latest(x)` read in a derivation vs at the leaf. |
| 49 | + E8: { a: {}, b: { remount: true }, quietOnly: true }, |
| 50 | + E9: { a: {}, b: { latestDerive: true } } |
| 51 | +}; |
| 52 | + |
| 53 | +/** What a screen shows, coarsely: an error, a fallback, or content. */ |
| 54 | +const shows = (x: string) => |
| 55 | + /\[(ROOT-)?E\]/.test(x) |
| 56 | + ? "error" |
| 57 | + : /\[(ROOT-)?L\]/.test(x) |
| 58 | + ? "loading" |
| 59 | + : /\[F\]/.test(x) |
| 60 | + ? "show-fallback" |
| 61 | + : "content"; |
| 62 | +/** How two screens differ, for grouping: by what they show, then by which |
| 63 | + * kind of read disagrees (value / latest / isPending). */ |
| 64 | +function kind(x: string, y: string): string { |
| 65 | + const [a, b] = [shows(x), shows(y)]; |
| 66 | + if (a !== b) return `${a} vs ${b}`; |
| 67 | + const reads = (s: string) => |
| 68 | + new Map([...s.matchAll(/([vlp])([sm]\d):([^ <]*)/g)].map(m => [m[1] + m[2], m[3]])); |
| 69 | + const [ra, rb] = [reads(x), reads(y)]; |
| 70 | + const diff = new Set<string>(); |
| 71 | + for (const [k, v] of ra) |
| 72 | + if (rb.get(k) !== v) diff.add(k[0] === "v" ? "value" : k[0] === "l" ? "latest" : "isPending"); |
| 73 | + for (const k of rb.keys()) if (!ra.has(k)) diff.add("presence"); |
| 74 | + return diff.size ? `${a}: ${[...diff].sort().join("+")} differ` : `${a}: structure differs`; |
| 75 | +} |
| 76 | + |
| 77 | +/** Runs the scenario's pairs; returns their findings, plus the base run's. */ |
| 78 | +async function runWithPairs(sc: Scenario): Promise<{ base: Result; findings: Finding[] }> { |
| 79 | + const base = await runScenario(sc); |
| 80 | + const findings = [...base.findings]; |
| 81 | + for (const name of PAIRS) { |
| 82 | + const def = PAIR_DEFS[name]; |
| 83 | + const a = Object.keys(def.a).length ? await runScenario(sc, def.a) : base; |
| 84 | + const b = await runScenario(sc, def.b); |
| 85 | + const diff = def.finalOnly |
| 86 | + ? a.final !== b.final |
| 87 | + ? a.screens.length - 1 |
| 88 | + : -1 |
| 89 | + : a.screens.findIndex( |
| 90 | + (x, i) => |
| 91 | + a.settled[i] && |
| 92 | + b.settled[i] && |
| 93 | + (!def.quietOnly || (a.quiet[i] && b.quiet[i])) && |
| 94 | + x !== b.screens[i] |
| 95 | + ); |
| 96 | + const at = diff >= 0 ? diff : a.final !== b.final ? a.screens.length : -1; |
| 97 | + if (at >= 0) { |
| 98 | + const x = at < a.screens.length ? a.screens[at] : a.final; |
| 99 | + const y = at < b.screens.length ? b.screens[at] : b.final; |
| 100 | + findings.push({ |
| 101 | + oracle: "equivalence", |
| 102 | + law: name, |
| 103 | + step: at - 1, |
| 104 | + detail: `${kind(x, y)} :: ${x} ≠ ${y}` |
| 105 | + }); |
| 106 | + } |
| 107 | + } |
| 108 | + return { base, findings }; |
| 109 | +} |
| 110 | +const MATCH = process.env.ASYNC_GEN_MATCH ? new RegExp(process.env.ASYNC_GEN_MATCH) : null; |
| 111 | +const real = (fs: Finding[]) => |
| 112 | + fs.filter(f => !IGNORE.has(f.law) && (!MATCH || MATCH.test(`${f.oracle}/${f.law}: ${f.detail}`))); |
| 113 | +/** A signature groups findings by law and the first words of the detail, |
| 114 | + * with numbers and values stripped. */ |
| 115 | +const signature = (f: Finding) => |
| 116 | + f.oracle === "equivalence" |
| 117 | + ? `${f.oracle}/${f.law}: ${f.detail.slice(0, f.detail.indexOf(" :: "))}` |
| 118 | + : `${f.oracle}/${f.law}: ${f.detail.replace(/\d+/g, "#").replace(/≠.*/, "≠ …").slice(0, 90)}`; |
| 119 | + |
| 120 | +describe("async-gen", () => { |
| 121 | + test("smoke: a settled async memo under Loading shows its value", async () => { |
| 122 | + const result = await runScenario({ |
| 123 | + signals: [1], |
| 124 | + memos: [{ async: true, deps: [{ src: { k: "s", i: 0 }, how: "plain" }], poison: null }], |
| 125 | + tree: { |
| 126 | + t: "leaf", |
| 127 | + text: [{ how: "plain", src: { k: "m", i: 0 } }], |
| 128 | + attrs: [], |
| 129 | + grouped: true |
| 130 | + }, |
| 131 | + steps: [{ t: "settle", pick: 0 }, { t: "task" }, { t: "write", s: 0, v: 2 }, { t: "task" }] |
| 132 | + }); |
| 133 | + expect(result.findings).toEqual([]); |
| 134 | + expect(result.final).toBe("<span>vm0:3 </span>"); |
| 135 | + }); |
| 136 | +}); |
| 137 | + |
| 138 | +describe.skipIf(!FUZZ)("async-gen campaign", () => { |
| 139 | + test(`seed ${SEED}, ${CASES} cases, mode ${MODE}`, { timeout: 0 }, async () => { |
| 140 | + const arb = scenarioArb(); |
| 141 | + if (MODE === "shrink") { |
| 142 | + try { |
| 143 | + await fc.assert( |
| 144 | + fc.asyncProperty(arb, async sc => { |
| 145 | + const r = await runWithPairs(sc); |
| 146 | + const f = real(r.findings)[0]; |
| 147 | + if (f) |
| 148 | + throw new Error( |
| 149 | + `${f.oracle}/${f.law} @${f.step}: ${f.detail}\n${r.base.description}\n${JSON.stringify(sc)}` |
| 150 | + ); |
| 151 | + }), |
| 152 | + { seed: SEED, numRuns: CASES, endOnFailure: true } |
| 153 | + ); |
| 154 | + out(`async-gen: seed ${SEED}, ${CASES} cases, no finding outside [${[...IGNORE]}]`); |
| 155 | + } catch (e: any) { |
| 156 | + out(String(e?.message ?? e)); |
| 157 | + throw e; |
| 158 | + } |
| 159 | + return; |
| 160 | + } |
| 161 | + const tally = new Map<string, { count: number; first: Scenario; detail: string }>(); |
| 162 | + let failing = 0; |
| 163 | + const traces: Record<number, Trace> = {}; |
| 164 | + let idx = 0; |
| 165 | + for (const sc of fc.sample(arb, { seed: SEED, numRuns: CASES })) { |
| 166 | + const r = await runWithPairs(sc); |
| 167 | + traces[idx++] = { |
| 168 | + screens: r.base.screens, |
| 169 | + final: r.base.final, |
| 170 | + laws: [...new Set(r.findings.map(f => `${f.oracle}/${f.law}`))].sort() |
| 171 | + }; |
| 172 | + const fs = real(r.findings); |
| 173 | + if (fs.length) failing++; |
| 174 | + const seen = new Set<string>(); |
| 175 | + for (const f of fs) { |
| 176 | + const sig = signature(f); |
| 177 | + if (seen.has(sig)) continue; |
| 178 | + seen.add(sig); |
| 179 | + const t = tally.get(sig); |
| 180 | + if (t) t.count++; |
| 181 | + else tally.set(sig, { count: 1, first: sc, detail: `@${f.step} ${f.detail}` }); |
| 182 | + } |
| 183 | + } |
| 184 | + const diffs = recordOrDiff(`client${PAIRS.length ? `+${PAIRS}` : ""}`, SEED, CASES, traces); |
| 185 | + if (diffs.length) |
| 186 | + out(`async-gen differential: ${diffs.length} cases differ\n ${diffs.join("\n ")}`); |
| 187 | + out( |
| 188 | + `async-gen survey: seed ${SEED}, ${CASES} cases, ${failing} with findings, ${tally.size} signatures` |
| 189 | + ); |
| 190 | + for (const [sig, t] of [...tally].sort((a, b) => b[1].count - a[1].count)) { |
| 191 | + out(` ${t.count}× ${sig}`); |
| 192 | + out(` ${describeScenario(t.first)}`); |
| 193 | + out(` ${t.detail.slice(0, 300)}`); |
| 194 | + if (MODE !== "minimize") continue; |
| 195 | + // Reduce while some finding keeps the same oracle/law. |
| 196 | + const law = sig.slice(0, sig.indexOf(":")); |
| 197 | + const min = await minimize(t.first, async sc => |
| 198 | + real((await runWithPairs(sc)).findings).some(f => `${f.oracle}/${f.law}` === law) |
| 199 | + ); |
| 200 | + const r = await runWithPairs(min); |
| 201 | + const f = real(r.findings).find(f => `${f.oracle}/${f.law}` === law)!; |
| 202 | + out(` MIN ${r.base.description}`); |
| 203 | + out(` MIN @${f.step} ${f.detail.slice(0, 300)}`); |
| 204 | + out(` MIN ${JSON.stringify(min)}`); |
| 205 | + } |
| 206 | + expect(diffs, "cases that differ from ASYNC_GEN_DIFF's recording").toEqual([]); |
| 207 | + }); |
| 208 | +}); |
0 commit comments