|
| 1 | +// Seeded concurrency torture lane for session / lease / lock invariants (#1416, |
| 2 | +// umbrella #1412 Track A). |
| 3 | +// |
| 4 | +// N concurrent clients drive randomized-but-SEEDED interleavings of |
| 5 | +// open / mutate / close / takeover / kill against fake providers (the real |
| 6 | +// `SessionStore` + `LeaseRegistry`, an in-memory device-claim model), with ALL |
| 7 | +// concurrency routed through a deterministic scheduler so a seed fully |
| 8 | +// determines execution order. After every run the harness asserts: |
| 9 | +// - no leaked leases or claims, |
| 10 | +// - no cross-session state bleed, |
| 11 | +// - every lock released after owner death, |
| 12 | +// - the session store stays consistent, |
| 13 | +// - same-device critical sections never overlap (this pins the router's |
| 14 | +// same-device open serialization under many interleavings). |
| 15 | +// |
| 16 | +// Seed replay (documented in docs/agents/testing.md): |
| 17 | +// TORTURE_SEED=1234 pnpm test:concurrency-torture |
| 18 | +// replays that exact interleaving deterministically. Otherwise the lane sweeps |
| 19 | +// TORTURE_RUNS seeds (default 128, ≥100 to satisfy the acceptance bar) starting |
| 20 | +// at TORTURE_SEED_START (default 0). Any failure prints the seed and the exact |
| 21 | +// replay command. |
| 22 | + |
| 23 | +import test from 'node:test'; |
| 24 | +import assert from 'node:assert/strict'; |
| 25 | + |
| 26 | +import { runTorture, type TortureRunResult } from './concurrency-torture/harness.ts'; |
| 27 | + |
| 28 | +function intFromEnv(name: string, fallback: number): number { |
| 29 | + const raw = process.env[name]?.trim(); |
| 30 | + if (!raw) return fallback; |
| 31 | + const parsed = Number(raw); |
| 32 | + if (!Number.isInteger(parsed) || parsed < 0) { |
| 33 | + throw new Error(`${name} must be a non-negative integer, got ${raw}`); |
| 34 | + } |
| 35 | + return parsed; |
| 36 | +} |
| 37 | + |
| 38 | +function optionalIntFromEnv(name: string): number | undefined { |
| 39 | + const raw = process.env[name]?.trim(); |
| 40 | + if (!raw) return undefined; |
| 41 | + const parsed = Number(raw); |
| 42 | + if (!Number.isInteger(parsed) || parsed <= 0) { |
| 43 | + throw new Error(`${name} must be a positive integer, got ${raw}`); |
| 44 | + } |
| 45 | + return parsed; |
| 46 | +} |
| 47 | + |
| 48 | +function replayHint(seed: number): string { |
| 49 | + return `Replay this exact interleaving with: TORTURE_SEED=${seed} pnpm test:concurrency-torture`; |
| 50 | +} |
| 51 | + |
| 52 | +function assertClean(result: TortureRunResult): void { |
| 53 | + if (result.failures.length === 0) return; |
| 54 | + const lines = result.failures.map((f) => ` - [${f.invariant}] ${f.detail}`); |
| 55 | + assert.fail( |
| 56 | + `Concurrency invariants violated on seed ${result.seed} ` + |
| 57 | + `(${result.clients} clients, ${result.ops} ops, ${result.scheduleLength} scheduled steps):\n` + |
| 58 | + `${lines.join('\n')}\n${replayHint(result.seed)}`, |
| 59 | + ); |
| 60 | +} |
| 61 | + |
| 62 | +const explicitSeed = optionalIntFromEnv('TORTURE_SEED'); |
| 63 | +const clients = optionalIntFromEnv('TORTURE_CLIENTS'); |
| 64 | +const opsPerClient = optionalIntFromEnv('TORTURE_OPS'); |
| 65 | + |
| 66 | +if (explicitSeed !== undefined) { |
| 67 | + test(`concurrency torture — replay seed ${explicitSeed}`, async () => { |
| 68 | + const result = await runTorture({ seed: explicitSeed, clients, opsPerClient }); |
| 69 | + // Determinism self-check: the same seed must reproduce the same schedule. |
| 70 | + const replay = await runTorture({ seed: explicitSeed, clients, opsPerClient }); |
| 71 | + assert.equal( |
| 72 | + replay.scheduleLength, |
| 73 | + result.scheduleLength, |
| 74 | + `seed ${explicitSeed} produced a different schedule length on replay ` + |
| 75 | + `(${result.scheduleLength} vs ${replay.scheduleLength}) — non-determinism`, |
| 76 | + ); |
| 77 | + assertClean(result); |
| 78 | + }); |
| 79 | +} else { |
| 80 | + const runs = intFromEnv('TORTURE_RUNS', 128); |
| 81 | + const seedStart = intFromEnv('TORTURE_SEED_START', 0); |
| 82 | + |
| 83 | + test(`concurrency torture — ${runs} seeded interleavings from ${seedStart}`, async () => { |
| 84 | + let exercisedSerialization = false; |
| 85 | + for (let i = 0; i < runs; i += 1) { |
| 86 | + const seed = seedStart + i; |
| 87 | + const result = await runTorture({ seed, clients, opsPerClient }); |
| 88 | + assertClean(result); |
| 89 | + if (Object.values(result.perDeviceMaxConcurrency).some((max) => max >= 1)) { |
| 90 | + exercisedSerialization = true; |
| 91 | + } |
| 92 | + } |
| 93 | + // Sanity: the lane must actually enter device critical sections, otherwise a |
| 94 | + // regression could hollow it out into a no-op that always "passes". |
| 95 | + assert.ok( |
| 96 | + exercisedSerialization, |
| 97 | + 'no device critical section was exercised across the sweep — the lane is not testing serialization', |
| 98 | + ); |
| 99 | + }); |
| 100 | +} |
0 commit comments