|
22 | 22 | // seeds (default 128, ≥100 to satisfy the acceptance bar) starting at |
23 | 23 | // TORTURE_SEED_START (default 0). Any failure prints the seed and replay command. |
24 | 24 |
|
25 | | -import test from 'node:test'; |
| 25 | +import test, { after } from 'node:test'; |
26 | 26 | import assert from 'node:assert/strict'; |
27 | 27 |
|
28 | 28 | import { runTorture, type ClientOp, type TortureRunResult } from './concurrency-torture/harness.ts'; |
| 29 | +import { DEVICE_POOL } from './concurrency-torture/bindings.ts'; |
29 | 30 | import { buildEnvelope, writeEnvelopeIfRequested } from './concurrency-torture/envelope.ts'; |
30 | 31 |
|
| 32 | +// The #1430 envelope must report the outcome of the WHOLE lane, not just the |
| 33 | +// sweep test. Every test records its result here and the envelope is written |
| 34 | +// once, after all tests settle, so a later failing guardrail can never be |
| 35 | +// published as a passing envelope. |
| 36 | +type EnvelopeRun = { seedStart: number; runs: number; durationMs: number }; |
| 37 | +let laneFailed = false; |
| 38 | +let sweepRun: EnvelopeRun | undefined; |
| 39 | + |
| 40 | +async function guard(body: () => Promise<void>): Promise<void> { |
| 41 | + try { |
| 42 | + await body(); |
| 43 | + } catch (error) { |
| 44 | + laneFailed = true; |
| 45 | + throw error; |
| 46 | + } |
| 47 | +} |
| 48 | + |
| 49 | +after(() => { |
| 50 | + if (!sweepRun) return; // only the sweep lane (nightly) publishes an envelope |
| 51 | + const envelope = buildEnvelope({ ...sweepRun, result: laneFailed ? 'fail' : 'pass' }); |
| 52 | + const written = writeEnvelopeIfRequested(envelope); |
| 53 | + if (written) console.log(`torture envelope → ${written}\n${JSON.stringify(envelope)}`); |
| 54 | +}); |
| 55 | + |
| 56 | +/** |
| 57 | + * Assert a seed replays bit-for-bit: same ordered scheduler trace, same terminal |
| 58 | + * invariant outcome, same contention profile. Run for EVERY seed in the sweep so |
| 59 | + * determinism is proven on the normal CI/nightly path, not only under TORTURE_SEED. |
| 60 | + */ |
| 61 | +function assertDeterministicReplay(result: TortureRunResult, replay: TortureRunResult): void { |
| 62 | + assert.equal( |
| 63 | + replay.traceSignature, |
| 64 | + result.traceSignature, |
| 65 | + `seed ${result.seed} produced a different scheduler trace on replay — non-determinism`, |
| 66 | + ); |
| 67 | + assert.deepEqual( |
| 68 | + replay.failures, |
| 69 | + result.failures, |
| 70 | + `seed ${result.seed} produced a different invariant outcome on replay — non-determinism`, |
| 71 | + ); |
| 72 | + assert.equal( |
| 73 | + replay.deviceContention, |
| 74 | + result.deviceContention, |
| 75 | + `seed ${result.seed} produced different device contention on replay — non-determinism`, |
| 76 | + ); |
| 77 | +} |
| 78 | + |
31 | 79 | function intFromEnv(name: string, fallback: number): number { |
32 | 80 | const raw = process.env[name]?.trim(); |
33 | 81 | if (!raw) return fallback; |
@@ -67,93 +115,73 @@ const clients = optionalIntFromEnv('TORTURE_CLIENTS'); |
67 | 115 | const opsPerClient = optionalIntFromEnv('TORTURE_OPS'); |
68 | 116 |
|
69 | 117 | if (explicitSeed !== undefined) { |
70 | | - test(`concurrency torture — replay seed ${explicitSeed}`, async () => { |
71 | | - const result = await runTorture({ seed: explicitSeed, clients, opsPerClient }); |
72 | | - // Determinism self-check: the same seed must reproduce the EXACT schedule, |
73 | | - // the same terminal invariant outcome, and the same contention profile. |
74 | | - const replay = await runTorture({ seed: explicitSeed, clients, opsPerClient }); |
75 | | - assert.equal( |
76 | | - replay.traceSignature, |
77 | | - result.traceSignature, |
78 | | - `seed ${explicitSeed} produced a different scheduler trace on replay — non-determinism`, |
79 | | - ); |
80 | | - assert.deepEqual( |
81 | | - replay.failures, |
82 | | - result.failures, |
83 | | - `seed ${explicitSeed} produced a different invariant outcome on replay — non-determinism`, |
84 | | - ); |
85 | | - assert.equal( |
86 | | - replay.deviceContention, |
87 | | - result.deviceContention, |
88 | | - `seed ${explicitSeed} produced different device contention on replay — non-determinism`, |
89 | | - ); |
90 | | - assertClean(result); |
91 | | - }); |
| 118 | + test(`concurrency torture — replay seed ${explicitSeed}`, () => |
| 119 | + guard(async () => { |
| 120 | + const result = await runTorture({ seed: explicitSeed, clients, opsPerClient }); |
| 121 | + const replay = await runTorture({ seed: explicitSeed, clients, opsPerClient }); |
| 122 | + assertDeterministicReplay(result, replay); |
| 123 | + assertClean(result); |
| 124 | + })); |
92 | 125 | } else { |
93 | 126 | const runs = intFromEnv('TORTURE_RUNS', 128); |
94 | 127 | const seedStart = intFromEnv('TORTURE_SEED_START', 0); |
95 | 128 |
|
96 | | - test(`concurrency torture — ${runs} seeded interleavings from ${seedStart}`, async () => { |
97 | | - const started = Date.now(); |
98 | | - let totalContention = 0; |
99 | | - let failed = false; |
100 | | - try { |
101 | | - for (let i = 0; i < runs; i += 1) { |
102 | | - const seed = seedStart + i; |
103 | | - const result = await runTorture({ seed, clients, opsPerClient }); |
104 | | - totalContention += result.deviceContention; |
105 | | - if (result.failures.length > 0) failed = true; |
106 | | - assertClean(result); |
| 129 | + test(`concurrency torture — ${runs} seeded interleavings from ${seedStart}`, () => |
| 130 | + guard(async () => { |
| 131 | + const started = Date.now(); |
| 132 | + let totalContention = 0; |
| 133 | + try { |
| 134 | + for (let i = 0; i < runs; i += 1) { |
| 135 | + const seed = seedStart + i; |
| 136 | + const result = await runTorture({ seed, clients, opsPerClient }); |
| 137 | + totalContention += result.deviceContention; |
| 138 | + // Prove exact replay on the NORMAL sweep path (not just TORTURE_SEED): |
| 139 | + // re-run the seed and assert the whole trace + outcome + contention match. |
| 140 | + assertDeterministicReplay(result, await runTorture({ seed, clients, opsPerClient })); |
| 141 | + assertClean(result); |
| 142 | + } |
| 143 | + // The sweep must actually PRODUCE same-device contention — two clients |
| 144 | + // parked on one device lock — or the lane is not exercising serialization |
| 145 | + // and a broken lock could pass unnoticed. |
| 146 | + assert.ok( |
| 147 | + totalContention > 0, |
| 148 | + 'no same-device lock contention occurred across the sweep — the lane is not testing serialization', |
| 149 | + ); |
| 150 | + } finally { |
| 151 | + sweepRun = { seedStart, runs, durationMs: Date.now() - started }; |
107 | 152 | } |
108 | | - // The sweep must actually PRODUCE same-device contention — two clients |
109 | | - // parked on one device lock — or the lane is not exercising serialization |
110 | | - // and a broken lock could pass unnoticed. |
111 | | - assert.ok( |
112 | | - totalContention > 0, |
113 | | - 'no same-device lock contention occurred across the sweep — the lane is not testing serialization', |
114 | | - ); |
115 | | - } catch (error) { |
116 | | - failed = true; |
117 | | - throw error; |
118 | | - } finally { |
119 | | - const envelope = buildEnvelope({ |
120 | | - seedStart, |
121 | | - runs, |
122 | | - durationMs: Date.now() - started, |
123 | | - result: failed ? 'fail' : 'pass', |
124 | | - }); |
125 | | - const written = writeEnvelopeIfRequested(envelope); |
126 | | - if (written) console.log(`torture envelope → ${written}\n${JSON.stringify(envelope)}`); |
127 | | - } |
128 | | - }); |
| 153 | + })); |
129 | 154 |
|
130 | 155 | // Forced same-device contention: two clients repeatedly open THE SAME pinned |
131 | | - // device. This deterministically drives both onto one `device:` lock so the |
132 | | - // overlap invariant is exercised, not just present. If the router stopped |
133 | | - // serializing same-device opens, `deviceContention` here would collapse and |
134 | | - // the overlap invariant would fire. |
135 | | - test('concurrency torture — forced two-client same-device contention', async () => { |
136 | | - const program: ClientOp[][] = [ |
137 | | - ['open', 'mutate', 'close', 'open', 'mutate'], |
138 | | - ['open', 'mutate', 'close', 'open', 'mutate'], |
139 | | - ]; |
140 | | - let contendedSeeds = 0; |
141 | | - for (let seed = 0; seed < 40; seed += 1) { |
142 | | - const result = await runTorture({ seed, program }); |
143 | | - assertClean(result); |
144 | | - const perDeviceMax = Object.values(result.perDeviceMaxConcurrency); |
| 156 | + // device (via `pinnedDevice`, so `doOpen` cannot randomly pick a different one). |
| 157 | + // This deterministically drives both onto one `device:` lock so the overlap |
| 158 | + // invariant is exercised, not just present. If the router stopped serializing |
| 159 | + // same-device opens, `deviceContention` here would collapse and the overlap |
| 160 | + // invariant would fire. |
| 161 | + test('concurrency torture — forced two-client same-device contention', () => |
| 162 | + guard(async () => { |
| 163 | + const pinnedDevice = DEVICE_POOL[0]; |
| 164 | + const program: ClientOp[][] = [ |
| 165 | + ['open', 'mutate', 'close', 'open', 'mutate'], |
| 166 | + ['open', 'mutate', 'close', 'open', 'mutate'], |
| 167 | + ]; |
| 168 | + let contendedSeeds = 0; |
| 169 | + for (let seed = 0; seed < 40; seed += 1) { |
| 170 | + const result = await runTorture({ seed, program, pinnedDevice }); |
| 171 | + assertClean(result); |
| 172 | + const perDeviceMax = Object.values(result.perDeviceMaxConcurrency); |
| 173 | + assert.ok( |
| 174 | + perDeviceMax.every((max) => max <= 1), |
| 175 | + `seed ${seed}: same-device critical sections overlapped — ${JSON.stringify( |
| 176 | + result.perDeviceMaxConcurrency, |
| 177 | + )}`, |
| 178 | + ); |
| 179 | + if (result.deviceContention > 0) contendedSeeds += 1; |
| 180 | + } |
145 | 181 | assert.ok( |
146 | | - perDeviceMax.every((max) => max <= 1), |
147 | | - `seed ${seed}: same-device critical sections overlapped — ${JSON.stringify( |
148 | | - result.perDeviceMaxConcurrency, |
149 | | - )}`, |
| 182 | + contendedSeeds > 0, |
| 183 | + 'two clients never actually contended for the pinned device across 40 seeds — ' + |
| 184 | + 'the forced-contention scenario is not exercising the device lock', |
150 | 185 | ); |
151 | | - if (result.deviceContention > 0) contendedSeeds += 1; |
152 | | - } |
153 | | - assert.ok( |
154 | | - contendedSeeds > 0, |
155 | | - 'two clients never actually contended for the pinned device across 40 seeds — ' + |
156 | | - 'the forced-contention scenario is not exercising the device lock', |
157 | | - ); |
158 | | - }); |
| 186 | + })); |
159 | 187 | } |
0 commit comments