55// open / mutate / close / takeover / kill against fake providers (the real
66// `SessionStore` + `LeaseRegistry`, an in-memory device-claim model), with ALL
77// concurrency routed through a deterministic scheduler so a seed fully
8- // determines execution order. After every run the harness asserts:
8+ // determines execution order. Each operation's lock plan is derived from the
9+ // production router primitive `resolveRequestExecutionLockKeys`, so reverting
10+ // the router's same-device serialization trips the overlap invariant. After
11+ // every run the harness asserts:
912// - no leaked leases or claims,
1013// - no cross-session state bleed,
1114// - every lock released after owner death,
1215// - the session store stays consistent,
13- // - same-device critical sections never overlap (this pins the router's
14- // same-device open serialization under many interleavings).
16+ // - same-device critical sections never overlap.
1517//
1618// Seed replay (documented in docs/agents/testing.md):
1719// TORTURE_SEED=1234 pnpm test:concurrency-torture
18- // replays that exact interleaving deterministically. Otherwise the lane sweeps
19- // TORTURE_RUNS seeds (default 128, ≥100 to satisfy the acceptance bar) starting
20- // at TORTURE_SEED_START (default 0). Any failure prints the seed and the exact
21- // replay command.
20+ // replays that exact interleaving deterministically — the whole scheduler trace,
21+ // not just its length, is asserted equal. Otherwise the lane sweeps TORTURE_RUNS
22+ // seeds (default 128, ≥100 to satisfy the acceptance bar) starting at
23+ // TORTURE_SEED_START (default 0). Any failure prints the seed and replay command.
2224
2325import test from 'node:test' ;
2426import assert from 'node:assert/strict' ;
2527
26- import { runTorture , type TortureRunResult } from './concurrency-torture/harness.ts' ;
28+ import { runTorture , type ClientOp , type TortureRunResult } from './concurrency-torture/harness.ts' ;
29+ import { buildEnvelope , writeEnvelopeIfRequested } from './concurrency-torture/envelope.ts' ;
2730
2831function intFromEnv ( name : string , fallback : number ) : number {
2932 const raw = process . env [ name ] ?. trim ( ) ;
@@ -66,13 +69,23 @@ const opsPerClient = optionalIntFromEnv('TORTURE_OPS');
6669if ( explicitSeed !== undefined ) {
6770 test ( `concurrency torture — replay seed ${ explicitSeed } ` , async ( ) => {
6871 const result = await runTorture ( { seed : explicitSeed , clients, opsPerClient } ) ;
69- // Determinism self-check: the same seed must reproduce the same schedule.
72+ // Determinism self-check: the same seed must reproduce the EXACT schedule,
73+ // the same terminal invariant outcome, and the same contention profile.
7074 const replay = await runTorture ( { seed : explicitSeed , clients, opsPerClient } ) ;
7175 assert . equal (
72- replay . scheduleLength ,
73- result . scheduleLength ,
74- `seed ${ explicitSeed } produced a different schedule length on replay ` +
75- `(${ result . scheduleLength } vs ${ replay . scheduleLength } ) — non-determinism` ,
76+ replay . traceSignature ,
77+ result . traceSignature ,
78+ `seed ${ explicitSeed } produced a different scheduler trace on replay — non-determinism` ,
79+ ) ;
80+ assert . deepEqual (
81+ replay . failures ,
82+ result . failures ,
83+ `seed ${ explicitSeed } produced a different invariant outcome on replay — non-determinism` ,
84+ ) ;
85+ assert . equal (
86+ replay . deviceContention ,
87+ result . deviceContention ,
88+ `seed ${ explicitSeed } produced different device contention on replay — non-determinism` ,
7689 ) ;
7790 assertClean ( result ) ;
7891 } ) ;
@@ -81,20 +94,66 @@ if (explicitSeed !== undefined) {
8194 const seedStart = intFromEnv ( 'TORTURE_SEED_START' , 0 ) ;
8295
8396 test ( `concurrency torture — ${ runs } seeded interleavings from ${ seedStart } ` , async ( ) => {
84- let exercisedSerialization = false ;
85- for ( let i = 0 ; i < runs ; i += 1 ) {
86- const seed = seedStart + i ;
87- const result = await runTorture ( { seed, clients, opsPerClient } ) ;
88- assertClean ( result ) ;
89- if ( Object . values ( result . perDeviceMaxConcurrency ) . some ( ( max ) => max >= 1 ) ) {
90- exercisedSerialization = true ;
97+ const started = Date . now ( ) ;
98+ let totalContention = 0 ;
99+ let failed = false ;
100+ try {
101+ for ( let i = 0 ; i < runs ; i += 1 ) {
102+ const seed = seedStart + i ;
103+ const result = await runTorture ( { seed, clients, opsPerClient } ) ;
104+ totalContention += result . deviceContention ;
105+ if ( result . failures . length > 0 ) failed = true ;
106+ assertClean ( result ) ;
91107 }
108+ // The sweep must actually PRODUCE same-device contention — two clients
109+ // parked on one device lock — or the lane is not exercising serialization
110+ // and a broken lock could pass unnoticed.
111+ assert . ok (
112+ totalContention > 0 ,
113+ 'no same-device lock contention occurred across the sweep — the lane is not testing serialization' ,
114+ ) ;
115+ } catch ( error ) {
116+ failed = true ;
117+ throw error ;
118+ } finally {
119+ const envelope = buildEnvelope ( {
120+ seedStart,
121+ runs,
122+ durationMs : Date . now ( ) - started ,
123+ result : failed ? 'fail' : 'pass' ,
124+ } ) ;
125+ const written = writeEnvelopeIfRequested ( envelope ) ;
126+ if ( written ) console . log ( `torture envelope → ${ written } \n${ JSON . stringify ( envelope ) } ` ) ;
127+ }
128+ } ) ;
129+
130+ // Forced same-device contention: two clients repeatedly open THE SAME pinned
131+ // device. This deterministically drives both onto one `device:` lock so the
132+ // overlap invariant is exercised, not just present. If the router stopped
133+ // serializing same-device opens, `deviceContention` here would collapse and
134+ // the overlap invariant would fire.
135+ test ( 'concurrency torture — forced two-client same-device contention' , async ( ) => {
136+ const program : ClientOp [ ] [ ] = [
137+ [ 'open' , 'mutate' , 'close' , 'open' , 'mutate' ] ,
138+ [ 'open' , 'mutate' , 'close' , 'open' , 'mutate' ] ,
139+ ] ;
140+ let contendedSeeds = 0 ;
141+ for ( let seed = 0 ; seed < 40 ; seed += 1 ) {
142+ const result = await runTorture ( { seed, program } ) ;
143+ assertClean ( result ) ;
144+ const perDeviceMax = Object . values ( result . perDeviceMaxConcurrency ) ;
145+ assert . ok (
146+ perDeviceMax . every ( ( max ) => max <= 1 ) ,
147+ `seed ${ seed } : same-device critical sections overlapped — ${ JSON . stringify (
148+ result . perDeviceMaxConcurrency ,
149+ ) } `,
150+ ) ;
151+ if ( result . deviceContention > 0 ) contendedSeeds += 1 ;
92152 }
93- // Sanity: the lane must actually enter device critical sections, otherwise a
94- // regression could hollow it out into a no-op that always "passes".
95153 assert . ok (
96- exercisedSerialization ,
97- 'no device critical section was exercised across the sweep — the lane is not testing serialization' ,
154+ contendedSeeds > 0 ,
155+ 'two clients never actually contended for the pinned device across 40 seeds — ' +
156+ 'the forced-contention scenario is not exercising the device lock' ,
98157 ) ;
99158 } ) ;
100159}
0 commit comments