Skip to content

Commit 8e9001e

Browse files
test(daemon): add real-scope runLocked serialization guard; whole-lane envelope duration
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
1 parent 0890152 commit 8e9001e

3 files changed

Lines changed: 133 additions & 24 deletions

File tree

‎docs/agents/testing.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,13 @@ and process "kill" — the production claim is a filesystem/OS lock and real pro
142142
scope for this scheduling lane and covered by their own unit tests. The full real-vs-modeled boundary
143143
is documented at the top of `concurrency-torture/harness.ts`.
144144

145+
Because the seeded sweep *models* the mutex grant, a separate **real-scope guard**
146+
(`concurrency-torture/real-scope-serialization.ts`) drives concurrent same-device opens through the
147+
actual `createRequestExecutionScope().runLocked()` → `withRequestExecutionLocks` → `withKeyedLock`
148+
and asserts the critical sections never overlap. This is intentionally not seeded (it exercises real
149+
event-loop scheduling); its job is to fail if the production lock *application* path regresses, which
150+
the modeled sweep alone could not catch.
151+
145152
```bash
146153
pnpm test:concurrency-torture # default sweep (TORTURE_RUNS=128 seeds from 0)
147154
TORTURE_SEED=1234 pnpm test:concurrency-torture # replay ONE seed's exact interleaving (seed-replay flag)

‎test/integration/concurrency-torture.test.ts‎

Lines changed: 44 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,17 @@ import assert from 'node:assert/strict';
2727

2828
import { runTorture, type ClientOp, type TortureRunResult } from './concurrency-torture/harness.ts';
2929
import { DEVICE_POOL } from './concurrency-torture/bindings.ts';
30+
import { measureRealScopeMaxOverlap } from './concurrency-torture/real-scope-serialization.ts';
3031
import { buildEnvelope, writeEnvelopeIfRequested } from './concurrency-torture/envelope.ts';
3132

3233
// The #1430 envelope must report the outcome of the WHOLE lane, not just the
3334
// sweep test. Every test records its result here and the envelope is written
3435
// once, after all tests settle, so a later failing guardrail can never be
3536
// published as a passing envelope.
36-
type EnvelopeRun = { seedStart: number; runs: number; durationMs: number };
37+
type SweepRange = { seedStart: number; runs: number };
38+
const laneStartedMs = Date.now();
3739
let laneFailed = false;
38-
let sweepRun: EnvelopeRun | undefined;
40+
let sweepRange: SweepRange | undefined;
3941

4042
async function guard(body: () => Promise<void>): Promise<void> {
4143
try {
@@ -47,8 +49,14 @@ async function guard(body: () => Promise<void>): Promise<void> {
4749
}
4850

4951
after(() => {
50-
if (!sweepRun) return; // only the sweep lane (nightly) publishes an envelope
51-
const envelope = buildEnvelope({ ...sweepRun, result: laneFailed ? 'fail' : 'pass' });
52+
if (!sweepRange) return; // only the sweep lane (nightly) publishes an envelope
53+
// Duration spans the WHOLE lane (sweep + replay + both serialization guards),
54+
// measured to this hook, so it can't understate work by excluding later tests.
55+
const envelope = buildEnvelope({
56+
...sweepRange,
57+
durationMs: Date.now() - laneStartedMs,
58+
result: laneFailed ? 'fail' : 'pass',
59+
});
5260
const written = writeEnvelopeIfRequested(envelope);
5361
if (written) console.log(`torture envelope → ${written}\n${JSON.stringify(envelope)}`);
5462
});
@@ -114,6 +122,22 @@ const explicitSeed = optionalIntFromEnv('TORTURE_SEED');
114122
const clients = optionalIntFromEnv('TORTURE_CLIENTS');
115123
const opsPerClient = optionalIntFromEnv('TORTURE_OPS');
116124

125+
// Guard the PRODUCTION lock APPLICATION path, not just the modeled sweep: drive
126+
// concurrent same-device opens through the real `createRequestExecutionScope()
127+
// .runLocked()` (→ `withRequestExecutionLocks` → `withKeyedLock`) and assert
128+
// they never overlap. Regressing production serialization trips this even
129+
// though the seeded sweep models the grant. Runs in every mode.
130+
test('concurrency torture — real-scope same-device serialization', () =>
131+
guard(async () => {
132+
const maxOverlap = await measureRealScopeMaxOverlap(4);
133+
assert.equal(
134+
maxOverlap,
135+
1,
136+
`real createRequestExecutionScope().runLocked() let ${maxOverlap} same-device critical ` +
137+
'sections overlap — production execution locking is not serializing',
138+
);
139+
}));
140+
117141
if (explicitSeed !== undefined) {
118142
test(`concurrency torture — replay seed ${explicitSeed}`, () =>
119143
guard(async () => {
@@ -128,28 +152,24 @@ if (explicitSeed !== undefined) {
128152

129153
test(`concurrency torture — ${runs} seeded interleavings from ${seedStart}`, () =>
130154
guard(async () => {
131-
const started = Date.now();
155+
sweepRange = { seedStart, runs };
132156
let totalContention = 0;
133-
try {
134-
for (let i = 0; i < runs; i += 1) {
135-
const seed = seedStart + i;
136-
const result = await runTorture({ seed, clients, opsPerClient });
137-
totalContention += result.deviceContention;
138-
// Prove exact replay on the NORMAL sweep path (not just TORTURE_SEED):
139-
// re-run the seed and assert the whole trace + outcome + contention match.
140-
assertDeterministicReplay(result, await runTorture({ seed, clients, opsPerClient }));
141-
assertClean(result);
142-
}
143-
// The sweep must actually PRODUCE same-device contention — two clients
144-
// parked on one device lock — or the lane is not exercising serialization
145-
// and a broken lock could pass unnoticed.
146-
assert.ok(
147-
totalContention > 0,
148-
'no same-device lock contention occurred across the sweep — the lane is not testing serialization',
149-
);
150-
} finally {
151-
sweepRun = { seedStart, runs, durationMs: Date.now() - started };
157+
for (let i = 0; i < runs; i += 1) {
158+
const seed = seedStart + i;
159+
const result = await runTorture({ seed, clients, opsPerClient });
160+
totalContention += result.deviceContention;
161+
// Prove exact replay on the NORMAL sweep path (not just TORTURE_SEED):
162+
// re-run the seed and assert the whole trace + outcome + contention match.
163+
assertDeterministicReplay(result, await runTorture({ seed, clients, opsPerClient }));
164+
assertClean(result);
152165
}
166+
// The sweep must actually PRODUCE same-device contention — two clients
167+
// parked on one device lock — or the lane is not exercising serialization
168+
// and a broken lock could pass unnoticed.
169+
assert.ok(
170+
totalContention > 0,
171+
'no same-device lock contention occurred across the sweep — the lane is not testing serialization',
172+
);
153173
}));
154174

155175
// Forced same-device contention: two clients repeatedly open THE SAME pinned
Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
// Real-scope same-device serialization guard for the concurrency torture lane
2+
// (#1416). The seeded sweep models the mutex GRANT (a seed cannot reproduce
3+
// `withKeyedLock`'s native microtask hand-off), so on its own it could stay
4+
// green if the PRODUCTION lock APPLICATION path regressed. This guard closes
5+
// that gap by driving two concurrent requests for the SAME device through the
6+
// real `createRequestExecutionScope().runLocked()` — i.e. the actual
7+
// `withRequestExecutionLocks` → `withKeyedLock` machinery, not a model — and
8+
// asserting their critical sections never overlap. It is intentionally NOT
9+
// seeded: it exercises real event-loop scheduling. Disabling production locking
10+
// (`shouldLockSessionExecution` → false, or dropping the `device:` key) makes
11+
// two sections overlap and trips this assertion.
12+
13+
import assert from 'node:assert/strict';
14+
import fs from 'node:fs';
15+
import os from 'node:os';
16+
import path from 'node:path';
17+
18+
import { createRequestExecutionScope } from '../../../src/daemon/request-execution-scope.ts';
19+
import { LeaseRegistry } from '../../../src/daemon/lease-registry.ts';
20+
import { SessionStore } from '../../../src/daemon/session-store.ts';
21+
import { withDeviceInventoryProvider } from '../../../src/core/dispatch-resolve.ts';
22+
import type { CommandFlags } from '../../../src/core/dispatch-context.ts';
23+
import type { DaemonRequest } from '../../../src/daemon/types.ts';
24+
25+
import { DEVICE_POOL } from './bindings.ts';
26+
27+
function openRequest(sessionName: string, deviceId: string): DaemonRequest {
28+
return {
29+
command: 'open',
30+
positionals: [],
31+
token: 'torture',
32+
session: sessionName,
33+
flags: { platform: 'ios', udid: deviceId } as CommandFlags,
34+
} as DaemonRequest;
35+
}
36+
37+
/**
38+
* Drive `concurrency` fresh opens of ONE device concurrently through the real
39+
* request execution scope and return the maximum number of critical sections
40+
* that were ever active at once. Production serialization ⇒ exactly 1.
41+
*/
42+
export async function measureRealScopeMaxOverlap(concurrency: number): Promise<number> {
43+
const device = DEVICE_POOL[0];
44+
assert.ok(device, 'DEVICE_POOL must not be empty');
45+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'torture-real-scope-'));
46+
const sessionStore = new SessionStore(dir);
47+
const leaseRegistry = new LeaseRegistry();
48+
49+
let active = 0;
50+
let maxActive = 0;
51+
const criticalSection = async (): Promise<void> => {
52+
active += 1;
53+
maxActive = Math.max(maxActive, active);
54+
// Yield across several event-loop turns so a NON-serializing lock would let
55+
// a second section observe `active > 1` here.
56+
for (let i = 0; i < 5; i += 1) await Promise.resolve();
57+
await new Promise((resolve) => setImmediate(resolve));
58+
active -= 1;
59+
};
60+
61+
try {
62+
await withDeviceInventoryProvider(
63+
async () => [device],
64+
async () => {
65+
const scopes = await Promise.all(
66+
Array.from({ length: concurrency }, (_, i) =>
67+
createRequestExecutionScope({
68+
req: openRequest(`real-${i}`, device.id),
69+
sessionStore,
70+
leaseRegistry,
71+
}),
72+
),
73+
);
74+
await Promise.all(scopes.map((scope) => scope.runLocked(criticalSection)));
75+
},
76+
);
77+
} finally {
78+
fs.rmSync(dir, { recursive: true, force: true });
79+
}
80+
81+
return maxActive;
82+
}

0 commit comments

Comments
 (0)