Skip to content

Commit 9f8fc2b

Browse files
committed
fix(test): isolate load-sensitive full-suite lanes
1 parent b15b208 commit 9f8fc2b

6 files changed

Lines changed: 669 additions & 199 deletions

File tree

‎scripts/test-run-lock.ts‎

Lines changed: 227 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
import { randomUUID } from "node:crypto";
2+
import {
3+
mkdirSync,
4+
readFileSync,
5+
readdirSync,
6+
renameSync,
7+
rmSync,
8+
statSync,
9+
writeFileSync,
10+
} from "node:fs";
11+
import { tmpdir } from "node:os";
12+
import { join } from "node:path";
13+
14+
export const TEST_RUN_ID_ENV = "OCX_TEST_RUN_ID";
15+
export const TEST_RUN_NO_QUEUE_ENV = "OCX_TEST_NO_QUEUE";
16+
const DEFAULT_LOCK_PATH = join(tmpdir(), "opencodex-bun-test.lock");
17+
const OWNER_FILE = "owner.json";
18+
const MEMBERS_DIR = "members";
19+
const INCOMPLETE_OWNER_GRACE_MS = 10_000;
20+
21+
export interface TestRunLockOwner {
22+
version: 1;
23+
runId: string;
24+
token: string;
25+
pid: number;
26+
acquiredAt: string;
27+
}
28+
29+
export interface TestRunLock {
30+
acquired: boolean;
31+
owner: TestRunLockOwner | null;
32+
release(): void;
33+
}
34+
35+
export interface AcquireTestRunLockOptions {
36+
runId: string;
37+
ownerPid?: number;
38+
lockPath?: string;
39+
pollMs?: number;
40+
maxWaitMs?: number;
41+
env?: NodeJS.ProcessEnv;
42+
onWait?: (owner: TestRunLockOwner | null) => void;
43+
onAcquiredAfterWait?: (elapsedMs: number) => void;
44+
}
45+
46+
export interface BareTestRunIdentity {
47+
ownerPid: number;
48+
runId: string;
49+
}
50+
51+
/**
52+
* Give one bare Bun invocation a stable identity without conflating sibling commands.
53+
*
54+
* Without `--parallel`, the preload runs in the test-runner process itself and its
55+
* parent may be a long-lived shell or agent host shared by many unrelated commands.
56+
* Bun parallel workers expose `BUN_TEST_WORKER_ID` and share one short-lived parent
57+
* controller, so only that case may safely rendezvous on the parent PID.
58+
*/
59+
export function resolveBareTestRunIdentity(options: {
60+
pid: number;
61+
ppid: number;
62+
workerId?: string;
63+
}): BareTestRunIdentity {
64+
const coordinatorPid = options.workerId ? options.ppid : options.pid;
65+
return { ownerPid: options.pid, runId: `bare-${coordinatorPid}` };
66+
}
67+
68+
function ownerPath(lockPath: string): string {
69+
return join(lockPath, OWNER_FILE);
70+
}
71+
72+
function memberPath(lockPath: string, owner: TestRunLockOwner, pid: number): string {
73+
return join(lockPath, MEMBERS_DIR, `${pid}-${owner.token}`);
74+
}
75+
76+
function readOwner(lockPath: string): TestRunLockOwner | null {
77+
try {
78+
const parsed = JSON.parse(readFileSync(ownerPath(lockPath), "utf8")) as Partial<TestRunLockOwner>;
79+
if (parsed.version !== 1 || typeof parsed.runId !== "string" || typeof parsed.token !== "string"
80+
|| !Number.isInteger(parsed.pid) || (parsed.pid ?? 0) <= 0 || typeof parsed.acquiredAt !== "string") {
81+
return null;
82+
}
83+
return parsed as TestRunLockOwner;
84+
} catch {
85+
return null;
86+
}
87+
}
88+
89+
function processIsAlive(pid: number): boolean {
90+
try {
91+
process.kill(pid, 0);
92+
return true;
93+
} catch (error) {
94+
return (error as NodeJS.ErrnoException).code === "EPERM";
95+
}
96+
}
97+
98+
function liveMemberExists(lockPath: string, owner: TestRunLockOwner): boolean {
99+
try {
100+
return readdirSync(join(lockPath, MEMBERS_DIR)).some(file => {
101+
const suffix = `-${owner.token}`;
102+
if (!file.endsWith(suffix)) return false;
103+
const pid = Number.parseInt(file.slice(0, -suffix.length), 10);
104+
return Number.isInteger(pid) && pid > 0 && processIsAlive(pid);
105+
});
106+
} catch {
107+
return false;
108+
}
109+
}
110+
111+
function lockIsLive(lockPath: string, owner: TestRunLockOwner): boolean {
112+
return processIsAlive(owner.pid) || liveMemberExists(lockPath, owner);
113+
}
114+
115+
function registerMember(lockPath: string, owner: TestRunLockOwner, pid: number): boolean {
116+
const membersPath = join(lockPath, MEMBERS_DIR);
117+
try {
118+
mkdirSync(membersPath, { recursive: true, mode: 0o700 });
119+
writeFileSync(memberPath(lockPath, owner, pid), "", { flag: "a", mode: 0o600 });
120+
} catch {
121+
return false;
122+
}
123+
if (ownsLock(lockPath, owner)) return true;
124+
rmSync(memberPath(lockPath, owner, pid), { force: true });
125+
return false;
126+
}
127+
128+
function incompleteOwnerIsRecent(lockPath: string): boolean {
129+
try {
130+
return Date.now() - statSync(lockPath).mtimeMs < INCOMPLETE_OWNER_GRACE_MS;
131+
} catch {
132+
return false;
133+
}
134+
}
135+
136+
function reclaimStaleLock(lockPath: string): boolean {
137+
const stalePath = `${lockPath}.stale-${process.pid}-${randomUUID()}`;
138+
try {
139+
renameSync(lockPath, stalePath);
140+
} catch (error) {
141+
if (["ENOENT", "EACCES", "EPERM"].includes((error as NodeJS.ErrnoException).code ?? "")) return false;
142+
throw error;
143+
}
144+
rmSync(stalePath, { recursive: true, force: true });
145+
return true;
146+
}
147+
148+
function ownsLock(lockPath: string, owner: TestRunLockOwner): boolean {
149+
const current = readOwner(lockPath);
150+
return current?.runId === owner.runId && current.token === owner.token && current.pid === owner.pid;
151+
}
152+
153+
/**
154+
* Acquire the machine-wide OpenCodex Bun-test lock.
155+
*
156+
* `mkdir` is the cross-platform atomic primitive. The owner PID makes a lock left by
157+
* SIGKILL recoverable, while the run ID lets every worker belonging to one bare
158+
* `bun test --parallel` invocation join the same lock without blocking its siblings.
159+
* Joiners register their own PIDs so a worker-owned lock remains live if its first
160+
* worker exits before the rest of the pool.
161+
*/
162+
export async function acquireTestRunLock(options: AcquireTestRunLockOptions): Promise<TestRunLock> {
163+
const env = options.env ?? process.env;
164+
if (env[TEST_RUN_NO_QUEUE_ENV] === "1") {
165+
return { acquired: false, owner: null, release() {} };
166+
}
167+
168+
const lockPath = options.lockPath ?? DEFAULT_LOCK_PATH;
169+
const ownerPid = options.ownerPid ?? process.pid;
170+
const pollMs = Math.max(1, options.pollMs ?? 5_000);
171+
const maxWaitMs = Math.max(pollMs, options.maxWaitMs ?? 45 * 60 * 1000);
172+
const startedAt = Date.now();
173+
let announced = false;
174+
175+
for (;;) {
176+
const owner: TestRunLockOwner = {
177+
version: 1,
178+
runId: options.runId,
179+
token: randomUUID(),
180+
pid: ownerPid,
181+
acquiredAt: new Date().toISOString(),
182+
};
183+
try {
184+
mkdirSync(lockPath, { mode: 0o700 });
185+
try {
186+
writeFileSync(ownerPath(lockPath), `${JSON.stringify(owner)}\n`, { encoding: "utf8", mode: 0o600, flag: "wx" });
187+
} catch (error) {
188+
rmSync(lockPath, { recursive: true, force: true });
189+
throw error;
190+
}
191+
if (announced) options.onAcquiredAfterWait?.(Date.now() - startedAt);
192+
return {
193+
acquired: true,
194+
owner,
195+
release() {
196+
if (!ownsLock(lockPath, owner)) return;
197+
reclaimStaleLock(lockPath);
198+
},
199+
};
200+
} catch (error) {
201+
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
202+
}
203+
204+
const current = readOwner(lockPath);
205+
if (current?.runId === options.runId && lockIsLive(lockPath, current)) {
206+
if (registerMember(lockPath, current, process.pid)) {
207+
return { acquired: false, owner: current, release() {} };
208+
}
209+
continue;
210+
}
211+
const ownerIsLive = current ? lockIsLive(lockPath, current) : incompleteOwnerIsRecent(lockPath);
212+
if (!ownerIsLive && reclaimStaleLock(lockPath)) continue;
213+
214+
if (!announced) {
215+
announced = true;
216+
options.onWait?.(current);
217+
}
218+
if (Date.now() - startedAt >= maxWaitMs) {
219+
const holder = current ? `pid ${current.pid} (run ${current.runId})` : "an initializing runner";
220+
throw new Error(
221+
`timed out after ${Math.round(maxWaitMs / 1000)}s waiting for ${holder} to release ${lockPath}; `
222+
+ `set ${TEST_RUN_NO_QUEUE_ENV}=1 only when overlapping test runners are intentional`,
223+
);
224+
}
225+
await Bun.sleep(pollMs);
226+
}
227+
}

0 commit comments

Comments
 (0)