Skip to content

Commit 9e78b03

Browse files
fix(cli): render lock never removes another render's takeover, knows a reused pid, works in a shared folder and on Windows
- A `.break` takeover file is removed only by the render that wrote it. One whose writer is gone stops the render with the file to delete, instead of being removed on an earlier reading, which could remove a live `.break` written since and let two renders hold the slot. The takeover re-reads the lock under its own `.break` right before replacing it. - The lock record keeps its holder's process start time, so a pid that now belongs to another process (an OOM-killed render in a container with a persisted cache) no longer holds the slot forever. - The `locks` folder is created writable for every account, so accounts sharing HYPERFRAMES_CACHE_DIR take turns instead of the second failing with EACCES; a folder this account cannot write in says how to fix it, and another account's file that cannot be removed is left, not fatal. - Windows: no PowerShell start-time lookup per check (pid liveness only), a file another handle holds (EPERM/EBUSY/EACCES) reads as held, a lock that vanished is retried once instead of in a tight loop, and a wait stops when its signal aborts. The tests end every acquire they start before removing their folder, and the paused contender writes through its open descriptor. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent f50f10b commit 9e78b03

6 files changed

Lines changed: 367 additions & 85 deletions

File tree

‎packages/cli/src/commands/render/renderSlot.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
RenderLockBusy,
88
type RenderLockOptions,
99
} from "../../utils/renderLock.js";
10+
import { RenderLockStaleBreak, RenderLockUnwritable } from "../../utils/renderLockClaim.js";
1011
import type { RenderPlan } from "./plan.js";
1112

1213
/** One local render at a time per user account: a second waits for the slot, or fails under --no-wait. */
@@ -30,6 +31,10 @@ export async function acquireRenderSlot(
3031
...lock,
3132
});
3233
} catch (error) {
34+
if (error instanceof RenderLockStaleBreak || error instanceof RenderLockUnwritable) {
35+
errorBox("The render slot cannot be taken", error.message);
36+
return failCommand(1, error);
37+
}
3338
if (!(error instanceof RenderLockBusy)) throw error;
3439
errorBox(
3540
"Another render is running for this user account",

‎packages/cli/src/utils/projectLock.ts‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,11 @@ export function alive(pid: number): boolean {
3737
const sameHolder = (a: Holder | null, b: Holder) =>
3838
a?.pid === b.pid && a.startedAt === b.startedAt && a.command === b.command;
3939

40-
export function takeOver(path: string, holder: Holder, stillStale: () => boolean): boolean {
40+
function breakStaleHolder(path: string, current: Holder, holder: Holder): boolean {
4141
const breakPath = `${path}.break`;
4242
if (!createNew(breakPath, holder)) return false;
4343
try {
44-
if (stillStale()) {
44+
if (sameHolder(readHolder(path), current)) {
4545
try {
4646
unlinkSync(path);
4747
} catch (error) {
@@ -64,12 +64,10 @@ function createNew(path: string, holder: Holder): boolean {
6464
}
6565
}
6666

67-
export function tryAcquire(path: string, holder: Holder): boolean {
67+
function tryAcquire(path: string, holder: Holder): boolean {
6868
if (createNew(path, holder)) return true;
6969
const current = readHolder(path);
70-
if (current && !alive(current.pid)) {
71-
return takeOver(path, holder, () => sameHolder(readHolder(path), current));
72-
}
70+
if (current && !alive(current.pid)) return breakStaleHolder(path, current, holder);
7371
return false;
7472
}
7573

‎packages/cli/src/utils/renderLock.test.ts‎

Lines changed: 189 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,24 @@
11
import { spawnSync } from "node:child_process";
22
import {
3+
chmodSync,
34
closeSync,
45
existsSync,
56
mkdirSync,
67
mkdtempSync,
78
openSync,
89
readdirSync,
10+
readFileSync,
911
rmSync,
12+
statSync,
1013
writeFileSync,
1114
} from "node:fs";
1215
import { tmpdir } from "node:os";
1316
import { join } from "node:path";
1417
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
15-
import { acquireRenderLock, RenderLockBusy, type RenderLockHolder } from "./renderLock.js";
18+
import { acquireRenderLock, RenderLockBusy, type RenderLockOptions } from "./renderLock.js";
19+
import { type RenderLockHolder } from "./renderLockClaim.js";
20+
import { startToken } from "./renderLockOwners.js";
21+
import { RenderLockStaleBreak } from "./renderLockClaim.js";
1622
import type { Holder } from "./projectLock.js";
1723
import { acquireRenderSlot } from "../commands/render/renderSlot.js";
1824

@@ -24,7 +30,19 @@ beforeEach(() => {
2430
path = join(dir, "locks", "render.lock");
2531
});
2632

27-
afterEach(() => {
33+
const stillWaiting: Array<{ stop: AbortController; settled: Promise<unknown> }> = [];
34+
35+
/** An acquire that every test ends before its folder is removed, so no loop outlives the test. */
36+
function startAcquire(options: Omit<RenderLockOptions, "signal">): Promise<() => void> {
37+
const stop = new AbortController();
38+
const acquiring = acquireRenderLock({ ...options, signal: stop.signal });
39+
stillWaiting.push({ stop, settled: acquiring.catch(() => undefined) });
40+
return acquiring;
41+
}
42+
43+
afterEach(async () => {
44+
for (const { stop } of stillWaiting) stop.abort(new Error("test over"));
45+
await Promise.all(stillWaiting.splice(0).map(({ settled }) => settled));
2846
vi.useRealTimers();
2947
vi.restoreAllMocks();
3048
rmSync(dir, { recursive: true, force: true });
@@ -49,7 +67,7 @@ describe("acquireRenderLock", () => {
4967
const release = await acquireRenderLock({ project: "/work/brand", wait: false, path, env: {} });
5068
const waits: Array<RenderLockHolder | null> = [];
5169
let started = false;
52-
const second = acquireRenderLock({
70+
const second = startAcquire({
5371
project: "/work/crime",
5472
wait: true,
5573
path,
@@ -85,21 +103,120 @@ describe("acquireRenderLock", () => {
85103
release();
86104
});
87105

88-
it("takes over when the render that was taking over a dead slot died too", async () => {
89-
const dead = (project: string): RenderLockHolder => ({
90-
pid: deadPid(),
106+
describe("a takeover file (.break)", () => {
107+
const record = (pid: number, project: string): RenderLockHolder => ({
108+
pid,
91109
command: "render",
92110
startedAt: 1,
93111
project,
112+
identity: startToken(pid),
94113
});
95-
await acquireRenderLock({ project: "/work/seed", wait: false, path, env: {} }).then((r) => r());
96-
writeFileSync(path, JSON.stringify(dead("/work/killed")));
97-
writeFileSync(`${path}.break`, JSON.stringify(dead("/work/breaker")));
114+
const seed = (breaker: string) => {
115+
mkdirSync(join(dir, "locks"), { recursive: true });
116+
writeFileSync(path, JSON.stringify(record(deadPid(), "/work/killed")));
117+
writeFileSync(`${path}.break`, breaker);
118+
};
98119

99-
const release = await acquireRenderLock({ project: "/work/next", wait: false, path, env: {} });
120+
it.each([
121+
["readable", () => JSON.stringify(record(deadPid(), "/work/breaker"))],
122+
["half-written", () => '{"pid":'],
123+
])(
124+
"left %s by a render that died is never deleted on an earlier reading; the render says what to delete",
125+
async (_kind, content) => {
126+
const left = content();
127+
seed(left);
128+
for (const wait of [false, true]) {
129+
await expect(
130+
startAcquire({ project: "/work/next", wait, path, env: {} }),
131+
).rejects.toBeInstanceOf(RenderLockStaleBreak);
132+
}
133+
expect(readFileSync(`${path}.break`, "utf8")).toBe(left);
134+
},
135+
);
100136

101-
release();
102-
expect(existsSync(`${path}.break`)).toBe(false);
137+
it("that a live render is still writing is left to it, and that render's takeover holds alone", async () => {
138+
vi.useFakeTimers({ now: Date.now() });
139+
mkdirSync(join(dir, "locks"), { recursive: true });
140+
writeFileSync(path, JSON.stringify(record(deadPid(), "/work/killed")));
141+
let resume: () => void = () => {};
142+
let heldBy: string[] = [];
143+
const first = startAcquire({
144+
project: "/work/one",
145+
wait: false,
146+
path,
147+
env: {},
148+
beforeReplace: () => new Promise<void>((resolve) => (resume = resolve)),
149+
}).then((release) => ((heldBy = [...heldBy, "one"]), release));
150+
await vi.waitFor(() => expect(existsSync(`${path}.break`)).toBe(true));
151+
const breakWhileReplacing = readFileSync(`${path}.break`, "utf8");
152+
153+
await expect(
154+
startAcquire({ project: "/work/two", wait: false, path, env: {} }),
155+
).rejects.toBeInstanceOf(RenderLockBusy);
156+
const second = startAcquire({ project: "/work/two", wait: true, path, env: {} }).then(
157+
(release) => ((heldBy = [...heldBy, "two"]), release),
158+
);
159+
await vi.advanceTimersByTimeAsync(30_000);
160+
expect(readFileSync(`${path}.break`, "utf8")).toBe(breakWhileReplacing);
161+
expect(heldBy).toEqual([]);
162+
163+
resume();
164+
const releaseFirst = await first;
165+
await vi.advanceTimersByTimeAsync(5_000);
166+
expect(heldBy).toEqual(["one"]);
167+
releaseFirst();
168+
await vi.advanceTimersByTimeAsync(1_000);
169+
expect(heldBy).toEqual(["one", "two"]);
170+
(await second)();
171+
});
172+
});
173+
174+
it.skipIf(process.platform === "win32")(
175+
"takes over a lock whose pid now belongs to a different process",
176+
async () => {
177+
mkdirSync(join(dir, "locks"), { recursive: true });
178+
const reused: RenderLockHolder = {
179+
pid: process.pid,
180+
command: "render",
181+
startedAt: 1,
182+
project: "/work/oom-killed",
183+
identity: "linux:1",
184+
};
185+
writeFileSync(path, JSON.stringify(reused));
186+
187+
const release = await acquireRenderLock({
188+
project: "/work/next",
189+
wait: false,
190+
path,
191+
env: {},
192+
});
193+
194+
expect(JSON.parse(readFileSync(path, "utf8")).project).toBe("/work/next");
195+
release();
196+
},
197+
);
198+
199+
describe.skipIf(process.platform === "win32")("a cache folder shared by accounts", () => {
200+
it("creates its locks folder writable for every account", async () => {
201+
const release = await acquireRenderLock({ project: "/work/a", wait: false, path, env: {} });
202+
expect(statSync(join(dir, "locks")).mode & 0o777).toBe(0o777);
203+
release();
204+
});
205+
206+
it.skipIf(process.getuid?.() === 0)(
207+
"says how to fix a locks folder this account cannot write in",
208+
async () => {
209+
mkdirSync(join(dir, "locks"), { mode: 0o755 });
210+
chmodSync(join(dir, "locks"), 0o555);
211+
try {
212+
await expect(
213+
acquireRenderLock({ project: "/work/b", wait: false, path, env: {} }),
214+
).rejects.toThrow(`chmod 777 ${join(dir, "locks")}`);
215+
} finally {
216+
chmodSync(join(dir, "locks"), 0o755);
217+
}
218+
},
219+
);
103220
});
104221

105222
it("keeps the slot in this user's HyperFrames cache folder, which HYPERFRAMES_CACHE_DIR can share", async () => {
@@ -116,24 +233,25 @@ describe("acquireRenderLock", () => {
116233
describe("a lock file that cannot be read yet", () => {
117234
/** A contender that opens the lock exclusively, then stalls before writing its holder. */
118235
const pausedAfterOpen = (gate: Promise<void>) => async (file: string, holder: Holder) => {
119-
closeSync(openSync(file, "wx"));
236+
const fd = openSync(file, "wx");
120237
await gate;
121-
writeFileSync(file, JSON.stringify(holder));
238+
writeFileSync(fd, JSON.stringify(holder));
239+
closeSync(fd);
122240
return true;
123241
};
124242

125243
it("is not taken from a live contender paused between opening and writing it", async () => {
126244
vi.useFakeTimers({ now: Date.now() });
127245
let resume: () => void = () => {};
128-
const writerA = acquireRenderLock({
246+
const writerA = startAcquire({
129247
project: "/work/a",
130248
wait: false,
131249
path,
132250
env: {},
133251
claim: pausedAfterOpen(new Promise<void>((resolve) => (resume = resolve))),
134252
});
135253
let heldByB = false;
136-
const writerB = acquireRenderLock({ project: "/work/b", wait: true, path, env: {} }).then(
254+
const writerB = startAcquire({ project: "/work/b", wait: true, path, env: {} }).then(
137255
(release) => {
138256
heldByB = true;
139257
return release;
@@ -157,13 +275,20 @@ describe("acquireRenderLock", () => {
157275
});
158276

159277
it("is reclaimed when the contender that opened it died before writing", async () => {
160-
void acquireRenderLock({
278+
let crash: (reason: Error) => void = () => {};
279+
const died = new Promise<void>((_, reject) => (crash = reject));
280+
const writerA = startAcquire({
161281
project: "/work/a",
162282
wait: false,
163283
path,
164284
env: {},
165285
pid: deadPid(),
166-
claim: pausedAfterOpen(new Promise<void>(() => {})),
286+
// A dead process holds no handle, so the opened file is closed, as the OS would close it.
287+
claim: async (file) => {
288+
closeSync(openSync(file, "wx"));
289+
await died;
290+
return true;
291+
},
167292
});
168293
await vi.waitFor(() => expect(existsSync(path)).toBe(true));
169294

@@ -172,6 +297,8 @@ describe("acquireRenderLock", () => {
172297
expect(readdirSync(join(dir, "locks")).filter((n) => n.includes(".owner-"))).toHaveLength(1);
173298
release();
174299
expect(readdirSync(join(dir, "locks"))).toEqual([]);
300+
crash(new Error("A is gone"));
301+
await expect(writerA).rejects.toThrow("A is gone");
175302
});
176303

177304
it("is reclaimed when no contender is left at all", async () => {
@@ -188,28 +315,53 @@ describe("acquireRenderLock", () => {
188315
release();
189316
expect(existsSync(path)).toBe(false);
190317
});
318+
});
191319

192-
it("reclaims a half-written takeover file next to a dead holder", async () => {
193-
const killed: RenderLockHolder = {
194-
pid: deadPid(),
195-
command: "render",
196-
startedAt: 1,
197-
project: "/work/killed",
198-
};
199-
mkdirSync(join(dir, "locks"), { recursive: true });
200-
writeFileSync(path, JSON.stringify(killed));
201-
writeFileSync(`${path}.break`, '{"pid":');
320+
it("stops waiting, and stops marking itself a contender, when its signal aborts", async () => {
321+
vi.useFakeTimers();
322+
const release = await acquireRenderLock({ project: "/work/brand", wait: false, path, env: {} });
323+
const stop = new AbortController();
324+
const waiting = acquireRenderLock({
325+
project: "/work/crime",
326+
wait: true,
327+
path,
328+
env: {},
329+
signal: stop.signal,
330+
});
331+
await vi.advanceTimersByTimeAsync(3_000);
202332

203-
const release = await acquireRenderLock({
204-
project: "/work/next",
333+
stop.abort(new Error("cancelled"));
334+
await expect(waiting).rejects.toThrow("cancelled");
335+
release();
336+
await vi.advanceTimersByTimeAsync(5_000);
337+
expect(readdirSync(join(dir, "locks"))).toEqual([]);
338+
});
339+
340+
it("reads a file another handle holds (Windows EPERM/EBUSY) as held, not as a failure", async () => {
341+
const busy = Object.assign(new Error("EBUSY: resource busy or locked"), { code: "EBUSY" });
342+
343+
await expect(
344+
acquireRenderLock({
345+
project: "/work/crime",
205346
wait: false,
206347
path,
207348
env: {},
208-
});
349+
claim: () => {
350+
throw busy;
351+
},
352+
}),
353+
).rejects.toBeInstanceOf(RenderLockBusy);
354+
expect(readdirSync(join(dir, "locks")).filter((n) => n.includes(".owner-"))).toEqual([]);
355+
});
209356

210-
release();
211-
expect(existsSync(`${path}.break`)).toBe(false);
212-
});
357+
it("never starts PowerShell for a start-time lookup on Windows", () => {
358+
const platform = Object.getOwnPropertyDescriptor(process, "platform")!;
359+
Object.defineProperty(process, "platform", { ...platform, value: "win32" });
360+
try {
361+
expect(startToken(process.pid)).toBeNull();
362+
} finally {
363+
Object.defineProperty(process, "platform", platform);
364+
}
213365
});
214366

215367
it("lets renders run side by side with HYPERFRAMES_RENDER_LOCK=off", async () => {
@@ -247,11 +399,13 @@ describe("acquireRenderSlot", () => {
247399
vi.spyOn(console, "error").mockImplementation(() => {});
248400
const release = await acquireRenderLock({ project: "/work/brand", wait: false, path, env: {} });
249401

402+
const stop = new AbortController();
250403
const slot = acquireRenderSlot(
251404
{ project: { dir: "/work/crime" } as never, waitForRenderSlot: true },
252-
undefined,
405+
stop.signal,
253406
{ path, env: {} },
254407
);
408+
stillWaiting.push({ stop, settled: slot.catch(() => undefined) });
255409
await vi.advanceTimersByTimeAsync(3_000);
256410

257411
const said = vi.mocked(console.error).mock.calls.flat().join("\n");

0 commit comments

Comments
 (0)