Skip to content

Commit 216b022

Browse files
committed
fix(producer): a render never reclaims a dir whose owner it cannot place
Without a boot identity, or on Linux without the pid namespace, two missing values matched, so on a shared output folder a same-hostname machine could delete another machine's live work dir. Such a scope is now unknown, and the dir is left alone; macOS, which has no pid namespace, still reclaims by boot.
1 parent 2fdbcf0 commit 216b022

2 files changed

Lines changed: 98 additions & 23 deletions

File tree

Lines changed: 84 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,24 @@
1-
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
2-
import { tmpdir } from "node:os";
3-
import { join } from "node:path";
4-
import { describe, expect, it, vi } from "vitest";
1+
import { spawnSync } from "node:child_process";
2+
import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
3+
import { hostname, tmpdir } from "node:os";
4+
import { basename, join } from "node:path";
5+
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
6+
7+
const probes = vi.hoisted(() => ({ uptime: false, pidns: false }));
58

69
// An agent's macOS seatbelt refuses uptime: "uv_uptime returned EPERM (operation not permitted)".
7-
vi.mock("node:os", async (actual) => ({
8-
...(await actual<typeof import("node:os")>()),
9-
uptime: () => {
10-
throw Object.assign(new Error("A system error occurred: uv_uptime returned EPERM"), {
11-
code: "ERR_SYSTEM_ERROR",
12-
});
13-
},
14-
}));
10+
vi.mock("node:os", async (actual) => {
11+
const os = await actual<typeof import("node:os")>();
12+
return {
13+
...os,
14+
uptime: () => {
15+
if (probes.uptime) return os.uptime();
16+
throw Object.assign(new Error("A system error occurred: uv_uptime returned EPERM"), {
17+
code: "ERR_SYSTEM_ERROR",
18+
});
19+
},
20+
};
21+
});
1522

1623
// Without /proc, as on macOS, so Linux CI takes the uptime path too.
1724
vi.mock("node:fs", async (actual) => {
@@ -23,20 +30,76 @@ vi.mock("node:fs", async (actual) => {
2330
throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
2431
return (fs.readFileSync as (...args: unknown[]) => unknown)(path, ...rest);
2532
}) as typeof fs.readFileSync,
33+
readlinkSync: ((path: string, ...rest: unknown[]) => {
34+
if (path === "/proc/self/ns/pid" && !probes.pidns)
35+
throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
36+
return (fs.readlinkSync as (...args: unknown[]) => unknown)(path, ...rest);
37+
}) as typeof fs.readlinkSync,
2638
};
2739
});
2840

29-
const { createRenderWorkDir } = await import("./renderDirOwner.js");
41+
const { createOwnedRenderDir, createRenderWorkDir } = await import("./renderDirOwner.js");
42+
43+
const JOB = "0f8c2b1e-5d3a-4c7b-9e21-6a4f0d8b3c19";
44+
const platform = process.platform;
3045

3146
describe("a render work dir where the system refuses uptime", () => {
47+
let parent: string;
48+
const deadPid = () => spawnSync(process.execPath, ["-e", ""]).pid!;
49+
const plant = (name: string, owner: object): string => {
50+
mkdirSync(join(parent, name));
51+
writeFileSync(join(parent, name, "owner.json"), JSON.stringify(owner));
52+
return name;
53+
};
54+
const ownStamp = () => {
55+
const own = createOwnedRenderDir(join(parent, "self-"));
56+
const stamp = JSON.parse(readFileSync(join(own, "owner.json"), "utf-8"));
57+
rmSync(own, { recursive: true });
58+
return stamp;
59+
};
60+
61+
beforeEach(() => {
62+
parent = mkdtempSync(join(tmpdir(), "hf-render-dir-sandbox-"));
63+
probes.uptime = false;
64+
probes.pidns = false;
65+
});
66+
afterEach(() => {
67+
Object.defineProperty(process, "platform", { value: platform });
68+
rmSync(parent, { recursive: true, force: true });
69+
});
70+
3271
it("is still created and stamped with its owner", () => {
33-
const parent = mkdtempSync(join(tmpdir(), "hf-render-dir-sandbox-"));
34-
try {
35-
const dir = createRenderWorkDir(join(parent, "work-"), parent);
36-
const owner = JSON.parse(readFileSync(join(dir, "owner.json"), "utf-8"));
37-
expect(owner.pid).toBe(process.pid);
38-
} finally {
39-
rmSync(parent, { recursive: true, force: true });
40-
}
72+
const dir = createRenderWorkDir(join(parent, "work-"), parent);
73+
const owner = JSON.parse(readFileSync(join(dir, "owner.json"), "utf-8"));
74+
expect(owner.pid).toBe(process.pid);
75+
});
76+
77+
it("keeps another same-hostname machine's dir when neither boot nor pid namespace can be read", () => {
78+
// Machine A's live render on a shared output dir; its pid does not exist here on machine B.
79+
const other = plant(`work-${JOB}-Aa0000`, { pid: deadPid(), host: hostname() });
80+
81+
const created = basename(createRenderWorkDir(join(parent, `work-${JOB}-`), parent));
82+
83+
expect(readdirSync(parent).sort()).toEqual([other, created].sort());
84+
});
85+
86+
it("on Linux, keeps it when only the pid namespace cannot be read", () => {
87+
Object.defineProperty(process, "platform", { value: "linux" });
88+
probes.uptime = true;
89+
const other = plant(`work-${JOB}-Bb0000`, { ...ownStamp(), pid: deadPid() });
90+
91+
const created = basename(createRenderWorkDir(join(parent, `work-${JOB}-`), parent));
92+
93+
expect(readdirSync(parent).sort()).toEqual([other, created].sort());
94+
});
95+
96+
it("on macOS, where there is no pid namespace, still reclaims a dead render's dir by boot time", () => {
97+
Object.defineProperty(process, "platform", { value: "darwin" });
98+
probes.uptime = true;
99+
plant(`work-${JOB}-Cc0000`, { ...ownStamp(), pid: deadPid() });
100+
101+
const created = basename(createRenderWorkDir(join(parent, `work-${JOB}-`), parent));
102+
103+
expect(readdirSync(parent)).toEqual([created]);
41104
});
42105
});

‎packages/producer/src/services/render/renderDirOwner.ts‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,21 @@ function alive(pid: number): boolean {
5959

6060
type OwnerRecord = { pid?: unknown; host?: unknown; boot?: unknown; pidns?: unknown };
6161

62-
function inThisPidScope(owner: OwnerRecord): boolean {
62+
function knownPidScope(): ReturnType<typeof pidScope> | undefined {
6363
const scope = pidScope();
64-
return owner.host === scope.host && owner.boot === scope.boot && owner.pidns === scope.pidns;
64+
const unknown =
65+
scope.boot === undefined || (process.platform === "linux" && scope.pidns === undefined);
66+
return unknown ? undefined : scope;
67+
}
68+
69+
function inThisPidScope(owner: OwnerRecord): boolean {
70+
const scope = knownPidScope();
71+
return (
72+
scope !== undefined &&
73+
owner.host === scope.host &&
74+
owner.boot === scope.boot &&
75+
owner.pidns === scope.pidns
76+
);
6577
}
6678

6779
/** "gone" only for an owner in this pid scope that has exited; "none" when there is no owner file, else "unknown". */

0 commit comments

Comments
 (0)