Skip to content

Commit 5298415

Browse files
committed
fix(server): defer disk-blocked launcher transitions
1 parent ec98abc commit 5298415

2 files changed

Lines changed: 248 additions & 59 deletions

File tree

‎apps/server/src/serviceLauncher.test.ts‎

Lines changed: 201 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
// @effect-diagnostics-next-line nodeBuiltinImport:off - observes the children the launcher spawns with Node built-ins.
22
import type * as NodeChildProcess from "node:child_process";
3+
// @effect-diagnostics-next-line nodeBuiltinImport:off - controls filesystem capacity without filling the test machine.
4+
import * as NodeFSP from "node:fs/promises";
35

46
import * as NodeServices from "@effect/platform-node/NodeServices";
57
import { assert, it } from "@effect/vitest";
@@ -15,7 +17,6 @@ import {
1517
main,
1618
readServiceEnvironment,
1719
type BackupSpaceProbe,
18-
serviceStartDiskSpaceBlocker,
1920
readServiceState,
2021
writeServiceState,
2122
} from "./serviceLauncher.ts";
@@ -29,12 +30,20 @@ import {
2930
} from "./cloud/serviceProtocol.ts";
3031

3132
const spawnedChildren = vi.hoisted((): Array<NodeChildProcess.ChildProcess> => []);
33+
const spawnHook = vi.hoisted(() => ({
34+
current: undefined as ((child: NodeChildProcess.ChildProcess) => void) | undefined,
35+
}));
36+
37+
vi.mock("node:fs/promises", async (importOriginal) => ({
38+
...(await importOriginal<typeof NodeFSP>()),
39+
}));
3240

3341
vi.mock("node:child_process", async (importOriginal) => {
3442
const actual = await importOriginal<typeof NodeChildProcess>();
3543
const spawn = (...args: Parameters<typeof actual.spawn>) => {
3644
const child = actual.spawn(...args);
3745
spawnedChildren.push(child);
46+
spawnHook.current?.(child);
3847
return child;
3948
};
4049
return { ...actual, spawn: spawn as typeof actual.spawn };
@@ -66,11 +75,6 @@ it("orders exact semantic versions without treating build metadata as precedence
6675
assert.equal(compareExactServiceVersions("2.0.0+one", "2.0.0+two"), 0);
6776
});
6877

69-
it("blocks service startup when free disk space is below the launch headroom", () => {
70-
assert.isUndefined(serviceStartDiskSpaceBlocker(64 * 1024 ** 2));
71-
assert.include(serviceStartDiskSpaceBlocker(0) ?? "", "before starting the server");
72-
});
73-
7478
it("rejects contradictory service state", () => {
7579
assert.isUndefined(
7680
decodeServiceState({
@@ -133,6 +137,197 @@ const writeFakeRuntime = (
133137
});
134138

135139
it.layer(NodeServices.layer)("service state persistence", (it) => {
140+
it.effect.each([
141+
"startup",
142+
"home probe error",
143+
"database probe error",
144+
"pending update",
145+
"update handoff",
146+
"trial exit",
147+
"failed trial start",
148+
"stop",
149+
] as const)("waits for disk recovery during %s without losing the update lifecycle", (scenario) =>
150+
Effect.gen(function* () {
151+
const fs = yield* FileSystem.FileSystem;
152+
const path = yield* Path.Path;
153+
const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-disk-" });
154+
const statePath = path.join(root, "runtime", "service-state.json");
155+
const databasePath = path.join(root, "userdata", "statev2.sqlite");
156+
let backupPath = path.join(root, "runtime", "db-backup", "disk-update");
157+
const pending = {
158+
id: "disk-update",
159+
fromVersion: "1.0.0",
160+
targetVersion: "1.1.0",
161+
dbPath: databasePath,
162+
status: "pending" as const,
163+
};
164+
yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
165+
yield* fs.writeFileString(databasePath, "before trial");
166+
const childSource = `
167+
const fs = require("node:fs");
168+
const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);
169+
if (context.update?.status === "pending") {
170+
if (context.childVersion !== context.update.targetVersion) throw new Error("wrong trial version");
171+
if (fs.readFileSync(${JSON.stringify(path.dirname(backupPath))} + "/" + context.update.id + "/database", "utf8") !== "before trial") {
172+
throw new Error("missing original backup");
173+
}
174+
if (${JSON.stringify(scenario)} === "trial exit") {
175+
fs.writeFileSync(context.update.dbPath, "migrated");
176+
process.on("message", () => process.exit(1));
177+
process.send("migrated");
178+
} else {
179+
process.send({ type: "prepared", updateId: context.update.id });
180+
process.on("message", (message) => {
181+
if (message.type === "committed") process.send("ready");
182+
});
183+
}
184+
} else if (${JSON.stringify(scenario)} === "update handoff" && context.update === undefined) {
185+
process.on("message", (message) => {
186+
if (message.type === "update-accepted") process.send("exiting");
187+
if (message === "exit") process.exit(0);
188+
});
189+
process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${JSON.stringify(databasePath)} });
190+
} else {
191+
process.send("ready");
192+
}
193+
setInterval(() => {}, 1_000);
194+
`;
195+
for (const version of ["1.0.0", "1.1.0"]) {
196+
yield* writeFakeRuntime(
197+
fs,
198+
path,
199+
path.join(root, "runtime", "versions", version),
200+
childSource,
201+
);
202+
}
203+
yield* Effect.promise(() =>
204+
writeServiceState(statePath, {
205+
protocol: SERVICE_LAUNCHER_PROTOCOL,
206+
activeVersion: "1.0.0",
207+
...([
208+
"startup",
209+
"home probe error",
210+
"database probe error",
211+
"update handoff",
212+
"stop",
213+
].includes(scenario)
214+
? {}
215+
: { update: pending }),
216+
}),
217+
);
218+
let low = [
219+
"startup",
220+
"home probe error",
221+
"database probe error",
222+
"pending update",
223+
"stop",
224+
].includes(scenario);
225+
let recovered = false;
226+
const blocked = Promise.withResolvers<void>();
227+
const ready = Promise.withResolvers<void>();
228+
const actualStatfs = NodeFSP.statfs;
229+
const stats = yield* Effect.promise(() => actualStatfs(root));
230+
yield* Effect.acquireRelease(
231+
Effect.sync(() => {
232+
vi.useFakeTimers({
233+
toFake: ["setInterval", "clearInterval", "setTimeout", "clearTimeout"],
234+
});
235+
const capacity = vi.spyOn(NodeFSP, "statfs").mockImplementation(async (directory) => {
236+
if (
237+
(scenario === "home probe error" && directory === root) ||
238+
(scenario === "database probe error" && directory === path.dirname(databasePath))
239+
) {
240+
throw new Error("statfs unavailable");
241+
}
242+
if (
243+
scenario === "failed trial start" &&
244+
!recovered &&
245+
(await fs.exists(backupPath).pipe(Effect.runPromise))
246+
)
247+
low = true;
248+
return { ...stats, bavail: low ? 0 : 64 * 1024 ** 2, bsize: 1 };
249+
});
250+
const log = vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
251+
if (String(chunk).includes("Waiting for free space")) blocked.resolve();
252+
return true;
253+
});
254+
spawnHook.current = (child) =>
255+
child.on("message", (message) => {
256+
if (message === "ready") ready.resolve();
257+
if (message === "migrated" || message === "exiting") {
258+
low = true;
259+
child.send("exit");
260+
}
261+
});
262+
return { capacity, log };
263+
}),
264+
({ capacity, log }) =>
265+
Effect.sync(() => {
266+
capacity.mockRestore();
267+
log.mockRestore();
268+
spawnHook.current = undefined;
269+
vi.useRealTimers();
270+
}),
271+
);
272+
const beforeSpawns = spawnedChildren.length;
273+
const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath)));
274+
const running = launcher.run();
275+
yield* Effect.addFinalizer(() =>
276+
Effect.promise(async () => {
277+
await launcher.stop("SIGTERM");
278+
await running;
279+
}),
280+
);
281+
yield* Effect.promise(() => Promise.race([blocked.promise, ready.promise, running]));
282+
assert.equal(
283+
spawnedChildren.length - beforeSpawns,
284+
["trial exit", "update handoff"].includes(scenario) ? 1 : 0,
285+
);
286+
if (scenario === "update handoff") {
287+
const state = yield* Effect.promise(() => readServiceState(statePath));
288+
backupPath = path.join(root, "runtime", "db-backup", state.update!.id);
289+
}
290+
assert.equal(
291+
yield* fs.exists(backupPath),
292+
scenario === "trial exit" || scenario === "failed trial start",
293+
);
294+
if (scenario === "trial exit")
295+
assert.equal(yield* fs.readFileString(databasePath), "migrated");
296+
if (scenario === "failed trial start") {
297+
yield* fs.remove(path.join(root, "runtime", "versions", "1.1.0"), { recursive: true });
298+
}
299+
if (scenario === "stop") {
300+
yield* Effect.promise(() => launcher.stop("SIGTERM"));
301+
yield* Effect.promise(() => running);
302+
assert.equal(vi.getTimerCount(), 0);
303+
return;
304+
}
305+
low = false;
306+
recovered = true;
307+
yield* Effect.promise(() => vi.advanceTimersByTimeAsync(30_000));
308+
yield* Effect.promise(() => Promise.race([ready.promise, running]));
309+
const state = yield* Effect.promise(() => readServiceState(statePath));
310+
assert.equal(
311+
state.activeVersion,
312+
["pending update", "update handoff"].includes(scenario) ? "1.1.0" : "1.0.0",
313+
);
314+
assert.equal(
315+
state.update?.status,
316+
{
317+
startup: undefined,
318+
"home probe error": undefined,
319+
"database probe error": undefined,
320+
"pending update": "committed",
321+
"update handoff": "committed",
322+
"trial exit": "rolled-back",
323+
"failed trial start": "failed",
324+
}[scenario],
325+
);
326+
assert.equal(yield* fs.readFileString(databasePath), "before trial");
327+
assert.isFalse(yield* fs.exists(backupPath));
328+
}),
329+
);
330+
136331
it.effect(
137332
"loads optional service.env literally without changing inherited or managed variables",
138333
() =>

0 commit comments

Comments
 (0)