Skip to content

Commit d0d0608

Browse files
fix(server): a remote update points the service unit at the runtime it committed (#19652)
Co-authored-by: Julius Marminge <[email protected]>
1 parent 729f675 commit d0d0608

9 files changed

Lines changed: 937 additions & 128 deletions

File tree

‎apps/server/src/cloud/bootService.test.ts‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,19 @@
1+
// @effect-diagnostics-next-line nodeBuiltinImport:off -- Cross-process launcher contention uses real filesystem leases and IPC barriers.
2+
import * as NodeChildProcess from "node:child_process";
13
import * as NodeServices from "@effect/platform-node/NodeServices";
24
import { expect, it } from "@effect/vitest";
35
import * as HostProcess from "@t3tools/shared/HostProcess";
46
import * as ConfigProvider from "effect/ConfigProvider";
57
import * as Duration from "effect/Duration";
68
import * as Effect from "effect/Effect";
79
import * as FileSystem from "effect/FileSystem";
10+
import * as Fiber from "effect/Fiber";
811
import * as Layer from "effect/Layer";
912
import * as Path from "effect/Path";
1013
import { CliError } from "effect/cli";
1114
import { HttpClient } from "effect/http";
1215
import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
16+
import { vi } from "vite-plus/test";
1317

1418
import * as ProcessRunner from "../processRunner.ts";
1519
import * as BootService from "./bootService.ts";
@@ -22,6 +26,24 @@ import {
2226
serviceStateHasPendingUpdate,
2327
} from "./serviceProtocol.ts";
2428

29+
const leaseObserver = vi.hoisted(() => ({ contended: (_destination: string) => {} }));
30+
vi.mock("node:fs", async (importOriginal) => {
31+
const actual = await importOriginal<typeof import("node:fs")>();
32+
return {
33+
...actual,
34+
renameSync: (...args: Parameters<typeof actual.renameSync>) => {
35+
try {
36+
return actual.renameSync(...args);
37+
} catch (error) {
38+
if (["EEXIST", "ENOTEMPTY"].includes((error as NodeJS.ErrnoException).code ?? "")) {
39+
leaseObserver.contended(String(args[1]));
40+
}
41+
throw error;
42+
}
43+
},
44+
};
45+
});
46+
2547
it("reads the served T3 home back out of a rendered unit or plist", () => {
2648
const plan = (baseDir: string) => ({
2749
program: [`${baseDir}/runtime/versions/1.2.3/t3`, "__service-launcher"],
@@ -442,6 +464,11 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
442464
current: false,
443465
installedVersion: "1.2.4-nightly.1",
444466
});
467+
yield* service.install({ allowDowngrade: true });
468+
expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({
469+
protocol: SERVICE_LAUNCHER_PROTOCOL,
470+
activeVersion: "1.2.3",
471+
});
445472
}
446473
}),
447474
);
@@ -575,6 +602,168 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
575602
expect(yield* newer.restart).toBe("restarted");
576603
expect((yield* newer.status).problems).not.toContain("restart-pending");
577604
expect((yield* newer.status).current).toBe(true);
605+
606+
const latest = yield* makeService(undefined, "1.2.5");
607+
yield* latest.install({ start: true });
608+
expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({
609+
protocol: SERVICE_LAUNCHER_PROTOCOL,
610+
activeVersion: "1.2.5",
611+
});
612+
expect(yield* fs.readFileString(plan.unitPath)).toContain("versions/1.2.5/t3");
613+
expect((yield* latest.status).current).toBe(true);
614+
expect(
615+
yield* fs.exists(path.join(path.dirname(statePath), SERVICE_RESTART_PENDING_FILE)),
616+
).toBe(false);
617+
}),
618+
);
619+
620+
it.effect(
621+
"serializes a local install with another process reconciling an older unit snapshot",
622+
() =>
623+
Effect.gen(function* () {
624+
const { service, fs, statePath, makeService, runtime } = yield* makeHarness();
625+
const path = yield* Path.Path;
626+
const plan = yield* service.install();
627+
const unit = yield* fs.readFileString(plan.unitPath);
628+
yield* fs.writeFileString(
629+
plan.unitPath,
630+
unit.replace("versions/1.2.3/t3", "versions/1.2.2/t3"),
631+
);
632+
yield* fs.writeFileString(
633+
statePath,
634+
JSON.stringify({
635+
protocol: SERVICE_LAUNCHER_PROTOCOL,
636+
activeVersion: "1.2.3",
637+
update: {
638+
id: "committed",
639+
fromVersion: "1.2.2",
640+
targetVersion: "1.2.3",
641+
status: "committed",
642+
},
643+
}),
644+
);
645+
const rejection = path.join(plan.baseDir, "rejected-update");
646+
const databasePath = path.join(plan.baseDir, "state.sqlite");
647+
yield* fs.writeFileString(databasePath, "database");
648+
yield* fs.writeFileString(
649+
runtime.entryPath,
650+
`#!${process.execPath}
651+
import { writeFileSync } from "node:fs";
652+
process.on("message", message => {
653+
if (message === "request-update") process.send({ type: "request-update", targetVersion: "1.2.5", dbPath: ${JSON.stringify(databasePath)} });
654+
if (message.type === "update-rejected") {
655+
writeFileSync(${JSON.stringify(rejection)}, message.reason);
656+
process.exit(0);
657+
}
658+
});
659+
`,
660+
);
661+
yield* fs.chmod(runtime.entryPath, 0o755);
662+
const newer = yield* makeService(undefined, "1.2.4");
663+
yield* makeService(undefined, "1.2.5");
664+
const snapshot = Promise.withResolvers<void>();
665+
const milestone = Promise.withResolvers<void>();
666+
const script = path.join(plan.baseDir, "launcher-contention.mjs");
667+
yield* fs.writeFileString(
668+
script,
669+
`
670+
import fs from "node:fs/promises";
671+
import cp from "node:child_process";
672+
import { syncBuiltinESMExports } from "node:module";
673+
const server = Promise.withResolvers();
674+
const originalSpawn = cp.spawn;
675+
cp.spawn = (...args) => { const child = originalSpawn(...args); server.resolve(child); return child; };
676+
process.on("message", async message => {
677+
if (message === "installed") (await server.promise).send("request-update");
678+
});
679+
const originalRead = fs.readFile;
680+
fs.readFile = async (...args) => {
681+
const contents = await originalRead(...args);
682+
if (args[0] === ${JSON.stringify(plan.unitPath)}) {
683+
fs.readFile = originalRead;
684+
syncBuiltinESMExports();
685+
const resume = new Promise(resolve => process.once("message", resolve));
686+
process.send("snapshot");
687+
await resume;
688+
}
689+
return contents;
690+
};
691+
syncBuiltinESMExports();
692+
const { Launcher, readServiceState } = await import(${JSON.stringify(import.meta.resolve("../serviceLauncher.ts"))});
693+
const launcher = new Launcher(${JSON.stringify(plan.baseDir)}, await readServiceState(${JSON.stringify(statePath)}), {
694+
serviceUnit: { path: ${JSON.stringify(plan.unitPath)}, reload: async () => {} },
695+
});
696+
try { await launcher.run(); throw Error("unexpected completion"); }
697+
catch (error) {
698+
if (!error.message.includes("Active child exited unexpectedly (0)")) throw error;
699+
}
700+
const { acquireServiceSelectionLock } = await import(${JSON.stringify(import.meta.resolve("./serviceSelectionLock.ts"))});
701+
// Exit owning the real lease, as a crashed process would.
702+
await acquireServiceSelectionLock(${JSON.stringify(plan.unitPath)});
703+
process.disconnect();
704+
`,
705+
);
706+
const child = yield* Effect.acquireRelease(
707+
Effect.sync(() =>
708+
NodeChildProcess.fork(script, [], {
709+
execArgv: [],
710+
stdio: ["ignore", "inherit", "inherit", "ipc"],
711+
}),
712+
),
713+
(child) =>
714+
Effect.sync(() => {
715+
if (child.exitCode === null && child.signalCode === null) child.kill();
716+
}),
717+
);
718+
const exited = new Promise<void>((resolve, reject) => {
719+
child.once("error", reject);
720+
child.once("exit", (code) =>
721+
code === 0 ? resolve() : reject(new Error(`launcher exited ${code}`)),
722+
);
723+
});
724+
child.on("message", (message) => {
725+
if (message === "snapshot") snapshot.resolve();
726+
});
727+
yield* Effect.promise(() =>
728+
Promise.race([
729+
snapshot.promise,
730+
exited.then(() => {
731+
throw new Error("launcher exited before snapshot");
732+
}),
733+
]),
734+
);
735+
leaseObserver.contended = (destination) => {
736+
if (destination === plan.unitPath + ".selection-lock") milestone.resolve();
737+
};
738+
yield* Effect.addFinalizer(() =>
739+
Effect.sync(() => {
740+
leaseObserver.contended = () => {};
741+
}),
742+
);
743+
const installing = yield* newer
744+
.install({ start: false })
745+
.pipe(Effect.ensuring(Effect.sync(() => milestone.resolve())), Effect.forkChild);
746+
// Resume the launcher once the installer contends for the unit lease, or once an
747+
// uncoordinated installer has finished, which makes the stale snapshot win.
748+
yield* Effect.promise(() => milestone.promise);
749+
child.send("resume");
750+
yield* Fiber.join(installing);
751+
child.send("installed");
752+
yield* Effect.promise(() => exited);
753+
expect(yield* fs.readFileString(rejection)).toContain("Service selection changed");
754+
expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({
755+
protocol: SERVICE_LAUNCHER_PROTOCOL,
756+
activeVersion: "1.2.4",
757+
});
758+
expect(yield* fs.readFileString(plan.unitPath)).toContain("versions/1.2.4/t3");
759+
expect(
760+
yield* fs.readFileString(
761+
path.join(path.dirname(statePath), SERVICE_RESTART_PENDING_FILE),
762+
),
763+
).toBe("1.2.4\n");
764+
// The next installer reclaims the dead owner and releases its own lease.
765+
yield* newer.install({ start: false });
766+
expect(yield* fs.exists(plan.unitPath + ".selection-lock")).toBe(false);
578767
}),
579768
);
580769

@@ -617,6 +806,11 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
617806
const error = yield* service.install({ start: false }).pipe(Effect.flip);
618807
expect(error._tag).toBe("BootServiceUpdatePendingError");
619808
expect(yield* fs.readFileString(statePath)).toBe(pendingState);
809+
yield* fs.remove((yield* service.status).unitPath);
810+
expect((yield* service.install({ start: false }).pipe(Effect.flip))._tag).toBe(
811+
"BootServiceUpdatePendingError",
812+
);
813+
expect(yield* fs.readFileString(statePath)).toBe(pendingState);
620814
}),
621815
);
622816

0 commit comments

Comments
 (0)