Skip to content

Commit 44ee2b9

Browse files
maria-rcksshivamhwpjuliusmarminge
authored
fix(server): restart the service after T3 home ownership clears (#25237)
Co-authored-by: shivam <[email protected]> Co-authored-by: Julius Marminge <[email protected]>
1 parent 9b242e4 commit 44ee2b9

4 files changed

Lines changed: 377 additions & 36 deletions

File tree

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// Effect-free so the service launcher shares the pre-lock owner identity check with server startup.
2+
3+
export const legacyOwnerStartTimeCommand = (pid: number, platform: NodeJS.Platform) => {
4+
const windows = platform === "win32";
5+
return {
6+
command: windows ? "powershell.exe" : "ps",
7+
args: windows
8+
? [
9+
"-NoProfile",
10+
"-NonInteractive",
11+
"-Command",
12+
`(Get-Process -Id ${pid} -ErrorAction Stop).StartTime.ToUniversalTime().ToString('o')`,
13+
]
14+
: ["-p", String(pid), "-o", "lstart="],
15+
env: { LC_ALL: "C", TZ: "UTC" },
16+
};
17+
};
18+
19+
// ps reports whole seconds. Unknown identity (no output) stays conservative.
20+
export const legacyOwnerStartedBeforeRecord = (
21+
output: string | undefined,
22+
recordedAt: number,
23+
platform: NodeJS.Platform,
24+
): boolean => {
25+
if (output === undefined) return true;
26+
const trimmed = output.trim();
27+
const startedAt = Date.parse(platform === "win32" ? trimmed : `${trimmed} UTC`);
28+
return !Number.isFinite(startedAt) || startedAt <= recordedAt + 1_000;
29+
};

‎apps/server/src/serverOwnership.ts‎

Lines changed: 13 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,10 @@ import {
2727
compareExactServiceVersions,
2828
serviceStateActiveVersion,
2929
} from "./cloud/serviceProtocol.ts";
30+
import {
31+
legacyOwnerStartedBeforeRecord,
32+
legacyOwnerStartTimeCommand,
33+
} from "./legacyServerOwner.ts";
3034
import * as ProcessRunner from "./processRunner.ts";
3135
import {
3236
clearPersistedServerRuntimeState,
@@ -80,29 +84,20 @@ const legacyOwnerIsLive = Effect.fn("legacyOwnerIsLive")(function* (
8084
if (!isProcessAlive(state.pid)) return false;
8185
const recordedAt = Date.parse(state.startedAt);
8286
if (!Number.isFinite(recordedAt)) return true;
83-
const windows = (yield* HostProcess.Platform) === "win32";
87+
const platform = yield* HostProcess.Platform;
8488
const runner = yield* ProcessRunner.ProcessRunner;
8589
const result = yield* runner
8690
.run({
87-
command: windows ? "powershell.exe" : "ps",
88-
args: windows
89-
? [
90-
"-NoProfile",
91-
"-NonInteractive",
92-
"-Command",
93-
`(Get-Process -Id ${state.pid} -ErrorAction Stop).StartTime.ToUniversalTime().ToString('o')`,
94-
]
95-
: ["-p", String(state.pid), "-o", "lstart="],
96-
env: { LC_ALL: "C", TZ: "UTC" },
91+
...legacyOwnerStartTimeCommand(state.pid, platform),
9792
timeout: Duration.seconds(2),
9893
maxOutputBytes: 16_384,
9994
})
10095
.pipe(Effect.option);
101-
if (Option.isNone(result) || result.value.code !== 0) return true;
102-
const output = result.value.stdout.trim();
103-
const startedAt = Date.parse(windows ? output : `${output} UTC`);
104-
// ps reports whole seconds. Unknown identity stays conservative.
105-
return !Number.isFinite(startedAt) || startedAt <= recordedAt + 1_000;
96+
return legacyOwnerStartedBeforeRecord(
97+
Option.isSome(result) && result.value.code === 0 ? result.value.stdout : undefined,
98+
recordedAt,
99+
platform,
100+
);
106101
});
107102

108103
// Pre-lock services become invisible when another server replaces their runtime
@@ -193,7 +188,8 @@ const openOwnershipLock = (stateDir: string) =>
193188
}
194189
const db = new NodeSqlite.DatabaseSync(lockPath);
195190
try {
196-
db.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;");
191+
// An idle service launcher probes this lock for a moment; wait out that probe.
192+
db.exec("PRAGMA busy_timeout = 250; BEGIN EXCLUSIVE;");
197193
const identity = NodeFS.statSync(lockPath, { bigint: true });
198194
return {
199195
db,

‎apps/server/src/serviceLauncher.test.ts‎

Lines changed: 234 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import {
3030
registerServiceTask,
3131
serviceLauncherIsRunning,
3232
serviceLauncherLockPath,
33+
stateDirOwned,
3334
writeServiceState,
3435
} from "./serviceLauncher.ts";
3536
import {
@@ -44,12 +45,27 @@ import {
4445
} from "./cloud/serviceProtocol.ts";
4546

4647
const spawnedChildren = vi.hoisted((): Array<NodeChildProcess.ChildProcess> => []);
48+
const spawnHook = vi.hoisted(() => ({
49+
current: undefined as ((child: NodeChildProcess.ChildProcess) => void) | undefined,
50+
}));
51+
52+
const failRm = vi.hoisted(() => ({ path: undefined as string | undefined }));
53+
54+
vi.mock("node:fs/promises", async (importOriginal) => {
55+
const actual = await importOriginal<typeof NodeFSP>();
56+
const rm: typeof actual.rm = (path, options) =>
57+
path === failRm.path
58+
? Promise.reject(Object.assign(new Error("EACCES"), { code: "EACCES" }))
59+
: actual.rm(path, options);
60+
return { ...actual, rm };
61+
});
4762

4863
vi.mock("node:child_process", async (importOriginal) => {
4964
const actual = await importOriginal<typeof NodeChildProcess>();
5065
const spawn = (...args: Parameters<typeof actual.spawn>) => {
5166
const child = actual.spawn(...args);
5267
spawnedChildren.push(child);
68+
spawnHook.current?.(child);
5369
return child;
5470
};
5571
return { ...actual, spawn: spawn as typeof actual.spawn };
@@ -578,25 +594,36 @@ process.exit(0);
578594
}),
579595
);
580596

581-
it.effect("stays idle instead of exiting when another server owns the T3 home", () =>
597+
it.effect("waits without exiting and restarts once the other owner releases the T3 home", () =>
582598
Effect.gen(function* () {
583599
const fs = yield* FileSystem.FileSystem;
584600
const path = yield* Path.Path;
585-
const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-refused-" });
601+
const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-reclaim-" });
586602
const statePath = path.join(root, "runtime", "service-state.json");
603+
const ownerPath = path.join(root, "userdata", "server-owner.sqlite");
604+
yield* fs.makeDirectory(path.dirname(ownerPath), { recursive: true });
587605
yield* writeFakeRuntime(
588606
fs,
589607
path,
590608
path.join(root, "runtime", "versions", "1.0.0"),
591-
`process.exit(${SERVER_EXIT_CODE_STATE_DIR_OWNED});\n`,
609+
[
610+
`const db = new (require("node:sqlite").DatabaseSync)(${JSON.stringify(ownerPath)});`,
611+
`try { db.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;"); }`,
612+
`catch { process.exit(${SERVER_EXIT_CODE_STATE_DIR_OWNED}); }`,
613+
'process.send("owner");',
614+
"setInterval(() => {}, 1_000);",
615+
].join("\n"),
592616
);
593617
yield* Effect.promise(() =>
594618
writeServiceState(statePath, {
595619
protocol: SERVICE_LAUNCHER_PROTOCOL,
596620
activeVersion: "1.0.0",
597621
}),
598622
);
623+
using otherOwner = new NodeSqlite.DatabaseSync(ownerPath);
624+
otherOwner.exec("BEGIN EXCLUSIVE;");
599625
const refused = Promise.withResolvers<void>();
626+
const owned = Promise.withResolvers<void>();
600627
yield* Effect.acquireRelease(
601628
Effect.sync(() =>
602629
vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
@@ -606,15 +633,218 @@ process.exit(0);
606633
),
607634
(spy) => Effect.sync(() => spy.mockRestore()),
608635
);
609-
const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath)));
636+
yield* Effect.acquireRelease(
637+
Effect.sync(() => {
638+
spawnHook.current = (child) => child.once("message", () => owned.resolve());
639+
}),
640+
() =>
641+
Effect.sync(() => {
642+
spawnHook.current = undefined;
643+
}),
644+
);
645+
const spawnsBefore = spawnedChildren.length;
646+
const launcher = new Launcher(
647+
root,
648+
yield* Effect.promise(() => readServiceState(statePath)),
649+
{ stateDirOwnerPollMs: 10 },
650+
);
610651
let completed = false;
611652
const running = launcher.run().finally(() => {
612653
completed = true;
613654
});
614655
yield* Effect.promise(() => refused.promise);
615656
assert.isFalse(completed);
657+
otherOwner.close();
658+
yield* Effect.promise(() => owned.promise);
659+
yield* Effect.promise(() => launcher.stop("SIGTERM"));
660+
yield* Effect.promise(() => running);
661+
assert.strictEqual(spawnedChildren.length - spawnsBefore, 2);
662+
}),
663+
);
664+
665+
it.effect(
666+
"keeps a pending update while owned and rolls a failed resumed trial back to the other owner's writes",
667+
() =>
668+
Effect.gen(function* () {
669+
const fs = yield* FileSystem.FileSystem;
670+
const path = yield* Path.Path;
671+
const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-reclaim-update-" });
672+
const statePath = path.join(root, "runtime", "service-state.json");
673+
const ownerPath = path.join(root, "userdata", "server-owner.sqlite");
674+
const databasePath = path.join(root, "userdata", "state.sqlite");
675+
const startsPath = path.join(root, "starts.log");
676+
yield* fs.makeDirectory(path.dirname(ownerPath), { recursive: true });
677+
yield* fs.writeFileString(databasePath, "before update");
678+
for (const version of ["1.0.0", "1.1.0"]) {
679+
yield* writeFakeRuntime(
680+
fs,
681+
path,
682+
path.join(root, "runtime", "versions", version),
683+
[
684+
`const db = new (require("node:sqlite").DatabaseSync)(${JSON.stringify(ownerPath)});`,
685+
`try { db.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;"); }`,
686+
`catch { process.exit(${SERVER_EXIT_CODE_STATE_DIR_OWNED}); }`,
687+
"const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);",
688+
`require("node:fs").appendFileSync(${JSON.stringify(startsPath)}, context.childVersion + "\\n");`,
689+
'if (context.update?.status === "pending") {',
690+
` require("node:fs").writeFileSync(${JSON.stringify(databasePath)}, "migrated by failed trial");`,
691+
" process.exit(1);",
692+
"}",
693+
'process.send("active");',
694+
"setInterval(() => {}, 1_000);",
695+
].join("\n"),
696+
);
697+
}
698+
yield* Effect.promise(() =>
699+
writeServiceState(statePath, {
700+
protocol: SERVICE_LAUNCHER_PROTOCOL,
701+
activeVersion: "1.0.0",
702+
update: {
703+
id: "update-1",
704+
fromVersion: "1.0.0",
705+
targetVersion: "1.1.0",
706+
dbPath: databasePath,
707+
status: "pending",
708+
},
709+
}),
710+
);
711+
using otherOwner = new NodeSqlite.DatabaseSync(ownerPath);
712+
otherOwner.exec("BEGIN EXCLUSIVE;");
713+
const refused = Promise.withResolvers<void>();
714+
const active = Promise.withResolvers<void>();
715+
yield* Effect.acquireRelease(
716+
Effect.sync(() =>
717+
vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
718+
if (String(chunk).includes("owns this T3 home")) refused.resolve();
719+
return true;
720+
}),
721+
),
722+
(spy) => Effect.sync(() => spy.mockRestore()),
723+
);
724+
yield* Effect.acquireRelease(
725+
Effect.sync(() => {
726+
spawnHook.current = (child) =>
727+
child.on("message", (message) => message === "active" && active.resolve());
728+
}),
729+
() =>
730+
Effect.sync(() => {
731+
spawnHook.current = undefined;
732+
}),
733+
);
734+
const launcher = new Launcher(
735+
root,
736+
yield* Effect.promise(() => readServiceState(statePath)),
737+
{ stateDirOwnerPollMs: 10 },
738+
);
739+
const running = launcher.run();
740+
yield* Effect.promise(() => refused.promise);
741+
const idle = yield* Effect.promise(() => readServiceState(statePath));
742+
assert.strictEqual(idle.activeVersion, "1.0.0");
743+
assert.strictEqual(idle.update?.status, "pending");
744+
assert.isFalse(yield* fs.exists(path.join(root, "runtime", "db-backup", "update-1")));
745+
746+
yield* fs.writeFileString(databasePath, "written by the other owner");
747+
otherOwner.close();
748+
yield* Effect.promise(() => active.promise);
749+
yield* Effect.promise(() => launcher.stop("SIGTERM"));
750+
yield* Effect.promise(() => running);
751+
752+
const state = yield* Effect.promise(() => readServiceState(statePath));
753+
assert.strictEqual(state.activeVersion, "1.0.0");
754+
assert.strictEqual(state.update?.status, "rolled-back");
755+
assert.strictEqual(yield* fs.readFileString(startsPath), "1.1.0\n1.0.0\n");
756+
assert.strictEqual(yield* fs.readFileString(databasePath), "written by the other owner");
757+
}),
758+
);
759+
760+
it.effect("fails a pending update when its backup cannot be dropped while owned", () =>
761+
Effect.gen(function* () {
762+
const fs = yield* FileSystem.FileSystem;
763+
const path = yield* Path.Path;
764+
const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-stale-backup-" });
765+
const statePath = path.join(root, "runtime", "service-state.json");
766+
const ownerPath = path.join(root, "userdata", "server-owner.sqlite");
767+
const databasePath = path.join(root, "userdata", "state.sqlite");
768+
yield* fs.makeDirectory(path.dirname(ownerPath), { recursive: true });
769+
yield* fs.writeFileString(databasePath, "before update");
770+
yield* writeFakeRuntime(
771+
fs,
772+
path,
773+
path.join(root, "runtime", "versions", "1.1.0"),
774+
`process.exit(${SERVER_EXIT_CODE_STATE_DIR_OWNED});\n`,
775+
);
776+
yield* Effect.promise(() =>
777+
writeServiceState(statePath, {
778+
protocol: SERVICE_LAUNCHER_PROTOCOL,
779+
activeVersion: "1.0.0",
780+
update: {
781+
id: "update-1",
782+
fromVersion: "1.0.0",
783+
targetVersion: "1.1.0",
784+
dbPath: databasePath,
785+
status: "pending",
786+
},
787+
}),
788+
);
789+
const refused = Promise.withResolvers<void>();
790+
yield* Effect.acquireRelease(
791+
Effect.sync(() =>
792+
vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
793+
if (String(chunk).includes("owns this T3 home")) refused.resolve();
794+
return true;
795+
}),
796+
),
797+
(spy) => Effect.sync(() => spy.mockRestore()),
798+
);
799+
yield* Effect.acquireRelease(
800+
Effect.sync(() => {
801+
failRm.path = path.join(root, "runtime", "db-backup", "update-1");
802+
}),
803+
() =>
804+
Effect.sync(() => {
805+
failRm.path = undefined;
806+
}),
807+
);
808+
const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath)));
809+
const running = launcher.run();
810+
yield* Effect.promise(() => refused.promise);
616811
yield* Effect.promise(() => launcher.stop("SIGTERM"));
617812
yield* Effect.promise(() => running);
813+
814+
const state = yield* Effect.promise(() => readServiceState(statePath));
815+
assert.strictEqual(state.activeVersion, "1.0.0");
816+
assert.strictEqual(state.update?.status, "failed");
817+
}),
818+
);
819+
820+
it.effect("treats the T3 home as owned while the lock or a pre-lock server holds it", () =>
821+
Effect.gen(function* () {
822+
const fs = yield* FileSystem.FileSystem;
823+
const path = yield* Path.Path;
824+
const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-owned-" });
825+
const stateDir = path.join(root, "userdata");
826+
const recordPath = path.join(stateDir, "server-runtime.json");
827+
yield* fs.makeDirectory(stateDir, { recursive: true });
828+
assert.isFalse(yield* Effect.promise(() => stateDirOwned(root)));
829+
830+
using owner = new NodeSqlite.DatabaseSync(path.join(stateDir, "server-owner.sqlite"));
831+
owner.exec("BEGIN EXCLUSIVE;");
832+
assert.isTrue(yield* Effect.promise(() => stateDirOwned(root)));
833+
owner.close();
834+
835+
const record = { version: 1, pid: process.pid, port: 1, origin: "", startedAt: "" };
836+
yield* fs.writeFileString(recordPath, JSON.stringify(record));
837+
assert.isTrue(yield* Effect.promise(() => stateDirOwned(root)));
838+
yield* fs.writeFileString(recordPath, JSON.stringify({ ...record, ownerId: "lock-aware" }));
839+
assert.isFalse(yield* Effect.promise(() => stateDirOwned(root)));
840+
// This process started after 1970, so its PID was reused since that record.
841+
yield* fs.writeFileString(
842+
recordPath,
843+
JSON.stringify({ ...record, startedAt: "1970-01-01T00:00:00Z" }),
844+
);
845+
assert.isFalse(yield* Effect.promise(() => stateDirOwned(root)));
846+
yield* fs.writeFileString(recordPath, JSON.stringify({ ...record, pid: 2 ** 22 + 1 }));
847+
assert.isFalse(yield* Effect.promise(() => stateDirOwned(root)));
618848
}),
619849
);
620850

0 commit comments

Comments
 (0)