Skip to content

Commit dfa6c7a

Browse files
PR Batch Testerclaude
andcommitted
fix(server): service launcher starts its server again after another server releases the T3 home
When the service child exits 78 because another server holds the owner lock, the launcher used to idle until someone restarted the unit. It now checks the lock every 30 s and starts the active version again once the lock is free. Polling inside the launcher keeps the no-restart-loop guarantee from #16102. Refs #24225 Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent e340aae commit dfa6c7a

2 files changed

Lines changed: 102 additions & 5 deletions

File tree

‎apps/server/src/serviceLauncher.test.ts‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
// @effect-diagnostics-next-line nodeBuiltinImport:off - observes the children the launcher spawns with Node built-ins.
22
import type * as NodeChildProcess from "node:child_process";
3+
import * as NodeSqlite from "node:sqlite";
34

45
import * as NodeServices from "@effect/platform-node/NodeServices";
56
import { assert, it } from "@effect/vitest";
@@ -253,6 +254,63 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {
253254
}),
254255
);
255256

257+
it.effect("starts the server again once the other owner releases the T3 home", () =>
258+
Effect.gen(function* () {
259+
const fs = yield* FileSystem.FileSystem;
260+
const path = yield* Path.Path;
261+
const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-reclaim-" });
262+
const statePath = path.join(root, "runtime", "service-state.json");
263+
const ownerPath = path.join(root, "userdata", "server-owner.sqlite");
264+
yield* fs.makeDirectory(path.dirname(ownerPath), { recursive: true });
265+
yield* writeFakeRuntime(
266+
fs,
267+
path,
268+
path.join(root, "runtime", "versions", "1.0.0"),
269+
[
270+
`const db = new (require("node:sqlite").DatabaseSync)(${JSON.stringify(ownerPath)});`,
271+
`try { db.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;"); }`,
272+
`catch { process.exit(${SERVER_EXIT_CODE_STATE_DIR_OWNED}); }`,
273+
"setInterval(() => {}, 1_000);",
274+
].join("\n"),
275+
);
276+
yield* Effect.promise(() =>
277+
writeServiceState(statePath, {
278+
protocol: SERVICE_LAUNCHER_PROTOCOL,
279+
activeVersion: "1.0.0",
280+
}),
281+
);
282+
const otherOwner = new NodeSqlite.DatabaseSync(ownerPath);
283+
otherOwner.exec("BEGIN EXCLUSIVE;");
284+
const refused = Promise.withResolvers<void>();
285+
const reclaimed = Promise.withResolvers<void>();
286+
yield* Effect.acquireRelease(
287+
Effect.sync(() =>
288+
vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
289+
if (String(chunk).includes("owns this T3 home")) refused.resolve();
290+
if (String(chunk).includes("T3 home is free again")) reclaimed.resolve();
291+
return true;
292+
}),
293+
),
294+
(spy) => Effect.sync(() => spy.mockRestore()),
295+
);
296+
const spawnsBefore = spawnedChildren.length;
297+
const launcher = new Launcher(
298+
root,
299+
yield* Effect.promise(() => readServiceState(statePath)),
300+
{
301+
stateDirOwnerPollMs: 10,
302+
},
303+
);
304+
const running = launcher.run();
305+
yield* Effect.promise(() => refused.promise);
306+
otherOwner.close();
307+
yield* Effect.promise(() => reclaimed.promise);
308+
yield* Effect.promise(() => launcher.stop("SIGTERM"));
309+
yield* Effect.promise(() => running);
310+
assert.strictEqual(spawnedChildren.length - spawnsBefore, 2);
311+
}),
312+
);
313+
256314
it.effect("commits only after the trial reports prepared", () =>
257315
Effect.gen(function* () {
258316
const fs = yield* FileSystem.FileSystem;

‎apps/server/src/serviceLauncher.ts‎

Lines changed: 44 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import * as NodeCrypto from "node:crypto";
99
import * as NodeFS from "node:fs";
1010
import * as NodeFSP from "node:fs/promises";
1111
import * as NodePath from "node:path";
12+
import * as NodeSqlite from "node:sqlite";
1213
import * as NodeUtil from "node:util";
1314

1415
import type {
@@ -42,6 +43,7 @@ const TERMINATE_GRACE_MS = 5_000;
4243
// Matches SERVER_EXIT_CODE_STATE_DIR_OWNED in contracts. The standalone
4344
// launcher stays independent of the Effect-backed contracts package.
4445
const STATE_DIR_OWNED_EXIT_CODE = 78;
46+
const STATE_DIR_OWNER_POLL_MS = 30_000;
4547

4648
type TerminalStatus = "committed" | "rolled-back" | "failed";
4749
type ChildRole = "active" | "trial";
@@ -458,6 +460,28 @@ async function terminateChild(
458460
}
459461
}
460462

463+
/**
464+
* Takes and drops the same exclusive transaction the server holds for its
465+
* lifetime (see `acquireServerOwnership`). Only SQLITE_BUSY counts as owned:
466+
* any other failure is left for the server's own startup to report.
467+
*/
468+
function stateDirOwned(baseDir: string): boolean {
469+
let db: NodeSqlite.DatabaseSync | undefined;
470+
try {
471+
db = new NodeSqlite.DatabaseSync(NodePath.join(baseDir, "userdata", "server-owner.sqlite"));
472+
db.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE; ROLLBACK;");
473+
return false;
474+
} catch (cause) {
475+
return (
476+
cause instanceof Error &&
477+
(("errcode" in cause && cause.errcode === 5) ||
478+
("code" in cause && cause.code === "SQLITE_BUSY"))
479+
);
480+
} finally {
481+
db?.close();
482+
}
483+
}
484+
461485
const stopMarkerPath = (baseDir: string) =>
462486
NodePath.join(baseDir, "runtime", SERVICE_STOP_MARKER_FILE);
463487
const restartPendingPath = (baseDir: string) =>
@@ -476,16 +500,19 @@ export class Launcher {
476500
readonly #completion = Promise.withResolvers<void>();
477501
readonly #backupSpaceProbe: Partial<BackupSpaceProbe> | undefined;
478502
readonly #environment: Readonly<NodeJS.ProcessEnv>;
503+
readonly #stateDirOwnerPollMs: number;
479504

480505
constructor(
481506
baseDir: string,
482507
state: ServiceState,
483508
options: {
484509
readonly backupSpaceProbe?: Partial<BackupSpaceProbe>;
485510
readonly environment?: Readonly<NodeJS.ProcessEnv>;
511+
readonly stateDirOwnerPollMs?: number;
486512
} = {},
487513
) {
488514
this.#baseDir = baseDir;
515+
this.#stateDirOwnerPollMs = options.stateDirOwnerPollMs ?? STATE_DIR_OWNER_POLL_MS;
489516
this.#backupSpaceProbe = options.backupSpaceProbe;
490517
this.#environment = { ...(options.environment ?? process.env) };
491518
this.#statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE);
@@ -814,8 +841,9 @@ export class Launcher {
814841

815842
/**
816843
* Exiting would make systemd or launchd start another refused server, so stay
817-
* idle until stopped or restarted. A pending update fails without restoring
818-
* its backup: the other owner may have written to the database since.
844+
* idle and start the server again once the other owner releases the lock. A
845+
* pending update fails without restoring its backup: the other owner may have
846+
* written to the database since.
819847
*/
820848
async #idleWhileStateDirOwned(): Promise<void> {
821849
this.#clearTimer();
@@ -830,13 +858,24 @@ export class Launcher {
830858
this.#state = next;
831859
await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined);
832860
}
833-
// Signal listeners alone do not keep Node alive.
834-
this.#timer = setInterval(() => {}, 2_147_483_647);
835861
process.stderr.write(
836-
"[service-launcher] Another T3 Code server owns this T3 home. Stop it, then restart this service.\n",
862+
"[service-launcher] Another T3 Code server owns this T3 home. This service starts again once it stops.\n",
863+
);
864+
// The interval also keeps Node alive: signal listeners alone do not.
865+
this.#timer = setInterval(
866+
() => this.#enqueue(() => this.#reclaimStateDir()),
867+
this.#stateDirOwnerPollMs,
837868
);
838869
}
839870

871+
async #reclaimStateDir(): Promise<void> {
872+
if (this.#timer === undefined || this.#child !== null || this.#stopping) return;
873+
if (stateDirOwned(this.#baseDir)) return;
874+
this.#clearTimer();
875+
process.stderr.write("[service-launcher] The T3 home is free again. Starting the server.\n");
876+
await this.#startChild(this.#state.activeVersion, "active", this.#state.update);
877+
}
878+
840879
async #returnToPrevious(
841880
pending: PendingServiceUpdate,
842881
status: "rolled-back" | "failed",

0 commit comments

Comments
 (0)