Skip to content

Commit c2f6f44

Browse files
committed
fix(server): a reused PID no longer keeps the service launcher waiting
1 parent 24dc26d commit c2f6f44

4 files changed

Lines changed: 102 additions & 39 deletions

File tree

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// Effect-free so the service launcher shares the pre-lock owner identity check with server startup.
2+
3+
export const legacyOwnerStartTimeCommand = (pid: number, platform: NodeJS.Platform) => {
4+
const windows = platform === "win32";
5+
return {
6+
command: windows ? "powershell.exe" : "ps",
7+
args: windows
8+
? [
9+
"-NoProfile",
10+
"-NonInteractive",
11+
"-Command",
12+
`(Get-Process -Id ${pid} -ErrorAction Stop).StartTime.ToUniversalTime().ToString('o')`,
13+
]
14+
: ["-p", String(pid), "-o", "lstart="],
15+
env: { LC_ALL: "C", TZ: "UTC" },
16+
};
17+
};
18+
19+
// ps reports whole seconds. Unknown identity (no output) stays conservative.
20+
export const legacyOwnerStartedBeforeRecord = (
21+
output: string | undefined,
22+
recordedAt: number,
23+
platform: NodeJS.Platform,
24+
): boolean => {
25+
if (output === undefined) return true;
26+
const trimmed = output.trim();
27+
const startedAt = Date.parse(platform === "win32" ? trimmed : `${trimmed} UTC`);
28+
return !Number.isFinite(startedAt) || startedAt <= recordedAt + 1_000;
29+
};

‎apps/server/src/serverOwnership.ts‎

Lines changed: 11 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,10 @@ import {
2727
compareExactServiceVersions,
2828
serviceStateActiveVersion,
2929
} from "./cloud/serviceProtocol.ts";
30+
import {
31+
legacyOwnerStartedBeforeRecord,
32+
legacyOwnerStartTimeCommand,
33+
} from "./legacyServerOwner.ts";
3034
import * as ProcessRunner from "./processRunner.ts";
3135
import {
3236
clearPersistedServerRuntimeState,
@@ -80,29 +84,20 @@ const legacyOwnerIsLive = Effect.fn("legacyOwnerIsLive")(function* (
8084
if (!isProcessAlive(state.pid)) return false;
8185
const recordedAt = Date.parse(state.startedAt);
8286
if (!Number.isFinite(recordedAt)) return true;
83-
const windows = (yield* HostProcess.Platform) === "win32";
87+
const platform = yield* HostProcess.Platform;
8488
const runner = yield* ProcessRunner.ProcessRunner;
8589
const result = yield* runner
8690
.run({
87-
command: windows ? "powershell.exe" : "ps",
88-
args: windows
89-
? [
90-
"-NoProfile",
91-
"-NonInteractive",
92-
"-Command",
93-
`(Get-Process -Id ${state.pid} -ErrorAction Stop).StartTime.ToUniversalTime().ToString('o')`,
94-
]
95-
: ["-p", String(state.pid), "-o", "lstart="],
96-
env: { LC_ALL: "C", TZ: "UTC" },
91+
...legacyOwnerStartTimeCommand(state.pid, platform),
9792
timeout: Duration.seconds(2),
9893
maxOutputBytes: 16_384,
9994
})
10095
.pipe(Effect.option);
101-
if (Option.isNone(result) || result.value.code !== 0) return true;
102-
const output = result.value.stdout.trim();
103-
const startedAt = Date.parse(windows ? output : `${output} UTC`);
104-
// ps reports whole seconds. Unknown identity stays conservative.
105-
return !Number.isFinite(startedAt) || startedAt <= recordedAt + 1_000;
96+
return legacyOwnerStartedBeforeRecord(
97+
Option.isSome(result) && result.value.code === 0 ? result.value.stdout : undefined,
98+
recordedAt,
99+
platform,
100+
);
106101
});
107102

108103
// Pre-lock services become invisible when another server replaces their runtime

‎apps/server/src/serviceLauncher.test.ts‎

Lines changed: 36 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -251,16 +251,27 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {
251251
using otherOwner = new NodeSqlite.DatabaseSync(ownerPath);
252252
otherOwner.exec("BEGIN EXCLUSIVE;");
253253
let refused = Promise.withResolvers<void>();
254+
let onRefused: (() => void) | undefined;
254255
const owned = Promise.withResolvers<void>();
255256
yield* Effect.acquireRelease(
256257
Effect.sync(() =>
257258
vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
258-
if (String(chunk).includes("owns this T3 home")) refused.resolve();
259+
if (!String(chunk).includes("owns this T3 home")) return true;
260+
onRefused?.();
261+
refused.resolve();
259262
return true;
260263
}),
261264
),
262265
(spy) => Effect.sync(() => spy.mockRestore()),
263266
);
267+
const setIntervalSpy = yield* Effect.acquireRelease(
268+
Effect.sync(() => vi.spyOn(globalThis, "setInterval")),
269+
(spy) => Effect.sync(() => spy.mockRestore()),
270+
);
271+
const clearTimeoutSpy = yield* Effect.acquireRelease(
272+
Effect.sync(() => vi.spyOn(globalThis, "clearTimeout")),
273+
(spy) => Effect.sync(() => spy.mockRestore()),
274+
);
264275
yield* Effect.acquireRelease(
265276
Effect.sync(() => {
266277
spawnHook.current = (child) => child.once("message", () => owned.resolve());
@@ -278,15 +289,14 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {
278289
stateDirOwnerPollMs: 10,
279290
},
280291
);
281-
let completed = false;
282-
const running = launcher.run().finally(() => {
283-
completed = true;
284-
});
285-
yield* Effect.promise(() => refused.promise);
286-
assert.isFalse(completed);
287-
yield* Effect.promise(() => launcher.stop("SIGTERM"));
288-
yield* Effect.promise(() => running);
289-
292+
// Stop lands after the exit-78 transition starts but before it installs the owner poll.
293+
onRefused = () => void launcher.stop("SIGTERM");
294+
yield* Effect.promise(() => launcher.run());
295+
const poll = setIntervalSpy.mock.results.at(-1)?.value;
296+
assert.isDefined(poll);
297+
assert.isTrue(clearTimeoutSpy.mock.calls.some(([timer]) => timer === poll));
298+
299+
onRefused = undefined;
290300
refused = Promise.withResolvers<void>();
291301
const restarted = new Launcher(
292302
root,
@@ -295,8 +305,12 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {
295305
stateDirOwnerPollMs: 10,
296306
},
297307
);
298-
const recovering = restarted.run();
308+
let completed = false;
309+
const recovering = restarted.run().finally(() => {
310+
completed = true;
311+
});
299312
yield* Effect.promise(() => refused.promise);
313+
assert.isFalse(completed);
300314
otherOwner.close();
301315
yield* Effect.promise(() => owned.promise);
302316
yield* Effect.promise(() => restarted.stop("SIGTERM"));
@@ -313,20 +327,26 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {
313327
const stateDir = path.join(root, "userdata");
314328
const recordPath = path.join(stateDir, "server-runtime.json");
315329
yield* fs.makeDirectory(stateDir, { recursive: true });
316-
assert.isFalse(stateDirOwned(root));
330+
assert.isFalse(yield* Effect.promise(() => stateDirOwned(root)));
317331

318332
using owner = new NodeSqlite.DatabaseSync(path.join(stateDir, "server-owner.sqlite"));
319333
owner.exec("BEGIN EXCLUSIVE;");
320-
assert.isTrue(stateDirOwned(root));
334+
assert.isTrue(yield* Effect.promise(() => stateDirOwned(root)));
321335
owner.close();
322336

323337
const record = { version: 1, pid: process.pid, port: 1, origin: "", startedAt: "" };
324338
yield* fs.writeFileString(recordPath, JSON.stringify(record));
325-
assert.isTrue(stateDirOwned(root));
339+
assert.isTrue(yield* Effect.promise(() => stateDirOwned(root)));
326340
yield* fs.writeFileString(recordPath, JSON.stringify({ ...record, ownerId: "lock-aware" }));
327-
assert.isFalse(stateDirOwned(root));
341+
assert.isFalse(yield* Effect.promise(() => stateDirOwned(root)));
342+
// This process started after 1970, so its PID was reused since that record.
343+
yield* fs.writeFileString(
344+
recordPath,
345+
JSON.stringify({ ...record, startedAt: "1970-01-01T00:00:00Z" }),
346+
);
347+
assert.isFalse(yield* Effect.promise(() => stateDirOwned(root)));
328348
yield* fs.writeFileString(recordPath, JSON.stringify({ ...record, pid: 2 ** 22 + 1 }));
329-
assert.isFalse(stateDirOwned(root));
349+
assert.isFalse(yield* Effect.promise(() => stateDirOwned(root)));
330350
}),
331351
);
332352

‎apps/server/src/serviceLauncher.ts‎

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,10 @@ import {
2929
SERVICE_RESTART_PENDING_FILE,
3030
SERVICE_STOP_MARKER_FILE,
3131
} from "./cloud/serviceProtocol.ts";
32+
import {
33+
legacyOwnerStartedBeforeRecord,
34+
legacyOwnerStartTimeCommand,
35+
} from "./legacyServerOwner.ts";
3236

3337
const HANDOFF_DELAY_MS = 2_000;
3438
/**
@@ -448,10 +452,9 @@ const isSqliteBusy = (cause: unknown) =>
448452
(("errcode" in cause && cause.errcode === 5) ||
449453
("code" in cause && cause.code === "SQLITE_BUSY"));
450454

451-
// Mirror acquireServerOwnership: the lock, then a live pre-lock server-runtime.json PID.
452-
// Skip its PID-reuse check so a reused PID waits instead of restarting a refused server;
455+
// Mirror acquireServerOwnership: the lock, then a live pre-lock server-runtime.json owner;
453456
// leave other failures for server startup to report.
454-
export function stateDirOwned(baseDir: string): boolean {
457+
export async function stateDirOwned(baseDir: string): Promise<boolean> {
455458
const stateDir = NodePath.join(baseDir, "userdata");
456459
let db: NodeSqlite.DatabaseSync | undefined;
457460
try {
@@ -465,7 +468,7 @@ export function stateDirOwned(baseDir: string): boolean {
465468
let record: unknown;
466469
try {
467470
record = JSON.parse(
468-
NodeFS.readFileSync(NodePath.join(stateDir, "server-runtime.json"), "utf8"),
471+
await NodeFSP.readFile(NodePath.join(stateDir, "server-runtime.json"), "utf8"),
469472
);
470473
} catch {
471474
return false;
@@ -475,10 +478,25 @@ export function stateDirOwned(baseDir: string): boolean {
475478
if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0) return false;
476479
try {
477480
process.kill(pid, 0);
478-
return true;
479481
} catch (error) {
480-
return error instanceof Error && "code" in error && error.code === "EPERM";
482+
if (!(error instanceof Error && "code" in error && error.code === "EPERM")) return false;
481483
}
484+
const recordedAt = Date.parse(
485+
"startedAt" in record && typeof record.startedAt === "string" ? record.startedAt : "",
486+
);
487+
if (!Number.isFinite(recordedAt)) return true;
488+
// oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
489+
const platform = process.platform;
490+
const { command, args, env } = legacyOwnerStartTimeCommand(pid, platform);
491+
const output = await new Promise<string | undefined>((resolve) =>
492+
NodeChildProcess.execFile(
493+
command,
494+
args,
495+
{ env: { ...process.env, ...env }, timeout: 2_000, maxBuffer: 16_384 },
496+
(error, stdout) => resolve(error === null ? stdout : undefined),
497+
),
498+
);
499+
return legacyOwnerStartedBeforeRecord(output, recordedAt, platform);
482500
}
483501

484502
const stopMarkerPath = (baseDir: string) =>
@@ -859,7 +877,8 @@ export class Launcher {
859877
return;
860878
}
861879
if (this.#timer === undefined || this.#child !== null) return;
862-
if (stateDirOwned(this.#baseDir)) return;
880+
// A stop during the probe clears the timer.
881+
if ((await stateDirOwned(this.#baseDir)) || this.#timer === undefined) return;
863882
this.#clearTimer();
864883
process.stderr.write("[service-launcher] The T3 home is free again. Starting the server.\n");
865884
await this.#startChild(this.#state.activeVersion, "active", this.#state.update);

0 commit comments

Comments
 (0)