Skip to content

Commit 0b1ed14

Browse files
fix(cli): make a deferred service restart survive failures and outside restarts
- The restart-pending marker is written before the state and unit, so a failed repoint never leaves the old process looking current. - The launcher removes the marker on startup, so a systemctl or launchctl restart done by hand counts as the restart too. - A no-start install re-reads the state after writing it and stops before touching the unit if the launcher wrote a remote update in between. - t3 update judges the service by status.current (stopped, disabled, and pending-restart included), not only by its recorded version. Co-Authored-By: Claude Fable 5 <[email protected]>
1 parent ba1835e commit 0b1ed14

6 files changed

Lines changed: 93 additions & 11 deletions

File tree

‎apps/server/src/cli/update.ts‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -410,11 +410,16 @@ const runUpdate = Effect.fn("cli.update.run")(function* (input: {
410410
const serviceVersion = serviceInstalled ? status.installedVersion : undefined;
411411
const executableCurrent = targetVersion === currentVersion;
412412
// A service whose recorded version is missing or unreadable is not known
413-
// to be current, so it gets the update rather than being skipped. So is one
414-
// whose files already name the target but whose process was never
415-
// restarted onto it (an earlier update where the restart was declined).
413+
// to be current, so it gets the update rather than being skipped. Nor is
414+
// one on the right version that is stopped, disabled, or still running the
415+
// version before it (an earlier update where the restart was declined):
416+
// `status.current` covers all of that when the target is this executable,
417+
// and the problem list is what can be judged for any other target.
416418
const restartPending = status.problems?.includes("restart-pending") === true;
417-
const serviceCurrent = !serviceInstalled || (serviceVersion === targetVersion && !restartPending);
419+
const serviceCurrent =
420+
!serviceInstalled ||
421+
(serviceVersion === targetVersion &&
422+
(executableCurrent ? status.current : (status.problems ?? []).length === 0));
418423
const newestInstalled =
419424
serviceVersion !== undefined && compareExactServiceVersions(serviceVersion, currentVersion) > 0
420425
? serviceVersion

‎apps/server/src/cloud/bootService.test.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import { pinnedRuntimePaths } from "./pinnedRuntime.ts";
2020
import {
2121
parseServiceState,
2222
SERVICE_LAUNCHER_PROTOCOL,
23+
SERVICE_RESTART_PENDING_FILE,
2324
serviceStateHasPendingUpdate,
2425
} from "./serviceProtocol.ts";
2526

@@ -520,6 +521,27 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
520521
}),
521522
);
522523

524+
it.effect("install with start=false keeps the marker when a later write fails", () =>
525+
Effect.gen(function* () {
526+
const { service, fs, statePath, makeService } = yield* makeHarness();
527+
const path = yield* Path.Path;
528+
yield* service.install();
529+
const newer = yield* makeService(undefined, "1.2.4");
530+
// A non-empty directory in the unit's place: it still counts as an
531+
// installed unit, and the rename that writes the new unit fails.
532+
const unitPath = (yield* service.status).unitPath;
533+
yield* fs.remove(unitPath);
534+
yield* fs.makeDirectory(unitPath);
535+
yield* fs.writeFileString(path.join(unitPath, "occupied"), "");
536+
537+
const error = yield* newer.install({ start: false }).pipe(Effect.flip);
538+
expect(error._tag).toBe("BootServiceInstallError");
539+
expect(
540+
yield* fs.exists(path.join(path.dirname(statePath), SERVICE_RESTART_PENDING_FILE)),
541+
).toBe(true);
542+
}),
543+
);
544+
523545
it.effect("install with start=false refuses while a remote update is pending", () =>
524546
Effect.gen(function* () {
525547
const { service, fs, statePath } = yield* makeHarness();

‎apps/server/src/cloud/bootService.ts‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -851,6 +851,13 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
851851
yield* fs
852852
.makeDirectory(path.dirname(unitPath), { recursive: true })
853853
.pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause })));
854+
if (!start && installed) {
855+
// Written first: once the files below name the new version, the
856+
// running service is behind them, and a failure between the two
857+
// writes must not leave it looking current. The launcher removes the
858+
// marker when it starts, `restart` and a started install do too.
859+
yield* fs.writeFileString(restartPendingPath, `${input.cliVersion}\n`, { mode: 0o600 });
860+
}
854861
yield* writeDurably(
855862
statePath,
856863
// @effect-diagnostics-next-line preferSchemaOverJson:off - fixed launcher-owned document.
@@ -863,15 +870,23 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
863870
2,
864871
)}\n`,
865872
);
873+
if (!start && installed) {
874+
// The launcher only writes this file while a remote update is in
875+
// flight. One that began after the check above lands either before
876+
// this write (then the launcher's copy in memory is what it keeps
877+
// acting on, and its next write puts its own outcome back) or after
878+
// it, which this read catches: the file no longer says what was just
879+
// written, so stop here before repointing the unit.
880+
const written = yield* fs.readFileString(statePath);
881+
if (serviceStateActiveVersion(written) !== input.cliVersion) {
882+
return yield* new BootServiceUpdatePendingError();
883+
}
884+
}
866885
yield* writeDurably(unitPath, manager.render(plan));
867886

868887
if (start) {
869888
yield* runSteps(manager.activate);
870889
yield* fs.remove(restartPendingPath, { force: true });
871-
} else if (installed) {
872-
// The running service is now behind its unit until someone restarts
873-
// it; the marker is what status and install use to remember that.
874-
yield* fs.writeFileString(restartPendingPath, `${input.cliVersion}\n`, { mode: 0o600 });
875890
}
876891
}).pipe(
877892
Effect.mapError((cause) =>

‎apps/server/src/cloud/serviceProtocol.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,9 @@ export const SERVICE_STATE_FILE = "service-state.json";
99
to start my replacement" while a pending update is recorded. */
1010
export const SERVICE_STOP_MARKER_FILE = ".service-stopping";
1111
/** Written by `t3 update` when the unit was repointed at a new version but the
12-
running service was deliberately left on the old one, and removed by the
13-
next restart. Its presence keeps the service from reporting as current. */
12+
running service was deliberately left on the old one. The launcher removes
13+
it when it starts (whoever restarted the service), so while it exists the
14+
service is known to be behind its unit and status reports it that way. */
1415
export const SERVICE_RESTART_PENDING_FILE = ".restart-pending";
1516

1617
export interface PendingServiceUpdate {

‎apps/server/src/serviceLauncher.test.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import {
1010
decodeServiceState,
1111
isExactServiceVersion,
1212
SERVICE_LAUNCHER_PROTOCOL,
13+
SERVICE_RESTART_PENDING_FILE,
1314
SERVICE_STOP_MARKER_FILE,
1415
} from "./cloud/serviceProtocol.ts";
1516

@@ -113,6 +114,38 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {
113114
}),
114115
);
115116

117+
it.effect("a fresh launcher clears a restart deferred by t3 update", () =>
118+
Effect.gen(function* () {
119+
const fs = yield* FileSystem.FileSystem;
120+
const path = yield* Path.Path;
121+
const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-restart-" });
122+
const statePath = path.join(root, "runtime", "service-state.json");
123+
const restartPending = path.join(root, "runtime", SERVICE_RESTART_PENDING_FILE);
124+
yield* writeFakeRuntime(
125+
fs,
126+
path,
127+
path.join(root, "runtime", "versions", "1.0.0"),
128+
"setInterval(() => {}, 1_000);\n",
129+
);
130+
yield* Effect.promise(() =>
131+
writeServiceState(statePath, {
132+
protocol: SERVICE_LAUNCHER_PROTOCOL,
133+
activeVersion: "1.0.0",
134+
}),
135+
);
136+
yield* fs.writeFileString(restartPending, "1.0.0\n");
137+
138+
const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath)));
139+
const running = launcher.run();
140+
const stopping = launcher.stop("SIGTERM");
141+
yield* Effect.promise(() => stopping);
142+
yield* Effect.promise(() => running);
143+
// Whoever restarted the service, the launcher now runs what the unit
144+
// names, so the deferred-restart marker is gone.
145+
assert.isFalse(yield* fs.exists(restartPending));
146+
}),
147+
);
148+
116149
it.effect("serializes shutdown with launcher recovery", () =>
117150
Effect.gen(function* () {
118151
const fs = yield* FileSystem.FileSystem;

‎apps/server/src/serviceLauncher.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import {
2626
SERVICE_LAUNCHER_CONTEXT_ENV,
2727
SERVICE_LAUNCHER_PROTOCOL,
2828
SERVICE_STATE_FILE,
29+
SERVICE_RESTART_PENDING_FILE,
2930
SERVICE_STOP_MARKER_FILE,
3031
} from "./cloud/serviceProtocol.ts";
3132

@@ -273,6 +274,8 @@ async function terminateChild(
273274

274275
const stopMarkerPath = (baseDir: string) =>
275276
NodePath.join(baseDir, "runtime", SERVICE_STOP_MARKER_FILE);
277+
const restartPendingPath = (baseDir: string) =>
278+
NodePath.join(baseDir, "runtime", SERVICE_RESTART_PENDING_FILE);
276279

277280
export class Launcher {
278281
readonly #baseDir: string;
@@ -365,8 +368,11 @@ export class Launcher {
365368
async #recover(): Promise<void> {
366369
// A fresh launcher means servers are running again: any stop marker from
367370
// a previous explicit stop is stale and must not make a future update
368-
// handoff release its tunnel.
371+
// handoff release its tunnel. A fresh launcher is also, by definition,
372+
// the one the unit names, so a restart deferred by `t3 update` is done
373+
// no matter who restarted the service.
369374
await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined);
375+
await NodeFSP.rm(restartPendingPath(this.#baseDir), { force: true }).catch(() => undefined);
370376
const update = this.#state.update;
371377
if (update?.status !== "pending") {
372378
if (update !== undefined) {

0 commit comments

Comments
 (0)