Skip to content

Commit 003aaa3

Browse files
committed
fix(server): pause service restart when disk is full
1 parent 1152e91 commit 003aaa3

2 files changed

Lines changed: 55 additions & 0 deletions

File tree

‎apps/server/src/serviceLauncher.test.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import {
1515
main,
1616
readServiceEnvironment,
1717
type BackupSpaceProbe,
18+
serviceStartDiskSpaceBlocker,
1819
readServiceState,
1920
writeServiceState,
2021
} from "./serviceLauncher.ts";
@@ -65,6 +66,11 @@ it("orders exact semantic versions without treating build metadata as precedence
6566
assert.equal(compareExactServiceVersions("2.0.0+one", "2.0.0+two"), 0);
6667
});
6768

69+
it("blocks service startup when free disk space is below the launch headroom", () => {
70+
assert.isUndefined(serviceStartDiskSpaceBlocker(64 * 1024 ** 2));
71+
assert.include(serviceStartDiskSpaceBlocker(0) ?? "", "before starting the server");
72+
});
73+
6874
it("rejects contradictory service state", () => {
6975
assert.isUndefined(
7076
decodeServiceState({

‎apps/server/src/serviceLauncher.ts‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@ const HANDOFF_DELAY_MS = 2_000;
3636
*/
3737
const PREPARED_TIMEOUT_MS = 30 * 60_000;
3838
const TERMINATE_GRACE_MS = 5_000;
39+
// Keep SQLite startup and journaling from turning a full filesystem into a restart loop.
40+
const MIN_SERVICE_START_FREE_BYTES = 64 * 1024 ** 2;
41+
const DISK_SPACE_RETRY_MS = 30_000;
3942
// Match contracts' SERVER_EXIT_CODE_STATE_DIR_OWNED without depending on Effect.
4043
const STATE_DIR_OWNED_EXIT_CODE = 78;
4144

@@ -219,6 +222,11 @@ const filesystemBackupSpaceProbe: BackupSpaceProbe = {
219222
canClone: backupCanClone,
220223
};
221224

225+
export const serviceStartDiskSpaceBlocker = (freeBytes: number): string | undefined =>
226+
freeBytes < MIN_SERVICE_START_FREE_BYTES
227+
? `T3 Code needs at least ${MIN_SERVICE_START_FREE_BYTES} bytes free before starting the server; ${freeBytes} bytes available.`
228+
: undefined;
229+
222230
/** Check before stopping the server so full disks refuse updates without an outage. */
223231
export async function databaseBackupBlocker(
224232
baseDir: string,
@@ -524,6 +532,36 @@ export class Launcher {
524532
this.#timer = undefined;
525533
}
526534

535+
async #serviceStartDiskSpaceBlocker(): Promise<string | undefined> {
536+
try {
537+
return serviceStartDiskSpaceBlocker(
538+
await filesystemBackupSpaceProbe.readFreeBytes(this.#baseDir),
539+
);
540+
} catch {
541+
return undefined;
542+
}
543+
}
544+
545+
#waitForDiskSpace(
546+
reason: string,
547+
version: string,
548+
role: ChildRole,
549+
update?: ServiceUpdateRecord,
550+
): void {
551+
if (this.#timer !== undefined || this.#stopping) return;
552+
process.stderr.write(`[service-launcher] ${reason} Waiting for free space before retrying.\n`);
553+
this.#timer = setInterval(
554+
() =>
555+
this.#enqueue(async () => {
556+
if (this.#child !== null || this.#stopping) return;
557+
if ((await this.#serviceStartDiskSpaceBlocker()) !== undefined) return;
558+
this.#clearTimer();
559+
await this.#startChild(version, role, update);
560+
}),
561+
DISK_SPACE_RETRY_MS,
562+
);
563+
}
564+
527565
async #recover(): Promise<void> {
528566
// Clear old stop markers to preserve future tunnels; clear restart-pending only for the awaited version to avoid CLI races.
529567
await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined);
@@ -567,6 +605,11 @@ export class Launcher {
567605

568606
async #startChild(version: string, role: ChildRole, update?: ServiceUpdateRecord): Promise<void> {
569607
if (this.#stopping) return;
608+
const diskSpaceBlocker = await this.#serviceStartDiskSpaceBlocker();
609+
if (diskSpaceBlocker !== undefined) {
610+
this.#waitForDiskSpace(diskSpaceBlocker, version, role, update);
611+
return;
612+
}
570613
if (!(await runtimeExists(this.#baseDir, version))) {
571614
throw new Error(`Selected t3@${version} runtime is missing or incomplete.`);
572615
}
@@ -749,6 +792,12 @@ export class Launcher {
749792
await this.#idleWhileStateDirOwned();
750793
return;
751794
}
795+
const diskSpaceBlocker = await this.#serviceStartDiskSpaceBlocker();
796+
if (diskSpaceBlocker !== undefined) {
797+
this.#clearTimer();
798+
this.#waitForDiskSpace(diskSpaceBlocker, child.version, child.role, this.#state.update);
799+
return;
800+
}
752801
if (child.role === "trial") {
753802
this.#clearTimer();
754803
const pending = this.#state.update;

0 commit comments

Comments
 (0)