Skip to content

Commit 5875ce8

Browse files
MarcoDotIOclaude
andcommitted
fix(server): server updates reclaim partial database backups left by killed updates
A launcher killed during the pre-update database backup skips the backup's own cleanup, and a retry only clears the staging directory of its own update id. A partial copy from any earlier update stayed on disk forever and counted against the free-space check, so it could refuse a later update. Remove abandoned `.staging` directories before measuring free space. Requests are refused while an update is pending, so none of them is still being written. Part of #27809. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent c8a601c commit 5875ce8

2 files changed

Lines changed: 43 additions & 0 deletions

File tree

‎apps/server/src/serviceLauncher.test.ts‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
// @effect-diagnostics-next-line nodeBuiltinImport:off - observes the children the launcher spawns with Node built-ins.
22
import type * as NodeChildProcess from "node:child_process";
3+
// @effect-diagnostics-next-line nodeBuiltinImport:off - the free-space probe is a plain Promise callback.
4+
import * as NodeFS from "node:fs";
35

46
import * as NodeServices from "@effect/platform-node/NodeServices";
57
import { assert, it } from "@effect/vitest";
@@ -618,6 +620,35 @@ setInterval(() => {}, 1_000);
618620
}),
619621
);
620622

623+
it.effect("reclaims partial backups left by killed updates before checking space", () =>
624+
Effect.gen(function* () {
625+
const fs = yield* FileSystem.FileSystem;
626+
const path = yield* Path.Path;
627+
const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-orphan-" });
628+
const databasePath = path.join(root, "state.sqlite");
629+
yield* writeSparseFile(databasePath, 10 * 1024 ** 3);
630+
const backupRoot = path.join(root, "runtime", "db-backup");
631+
const orphan = path.join(backupRoot, "earlier-update.staging");
632+
const completed = path.join(backupRoot, "completed-update");
633+
yield* fs.makeDirectory(orphan, { recursive: true });
634+
yield* fs.writeFileString(path.join(orphan, "database"), "partial copy");
635+
yield* fs.makeDirectory(completed);
636+
yield* fs.writeFileString(path.join(completed, "database"), "complete copy");
637+
638+
// The orphan is what keeps this disk too full for a full copy.
639+
const blocker = yield* Effect.promise(() =>
640+
databaseBackupBlocker(root, databasePath, {
641+
readFreeBytes: () => Promise.resolve((NodeFS.existsSync(orphan) ? 8 : 11) * 1024 ** 3),
642+
canClone: () => Promise.resolve(false),
643+
}),
644+
);
645+
646+
assert.isUndefined(blocker);
647+
assert.isFalse(yield* fs.exists(orphan));
648+
assert.isTrue(yield* fs.exists(path.join(completed, "database")));
649+
}),
650+
);
651+
621652
it.effect("checks database backup space against clone or full-copy needs", () =>
622653
Effect.gen(function* () {
623654
const fs = yield* FileSystem.FileSystem;

‎apps/server/src/serviceLauncher.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,17 @@ const filesystemBackupSpaceProbe: BackupSpaceProbe = {
219219
canClone: backupCanClone,
220220
};
221221

222+
// A killed launcher skips the backup's cleanup, and retries only clear their own update's copy.
223+
// Update requests are refused while one is pending, so no staging directory is still in use here.
224+
async function removeAbandonedBackupCopies(backupRoot: string): Promise<void> {
225+
for (const entry of await NodeFSP.readdir(backupRoot)) {
226+
if (!entry.endsWith(".staging")) continue;
227+
await NodeFSP.rm(NodePath.join(backupRoot, entry), { recursive: true, force: true }).catch(
228+
() => undefined,
229+
);
230+
}
231+
}
232+
222233
/** Check before stopping the server so full disks refuse updates without an outage. */
223234
export async function databaseBackupBlocker(
224235
baseDir: string,
@@ -228,6 +239,7 @@ export async function databaseBackupBlocker(
228239
const { readFreeBytes, canClone } = { ...filesystemBackupSpaceProbe, ...probe };
229240
const backupRoot = NodePath.join(baseDir, "runtime", "db-backup");
230241
await NodeFSP.mkdir(backupRoot, { recursive: true, mode: 0o700 });
242+
await removeAbandonedBackupCopies(backupRoot);
231243
const [files, free] = await Promise.all([statDatabaseFiles(dbPath), readFreeBytes(backupRoot)]);
232244
const size = files.reduce((total, file) => total + file.size, 0);
233245
const fullCopyBytes = size * 1.05;

0 commit comments

Comments
 (0)