Skip to content

Commit bd21af0

Browse files
committed
fix(server): migrated databases can return freed space to disk
1 parent cefb544 commit bd21af0

2 files changed

Lines changed: 32 additions & 10 deletions

File tree

‎apps/server/src/persistence/initializeV2Database.test.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ it.effect("includes committed WAL data and does not publish a failed snapshot",
130130
const source = new NodeSqlite.DatabaseSync(sourcePath);
131131
try {
132132
source.exec(
133-
"PRAGMA journal_mode=WAL; CREATE TABLE messages(text TEXT); INSERT INTO messages VALUES ('committed'); BEGIN; INSERT INTO messages VALUES ('uncommitted');",
133+
"PRAGMA journal_mode=WAL; CREATE TABLE messages(text TEXT); INSERT INTO messages VALUES ('committed'); CREATE TABLE dropped(blob BLOB); INSERT INTO dropped VALUES (randomblob(65536)); DROP TABLE dropped; BEGIN; INSERT INTO messages VALUES ('uncommitted');",
134134
);
135135
yield* initializeV2Database(destinationPath);
136136
const copy = new NodeSqlite.DatabaseSync(destinationPath, { readOnly: true });
@@ -142,6 +142,8 @@ it.effect("includes committed WAL data and does not publish a failed snapshot",
142142
.map((row) => row.text),
143143
["committed"],
144144
);
145+
assert.deepEqual(copy.prepare("PRAGMA auto_vacuum").get(), { auto_vacuum: 2 });
146+
assert.deepEqual(copy.prepare("PRAGMA freelist_count").get(), { freelist_count: 0 });
145147
} finally {
146148
copy.close();
147149
}

‎apps/server/src/persistence/initializeV2Database.ts‎

Lines changed: 29 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import * as NodeSqlite from "node:sqlite";
1+
import * as NodeWorkerThreads from "node:worker_threads";
22

33
import * as Effect from "effect/Effect";
44
import * as FileSystem from "effect/FileSystem";
@@ -24,6 +24,20 @@ const isProcessAlive = (pid: number): boolean => {
2424
}
2525
};
2626

27+
// V1 files predate auto_vacuum, and a page backup would keep that. VACUUM INTO compacts the copy and
28+
// enables incremental mode, so the server can return freed pages; the worker keeps it off the event loop.
29+
const snapshotWorkerSource = `
30+
const { workerData } = require("node:worker_threads");
31+
const { DatabaseSync } = require("node:sqlite");
32+
const database = new DatabaseSync(workerData.sourcePath, { readOnly: true });
33+
try {
34+
database.exec("PRAGMA auto_vacuum = INCREMENTAL");
35+
database.prepare("VACUUM INTO ?").run(workerData.snapshotPath);
36+
} finally {
37+
database.close();
38+
}
39+
`;
40+
2741
// Legacy names have no PID; newer copies stay while their process is alive.
2842
const removeAbandonedCopies = Effect.fn("removeAbandonedCopies")(function* (directory: string) {
2943
const fs = yield* FileSystem.FileSystem;
@@ -58,14 +72,20 @@ export const initializeV2Database = Effect.fn("initializeV2Database")(function*
5872
prefix: `.v2-import-${process.pid}-`,
5973
});
6074
const snapshotPath = path.join(temporaryDirectory, "snapshot.sqlite");
61-
yield* Effect.tryPromise(async () => {
62-
const database = new NodeSqlite.DatabaseSync(sourcePath, { readOnly: true });
63-
try {
64-
await NodeSqlite.backup(database, snapshotPath);
65-
} finally {
66-
database.close();
67-
}
68-
});
75+
yield* Effect.tryPromise(
76+
(signal) =>
77+
new Promise<void>((resolve, reject) => {
78+
const worker = new NodeWorkerThreads.Worker(snapshotWorkerSource, {
79+
eval: true,
80+
workerData: { sourcePath, snapshotPath },
81+
});
82+
signal.addEventListener("abort", () => void worker.terminate(), { once: true });
83+
worker.once("error", reject);
84+
worker.once("exit", (code) =>
85+
code === 0 ? resolve() : reject(new Error(`Snapshot worker exited with ${code}`)),
86+
);
87+
}),
88+
);
6989
// Publish only a complete snapshot, without replacing an existing V2 database.
7090
yield* fs.link(snapshotPath, destinationPath).pipe(
7191
Effect.catchIf(

0 commit comments

Comments
 (0)