Skip to content

Commit 224740a

Browse files
t3dotggclaude
andcommitted
feat: Settings reclaims freed database space in one click
Databases created before automatic shrinking keep the space that history cleanup frees, and the only way to get it back was quitting T3 Code and running `t3 doctor` in a terminal. Settings > Maintenance now offers a Reclaim button when the desktop app or background service runs the server. It checks free disk space, stops the server like a SIGTERM, runs the existing repair once the database is closed, and lets the supervisor start it again. Failures show in Settings. Servers started by hand keep the command hint. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 6417151 commit 224740a

16 files changed

Lines changed: 669 additions & 75 deletions

File tree

‎apps/mobile/src/features/settings/components/DatabaseStorageNotice.tsx‎

Lines changed: 87 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,24 @@
11
import { useAtomSet, useAtomValue } from "@effect/atom-react";
2+
import {
3+
isAtomCommandInterrupted,
4+
squashAtomCommandFailure,
5+
} from "@t3tools/client-runtime/state/runtime";
6+
import { databaseReclaimMinutes, type DatabaseStorageStatus } from "@t3tools/contracts";
27
import { AsyncResult } from "effect/reactivity";
8+
import { useState } from "react";
9+
import { Alert } from "react-native";
310

411
import { copyTextWithHaptic } from "../../../lib/copyTextWithHaptic";
512
import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../../state/preferences";
613
import { serverEnvironment } from "../../../state/server";
14+
import { useAtomCommand } from "../../../state/use-atom-command";
715
import type { SettingsTarget } from "../settings-environment-filter";
816
import { ListActionRow as SettingsActionRow } from "../../../components/ui/ListActionRow";
917
import { SettingsSection } from "./SettingsSection";
1018

11-
/** Suggests `t3 doctor` on every machine whose database still holds at least 1 GiB of freed pages. */
19+
const formatGiB = (bytes: number) => `${(bytes / 1024 ** 3).toFixed(1)} GiB`;
20+
21+
/** Offers to reclaim the space on every machine whose database still holds at least 1 GiB of freed pages. */
1222
export function DatabaseStorageNotice({
1323
targets,
1424
}: {
@@ -31,18 +41,16 @@ export function DatabaseStorageNotice({
3141
return (
3242
<SettingsSection
3343
title="Reclaim database space"
34-
footer="These databases still hold space freed by history cleanup. Run the command on each machine to reclaim it. It restarts the background service. Quit the desktop app first if it hosts the environment."
44+
footer={`These databases still hold space freed by history cleanup.${
45+
visible.every((recommendation) => recommendation.status.canReclaim === true)
46+
? ""
47+
: " Run the command on each machine to reclaim it. It restarts the background service. Quit the desktop app first if it hosts the environment."
48+
}`}
3549
>
36-
{visible.map(({ environmentId, status }) => (
37-
<SettingsActionRow
38-
key={environmentId}
39-
icon="doc.on.doc"
40-
label={targets.find((target) => target.environmentId === environmentId)?.label ?? ""}
41-
description={status.compactCommand}
42-
detail={`${(status.reclaimableBytes / 1024 ** 3).toFixed(1)} GiB`}
43-
onPress={() => copyTextWithHaptic(status.compactCommand, { target: "command" })}
44-
/>
45-
))}
50+
{visible.map(({ environmentId, status }) => {
51+
const target = targets.find((candidate) => candidate.environmentId === environmentId);
52+
return target ? <DatabaseRow key={environmentId} target={target} status={status} /> : null;
53+
})}
4654
<SettingsActionRow
4755
icon="xmark"
4856
label="Dismiss"
@@ -62,3 +70,70 @@ export function DatabaseStorageNotice({
6270
</SettingsSection>
6371
);
6472
}
73+
74+
function DatabaseRow({
75+
target,
76+
status,
77+
}: {
78+
readonly target: SettingsTarget;
79+
readonly status: DatabaseStorageStatus;
80+
}) {
81+
const { environmentId, label } = target;
82+
const allowed = useAtomValue(
83+
serverEnvironment.reclaimDatabaseSpace.permissionAtom(environmentId),
84+
);
85+
const reclaim = useAtomCommand(serverEnvironment.reclaimDatabaseSpace, { reportFailure: false });
86+
const [requested, setRequested] = useState(false);
87+
const amount = formatGiB(status.reclaimableBytes);
88+
if (status.canReclaim !== true || !allowed) {
89+
return (
90+
<SettingsActionRow
91+
icon="doc.on.doc"
92+
label={label}
93+
description={status.compactCommand}
94+
detail={amount}
95+
onPress={() => copyTextWithHaptic(status.compactCommand, { target: "command" })}
96+
/>
97+
);
98+
}
99+
// A failed repair reports back after the restart, which ends the wait.
100+
const restarting = requested && status.reclaimFailure === undefined;
101+
const minutes = databaseReclaimMinutes(status);
102+
const start = async () => {
103+
setRequested(true);
104+
const result = await reclaim({ environmentId, input: {} });
105+
if (result._tag === "Success") return;
106+
setRequested(false);
107+
if (isAtomCommandInterrupted(result)) return;
108+
const error = squashAtomCommandFailure(result);
109+
Alert.alert(
110+
`Could not reclaim space on ${label}`,
111+
error instanceof Error ? error.message : "An error occurred.",
112+
);
113+
};
114+
return (
115+
<SettingsActionRow
116+
icon="arrow.clockwise"
117+
label={label}
118+
description={
119+
restarting
120+
? `Restarting. Back in about ${minutes} min.`
121+
: status.reclaimFailure === undefined
122+
? `Restarts T3 Code there. It is offline for about ${minutes} min.`
123+
: `The last attempt failed: ${status.reclaimFailure}`
124+
}
125+
detail={amount}
126+
loading={restarting}
127+
onPress={() =>
128+
Alert.alert(
129+
`Reclaim ${amount} on ${label}?`,
130+
`T3 Code on ${label} stops, rebuilds its database, and starts again. It is offline for about ${minutes} min. Running agents stop when it restarts.`,
131+
[
132+
{ text: "Cancel", style: "cancel" },
133+
{ text: "Restart and reclaim", onPress: () => void start() },
134+
],
135+
)
136+
}
137+
/>
138+
);
139+
}

‎apps/server/src/cli/server.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import * as CliError from "effect/cli/CliError";
88

99
import * as ServerConfig from "../config.ts";
1010
import { awaitLauncherLoss } from "../launcherWatch.ts";
11+
import * as DatabaseMaintenance from "../persistence/DatabaseMaintenance.ts";
1112
import * as ProcessRunner from "../processRunner.ts";
1213
import * as AutomaticPort from "./automaticPort.ts";
1314
import { type CliServerFlags, resolveServerConfig, sharedServerCommandFlags } from "./config.ts";
@@ -34,7 +35,15 @@ const runServerCommand = Effect.fn("server.runServerCommand")(function* (
3435
const logLevel = yield* GlobalFlag.LogLevel;
3536
const selectAutomaticPort = yield* AutomaticPort.make.pipe(Effect.provide(ProcessRunner.layer));
3637
const config = yield* resolveServerConfig(flags, logLevel, { ...options, selectAutomaticPort });
37-
const run = runServer.pipe(Effect.provideService(ServerConfig.ServerConfig, config));
38+
const run = runServer.pipe(
39+
Effect.provideService(ServerConfig.ServerConfig, config),
40+
// Settings' Reclaim stops the server; repair once the database is closed, before the restart.
41+
Effect.ensuring(
42+
DatabaseMaintenance.runRequestedReclaim(config.dbPath).pipe(
43+
Effect.provide(DatabaseMaintenance.layer),
44+
),
45+
),
46+
);
3847
if (config.mode !== "desktop" || config.desktopBootstrapToken === undefined) return yield* run;
3948
yield* Effect.sync(() => {
4049
process.stdout.on("error", ignoreClosedLauncherPipe);

‎apps/server/src/observability/RpcInstrumentation.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ const RPC_AGGREGATES = {
6161
[WS_METHODS.serverRunStorageCleanup]: "server",
6262
[WS_METHODS.serverGetStorageCleanupReport]: "server",
6363
[WS_METHODS.serverGetDatabaseStorage]: "server",
64+
[WS_METHODS.serverReclaimDatabaseSpace]: "server",
6465
[WS_METHODS.peerLinksList]: "peerLinks",
6566
[WS_METHODS.peerLinksLink]: "peerLinks",
6667
[WS_METHODS.peerLinksUnlink]: "peerLinks",

‎apps/server/src/persistence/DatabaseMaintenance.test.ts‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,77 @@ it.layer(DatabaseMaintenance.layer.pipe(Layer.provideMerge(NodeServices.layer)))
101101
}).pipe(Effect.scoped),
102102
);
103103

104+
it.effect("runs a repair requested from Settings once and keeps its failure for Settings", () =>
105+
Effect.gen(function* () {
106+
const fs = yield* FileSystem.FileSystem;
107+
const path = yield* Path.Path;
108+
const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-requested-reclaim-" });
109+
const dbPath = path.join(baseDir, "userdata", "statev2.sqlite");
110+
const files = DatabaseMaintenance.reclaimFiles(path, dbPath);
111+
yield* Effect.void.pipe(Effect.provide(Sqlite.layerFromPath(dbPath)));
112+
const autoVacuum = Effect.flatMap(
113+
SqlClient.SqlClient,
114+
(sql) => sql<{ readonly auto_vacuum: number }>`PRAGMA auto_vacuum`,
115+
).pipe(
116+
Effect.map((rows) => rows[0]?.auto_vacuum),
117+
Effect.provide(NodeSqliteClient.layer({ filename: dbPath, readonly: true })),
118+
);
119+
yield* Effect.gen(function* () {
120+
const sql = yield* SqlClient.SqlClient;
121+
yield* sql`PRAGMA auto_vacuum = NONE`;
122+
yield* sql`VACUUM`;
123+
yield* sql`CREATE TABLE retained_data (value TEXT)`;
124+
yield* sql`INSERT INTO retained_data VALUES ('keep')`;
125+
}).pipe(Effect.provide(NodeSqliteClient.layer({ filename: dbPath })));
126+
127+
// Every server exit runs this; without a request it must leave the database alone.
128+
yield* DatabaseMaintenance.runRequestedReclaim(dbPath);
129+
assert.equal(yield* autoVacuum, 0);
130+
131+
yield* fs.writeFileString(files.request, "");
132+
yield* fs.writeFileString(files.failure, "an earlier failure");
133+
yield* DatabaseMaintenance.runRequestedReclaim(dbPath);
134+
assert.isFalse(yield* fs.exists(files.request));
135+
assert.isFalse(yield* fs.exists(files.failure));
136+
assert.equal(yield* autoVacuum, 2);
137+
yield* Effect.gen(function* () {
138+
const sql = yield* SqlClient.SqlClient;
139+
assert.deepEqual(yield* sql`SELECT value FROM retained_data`, [{ value: "keep" }]);
140+
}).pipe(Effect.provide(NodeSqliteClient.layer({ filename: dbPath, readonly: true })));
141+
142+
const brokenPath = path.join(baseDir, "broken", "statev2.sqlite");
143+
const broken = DatabaseMaintenance.reclaimFiles(path, brokenPath);
144+
yield* fs.makeDirectory(path.dirname(brokenPath));
145+
yield* fs.writeFileString(brokenPath, "not a database");
146+
yield* fs.writeFileString(broken.request, "");
147+
yield* DatabaseMaintenance.runRequestedReclaim(brokenPath);
148+
assert.isFalse(
149+
yield* fs.exists(broken.request),
150+
"a failed repair must not rerun on every exit",
151+
);
152+
assert.equal(
153+
yield* fs.readFileString(broken.failure),
154+
new DatabaseMaintenance.DatabaseCompactError({ path: brokenPath, cause: null }).message,
155+
);
156+
}).pipe(Effect.scoped),
157+
);
158+
159+
it.effect("refuses a repair whose copies would not fit on disk", () =>
160+
Effect.gen(function* () {
161+
const fs = yield* FileSystem.FileSystem;
162+
const path = yield* Path.Path;
163+
const dbPath = path.join(
164+
yield* fs.makeTempDirectoryScoped({ prefix: "t3-reclaim-space-" }),
165+
"statev2.sqlite",
166+
);
167+
assert.isNull(yield* DatabaseMaintenance.reclaimSpaceShortfall(dbPath, 1024));
168+
const huge = 2 ** 60;
169+
const shortfall = yield* DatabaseMaintenance.reclaimSpaceShortfall(dbPath, huge);
170+
assert.isAtLeast(shortfall?.neededBytes ?? 0, huge);
171+
assert.isBelow(shortfall?.freeBytes ?? Infinity, huge);
172+
}).pipe(Effect.scoped),
173+
);
174+
104175
it.effect("compacts an existing database offline only once its server has stopped", () =>
105176
Effect.gen(function* () {
106177
const fs = yield* FileSystem.FileSystem;

‎apps/server/src/persistence/DatabaseMaintenance.ts‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,13 @@ import * as Context from "effect/Context";
44
import * as Effect from "effect/Effect";
55
import * as FileSystem from "effect/FileSystem";
66
import * as Layer from "effect/Layer";
7+
import * as Option from "effect/Option";
78
import * as Path from "effect/Path";
9+
import * as Schedule from "effect/Schedule";
810
import * as Schema from "effect/Schema";
11+
// @effect-diagnostics-next-line nodeBuiltinImport:off -- Effect has no free-space query.
12+
import * as NodeFSP from "node:fs/promises";
13+
import * as NodeOS from "node:os";
914
import * as SqlClient from "effect/sql/SqlClient";
1015
import { isSqlError } from "effect/sql/SqlError";
1116

@@ -57,6 +62,15 @@ export class LegacyDatabaseRemoveError extends Schema.TaggedError<LegacyDatabase
5762
}
5863
}
5964

65+
export class DiskSpaceCheckError extends Schema.TaggedError<DiskSpaceCheckError>()(
66+
"DiskSpaceCheckError",
67+
{ path: Schema.String, cause: Schema.Defect() },
68+
) {
69+
override get message() {
70+
return "Could not check free disk space.";
71+
}
72+
}
73+
6074
export class DatabaseMaintenance extends Context.Service<
6175
DatabaseMaintenance,
6276
{
@@ -230,3 +244,94 @@ const make = Effect.gen(function* () {
230244
});
231245

232246
export const layer = Layer.effect(DatabaseMaintenance, make);
247+
248+
/** Settings' Reclaim writes the request; the failure stays for Settings until the next request. */
249+
export const reclaimFiles = (path: Path.Path, databasePath: string) => ({
250+
request: path.join(path.dirname(databasePath), "reclaim-requested"),
251+
failure: path.join(path.dirname(databasePath), "reclaim-failed"),
252+
});
253+
254+
/** Runs a repair that Settings requested. The server calls this on exit, once its database is closed. */
255+
export const runRequestedReclaim = Effect.fn("DatabaseMaintenance.runRequestedReclaim")(
256+
function* (databasePath: string) {
257+
const fs = yield* FileSystem.FileSystem;
258+
const files = reclaimFiles(yield* Path.Path, databasePath);
259+
if (!(yield* fs.exists(files.request))) return;
260+
// One attempt per request, so a repair killed midway (it rolls back) cannot delay later exits.
261+
yield* fs.remove(files.request, { force: true });
262+
yield* Effect.logInfo("Reclaiming database space before the restart. This can take minutes.");
263+
const maintenance = yield* DatabaseMaintenance;
264+
yield* maintenance.compact(databasePath).pipe(
265+
// Worker threads and agent processes can hold the file for a moment after the server stops.
266+
Effect.retry({
267+
times: 10,
268+
while: (error) => error._tag === "DatabaseInUseError",
269+
schedule: Schedule.spaced("1 second"),
270+
}),
271+
Effect.matchEffect({
272+
onSuccess: (result) =>
273+
fs
274+
.remove(files.failure, { force: true })
275+
.pipe(Effect.andThen(Effect.logInfo("Reclaimed database space", result))),
276+
onFailure: (error) =>
277+
fs
278+
.writeFileString(
279+
files.failure,
280+
error._tag === "DatabaseInUseError"
281+
? "Another program kept the database open. Close other T3 Code apps and t3 commands on this machine, then try again."
282+
: error.message,
283+
)
284+
.pipe(
285+
Effect.andThen(Effect.logWarning("Database space reclamation failed", { error })),
286+
),
287+
}),
288+
);
289+
},
290+
Effect.catchCause((cause) => Effect.logWarning("Database space reclamation failed", { cause })),
291+
);
292+
293+
// SQLite's search order for its temporary files (unixTempFileDir in os_unix.c).
294+
const sqliteTempDirectory = Effect.gen(function* () {
295+
if ((yield* HostProcess.Platform) === "win32") return NodeOS.tmpdir();
296+
const fs = yield* FileSystem.FileSystem;
297+
const env = yield* HostProcess.Environment;
298+
for (const directory of [env.SQLITE_TMPDIR, env.TMPDIR, "/var/tmp", "/usr/tmp", "/tmp"]) {
299+
if (directory === undefined || directory === "") continue;
300+
const info = yield* fs.stat(directory).pipe(Effect.option);
301+
if (Option.isSome(info) && info.value.type === "Directory") return directory;
302+
}
303+
return ".";
304+
});
305+
306+
const freeBytes = (directory: string) =>
307+
Effect.tryPromise({
308+
try: () => NodeFSP.statfs(directory),
309+
catch: (cause) => new DiskSpaceCheckError({ path: directory, cause }),
310+
}).pipe(Effect.map((stats) => stats.bavail * stats.bsize));
311+
312+
/**
313+
* VACUUM copies the live pages to a temporary file, then writes them again to the WAL.
314+
* Returns the free space the repair needs when either file system has less, else null.
315+
*/
316+
export const reclaimSpaceShortfall = Effect.fn("DatabaseMaintenance.reclaimSpaceShortfall")(
317+
function* (databasePath: string, liveBytes: number) {
318+
const fs = yield* FileSystem.FileSystem;
319+
const path = yield* Path.Path;
320+
const databaseDirectory = path.dirname(databasePath);
321+
const tempDirectory = yield* sqliteTempDirectory;
322+
const copyBytes = Math.ceil(liveBytes * 1.1);
323+
const [databaseFree, tempFree, databaseDevice, tempDevice] = yield* Effect.all([
324+
freeBytes(databaseDirectory),
325+
freeBytes(tempDirectory),
326+
fs.stat(databaseDirectory).pipe(Effect.map((info) => info.dev)),
327+
fs.stat(tempDirectory).pipe(Effect.map((info) => info.dev)),
328+
]);
329+
if (databaseDevice === tempDevice) {
330+
return databaseFree < 2 * copyBytes
331+
? { neededBytes: 2 * copyBytes, freeBytes: databaseFree }
332+
: null;
333+
}
334+
if (databaseFree < copyBytes) return { neededBytes: copyBytes, freeBytes: databaseFree };
335+
return tempFree < copyBytes ? { neededBytes: copyBytes, freeBytes: tempFree } : null;
336+
},
337+
);

0 commit comments

Comments
 (0)