Skip to content

Commit 2adda57

Browse files
fix(server): t3 doctor holds off server launches and names the holder (#24963)
Co-authored-by: Julius Marminge <[email protected]>
1 parent 5f43681 commit 2adda57

4 files changed

Lines changed: 140 additions & 31 deletions

File tree

‎apps/server/src/cli/doctor.test.ts‎

Lines changed: 85 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,59 @@
1+
import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
12
import * as NodeServices from "@effect/platform-node/NodeServices";
3+
import * as HostProcess from "@t3tools/shared/HostProcess";
24
import { assert, it } from "@effect/vitest";
35
import * as Console from "effect/Console";
46
import * as Effect from "effect/Effect";
7+
import * as FileSystem from "effect/FileSystem";
58
import * as Layer from "effect/Layer";
9+
import * as Path from "effect/Path";
610
import * as TestClock from "effect/testing/TestClock";
711

812
import * as BootService from "../cloud/bootService.ts";
913
import { runDoctor } from "./doctor.ts";
1014
import { DatabaseMaintenance } from "../persistence/DatabaseMaintenance.ts";
15+
import * as ProcessRunner from "../processRunner.ts";
16+
import { acquireServerOwnership } from "../serverOwnership.ts";
17+
import { makePersistedServerRuntimeState } from "../serverRuntimeState.ts";
18+
19+
// An empty home, so no boot service unit installed on this machine is read.
20+
const layerHost = Layer.mergeAll(
21+
ProcessRunner.layer,
22+
Layer.effect(
23+
HostProcess.HomeDirectory,
24+
Effect.gen(function* () {
25+
return yield* (yield* FileSystem.FileSystem).makeTempDirectoryScoped();
26+
}),
27+
),
28+
Layer.succeed(HostProcess.Environment, {}),
29+
).pipe(Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)));
1130

1231
it.effect(
13-
"--fix reclaims space with the service paused and keeps the V1 database, and --dry-run changes nothing",
32+
"--fix locks the home while the service is paused, names a server that holds it, keeps the V1 database, and --dry-run changes nothing",
1433
() =>
1534
Effect.gen(function* () {
35+
const fs = yield* FileSystem.FileSystem;
36+
const path = yield* Path.Path;
37+
const stateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-doctor-" });
38+
const serverRuntimeStatePath = path.join(stateDir, "server-runtime.json");
1639
const calls: string[] = [];
40+
const services =
41+
yield* Effect.context<Effect.Services<ReturnType<typeof acquireServerOwnership>>>();
42+
// Stands in for a server launched for the same home, for example over SSH.
43+
const startServer = Effect.scoped(acquireServerOwnership(serverRuntimeStatePath)).pipe(
44+
Effect.result,
45+
Effect.map((result) =>
46+
calls.push(result._tag === "Success" ? "server started" : result.failure._tag),
47+
),
48+
Effect.provide(services),
49+
);
1750
const doctor = (dryRun: boolean) =>
18-
runDoctor("/test/statev2.sqlite", { fix: true, dryRun, interactive: false }).pipe(
51+
runDoctor(
52+
{ dbPath: path.join(stateDir, "statev2.sqlite"), serverRuntimeStatePath },
53+
{ fix: true, dryRun, interactive: false },
54+
).pipe(
1955
Effect.provide(
2056
Layer.mergeAll(
21-
NodeServices.layer,
2257
Layer.mock(BootService.BootService)({
2358
withStopped: (operation) =>
2459
Effect.sync(() => calls.push("pause")).pipe(
@@ -35,15 +70,17 @@ it.effect(
3570
legacyDatabaseBytes: 4000,
3671
}),
3772
compact: () =>
38-
Effect.sync(() => {
39-
calls.push("compact");
40-
return {
41-
beforeBytes: 2000,
42-
afterBytes: 1000,
43-
deletedEventCount: 1,
44-
deletedReceiptCount: 0,
45-
};
46-
}),
73+
startServer.pipe(
74+
Effect.map(() => {
75+
calls.push("compact");
76+
return {
77+
beforeBytes: 2000,
78+
afterBytes: 1000,
79+
deletedEventCount: 1,
80+
deletedReceiptCount: 0,
81+
};
82+
}),
83+
),
4784
removeLegacyDatabase: () => Effect.sync(() => calls.push("remove legacy")),
4885
}),
4986
),
@@ -52,8 +89,32 @@ it.effect(
5289
yield* doctor(true);
5390
assert.deepEqual(calls, []);
5491
yield* doctor(false);
55-
assert.deepEqual(calls, ["pause", "compact", "resume"]);
56-
}),
92+
yield* startServer;
93+
assert.deepEqual(calls, [
94+
"pause",
95+
"ServerAlreadyRunningError",
96+
"compact",
97+
"resume",
98+
"server started",
99+
]);
100+
101+
calls.length = 0;
102+
const state = yield* makePersistedServerRuntimeState({
103+
config: { host: undefined, devUrl: undefined },
104+
port: 3773,
105+
});
106+
const refused = yield* Effect.scoped(
107+
acquireServerOwnership(serverRuntimeStatePath).pipe(
108+
Effect.tap((owner) => owner.publish(state)),
109+
Effect.andThen(doctor(false)),
110+
),
111+
).pipe(Effect.flip);
112+
assert.equal(
113+
refused.message,
114+
`Stop T3 Code server process ${state.pid}, which the background service did not start, before repairing the database.`,
115+
);
116+
assert.deepEqual(calls, ["pause", "resume"]);
117+
}).pipe(Effect.provide(layerHost)),
57118
);
58119

59120
it.effect("warns how long reclaiming can take and reports each step with its elapsed time", () =>
@@ -65,10 +126,17 @@ it.effect("warns how long reclaiming can take and reports each step with its ela
65126
["rebuild-search", "1 second"],
66127
["checkpoint", "1 second"],
67128
] as const;
68-
yield* runDoctor("/test/statev2.sqlite", { fix: true, dryRun: false, interactive: false }).pipe(
129+
const stateDir = yield* (yield* FileSystem.FileSystem).makeTempDirectoryScoped();
130+
const path = yield* Path.Path;
131+
yield* runDoctor(
132+
{
133+
dbPath: path.join(stateDir, "statev2.sqlite"),
134+
serverRuntimeStatePath: path.join(stateDir, "server-runtime.json"),
135+
},
136+
{ fix: true, dryRun: false, interactive: false },
137+
).pipe(
69138
Effect.provide(
70139
Layer.mergeAll(
71-
NodeServices.layer,
72140
Layer.mock(BootService.BootService)({
73141
withStopped: (operation) =>
74142
Effect.sync(() => output.push("<pause>")).pipe(
@@ -125,5 +193,5 @@ it.effect("warns how long reclaiming can take and reports each step with its ela
125193
["0:00", "1:05", "12:46", "12:47"],
126194
"each compaction step must print one line with the elapsed time as it starts",
127195
);
128-
}),
196+
}).pipe(Effect.provide(layerHost)),
129197
);

‎apps/server/src/cli/doctor.ts‎

Lines changed: 41 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,13 @@ import * as Clock from "effect/Clock";
33
import * as Console from "effect/Console";
44
import * as Effect from "effect/Effect";
55
import * as Layer from "effect/Layer";
6+
import * as Option from "effect/Option";
67
import { Command, Flag, GlobalFlag, Prompt } from "effect/cli";
78

89
import * as BootService from "../cloud/bootService.ts";
910
import * as DatabaseMaintenance from "../persistence/DatabaseMaintenance.ts";
11+
import * as ServerOwnership from "../serverOwnership.ts";
12+
import { isProcessAlive, readPersistedServerRuntimeState } from "../serverRuntimeState.ts";
1013
import { projectLocationFlags, resolveCliAuthConfig } from "./config.ts";
1114
import * as CliService from "./service.ts";
1215

@@ -29,10 +32,37 @@ const COMPACT_STEP_LABELS: Record<DatabaseMaintenance.DatabaseCompactStep, strin
2932
checkpoint: "Flushing the write-ahead log",
3033
};
3134

35+
// Holding the server lock refuses a server launched mid-repair, such as one started over SSH.
36+
// Lock only: acquireServerOwnership would also refuse the older service doctor just stopped.
37+
const lockServerHome = Effect.fn("runDoctor.lockServerHome")(function* (paths: {
38+
readonly dbPath: string;
39+
readonly serverRuntimeStatePath: string;
40+
}) {
41+
yield* ServerOwnership.holdServerOwnershipLock(paths.serverRuntimeStatePath).pipe(
42+
Effect.catchTags({
43+
ServerAlreadyRunningError: Effect.fn("runDoctor.nameServerHolder")(function* (cause) {
44+
const owner = yield* readPersistedServerRuntimeState(paths.serverRuntimeStatePath).pipe(
45+
Effect.map(Option.filter((state) => isProcessAlive(state.pid))),
46+
Effect.orElseSucceed(() => Option.none()),
47+
);
48+
return yield* new DatabaseMaintenance.DatabaseInUseError({
49+
path: paths.dbPath,
50+
cause,
51+
...(Option.isSome(owner) && {
52+
pid: owner.value.pid,
53+
serviceManaged: owner.value.serviceManaged === true,
54+
}),
55+
});
56+
}),
57+
}),
58+
);
59+
});
60+
3261
export const runDoctor = Effect.fn("runDoctor")(function* (
33-
databasePath: string,
62+
paths: { readonly dbPath: string; readonly serverRuntimeStatePath: string },
3463
options: { readonly fix: boolean; readonly dryRun: boolean; readonly interactive?: boolean },
3564
) {
65+
const databasePath = paths.dbPath;
3666
const maintenance = yield* DatabaseMaintenance.DatabaseMaintenance;
3767
const status = yield* maintenance.inspect(databasePath);
3868
yield* Console.log(
@@ -123,19 +153,19 @@ export const runDoctor = Effect.fn("runDoctor")(function* (
123153
);
124154
// Report the repair before restarting, which can fail independently.
125155
yield* service.withStopped(
126-
maintenance
127-
.compact(databasePath, reportStep)
128-
.pipe(
129-
Effect.tap((result) =>
130-
Clock.monotonicTimeNanos.pipe(
131-
Effect.flatMap((now) =>
132-
Console.log(
133-
`Done in ${formatElapsed(now - startedAt)}: ${formatBytes(result.beforeBytes)} -> ${formatBytes(result.afterBytes)}, ${result.deletedEventCount} superseded events removed.`,
134-
),
156+
lockServerHome(paths).pipe(
157+
Effect.andThen(maintenance.compact(databasePath, reportStep)),
158+
Effect.tap((result) =>
159+
Clock.monotonicTimeNanos.pipe(
160+
Effect.flatMap((now) =>
161+
Console.log(
162+
`Done in ${formatElapsed(now - startedAt)}: ${formatBytes(result.beforeBytes)} -> ${formatBytes(result.afterBytes)}, ${result.deletedEventCount} superseded events removed.`,
135163
),
136164
),
137165
),
138166
),
167+
Effect.scoped,
168+
),
139169
);
140170
}
141171
if (selected.includes("legacy")) {
@@ -163,7 +193,7 @@ export const doctorCommand = Command.make("doctor", {
163193
Effect.gen(function* () {
164194
const logLevel = yield* GlobalFlag.LogLevel;
165195
const config = yield* resolveCliAuthConfig(flags, logLevel);
166-
yield* runDoctor(config.dbPath, { fix: flags.fix, dryRun: flags.dryRun }).pipe(
196+
yield* runDoctor(config, { fix: flags.fix, dryRun: flags.dryRun }).pipe(
167197
Effect.provide(Layer.merge(DatabaseMaintenance.layer, CliService.layer(config))),
168198
);
169199
}),

‎apps/server/src/persistence/DatabaseMaintenance.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,17 @@ export class DatabaseMissingError extends Schema.TaggedError<DatabaseMissingErro
2828

2929
export class DatabaseInUseError extends Schema.TaggedError<DatabaseInUseError>()(
3030
"DatabaseInUseError",
31-
{ path: Schema.String, cause: Schema.Defect() },
31+
{
32+
path: Schema.String,
33+
cause: Schema.Defect(),
34+
pid: Schema.optional(Schema.Int),
35+
serviceManaged: Schema.optional(Schema.Boolean),
36+
},
3237
) {
3338
override get message() {
39+
if (this.pid !== undefined) {
40+
return `Stop T3 Code server process ${this.pid}${this.serviceManaged ? "" : ", which the background service did not start,"} before repairing the database.`;
41+
}
3442
return "Stop the T3 Code server and desktop app, including any version before 0.1.0, before repairing the database.";
3543
}
3644
}

‎docs/user/background-service.md‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -182,8 +182,11 @@ pass `--fix` to run the recommended ones without a prompt (for scripts), or
182182
`--dry-run` to only report. An installed background service for the same data
183183
directory is stopped for the repair and started again afterwards, also when
184184
the repair fails. Quit a server you started by hand, or the desktop app, first:
185-
the repair refuses to touch a database that is in use. Use `--base-dir` for a
186-
different data directory.
185+
the repair refuses to touch a database that is in use, and names the server
186+
process that holds it when it can tell. While the repair runs, a server started
187+
for the same data directory, for example by a desktop app connecting over SSH,
188+
is refused until the repair ends. Use `--base-dir` for a different data
189+
directory.
187190

188191
The repair's time grows with the size of the whole database, not only the
189192
space it reclaims, and can exceed an hour on a large database or a slow disk;

0 commit comments

Comments
 (0)