Skip to content

Commit bcb31ba

Browse files
committed
fix(server): harden graceful shutdown drain completion
Keep manual queue holds when completing a scoped drain, clear drains on update failure, and stop idle/worktree turns from starting during drain.
1 parent 8b4cb8f commit bcb31ba

8 files changed

Lines changed: 505 additions & 285 deletions

File tree

‎apps/server/src/cli/shutdown.ts‎

Lines changed: 47 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -154,16 +154,30 @@ const formatResult = (result: GracefulShutdownResult) =>
154154

155155
const stopServerProcess = Effect.fn("cli.shutdown.stopServerProcess")(function* (flags: {
156156
readonly baseDir: Option.Option<string>;
157+
readonly json: boolean;
157158
}) {
158159
const logLevel = yield* GlobalFlag.LogLevel;
159160
const config = yield* resolveCliAuthConfig(flags, logLevel);
160161
const runtimeState = yield* readPersistedServerRuntimeState(config.serverRuntimeStatePath);
161-
if (Option.isNone(runtimeState)) return;
162+
if (Option.isNone(runtimeState)) return null;
162163
const pid = runtimeState.value.pid;
163-
if (typeof pid === "number" && isProcessAlive(pid)) {
164-
process.kill(pid, "SIGTERM");
164+
if (typeof pid !== "number" || !isProcessAlive(pid)) return null;
165+
yield* Effect.try({
166+
try: () => {
167+
process.kill(pid, "SIGTERM");
168+
},
169+
catch: (cause) =>
170+
new ShutdownCliError({
171+
reason:
172+
cause instanceof Error
173+
? `Could not send SIGTERM to server process ${pid}: ${cause.message}`
174+
: `Could not send SIGTERM to server process ${pid}.`,
175+
}),
176+
});
177+
if (!flags.json) {
165178
yield* Console.log(`Sent SIGTERM to server process ${pid}.`);
166179
}
180+
return pid;
167181
});
168182

169183
export const shutdownCommand = Command.make("shutdown", {
@@ -211,6 +225,12 @@ export const shutdownCommand = Command.make("shutdown", {
211225
}
212226

213227
const scope = yield* parseScope(flags);
228+
if (flags.stop && scope.type !== "environment") {
229+
return yield* new ShutdownCliError({
230+
reason: "--stop is only valid for an environment-wide drain.",
231+
});
232+
}
233+
214234
const timeoutMs = Math.max(1_000, flags.timeout * 1000);
215235
const input: GracefulShutdownInput = {
216236
scope,
@@ -238,32 +258,37 @@ export const shutdownCommand = Command.make("shutdown", {
238258
),
239259
);
240260

261+
if (result.timedOut) {
262+
if (flags.json) {
263+
yield* printJson(result);
264+
} else {
265+
yield* Console.log(formatResult(result));
266+
if (result.saveWorkPrompted) {
267+
yield* Console.log(`Save-work turns prompted: ${result.saveWorkTurnCount}.`);
268+
}
269+
yield* Console.log(
270+
"Drain remains active so new turns stay queued. Run `t3 shutdown --complete` to resume, or restart the server.",
271+
);
272+
}
273+
return yield* new ShutdownCliError({
274+
reason: formatResult(result),
275+
});
276+
}
277+
278+
const stoppedPid = flags.stop ? yield* stopServerProcess(flags) : null;
279+
241280
if (flags.json) {
242-
yield* printJson(result);
281+
yield* printJson(stoppedPid === null ? result : { ...result, stoppedPid });
243282
} else {
244283
yield* Console.log(formatResult(result));
245284
if (result.saveWorkPrompted) {
246285
yield* Console.log(`Save-work turns prompted: ${result.saveWorkTurnCount}.`);
247286
}
248-
}
249-
250-
if (flags.stop) {
251-
if (scope.type !== "environment") {
252-
return yield* new ShutdownCliError({
253-
reason: "--stop is only valid for an environment-wide drain.",
254-
});
287+
if (!flags.stop) {
288+
yield* Console.log(
289+
"Drain remains active so new turns stay queued. Run `t3 shutdown --complete` to resume, or restart the server.",
290+
);
255291
}
256-
yield* stopServerProcess(flags);
257-
} else if (!flags.json) {
258-
yield* Console.log(
259-
"Drain remains active so new turns stay queued. Run `t3 shutdown --complete` to resume, or restart the server.",
260-
);
261-
}
262-
263-
if (result.timedOut) {
264-
return yield* new ShutdownCliError({
265-
reason: formatResult(result),
266-
});
267292
}
268293
}),
269294
),

0 commit comments

Comments
 (0)