Skip to content

Commit 8b4cb8f

Browse files
committed
feat(server): gracefully drain environments before shutdown
- Add scoped graceful shutdown controls through the CLI and RPC - Queue new turns during drains and prompt active agents to save work - Drain active runs before server updates
1 parent 6577c6b commit 8b4cb8f

29 files changed

Lines changed: 1340 additions & 7 deletions

‎apps/server/src/auth/RpcAuthorization.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,9 @@ export const RPC_REQUIRED_SCOPES = {
8383
[WS_METHODS.serverUpdateServer]: AuthEnvironmentMaintainScope,
8484
[WS_METHODS.serverUpdateServerWithProgress]: AuthEnvironmentMaintainScope,
8585
[WS_METHODS.serverCommitDesktopUpdate]: AuthEnvironmentMaintainScope,
86+
[WS_METHODS.serverGracefulShutdown]: AuthEnvironmentMaintainScope,
87+
[WS_METHODS.serverGracefulShutdownStatus]: AuthEnvironmentMaintainScope,
88+
[WS_METHODS.serverGracefulShutdownComplete]: AuthEnvironmentMaintainScope,
8689
[WS_METHODS.serverUpsertKeybinding]: AuthSettingsWriteScope,
8790
[WS_METHODS.serverRemoveKeybinding]: AuthSettingsWriteScope,
8891
[WS_METHODS.serverGetStorageCleanupReport]: AuthOrchestrationReadScope,

‎apps/server/src/binCli.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import { projectCommand } from "./cli/project.ts";
2323
import { threadCommand } from "./cli/thread.ts";
2424
import { runDefaultServerCommand, serveCommand, startCommand } from "./cli/server.ts";
2525
import { updateCommand } from "./cli/update.ts";
26+
import { shutdownCommand } from "./cli/shutdown.ts";
2627
import { uninstallCommand } from "./cli/uninstall.ts";
2728
import { runServiceLauncherCli } from "./cli/serviceLauncher.ts";
2829
import { speechWorkerCommand } from "./cli/speechWorker.ts";
@@ -89,6 +90,7 @@ export const makeCli = ({ cloudEnabled = hasCloudPublicConfig } = {}) =>
8990
statusCommand,
9091
serviceCommand,
9192
updateCommand,
93+
shutdownCommand,
9294
uninstallCommand,
9395
// bin.ts runs the bare service invocation; this keeps flags and help working like other commands.
9496
Command.make("__service-launcher").pipe(

‎apps/server/src/cli/shutdown.ts‎

Lines changed: 270 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,270 @@
1+
import {
2+
AuthEnvironmentMaintainScope,
3+
DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT_MS,
4+
EnvironmentHttpApi,
5+
type GracefulShutdownInput,
6+
type GracefulShutdownResult,
7+
type GracefulShutdownScope,
8+
ProviderInstanceId,
9+
ProviderSessionId,
10+
} from "@t3tools/contracts";
11+
import * as Console from "effect/Console";
12+
import * as Duration from "effect/Duration";
13+
import * as Effect from "effect/Effect";
14+
import * as Layer from "effect/Layer";
15+
import * as Option from "effect/Option";
16+
import * as References from "effect/References";
17+
import * as Schema from "effect/Schema";
18+
import { Command, Flag, GlobalFlag } from "effect/cli";
19+
import { FetchHttpClient } from "effect/http";
20+
import * as HttpApiClient from "effect/http-api/HttpApiClient";
21+
22+
const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown));
23+
const printJson = (value: unknown) => Console.log(encodeJson(value));
24+
25+
import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts";
26+
import * as ServerConfig from "../config.ts";
27+
import { isProcessAlive, readPersistedServerRuntimeState } from "../serverRuntimeState.ts";
28+
import { projectLocationFlags, resolveCliAuthConfig } from "./config.ts";
29+
30+
export class ShutdownServerNotRunningError extends Schema.TaggedError<ShutdownServerNotRunningError>()(
31+
"ShutdownServerNotRunningError",
32+
{ baseDir: Schema.String },
33+
) {
34+
override get message(): string {
35+
return `No running T3 Code server uses ${this.baseDir}. Start T3 Code, then retry.`;
36+
}
37+
}
38+
39+
export class ShutdownCliError extends Schema.TaggedError<ShutdownCliError>()("ShutdownCliError", {
40+
reason: Schema.String,
41+
}) {
42+
override get message(): string {
43+
return this.reason;
44+
}
45+
}
46+
47+
const timeoutSecondsFlag = Flag.Int("timeout").pipe(
48+
Flag.withSchema(Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 30 * 60 }))),
49+
Flag.withDescription(
50+
"Seconds to wait for in-scope agents to finish after the save prompt (default 120).",
51+
),
52+
Flag.withDefault(Math.floor(DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT_MS / 1000)),
53+
);
54+
55+
const providerFlag = Flag.String("provider").pipe(
56+
Flag.withDescription("Drain only sessions for this provider instance id."),
57+
Flag.optional,
58+
);
59+
60+
const sessionFlag = Flag.String("session").pipe(
61+
Flag.withDescription("Drain only this provider session id."),
62+
Flag.optional,
63+
);
64+
65+
const noSaveFlag = Flag.Boolean("no-save").pipe(
66+
Flag.withDescription("Skip the save-work prompt even when the environment setting is on."),
67+
Flag.withDefault(false),
68+
);
69+
70+
const completeFlag = Flag.Boolean("complete").pipe(
71+
Flag.withDescription("Clear an active drain and resume queued turns instead of starting one."),
72+
Flag.withDefault(false),
73+
);
74+
75+
const stopFlag = Flag.Boolean("stop").pipe(
76+
Flag.withDescription("After a successful drain, send SIGTERM to the server process."),
77+
Flag.withDefault(false),
78+
);
79+
80+
const jsonFlag = Flag.Boolean("json").pipe(
81+
Flag.withDescription("Emit JSON instead of human-readable output."),
82+
Flag.withDefault(false),
83+
);
84+
85+
const parseScope = (input: {
86+
readonly provider: Option.Option<string>;
87+
readonly session: Option.Option<string>;
88+
}): Effect.Effect<GracefulShutdownScope, ShutdownCliError> => {
89+
if (Option.isSome(input.provider) && Option.isSome(input.session)) {
90+
return Effect.fail(
91+
new ShutdownCliError({
92+
reason: "Pass either --provider or --session, not both.",
93+
}),
94+
);
95+
}
96+
if (Option.isSome(input.session)) {
97+
return Effect.succeed({
98+
type: "session" as const,
99+
providerSessionId: ProviderSessionId.make(input.session.value),
100+
});
101+
}
102+
if (Option.isSome(input.provider)) {
103+
return Effect.succeed({
104+
type: "provider" as const,
105+
providerInstanceId: ProviderInstanceId.make(input.provider.value),
106+
});
107+
}
108+
return Effect.succeed({ type: "environment" as const });
109+
};
110+
111+
const makeLiveServerClient = (origin: string) =>
112+
HttpApiClient.make(EnvironmentHttpApi, { baseUrl: origin });
113+
114+
const withMaintainSession = Effect.fn("cli.shutdown.withMaintainSession")(function* <A, E, R>(
115+
flags: { readonly baseDir: Option.Option<string> },
116+
run: (input: { readonly origin: string; readonly token: string }) => Effect.Effect<A, E, R>,
117+
) {
118+
const logLevel = yield* GlobalFlag.LogLevel;
119+
const config = yield* resolveCliAuthConfig(flags, logLevel);
120+
return yield* Effect.gen(function* () {
121+
const runtimeState = yield* readPersistedServerRuntimeState(config.serverRuntimeStatePath);
122+
if (Option.isNone(runtimeState)) {
123+
return yield* new ShutdownServerNotRunningError({ baseDir: config.baseDir });
124+
}
125+
const origin = runtimeState.value.origin;
126+
const environmentAuth = yield* EnvironmentAuth.EnvironmentAuth;
127+
return yield* Effect.acquireUseRelease(
128+
environmentAuth.issueSession({
129+
scopes: [AuthEnvironmentMaintainScope],
130+
label: "t3 shutdown cli",
131+
}),
132+
(issued) => run({ origin, token: issued.token }),
133+
(issued) =>
134+
environmentAuth.revokeSession(issued.sessionId).pipe(Effect.ignore({ log: true })),
135+
);
136+
}).pipe(
137+
Effect.provide(
138+
Layer.mergeAll(
139+
FetchHttpClient.layer,
140+
EnvironmentAuth.layerRuntime.pipe(
141+
Layer.provide(ServerConfig.layer(config)),
142+
Layer.provide(FetchHttpClient.layer),
143+
Layer.provide(Layer.succeed(References.MinimumLogLevel, config.logLevel)),
144+
),
145+
),
146+
),
147+
);
148+
});
149+
150+
const formatResult = (result: GracefulShutdownResult) =>
151+
result.timedOut
152+
? `Graceful shutdown timed out after ${result.elapsedMs}ms with ${result.activeRunCount} active run(s) still in scope (${result.shutdownId}).`
153+
: `Graceful shutdown quiesced in ${result.elapsedMs}ms (${result.shutdownId}).`;
154+
155+
const stopServerProcess = Effect.fn("cli.shutdown.stopServerProcess")(function* (flags: {
156+
readonly baseDir: Option.Option<string>;
157+
}) {
158+
const logLevel = yield* GlobalFlag.LogLevel;
159+
const config = yield* resolveCliAuthConfig(flags, logLevel);
160+
const runtimeState = yield* readPersistedServerRuntimeState(config.serverRuntimeStatePath);
161+
if (Option.isNone(runtimeState)) return;
162+
const pid = runtimeState.value.pid;
163+
if (typeof pid === "number" && isProcessAlive(pid)) {
164+
process.kill(pid, "SIGTERM");
165+
yield* Console.log(`Sent SIGTERM to server process ${pid}.`);
166+
}
167+
});
168+
169+
export const shutdownCommand = Command.make("shutdown", {
170+
...projectLocationFlags,
171+
timeout: timeoutSecondsFlag,
172+
provider: providerFlag,
173+
session: sessionFlag,
174+
noSave: noSaveFlag,
175+
complete: completeFlag,
176+
stop: stopFlag,
177+
json: jsonFlag,
178+
}).pipe(
179+
Command.withDescription(
180+
"Gracefully drain the environment, a provider, or a provider session so updates and other tools can take over.",
181+
),
182+
Command.withHandler((flags) =>
183+
Effect.gen(function* () {
184+
if (flags.complete) {
185+
yield* withMaintainSession(flags, ({ origin, token }) =>
186+
Effect.gen(function* () {
187+
const client = yield* makeLiveServerClient(origin);
188+
yield* client.maintain.gracefulShutdownComplete({
189+
headers: { authorization: `Bearer ${token}` },
190+
payload: {},
191+
});
192+
}).pipe(
193+
Effect.timeout(Duration.seconds(30)),
194+
Effect.mapError(
195+
(cause) =>
196+
new ShutdownCliError({
197+
reason:
198+
cause instanceof Error
199+
? cause.message
200+
: "Could not complete the graceful shutdown.",
201+
}),
202+
),
203+
),
204+
);
205+
if (flags.json) {
206+
yield* printJson({ ok: true, completed: true });
207+
} else {
208+
yield* Console.log("Cleared graceful shutdown drain; queued turns may start again.");
209+
}
210+
return;
211+
}
212+
213+
const scope = yield* parseScope(flags);
214+
const timeoutMs = Math.max(1_000, flags.timeout * 1000);
215+
const input: GracefulShutdownInput = {
216+
scope,
217+
timeoutMs,
218+
...(flags.noSave ? { promptSaveWork: false } : {}),
219+
reason: "cli-shutdown",
220+
};
221+
222+
const result = yield* withMaintainSession(flags, ({ origin, token }) =>
223+
Effect.gen(function* () {
224+
const client = yield* makeLiveServerClient(origin);
225+
return yield* client.maintain.gracefulShutdown({
226+
headers: { authorization: `Bearer ${token}` },
227+
payload: input,
228+
});
229+
}).pipe(
230+
Effect.timeout(Duration.millis(timeoutMs + 30_000)),
231+
Effect.mapError(
232+
(cause) =>
233+
new ShutdownCliError({
234+
reason:
235+
cause instanceof Error ? cause.message : "Graceful shutdown request failed.",
236+
}),
237+
),
238+
),
239+
);
240+
241+
if (flags.json) {
242+
yield* printJson(result);
243+
} else {
244+
yield* Console.log(formatResult(result));
245+
if (result.saveWorkPrompted) {
246+
yield* Console.log(`Save-work turns prompted: ${result.saveWorkTurnCount}.`);
247+
}
248+
}
249+
250+
if (flags.stop) {
251+
if (scope.type !== "environment") {
252+
return yield* new ShutdownCliError({
253+
reason: "--stop is only valid for an environment-wide drain.",
254+
});
255+
}
256+
yield* stopServerProcess(flags);
257+
} else if (!flags.json) {
258+
yield* Console.log(
259+
"Drain remains active so new turns stay queued. Run `t3 shutdown --complete` to resume, or restart the server.",
260+
);
261+
}
262+
263+
if (result.timedOut) {
264+
return yield* new ShutdownCliError({
265+
reason: formatResult(result),
266+
});
267+
}
268+
}),
269+
),
270+
);

‎apps/server/src/cloud/selfUpdate.test.ts‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import * as NodeServices from "@effect/platform-node/NodeServices";
22
import { expect, it } from "@effect/vitest";
3-
import type { ManagedPolicyDocument } from "@t3tools/contracts";
3+
import { GracefulShutdownId, type ManagedPolicyDocument } from "@t3tools/contracts";
44
import * as HostProcess from "@t3tools/shared/HostProcess";
55
import * as Deferred from "effect/Deferred";
66
import * as Duration from "effect/Duration";
@@ -17,12 +17,37 @@ import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
1717
import packageJson from "../../package.json" with { type: "json" };
1818
import * as ServerConfig from "../config.ts";
1919
import * as DesktopAppUpdate from "../desktopUpdate/DesktopAppUpdate.ts";
20+
import * as GracefulShutdown from "../orchestration-v2/GracefulShutdown.ts";
2021
import * as ManagedPolicy from "../policy/ManagedPolicy.ts";
2122
import * as ProcessRunner from "../processRunner.ts";
2223
import * as ServiceLauncherClient from "./serviceLauncherClient.ts";
2324
import { SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts";
2425
import * as ServerSelfUpdate from "./selfUpdate.ts";
2526

27+
const layerGracefulShutdownTest = Layer.succeed(
28+
GracefulShutdown.GracefulShutdown,
29+
GracefulShutdown.GracefulShutdown.of({
30+
status: Effect.succeed({
31+
phase: "idle",
32+
activeRunCount: 0,
33+
saveWorkPrompted: false,
34+
saveWorkTurnCount: 0,
35+
}),
36+
beginAndAwait: () =>
37+
Effect.succeed({
38+
shutdownId: GracefulShutdownId.make("test-shutdown"),
39+
phase: "quiesced",
40+
scope: { type: "environment" },
41+
timedOut: false,
42+
activeRunCount: 0,
43+
saveWorkPrompted: false,
44+
saveWorkTurnCount: 0,
45+
elapsedMs: 0,
46+
}),
47+
complete: () => Effect.void,
48+
}),
49+
);
50+
2651
interface HarnessOptions {
2752
readonly mode?: "web" | "desktop";
2853
readonly managed?: boolean;
@@ -150,7 +175,8 @@ const makeHarness = Effect.fn("test.make_self_update_harness")(function* (
150175
Effect.provideService(HostProcess.Platform, "linux"),
151176
Effect.provideService(HostProcess.Architecture, "x64"),
152177
Effect.provide(
153-
Layer.merge(
178+
Layer.mergeAll(
179+
layerGracefulShutdownTest,
154180
ServerConfig.layer({ ...config, mode: options.mode ?? "web" }),
155181
ManagedPolicy.layerTest(options.policy),
156182
),
@@ -252,7 +278,7 @@ it.layer(NodeServices.layer)("server self update", (it) => {
252278
Effect.sync(() => void stages.push(stage)),
253279
);
254280
expect(result).toEqual({ targetVersion: "1.2.0", method: "desktop-app" });
255-
expect(stages).toEqual(["downloading", "installing"]);
281+
expect(stages).toEqual(["draining", "downloading", "installing"]);
256282
expect(order).toEqual([]);
257283
}),
258284
);

0 commit comments

Comments
 (0)