Skip to content

Commit e2f4f90

Browse files
committed
fix(server): harden drain admission and provider-update prechecks
Make drain start atomic, propagate quiescence failures, keep queue order except for save-work, and only drain provider updates that will install.
1 parent bcb31ba commit e2f4f90

6 files changed

Lines changed: 219 additions & 119 deletions

File tree

‎apps/server/src/orchestration-v2/GracefulShutdown.ts‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -238,13 +238,6 @@ const make = Effect.fn("GracefulShutdown.make")(function* () {
238238

239239
const beginAndAwait = Effect.fn("GracefulShutdown.beginAndAwait")(
240240
function* (input: GracefulShutdownInput) {
241-
const existing = yield* gate.getState;
242-
if (GracefulShutdownGate.isDrainActive(existing)) {
243-
return yield* new GracefulShutdownError({
244-
reason: "A graceful shutdown is already in progress.",
245-
});
246-
}
247-
248241
const sequence = yield* Ref.getAndUpdate(shutdownSequence, (value) => value + 1);
249242
const started = yield* DateTime.now;
250243
const shutdownId =
@@ -275,7 +268,7 @@ const make = Effect.fn("GracefulShutdown.make")(function* () {
275268
}
276269
}
277270

278-
yield* gate.setState({
271+
const admitted = yield* gate.tryBegin({
279272
phase: "draining",
280273
shutdownId,
281274
scope: input.scope,
@@ -286,6 +279,11 @@ const make = Effect.fn("GracefulShutdown.make")(function* () {
286279
saveWorkMessageIds,
287280
saveWorkThreadIds,
288281
});
282+
if (!admitted) {
283+
return yield* new GracefulShutdownError({
284+
reason: "A graceful shutdown is already in progress.",
285+
});
286+
}
289287

290288
const drainBody = Effect.gen(function* () {
291289
if (shouldPrompt) {
@@ -329,7 +327,7 @@ const make = Effect.fn("GracefulShutdown.make")(function* () {
329327
}
330328
});
331329

332-
const outcome = yield* Effect.race(
330+
const outcome = yield* Effect.raceFirst(
333331
waitQuiesced,
334332
Effect.sleep(Duration.millis(timeoutMs)).pipe(Effect.as("timed_out" as const)),
335333
);

‎apps/server/src/orchestration-v2/GracefulShutdownGate.test.ts‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,4 +120,35 @@ it.layer(layer)("GracefulShutdownGate", (it) => {
120120
);
121121
}),
122122
);
123+
124+
it.effect("tryBegin admits only one concurrent drain", () =>
125+
Effect.gen(function* () {
126+
const gate = yield* GracefulShutdownGate.GracefulShutdownGate;
127+
yield* gate.clear;
128+
const first = {
129+
phase: "draining" as const,
130+
shutdownId: "shutdown-first",
131+
scope: { type: "environment" as const },
132+
reason: "first",
133+
startedAt: "2026-01-01T00:00:00.000Z",
134+
deadlineAt: "2026-01-01T00:02:00.000Z",
135+
saveWorkPrompted: false,
136+
saveWorkMessageIds: new Set<MessageId>(),
137+
saveWorkThreadIds: new Set<ThreadId>(),
138+
};
139+
const second = {
140+
...first,
141+
shutdownId: "shutdown-second",
142+
reason: "second",
143+
};
144+
145+
assert.isTrue(yield* gate.tryBegin(first));
146+
assert.isFalse(yield* gate.tryBegin(second));
147+
assert.strictEqual((yield* gate.getState).shutdownId, "shutdown-first");
148+
149+
yield* gate.clear;
150+
assert.isTrue(yield* gate.tryBegin(second));
151+
assert.strictEqual((yield* gate.getState).shutdownId, "shutdown-second");
152+
}),
153+
);
123154
});

‎apps/server/src/orchestration-v2/GracefulShutdownGate.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ export class GracefulShutdownGate extends Context.Service<
4646
{
4747
readonly getState: Effect.Effect<GracefulShutdownGateState>;
4848
readonly setState: (state: GracefulShutdownGateState) => Effect.Effect<void>;
49+
/** Atomically install drain state when idle; false if a drain is already active. */
50+
readonly tryBegin: (state: GracefulShutdownGateState) => Effect.Effect<boolean>;
4951
readonly clear: Effect.Effect<void>;
5052
readonly isSaveWorkMessage: (messageId: MessageId) => Effect.Effect<boolean>;
5153
/** True when new turns for this thread must queue (steers still allowed). */
@@ -64,6 +66,11 @@ const make = Effect.fn("GracefulShutdownGate.make")(function* () {
6466

6567
const getState = Ref.get(stateRef);
6668
const setState = (state: GracefulShutdownGateState) => Ref.set(stateRef, state);
69+
const tryBegin = (state: GracefulShutdownGateState) =>
70+
Ref.modify(stateRef, (current) => {
71+
if (isDrainActive(current)) return [false, current] as const;
72+
return [true, state] as const;
73+
});
6774
const clear = Ref.set(stateRef, IDLE_STATE);
6875

6976
const isSaveWorkMessage = Effect.fn("GracefulShutdownGate.isSaveWorkMessage")(function* (
@@ -114,6 +121,7 @@ const make = Effect.fn("GracefulShutdownGate.make")(function* () {
114121
return GracefulShutdownGate.of({
115122
getState,
116123
setState,
124+
tryBegin,
117125
clear,
118126
isSaveWorkMessage,
119127
shouldQueueNewTurns,

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 46 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -823,12 +823,6 @@ function worktreeSetupPending(
823823
);
824824
}
825825

826-
function nextQueuedRun(
827-
projection: Pick<OrchestrationV2ThreadProjection, "runs" | "messages">,
828-
): OrchestrationV2Run | undefined {
829-
return queuedRunsInDeliveryOrder(projection).find((run) => run.queueHeld !== true);
830-
}
831-
832826
function followsCurrentConversation(
833827
message: OrchestrationV2ConversationMessage | undefined,
834828
): boolean {
@@ -1598,19 +1592,60 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
15981592
},
15991593
);
16001594

1595+
// Drain may skip blocked candidates; only save-work may bypass an earlier blocked run.
1596+
const selectStartableQueuedRun = Effect.fnUntraced(function* (
1597+
threadId: ThreadId,
1598+
projection: Pick<
1599+
OrchestrationV2ThreadProjection,
1600+
"thread" | "messages" | "providerSessions" | "runs"
1601+
>,
1602+
queuedCandidates: ReadonlyArray<OrchestrationV2Run>,
1603+
) {
1604+
let blockedCandidate = false;
1605+
for (const candidate of queuedCandidates) {
1606+
const candidateSelection = queuedRunModelSelection(projection, candidate);
1607+
if (
1608+
yield* gracefulShutdownGate.shouldQueueNewTurns({
1609+
threadId,
1610+
providerInstanceId: candidateSelection.instanceId,
1611+
providerSessionIds: projection.providerSessions.map((session) => session.id),
1612+
messageId: candidate.userMessageId,
1613+
})
1614+
) {
1615+
blockedCandidate = true;
1616+
continue;
1617+
}
1618+
if (
1619+
blockedCandidate &&
1620+
!(yield* gracefulShutdownGate.isSaveWorkMessage(candidate.userMessageId))
1621+
) {
1622+
continue;
1623+
}
1624+
return {
1625+
storedQueuedRun: candidate,
1626+
modelSelection: candidateSelection,
1627+
} as const;
1628+
}
1629+
return undefined;
1630+
});
1631+
16011632
const failQueuedRunStart = Effect.fn("Orchestrator.failQueuedRunStart")(function* (
16021633
threadId: ThreadId,
16031634
cause: unknown,
16041635
queuedRunId?: RunId,
16051636
) {
16061637
const projection = yield* projectionStore.getThreadRecords(
16071638
threadId,
1608-
["runs", "nodes", "attempts", "providerThreads", "turnItems", "messages"],
1639+
["runs", "nodes", "attempts", "providerThreads", "providerSessions", "turnItems", "messages"],
16091640
{ turnItemTypes: [], messageRoles: ["user"] },
16101641
);
16111642
const queuedRun =
16121643
queuedRunId === undefined
1613-
? nextQueuedRun(projection)
1644+
? (yield* selectStartableQueuedRun(
1645+
threadId,
1646+
projection,
1647+
queuedRunsInDeliveryOrder(projection).filter((run) => run.queueHeld !== true),
1648+
))?.storedQueuedRun
16141649
: projection.runs.find((run) => run.id === queuedRunId && run.status === "queued");
16151650
if (queuedRun === undefined) return;
16161651
const now = yield* DateTime.now;
@@ -1738,27 +1773,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
17381773
selectedQueuedRun === undefined
17391774
? queuedRunsInDeliveryOrder(projection).filter((run) => run.queueHeld !== true)
17401775
: [selectedQueuedRun];
1741-
let storedQueuedRun: (typeof queuedCandidates)[number] | undefined;
1742-
let modelSelection: ReturnType<typeof queuedRunModelSelection> | undefined;
1743-
for (const candidate of queuedCandidates) {
1744-
const candidateSelection = queuedRunModelSelection(projection, candidate);
1745-
if (
1746-
yield* gracefulShutdownGate.shouldQueueNewTurns({
1747-
threadId,
1748-
providerInstanceId: candidateSelection.instanceId,
1749-
providerSessionIds: projection.providerSessions.map((session) => session.id),
1750-
messageId: candidate.userMessageId,
1751-
})
1752-
) {
1753-
continue;
1754-
}
1755-
storedQueuedRun = candidate;
1756-
modelSelection = candidateSelection;
1757-
break;
1758-
}
1759-
if (storedQueuedRun === undefined || modelSelection === undefined) {
1776+
const selected = yield* selectStartableQueuedRun(threadId, projection, queuedCandidates);
1777+
if (selected === undefined) {
17601778
return;
17611779
}
1780+
const { storedQueuedRun, modelSelection } = selected;
17621781
const queuedMessage = projection.messages.find(
17631782
(candidate) => candidate.id === storedQueuedRun.userMessageId,
17641783
);

‎apps/server/src/provider/providerMaintenanceRunner.test.ts‎

Lines changed: 51 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,7 @@ const makeTestRunner = (
240240
})),
241241
},
242242
latestVersions: Layer.Layer<ProviderLatestVersions.ProviderLatestVersions> = ProviderLatestVersions.layer,
243+
drainBegins?: Ref.Ref<number>,
243244
) =>
244245
Effect.service(ProviderMaintenanceRunner.ProviderMaintenanceRunner).pipe(
245246
Effect.provide(
@@ -264,16 +265,21 @@ const makeTestRunner = (
264265
saveWorkTurnCount: 0,
265266
}),
266267
beginAndAwait: () =>
267-
Effect.succeed({
268-
shutdownId: GracefulShutdownId.make("test-shutdown"),
269-
phase: "quiesced",
270-
scope: { type: "environment" },
271-
timedOut: false,
272-
activeRunCount: 0,
273-
saveWorkPrompted: false,
274-
saveWorkTurnCount: 0,
275-
elapsedMs: 0,
276-
}),
268+
(drainBegins === undefined
269+
? Effect.void
270+
: Ref.update(drainBegins, (count) => count + 1)
271+
).pipe(
272+
Effect.as({
273+
shutdownId: GracefulShutdownId.make("test-shutdown"),
274+
phase: "quiesced" as const,
275+
scope: { type: "environment" as const },
276+
timedOut: false,
277+
activeRunCount: 0,
278+
saveWorkPrompted: false,
279+
saveWorkTurnCount: 0,
280+
elapsedMs: 0,
281+
}),
282+
),
277283
complete: () => Effect.void,
278284
}),
279285
),
@@ -318,6 +324,7 @@ describe("providerMaintenanceRunner", () => {
318324
[
319325
["queued", "Waiting for another provider update to finish."],
320326
["running", "Checking for the latest version"],
327+
["running", "Waiting for agents to finish before updating"],
321328
["running", "Running native-agent update"],
322329
["running", "Verifying the installed version"],
323330
["succeeded", "Provider updated."],
@@ -353,6 +360,7 @@ describe("providerMaintenanceRunner", () => {
353360
const calls: Array<{ command: string; args: ReadonlyArray<string> }> = [];
354361
const refreshedInstanceIds: Array<ProviderInstanceId> = [];
355362
return Effect.gen(function* () {
363+
const drainBegins = yield* Ref.make(0);
356364
const { registry, updateStatesRef, providersRef } = yield* makeRegistry({
357365
...baseProvider,
358366
version: "2.1.309",
@@ -366,38 +374,44 @@ describe("providerMaintenanceRunner", () => {
366374
message: "Update available.",
367375
},
368376
});
369-
const updater = yield* makeTestRunner({
370-
...registry,
371-
refreshInstance: (instanceId) => {
372-
refreshedInstanceIds.push(instanceId);
373-
return Ref.updateAndGet(providersRef, (providers) =>
374-
providers.map((provider) =>
375-
provider.instanceId === instanceId
376-
? {
377-
...provider,
378-
version: "2.1.310",
379-
...probe,
380-
checkedAt: "2026-04-10T00:00:01.000Z",
381-
}
382-
: provider,
377+
const updater = yield* makeTestRunner(
378+
{
379+
...registry,
380+
refreshInstance: (instanceId) => {
381+
refreshedInstanceIds.push(instanceId);
382+
return Ref.updateAndGet(providersRef, (providers) =>
383+
providers.map((provider) =>
384+
provider.instanceId === instanceId
385+
? {
386+
...provider,
387+
version: "2.1.310",
388+
...probe,
389+
checkedAt: "2026-04-10T00:00:01.000Z",
390+
}
391+
: provider,
392+
),
393+
);
394+
},
395+
getProviderMaintenanceCapabilitiesForInstance: () =>
396+
Effect.succeed(
397+
makeProviderMaintenanceCapabilities({
398+
provider: CODEX_DRIVER,
399+
packageName: "@anthropic-ai/claude-code",
400+
updateExecutable: "claude",
401+
updateArgs: ["update"],
402+
updateLockKey: "claudeAgent-native",
403+
}),
383404
),
384-
);
385405
},
386-
getProviderMaintenanceCapabilitiesForInstance: () =>
387-
Effect.succeed(
388-
makeProviderMaintenanceCapabilities({
389-
provider: CODEX_DRIVER,
390-
packageName: "@anthropic-ai/claude-code",
391-
updateExecutable: "claude",
392-
updateArgs: ["update"],
393-
updateLockKey: "claudeAgent-native",
394-
}),
395-
),
396-
});
406+
undefined,
407+
undefined,
408+
drainBegins,
409+
);
397410

398411
const result = yield* updater.updateProvider(CODEX_DRIVER);
399412
assert.deepStrictEqual(refreshedInstanceIds, [CODEX_INSTANCE_ID]);
400413
assert.deepStrictEqual(calls, skip ? [] : [{ command: "claude", args: ["update"] }]);
414+
assert.strictEqual(yield* Ref.get(drainBegins), skip ? 0 : 1);
401415
if (!skip) return;
402416
assert.strictEqual(result.providers[0]?.updateState?.status, "succeeded");
403417
assert.strictEqual(result.providers[0]?.updateState?.message, "Already up to date.");
@@ -604,7 +618,7 @@ describe("providerMaintenanceRunner", () => {
604618
});
605619

606620
yield* updater.updateProvider(CODEX_DRIVER);
607-
assert.deepStrictEqual(fresh, [false, true, true]);
621+
assert.deepStrictEqual(fresh, [false, true, true, true]);
608622
assert.deepStrictEqual(calls, [
609623
{ command: "/opt/homebrew/bin/brew", args: ["upgrade", "--cask", "codex"] },
610624
]);

0 commit comments

Comments
 (0)