Skip to content

Commit d77541f

Browse files
rileyjshawclaude
andcommitted
fix(server): report session exits when a provider instance is torn down
Removing or replacing a provider instance (any `providerInstances` settings edit) closed the instance scope while threads were mid-turn, and the orchestrator never heard about it: those threads stayed "running" forever. - CodexSessionRuntime.close emitted `session/closed` and then immediately `Queue.shutdown` the events queue, which clears buffered messages, so the exit was dropped before the adapter forwarder took it. End the queue instead so the consumer drains it. - CodexAdapter.stopSessionInternal closed the session scope (interrupting the forwarder) right after `close`; wait for the forwarder to drain first. Both adapters now end their runtime event queue on release instead of shutting it down, so already-emitted exits stay readable. - ProviderInstanceRegistry.reconcile now calls `adapter.stopAll()` before closing a removed/replaced instance's scope. Adapter finalizers stop sessions silently on purpose (server shutdown keeps threads resumable); a settings edit is not a shutdown. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 67b175a commit d77541f

8 files changed

Lines changed: 180 additions & 15 deletions

‎apps/server/src/provider/Layers/ClaudeAdapter.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2133,7 +2133,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* (
21332133
}) as ClaudeQueryRuntime);
21342134

21352135
const sessions = new Map<ThreadId, ClaudeSessionContext>();
2136-
const runtimeEventQueue = yield* Queue.unbounded<ProviderRuntimeEvent>();
2136+
const runtimeEventQueue = yield* Queue.unbounded<ProviderRuntimeEvent, Cause.Done>();
21372137

21382138
const nowIso = Effect.map(DateTime.now, DateTime.formatIso);
21392139
const randomUUIDv4 = crypto.randomUUIDv4.pipe(
@@ -5611,7 +5611,8 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* (
56115611
Effect.catch((cause) =>
56125612
Effect.logError("Failed to emit Claude session shutdown event.", { cause }),
56135613
),
5614-
Effect.tap(() => Queue.shutdown(runtimeEventQueue)),
5614+
// Keep already-emitted exit events readable for the provider service.
5615+
Effect.tap(() => Queue.end(runtimeEventQueue)),
56155616
Effect.tap(() => managedNativeEventLogger?.close() ?? Effect.void),
56165617
),
56175618
);

‎apps/server/src/provider/Layers/CodexAdapter.test.ts‎

Lines changed: 56 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import { createModelSelection } from "@t3tools/shared/model";
2222
import * as NodeServices from "@effect/platform-node/NodeServices";
2323
import { it, vi } from "@effect/vitest";
2424

25+
import * as Cause from "effect/Cause";
2526
import * as Context from "effect/Context";
2627
import * as Effect from "effect/Effect";
2728
import * as Exit from "effect/Exit";
@@ -60,7 +61,7 @@ const asEventId = (value: string): EventId => EventId.make(value);
6061
const asItemId = (value: string): ProviderItemId => ProviderItemId.make(value);
6162

6263
class FakeCodexRuntime implements CodexSessionRuntimeShape {
63-
private readonly eventQueue = Effect.runSync(Queue.unbounded<ProviderEvent>());
64+
private readonly eventQueue = Effect.runSync(Queue.unbounded<ProviderEvent, Cause.Done>());
6465
private readonly now = "2026-01-01T00:00:00.000Z";
6566

6667
public readonly startImpl = vi.fn(() =>
@@ -119,6 +120,8 @@ class FakeCodexRuntime implements CodexSessionRuntimeShape {
119120
);
120121

121122
public readonly closeImpl = vi.fn(() => Promise.resolve(undefined));
123+
/** Mirror the real runtime: `close` emits `session/closed` before ending `events`. */
124+
public closeEmitsSessionClosed = false;
122125

123126
readonly options: CodexSessionRuntimeOptions;
124127

@@ -162,7 +165,26 @@ class FakeCodexRuntime implements CodexSessionRuntimeShape {
162165
return Stream.fromQueue(this.eventQueue);
163166
}
164167

165-
close = Effect.promise(() => this.closeImpl());
168+
get close() {
169+
return Effect.promise(() => this.closeImpl()).pipe(
170+
Effect.andThen(() =>
171+
this.closeEmitsSessionClosed
172+
? this.emit({
173+
id: asEventId(`evt-${this.options.threadId}-closed`),
174+
kind: "session",
175+
provider: ProviderDriverKind.make("codex"),
176+
threadId: this.options.threadId,
177+
createdAt: this.now,
178+
method: "session/closed",
179+
message: "Session stopped",
180+
})
181+
: Effect.void,
182+
),
183+
// The real runtime ends (not shuts down) its event queue on close.
184+
Effect.andThen(() => Queue.end(this.eventQueue)),
185+
Effect.asVoid,
186+
);
187+
}
166188

167189
emit(event: ProviderEvent) {
168190
return Queue.offer(this.eventQueue, event).pipe(Effect.asVoid);
@@ -1792,6 +1814,38 @@ lifecycleLayer("CodexAdapterLive lifecycle", (it) => {
17921814
}),
17931815
);
17941816

1817+
it.effect("delivers the runtime's session/closed emitted during stopSession", () =>
1818+
Effect.gen(function* () {
1819+
const adapter = yield* CodexAdapter;
1820+
const threadId = asThreadId("thread-stop-drain");
1821+
yield* adapter.startSession({
1822+
provider: ProviderDriverKind.make("codex"),
1823+
threadId,
1824+
runtimeMode: "full-access",
1825+
});
1826+
const runtime = lifecycleRuntimeFactory.lastRuntime;
1827+
NodeAssert.ok(runtime);
1828+
runtime.closeEmitsSessionClosed = true;
1829+
const exitedFiber = yield* adapter.streamEvents.pipe(
1830+
Stream.filter((event) => event.type === "session.exited" && event.threadId === threadId),
1831+
Stream.runHead,
1832+
Effect.forkChild,
1833+
);
1834+
1835+
// Tearing the session down (settings reconcile, stopAll) must not lose
1836+
// the exit the runtime emits on close, or the thread stays "running".
1837+
yield* adapter.stopSession(threadId);
1838+
1839+
const exited = yield* Fiber.join(exitedFiber);
1840+
NodeAssert.equal(exited._tag, "Some");
1841+
if (exited._tag !== "Some" || exited.value.type !== "session.exited") {
1842+
return;
1843+
}
1844+
NodeAssert.equal(exited.value.payload.exitKind, "graceful");
1845+
NodeAssert.equal(yield* adapter.hasSession(threadId), false);
1846+
}),
1847+
);
1848+
17951849
it.effect("maps retryable Codex error notifications to runtime.warning", () =>
17961850
Effect.gen(function* () {
17971851
const { adapter, runtime } = yield* startLifecycleRuntime();

‎apps/server/src/provider/Layers/CodexAdapter.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import {
3232
ThreadId,
3333
ProviderSendTurnInput,
3434
} from "@t3tools/contracts";
35+
import * as Cause from "effect/Cause";
3536
import * as Effect from "effect/Effect";
3637
import * as NodeCrypto from "node:crypto";
3738
import * as Crypto from "effect/Crypto";
@@ -2259,7 +2260,7 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* (
22592260
: undefined);
22602261
const managedNativeEventLogger =
22612262
options?.nativeEventLogger === undefined ? nativeEventLogger : undefined;
2262-
const runtimeEventQueue = yield* Queue.unbounded<ProviderRuntimeEvent>();
2263+
const runtimeEventQueue = yield* Queue.unbounded<ProviderRuntimeEvent, Cause.Done>();
22632264
const sessions = new Map<ThreadId, CodexAdapterSessionContext>();
22642265

22652266
const startSession: CodexAdapterShape["startSession"] = (input) =>
@@ -2762,6 +2763,10 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* (
27622763
session.stopped = true;
27632764
sessions.delete(session.threadId);
27642765
yield* session.runtime.close.pipe(Effect.ignore);
2766+
// `close` ends the runtime event stream after emitting `session/closed`.
2767+
// Let the forwarder drain it before its scope is closed, so the exit
2768+
// reaches the orchestrator and the thread does not stay "running".
2769+
yield* Fiber.join(session.eventFiber).pipe(Effect.timeoutOption("5 seconds"), Effect.ignore);
27652770
yield* Effect.ignore(Scope.close(session.scope, Exit.void));
27662771
yield* Fiber.interrupt(session.eventFiber).pipe(Effect.ignore);
27672772
});
@@ -2793,7 +2798,8 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* (
27932798

27942799
yield* Effect.acquireRelease(Effect.void, () =>
27952800
stopAll().pipe(
2796-
Effect.andThen(Queue.shutdown(runtimeEventQueue)),
2801+
// Keep already-emitted exit events readable for the provider service.
2802+
Effect.andThen(Queue.end(runtimeEventQueue)),
27972803
Effect.andThen(managedNativeEventLogger?.close() ?? Effect.void),
27982804
Effect.ignore,
27992805
),

‎apps/server/src/provider/Layers/CodexCollabRuntime.integration.test.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -867,6 +867,35 @@ describe("CodexSessionRuntime collab integration", () => {
867867
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
868868
);
869869
}
870+
871+
it.effect("keeps the session/closed event readable after close", () =>
872+
Effect.gen(function* () {
873+
// @effect-diagnostics-next-line preferSchemaOverJson:off
874+
NodeFS.writeFileSync(scriptPath, JSON.stringify({ rootThreadId: ROOT, notifications: [] }));
875+
yield* Effect.addFinalizer(() =>
876+
Effect.sync(() => NodeFS.rmSync(scriptPath, { force: true })),
877+
);
878+
879+
const runtime = yield* makeCodexSessionRuntime({
880+
threadId: ThreadId.make("thread-close-drain"),
881+
binaryPath: peerPath,
882+
cwd: NodeOS.tmpdir(),
883+
runtimeMode: "full-access",
884+
environment: { ...process.env, T3_CODEX_COLLAB_SCRIPT: scriptPath },
885+
});
886+
yield* runtime.start();
887+
888+
// Nothing is consuming `events` yet: the same window the adapter's
889+
// forwarder can be in when a session is torn down. Ending the queue
890+
// (instead of shutting it down) keeps the buffered exit readable.
891+
yield* runtime.close;
892+
893+
const sessionEvents = Array.from(yield* Stream.runCollect(runtime.events))
894+
.filter((event) => event.kind === "session")
895+
.map((event) => event.method);
896+
assert.deepEqual(sessionEvents, ["session/connecting", "session/ready", "session/closed"]);
897+
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
898+
);
870899
});
871900

872901
describe("CodexSessionRuntime compaction", () => {

‎apps/server/src/provider/Layers/CodexSessionRuntime.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import {
2020
} from "@t3tools/contracts";
2121
import { resolveSpawnCommand } from "@t3tools/shared/shell";
2222
import { normalizeModelSlug } from "@t3tools/shared/model";
23+
import * as Cause from "effect/Cause";
2324
import * as Crypto from "effect/Crypto";
2425
import * as DateTime from "effect/DateTime";
2526
import * as Deferred from "effect/Deferred";
@@ -1307,7 +1308,9 @@ export const makeCodexSessionRuntime = (
13071308
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
13081309
const runtimeScope = yield* Scope.Scope;
13091310
const crypto = yield* Crypto.Crypto;
1310-
const events = yield* Queue.unbounded<ProviderEvent>();
1311+
// `Done`-terminated so `close` can end the queue gracefully: the final
1312+
// `session/closed` event stays readable instead of being cleared.
1313+
const events = yield* Queue.unbounded<ProviderEvent, Cause.Done>();
13111314
const pendingApprovalsRef = yield* Ref.make(new Map<ApprovalRequestId, PendingApproval>());
13121315
const approvalCorrelationsRef = yield* Ref.make(new Map<string, ApprovalCorrelation>());
13131316
const pendingUserInputsRef = yield* Ref.make(new Map<ApprovalRequestId, PendingUserInput>());
@@ -2537,7 +2540,10 @@ export const makeCodexSessionRuntime = (
25372540
);
25382541
yield* Scope.close(runtimeScope, Exit.void);
25392542
yield* Queue.shutdown(serverNotifications);
2540-
yield* Queue.shutdown(events);
2543+
// `Queue.shutdown` clears buffered messages, which dropped the
2544+
// `session/closed` emitted just above whenever the consumer had not
2545+
// taken it yet. `end` lets the consumer drain, then completes.
2546+
yield* Queue.end(events);
25412547
});
25422548

25432549
return {

‎apps/server/src/provider/Layers/ProviderInstanceRegistryLive.test.ts‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ import * as Effect from "effect/Effect";
4444
import * as FileSystem from "effect/FileSystem";
4545
import * as Layer from "effect/Layer";
4646
import * as Path from "effect/Path";
47+
import * as Schema from "effect/Schema";
4748
import * as Stream from "effect/Stream";
4849
import { HttpClient, HttpClientResponse } from "effect/unstable/http";
4950

@@ -60,6 +61,7 @@ import { GrokDriver } from "../Drivers/GrokDriver.ts";
6061
import { OpenCodeDriver } from "../Drivers/OpenCodeDriver.ts";
6162
import * as ModelManifest from "../ModelManifest.ts";
6263
import { OpenCodeRuntimeLive } from "../opencodeRuntime.ts";
64+
import type { ProviderDriver, ProviderInstance } from "../ProviderDriver.ts";
6365
import * as ResetCreditCoordinator from "./resetCreditCoordinator.ts";
6466
import { NoOpProviderEventLoggers, ProviderEventLoggers } from "./ProviderEventLoggers.ts";
6567
import { makeProviderInstanceRegistry } from "./ProviderInstanceRegistryLive.ts";
@@ -799,3 +801,57 @@ describe("ProviderInstanceRegistryLive — all drivers slice", () => {
799801
}).pipe(Effect.provide(testLayer)),
800802
);
801803
});
804+
805+
describe("ProviderInstanceRegistryLive — reconcile teardown", () => {
806+
it.effect("stops live sessions before closing a removed instance's scope", () =>
807+
Effect.gen(function* () {
808+
const driverKind = ProviderDriverKind.make("codex");
809+
const instanceId = ProviderInstanceId.make("codex_removed");
810+
const calls: Array<string> = [];
811+
812+
const fakeDriver: ProviderDriver<unknown> = {
813+
driverKind,
814+
metadata: { displayName: "Fake" },
815+
configSchema: Schema.Unknown,
816+
defaultConfig: () => ({}),
817+
create: (input) =>
818+
Effect.gen(function* () {
819+
yield* Effect.addFinalizer(() => Effect.sync(() => void calls.push("scope-closed")));
820+
const instance: ProviderInstance = {
821+
instanceId: input.instanceId,
822+
driverKind,
823+
continuationIdentity: { driverKind, continuationKey: input.instanceId },
824+
displayName: input.displayName,
825+
enabled: true,
826+
get snapshot(): never {
827+
throw new Error("reconcile must not read the provider snapshot.");
828+
},
829+
// Only `stopAll` is reachable from `reconcile`.
830+
adapter: {
831+
stopAll: () => Effect.sync(() => void calls.push("stopAll")),
832+
} as unknown as ProviderInstance["adapter"],
833+
get textGeneration(): never {
834+
throw new Error("reconcile must not generate text.");
835+
},
836+
};
837+
return instance;
838+
}),
839+
};
840+
841+
const { registry, mutator } = yield* makeProviderInstanceRegistry({
842+
drivers: [fakeDriver],
843+
configMap: { [instanceId]: { driver: driverKind, config: {} } },
844+
});
845+
expect((yield* registry.listInstances).map((instance) => instance.instanceId)).toEqual([
846+
instanceId,
847+
]);
848+
849+
// Removing the instance (a settings edit) must report the exit of every
850+
// live session before the adapter is torn down with its scope.
851+
yield* mutator.reconcile({});
852+
853+
expect(calls).toEqual(["stopAll", "scope-closed"]);
854+
expect(yield* registry.listInstances).toEqual([]);
855+
}).pipe(Effect.scoped),
856+
);
857+
});

‎apps/server/src/provider/Layers/ProviderInstanceRegistryLive.ts‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -248,9 +248,21 @@ const makeReconcile = <R>(input: {
248248
}
249249
for (const id of [...removedIds, ...replacedIds]) {
250250
const live = previousEntries.get(id);
251-
if (live) {
252-
yield* Scope.close(live.scope, Exit.void).pipe(Effect.ignore);
253-
}
251+
if (!live) continue;
252+
// Stop sessions explicitly before the scope teardown. Adapter
253+
// finalizers stop sessions silently (server shutdown keeps threads
254+
// resumable); a settings edit is not a shutdown, and threads with
255+
// an active turn must learn their provider is gone or they stay
256+
// "running" forever.
257+
yield* live.instance.adapter.stopAll().pipe(
258+
Effect.catchCause((cause) =>
259+
Effect.logWarning("Failed to stop provider instance sessions before teardown", {
260+
instanceId: id,
261+
cause,
262+
}),
263+
),
264+
);
265+
yield* Scope.close(live.scope, Exit.void).pipe(Effect.ignore);
254266
}
255267

256268
// 2. Build additions and replacements. Walk `nextRaw` so the final

‎apps/server/src/provider/Services/ProviderInstanceRegistryMutator.ts‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,11 @@
1212
* The mutator exposes a single entry point, `reconcile(configMap)`, which:
1313
*
1414
* 1. Diffs the incoming map against the live one keyed by instance id.
15-
* 2. Closes the per-instance `Scope` of every removed or replaced entry
16-
* (tearing down adapter processes, refresh fibres, temp files) BEFORE
17-
* creating the replacement — `reconcile` guarantees "at most one live
18-
* instance per id" at all times.
15+
* 2. Stops every live session of each removed or replaced entry (so the
16+
* orchestrator sees `session.exited` for threads mid-turn), then closes
17+
* its per-instance `Scope` (tearing down adapter processes, refresh
18+
* fibres, temp files) BEFORE creating the replacement — `reconcile`
19+
* guarantees "at most one live instance per id" at all times.
1920
* 3. Opens a fresh child `Scope` for every added or replaced entry, runs
2021
* the driver's `create`, and stores the resulting `ProviderInstance`
2122
* plus its scope.

0 commit comments

Comments
 (0)