Skip to content

Commit d87cea0

Browse files
fix(server): release Codex sessions after transport termination
1 parent 13aa882 commit d87cea0

4 files changed

Lines changed: 302 additions & 27 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1791,7 +1791,25 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
17911791
model: input.modelSelection.model,
17921792
now,
17931793
});
1794-
const events = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>();
1794+
const events = yield* Queue.unbounded<
1795+
ProviderAdapter.ProviderAdapterV2Event,
1796+
ProviderAdapter.ProviderAdapterEventStreamError
1797+
>();
1798+
// Drain decoded events before reporting transport death. The manager
1799+
// owns eviction and cleanup; never replay a possibly accepted turn.
1800+
yield* client.termination.pipe(
1801+
Effect.catch((cause) =>
1802+
Queue.fail(
1803+
events,
1804+
new ProviderAdapter.ProviderAdapterEventStreamError({
1805+
driver: CODEX_PROVIDER,
1806+
providerSessionId: input.providerSessionId,
1807+
cause,
1808+
}),
1809+
),
1810+
),
1811+
Effect.forkScoped,
1812+
);
17951813
const rateLimitSnapshot = yield* Ref.make<CodexRateLimitSnapshot | undefined>(undefined);
17961814
const limitedTurnItems = yield* Ref.make(
17971815
new Map<ProviderThreadId, Extract<OrchestrationV2TurnItem, { type: "error" }>>(),
@@ -6405,7 +6423,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
64056423
driver: CODEX_PROVIDER,
64066424
providerSessionId: input.providerSessionId,
64076425
providerSession: session,
6408-
events: Stream.fromEffectRepeat(Queue.take(events)),
6426+
events: Stream.fromQueue(events),
64096427
canReuseContextUsage: canReuseCodexContextUsage,
64106428
// Known gap: a subagent that Codex resumes later reads as completed
64116429
// (not pending) between turns, so idle release can win the race

‎apps/server/src/orchestration-v2/ProviderSessionManager.test.ts‎

Lines changed: 255 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices";
33
import { assert, it } from "@effect/vitest";
44
import { AgentScope } from "@t3tools/shared/AgentScope";
55
import {
6+
CodexSettings,
67
EnvironmentId,
78
type ModelSelection,
89
type OrchestrationV2AppThread,
@@ -17,6 +18,15 @@ import {
1718
type ProviderSessionId,
1819
ThreadId,
1920
} from "@t3tools/contracts";
21+
import * as CodexClient from "effect-codex-app-server/client";
22+
import * as CodexError from "effect-codex-app-server/errors";
23+
import * as Crypto from "effect/Crypto";
24+
import * as Schema from "effect/Schema";
25+
import * as Path from "effect/Path";
26+
import * as Sink from "effect/Sink";
27+
import * as Stdio from "effect/Stdio";
28+
import * as CodexAdapter from "./Adapters/CodexAdapterV2.ts";
29+
import { makeReplayServerConfig } from "./Adapters/CodexAdapterV2.testkit.ts";
2030
import * as Cause from "effect/Cause";
2131
import * as DateTime from "effect/DateTime";
2232
import * as Deferred from "effect/Deferred";
@@ -160,6 +170,8 @@ const layerBeforeUnchangedWriteEventSink = (before: Ref.Ref<Effect.Effect<void>>
160170
}),
161171
).pipe(Layer.provide(layerTestEventSink));
162172

173+
const decodeCodexSettings = Schema.decodeEffect(CodexSettings);
174+
163175
const CodexCapabilities: OrchestrationV2ProviderCapabilities = CodexProviderCapabilitiesV2;
164176
const ExclusiveCapabilities: OrchestrationV2ProviderCapabilities = {
165177
...CodexCapabilities,
@@ -458,6 +470,15 @@ const makeProviderAdapter = Effect.fnUntraced(function* (
458470
});
459471

460472
function layerTest(input: {
473+
readonly adapters?: Effect.Effect<
474+
ReadonlyArray<ProviderAdapter.ProviderAdapterV2["Service"]>,
475+
never,
476+
| McpProviderSessions.McpProviderSessions
477+
| FileSystem.FileSystem
478+
| Path.Path
479+
| Crypto.Crypto
480+
| IdAllocator.IdAllocatorV2
481+
>;
461482
readonly state: Ref.Ref<TestProviderRuntimeState>;
462483
readonly idleTimeoutMs: number;
463484
readonly maxIdlePinMs?: number;
@@ -500,30 +521,33 @@ function layerTest(input: {
500521
? layerFailingReleaseEventSink
501522
: layerTestEventSink;
502523
const layerRegistry = Layer.unwrap(
503-
makeProviderAdapter(input.state, {
504-
failEventStream: input.failEventStream ?? false,
505-
...(input.capabilities === undefined ? {} : { capabilities: input.capabilities }),
506-
...(input.mcpConfigs === undefined ? {} : { mcpConfigs: input.mcpConfigs }),
507-
...(input.beforeOpen === undefined ? {} : { beforeOpen: input.beforeOpen }),
508-
...(input.hasPendingBackgroundWork === undefined
509-
? {}
510-
: { hasPendingBackgroundWork: input.hasPendingBackgroundWork }),
511-
...(input.hasPendingBackgroundWorkForThread === undefined
512-
? {}
513-
: { hasPendingBackgroundWorkForThread: input.hasPendingBackgroundWorkForThread }),
514-
...(input.backgroundTasksEndWithProcess === undefined
515-
? {}
516-
: { backgroundTasksEndWithProcess: input.backgroundTasksEndWithProcess }),
517-
...(input.hangSessionScopeClose === undefined
518-
? {}
519-
: { hangSessionScopeClose: input.hangSessionScopeClose }),
520-
...(input.startTurn === undefined ? {} : { startTurn: input.startTurn }),
521-
...(input.beforeUnload === undefined ? {} : { beforeUnload: input.beforeUnload }),
522-
...(input.spawnBeforeOpen === undefined ? {} : { spawnBeforeOpen: input.spawnBeforeOpen }),
523-
...(input.scopeCloseReached === undefined
524-
? {}
525-
: { scopeCloseReached: input.scopeCloseReached }),
526-
}).pipe(Effect.map(ProviderAdapterRegistry.layerSingle)),
524+
(
525+
input.adapters ??
526+
makeProviderAdapter(input.state, {
527+
failEventStream: input.failEventStream ?? false,
528+
...(input.capabilities === undefined ? {} : { capabilities: input.capabilities }),
529+
...(input.mcpConfigs === undefined ? {} : { mcpConfigs: input.mcpConfigs }),
530+
...(input.beforeOpen === undefined ? {} : { beforeOpen: input.beforeOpen }),
531+
...(input.hasPendingBackgroundWork === undefined
532+
? {}
533+
: { hasPendingBackgroundWork: input.hasPendingBackgroundWork }),
534+
...(input.hasPendingBackgroundWorkForThread === undefined
535+
? {}
536+
: { hasPendingBackgroundWorkForThread: input.hasPendingBackgroundWorkForThread }),
537+
...(input.backgroundTasksEndWithProcess === undefined
538+
? {}
539+
: { backgroundTasksEndWithProcess: input.backgroundTasksEndWithProcess }),
540+
...(input.hangSessionScopeClose === undefined
541+
? {}
542+
: { hangSessionScopeClose: input.hangSessionScopeClose }),
543+
...(input.startTurn === undefined ? {} : { startTurn: input.startTurn }),
544+
...(input.beforeUnload === undefined ? {} : { beforeUnload: input.beforeUnload }),
545+
...(input.spawnBeforeOpen === undefined ? {} : { spawnBeforeOpen: input.spawnBeforeOpen }),
546+
...(input.scopeCloseReached === undefined
547+
? {}
548+
: { scopeCloseReached: input.scopeCloseReached }),
549+
}).pipe(Effect.map((adapter) => [adapter]))
550+
).pipe(Effect.map(ProviderAdapterRegistry.layerFromAdapters)),
527551
);
528552
const layerConfiguredMcpRegistry =
529553
input.pauseResolve === undefined
@@ -554,7 +578,10 @@ function layerTest(input: {
554578
}).pipe(
555579
Layer.provide(
556580
Layer.mergeAll(
557-
layerRegistry.pipe(Layer.provideMerge(McpProviderSessions.layer)),
581+
layerRegistry.pipe(
582+
Layer.provide(IdAllocator.layer),
583+
Layer.provideMerge(McpProviderSessions.layer),
584+
),
558585
CommandOutputHub.layer.pipe(
559586
Layer.provide(Layer.merge(layerConfiguredEventSink, layerTestStores)),
560587
),
@@ -4908,3 +4935,206 @@ it.effect(
49084935
assert.isFalse(denied?.capabilities?.has("device"));
49094936
}),
49104937
);
4938+
4939+
it.effect.each([false, true])(
4940+
"Codex transport death reopens the saved native thread (in flight: %s)",
4941+
(inFlight) =>
4942+
Effect.gen(function* () {
4943+
const state = yield* Ref.make(emptyState);
4944+
const frames: Array<{ process: number; method: string; params?: { threadId?: string } }> = [];
4945+
const processes: Array<{
4946+
input: Queue.Queue<Uint8Array, Cause.Done>;
4947+
closed: Deferred.Deferred<void>;
4948+
submitted: Deferred.Deferred<void>;
4949+
}> = [];
4950+
const healthyInstanceId = ProviderInstanceId.make("codex-healthy");
4951+
const settings = yield* decodeCodexSettings({
4952+
homePath: "/custom-codex-home",
4953+
}).pipe(Effect.orDie);
4954+
const openedSettings: Array<CodexSettings> = [];
4955+
const adapters = Effect.forEach(
4956+
[modelSelection.instanceId, healthyInstanceId],
4957+
(instanceId) =>
4958+
Effect.gen(function* () {
4959+
return yield* CodexAdapter.makeCodexAdapterV2({
4960+
instanceId,
4961+
settings,
4962+
environment: {},
4963+
crypto: yield* Crypto.Crypto,
4964+
fileSystem: yield* FileSystem.FileSystem,
4965+
idAllocator: yield* IdAllocator.IdAllocatorV2,
4966+
serverConfig: yield* makeReplayServerConfig("dead-session").pipe(Effect.orDie),
4967+
clientFactory: {
4968+
open: (openInput) =>
4969+
Effect.gen(function* () {
4970+
openedSettings.push(openInput.settings);
4971+
const processIndex = processes.length;
4972+
const input = yield* Queue.unbounded<Uint8Array, Cause.Done>();
4973+
const closed = yield* Deferred.make<void>();
4974+
const submitted = yield* Deferred.make<void>();
4975+
processes.push({ input, closed, submitted });
4976+
yield* Effect.addFinalizer(() => Deferred.succeed(closed, undefined));
4977+
const encoder = new TextEncoder();
4978+
return yield* CodexClient.make(
4979+
Stdio.make({
4980+
args: Effect.succeed([]),
4981+
stdin: Stream.fromQueue(input),
4982+
stdout: () =>
4983+
Sink.forEach((chunk: string | Uint8Array) =>
4984+
Effect.gen(function* () {
4985+
const frame = JSON.parse(
4986+
typeof chunk === "string" ? chunk : new TextDecoder().decode(chunk),
4987+
);
4988+
frames.push({
4989+
process: processIndex,
4990+
method: frame.method,
4991+
params: frame.params,
4992+
});
4993+
if (frame.id === undefined) return;
4994+
if (frame.method === "turn/start") {
4995+
yield* Deferred.succeed(submitted, undefined);
4996+
return; // Accepted, but the reply is lost when the process dies.
4997+
}
4998+
const result =
4999+
frame.method === "initialize"
5000+
? {
5001+
userAgent: "test",
5002+
codexHome: process.cwd(),
5003+
platformFamily: "unix",
5004+
platformOs: "linux",
5005+
}
5006+
: { thread: { id: frame.params.threadId, updatedAt: 1 } };
5007+
yield* Queue.offer(
5008+
input,
5009+
encoder.encode(JSON.stringify({ id: frame.id, result }) + "\n"),
5010+
);
5011+
}),
5012+
),
5013+
stderr: () => Sink.drain,
5014+
}),
5015+
{},
5016+
Effect.succeed(
5017+
new CodexError.CodexAppServerTransportError({
5018+
operation: "read-process-exit-status",
5019+
cause: "SIGKILL",
5020+
}),
5021+
),
5022+
);
5023+
}),
5024+
},
5025+
});
5026+
}),
5027+
);
5028+
yield* Effect.gen(function* () {
5029+
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
5030+
const eventSink = yield* EventSink.EventSinkV2;
5031+
const allocator = yield* IdAllocator.IdAllocatorV2;
5032+
const projections = yield* ProjectionStore.ProjectionStoreV2;
5033+
const now = yield* DateTime.now;
5034+
const threadId = ThreadId.make("dead-codex");
5035+
const healthyThreadId = ThreadId.make("healthy-codex");
5036+
for (const id of [threadId, healthyThreadId]) {
5037+
yield* eventSink.write({
5038+
events: [yield* makeThreadCreatedEvent({ idAllocator: allocator, threadId: id, now })],
5039+
});
5040+
}
5041+
const providerSessionId = yield* allocator.allocate.providerSession({
5042+
providerInstanceId: modelSelection.instanceId,
5043+
threadId,
5044+
});
5045+
const healthySessionId = yield* allocator.allocate.providerSession({
5046+
providerInstanceId: healthyInstanceId,
5047+
threadId: healthyThreadId,
5048+
});
5049+
const openInput = { threadId, providerSessionId, modelSelection, runtimePolicy };
5050+
const first = yield* manager.open(openInput);
5051+
const saved = makeProviderThread({
5052+
idAllocator: allocator,
5053+
threadId,
5054+
providerSessionId,
5055+
now,
5056+
nativeThreadId: "saved-native-id",
5057+
});
5058+
const resumeInput = { threadId, providerThread: saved, modelSelection, runtimePolicy };
5059+
yield* first.resumeThread(resumeInput);
5060+
const healthy = yield* manager.open({
5061+
...openInput,
5062+
threadId: healthyThreadId,
5063+
providerSessionId: healthySessionId,
5064+
modelSelection: { ...modelSelection, instanceId: healthyInstanceId },
5065+
});
5066+
const subscription = yield* first.subscribeEvents!;
5067+
const streamExit = yield* subscription.events.pipe(
5068+
Stream.runDrain,
5069+
Effect.exit,
5070+
Effect.forkChild,
5071+
);
5072+
const runId = allocator.derive.run({ threadId, ordinal: 1 });
5073+
const starting = inFlight
5074+
? yield* first
5075+
.startTurn({
5076+
appThread: (yield* projections.getThreadProjection(threadId)).thread,
5077+
threadId,
5078+
runId,
5079+
runOrdinal: 1,
5080+
providerTurnOrdinal: 1,
5081+
attemptId: allocator.derive.runAttempt({ runId, attemptOrdinal: 1 }),
5082+
rootNodeId: allocator.derive.rootNode({ runId }),
5083+
providerThread: saved,
5084+
message: {
5085+
createdBy: "user",
5086+
creationSource: "web",
5087+
messageId: yield* allocator.allocate.message({ threadId, ordinal: 1 }),
5088+
text: "do work once",
5089+
attachments: [],
5090+
},
5091+
modelSelection,
5092+
runtimePolicy,
5093+
})
5094+
.pipe(Effect.exit, Effect.forkChild)
5095+
: undefined;
5096+
if (starting) yield* Deferred.await(processes[0]!.submitted);
5097+
yield* Queue.end(processes[0]!.input);
5098+
assert.isTrue(Exit.isFailure(yield* Fiber.join(streamExit)));
5099+
if (starting) assert.isTrue(Exit.isFailure(yield* Fiber.join(starting)));
5100+
yield* Deferred.await(processes[0]!.closed);
5101+
assert.isTrue(Option.isNone(yield* manager.get(providerSessionId)));
5102+
assert.strictEqual(Option.getOrThrow(yield* manager.get(healthySessionId)), healthy);
5103+
assert.isFalse(yield* Deferred.isDone(processes[1]!.closed));
5104+
yield* healthy.resumeThread({
5105+
threadId: healthyThreadId,
5106+
providerThread: makeProviderThread({
5107+
idAllocator: allocator,
5108+
threadId: healthyThreadId,
5109+
providerSessionId: healthySessionId,
5110+
now,
5111+
nativeThreadId: "healthy-native-id",
5112+
}),
5113+
modelSelection: { ...modelSelection, instanceId: healthyInstanceId },
5114+
runtimePolicy,
5115+
});
5116+
const replacement = yield* manager.open(openInput);
5117+
const resumed = yield* replacement.resumeThread(resumeInput);
5118+
assert.notStrictEqual(replacement, first);
5119+
assert.equal(resumed.nativeThreadRef?.nativeId, "saved-native-id");
5120+
assert.equal(processes.length, 3);
5121+
assert.deepEqual(openedSettings, [settings, settings, settings]);
5122+
assert.deepEqual(
5123+
frames
5124+
.filter((f) => f.method === "thread/resume")
5125+
.map((f) => [f.process, f.params?.threadId]),
5126+
[
5127+
[0, "saved-native-id"],
5128+
[1, "healthy-native-id"],
5129+
[2, "saved-native-id"],
5130+
],
5131+
);
5132+
assert.equal(frames.filter((f) => f.method === "thread/start").length, 0);
5133+
assert.equal(frames.filter((f) => f.method === "turn/start").length, inFlight ? 1 : 0);
5134+
assert.deepEqual(
5135+
frames.filter((f) => f.method === "initialize").map((f) => f.process),
5136+
[0, 1, 2],
5137+
);
5138+
}).pipe(Effect.provide(layerTest({ state, adapters, idleTimeoutMs: 60_000 })));
5139+
}),
5140+
);

‎packages/effect-codex-app-server/src/client.test.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import * as Fiber from "effect/Fiber";
12
import * as Exit from "effect/Exit";
23
import * as Layer from "effect/Layer";
34
import * as Path from "effect/Path";
@@ -124,6 +125,26 @@ it.layer(NodeServices.layer)("effect-codex-app-server client", (it) => {
124125
]);
125126
}),
126127
);
128+
it.effect("reports SIGKILL between requests to every termination observer", () =>
129+
Effect.gen(function* () {
130+
const handle = yield* makeHandle();
131+
const context = yield* Layer.build(CodexClient.layerChildProcess(handle));
132+
const client = yield* Effect.service(CodexClient.CodexAppServerClient).pipe(
133+
Effect.provide(context),
134+
);
135+
yield* client.request("initialize", {
136+
clientInfo: { name: "termination-test", version: "0.0.0" },
137+
capabilities: { experimentalApi: true, optOutNotificationMethods: null },
138+
});
139+
const observer = yield* client.termination.pipe(Effect.exit, Effect.forkChild);
140+
yield* handle.kill({ killSignal: "SIGKILL" });
141+
const exit = yield* Fiber.join(observer);
142+
assert.isTrue(Exit.isFailure(exit));
143+
// The signal is durable even for observers attached after the death.
144+
assert.deepEqual(yield* Effect.exit(client.termination), exit);
145+
assert.isTrue(Exit.isFailure(yield* Effect.exit(client.raw.request("account/read", {}))));
146+
}),
147+
);
127148
it.effect("drains child stderr so large diagnostics cannot block protocol responses", () =>
128149
Effect.gen(function* () {
129150
const handle = yield* makeHandle({

0 commit comments

Comments
 (0)