Skip to content

Commit 03f9cfd

Browse files
SunkenInTimeclaude
andcommitted
fix(server): Codex runs fail and recover instead of hanging when the app-server dies or stops answering
Every Codex thread shares one app-server process. When it stopped answering or exited mid-turn, T3 never noticed: running turns stayed "running" forever, delegated tasks never woke their parent, and new Codex threads hung at "starting" until T3 restarted. The adapter now fails its session when the app-server connection ends, and when the app-server does not answer a once-a-minute thread/loaded/list probe within a minute. The session manager then fails the session's runs and the next turn starts a fresh app-server. Events Codex sent before the failure are kept. The protocol also ends the connection 2 s after the process exits, since a command Codex started can keep its stdout open, and fails requests still waiting for a reply when the client closes. A release no longer overwrites the status of a replacement session that opened under the same id meanwhile. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 0e7346c commit 03f9cfd

7 files changed

Lines changed: 301 additions & 3 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ import * as Path from "effect/Path";
4848
import * as PlatformError from "effect/PlatformError";
4949
import * as Ref from "effect/Ref";
5050
import * as Schema from "effect/Schema";
51+
import * as Sink from "effect/Sink";
52+
import * as Stdio from "effect/Stdio";
5153
import * as Stream from "effect/Stream";
5254
import { TestClock } from "effect/testing";
5355
import { ChildProcess, ChildProcessSpawner } from "effect/process";
@@ -1897,6 +1899,71 @@ describe("CodexAdapterV2 session initialize", () => {
18971899
);
18981900
});
18991901

1902+
describe("CodexAdapterV2 lost app-server", () => {
1903+
const sessionFailureOver = (stdin: Stream.Stream<Uint8Array>) =>
1904+
Effect.gen(function* () {
1905+
const adapter = yield* CodexAdapterV2.makeCodexAdapterV2({
1906+
instanceId: CodexAdapterV2.CODEX_DEFAULT_INSTANCE_ID,
1907+
settings: DEFAULT_CODEX_SETTINGS,
1908+
environment: {},
1909+
clientFactory: {
1910+
open: () =>
1911+
CodexClient.make(
1912+
Stdio.make({
1913+
args: Effect.succeed([]),
1914+
stdin,
1915+
stdout: () => Sink.drain,
1916+
stderr: () => Sink.drain,
1917+
}),
1918+
),
1919+
},
1920+
crypto: yield* Crypto.Crypto,
1921+
fileSystem: yield* FileSystem.FileSystem,
1922+
idAllocator: yield* IdAllocator.IdAllocatorV2,
1923+
serverConfig: yield* makeReplayServerConfig("codex-lost-app-server").pipe(Effect.orDie),
1924+
});
1925+
const runtime = yield* adapter.openSession({
1926+
threadId: ThreadId.make("thread-codex-lost-app-server"),
1927+
providerSessionId: ProviderSessionId.make("provider-session-codex-lost-app-server"),
1928+
modelSelection: CODEX_TEST_MODEL_SELECTION,
1929+
runtimePolicy: CODEX_TEST_RUNTIME_POLICY,
1930+
});
1931+
return yield* runtime.events.pipe(
1932+
Stream.runDrain,
1933+
Effect.flip,
1934+
Effect.forkChild({ startImmediately: true }),
1935+
);
1936+
});
1937+
1938+
it.effect("fails the session when the app-server stops answering", () =>
1939+
Effect.gen(function* () {
1940+
const failure = yield* sessionFailureOver(Stream.never);
1941+
yield* TestClock.adjust("1 minute");
1942+
yield* TestClock.adjust("1 minute");
1943+
1944+
const error = yield* Fiber.join(failure);
1945+
assert.instanceOf(error, ProviderAdapter.ProviderAdapterEventStreamError);
1946+
assert.include(String(error.cause), "Codex stopped responding");
1947+
}).pipe(
1948+
Effect.provide(
1949+
Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer),
1950+
),
1951+
),
1952+
);
1953+
1954+
it.effect("fails the session when the app-server connection ends", () =>
1955+
Effect.gen(function* () {
1956+
const error = yield* Fiber.join(yield* sessionFailureOver(Stream.empty));
1957+
assert.instanceOf(error, ProviderAdapter.ProviderAdapterEventStreamError);
1958+
assert.include(String(error.cause), "Codex app-server connection ended");
1959+
}).pipe(
1960+
Effect.provide(
1961+
Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer),
1962+
),
1963+
),
1964+
);
1965+
});
1966+
19001967
describe("CodexAdapterV2 post-settle continuation", () => {
19011968
const awaitUntil = (predicate: () => boolean, label: string): Effect.Effect<void> =>
19021969
Effect.gen(function* () {

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts‎

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -468,6 +468,11 @@ function codexMcpToolOutput(
468468
// Reading an app's resource is one MCP round trip; a server that hangs must not
469469
// hold the item (and the run's ingestion) open.
470470
const MCP_APP_CAPTURE_TIMEOUT = "20 seconds";
471+
// A wedged app-server stays alive but stops answering. The probe,
472+
// thread/loaded/list, only reads Codex's in-memory thread list, and any reply
473+
// to it, even an error, proves the app-server is still reading.
474+
const CODEX_LIVENESS_PROBE_INTERVAL = "1 minute";
475+
const CODEX_LIVENESS_PROBE_TIMEOUT = "1 minute";
471476

472477
/** The UI resource a completed Codex MCP tool call names, under any field Codex reports it in. */
473478
function codexMcpAppResourceUri(item: CodexDynamicToolItem): string | undefined {
@@ -1806,7 +1811,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
18061811
model: input.modelSelection.model,
18071812
now,
18081813
});
1809-
const events = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>();
1814+
const events = yield* Queue.unbounded<
1815+
ProviderAdapter.ProviderAdapterV2Event,
1816+
ProviderAdapter.ProviderAdapterEventStreamError
1817+
>();
18101818
const rateLimitSnapshot = yield* Ref.make<CodexRateLimitSnapshot | undefined>(undefined);
18111819
const limitedTurnItems = yield* Ref.make(
18121820
new Map<ProviderThreadId, Extract<OrchestrationV2TurnItem, { type: "error" }>>(),
@@ -6409,6 +6417,43 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
64096417
}).pipe(Effect.mapError(mcpAppsError("Codex MCP resource read failed."))),
64106418
};
64116419

6420+
// Every thread on this instance shares the app-server, so losing it
6421+
// fails the session after the events it already sent: its runs fail
6422+
// and the next turn starts a new app-server.
6423+
const unresponsive = Effect.gen(function* () {
6424+
while (true) {
6425+
yield* Effect.sleep(CODEX_LIVENESS_PROBE_INTERVAL);
6426+
const answered = yield* client
6427+
.request("thread/loaded/list", { limit: 1 })
6428+
.pipe(
6429+
Effect.catchTags({ CodexAppServerRequestError: () => Effect.void }),
6430+
Effect.timeoutOption(CODEX_LIVENESS_PROBE_TIMEOUT),
6431+
);
6432+
if (Option.isNone(answered)) {
6433+
return `Codex stopped responding: it did not answer a liveness check within ${CODEX_LIVENESS_PROBE_TIMEOUT}.`;
6434+
}
6435+
}
6436+
});
6437+
yield* Effect.raceFirst(
6438+
client.terminated.pipe(
6439+
Effect.map((error) => `Codex app-server connection ended: ${error.message}`),
6440+
),
6441+
// A probe that fails with a transport error is the termination above.
6442+
unresponsive.pipe(Effect.catch(() => Effect.never)),
6443+
).pipe(
6444+
Effect.flatMap((detail) =>
6445+
Queue.fail(
6446+
events,
6447+
new ProviderAdapter.ProviderAdapterEventStreamError({
6448+
driver: CODEX_PROVIDER,
6449+
providerSessionId: input.providerSessionId,
6450+
cause: detail,
6451+
}),
6452+
),
6453+
),
6454+
Effect.forkIn(scope),
6455+
);
6456+
64126457
const runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime = {
64136458
instanceId: adapterOptions.instanceId,
64146459
driver: CODEX_PROVIDER,

‎apps/server/src/orchestration-v2/ProviderSessionManager.test.ts‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3531,6 +3531,81 @@ it.effect("ProviderSessionManagerV2 release retries leave a replacement session
35313531
}),
35323532
);
35333533

3534+
it.effect(
3535+
"ProviderSessionManagerV2 leaves a replacement's status to it when a failed release ends",
3536+
() =>
3537+
Effect.gen(function* () {
3538+
const state = yield* Ref.make(emptyState);
3539+
const scopeCloseReached = yield* Deferred.make<void>();
3540+
yield* Effect.gen(function* () {
3541+
const eventSink = yield* EventSink.EventSinkV2;
3542+
const idAllocator = yield* IdAllocator.IdAllocatorV2;
3543+
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
3544+
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
3545+
const threadId = ThreadId.make("thread-provider-session-manager-failed-replacement");
3546+
const providerSessionId = yield* idAllocator.allocate.providerSession({
3547+
providerInstanceId: modelSelection.instanceId,
3548+
threadId,
3549+
});
3550+
const now = yield* DateTime.now;
3551+
// The release settles this request right after recording the session.
3552+
const request = yield* makePendingRuntimeRequestEvents({
3553+
idAllocator,
3554+
threadId,
3555+
providerSessionId,
3556+
providerThread: makeProviderThread({ idAllocator, threadId, providerSessionId, now }),
3557+
now,
3558+
});
3559+
yield* eventSink.write({
3560+
events: [
3561+
yield* makeThreadCreatedEvent({ idAllocator, threadId, now }),
3562+
...request.events,
3563+
],
3564+
});
3565+
yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy });
3566+
3567+
// The runtime's events end unexpectedly, and its release stalls in scope close.
3568+
const events = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId));
3569+
assert.isDefined(events);
3570+
yield* Queue.end(events!);
3571+
yield* Deferred.await(scopeCloseReached);
3572+
3573+
// Another thread on the same runtime reopens it meanwhile; then the
3574+
// time-boxed close lets the release write its records.
3575+
yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy });
3576+
const settled = yield* eventSink
3577+
.stream({
3578+
threadId,
3579+
afterSequence: yield* eventSink.latestSequence({ threadId }),
3580+
eventType: "runtime-request.updated",
3581+
})
3582+
.pipe(Stream.runHead, Effect.forkScoped);
3583+
yield* TestClock.adjust("30 seconds");
3584+
yield* Fiber.join(settled);
3585+
const status = (yield* projectionStore.getThreadProjection(threadId)).providerSessions.at(
3586+
-1,
3587+
)?.status;
3588+
3589+
// The replacement's close hangs as well; release it while the clock can move.
3590+
const stopping = yield* manager.shutdown.pipe(Effect.forkChild);
3591+
yield* TestClock.adjust("30 seconds");
3592+
yield* Fiber.join(stopping);
3593+
3594+
assert.isDefined(status);
3595+
assert.notEqual(status, "error");
3596+
}).pipe(
3597+
Effect.provide(
3598+
layerTest({
3599+
state,
3600+
idleTimeoutMs: 60_000,
3601+
hangSessionScopeClose: true,
3602+
scopeCloseReached,
3603+
}),
3604+
),
3605+
);
3606+
}),
3607+
);
3608+
35343609
it.effect("ProviderSessionManagerV2 keeps each failed release's cleanup", () =>
35353610
Effect.gen(function* () {
35363611
const state = yield* Ref.make(emptyState);

‎apps/server/src/orchestration-v2/ProviderSessionManager.ts‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -940,6 +940,8 @@ export const layerWithOptions = (
940940
readonly cancelIdleFiber?: boolean;
941941
readonly onlyIfIdleGeneration?: number;
942942
readonly gracefulSubscribers?: boolean;
943+
/** Set by callers inside `open`, which already hold this session's open lock. */
944+
readonly openLockHeld?: boolean;
943945
}) =>
944946
Effect.acquireUseRelease(
945947
removeLiveEntry(input),
@@ -971,8 +973,19 @@ export const layerWithOptions = (
971973
...(input.detail === undefined ? {} : { detail: input.detail }),
972974
releasedAt,
973975
};
976+
// A thread of a shared runtime can reopen it under the same
977+
// id while this closes. The open lock orders the check and
978+
// write against that, so a replacement keeps its status.
979+
const writeRecords = Effect.gen(function* () {
980+
const replaced = (yield* Ref.get(sessions)).has(
981+
sessionKey(input.providerSessionId),
982+
);
983+
yield* writeReleaseRecords({ ...records, replaced });
984+
});
974985
const recorded = yield* Effect.exit(
975-
writeReleaseRecords({ ...records, replaced: false }),
986+
input.openLockHeld === true
987+
? writeRecords
988+
: sessionOpen.withLock(input.providerSessionId, writeRecords),
976989
);
977990
if (Exit.isFailure(recorded)) {
978991
yield* retryReleaseRecords(records);
@@ -2209,11 +2222,13 @@ export const layerWithOptions = (
22092222
providerSessionId: input.providerSessionId,
22102223
reason: "manual_shutdown",
22112224
detail: "The provider session start was interrupted.",
2225+
openLockHeld: true,
22122226
}
22132227
: {
22142228
providerSessionId: input.providerSessionId,
22152229
reason: "runtime_error",
22162230
detail: "Failed to persist the provider-session attachment.",
2231+
openLockHeld: true,
22172232
},
22182233
).pipe(logReleaseFailure(input.providerSessionId)),
22192234
),

‎packages/effect-codex-app-server/src/client.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import * as Context from "effect/Context";
2+
import * as Deferred from "effect/Deferred";
23
import * as Effect from "effect/Effect";
34
import * as Layer from "effect/Layer";
45
import * as Schema from "effect/Schema";
@@ -75,6 +76,8 @@ export class CodexAppServerClient extends Context.Service<
7576
params: unknown,
7677
) => Effect.Effect<void, CodexError.CodexAppServerError>,
7778
) => Effect.Effect<void>;
79+
/** Resolves with the reason once the app-server connection ends, such as the process exiting. */
80+
readonly terminated: Effect.Effect<CodexError.CodexAppServerError>;
7881
}
7982
>()("effect-codex-app-server/client/CodexAppServerClient") {}
8083

@@ -98,6 +101,7 @@ export const make = Effect.fn("effect-codex-app-server/CodexAppServerClient.make
98101
stdio: Stdio.Stdio,
99102
options: CodexAppServerClientOptions = {},
100103
terminationError?: Effect.Effect<CodexError.CodexAppServerError>,
104+
exited?: Effect.Effect<CodexError.CodexAppServerError>,
101105
): Effect.fn.Return<CodexAppServerClient["Service"], never, Scope.Scope> {
102106
const requestHandlers = new Map<string, ServerRequestHandler>();
103107
const notificationHandlers = new Map<string, Array<ServerNotificationHandler>>();
@@ -195,9 +199,12 @@ export const make = Effect.fn("effect-codex-app-server/CodexAppServerClient.make
195199
: Effect.fail(CodexError.CodexAppServerRequestError.methodNotFound(request.method));
196200
};
197201

202+
const termination = yield* Deferred.make<CodexError.CodexAppServerError>();
198203
const transport = yield* CodexProtocol.makeCodexAppServerPatchedProtocol({
199204
stdio,
205+
onTermination: (error) => Deferred.succeed(termination, error).pipe(Effect.asVoid),
200206
...(terminationError ? { terminationError } : {}),
207+
...(exited ? { exited } : {}),
201208
...(options.logIncoming !== undefined ? { logIncoming: options.logIncoming } : {}),
202209
...(options.logOutgoing !== undefined ? { logOutgoing: options.logOutgoing } : {}),
203210
...(options.logger ? { logger: options.logger } : {}),
@@ -258,6 +265,7 @@ export const make = Effect.fn("effect-codex-app-server/CodexAppServerClient.make
258265
Effect.sync(() => {
259266
unknownNotificationHandler = handler;
260267
}),
268+
terminated: Deferred.await(termination),
261269
});
262270
});
263271

@@ -271,5 +279,6 @@ const makeChildProcessClient = Effect.fn(
271279
"effect-codex-app-server/CodexAppServerClient.makeChildProcessClient",
272280
)(function* (handle: ChildProcessSpawner.ChildProcessHandle, options: CodexAppServerClientOptions) {
273281
yield* Stream.runDrain(handle.stderr).pipe(Effect.ignore, Effect.forkScoped);
274-
return yield* make(makeChildStdio(handle), options, makeTerminationError(handle));
282+
const terminationError = makeTerminationError(handle);
283+
return yield* make(makeChildStdio(handle), options, terminationError, terminationError);
275284
});

0 commit comments

Comments
 (0)