Skip to content

Commit 3a2f884

Browse files
omar7550claude
andcommitted
fix(server): Codex runs and delegated tasks settle when the shared app-server exits
Every Codex thread shares one app-server. When it died, the client never reported the termination and the adapter's event stream never ended, so runs stayed running forever and later turns reused the dead session. - The Codex client exposes awaitTermination; the adapter fails its event stream on it, so the session manager fails the runs and releases the session. - A finalized run's still-open native subagents are failed when the provider stream dies, and the orchestrator re-finalizes the delegated task waiting on them. - A release no longer overwrites the status of a shared session reopened while the old one was closing. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 5307c23 commit 3a2f884

11 files changed

Lines changed: 746 additions & 25 deletions

‎apps/server/src/mcp/DelegatedTaskProviderExit.integration.test.ts‎

Lines changed: 398 additions & 0 deletions
Large diffs are not rendered by default.

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1791,7 +1791,32 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
17911791
model: input.modelSelection.model,
17921792
now,
17931793
});
1794-
const events = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>();
1794+
const events = yield* Queue.unbounded<
1795+
ProviderAdapter.ProviderAdapterV2Event,
1796+
ProviderAdapter.ProviderAdapterEventStreamError
1797+
>();
1798+
// Every thread in this session shares one app-server. Once it is gone,
1799+
// fail the session's stream: the session manager then fails its runs
1800+
// and releases the session, so the next turn starts a new app-server.
1801+
yield* client.awaitTermination.pipe(
1802+
Effect.tap((cause) =>
1803+
Effect.logWarning("Codex app-server exited; failing its provider session.", {
1804+
providerSessionId: input.providerSessionId,
1805+
cause,
1806+
}),
1807+
),
1808+
Effect.flatMap((cause) =>
1809+
Queue.fail(
1810+
events,
1811+
new ProviderAdapter.ProviderAdapterEventStreamError({
1812+
driver: CODEX_PROVIDER,
1813+
providerSessionId: input.providerSessionId,
1814+
cause,
1815+
}),
1816+
),
1817+
),
1818+
Effect.forkIn(scope),
1819+
);
17951820
const rateLimitSnapshot = yield* Ref.make<CodexRateLimitSnapshot | undefined>(undefined);
17961821
const limitedTurnItems = yield* Ref.make(
17971822
new Map<ProviderThreadId, Extract<OrchestrationV2TurnItem, { type: "error" }>>(),
@@ -6413,7 +6438,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
64136438
driver: CODEX_PROVIDER,
64146439
providerSessionId: input.providerSessionId,
64156440
providerSession: session,
6416-
events: Stream.fromEffectRepeat(Queue.take(events)),
6441+
events: Stream.fromQueue(events),
64176442
canReuseContextUsage: canReuseCodexContextUsage,
64186443
// Known gap: a subagent that Codex resumes later reads as completed
64196444
// (not pending) between turns, so idle release can win the race

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 53 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -11499,27 +11499,62 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1149911499
// below. Replaying the full event table on every server start delays live
1150011500
// queue promotion in proportion to the lifetime size of the database.
1150111501
const terminalEventsAfterSequence = yield* eventSink.latestSequence().pipe(Effect.orDie);
11502+
// See PROVIDER_EXIT_SUBAGENT_CASCADE_COMMAND_PREFIX: those subagent rows settle
11503+
// without a run update, so the child's delegated task is reconsidered here.
11504+
const settleDelegatedTaskAfterProviderExit = (stored: OrchestrationV2StoredEvent) =>
11505+
Effect.gen(function* () {
11506+
const threadId = stored.event.threadId;
11507+
const parentThreadId = yield* appOwnedSubagentParentThreadId(threadId);
11508+
if (parentThreadId === undefined) return;
11509+
yield* threadDispatch.withLock(parentThreadId, finalizeAppOwnedSubagent(threadId));
11510+
}).pipe(
11511+
Effect.catchCause((cause) =>
11512+
Effect.logWarning("Failed to settle a delegated task after its provider exited", {
11513+
threadId: stored.event.threadId,
11514+
cause,
11515+
}),
11516+
),
11517+
);
11518+
1150211519
// Queue promotion can wait on a provider or a thread lock. Subscribe to run
1150311520
// updates before buffering so that wait never retains unrelated tool bodies.
11504-
yield* eventSink
11505-
.stream({ afterSequence: terminalEventsAfterSequence, eventType: "run.updated" })
11506-
.pipe(
11507-
Stream.filter(
11508-
(stored) =>
11509-
stored.event.type === "run.updated" &&
11510-
!String(stored.commandId).startsWith("command:runtime-reconcile:") &&
11511-
((stored.event.payload.delegatedTaskId !== undefined &&
11512-
(stored.event.payload.status === "queued" ||
11513-
stored.event.payload.status === "starting")) ||
11514-
stored.event.payload.status === "completed" ||
11515-
stored.event.payload.status === "interrupted" ||
11516-
stored.event.payload.status === "failed" ||
11517-
stored.event.payload.status === "cancelled" ||
11518-
stored.event.payload.status === "rolled_back"),
11521+
// One consumer serializes both reactions.
11522+
yield* Stream.merge(
11523+
eventSink
11524+
.stream({ afterSequence: terminalEventsAfterSequence, eventType: "run.updated" })
11525+
.pipe(
11526+
Stream.filter(
11527+
(stored) =>
11528+
stored.event.type === "run.updated" &&
11529+
!String(stored.commandId).startsWith("command:runtime-reconcile:") &&
11530+
((stored.event.payload.delegatedTaskId !== undefined &&
11531+
(stored.event.payload.status === "queued" ||
11532+
stored.event.payload.status === "starting")) ||
11533+
stored.event.payload.status === "completed" ||
11534+
stored.event.payload.status === "interrupted" ||
11535+
stored.event.payload.status === "failed" ||
11536+
stored.event.payload.status === "cancelled" ||
11537+
stored.event.payload.status === "rolled_back"),
11538+
),
1151911539
),
11520-
Stream.runForEach(handleRunUpdate),
11521-
Effect.forkDetach,
11522-
);
11540+
eventSink
11541+
.stream({ afterSequence: terminalEventsAfterSequence, eventType: "subagent.updated" })
11542+
.pipe(
11543+
Stream.filter((stored) =>
11544+
String(stored.commandId).startsWith(
11545+
RunExecutionService.PROVIDER_EXIT_SUBAGENT_CASCADE_COMMAND_PREFIX,
11546+
),
11547+
),
11548+
Stream.changesWith((left, right) => left.commandId === right.commandId),
11549+
),
11550+
).pipe(
11551+
Stream.runForEach((stored) =>
11552+
stored.event.type === "run.updated"
11553+
? handleRunUpdate(stored)
11554+
: settleDelegatedTaskAfterProviderExit(stored),
11555+
),
11556+
Effect.forkDetach,
11557+
);
1152311558

1152411559
const recoverDelegatedTasks = Effect.gen(function* () {
1152511560
yield* projectionStore.getRecoveryThreadIds("subagent-results").pipe(

‎apps/server/src/orchestration-v2/ProviderSessionManager.test.ts‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2691,6 +2691,55 @@ it.effect("ProviderSessionManagerV2 releases idle sessions without sweeping all
26912691
}),
26922692
);
26932693

2694+
it.effect(
2695+
"ProviderSessionManagerV2 keeps a reopened session's status when the old release finishes later",
2696+
() =>
2697+
Effect.gen(function* () {
2698+
const state = yield* Ref.make(emptyState);
2699+
const scopeCloseReached = yield* Deferred.make<void>();
2700+
yield* Effect.gen(function* () {
2701+
const eventSink = yield* EventSink.EventSinkV2;
2702+
const idAllocator = yield* IdAllocator.IdAllocatorV2;
2703+
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
2704+
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
2705+
const now = yield* DateTime.now;
2706+
const threadId = ThreadId.make("thread-provider-session-manager-reopened-during-release");
2707+
const providerSessionId = yield* idAllocator.allocate.providerSession({
2708+
providerInstanceId: modelSelection.instanceId,
2709+
threadId,
2710+
});
2711+
yield* eventSink.write({
2712+
events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })],
2713+
});
2714+
yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy });
2715+
2716+
// The old release stalls closing its process; a new turn reopens the
2717+
// shared session id meanwhile, as a completion wake does.
2718+
const release = yield* manager.close(providerSessionId).pipe(Effect.forkChild);
2719+
yield* Deferred.await(scopeCloseReached);
2720+
yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy });
2721+
yield* TestClock.adjust("30 seconds");
2722+
yield* Fiber.join(release);
2723+
2724+
const projection = yield* projectionStore.getThreadProjection(threadId);
2725+
const stopping = yield* manager.shutdown.pipe(Effect.forkChild);
2726+
yield* TestClock.adjust("30 seconds");
2727+
yield* Fiber.join(stopping);
2728+
assert.equal(projection.providerSessions.at(-1)?.status, "ready");
2729+
}).pipe(
2730+
Effect.provide(
2731+
layerTest({
2732+
state,
2733+
idleTimeoutMs: 60_000,
2734+
spawnBeforeOpen: true,
2735+
hangSessionScopeClose: true,
2736+
scopeCloseReached,
2737+
}),
2738+
),
2739+
);
2740+
}),
2741+
);
2742+
26942743
it.effect("ProviderSessionManagerV2 persists release when session scope close hangs", () =>
26952744
Effect.gen(function* () {
26962745
const state = yield* Ref.make(emptyState);

‎apps/server/src/orchestration-v2/ProviderSessionManager.ts‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -989,6 +989,8 @@ export const layerWithOptions = (
989989
readonly cancelIdleFiber?: boolean;
990990
readonly onlyIfIdleGeneration?: number;
991991
readonly gracefulSubscribers?: boolean;
992+
/** open() releases its own failed start while holding sessionOpen. */
993+
readonly holdsSessionOpenLock?: boolean;
992994
}) =>
993995
Effect.acquireUseRelease(
994996
removeLiveEntry(input),
@@ -1028,8 +1030,19 @@ export const layerWithOptions = (
10281030
releasedBackgroundTasks,
10291031
releasedAt,
10301032
};
1033+
// Closing the scope can take a while. A shared session reopened
1034+
// meanwhile owns the session row, so leave its status alone;
1035+
// sessionOpen keeps a reopen from landing between check and write.
1036+
const writeRecords = Effect.gen(function* () {
1037+
const replaced = (yield* Ref.get(sessions)).has(
1038+
sessionKey(input.providerSessionId),
1039+
);
1040+
yield* writeReleaseRecords({ ...records, replaced });
1041+
});
10311042
const recorded = yield* Effect.exit(
1032-
writeReleaseRecords({ ...records, replaced: false }),
1043+
input.holdsSessionOpenLock === true
1044+
? writeRecords
1045+
: sessionOpen.withLock(input.providerSessionId, writeRecords),
10331046
);
10341047
if (Exit.isFailure(recorded)) {
10351048
yield* retryReleaseRecords(records);
@@ -2230,11 +2243,13 @@ export const layerWithOptions = (
22302243
providerSessionId: input.providerSessionId,
22312244
reason: "manual_shutdown",
22322245
detail: "The provider session start was interrupted.",
2246+
holdsSessionOpenLock: true,
22332247
}
22342248
: {
22352249
providerSessionId: input.providerSessionId,
22362250
reason: "runtime_error",
22372251
detail: "Failed to persist the provider-session attachment.",
2252+
holdsSessionOpenLock: true,
22382253
},
22392254
).pipe(logReleaseFailure(input.providerSessionId)),
22402255
),

‎apps/server/src/orchestration-v2/RunExecutionService.test.ts‎

Lines changed: 48 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4373,6 +4373,8 @@ function runBackgroundItemScenario(
43734373
makeEvents: (ids: BackgroundScenarioIds) => ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>,
43744374
options?: {
43754375
readonly keepEventStreamOpen?: boolean;
4376+
readonly failEventStream?: ProviderAdapter.ProviderAdapterEventStreamError;
4377+
readonly onWrite?: (events: ReadonlyArray<OrchestrationV2DomainEvent>) => Effect.Effect<void>;
43764378
readonly loadInheritedBackgroundTurnItems?: () => Effect.Effect<
43774379
ReadonlyArray<{ readonly id: TurnItemId; readonly runId: RunId }>
43784380
>;
@@ -4397,6 +4399,7 @@ function runBackgroundItemScenario(
43974399
write: () => Effect.succeed([]),
43984400
writeWithEffects: (input) =>
43994401
Effect.gen(function* () {
4402+
yield* options?.onWrite?.(input.events) ?? Effect.void;
44004403
if (
44014404
input.events.some(
44024405
(event) => event.type === "run.updated" && event.runId === ids.runId,
@@ -4447,9 +4450,11 @@ function runBackgroundItemScenario(
44474450
const events = Stream.fromIterable(makeEvents(ids));
44484451
return {
44494452
events:
4450-
options?.keepEventStreamOpen === true
4451-
? events.pipe(Stream.concat(Stream.never))
4452-
: events,
4453+
options?.failEventStream !== undefined
4454+
? events.pipe(Stream.concat(Stream.fail(options.failEventStream)))
4455+
: options?.keepEventStreamOpen === true
4456+
? events.pipe(Stream.concat(Stream.never))
4457+
: events,
44534458
close: Deferred.succeed(ingestionDone, undefined),
44544459
};
44554460
}),
@@ -4556,6 +4561,46 @@ it.effect("keeps a failed root's still-running native children live while the ru
45564561
}),
45574562
);
45584563

4564+
it.effect("fails a completed root's native subagents when the provider stream dies", () =>
4565+
Effect.gen(function* () {
4566+
const written = yield* Ref.make<ReadonlyArray<OrchestrationV2DomainEvent>>([]);
4567+
const observed = yield* runBackgroundItemScenario(
4568+
"completed-root-orphaned-child",
4569+
(ids) => [
4570+
childThreadCreatedEvent(ids),
4571+
subagentEvent(ids, "running"),
4572+
backgroundTurnItemEvent(ids, "subagent", "running", 1),
4573+
rootTerminalEvent(ids, "completed"),
4574+
],
4575+
{
4576+
failEventStream: new ProviderAdapter.ProviderAdapterEventStreamError({
4577+
driver,
4578+
providerSessionId: ProviderSessionId.make("session:completed-root-orphaned-child"),
4579+
cause: "Codex App Server process exited with code 137",
4580+
}),
4581+
onWrite: (events) => Ref.update(written, (current) => [...current, ...events]),
4582+
},
4583+
);
4584+
4585+
assert.deepEqual(observed, ["subagent:running", "turn_item:running", "root-finalized"]);
4586+
const events = yield* Ref.get(written);
4587+
assert.deepEqual(
4588+
events.flatMap((event) => (event.type === "run.updated" ? [event.payload.status] : [])),
4589+
["waiting"],
4590+
"the root turn already completed; its own result stands",
4591+
);
4592+
assert.deepEqual(
4593+
events.flatMap((event) =>
4594+
event.type === "subagent.updated" || event.type === "turn-item.updated"
4595+
? [`${event.type}:${event.payload.status}`]
4596+
: [],
4597+
),
4598+
["subagent.updated:failed", "turn-item.updated:failed"],
4599+
"a subagent whose provider died must not stay running",
4600+
);
4601+
}),
4602+
);
4603+
45594604
it.effect("releases ingestion after idle subagent rows and items settle", () =>
45604605
Effect.gen(function* () {
45614606
const observed = yield* runBackgroundItemScenario(

‎apps/server/src/orchestration-v2/RunExecutionService.ts‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,13 @@ function isRunOwnedSubagentTerminalStatus(
167167
return status === "interrupted" || status === "failed" || status === "cancelled";
168168
}
169169

170+
/**
171+
* Tags the write that fails a finalized run's native subagents after its
172+
* provider died. No run update follows it, so the orchestrator watches this tag
173+
* to settle the delegated task waiting on those subagents.
174+
*/
175+
export const PROVIDER_EXIT_SUBAGENT_CASCADE_COMMAND_PREFIX = "command:provider-exit-subagents:";
176+
170177
function childTurnsContinue(terminal: ProviderTerminalEvent): boolean {
171178
return terminal.status === "failed" && terminal.childTurnsContinue === true;
172179
}
@@ -1165,6 +1172,29 @@ export const layer: Layer.Layer<
11651172
}
11661173
}
11671174
});
1175+
// A completed root keeps ingesting while its native subagents run on.
1176+
// If the provider dies then, no event will ever settle them.
1177+
const failOpenRunOwnedSubagentsAfterFinalization = Effect.gen(function* () {
1178+
const open = yield* Ref.getAndSet(
1179+
openRunOwnedSubagents,
1180+
emptyOpenRunOwnedSubagentProjection(),
1181+
);
1182+
const events = yield* cascadeTerminalizeRunOwnedSubagents({
1183+
run: input.run,
1184+
open,
1185+
status: "failed",
1186+
completedAt: yield* DateTime.now,
1187+
allocateEventId: () => idAllocator.allocate.event({ threadId: input.run.threadId }),
1188+
});
1189+
if (events.length === 0) return;
1190+
yield* eventSink.writeWithEffects({
1191+
commandId: CommandId.make(
1192+
`${PROVIDER_EXIT_SUBAGENT_CASCADE_COMMAND_PREFIX}${input.run.id}`,
1193+
),
1194+
events,
1195+
effects: [],
1196+
});
1197+
});
11681198
const shouldStopProviderEventIngestion = Effect.gen(function* () {
11691199
if (!(yield* Ref.get(rootTerminalSeen))) {
11701200
return false;
@@ -1339,7 +1369,7 @@ export const layer: Layer.Layer<
13391369
}).pipe(
13401370
Effect.andThen(
13411371
finalized
1342-
? Effect.void
1372+
? failOpenRunOwnedSubagentsAfterFinalization
13431373
: Ref.get(latestProviderThread).pipe(
13441374
Effect.flatMap((providerThread) =>
13451375
Ref.get(latestTurnItemOrdinal).pipe(

0 commit comments

Comments
 (0)