Skip to content

Commit 07d38b8

Browse files
fix(server): a failed Codex turn no longer traps the queue behind its subagents (#21366)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent cab362d commit 07d38b8

12 files changed

Lines changed: 261 additions & 4 deletions

File tree

‎apps/mobile/src/lib/threadActivity.test.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1314,6 +1314,20 @@ describe("buildThreadFeed", () => {
13141314
).toEqual([{ type: "thinking", id: "live-activity-row", createdAt: startedAt, runId }]);
13151315
});
13161316

1317+
it("does not show Thinking once the open run's provider turn has ended", () => {
1318+
const startedAt = "2026-04-01T00:00:01.000Z";
1319+
const run = { runId, status: "running" as const, startedAt, completedAt: null };
1320+
expect(
1321+
deriveThreadFeedPresentation(
1322+
[],
1323+
{ ...run, turnEnded: true },
1324+
new Set(),
1325+
new Set(),
1326+
startedAt,
1327+
),
1328+
).toEqual([]);
1329+
});
1330+
13171331
it("uses a stable Thinking row while work has started without a projected item", () => {
13181332
const startedAt = "2026-04-01T00:00:01.000Z";
13191333
const presented = deriveThreadFeedPresentation([], null, new Set(), new Set(), startedAt);

‎apps/mobile/src/lib/threadActivity.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,7 @@ export interface ThreadFeedLatestRun {
240240
readonly status: OrchestrationV2RunStatus;
241241
readonly startedAt: string | null;
242242
readonly completedAt: string | null;
243+
readonly turnEnded?: boolean;
243244
}
244245

245246
export interface AgentSpawnSummary {
@@ -1343,6 +1344,7 @@ export function deriveThreadFeedPresentation(
13431344
if (
13441345
isWorking &&
13451346
activeWorkStartedAt !== null &&
1347+
latestRun?.turnEnded !== true &&
13461348
!result.some(
13471349
(row) =>
13481350
row.id === LIVE_ACTIVITY_ROW_ID ||

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5804,6 +5804,87 @@ describe("CodexAdapterV2 post-settle continuation", () => {
58045804
),
58055805
);
58065806

5807+
const childTurnStartedIndex = interruptSubagentCommandTranscript.entries.findIndex(
5808+
(entry) => entry.type === "emit_inbound" && entry.label === "turn/started/child",
5809+
);
5810+
const failedRootLiveChildTranscript = makeCodexReplayTranscript({
5811+
scenario: "codex-failed-root-live-subagent",
5812+
entries: [
5813+
...interruptSubagentCommandTranscript.entries.slice(0, childTurnStartedIndex + 1),
5814+
{
5815+
type: "emit_inbound",
5816+
label: "turn/completed/root-failed",
5817+
frame: {
5818+
method: "turn/completed",
5819+
params: {
5820+
threadId: INTERRUPT_NATIVE_THREAD,
5821+
turn: {
5822+
...makeCodexReplayTurn({ id: INTERRUPT_NATIVE_TURN, status: "failed" }),
5823+
error: {
5824+
message: "unsupported websocket request type: response.interrupt",
5825+
codexErrorInfo: "other",
5826+
additionalDetails: null,
5827+
},
5828+
},
5829+
},
5830+
},
5831+
},
5832+
{
5833+
type: "emit_inbound",
5834+
label: "turn/completed/child",
5835+
afterMs: 30_000,
5836+
frame: {
5837+
method: "turn/completed",
5838+
params: {
5839+
threadId: INTERRUPT_CHILD_NATIVE_THREAD,
5840+
turn: makeCodexReplayTurn({ id: INTERRUPT_CHILD_NATIVE_TURN, status: "completed" }),
5841+
},
5842+
},
5843+
},
5844+
{ type: "runtime_exit", status: "success" },
5845+
],
5846+
});
5847+
5848+
it.effect("settles a failed root without waiting for subagents that keep running", () =>
5849+
Effect.scoped(
5850+
Effect.gen(function* () {
5851+
const harness = yield* makeCodexReplayHarness(failedRootLiveChildTranscript);
5852+
const now = yield* DateTime.now;
5853+
const childTurnCompleted = () =>
5854+
harness.events.some(
5855+
(event) =>
5856+
event.type === "provider_turn.updated" &&
5857+
event.threadId !== harness.threadId &&
5858+
event.providerTurn.status === "completed",
5859+
);
5860+
5861+
yield* harness.runtime.startTurn(
5862+
makeCodexTestTurnInput({
5863+
threadId: harness.threadId,
5864+
providerThread: harness.providerThread,
5865+
now,
5866+
attemptId: RunAttemptId.make("attempt-codex-failed-root-live-subagent"),
5867+
text: INTERRUPT_PROMPT,
5868+
}),
5869+
);
5870+
yield* awaitUntil(() => harness.terminalEvents().length === 1, "failed root terminal");
5871+
5872+
const terminal = harness.terminalEvents()[0];
5873+
assert.equal(terminal?.status, "failed");
5874+
assert.isTrue(terminal?.status === "failed" && terminal.childTurnsContinue === true);
5875+
assert.isFalse(childTurnCompleted(), "the subagent is still running");
5876+
5877+
yield* TestClock.adjust("30 seconds");
5878+
yield* awaitUntil(childTurnCompleted, "subagent turn completion");
5879+
assert.lengthOf(harness.terminalEvents(), 1);
5880+
}).pipe(
5881+
Effect.provide(
5882+
Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer),
5883+
),
5884+
),
5885+
),
5886+
);
5887+
58075888
const childTerminationResponseIndex = interruptSubagentCommandTranscript.entries.findIndex(
58085889
(entry) =>
58095890
entry.type === "emit_inbound" && entry.label === "thread/backgroundTerminals/terminate/child",

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5637,6 +5637,14 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
56375637
const hasActiveDescendants = Array.from((yield* Ref.get(activeTurns)).values()).some(
56385638
(candidate) => isDescendantCodexTurn(candidate, input.context),
56395639
);
5640+
// Native subagents run on their own threads, so a failed root leaves
5641+
// them working. Holding the root until they finish would keep the
5642+
// run open with no turn to steer. Stop interrupts the whole lineage,
5643+
// so an interrupted root still waits for its descendants to settle.
5644+
if (event.status === "failed" && hasActiveDescendants) {
5645+
yield* emitRootTerminal(input.context, { ...event, childTurnsContinue: true });
5646+
return;
5647+
}
56405648
if (event.status !== "completed" && hasActiveDescendants) {
56415649
yield* Ref.update(deferredRootTerminals, (current) => {
56425650
const updated = new Map(current);

‎apps/server/src/orchestration-v2/RunExecutionService.test.ts‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4377,6 +4377,9 @@ function runBackgroundItemScenario(
43774377
ReadonlyArray<{ readonly id: TurnItemId; readonly runId: RunId }>
43784378
>;
43794379
readonly onSubscribe?: Effect.Effect<void>;
4380+
readonly onFinalized?: (
4381+
events: ReadonlyArray<OrchestrationV2DomainEvent>,
4382+
) => Effect.Effect<void>;
43804383
},
43814384
) {
43824385
return Effect.gen(function* () {
@@ -4400,6 +4403,7 @@ function runBackgroundItemScenario(
44004403
)
44014404
) {
44024405
yield* Ref.update(observed, (current) => [...current, "root-finalized"]);
4406+
yield* options?.onFinalized?.(input.events) ?? Effect.void;
44034407
}
44044408
return [];
44054409
}),
@@ -4504,6 +4508,54 @@ function runBackgroundItemScenario(
45044508
});
45054509
}
45064510

4511+
it.effect("keeps a failed root's still-running native children live while the run settles", () =>
4512+
Effect.gen(function* () {
4513+
const finalized = yield* Ref.make<ReadonlyArray<OrchestrationV2DomainEvent>>([]);
4514+
const childTurn = (
4515+
ids: BackgroundScenarioIds,
4516+
status: "running" | "completed",
4517+
): ProviderAdapter.ProviderAdapterV2Event =>
4518+
({
4519+
type: "provider_turn.updated",
4520+
driver,
4521+
threadId: ids.childThreadId,
4522+
providerTurn: {
4523+
id: ProviderTurnId.make(`${ids.rootProviderTurnId}:child`),
4524+
providerThreadId: ProviderThreadId.make(`${ids.providerThreadId}:child`),
4525+
nodeId: NodeId.make(`${ids.subagentNodeId}:child-root`),
4526+
runAttemptId: null,
4527+
status,
4528+
},
4529+
}) as ProviderAdapter.ProviderAdapterV2Event;
4530+
const observed = yield* runBackgroundItemScenario(
4531+
"failed-root-live-child",
4532+
(ids) => [
4533+
childThreadCreatedEvent(ids),
4534+
subagentEvent(ids, "running"),
4535+
childTurn(ids, "running"),
4536+
{ ...rootTerminalEvent(ids, "failed"), childTurnsContinue: true },
4537+
subagentEvent(ids, "completed"),
4538+
childTurn(ids, "completed"),
4539+
],
4540+
{
4541+
keepEventStreamOpen: true,
4542+
onFinalized: (events) => Ref.set(finalized, events),
4543+
},
4544+
);
4545+
4546+
assert.deepEqual(observed, ["subagent:running", "root-finalized", "subagent:completed"]);
4547+
const written = yield* Ref.get(finalized);
4548+
assert.deepEqual(
4549+
written.flatMap((event) => (event.type === "run.updated" ? [event.payload.status] : [])),
4550+
["failed"],
4551+
);
4552+
assert.isFalse(
4553+
written.some((event) => event.type === "subagent.updated"),
4554+
"a child that outlives the failed root must not be settled with it",
4555+
);
4556+
}),
4557+
);
4558+
45074559
it.effect("releases ingestion after idle subagent rows and items settle", () =>
45084560
Effect.gen(function* () {
45094561
const observed = yield* runBackgroundItemScenario(

‎apps/server/src/orchestration-v2/RunExecutionService.ts‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,10 @@ function isRunOwnedSubagentTerminalStatus(
167167
return status === "interrupted" || status === "failed" || status === "cancelled";
168168
}
169169

170+
function childTurnsContinue(terminal: ProviderTerminalEvent): boolean {
171+
return terminal.status === "failed" && terminal.childTurnsContinue === true;
172+
}
173+
170174
/**
171175
* A running subagent stays with the run that launched it, which keeps ingesting until it ends;
172176
* taking it over too would store its events twice.
@@ -645,7 +649,9 @@ export const layer: Layer.Layer<
645649
open.childTurnItems.size > 0 ||
646650
open.nodes.size > 0;
647651
const cascadedSubagentEvents =
648-
isRunOwnedSubagentTerminalStatus(input.terminal.status) && hasOpenSubagentProjection
652+
isRunOwnedSubagentTerminalStatus(input.terminal.status) &&
653+
!childTurnsContinue(input.terminal) &&
654+
hasOpenSubagentProjection
649655
? yield* cascadeTerminalizeRunOwnedSubagents({
650656
run: input.run,
651657
open,
@@ -1011,7 +1017,10 @@ export const layer: Layer.Layer<
10111017
(cause) => new RunExecutionIngestError({ runId: input.run.id, cause }),
10121018
),
10131019
);
1014-
if (isRunOwnedSubagentTerminalStatus(terminal.status)) {
1020+
if (
1021+
isRunOwnedSubagentTerminalStatus(terminal.status) &&
1022+
!childTurnsContinue(terminal)
1023+
) {
10151024
yield* Ref.set(openRunOwnedSubagents, emptyOpenRunOwnedSubagentProjection());
10161025
}
10171026
yield* Ref.set(rootRunFinalized, true);
@@ -1161,7 +1170,11 @@ export const layer: Layer.Layer<
11611170
return false;
11621171
}
11631172
const terminal = yield* Ref.get(terminalEvent);
1164-
if (terminal !== null && terminal.status !== "completed") {
1173+
if (
1174+
terminal !== null &&
1175+
terminal.status !== "completed" &&
1176+
!childTurnsContinue(terminal)
1177+
) {
11651178
return true;
11661179
}
11671180
const childProviderTurns = yield* Ref.get(activeChildProviderTurns);

‎apps/web/src/components/chat/MessagesTimeline.logic.test.ts‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1929,6 +1929,44 @@ describe("deriveMessagesTimelineRows", () => {
19291929
expect(rows.at(-1)).toMatchObject({ kind: "thinking" });
19301930
});
19311931

1932+
it("does not show Thinking once the open run's provider turn has ended", () => {
1933+
const input = {
1934+
timelineEntries: [
1935+
{
1936+
id: "user-entry",
1937+
kind: "message" as const,
1938+
createdAt: "2026-01-01T00:00:00Z",
1939+
message: {
1940+
id: "user" as never,
1941+
role: "user" as const,
1942+
text: "Take over the queue",
1943+
runId: "turn-1" as never,
1944+
createdAt: "2026-01-01T00:00:00Z",
1945+
updatedAt: "2026-01-01T00:00:00Z",
1946+
streaming: false,
1947+
},
1948+
},
1949+
],
1950+
latestRun: {
1951+
runId: "turn-1" as never,
1952+
status: "running" as const,
1953+
startedAt: "2026-01-01T00:00:00Z",
1954+
completedAt: null,
1955+
},
1956+
isWorking: true,
1957+
activeTurnStartedAt: "2026-01-01T00:00:00Z",
1958+
turnDiffSummaries: [],
1959+
supportsConversationRollback: false,
1960+
};
1961+
1962+
expect(deriveMessagesTimelineRows(input).at(-1)?.kind).toBe("thinking");
1963+
const ended = deriveMessagesTimelineRows({
1964+
...input,
1965+
latestRun: { ...input.latestRun, turnEnded: true },
1966+
});
1967+
expect(ended.some((row) => row.kind === "thinking")).toBe(false);
1968+
});
1969+
19321970
it("does not fold the active in-progress turn", () => {
19331971
const rows = deriveMessagesTimelineRows({
19341972
timelineEntries: [

‎apps/web/src/components/chat/MessagesTimeline.logic.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -470,7 +470,7 @@ export interface TimelineDurationMessage {
470470

471471
export type TimelineLatestRun = Pick<
472472
ThreadRunSummary,
473-
"runId" | "status" | "startedAt" | "completedAt"
473+
"runId" | "status" | "startedAt" | "completedAt" | "turnEnded"
474474
>;
475475

476476
const LIVE_ACTIVITY_ROW_ID = "live-activity-row";
@@ -1874,6 +1874,7 @@ export function deriveMessagesTimelineRows(input: {
18741874
}
18751875
if (
18761876
input.isWorking &&
1877+
input.latestRun?.turnEnded !== true &&
18771878
!setupOwnsWorkingSlot &&
18781879
!hasActiveCompaction &&
18791880
(!hasActivityRow || latestToolFailed)

‎packages/client-runtime/src/state/models.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,11 @@ export interface ThreadRunSummary {
4141
readonly startedAt: string | null;
4242
readonly completedAt: string | null;
4343
readonly assistantMessageId: MessageId | null;
44+
/**
45+
* The run is still open, but its provider turn already ended, so nothing is
46+
* generating while the server settles it.
47+
*/
48+
readonly turnEnded?: boolean;
4449
readonly sourcePlanRef?: {
4550
readonly threadId: ThreadId;
4651
readonly planId: PlanId;

‎packages/client-runtime/src/state/threadExecution.test.ts‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ import {
66
ProviderThreadId,
77
ProviderSessionId,
88
ProviderDriverKind,
9+
ProviderTurnId,
10+
RunAttemptId,
911
RunId,
1012
ThreadId,
1113
type OrchestrationV2ExecutionNode,
@@ -191,6 +193,34 @@ describe("thread execution presentation", () => {
191193
expect(threadRuntimeHasInterruptibleRun(runtime)).toBe(true);
192194
});
193195

196+
it("marks an open run whose provider turn already ended", () => {
197+
const attemptId = RunAttemptId.make("attempt-open");
198+
const openRun = { ...run("run-open", 1, "running"), activeAttemptId: attemptId };
199+
const providerTurn = (status: "running" | "failed") => ({
200+
id: ProviderTurnId.make("provider-turn-open"),
201+
providerThreadId: ProviderThreadId.make("provider-thread-open"),
202+
nodeId: NodeId.make("node-open"),
203+
runAttemptId: attemptId,
204+
nativeTurnRef: null,
205+
ordinal: 1,
206+
status,
207+
startedAt: now,
208+
completedAt: status === "running" ? null : now,
209+
});
210+
const projection = (status: "running" | "failed") => ({
211+
...v2Projection,
212+
runs: [openRun],
213+
providerTurns: [providerTurn(status)],
214+
updatedAt: now,
215+
});
216+
217+
expect(deriveThreadActivityRun(projection("running"))?.turnEnded).toBeUndefined();
218+
expect(deriveThreadActivityRun(projection("failed"))).toMatchObject({
219+
status: "running",
220+
turnEnded: true,
221+
});
222+
});
223+
194224
it("presents a held queue as the stopped run instead of queued work", () => {
195225
const interrupted = { ...run("run-interrupted", 1, "interrupted"), completedAt: now };
196226
const held = { ...run("run-held", 2, "queued"), queueHeld: true };

0 commit comments

Comments
 (0)