Skip to content

Commit 2118005

Browse files
shivamhwpclaude
andcommitted
fix(codex): settle stopped children and retain failed archive cleanup
Settle a rejected late child only on its completion, reopen it on a later native turn, and preserve newer task state against stale completions. Fall back to bounded unload after sweep archive failure while preserving replacement tracking. Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent c47854a commit 2118005

4 files changed

Lines changed: 282 additions & 22 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts‎

Lines changed: 184 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5430,17 +5430,70 @@ describe("CodexAdapterV2 post-settle continuation", () => {
54305430
),
54315431
);
54325432

5433+
const INTERRUPT_PENDING_CHILD_NATIVE_THREAD = "native-codex-interrupt-pending-child-thread";
5434+
const INTERRUPT_RESUMED_ROOT_TURN = "native-codex-interrupt-resumed-root-turn";
5435+
const INTERRUPT_RESUMED_CHILD_TURN = "native-codex-interrupt-resumed-child-turn";
5436+
const INTERRUPT_RESUME_PROMPT = "Continue the stopped child.";
5437+
// Notifications run in order, so the model update fences everything before it.
5438+
const pendingChildFence = (model: string): CodexReplay.CodexAppServerReplayEntry => ({
5439+
type: "emit_inbound",
5440+
frame: {
5441+
method: "thread/settings/updated",
5442+
params: {
5443+
threadId: INTERRUPT_PENDING_CHILD_NATIVE_THREAD,
5444+
threadSettings: {
5445+
model,
5446+
effort: "low",
5447+
serviceTier: null,
5448+
modelProvider: "openai",
5449+
cwd: "/workspace",
5450+
approvalPolicy: "never",
5451+
approvalsReviewer: "auto_review",
5452+
collaborationMode: { mode: "default", settings: { model } },
5453+
sandboxPolicy: { type: "dangerFullAccess" },
5454+
},
5455+
},
5456+
},
5457+
});
5458+
const childSpawnIndex = interruptSubagentCommandTranscript.entries.findIndex(
5459+
(entry) =>
5460+
entry.type === "emit_inbound" && entry.label === "item/completed/subAgentActivity-started",
5461+
);
54335462
const interruptSubagentTurnAfterStopTranscript = makeCodexReplayTranscript({
54345463
scenario: "codex-interrupt-subagent-turn-after-stop",
54355464
entries: [
5436-
...interruptSubagentCommandTranscript.entries.slice(0, -1),
5465+
...interruptSubagentCommandTranscript.entries.slice(0, childSpawnIndex + 1),
5466+
{
5467+
type: "emit_inbound",
5468+
label: "item/completed/collab-spawn-pending-child",
5469+
frame: {
5470+
method: "item/completed",
5471+
params: {
5472+
item: {
5473+
type: "collabAgentToolCall",
5474+
id: "call-codex-interrupt-pending-child",
5475+
tool: "spawnAgent",
5476+
status: "completed",
5477+
senderThreadId: INTERRUPT_NATIVE_THREAD,
5478+
receiverThreadIds: [INTERRUPT_PENDING_CHILD_NATIVE_THREAD],
5479+
agentsStates: {
5480+
[INTERRUPT_PENDING_CHILD_NATIVE_THREAD]: { status: "pendingInit", message: null },
5481+
},
5482+
},
5483+
threadId: INTERRUPT_NATIVE_THREAD,
5484+
turnId: INTERRUPT_NATIVE_TURN,
5485+
completedAtMs: 1782622441200,
5486+
},
5487+
},
5488+
},
5489+
...interruptSubagentCommandTranscript.entries.slice(childSpawnIndex + 1, -1),
54375490
{
54385491
type: "emit_inbound",
54395492
label: "turn/started/child-after-stop",
54405493
frame: {
54415494
method: "turn/started",
54425495
params: {
5443-
threadId: INTERRUPT_CHILD_NATIVE_THREAD,
5496+
threadId: INTERRUPT_PENDING_CHILD_NATIVE_THREAD,
54445497
turn: makeCodexReplayTurn({
54455498
id: INTERRUPT_LATE_CHILD_NATIVE_TURN,
54465499
status: "inProgress",
@@ -5455,16 +5508,19 @@ describe("CodexAdapterV2 post-settle continuation", () => {
54555508
id: 7,
54565509
method: "turn/interrupt",
54575510
params: {
5458-
threadId: INTERRUPT_CHILD_NATIVE_THREAD,
5511+
threadId: INTERRUPT_PENDING_CHILD_NATIVE_THREAD,
54595512
turnId: INTERRUPT_LATE_CHILD_NATIVE_TURN,
54605513
},
54615514
},
54625515
},
5516+
pendingChildFence("gpt-fence-interrupt-requested"),
54635517
{
54645518
type: "emit_inbound",
54655519
label: "turn/interrupt/child-after-stop",
5520+
afterMs: 4_000,
54665521
frame: { id: 7, result: {} },
54675522
},
5523+
pendingChildFence("gpt-fence-interrupt-acked"),
54685524
{
54695525
type: "emit_inbound",
54705526
label: "thread/goal/updated/root",
@@ -5486,6 +5542,82 @@ describe("CodexAdapterV2 post-settle continuation", () => {
54865542
},
54875543
},
54885544
},
5545+
{
5546+
type: "emit_inbound",
5547+
label: "turn/completed/child-after-stop",
5548+
afterMs: 4_000,
5549+
frame: {
5550+
method: "turn/completed",
5551+
params: {
5552+
threadId: INTERRUPT_PENDING_CHILD_NATIVE_THREAD,
5553+
turn: makeCodexReplayTurn({
5554+
id: INTERRUPT_LATE_CHILD_NATIVE_TURN,
5555+
status: "interrupted",
5556+
}),
5557+
},
5558+
},
5559+
},
5560+
pendingChildFence("gpt-fence-late-turn-completed"),
5561+
...codexReplayPreamble({
5562+
nativeThreadId: INTERRUPT_NATIVE_THREAD,
5563+
nativeTurnId: INTERRUPT_RESUMED_ROOT_TURN,
5564+
prompt: INTERRUPT_RESUME_PROMPT,
5565+
})
5566+
.slice(-3)
5567+
.map((entry) =>
5568+
entry.type !== "runtime_exit" &&
5569+
typeof entry.frame === "object" &&
5570+
entry.frame !== null &&
5571+
"id" in entry.frame
5572+
? { ...entry, frame: { ...entry.frame, id: 8 } }
5573+
: entry,
5574+
),
5575+
{
5576+
type: "emit_inbound",
5577+
frame: {
5578+
method: "turn/started",
5579+
params: {
5580+
threadId: INTERRUPT_PENDING_CHILD_NATIVE_THREAD,
5581+
turn: makeCodexReplayTurn({ id: INTERRUPT_RESUMED_CHILD_TURN, status: "inProgress" }),
5582+
},
5583+
},
5584+
},
5585+
pendingChildFence("gpt-fence-child-resumed"),
5586+
{
5587+
type: "emit_inbound",
5588+
frame: {
5589+
method: "turn/completed",
5590+
params: {
5591+
threadId: INTERRUPT_PENDING_CHILD_NATIVE_THREAD,
5592+
turn: makeCodexReplayTurn({
5593+
id: INTERRUPT_LATE_CHILD_NATIVE_TURN,
5594+
status: "interrupted",
5595+
}),
5596+
},
5597+
},
5598+
},
5599+
pendingChildFence("gpt-fence-old-completion"),
5600+
{
5601+
type: "emit_inbound",
5602+
afterMs: 1_000,
5603+
frame: {
5604+
method: "turn/completed",
5605+
params: {
5606+
threadId: INTERRUPT_PENDING_CHILD_NATIVE_THREAD,
5607+
turn: makeCodexReplayTurn({ id: INTERRUPT_RESUMED_CHILD_TURN, status: "completed" }),
5608+
},
5609+
},
5610+
},
5611+
{
5612+
type: "emit_inbound",
5613+
frame: {
5614+
method: "turn/completed",
5615+
params: {
5616+
threadId: INTERRUPT_NATIVE_THREAD,
5617+
turn: makeCodexReplayTurn({ id: INTERRUPT_RESUMED_ROOT_TURN, status: "completed" }),
5618+
},
5619+
},
5620+
},
54895621
{ type: "runtime_exit", status: "success" },
54905622
],
54915623
});
@@ -5532,22 +5664,71 @@ describe("CodexAdapterV2 post-settle continuation", () => {
55325664
});
55335665
yield* harness.awaitTerminal(1);
55345666

5667+
const pendingChildUpdates = () =>
5668+
harness
5669+
.subagentUpdates()
5670+
.filter(
5671+
(event) =>
5672+
event.subagent.nativeTaskRef?.nativeId ===
5673+
`call-codex-interrupt-pending-child:${INTERRUPT_PENDING_CHILD_NATIVE_THREAD}`,
5674+
);
5675+
const fenced = (model: string) =>
5676+
harness.until(() =>
5677+
pendingChildUpdates().some((event) => event.subagent.model === model),
5678+
);
5679+
yield* fenced("gpt-fence-interrupt-requested");
5680+
assert.equal(pendingChildUpdates().at(-1)?.subagent.status, "pending");
5681+
assert.isTrue(yield* harness.hasPendingBackgroundWork);
5682+
55355683
// The TestClock never reaches the interrupt's timeout, so this arrives only
55365684
// if the reader is free to read the interrupt response and the frame after it.
5685+
yield* TestClock.adjust("4 seconds");
5686+
yield* fenced("gpt-fence-interrupt-acked");
55375687
yield* harness.until(() =>
55385688
harness.events.some(
55395689
(event) =>
55405690
event.type === "provider_thread.updated" &&
55415691
event.providerThread.goal?.status === "paused",
55425692
),
55435693
);
5694+
assert.equal(pendingChildUpdates().at(-1)?.subagent.status, "pending");
5695+
assert.isTrue(yield* harness.hasPendingBackgroundWork);
5696+
5697+
yield* TestClock.adjust("4 seconds");
5698+
yield* fenced("gpt-fence-late-turn-completed");
5699+
assert.equal(pendingChildUpdates().at(-1)?.subagent.status, "interrupted");
5700+
assert.isFalse(yield* harness.hasPendingBackgroundWork);
5701+
assert.lengthOf(harness.continuationRequests, 0);
55445702
assert.isFalse(
55455703
harness.events.some(
55465704
(event) =>
55475705
event.type === "provider_turn.updated" &&
55485706
event.providerTurn.nativeTurnRef?.nativeId === INTERRUPT_LATE_CHILD_NATIVE_TURN,
55495707
),
55505708
);
5709+
yield* harness.runtime.startTurn({
5710+
...makeCodexTestTurnInput({
5711+
threadId: harness.threadId,
5712+
providerThread: harness.providerThread,
5713+
now,
5714+
attemptId: RunAttemptId.make("attempt-codex-interrupt-resumed"),
5715+
text: INTERRUPT_RESUME_PROMPT,
5716+
}),
5717+
runOrdinal: 2,
5718+
providerTurnOrdinal: 2,
5719+
});
5720+
yield* fenced("gpt-fence-child-resumed");
5721+
assert.equal(pendingChildUpdates().at(-1)?.subagent.status, "running");
5722+
assert.isNull(pendingChildUpdates().at(-1)?.subagent.completedAt);
5723+
assert.isTrue(yield* harness.hasPendingBackgroundWork);
5724+
yield* fenced("gpt-fence-old-completion");
5725+
assert.equal(pendingChildUpdates().at(-1)?.subagent.status, "running");
5726+
assert.isTrue(yield* harness.hasPendingBackgroundWork);
5727+
yield* TestClock.adjust("1 second");
5728+
yield* harness.awaitTerminal(2);
5729+
assert.equal(pendingChildUpdates().at(-1)?.subagent.status, "completed");
5730+
assert.isFalse(yield* harness.hasPendingBackgroundWork);
5731+
assert.lengthOf(harness.continuationRequests, 0);
55515732
}).pipe(
55525733
Effect.provide(
55535734
Layer.mergeAll(

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2655,6 +2655,9 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
26552655
let ancestor: ActiveCodexTurnContext | undefined = subagent.parentContext;
26562656
while (ancestor !== undefined) {
26572657
if (terminalizedNativeTurns.has(ancestor.nativeTurnId)) {
2658+
yield* Ref.update(terminalizedNonCompletedNativeTurns, (current) =>
2659+
new Set(current).add(turn.nativeTurnId),
2660+
);
26582661
const nativeThreadId = yield* getNativeThreadId(subagent.providerThread);
26592662
// Fork requests off the notification reader so it can read their responses.
26602663
yield* client
@@ -2732,7 +2735,9 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
27322735
subagent,
27332736
status: "running",
27342737
startedAt: turn.startedAt,
2735-
...(providerTurnOrdinal > 1 ? { reopen: true, result: null } : {}),
2738+
...(providerTurnOrdinal > 1 || subagent.nativeTurnIds.size > 1
2739+
? { reopen: true, result: null }
2740+
: {}),
27362741
});
27372742
const now = yield* DateTime.now;
27382743
yield* emitProviderEvent({
@@ -6551,6 +6556,20 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
65516556
Effect.gen(function* () {
65526557
const context = (yield* Ref.get(activeTurns)).get(payload.turn.id);
65536558
if (context === undefined) {
6559+
// A child turn rejected after Stop has no context; its own completion settles the task.
6560+
const subagent = (yield* Ref.get(subagentThreads)).get(payload.threadId);
6561+
if (
6562+
subagent !== undefined &&
6563+
isOrchestrationV2WorkActive(subagent.task.status) &&
6564+
Array.from(subagent.nativeTurnIds).at(-1) === payload.turn.id &&
6565+
(yield* Ref.get(terminalizedNonCompletedNativeTurns)).has(payload.turn.id)
6566+
) {
6567+
yield* emitSubagentTaskUpdate({
6568+
subagent,
6569+
status: "interrupted",
6570+
completedAt: codexTimestamp(payload.turn.completedAt),
6571+
});
6572+
}
65546573
return;
65556574
}
65566575
const nativeStatus = mapCodexTurnStatus(payload.turn.status);

‎apps/server/src/orchestration-v2/ProviderSessionManager.test.ts‎

Lines changed: 41 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -441,6 +441,9 @@ const makeProviderAdapter = Effect.fnUntraced(function* (
441441
readonly startTurn?: Effect.Effect<void>;
442442
readonly interruptTurn?: Effect.Effect<void>;
443443
readonly beforeUnload?: Effect.Effect<void>;
444+
readonly beforeArchive?: (
445+
providerThread: OrchestrationV2ProviderThread,
446+
) => Effect.Effect<void, ProviderAdapter.ProviderAdapterProtocolError>;
444447
readonly spawnBeforeOpen?: boolean;
445448
readonly scopeCloseReached?: Deferred.Deferred<void>;
446449
/** Offers `unload:<native id>`, `archive:<native id>` and `close`; also gives the runtime archiveThread. */
@@ -576,10 +579,15 @@ const makeProviderAdapter = Effect.fnUntraced(function* (
576579
}: {
577580
providerThread: OrchestrationV2ProviderThread;
578581
}) =>
579-
Queue.offer(
580-
options.nativeThreadExits!,
581-
`archive:${providerThread.nativeThreadRef?.nativeId}`,
582-
).pipe(Effect.asVoid),
582+
(options.beforeArchive?.(providerThread) ?? Effect.void).pipe(
583+
Effect.andThen(
584+
Queue.offer(
585+
options.nativeThreadExits!,
586+
`archive:${providerThread.nativeThreadRef?.nativeId}`,
587+
),
588+
),
589+
Effect.asVoid,
590+
),
583591
}),
584592
respondToRuntimeRequest: () => Effect.void,
585593
readThreadSnapshot: () => unimplemented("readThreadSnapshot unused in test"),
@@ -618,6 +626,9 @@ function layerTest(input: {
618626
readonly startTurn?: Effect.Effect<void>;
619627
readonly interruptTurn?: Effect.Effect<void>;
620628
readonly beforeUnload?: Effect.Effect<void>;
629+
readonly beforeArchive?: (
630+
providerThread: OrchestrationV2ProviderThread,
631+
) => Effect.Effect<void, ProviderAdapter.ProviderAdapterProtocolError>;
621632
readonly spawnBeforeOpen?: boolean;
622633
readonly scopeCloseReached?: Deferred.Deferred<void>;
623634
readonly nativeThreadExits?: Queue.Queue<string>;
@@ -655,6 +666,7 @@ function layerTest(input: {
655666
...(input.startTurn === undefined ? {} : { startTurn: input.startTurn }),
656667
...(input.interruptTurn === undefined ? {} : { interruptTurn: input.interruptTurn }),
657668
...(input.beforeUnload === undefined ? {} : { beforeUnload: input.beforeUnload }),
669+
...(input.beforeArchive === undefined ? {} : { beforeArchive: input.beforeArchive }),
658670
...(input.spawnBeforeOpen === undefined ? {} : { spawnBeforeOpen: input.spawnBeforeOpen }),
659671
...(input.scopeCloseReached === undefined
660672
? {}
@@ -3796,9 +3808,9 @@ it.effect(
37963808
}),
37973809
);
37983810

3799-
it.effect(
3800-
"ProviderSessionManagerV2 archives a delegated child only once its parent took its result",
3801-
() =>
3811+
it.effect.each([false, true])(
3812+
"ProviderSessionManagerV2 archives a delegated child only once its parent took its result (archive fails: %s)",
3813+
(archiveFails) =>
38023814
Effect.gen(function* () {
38033815
const state = yield* Ref.make(emptyState);
38043816
const exits = yield* Queue.unbounded<string>();
@@ -4015,15 +4027,32 @@ it.effect(
40154027
});
40164028
yield* endTurn(parentThread);
40174029
yield* TestClock.adjust("1 second");
4018-
// Each child archives once before the session closes.
4019-
assert.deepEqual(
4020-
[yield* Queue.take(exits), yield* Queue.take(exits), yield* Queue.take(exits)],
4021-
["archive:native-delivered", "archive:native-late", "close"],
4030+
// Failed sweep archives fall back to unload before the session closes.
4031+
assert.equal(yield* Queue.take(exits), "archive:native-delivered");
4032+
assert.equal(
4033+
yield* Queue.take(exits),
4034+
archiveFails ? "unload:native-late" : "archive:native-late",
40224035
);
4036+
assert.equal(yield* Queue.take(exits), "close");
40234037
});
40244038

40254039
yield* effect.pipe(
4026-
Effect.provide(layerTest({ state, idleTimeoutMs: 1000, nativeThreadExits: exits })),
4040+
Effect.provide(
4041+
layerTest({
4042+
state,
4043+
idleTimeoutMs: 1000,
4044+
nativeThreadExits: exits,
4045+
beforeArchive: (providerThread) =>
4046+
archiveFails && providerThread.nativeThreadRef?.nativeId === "native-late"
4047+
? Effect.fail(
4048+
new ProviderAdapter.ProviderAdapterProtocolError({
4049+
driver: CODEX_DRIVER,
4050+
detail: "test archive failed",
4051+
}),
4052+
)
4053+
: Effect.void,
4054+
}),
4055+
),
40274056
);
40284057
}),
40294058
);

0 commit comments

Comments
 (0)