Skip to content

Commit 9819538

Browse files
committed
fix(server): keep Codex capacity retry startup responsive
1 parent 2395dc9 commit 9819538

2 files changed

Lines changed: 299 additions & 59 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts‎

Lines changed: 201 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2220,6 +2220,9 @@ describe("CodexAdapterV2 post-settle continuation", () => {
22202220
"exhaust",
22212221
"stop",
22222222
"stop-continued",
2223+
"stop-starting",
2224+
"started-first",
2225+
"completed-first",
22232226
"steer",
22242227
"start-fails",
22252228
] as const)("handles terminal Codex capacity failures when retries %s", (outcome) =>
@@ -2277,6 +2280,13 @@ describe("CodexAdapterV2 post-settle continuation", () => {
22772280
nativeTurnId: nativeTurnId(attempt),
22782281
prompt,
22792282
}).slice(-3);
2283+
if (
2284+
outcome === "started-first" ||
2285+
outcome === "stop-starting" ||
2286+
outcome === "completed-first"
2287+
) {
2288+
start.splice(1, 0, ...start.splice(2, 1));
2289+
}
22802290
for (const entry of start) {
22812291
if (entry.type === "expect_outbound") {
22822292
entries.push({
@@ -2298,6 +2308,17 @@ describe("CodexAdapterV2 post-settle continuation", () => {
22982308
},
22992309
});
23002310
} else if (entry.type === "emit_inbound" && entry.label === "turn/start") {
2311+
if (outcome === "completed-first")
2312+
entries.push({
2313+
type: "emit_inbound",
2314+
frame: {
2315+
method: "turn/completed",
2316+
params: {
2317+
threadId: nativeThreadId,
2318+
turn: makeCodexReplayTurn({ id: nativeTurnId(attempt), status: "completed" }),
2319+
},
2320+
},
2321+
});
23012322
entries.push(
23022323
outcome === "start-fails"
23032324
? {
@@ -2310,18 +2331,43 @@ describe("CodexAdapterV2 post-settle continuation", () => {
23102331
: withReplayRequestId(entry, 3 + attempt),
23112332
);
23122333
} else if (outcome !== "start-fails") {
2313-
entries.push(entry);
2334+
entries.push(
2335+
outcome === "stop-starting" && entry.type === "emit_inbound"
2336+
? { ...entry, afterMs: 1_000 }
2337+
: entry,
2338+
);
2339+
if (outcome === "stop-starting") {
2340+
entries.push(
2341+
{
2342+
type: "expect_outbound",
2343+
frame: {
2344+
id: 5,
2345+
method: "turn/interrupt",
2346+
params: { threadId: nativeThreadId, turnId: nativeTurnId(attempt) },
2347+
},
2348+
},
2349+
{ type: "emit_inbound", frame: { id: 5, result: {} } },
2350+
);
2351+
}
23142352
}
23152353
}
23162354
if (outcome === "exhaust") entries.push(...failureEntries(attempt));
2317-
if (outcome === "recover" || outcome === "completion-only") {
2355+
if (
2356+
outcome === "recover" ||
2357+
outcome === "completion-only" ||
2358+
outcome === "started-first" ||
2359+
outcome === "stop-starting"
2360+
) {
23182361
entries.push({
23192362
type: "emit_inbound",
23202363
frame: {
23212364
method: "turn/completed",
23222365
params: {
23232366
threadId: nativeThreadId,
2324-
turn: makeCodexReplayTurn({ id: nativeTurnId(attempt), status: "completed" }),
2367+
turn: makeCodexReplayTurn({
2368+
id: nativeTurnId(attempt),
2369+
status: outcome === "stop-starting" ? "interrupted" : "completed",
2370+
}),
23252371
},
23262372
},
23272373
});
@@ -2365,6 +2411,7 @@ describe("CodexAdapterV2 post-settle continuation", () => {
23652411
}
23662412
const milestones = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>();
23672413
const starts: Array<unknown> = [];
2414+
const retryStarted = yield* Deferred.make<void>();
23682415
const harness = yield* makeCodexReplayHarness(
23692416
makeCodexReplayTranscript({ scenario, entries }),
23702417
(event) =>
@@ -2375,8 +2422,11 @@ describe("CodexAdapterV2 post-settle continuation", () => {
23752422
? Queue.offer(milestones, event)
23762423
: Effect.void,
23772424
(method, params) =>
2378-
Effect.sync(() => {
2379-
if (method === "turn/start") starts.push(params);
2425+
Effect.gen(function* () {
2426+
if (method === "turn/start") {
2427+
starts.push(params);
2428+
if (starts.length === 2) yield* Deferred.succeed(retryStarted, undefined);
2429+
}
23802430
}),
23812431
);
23822432
const turnInput = makeCodexTestTurnInput({
@@ -2417,6 +2467,14 @@ describe("CodexAdapterV2 post-settle continuation", () => {
24172467
.pipe(Effect.forkScoped)
24182468
: undefined;
24192469
yield* TestClock.adjust(Math.min(2_000 * 2 ** (attempt - 1), 30_000));
2470+
if (outcome === "stop-starting") {
2471+
yield* Deferred.await(retryStarted);
2472+
const stopping = yield* harness.runtime
2473+
.interruptTurn({ providerThread: harness.providerThread, providerTurnId })
2474+
.pipe(Effect.forkScoped);
2475+
yield* TestClock.adjust("1 second");
2476+
yield* Fiber.join(stopping);
2477+
}
24202478
if (steering !== undefined) yield* Fiber.join(steering);
24212479
if (outcome === "stop-continued") {
24222480
yield* harness.until(() =>
@@ -2436,14 +2494,25 @@ describe("CodexAdapterV2 post-settle continuation", () => {
24362494
const terminal = harness.terminalEvents()[0];
24372495
assert.equal(
24382496
terminal?.status,
2439-
outcome === "recover" || outcome === "completion-only" || outcome === "steer"
2497+
outcome === "recover" ||
2498+
outcome === "completion-only" ||
2499+
outcome === "steer" ||
2500+
outcome === "started-first" ||
2501+
outcome === "completed-first"
24402502
? "completed"
2441-
: outcome === "stop" || outcome === "stop-continued"
2503+
: outcome === "stop" || outcome === "stop-continued" || outcome === "stop-starting"
24422504
? "interrupted"
24432505
: "failed",
24442506
);
24452507
assert.lengthOf(starts, 1 + retries);
24462508
assert.lengthOf(harness.terminalEvents(), 1);
2509+
assert.lengthOf(
2510+
harness.events.filter(
2511+
(event) =>
2512+
event.type === "provider_turn.updated" && event.providerTurn.status === "running",
2513+
),
2514+
outcome === "start-fails" ? 1 : 1 + retries,
2515+
);
24472516
const retryItems = harness.events.flatMap((event) =>
24482517
event.type === "turn_item.updated" && event.turnItem.type === "error"
24492518
? [event.turnItem]
@@ -5341,6 +5410,131 @@ describe("CodexAdapterV2 post-settle continuation", () => {
53415410
};
53425411
};
53435412

5413+
it.effect.each(["completed", "failed"] as const)(
5414+
"handles a late native child after the capacity retry %s",
5415+
(parentStatus) =>
5416+
Effect.scoped(
5417+
Effect.gen(function* () {
5418+
const retriedTurnId = "native-capacity-recovered-parent";
5419+
const retrying = yield* Deferred.make<void>();
5420+
const harness = yield* makeCodexReplayHarness(
5421+
makeCodexReplayTranscript({
5422+
scenario: "capacity-late-child",
5423+
entries: [
5424+
...lateChildPrefix().spawned,
5425+
{
5426+
type: "emit_inbound",
5427+
frame: {
5428+
method: "turn/completed",
5429+
params: {
5430+
threadId: RESUME_NATIVE_THREAD,
5431+
turn: {
5432+
...makeCodexReplayTurn({ id: RESUME_NATIVE_TURN, status: "failed" }),
5433+
error: {
5434+
message: "Selected model is at capacity.",
5435+
codexErrorInfo: "serverOverloaded",
5436+
},
5437+
},
5438+
},
5439+
},
5440+
},
5441+
{
5442+
type: "expect_outbound",
5443+
frame: {
5444+
id: 4,
5445+
method: "turn/start",
5446+
params: {
5447+
threadId: RESUME_NATIVE_THREAD,
5448+
input: [],
5449+
cwd: "/workspace",
5450+
model: "gpt-5.4",
5451+
approvalPolicy: "never",
5452+
approvalsReviewer: "user",
5453+
sandboxPolicy: { type: "dangerFullAccess" },
5454+
summary: "detailed",
5455+
collaborationMode: { mode: "default", settings: { model: "gpt-5.4" } },
5456+
},
5457+
},
5458+
},
5459+
{
5460+
type: "emit_inbound",
5461+
frame: {
5462+
id: 4,
5463+
result: {
5464+
turn: makeCodexReplayTurn({ id: retriedTurnId, status: "inProgress" }),
5465+
},
5466+
},
5467+
},
5468+
{
5469+
type: "emit_inbound",
5470+
frame: {
5471+
method: "turn/started",
5472+
params: {
5473+
threadId: RESUME_NATIVE_THREAD,
5474+
turn: makeCodexReplayTurn({ id: retriedTurnId, status: "inProgress" }),
5475+
},
5476+
},
5477+
},
5478+
{
5479+
type: "emit_inbound",
5480+
frame: {
5481+
method: "turn/completed",
5482+
params: {
5483+
threadId: RESUME_NATIVE_THREAD,
5484+
turn: {
5485+
...makeCodexReplayTurn({ id: retriedTurnId, status: parentStatus }),
5486+
error:
5487+
parentStatus === "failed"
5488+
? { message: "Recovery failed.", codexErrorInfo: "internalServerError" }
5489+
: null,
5490+
},
5491+
},
5492+
},
5493+
},
5494+
childAgentMessage({
5495+
id: "child-after-capacity",
5496+
text: "CAPACITY_CHILD_DONE",
5497+
turnId: RESUME_CHILD_TURN_1,
5498+
completedAtMs: 1782622480000,
5499+
afterMs: 60_000,
5500+
}),
5501+
childTurnCompleted(RESUME_CHILD_TURN_1),
5502+
],
5503+
}),
5504+
(event) =>
5505+
event.type === "turn_item.updated" &&
5506+
event.turnItem.type === "error" &&
5507+
event.turnItem.status === "running"
5508+
? Deferred.succeed(retrying, undefined)
5509+
: Effect.void,
5510+
);
5511+
yield* harness.runtime.startTurn(
5512+
makeCodexTestTurnInput({
5513+
threadId: harness.threadId,
5514+
providerThread: harness.providerThread,
5515+
now: yield* DateTime.now,
5516+
attemptId: RunAttemptId.make("capacity-late-child"),
5517+
text: RESUME_PROMPT,
5518+
}),
5519+
);
5520+
yield* Deferred.await(retrying);
5521+
yield* TestClock.adjust("2 seconds");
5522+
yield* harness.awaitTerminal(1);
5523+
yield* TestClock.adjust("60 seconds");
5524+
yield* harness.until(
5525+
() => harness.subagentUpdates().at(-1)?.subagent.status === "completed",
5526+
);
5527+
assert.lengthOf(harness.continuationRequests, parentStatus === "completed" ? 1 : 0);
5528+
if (parentStatus === "completed")
5529+
assert.include(harness.continuationRequests[0]?.detail, "CAPACITY_CHILD_DONE");
5530+
}).pipe(
5531+
Effect.provide(
5532+
Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer),
5533+
),
5534+
),
5535+
),
5536+
);
5537+
53445538
it.effect("names a late child spawned without a prompt in its wake", () =>
53455539
Effect.scoped(
53465540
Effect.gen(function* () {

0 commit comments

Comments
 (0)