Skip to content

Commit 02a2479

Browse files
committed
fix(server): retry Codex capacity failures with empty input
- Continue failed overloaded turns from persisted history, with bounded backoff - Keep retries interruptible and preserve one user message across continuations
1 parent b6994c5 commit 02a2479

2 files changed

Lines changed: 569 additions & 61 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts‎

Lines changed: 334 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2214,6 +2214,340 @@ describe("CodexAdapterV2 post-settle continuation", () => {
22142214
),
22152215
);
22162216

2217+
it.effect.each([
2218+
"recover",
2219+
"completion-only",
2220+
"exhaust",
2221+
"stop",
2222+
"stop-continued",
2223+
"steer",
2224+
"start-fails",
2225+
] as const)("handles terminal Codex capacity failures when retries %s", (outcome) =>
2226+
Effect.scoped(
2227+
Effect.gen(function* () {
2228+
const scenario = `capacity-${outcome}`;
2229+
const nativeThreadId = `native-${scenario}`;
2230+
const nativeTurnId = (attempt: number) => `${nativeThreadId}-${attempt}`;
2231+
const prompt = "Continue the task.";
2232+
const capacityError = {
2233+
message: "Selected model is at capacity. Please try a different model.",
2234+
codexErrorInfo: "serverOverloaded",
2235+
};
2236+
const failureEntries = (attempt: number): Array<CodexReplay.CodexAppServerReplayEntry> => [
2237+
{
2238+
type: "emit_inbound",
2239+
frame: {
2240+
method: "error",
2241+
params: {
2242+
threadId: nativeThreadId,
2243+
turnId: nativeTurnId(attempt),
2244+
willRetry: false,
2245+
error: capacityError,
2246+
},
2247+
},
2248+
},
2249+
{
2250+
type: "emit_inbound",
2251+
frame: {
2252+
method: "turn/completed",
2253+
params: {
2254+
threadId: nativeThreadId,
2255+
turn: {
2256+
...makeCodexReplayTurn({ id: nativeTurnId(attempt), status: "failed" }),
2257+
error: capacityError,
2258+
},
2259+
},
2260+
},
2261+
},
2262+
];
2263+
const entries = codexReplayPreamble({
2264+
nativeThreadId,
2265+
nativeTurnId: nativeTurnId(0),
2266+
prompt,
2267+
});
2268+
entries.push(
2269+
...(outcome === "completion-only" ? failureEntries(0).slice(1) : failureEntries(0)),
2270+
);
2271+
if (outcome === "completion-only") entries.push(...failureEntries(0).slice(1));
2272+
const retries = outcome === "exhaust" ? 5 : outcome === "stop" ? 0 : 1;
2273+
const steeredText = "Use the smaller change.";
2274+
for (let attempt = 1; attempt <= retries; attempt++) {
2275+
const start = codexReplayPreamble({
2276+
nativeThreadId,
2277+
nativeTurnId: nativeTurnId(attempt),
2278+
prompt,
2279+
}).slice(-3);
2280+
for (const entry of start) {
2281+
if (entry.type === "expect_outbound") {
2282+
entries.push({
2283+
...entry,
2284+
frame: {
2285+
id: 3 + attempt,
2286+
method: "turn/start",
2287+
params: {
2288+
threadId: nativeThreadId,
2289+
input: [],
2290+
cwd: "/workspace",
2291+
model: "gpt-5.4",
2292+
approvalPolicy: "never",
2293+
approvalsReviewer: "user",
2294+
sandboxPolicy: { type: "dangerFullAccess" },
2295+
summary: "detailed",
2296+
collaborationMode: { mode: "default", settings: { model: "gpt-5.4" } },
2297+
},
2298+
},
2299+
});
2300+
} else if (entry.type === "emit_inbound" && entry.label === "turn/start") {
2301+
entries.push(
2302+
outcome === "start-fails"
2303+
? {
2304+
type: "emit_inbound",
2305+
frame: {
2306+
id: 3 + attempt,
2307+
error: { code: -32603, message: "Start failed" },
2308+
},
2309+
}
2310+
: withReplayRequestId(entry, 3 + attempt),
2311+
);
2312+
} else if (outcome !== "start-fails") {
2313+
entries.push(entry);
2314+
}
2315+
}
2316+
if (outcome === "exhaust") entries.push(...failureEntries(attempt));
2317+
if (outcome === "recover" || outcome === "completion-only") {
2318+
entries.push({
2319+
type: "emit_inbound",
2320+
frame: {
2321+
method: "turn/completed",
2322+
params: {
2323+
threadId: nativeThreadId,
2324+
turn: makeCodexReplayTurn({ id: nativeTurnId(attempt), status: "completed" }),
2325+
},
2326+
},
2327+
});
2328+
}
2329+
if (outcome === "stop-continued" || outcome === "steer") {
2330+
const steering = outcome === "steer";
2331+
entries.push(
2332+
{
2333+
type: "expect_outbound",
2334+
frame: {
2335+
id: 5,
2336+
method: steering ? "turn/steer" : "turn/interrupt",
2337+
params: steering
2338+
? {
2339+
threadId: nativeThreadId,
2340+
expectedTurnId: nativeTurnId(attempt),
2341+
input: [{ type: "text", text: steeredText }],
2342+
}
2343+
: { threadId: nativeThreadId, turnId: nativeTurnId(attempt) },
2344+
},
2345+
},
2346+
{
2347+
type: "emit_inbound",
2348+
frame: { id: 5, result: steering ? { turnId: nativeTurnId(attempt) } : {} },
2349+
},
2350+
{
2351+
type: "emit_inbound",
2352+
frame: {
2353+
method: "turn/completed",
2354+
params: {
2355+
threadId: nativeThreadId,
2356+
turn: makeCodexReplayTurn({
2357+
id: nativeTurnId(attempt),
2358+
status: steering ? "completed" : "interrupted",
2359+
}),
2360+
},
2361+
},
2362+
},
2363+
);
2364+
}
2365+
}
2366+
const milestones = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>();
2367+
const starts: Array<unknown> = [];
2368+
const harness = yield* makeCodexReplayHarness(
2369+
makeCodexReplayTranscript({ scenario, entries }),
2370+
(event) =>
2371+
event.type === "turn.terminal" ||
2372+
(event.type === "turn_item.updated" &&
2373+
event.turnItem.type === "error" &&
2374+
event.turnItem.status === "running")
2375+
? Queue.offer(milestones, event)
2376+
: Effect.void,
2377+
(method, params) =>
2378+
Effect.sync(() => {
2379+
if (method === "turn/start") starts.push(params);
2380+
}),
2381+
);
2382+
const turnInput = makeCodexTestTurnInput({
2383+
threadId: harness.threadId,
2384+
providerThread: harness.providerThread,
2385+
now: yield* DateTime.now,
2386+
attemptId: RunAttemptId.make(scenario),
2387+
text: prompt,
2388+
});
2389+
yield* harness.runtime.startTurn(turnInput);
2390+
for (let attempt = 1; attempt <= Math.max(retries, 1); attempt++) {
2391+
const milestone = yield* Queue.take(milestones);
2392+
assert.equal(milestone.type, "turn_item.updated");
2393+
if (milestone.type !== "turn_item.updated" || milestone.turnItem.type !== "error") return;
2394+
const providerTurnId = milestone.turnItem.providerTurnId;
2395+
assert.isNotNull(providerTurnId);
2396+
if (providerTurnId === null) return;
2397+
assert.deepEqual(milestone.turnItem.retry, {
2398+
attempt,
2399+
maxAttempts: 5,
2400+
retryDelayMs: Math.min(2_000 * 2 ** (attempt - 1), 30_000),
2401+
});
2402+
assert.isEmpty(harness.terminalEvents());
2403+
if (outcome === "stop") {
2404+
yield* harness.runtime.interruptTurn({
2405+
providerThread: harness.providerThread,
2406+
providerTurnId,
2407+
});
2408+
}
2409+
const steering =
2410+
outcome === "steer"
2411+
? yield* harness.runtime
2412+
.steerTurn({
2413+
...turnInput,
2414+
providerTurnId,
2415+
message: { ...turnInput.message, text: steeredText },
2416+
})
2417+
.pipe(Effect.forkScoped)
2418+
: undefined;
2419+
yield* TestClock.adjust(Math.min(2_000 * 2 ** (attempt - 1), 30_000));
2420+
if (steering !== undefined) yield* Fiber.join(steering);
2421+
if (outcome === "stop-continued") {
2422+
yield* harness.until(() =>
2423+
harness.events.some(
2424+
(event) =>
2425+
event.type === "provider_turn.updated" &&
2426+
event.providerTurn.nativeTurnRef?.nativeId === nativeTurnId(1),
2427+
),
2428+
);
2429+
yield* harness.runtime.interruptTurn({
2430+
providerThread: harness.providerThread,
2431+
providerTurnId,
2432+
});
2433+
}
2434+
}
2435+
yield* harness.awaitTerminal(1);
2436+
const terminal = harness.terminalEvents()[0];
2437+
assert.equal(
2438+
terminal?.status,
2439+
outcome === "recover" || outcome === "completion-only" || outcome === "steer"
2440+
? "completed"
2441+
: outcome === "stop" || outcome === "stop-continued"
2442+
? "interrupted"
2443+
: "failed",
2444+
);
2445+
assert.lengthOf(starts, 1 + retries);
2446+
assert.lengthOf(harness.terminalEvents(), 1);
2447+
const retryItems = harness.events.flatMap((event) =>
2448+
event.type === "turn_item.updated" && event.turnItem.type === "error"
2449+
? [event.turnItem]
2450+
: [],
2451+
);
2452+
assert.isTrue(
2453+
["completed", "failed", "interrupted"].includes(retryItems.at(-1)?.status ?? ""),
2454+
);
2455+
assert.lengthOf(new Set(retryItems.map((item) => item.id)), 1);
2456+
2457+
if (outcome === "recover") {
2458+
const fs = yield* FileSystem.FileSystem;
2459+
const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-capacity-retry-" });
2460+
const localTranscript = yield* decodeReplayTranscriptJson(
2461+
(yield* encodeReplayTranscriptJson(
2462+
makeCodexReplayTranscript({ scenario, entries }),
2463+
)).replaceAll(yield* encodeStringJson("/workspace"), yield* encodeStringJson(cwd)),
2464+
);
2465+
const driver = yield* CodexReplay.makeReplayDriver(localTranscript);
2466+
yield* Effect.gen(function* () {
2467+
const orchestrator = yield* Orchestrator.OrchestratorV2;
2468+
const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
2469+
const threadId = ThreadId.make("thread:capacity-retry");
2470+
yield* orchestrator.dispatch({
2471+
type: "thread.create",
2472+
commandId: CommandId.make("create-capacity-retry"),
2473+
threadId,
2474+
projectId: ProjectId.make("project:capacity-retry"),
2475+
title: "Capacity retry",
2476+
modelSelection: CODEX_TEST_MODEL_SELECTION,
2477+
runtimeMode: "full-access",
2478+
interactionMode: "default",
2479+
branch: null,
2480+
worktreePath: cwd,
2481+
createdBy: "user",
2482+
creationSource: "web",
2483+
});
2484+
const retrying = yield* orchestrator.streamDomainEvents.pipe(
2485+
Stream.filter(
2486+
(event) =>
2487+
event.type === "turn-item.updated" &&
2488+
event.payload.type === "error" &&
2489+
event.payload.retry != null &&
2490+
event.payload.status === "running",
2491+
),
2492+
Stream.runHead,
2493+
Effect.forkChild({ startImmediately: true }),
2494+
);
2495+
yield* orchestrator.dispatch({
2496+
type: "message.dispatch",
2497+
commandId: CommandId.make("start-capacity-retry"),
2498+
threadId,
2499+
messageId: MessageId.make("message:capacity-retry"),
2500+
text: prompt,
2501+
attachments: [],
2502+
createdBy: "user",
2503+
creationSource: "web",
2504+
dispatchMode: { type: "start_immediately" },
2505+
});
2506+
yield* worker.drain();
2507+
yield* Fiber.join(retrying);
2508+
assert.equal(
2509+
(yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)?.status,
2510+
"running",
2511+
);
2512+
const completed = yield* orchestrator.streamDomainEvents.pipe(
2513+
Stream.filter(
2514+
(event) => event.type === "run.updated" && event.payload.status === "waiting",
2515+
),
2516+
Stream.runHead,
2517+
Effect.forkChild({ startImmediately: true }),
2518+
);
2519+
yield* TestClock.adjust("2 seconds");
2520+
yield* Fiber.join(completed);
2521+
yield* worker.drain();
2522+
const projection = yield* orchestrator.getThreadProjection(threadId);
2523+
assert.equal(projection.runs.at(-1)?.status, "completed");
2524+
assert.deepEqual(
2525+
projection.providerTurns.map((turn) => turn.status),
2526+
["failed", "completed"],
2527+
);
2528+
assert.lengthOf(
2529+
projection.messages.filter((message) => message.role === "user"),
2530+
1,
2531+
);
2532+
assert.isNull((yield* Ref.get(driver.state)).failure);
2533+
}).pipe(
2534+
Effect.provide(
2535+
ProviderReplayHarness.layerWithRegistry(
2536+
{ name: "codex-capacity-retry", runtimePolicyOverride: { cwd } },
2537+
CodexAdapterV2Testkit.layer({ transcript: localTranscript, driver }),
2538+
{ runEffectWorker: false },
2539+
),
2540+
),
2541+
);
2542+
}
2543+
}).pipe(
2544+
Effect.provide(
2545+
Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer),
2546+
),
2547+
),
2548+
),
2549+
);
2550+
22172551
it.effect("resolves retryable app-server errors on resumed provider activity", () =>
22182552
Effect.scoped(
22192553
Effect.gen(function* () {

0 commit comments

Comments
 (0)