Skip to content

Commit 57e3bdf

Browse files
committed
fix(codex): retry model capacity failures after 30 seconds
1 parent 0bdf33a commit 57e3bdf

25 files changed

Lines changed: 453 additions & 209 deletions

‎apps/mobile/src/features/threads/UsageLimitRecoveryCard.tsx‎

Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ export function UsageLimitRecoveryCard({
2121
const [pending, setPending] = useState(false);
2222
const [error, setError] = useState<string | null>(null);
2323
const resetAt = thread.runtime?.usageLimitResetAt ?? null;
24+
const modelAtCapacity = thread.runtime?.lastErrorClass === "model_capacity";
2425
const canSchedule =
2526
resetAt !== null &&
2627
Date.parse(resetAt) > Date.parse(thread.latestRun?.completedAt ?? thread.updatedAt);
@@ -49,7 +50,8 @@ export function UsageLimitRecoveryCard({
4950
recovery?.runId === runId && recovery?.resetAt === resetAt && recovery?.autoResume;
5051
if (
5152
thread.runtime?.status !== "failed" ||
52-
thread.runtime.lastErrorClass !== "usage_limit" ||
53+
(thread.runtime.lastErrorClass !== "usage_limit" &&
54+
thread.runtime.lastErrorClass !== "model_capacity") ||
5355
!runId
5456
)
5557
return null;
@@ -88,24 +90,36 @@ export function UsageLimitRecoveryCard({
8890
return (
8991
<View className="mx-3 mb-2 gap-2 rounded-xl border border-warning-foreground/25 bg-screen p-3">
9092
<Text className="text-sm text-warning-foreground">
91-
{resetAt
92-
? `Usage limit resets ${DateTime.toDateUtc(DateTime.makeUnsafe(resetAt)).toLocaleString()}.`
93-
: "The provider did not report a reset time. Retry manually when your limit is available."}
93+
{modelAtCapacity && resetAt
94+
? `Model at capacity. ${scheduled ? "Retrying" : "Retry available"} at ${DateTime.toDateUtc(DateTime.makeUnsafe(resetAt)).toLocaleTimeString()}.`
95+
: resetAt
96+
? `Usage limit resets ${DateTime.toDateUtc(DateTime.makeUnsafe(resetAt)).toLocaleString()}.`
97+
: "The provider did not report a reset time. Retry manually when your limit is available."}
9498
</Text>
9599
{canSchedule ? (
96100
<View className="flex-row flex-wrap gap-2">
97101
<Button
98-
label={scheduled ? "Cancel auto-resume" : "Resume at reset"}
102+
label={
103+
scheduled
104+
? modelAtCapacity
105+
? "Cancel auto-retry"
106+
: "Cancel auto-resume"
107+
: modelAtCapacity
108+
? "Schedule retry"
109+
: "Resume at reset"
110+
}
99111
tone="secondary"
100112
disabled={pending}
101113
onPress={() => void toggle("resume")}
102114
/>
103-
<Button
104-
label={snoozed ? "Wake now" : "Snooze until reset"}
105-
tone="secondary"
106-
disabled={pending || (!snoozed && resetPassed)}
107-
onPress={() => void toggle("snooze")}
108-
/>
115+
{!modelAtCapacity ? (
116+
<Button
117+
label={snoozed ? "Wake now" : "Snooze until reset"}
118+
tone="secondary"
119+
disabled={pending || (!snoozed && resetPassed)}
120+
onPress={() => void toggle("snooze")}
121+
/>
122+
) : null}
109123
</View>
110124
) : null}
111125
{error ? (

‎apps/mobile/src/features/threads/thread-work-log.tsx‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -899,7 +899,8 @@ const ThreadWorkLogRow = function ThreadWorkLogRow(
899899
);
900900
const failureItem = row.projectedItem.item;
901901
if (failureItem.type === "error" && failureItem.status === "failed") {
902-
const warning = failureItem.failure.class === "usage_limit";
902+
const warning =
903+
failureItem.failure.class === "usage_limit" || failureItem.failure.class === "model_capacity";
903904
const timestamp = new Date(row.createdAt);
904905
const resetAt = failureItem.failure.resetAt;
905906
const resetTime = resetAt
@@ -1033,7 +1034,8 @@ const ThreadWorkLogRow = function ThreadWorkLogRow(
10331034
const isSystemNotice = row.projectedItem.item.type === "system_notice";
10341035
const isUsageLimit =
10351036
row.projectedItem.item.type === "error" &&
1036-
row.projectedItem.item.failure.class === "usage_limit" &&
1037+
(row.projectedItem.item.failure.class === "usage_limit" ||
1038+
row.projectedItem.item.failure.class === "model_capacity") &&
10371039
row.projectedItem.item.status !== "completed";
10381040
const iconIsDestructive =
10391041
!isSystemNotice && !isUsageLimit && (row.icon === "alert" || row.icon === "warning");

‎apps/mobile/src/features/threads/threadListV2.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,10 @@ export function resolveThreadListV2Status(
195195
return "waiting";
196196
}
197197
if (thread.runtime?.status === "failed") {
198-
return thread.runtime.lastErrorClass === "usage_limit" ? "limited" : "failed";
198+
return thread.runtime.lastErrorClass === "usage_limit" ||
199+
thread.runtime.lastErrorClass === "model_capacity"
200+
? "limited"
201+
: "failed";
199202
}
200203
return "ready";
201204
}

‎apps/mobile/src/lib/threadActivity.test.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1348,7 +1348,7 @@ it("stops stranded thinking after a steer and follows the next thought or tool",
13481348
expect(first.status).toBe("running");
13491349
});
13501350

1351-
it.each(["provider_error", "usage_limit"] as const)(
1351+
it.each(["provider_error", "usage_limit", "model_capacity"] as const)(
13521352
"keeps a historical %s failure and preceding work visible without disclosures",
13531353
(failureClass) => {
13541354
const at = "2026-06-20T00:00:03.000Z";
@@ -1403,6 +1403,8 @@ it.each(["provider_error", "usage_limit"] as const)(
14031403
createdAt: at,
14041404
canExpand: false,
14051405
prominent: true,
1406+
status: failureClass === "provider_error" ? "failure" : "neutral",
1407+
icon: failureClass === "provider_error" ? "alert" : "warning",
14061408
});
14071409
},
14081410
);

‎apps/mobile/src/lib/threadActivity.ts‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -481,7 +481,9 @@ function itemStatus(item: OrchestrationV2TurnItem): ThreadFeedActivity["status"]
481481
if (item.type === "notification") return item.outcome === "failed" ? "failure" : null;
482482
if (item.type === "error") {
483483
if (item.status === "failed")
484-
return item.failure.class === "usage_limit" ? "neutral" : "failure";
484+
return item.failure.class === "usage_limit" || item.failure.class === "model_capacity"
485+
? "neutral"
486+
: "failure";
485487
return item.status === "completed" ? "success" : "neutral";
486488
}
487489
if (!itemIsToolLike(item)) return null;
@@ -578,7 +580,7 @@ function itemIcon(item: OrchestrationV2TurnItem): ThreadFeedActivity["icon"] {
578580
case "system_notice":
579581
return "warning";
580582
case "error":
581-
return item.failure.class === "usage_limit"
583+
return item.failure.class === "usage_limit" || item.failure.class === "model_capacity"
582584
? item.status === "completed"
583585
? "check"
584586
: "warning"
@@ -641,7 +643,11 @@ function itemSummary(
641643
case "run_interrupt_result":
642644
return "Run interrupted";
643645
case "error":
644-
return item.failure.class === "usage_limit" ? "Usage limit reached" : "Provider error";
646+
return item.failure.class === "model_capacity"
647+
? "Model at capacity"
648+
: item.failure.class === "usage_limit"
649+
? "Usage limit reached"
650+
: "Provider error";
645651
case "handoff":
646652
return "Context handed off";
647653
case "fork":
@@ -867,7 +873,8 @@ function toFeedActivity(
867873
toolLike: itemIsToolLike(item),
868874
prominent: itemIsProminent(item) || (item.type === "error" && item.status === "failed"),
869875
status:
870-
item.type === "error" && item.failure.class === "usage_limit"
876+
item.type === "error" &&
877+
(item.failure.class === "usage_limit" || item.failure.class === "model_capacity")
871878
? itemStatus(item)
872879
: workEntryDisplayIndicatesToolFailure(workEntry)
873880
? "failure"

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3877,6 +3877,24 @@ describe("CodexAdapterV2 post-settle continuation", () => {
38773877
);
38783878

38793879
it.effect.each([
3880+
{
3881+
name: "capacity",
3882+
code: "serverOverloaded",
3883+
notification: false,
3884+
expectedClass: "model_capacity",
3885+
},
3886+
{
3887+
name: "capacity-notification",
3888+
code: "serverOverloaded",
3889+
notification: true,
3890+
expectedClass: "model_capacity",
3891+
},
3892+
{
3893+
name: "capacity-message",
3894+
code: "other",
3895+
notification: false,
3896+
expectedClass: "model_capacity",
3897+
},
38803898
{
38813899
name: "usage",
38823900
code: "usageLimitExceeded",
@@ -3927,7 +3945,10 @@ describe("CodexAdapterV2 post-settle continuation", () => {
39273945
Effect.gen(function* () {
39283946
const nativeThreadId = `native-limit-${scenario.name}`;
39293947
const nativeTurnId = `turn-limit-${scenario.name}`;
3930-
const message = "Provider stopped this request.";
3948+
const message =
3949+
scenario.name === "capacity-message"
3950+
? "Selected model is at capacity. Please try a different model."
3951+
: "Provider stopped this request.";
39313952
const errorInfo = "errorInfo" in scenario ? scenario.errorInfo : scenario.code;
39323953
const resetAt = "2033-05-19T07:20:00.000Z";
39333954
const snapshot = {
@@ -4072,6 +4093,7 @@ describe("CodexAdapterV2 post-settle continuation", () => {
40724093
? Deferred.succeed(resetReceipt, undefined)
40734094
: Effect.void,
40744095
);
4096+
const startedAt = yield* DateTime.now;
40754097
yield* harness.runtime.startTurn(
40764098
makeCodexTestTurnInput({
40774099
threadId: harness.threadId,
@@ -4086,6 +4108,13 @@ describe("CodexAdapterV2 post-settle continuation", () => {
40864108
assert.equal(terminal?.status, "failed");
40874109
if (terminal?.status !== "failed") return;
40884110
assert.equal(terminal.failure.class, scenario.expectedClass);
4111+
if (scenario.expectedClass === "model_capacity") {
4112+
assert.equal(terminal.failure.retryable, true);
4113+
assert.equal(
4114+
terminal.failure.resetAt,
4115+
DateTime.formatIso(DateTime.add(startedAt, { seconds: 30 })),
4116+
);
4117+
}
40894118
assert.equal(terminal.threadDisposition, "reusable");
40904119
if (scenario.name === "deferred-reset") assert.equal(terminal.failure.resetAt, resetAt);
40914120
if (scenario.name === "matching-details")

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts‎

Lines changed: 38 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1134,6 +1134,10 @@ function codexUsageLimitError(
11341134
);
11351135
}
11361136

1137+
function codexModelCapacityError(code: string | null, message: string | undefined): boolean {
1138+
return code === "serverOverloaded" || /^Selected model is at capacity\b/iu.test(message ?? "");
1139+
}
1140+
11371141
interface ActiveCodexTurnContext {
11381142
closedCommandItemIds?: ReadonlySet<string>;
11391143
latestProviderFailure?: {
@@ -4566,9 +4570,11 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
45664570
failure: makeProviderFailure({
45674571
message: payload.error.additionalDetails?.trim() || payload.error.message,
45684572
code: notificationCode,
4569-
class: codexUsageLimitError(notificationCode, payload.error.codexErrorInfo)
4570-
? "usage_limit"
4571-
: "provider_error",
4573+
class: codexModelCapacityError(notificationCode, payload.error.message)
4574+
? "model_capacity"
4575+
: codexUsageLimitError(notificationCode, payload.error.codexErrorInfo)
4576+
? "usage_limit"
4577+
: "provider_error",
45724578
}),
45734579
};
45744580
return;
@@ -4589,11 +4595,13 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
45894595
? payload.error.message
45904596
: additionalDetails,
45914597
code,
4592-
class: codexUsageLimitError(code, payload.error.codexErrorInfo)
4593-
? "usage_limit"
4594-
: code?.startsWith("http") === true || code?.startsWith("responseStream") === true
4595-
? "transport_error"
4596-
: "provider_error",
4598+
class: codexModelCapacityError(code, payload.error.message)
4599+
? "model_capacity"
4600+
: codexUsageLimitError(code, payload.error.codexErrorInfo)
4601+
? "usage_limit"
4602+
: code?.startsWith("http") === true || code?.startsWith("responseStream") === true
4603+
? "transport_error"
4604+
: "provider_error",
45974605
retryable: true,
45984606
});
45994607
const itemOrdinal =
@@ -5645,9 +5653,14 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
56455653
input.failureErrorInfo === undefined
56465654
? undefined
56475655
: codexErrorInfoCode(input.failureErrorInfo);
5648-
const failureClass = codexUsageLimitError(failureCode ?? null, input.failureErrorInfo)
5649-
? "usage_limit"
5650-
: "provider_error";
5656+
const failureClass = codexModelCapacityError(
5657+
failureCode ?? null,
5658+
input.failureMessage,
5659+
)
5660+
? "model_capacity"
5661+
: codexUsageLimitError(failureCode ?? null, input.failureErrorInfo)
5662+
? "usage_limit"
5663+
: "provider_error";
56515664
// The notification carries the richer message, so it wins when the
56525665
// completion repeats it. An explicit exhaustion status on the
56535666
// completion still decides the class; a missing one is not evidence.
@@ -5709,17 +5722,26 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
57095722
event: CodexRootTerminalEvent,
57105723
) {
57115724
const current =
5712-
event.status === "failed" && event.failure.class === "usage_limit"
5725+
event.status === "failed" && event.failure.class === "model_capacity"
57135726
? {
57145727
...event,
57155728
failure: {
57165729
...event.failure,
5717-
resetAt:
5718-
event.failure.resetAt ??
5719-
codexUsageLimitResetAt(yield* Ref.get(rateLimitSnapshot)),
5730+
retryable: true,
5731+
resetAt: DateTime.formatIso(DateTime.add(yield* DateTime.now, { seconds: 30 })),
57205732
},
57215733
}
5722-
: event;
5734+
: event.status === "failed" && event.failure.class === "usage_limit"
5735+
? {
5736+
...event,
5737+
failure: {
5738+
...event.failure,
5739+
resetAt:
5740+
event.failure.resetAt ??
5741+
codexUsageLimitResetAt(yield* Ref.get(rateLimitSnapshot)),
5742+
},
5743+
}
5744+
: event;
57235745
yield* emitProviderEvent(current);
57245746
for (const turn of goalRuns.get(context.providerTurnId) ?? []) {
57255747
goalRuns.delete(turn.providerTurnId);

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2978,7 +2978,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
29782978
thread.archivedAt !== null ||
29792979
thread.settledOverride === "settled" ||
29802980
run?.id !== command.limitRecovery.runId ||
2981-
failure?.class !== "usage_limit" ||
2981+
(failure?.class !== "usage_limit" && failure?.class !== "model_capacity") ||
29822982
failure.resetAt !== command.limitRecovery.resetAt ||
29832983
resetMs <= DateTime.toEpochMillis(run.completedAt ?? run.requestedAt) ||
29842984
projection.runtimeRequests.some((request) => request.status === "pending")
@@ -4813,7 +4813,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
48134813
command.dispatchMode.type !== "start_immediately" ||
48144814
source === undefined ||
48154815
(source.status !== "interrupted" &&
4816-
!(source.status === "failed" && limited?.class === "usage_limit")) ||
4816+
!(
4817+
source.status === "failed" &&
4818+
(limited?.class === "usage_limit" || limited?.class === "model_capacity")
4819+
)) ||
48174820
latestExecutedRun(projection.runs)?.id !== source.id ||
48184821
projection.thread.archivedAt !== null ||
48194822
projection.thread.deletedAt !== null ||
@@ -4833,8 +4836,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
48334836
const now = yield* DateTime.now;
48344837
if (
48354838
run?.id !== command.usageLimitContinuationOfRunId ||
4836-
failure?.class !== "usage_limit" ||
4837-
threadShellFromProjection(projection).lastErrorClass !== "usage_limit" ||
4839+
(failure?.class !== "usage_limit" && failure?.class !== "model_capacity") ||
4840+
!["usage_limit", "model_capacity"].includes(
4841+
threadShellFromProjection(projection).lastErrorClass ?? "",
4842+
) ||
48384843
!recovery?.autoResume ||
48394844
recovery.requestId !== command.usageLimitRecoveryRequestId ||
48404845
recovery.runId !== run.id ||
@@ -9151,7 +9156,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
91519156
.pipe(mapDispatchError(input.command));
91529157
const failure = latestRootProviderFailure(latest, turnItems);
91539158
if (
9154-
failure?.class === "usage_limit" &&
9159+
(failure?.class === "usage_limit" || failure?.class === "model_capacity") &&
91559160
failure.resetAt != null &&
91569161
Number.isFinite(Date.parse(failure.resetAt)) &&
91579162
(recovery?.runId !== latest.id || recovery.resetAt !== failure.resetAt)

0 commit comments

Comments
 (0)