Skip to content

Commit 6528a5f

Browse files
ThallesPclaude
andcommitted
fix(server): auto resume Claude gateway 429s that report a reset time
Gateways that pool Claude accounts, such as CLIProxyAPI, answer a usage limit with a plain 429. The Claude CLI drops Retry-After and emits no rate_limit_event for API-key auth, so the run failed with no resetAt and the limit banner could not schedule a resume. Read the gateway's "(reset after <duration>)" cooldown from 429 results too, sharing the parser with the 9router 503 case. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent d6fb62a commit 6528a5f

2 files changed

Lines changed: 81 additions & 20 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5526,6 +5526,60 @@ describe("ClaudeAdapterV2 background wake turns", () => {
55265526
),
55275527
);
55285528

5529+
it.effect.each([
5530+
{ suffix: " (reset after 2h52m31s)", seconds: 10351 },
5531+
{ suffix: " (reset after 72h0m0s)", seconds: 259200 },
5532+
{ suffix: " (reset after 0s)", seconds: 0 },
5533+
{ suffix: "", seconds: 0 },
5534+
])("reads a gateway cooldown from a 429 result$suffix", (input) =>
5535+
Effect.gen(function* () {
5536+
const harness = yield* makeWakeHarness;
5537+
const now = yield* DateTime.now;
5538+
yield* harness.runtime.startTurn(
5539+
makeClaudeTestTurnInput({
5540+
threadId: harness.threadId,
5541+
providerThread: harness.providerThread,
5542+
now,
5543+
attemptId: RunAttemptId.make("attempt-gateway-429"),
5544+
text: "Continue.",
5545+
attachments: [],
5546+
}),
5547+
);
5548+
// CLIProxyAPI's answer once every pooled account is cooling down.
5549+
const text = `API Error: Request rejected (429) · All credentials for model claude-opus-5-5 are cooling down (last error: rate_limit_error: This request would exceed your account's rate limit. Please try again later.)${input.suffix}`;
5550+
yield* Queue.offerAll(harness.sdkMessages, [
5551+
makeAssistantErrorFrame({
5552+
uuid: "00000000-0000-4000-8000-000000000654",
5553+
error: "rate_limit",
5554+
text,
5555+
}),
5556+
makeResultFrame({
5557+
uuid: "00000000-0000-4000-8000-000000000655",
5558+
result: text,
5559+
terminalReason: "api_error",
5560+
isError: true,
5561+
apiErrorStatus: 429,
5562+
}),
5563+
]);
5564+
const terminal = yield* Queue.take(harness.terminalReceipts);
5565+
assert.equal(terminal.status, "failed");
5566+
if (terminal.status !== "failed") return;
5567+
assert.equal(terminal.failure.class, "usage_limit");
5568+
assert.equal(
5569+
terminal.failure.resetAt,
5570+
input.seconds > 0
5571+
? DateTime.formatIso(
5572+
DateTime.makeUnsafe(DateTime.toEpochMillis(now) + input.seconds * 1_000),
5573+
)
5574+
: null,
5575+
);
5576+
}).pipe(
5577+
Effect.provide(
5578+
Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer),
5579+
),
5580+
),
5581+
);
5582+
55295583
it.effect("keeps the measured context when a usage limit ends the turn", () =>
55305584
Effect.gen(function* () {
55315585
const harness = yield* makeWakeHarness;

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts‎

Lines changed: 27 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -3516,37 +3516,43 @@ function claudeUsageLimitResetAt(resetsAt: unknown): string | null {
35163516
: null;
35173517
}
35183518

3519+
// Gateways that pool Claude accounts (9router, CLIProxyAPI) answer a limit with their own
3520+
// cooldown in the error text, e.g. "(reset after 1h 2m 3s)", since the CLI drops Retry-After.
3521+
function gatewayCooldownResetAt(text: string, completedAt: DateTime.Utc): string | null {
3522+
const cooldown = /\(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec(text);
3523+
if (!cooldown) return null;
3524+
const seconds =
3525+
Number(cooldown[1] ?? 0) * 3_600 + Number(cooldown[2] ?? 0) * 60 + Number(cooldown[3] ?? 0);
3526+
const resetMs = DateTime.toEpochMillis(completedAt) + seconds * 1_000;
3527+
return seconds > 0 && resetMs < 8.64e15 ? DateTime.formatIso(DateTime.makeUnsafe(resetMs)) : null;
3528+
}
3529+
35193530
function providerFailureFromResult(
35203531
message: SDKResultMessage,
35213532
completedAt: DateTime.Utc,
35223533
failureHint?: string,
35233534
usageLimited = false,
35243535
): OrchestrationV2ProviderFailure | null {
3525-
const gatewayLimit =
3536+
// 9router wraps the upstream 429 in a 503.
3537+
const wrappedGatewayResetAt =
35263538
failureHint === undefined &&
35273539
message.subtype === "success" &&
35283540
message.is_error &&
35293541
message.terminal_reason === "api_error" &&
3530-
message.api_error_status === 503
3531-
? /^API Error: 503 \[[^\]\r\n]+\] \[429\]: \{"type":"error","error":\{"type":"rate_limit_error",[^\r\n]*\} \(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec(
3532-
message.result,
3533-
)
3542+
message.api_error_status === 503 &&
3543+
/^API Error: 503 \[[^\]\r\n]+\] \[429\]: \{"type":"error","error":\{"type":"rate_limit_error",[^\r\n]*\} \(reset after /.test(
3544+
message.result,
3545+
)
3546+
? gatewayCooldownResetAt(message.result, completedAt)
35343547
: null;
3535-
if (gatewayLimit) {
3536-
const seconds =
3537-
Number(gatewayLimit[1] ?? 0) * 3_600 +
3538-
Number(gatewayLimit[2] ?? 0) * 60 +
3539-
Number(gatewayLimit[3] ?? 0);
3540-
const resetMs = DateTime.toEpochMillis(completedAt) + seconds * 1_000;
3541-
if (seconds > 0 && resetMs < 8.64e15) {
3542-
return makeProviderFailure({
3543-
class: "usage_limit",
3544-
message: "Claude API rate limit reached. Try again later.",
3545-
code: "api_error_429",
3546-
retryable: true,
3547-
resetAt: DateTime.formatIso(DateTime.makeUnsafe(resetMs)),
3548-
});
3549-
}
3548+
if (wrappedGatewayResetAt !== null) {
3549+
return makeProviderFailure({
3550+
class: "usage_limit",
3551+
message: "Claude API rate limit reached. Try again later.",
3552+
code: "api_error_429",
3553+
retryable: true,
3554+
resetAt: wrappedGatewayResetAt,
3555+
});
35503556
}
35513557
const failureClass =
35523558
(message.subtype === "success" && message.api_error_status === 429) || usageLimited
@@ -3583,6 +3589,7 @@ function providerFailureFromResult(
35833589
: `api_error_${apiErrorStatus}`,
35843590
class: failureClass,
35853591
retryable: apiErrorStatus === 429 || apiErrorStatus === 529 ? true : null,
3592+
resetAt: apiErrorStatus === 429 ? gatewayCooldownResetAt(message.result, completedAt) : null,
35863593
});
35873594
}
35883595

0 commit comments

Comments
 (0)