Skip to content

Commit 88f6113

Browse files
fix(server): keep a gateway's Claude reset time on a direct 429 (#25438)
Co-authored-by: maria-rcks <[email protected]>
1 parent 01a93e7 commit 88f6113

2 files changed

Lines changed: 90 additions & 19 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6522,6 +6522,59 @@ describe("ClaudeAdapterV2 background wake turns", () => {
65226522
),
65236523
);
65246524

6525+
it.effect.each([
6526+
{ cooldown: " (reset after 1h2m3s)", seconds: 3723 },
6527+
{ cooldown: " (reset after 4m 6s) (last error: rate limited)", seconds: 246 },
6528+
{ cooldown: "", seconds: 0 },
6529+
])("keeps a gateway's reset on a direct 429 with cooldown '$cooldown'", (input) =>
6530+
Effect.gen(function* () {
6531+
const harness = yield* makeWakeHarness;
6532+
const now = yield* DateTime.now;
6533+
yield* harness.runtime.startTurn(
6534+
makeClaudeTestTurnInput({
6535+
threadId: harness.threadId,
6536+
providerThread: harness.providerThread,
6537+
now,
6538+
attemptId: RunAttemptId.make("attempt-direct-gateway-limit"),
6539+
text: "Continue.",
6540+
attachments: [],
6541+
}),
6542+
);
6543+
// An API-key session reports a gateway 429 only as text, with no rate_limit_info.
6544+
const result = `API Error: Request rejected (429) · All credentials for model claude-opus-5-5 are cooling down${input.cooldown}`;
6545+
yield* Queue.offerAll(harness.sdkMessages, [
6546+
makeAssistantErrorFrame({
6547+
uuid: "00000000-0000-4000-8000-000000000654",
6548+
error: "rate_limit",
6549+
text: result,
6550+
}),
6551+
makeResultFrame({
6552+
uuid: "00000000-0000-4000-8000-000000000655",
6553+
result,
6554+
terminalReason: "api_error",
6555+
isError: true,
6556+
apiErrorStatus: 429,
6557+
}),
6558+
]);
6559+
const terminal = yield* Queue.take(harness.terminalReceipts);
6560+
assert.equal(terminal.status, "failed");
6561+
if (terminal.status !== "failed") return;
6562+
assert.equal(terminal.failure.class, "usage_limit");
6563+
assert.equal(
6564+
terminal.failure.resetAt,
6565+
input.seconds > 0
6566+
? DateTime.formatIso(
6567+
DateTime.makeUnsafe(DateTime.toEpochMillis(now) + input.seconds * 1_000),
6568+
)
6569+
: null,
6570+
);
6571+
}).pipe(
6572+
Effect.provide(
6573+
Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer),
6574+
),
6575+
),
6576+
);
6577+
65256578
it.effect("keeps the measured context when a usage limit ends the turn", () =>
65266579
Effect.gen(function* () {
65276580
const harness = yield* makeWakeHarness;

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts‎

Lines changed: 37 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -3760,37 +3760,46 @@ function claudeUsageLimitResetAt(resetsAt: unknown): string | null {
37603760
: null;
37613761
}
37623762

3763+
function gatewayCooldownResetAt(
3764+
cooldown: RegExpExecArray | null,
3765+
completedAt: DateTime.Utc,
3766+
): string | undefined {
3767+
if (cooldown === null) return undefined;
3768+
const seconds =
3769+
Number(cooldown[1] ?? 0) * 3_600 + Number(cooldown[2] ?? 0) * 60 + Number(cooldown[3] ?? 0);
3770+
const resetMs = DateTime.toEpochMillis(completedAt) + seconds * 1_000;
3771+
return seconds > 0 && resetMs < 8.64e15
3772+
? DateTime.formatIso(DateTime.makeUnsafe(resetMs))
3773+
: undefined;
3774+
}
3775+
37633776
function providerFailureFromResult(
37643777
message: SDKResultMessage,
37653778
completedAt: DateTime.Utc,
37663779
failureHint?: string,
37673780
usageLimited = false,
37683781
): OrchestrationV2ProviderFailure | null {
3769-
const gatewayLimit =
3782+
const gatewayResetAt =
37703783
failureHint === undefined &&
37713784
message.subtype === "success" &&
37723785
message.is_error &&
37733786
message.terminal_reason === "api_error" &&
37743787
message.api_error_status === 503
3775-
? /^API Error: 503 \[[^\]\r\n]+\] \[429\]: \{"type":"error","error":\{"type":"rate_limit_error",[^\r\n]*\} \(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec(
3776-
message.result,
3788+
? gatewayCooldownResetAt(
3789+
/^API Error: 503 \[[^\]\r\n]+\] \[429\]: \{"type":"error","error":\{"type":"rate_limit_error",[^\r\n]*\} \(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec(
3790+
message.result,
3791+
),
3792+
completedAt,
37773793
)
3778-
: null;
3779-
if (gatewayLimit) {
3780-
const seconds =
3781-
Number(gatewayLimit[1] ?? 0) * 3_600 +
3782-
Number(gatewayLimit[2] ?? 0) * 60 +
3783-
Number(gatewayLimit[3] ?? 0);
3784-
const resetMs = DateTime.toEpochMillis(completedAt) + seconds * 1_000;
3785-
if (seconds > 0 && resetMs < 8.64e15) {
3786-
return makeProviderFailure({
3787-
class: "usage_limit",
3788-
message: "Claude API rate limit reached. Try again later.",
3789-
code: "api_error_429",
3790-
retryable: true,
3791-
resetAt: DateTime.formatIso(DateTime.makeUnsafe(resetMs)),
3792-
});
3793-
}
3794+
: undefined;
3795+
if (gatewayResetAt !== undefined) {
3796+
return makeProviderFailure({
3797+
class: "usage_limit",
3798+
message: "Claude API rate limit reached. Try again later.",
3799+
code: "api_error_429",
3800+
retryable: true,
3801+
resetAt: gatewayResetAt,
3802+
});
37943803
}
37953804
const failureClass =
37963805
(message.subtype === "success" && message.api_error_status === 429) || usageLimited
@@ -3819,6 +3828,14 @@ function providerFailureFromResult(
38193828
return null;
38203829
}
38213830
const apiErrorStatus = message.api_error_status ?? null;
3831+
// API-key sessions get no rate_limit_info, so a gateway's 429 carries its reset only as text.
3832+
const resetAt =
3833+
apiErrorStatus === 429 && message.result.startsWith("API Error:")
3834+
? gatewayCooldownResetAt(
3835+
/\(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec(message.result),
3836+
completedAt,
3837+
)
3838+
: undefined;
38223839
return makeProviderFailure({
38233840
message: listedError ?? structuredError ?? failureHint ?? message.result,
38243841
code:
@@ -3827,6 +3844,7 @@ function providerFailureFromResult(
38273844
: `api_error_${apiErrorStatus}`,
38283845
class: failureClass,
38293846
retryable: apiErrorStatus === 429 || apiErrorStatus === 529 ? true : null,
3847+
resetAt: resetAt ?? null,
38303848
});
38313849
}
38323850

0 commit comments

Comments
 (0)