Skip to content

Commit df4e823

Browse files
Pick-cataltaywtf
Pick-cat
and
altaywtf
committed
fix(agents): classify provider-native failover errors
Co-authored-by: Altay <[email protected]>
1 parent 59e95fe commit df4e823

9 files changed

Lines changed: 224 additions & 2 deletions

‎extensions/anthropic/index.test.ts‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,48 @@ describe("anthropic provider replay hooks", () => {
117117
).toBe("native");
118118
});
119119

120+
it("classifies Anthropic-native structured failover errors", async () => {
121+
const provider = await registerSingleProviderPlugin(anthropicPlugin);
122+
123+
expect(
124+
provider.classifyFailoverReason?.({
125+
provider: "anthropic",
126+
errorMessage: "",
127+
errorType: "rate_limit_error",
128+
}),
129+
).toBe("rate_limit");
130+
expect(
131+
provider.classifyFailoverReason?.({
132+
provider: "anthropic",
133+
errorMessage: "",
134+
errorType: "api_error",
135+
}),
136+
).toBe("timeout");
137+
expect(
138+
provider.classifyFailoverReason?.({
139+
provider: "anthropic",
140+
errorMessage: "",
141+
errorType: "rate_limit_error",
142+
code: "API_ERROR",
143+
}),
144+
).toBe("rate_limit");
145+
expect(
146+
provider.classifyFailoverReason?.({
147+
provider: "anthropic",
148+
errorMessage: "",
149+
code: "RATE_LIMIT_ERROR",
150+
}),
151+
).toBe("rate_limit");
152+
expect(
153+
provider.classifyFailoverReason?.({
154+
provider: "anthropic",
155+
errorMessage: "",
156+
errorType: "UNKNOWN_ERROR",
157+
code: "INSUFFICIENT_QUOTA",
158+
}),
159+
).toBeUndefined();
160+
});
161+
120162
it("owns replay policy for Claude transports", async () => {
121163
const provider = await registerSingleProviderPlugin(anthropicPlugin);
122164

‎extensions/anthropic/register.runtime.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,18 @@ import { wrapAnthropicProviderStream } from "./stream-wrappers.js";
5858
import { fetchAnthropicUsage, resolveAnthropicUsageAuth } from "./usage.js";
5959

6060
const PROVIDER_ID = "anthropic";
61+
62+
// Anthropic-native error descriptors stay with the Anthropic provider hook.
63+
function classifyAnthropicFailoverDescriptor(value: string | undefined) {
64+
switch (value?.trim().toUpperCase()) {
65+
case "RATE_LIMIT_ERROR":
66+
return "rate_limit" as const;
67+
case "API_ERROR":
68+
return "timeout" as const;
69+
default:
70+
return undefined;
71+
}
72+
}
6173
type UpsertAuthProfileParams = Parameters<typeof upsertAuthProfileWithLock>[0];
6274
const DEFAULT_ANTHROPIC_MODEL = "anthropic/claude-opus-4-8";
6375
const ANTHROPIC_OPUS_48_MODEL_ID = "claude-opus-4-8";
@@ -894,6 +906,11 @@ export function buildAnthropicProvider(): ProviderPlugin {
894906
(!isAnthropicMandatoryClaude5Model(modelId) ||
895907
normalizeLowercaseStringOrEmpty(provider) === PROVIDER_ID),
896908
resolveReasoningOutputMode: () => "native",
909+
classifyFailoverReason: ({ provider, code, errorType }) =>
910+
normalizeLowercaseStringOrEmpty(provider) === PROVIDER_ID
911+
? (classifyAnthropicFailoverDescriptor(errorType) ??
912+
classifyAnthropicFailoverDescriptor(code))
913+
: undefined,
897914
resolveThinkingProfile: ({ provider, modelId, params }) => {
898915
const contractModelId = resolveClaudeModelIdentity({ id: modelId, params });
899916
return isAnthropicMandatoryClaude5Model(contractModelId) &&

‎extensions/google/provider-hooks.test.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import { describe, expect, it } from "vitest";
2+
import { GOOGLE_GEMINI_PROVIDER_HOOKS } from "./provider-hooks.js";
3+
4+
describe("GOOGLE_GEMINI_PROVIDER_HOOKS.classifyFailoverReason", () => {
5+
it.each([
6+
{ provider: "google", code: "UNAVAILABLE", expected: "overloaded" },
7+
{ provider: "google-vertex", code: "DEADLINE_EXCEEDED", expected: "timeout" },
8+
{ provider: "google-antigravity", code: "INTERNAL", expected: "server_error" },
9+
{ provider: "google-gemini-cli", code: "UNAVAILABLE", expected: "overloaded" },
10+
] as const)("classifies $provider $code as $expected", ({ provider, code, expected }) => {
11+
expect(
12+
GOOGLE_GEMINI_PROVIDER_HOOKS.classifyFailoverReason({
13+
provider,
14+
errorMessage: "",
15+
code,
16+
}),
17+
).toBe(expected);
18+
});
19+
20+
it("leaves unknown codes for generic classification", () => {
21+
expect(
22+
GOOGLE_GEMINI_PROVIDER_HOOKS.classifyFailoverReason({
23+
provider: "google-vertex",
24+
errorMessage: "",
25+
code: "INSUFFICIENT_QUOTA",
26+
}),
27+
).toBeUndefined();
28+
});
29+
});

‎extensions/google/provider-hooks.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,26 @@ import type {
33
ProviderDefaultThinkingPolicyContext,
44
ProviderThinkingProfile,
55
} from "openclaw/plugin-sdk/core";
6+
import type { ProviderFailoverErrorContext } from "openclaw/plugin-sdk/plugin-entry";
67
import { buildProviderReplayFamilyHooks } from "openclaw/plugin-sdk/provider-model-shared";
78
import { buildProviderToolCompatFamilyHooks } from "openclaw/plugin-sdk/provider-tools";
89
import { resolveGoogleThinkingProfile } from "./provider-policy.js";
910
import { createGoogleThinkingStreamWrapper } from "./thinking-api.js";
1011

12+
// Google-family gRPC status codes stay with the shared Gemini provider hook.
13+
function classifyGoogleFailoverCode(code: string | undefined) {
14+
switch (code?.trim().toUpperCase()) {
15+
case "UNAVAILABLE":
16+
return "overloaded" as const;
17+
case "DEADLINE_EXCEEDED":
18+
return "timeout" as const;
19+
case "INTERNAL":
20+
return "server_error" as const;
21+
default:
22+
return undefined;
23+
}
24+
}
25+
1126
export const GOOGLE_GEMINI_PROVIDER_HOOKS = {
1227
...buildProviderReplayFamilyHooks({
1328
family: "google-gemini",
@@ -16,4 +31,6 @@ export const GOOGLE_GEMINI_PROVIDER_HOOKS = {
1631
resolveThinkingProfile: (context: ProviderDefaultThinkingPolicyContext) =>
1732
resolveGoogleThinkingProfile(context) satisfies ProviderThinkingProfile | undefined,
1833
wrapStreamFn: createGoogleThinkingStreamWrapper,
34+
classifyFailoverReason: ({ code }: ProviderFailoverErrorContext) =>
35+
classifyGoogleFailoverCode(code),
1936
};

‎extensions/openai/openai-provider.test.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,33 @@ describe("buildOpenAIProvider", () => {
204204
});
205205
});
206206

207+
it("classifies OpenAI-native code-only failover errors", () => {
208+
const provider = buildOpenAIProvider();
209+
210+
expect(
211+
provider.classifyFailoverReason?.({
212+
provider: "openai",
213+
errorMessage: "",
214+
code: "SERVER_ERROR",
215+
}),
216+
).toBe("server_error");
217+
expect(
218+
provider.classifyFailoverReason?.({
219+
provider: "openai",
220+
errorMessage: "",
221+
code: "INSUFFICIENT_QUOTA",
222+
}),
223+
).toBe("billing");
224+
// API_ERROR is an Anthropic-native code, not OpenAI's: fall through to generic.
225+
expect(
226+
provider.classifyFailoverReason?.({
227+
provider: "openai",
228+
errorMessage: "",
229+
code: "API_ERROR",
230+
}),
231+
).toBeUndefined();
232+
});
233+
207234
it("marks the OpenAI manifest catalog as runtime-discovered", () => {
208235
expect(manifest.modelCatalog.discovery.openai).toBe("runtime");
209236
});

‎extensions/openai/openai-provider.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,18 @@ import {
4949
import { resolveUnifiedOpenAIThinkingProfile } from "./thinking-policy.js";
5050

5151
const PROVIDER_ID = "openai";
52+
53+
// OpenAI-native error codes stay with the OpenAI provider hook.
54+
function classifyOpenAiFailoverCode(code: string | undefined) {
55+
switch (code?.trim().toUpperCase()) {
56+
case "SERVER_ERROR":
57+
return "server_error" as const;
58+
case "INSUFFICIENT_QUOTA":
59+
return "billing" as const;
60+
default:
61+
return undefined;
62+
}
63+
}
5264
const OPENAI_MODELS_ENDPOINT = "https://api.openai.com/v1/models";
5365
const OPENAI_CODEX_MODELS_ENDPOINT = `${OPENAI_CODEX_RESPONSES_BASE_URL}/models?client_version=1.0.0`;
5466
const OPENAI_MODELS_CACHE_TTL_MS = 60_000;
@@ -928,6 +940,10 @@ export function buildOpenAIProvider(): ProviderPlugin {
928940
},
929941
matchesContextOverflowError: ({ errorMessage }) =>
930942
/content_filter.*(?:prompt|input).*(?:too long|exceed)/i.test(errorMessage),
943+
classifyFailoverReason: ({ provider, code }) =>
944+
normalizeProviderId(provider ?? "") === PROVIDER_ID
945+
? classifyOpenAiFailoverCode(code)
946+
: undefined,
931947
resolveReasoningOutputMode: () => "native",
932948
resolveThinkingProfile: ({ provider, modelId, agentRuntime, compat }) =>
933949
normalizeProviderId(provider) === PROVIDER_ID

‎src/agents/embedded-agent-helpers.isbillingerrormessage.test.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1499,6 +1499,28 @@ describe("classifyFailoverReason provider messages", () => {
14991499
});
15001500

15011501
describe("classifyProviderRuntimeFailureKind", () => {
1502+
it("classifies generic resource-exhausted codes as rate_limit", () => {
1503+
expect(
1504+
classifyProviderRuntimeFailureKind({
1505+
provider: "openai",
1506+
code: "RESOURCE_EXHAUSTED",
1507+
message: "",
1508+
}),
1509+
).toBe("rate_limit");
1510+
});
1511+
1512+
it.each([
1513+
{ provider: "openai", code: "SERVER_ERROR" },
1514+
{ provider: "google", code: "UNAVAILABLE" },
1515+
{ provider: "anthropic", code: "RATE_LIMIT_ERROR" },
1516+
] as const)(
1517+
"does not report code-only $provider $code failures as empty responses",
1518+
({ provider, code }) => {
1519+
expect(classifyProviderRuntimeFailureKind({ provider, code, message: "" })).not.toBe(
1520+
"empty_response",
1521+
);
1522+
},
1523+
);
15021524
it("classifies missing scope failures", () => {
15031525
expect(
15041526
classifyProviderRuntimeFailureKind({

‎src/agents/embedded-agent-helpers/errors-provider-structured-signals.test.ts‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
11
// Covers provider hook structured failover signals.
22
import { beforeEach, describe, expect, it, vi } from "vitest";
33
import { resolveFailoverReasonFromError } from "../failover-error.js";
4-
import { classifyFailoverSignal } from "./errors.js";
4+
import { makeAssistantMessageFixture } from "../test-helpers/assistant-message-fixtures.js";
5+
import {
6+
classifyAssistantFailoverReason,
7+
classifyProviderRuntimeFailureKind,
8+
classifyFailoverSignal,
9+
} from "./errors.js";
510

611
const providerRuntimeMocks = vi.hoisted(() => ({
712
classifyProviderPluginError: vi.fn(),
@@ -111,6 +116,40 @@ describe("provider failover hook structured signals", () => {
111116
).toBe("billing");
112117
});
113118

119+
it.each([
120+
{ errorType: "rate_limit_error", reason: "rate_limit", runtimeKind: "rate_limit" },
121+
{ errorType: "api_error", reason: "timeout", runtimeKind: "timeout" },
122+
] as const)(
123+
"classifies message-less Anthropic $errorType assistant failures",
124+
({ errorType, reason, runtimeKind }) => {
125+
providerRuntimeMocks.classifyProviderPluginError.mockImplementation((context) => {
126+
if (context.provider !== "anthropic") {
127+
return undefined;
128+
}
129+
if (context.errorType === "rate_limit_error") {
130+
return "rate_limit";
131+
}
132+
return context.errorType === "api_error" ? "timeout" : undefined;
133+
});
134+
135+
const message = makeAssistantMessageFixture({
136+
provider: "anthropic",
137+
errorMessage: undefined,
138+
errorType,
139+
content: [],
140+
});
141+
142+
expect(classifyAssistantFailoverReason(message)).toBe(reason);
143+
expect(
144+
classifyProviderRuntimeFailureKind({
145+
provider: "anthropic",
146+
message: "",
147+
errorType,
148+
}),
149+
).toBe(runtimeKind);
150+
},
151+
);
152+
114153
it("does not promote generic SDK type strings as structured provider descriptors", () => {
115154
providerRuntimeMocks.classifyProviderPluginError.mockReturnValue("billing");
116155

‎src/agents/embedded-agent-helpers/errors.ts‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -907,6 +907,8 @@ function classifyFailoverClassificationFromHttpStatus(
907907
return null;
908908
}
909909

910+
// Only cross-provider structured codes classify in core; provider-native
911+
// mappings belong to provider hooks.
910912
function classifyFailoverReasonFromCode(raw: string | undefined): FailoverReason | null {
911913
const normalized = raw?.trim().toUpperCase();
912914
if (!normalized) {
@@ -917,14 +919,24 @@ function classifyFailoverReasonFromCode(raw: string | undefined): FailoverReason
917919
case "RATE_LIMIT":
918920
case "RATE_LIMITED":
919921
case "RATE_LIMIT_EXCEEDED":
922+
case "RATE_LIMIT_ERROR":
920923
case "TOO_MANY_REQUESTS":
921924
case "THROTTLED":
922925
case "THROTTLING":
923926
case "THROTTLINGEXCEPTION":
924927
case "THROTTLING_EXCEPTION":
925928
return "rate_limit";
929+
case "INSUFFICIENT_QUOTA":
930+
return "billing";
926931
case "DEACTIVATED_WORKSPACE":
927932
return "auth_permanent";
933+
case "INTERNAL":
934+
case "SERVER_ERROR":
935+
return "server_error";
936+
case "API_ERROR":
937+
case "DEADLINE_EXCEEDED":
938+
return "timeout";
939+
case "UNAVAILABLE":
928940
case "OVERLOADED":
929941
case "OVERLOADED_ERROR":
930942
return "overloaded";
@@ -1247,8 +1259,9 @@ export function classifyProviderRuntimeFailureKind(
12471259
const normalizedSignal = typeof signal === "string" ? { message: signal } : signal;
12481260
const message = normalizedSignal.message?.trim() ?? "";
12491261
const status = inferSignalStatus(normalizedSignal);
1262+
const hasStructuredErrorSignal = Boolean(normalizedSignal.code || normalizedSignal.errorType);
12501263

1251-
if (!message && typeof status !== "number") {
1264+
if (!message && typeof status !== "number" && !hasStructuredErrorSignal) {
12521265
return "empty_response";
12531266
}
12541267
if (normalizedSignal.code === "refresh_contention") {

0 commit comments

Comments
 (0)