Skip to content

Commit cfd8ba8

Browse files
mikasa0818altaywtf
mikasa0818
and
altaywtf
authored
fix #95519: [Bug]: Fallback should trigger on provider upstream_error / LLM request failed (#95542)
* fix(agents): fallback on upstream provider errors * refactor(agents): clarify provider fallback classification --------- Co-authored-by: Altay <[email protected]>
1 parent 389c355 commit cfd8ba8

4 files changed

Lines changed: 118 additions & 6 deletions

File tree

‎src/agents/embedded-agent-helpers.formatassistanterrortext.test.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { describe, expect, it } from "vitest";
44
import { MALFORMED_STREAMING_FRAGMENT_ERROR_MESSAGE } from "../shared/assistant-error-format.js";
55
import {
66
BILLING_ERROR_USER_MESSAGE,
7+
classifyAssistantFailoverReason,
78
formatBillingErrorMessage,
89
formatAssistantErrorText,
910
formatUserFacingAssistantErrorText,
@@ -116,6 +117,21 @@ describe("formatAssistantErrorText", () => {
116117
);
117118
expect(formatAssistantErrorText(msg)).toBe("LLM error server_error: Something exploded");
118119
});
120+
it("classifies provider upstream_error payloads as server errors for fallback", () => {
121+
const msg = makeAssistantMessageFixture({
122+
errorMessage: "Upstream request failed",
123+
errorType: "upstream_error",
124+
});
125+
126+
expect(classifyAssistantFailoverReason(msg, { provider: "openai" })).toBe("server_error");
127+
expect(
128+
classifyAssistantFailoverReason(
129+
makeAssistantError(
130+
'{"error":{"message":"Upstream request failed","type":"upstream_error","param":"","code":null}}',
131+
),
132+
),
133+
).toBe("server_error");
134+
});
119135
it("uses generic user-facing copy for escaped structured provider messages", () => {
120136
// The internal formatter keeps detail for logs, while user-facing text must
121137
// not expose arbitrary provider-controlled structured payload content.

‎src/agents/embedded-agent-helpers/errors.ts‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -922,6 +922,26 @@ function classifyFailoverReasonFromCode(raw: string | undefined): FailoverReason
922922
}
923923
}
924924

925+
function classifyFailoverReasonFromErrorType(raw: string | undefined): FailoverReason | null {
926+
const normalized = normalizeOptionalLowercaseString(raw);
927+
switch (normalized) {
928+
case "server_error":
929+
case "upstream_error":
930+
return "server_error";
931+
case "overloaded_error":
932+
return "overloaded";
933+
default:
934+
return null;
935+
}
936+
}
937+
938+
function classifyFailoverClassificationFromErrorType(
939+
raw: string | undefined,
940+
): FailoverClassification | null {
941+
const reason = classifyFailoverReasonFromErrorType(raw);
942+
return reason ? toReasonClassification(reason) : null;
943+
}
944+
925945
function isProvider(provider: string | undefined, match: string): boolean {
926946
const normalized = normalizeOptionalLowercaseString(provider);
927947
return Boolean(normalized && normalized.includes(match));
@@ -1181,9 +1201,14 @@ export function classifyFailoverSignal(signal: FailoverSignal): FailoverClassifi
11811201
errorType: signal.errorType,
11821202
})
11831203
: null;
1204+
const messageOrDetailClassification = mergeMessageAndDetailClassification(
1205+
messageClassification,
1206+
detailClassification,
1207+
);
1208+
const errorTypeClassification = classifyFailoverClassificationFromErrorType(signal.errorType);
11841209
const effectiveMessageClassification = providerPluginReason
11851210
? toReasonClassification(providerPluginReason)
1186-
: mergeMessageAndDetailClassification(messageClassification, detailClassification);
1211+
: (messageOrDetailClassification ?? errorTypeClassification);
11871212
const codeReason = classifyFailoverReasonFromCode(signal.code);
11881213
if (codeReason === "auth_permanent") {
11891214
return toReasonClassification(codeReason);
@@ -1658,8 +1683,17 @@ function isStructuredServerErrorMessage(raw: string): boolean {
16581683
if (!raw) {
16591684
return false;
16601685
}
1686+
const parsedType = normalizeOptionalLowercaseString(parseApiErrorInfo(raw)?.type);
1687+
if (parsedType === "server_error" || parsedType === "upstream_error") {
1688+
return true;
1689+
}
16611690
const value = normalizeLowercaseStringOrEmpty(raw);
1662-
return value.includes('"type":"server_error"') || value.includes('"code":"server_error"');
1691+
return (
1692+
value.includes('"type":"server_error"') ||
1693+
value.includes('"code":"server_error"') ||
1694+
value.includes('"type":"upstream_error"') ||
1695+
value.includes('"code":"upstream_error"')
1696+
);
16631697
}
16641698

16651699
export function parseImageDimensionError(raw: string): {

‎src/agents/embedded-agent-runner/result-fallback-classifier.test.ts‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,62 @@ describe("classifyEmbeddedAgentRunResultForModelFallback", () => {
5050
});
5151
});
5252

53+
it("classifies structured provider upstream_error payloads as fallback-worthy", () => {
54+
const rawError =
55+
'{"error":{"message":"Upstream request failed","type":"upstream_error","param":"","code":null}}';
56+
57+
const result = classifyEmbeddedAgentRunResultForModelFallback({
58+
provider: "openai-compatible",
59+
model: "primary-model",
60+
result: {
61+
payloads: [
62+
{
63+
isError: true,
64+
text: rawError,
65+
},
66+
],
67+
meta: {
68+
durationMs: 42,
69+
},
70+
},
71+
});
72+
73+
expect(result).toEqual({
74+
message: `openai-compatible/primary-model ended with a provider error: ${rawError}`,
75+
reason: "server_error",
76+
code: "embedded_error_payload",
77+
rawError,
78+
});
79+
});
80+
81+
it("classifies structured provider overloaded_error payloads as fallback-worthy", () => {
82+
const rawError =
83+
'{"error":{"message":"Provider overloaded","type":"overloaded_error","param":"","code":null}}';
84+
85+
const result = classifyEmbeddedAgentRunResultForModelFallback({
86+
provider: "openai-compatible",
87+
model: "primary-model",
88+
result: {
89+
payloads: [
90+
{
91+
isError: true,
92+
text: rawError,
93+
},
94+
],
95+
meta: {
96+
durationMs: 42,
97+
},
98+
},
99+
});
100+
101+
expect(result).toEqual({
102+
message: `openai-compatible/primary-model ended with a provider error: ${rawError}`,
103+
reason: "overloaded",
104+
code: "embedded_error_payload",
105+
rawError,
106+
});
107+
});
108+
53109
it("classifies generic external runner failure text as fallback-worthy", () => {
54110
const result = classifyEmbeddedAgentRunResultForModelFallback({
55111
provider: "claude-cli",

‎src/agents/embedded-agent-runner/result-fallback-classifier.ts‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,11 @@ import {
1313
} from "./delivery-evidence.js";
1414
import type { EmbeddedAgentRunResult } from "./types.js";
1515

16+
type ProviderErrorPayloadFailoverReason = Extract<
17+
FailoverReason,
18+
"auth" | "auth_permanent" | "billing" | "rate_limit" | "server_error" | "overloaded"
19+
>;
20+
1621
/**
1722
* Classifies embedded-agent terminal results for model fallback decisions.
1823
*
@@ -146,11 +151,10 @@ function classifyHarnessResult(params: {
146151
}
147152
}
148153

149-
/** Maps provider error payloads to fallback-safe business reasons. */
150-
function classifyBusinessDenialErrorPayloadReason(
154+
function classifyProviderErrorPayloadReason(
151155
errorText: string,
152156
provider: string,
153-
): Extract<FailoverReason, "auth" | "auth_permanent" | "billing" | "rate_limit"> | null {
157+
): ProviderErrorPayloadFailoverReason | null {
154158
if (!errorText.trim()) {
155159
return null;
156160
}
@@ -160,6 +164,8 @@ function classifyBusinessDenialErrorPayloadReason(
160164
case "auth_permanent":
161165
case "billing":
162166
case "rate_limit":
167+
case "server_error":
168+
case "overloaded":
163169
return failoverReason;
164170
default:
165171
return null;
@@ -252,7 +258,7 @@ export function classifyEmbeddedAgentRunResultForModelFallback(params: {
252258
.filter((payload) => payload?.isError === true)
253259
.map((payload) => (typeof payload.text === "string" ? payload.text : ""))
254260
.join("\n");
255-
const failoverReason = classifyBusinessDenialErrorPayloadReason(errorText, params.provider);
261+
const failoverReason = classifyProviderErrorPayloadReason(errorText, params.provider);
256262
if (failoverReason) {
257263
return {
258264
message: `${params.provider}/${params.model} ended with a provider error: ${errorText}`,

0 commit comments

Comments
 (0)