Skip to content

Commit f80f472

Browse files
joshavantjoshavant
authored
fix(agents): classify structured unsupported model errors (#92280)
* fix(agents): classify structured unsupported model errors * test(agents): update embedded harness helper mock
1 parent 3643de4 commit f80f472

13 files changed

Lines changed: 411 additions & 16 deletions

‎packages/llm-core/src/types.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -296,6 +296,9 @@ export interface AssistantMessage {
296296
usage: Usage;
297297
stopReason: StopReason;
298298
errorMessage?: string;
299+
errorCode?: string;
300+
errorType?: string;
301+
errorBody?: string;
299302
timestamp: number; // Unix timestamp in milliseconds
300303
}
301304

‎src/agents/embedded-agent-helpers.formatassistanterrortext.test.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -571,6 +571,23 @@ describe("formatAssistantErrorText", () => {
571571
"LLM request failed: provider rejected the request schema or tool payload.",
572572
);
573573
});
574+
575+
it("uses structured error body detail for model-not-found copy", () => {
576+
const msg = makeAssistantMessageFixture({
577+
errorMessage: "400 Param Incorrect",
578+
errorCode: "400",
579+
errorBody:
580+
'{"code":"400","message":"Param Incorrect","param":"Not supported model some-model-id"}',
581+
content: [],
582+
});
583+
584+
expect(formatAssistantErrorText(msg)).toBe(
585+
"The selected model was not found by the provider. Check the model id or choose a different model.",
586+
);
587+
expect(formatUserFacingAssistantErrorText(msg)).toBe(
588+
"The selected model was not found by the provider. Check the model id or choose a different model.",
589+
);
590+
});
574591
});
575592

576593
describe("formatRawAssistantErrorForUi", () => {

‎src/agents/embedded-agent-helpers.isbillingerrormessage.test.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// Covers provider error classifiers and failover reason mapping.
22
import { describe, expect, it } from "vitest";
33
import {
4+
classifyAssistantFailoverReason,
45
classifyProviderRuntimeFailureKind,
56
classifyFailoverReason,
67
classifyFailoverReasonFromHttpStatus,
@@ -1037,6 +1038,34 @@ describe("image dimension errors", () => {
10371038
});
10381039
});
10391040

1041+
describe("classifyAssistantFailoverReason", () => {
1042+
it("uses structured assistant error bodies for model-not-found 400s", () => {
1043+
expect(
1044+
classifyAssistantFailoverReason({
1045+
role: "assistant",
1046+
api: "openai-completions",
1047+
provider: "openai",
1048+
model: "some-model-id",
1049+
usage: {
1050+
input: 0,
1051+
output: 0,
1052+
cacheRead: 0,
1053+
cacheWrite: 0,
1054+
totalTokens: 0,
1055+
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
1056+
},
1057+
stopReason: "error",
1058+
errorMessage: "400 Param Incorrect",
1059+
errorCode: "400",
1060+
errorBody:
1061+
'{"code":"400","message":"Param Incorrect","param":"Not supported model some-model-id"}',
1062+
content: [],
1063+
timestamp: 0,
1064+
}),
1065+
).toBe("model_not_found");
1066+
});
1067+
});
1068+
10401069
describe("classifyFailoverReasonFromHttpStatus – 402 temporary limits", () => {
10411070
it("reclassifies periodic usage limits as rate_limit", () => {
10421071
const samples = [

‎src/agents/embedded-agent-helpers.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ export {
1212
} from "./embedded-agent-helpers/bootstrap.js";
1313
export {
1414
BILLING_ERROR_USER_MESSAGE,
15+
classifyAssistantFailoverReason,
1516
classifyProviderRuntimeFailureKind,
1617
formatBillingErrorMessage,
1718
formatRateLimitOrOverloadedErrorCopy,

‎src/agents/embedded-agent-helpers/errors.ts‎

Lines changed: 174 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,10 @@ const sandboxToolPolicyAuditMessages = new WeakSet<AssistantMessage>();
7777
export const GENERIC_ASSISTANT_ERROR_TEXT = "LLM request failed.";
7878
const PROVIDER_SCHEMA_REJECTION_USER_TEXT =
7979
"LLM request failed: provider rejected the request schema or tool payload.";
80+
const MODEL_NOT_FOUND_USER_TEXT =
81+
"The selected model was not found by the provider. Check the model id or choose a different model.";
82+
const MAX_FAILOVER_DETAIL_CANDIDATES = 12;
83+
const MAX_FAILOVER_DETAIL_CHARS = 1_000;
8084

8185
/** Detect provider errors that require reasoning to stay enabled. */
8286
export function isReasoningConstraintErrorMessage(raw: string): boolean {
@@ -276,6 +280,7 @@ export type FailoverSignal = {
276280
errorType?: string;
277281
message?: string;
278282
provider?: string;
283+
details?: readonly string[];
279284
};
280285

281286
export type FailoverClassification =
@@ -287,6 +292,87 @@ export type FailoverClassification =
287292
kind: "context_overflow";
288293
};
289294

295+
// Provider SDKs often keep semantic error fields outside Error.message.
296+
// These bounded candidates feed classification only; user-facing copy still
297+
// comes from the normal sanitized formatter path.
298+
function normalizeFailoverDetailString(value: string | undefined): string | undefined {
299+
const trimmed = value?.trim();
300+
if (!trimmed) {
301+
return undefined;
302+
}
303+
return trimmed.length > MAX_FAILOVER_DETAIL_CHARS
304+
? trimmed.slice(0, MAX_FAILOVER_DETAIL_CHARS)
305+
: trimmed;
306+
}
307+
308+
function appendFailoverDetailCandidate(candidates: string[], value: unknown): void {
309+
const normalized =
310+
typeof value === "string" || typeof value === "number" || typeof value === "boolean"
311+
? normalizeFailoverDetailString(String(value))
312+
: undefined;
313+
if (!normalized || candidates.includes(normalized)) {
314+
return;
315+
}
316+
candidates.push(normalized);
317+
}
318+
319+
function collectFailoverDetailCandidates(
320+
value: unknown,
321+
candidates: string[],
322+
seen: Set<object>,
323+
): void {
324+
if (
325+
candidates.length >= MAX_FAILOVER_DETAIL_CANDIDATES ||
326+
value === undefined ||
327+
value === null
328+
) {
329+
return;
330+
}
331+
if (typeof value === "string") {
332+
appendFailoverDetailCandidate(candidates, value);
333+
const trimmed = value.trim();
334+
if (!trimmed.startsWith("{") || !trimmed.endsWith("}")) {
335+
return;
336+
}
337+
try {
338+
collectFailoverDetailCandidates(JSON.parse(trimmed) as unknown, candidates, seen);
339+
} catch {
340+
// Non-JSON detail strings are still useful as direct classifier candidates.
341+
}
342+
return;
343+
}
344+
if (typeof value === "number" || typeof value === "boolean") {
345+
appendFailoverDetailCandidate(candidates, value);
346+
return;
347+
}
348+
if (!value || typeof value !== "object" || Array.isArray(value)) {
349+
return;
350+
}
351+
if (seen.has(value)) {
352+
return;
353+
}
354+
seen.add(value);
355+
const record = value as Record<string, unknown>;
356+
for (const key of ["message", "param", "code", "type", "error", "detail", "body"]) {
357+
collectFailoverDetailCandidates(record[key], candidates, seen);
358+
if (candidates.length >= MAX_FAILOVER_DETAIL_CANDIDATES) {
359+
return;
360+
}
361+
}
362+
}
363+
364+
export function extractFailoverSignalDetails(...values: unknown[]): string[] | undefined {
365+
const candidates: string[] = [];
366+
const seen = new Set<object>();
367+
for (const value of values) {
368+
collectFailoverDetailCandidates(value, candidates, seen);
369+
if (candidates.length >= MAX_FAILOVER_DETAIL_CANDIDATES) {
370+
break;
371+
}
372+
}
373+
return candidates.length > 0 ? candidates : undefined;
374+
}
375+
290376
export type ProviderRuntimeFailureKind =
291377
| "auth_scope"
292378
| "auth_refresh"
@@ -302,6 +388,7 @@ export type ProviderRuntimeFailureKind =
302388
| "rate_limit"
303389
| "dns"
304390
| "timeout"
391+
| "model_not_found"
305392
| "schema"
306393
| "sandbox_blocked"
307394
| "replay_invalid"
@@ -1009,6 +1096,49 @@ function classifyFailoverClassificationFromMessage(
10091096
return null;
10101097
}
10111098

1099+
function classificationReason(
1100+
classification: FailoverClassification | null,
1101+
): FailoverReason | undefined {
1102+
return classification?.kind === "reason" ? classification.reason : undefined;
1103+
}
1104+
1105+
function classifyFailoverDetailCandidates(
1106+
details: readonly string[] | undefined,
1107+
provider: string | undefined,
1108+
includeProviderPluginHooks: boolean,
1109+
): FailoverClassification | null {
1110+
for (const detail of details ?? []) {
1111+
const classification = classifyFailoverClassificationFromMessage(detail, provider, {
1112+
includeProviderPluginHooks,
1113+
});
1114+
if (classification) {
1115+
return classification;
1116+
}
1117+
}
1118+
return null;
1119+
}
1120+
1121+
function mergeMessageAndDetailClassification(
1122+
messageClassification: FailoverClassification | null,
1123+
detailClassification: FailoverClassification | null,
1124+
): FailoverClassification | null {
1125+
if (!messageClassification) {
1126+
return detailClassification;
1127+
}
1128+
if (!detailClassification) {
1129+
return messageClassification;
1130+
}
1131+
if (messageClassification.kind === "context_overflow") {
1132+
return messageClassification;
1133+
}
1134+
if (detailClassification.kind === "context_overflow") {
1135+
return detailClassification;
1136+
}
1137+
return classificationReason(messageClassification) === "format"
1138+
? detailClassification
1139+
: messageClassification;
1140+
}
1141+
10121142
export function classifyFailoverSignal(signal: FailoverSignal): FailoverClassification | null {
10131143
const inferredStatus = inferSignalStatus(signal);
10141144
const explicitStatus =
@@ -1029,6 +1159,11 @@ export function classifyFailoverSignal(signal: FailoverSignal): FailoverClassifi
10291159
includeProviderPluginHooks: !hasStructuredProviderSignal,
10301160
})
10311161
: null;
1162+
const detailClassification = classifyFailoverDetailCandidates(
1163+
signal.details,
1164+
signal.provider,
1165+
!hasStructuredProviderSignal,
1166+
);
10321167
const providerPluginReason =
10331168
hasStructuredProviderSignal &&
10341169
signal.provider &&
@@ -1043,7 +1178,7 @@ export function classifyFailoverSignal(signal: FailoverSignal): FailoverClassifi
10431178
: null;
10441179
const effectiveMessageClassification = providerPluginReason
10451180
? toReasonClassification(providerPluginReason)
1046-
: messageClassification;
1181+
: mergeMessageAndDetailClassification(messageClassification, detailClassification);
10471182
const codeReason = classifyFailoverReasonFromCode(signal.code);
10481183
if (codeReason === "auth_permanent") {
10491184
return toReasonClassification(codeReason);
@@ -1110,6 +1245,12 @@ export function classifyProviderRuntimeFailureKind(
11101245
if (failoverClassification?.kind === "reason" && failoverClassification.reason === "rate_limit") {
11111246
return "rate_limit";
11121247
}
1248+
if (
1249+
failoverClassification?.kind === "reason" &&
1250+
failoverClassification.reason === "model_not_found"
1251+
) {
1252+
return "model_not_found";
1253+
}
11131254
if (message && isDnsTransportErrorMessage(message)) {
11141255
return "dns";
11151256
}
@@ -1155,6 +1296,32 @@ export function classifyProviderRuntimeFailureKind(
11551296
return "unclassified";
11561297
}
11571298

1299+
function buildAssistantFailoverSignal(
1300+
msg: AssistantMessage,
1301+
opts?: { provider?: string },
1302+
): FailoverSignal {
1303+
return {
1304+
status: extractLeadingHttpStatus(msg.errorMessage?.trim() ?? "")?.code,
1305+
code: msg.errorCode,
1306+
errorType: msg.errorType,
1307+
message: msg.errorMessage?.trim() || undefined,
1308+
provider: opts?.provider ?? msg.provider,
1309+
details: extractFailoverSignalDetails(msg.errorBody),
1310+
};
1311+
}
1312+
1313+
export function classifyAssistantFailoverReason(
1314+
msg: AssistantMessage | undefined,
1315+
opts?: { provider?: string },
1316+
): FailoverReason | null {
1317+
if (!msg || msg.stopReason !== "error") {
1318+
return null;
1319+
}
1320+
return failoverReasonFromClassification(
1321+
classifyFailoverSignal(buildAssistantFailoverSignal(msg, opts)),
1322+
);
1323+
}
1324+
11581325
export function formatAssistantErrorText(
11591326
msg: AssistantMessage,
11601327
opts?: { cfg?: OpenClawConfig; sessionKey?: string; provider?: string; model?: string },
@@ -1169,9 +1336,8 @@ export function formatAssistantErrorText(
11691336
}
11701337

11711338
const providerRuntimeFailureKind = classifyProviderRuntimeFailureKind({
1172-
status: extractLeadingHttpStatus(raw)?.code,
1339+
...buildAssistantFailoverSignal(msg, { provider: opts?.provider }),
11731340
message: raw,
1174-
provider: opts?.provider ?? msg.provider,
11751341
});
11761342

11771343
const unknownTool =
@@ -1264,6 +1430,10 @@ export function formatAssistantErrorText(
12641430
return "LLM request failed: proxy or tunnel configuration blocked the provider request.";
12651431
}
12661432

1433+
if (providerRuntimeFailureKind === "model_not_found") {
1434+
return MODEL_NOT_FOUND_USER_TEXT;
1435+
}
1436+
12671437
if (isContextOverflowError(raw)) {
12681438
return (
12691439
"Context overflow: prompt too large for the model. " +
@@ -1580,8 +1750,5 @@ export function isFailoverErrorMessage(raw: string, opts?: { provider?: string }
15801750
}
15811751

15821752
export function isFailoverAssistantError(msg: AssistantMessage | undefined): boolean {
1583-
if (!msg || msg.stopReason !== "error") {
1584-
return false;
1585-
}
1586-
return isFailoverErrorMessage(msg.errorMessage ?? "", { provider: msg.provider });
1753+
return classifyAssistantFailoverReason(msg) !== null;
15871754
}

‎src/agents/embedded-agent-runner/run.overflow-compaction.harness.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,10 @@ export const mockedFormatBillingErrorMessage = vi.fn(() => "");
211211
export const mockedClassifyFailoverReason = vi.fn<(raw: string) => FailoverReason | null>(
212212
() => null,
213213
);
214+
export const mockedClassifyAssistantFailoverReason = vi.fn(
215+
(assistant?: { errorMessage?: string | null }): FailoverReason | null =>
216+
mockedClassifyFailoverReason(assistant?.errorMessage ?? ""),
217+
);
214218
export const mockedExtractObservedOverflowTokenCount = vi.fn((msg?: string) => {
215219
const match = msg?.match(/prompt is too long:\s*([\d,]+)\s+tokens\s*>\s*[\d,]+\s+maximum/i);
216220
return match?.[1] ? Number(match[1].replaceAll(",", "")) : undefined;
@@ -384,6 +388,11 @@ export function resetRunOverflowCompactionHarnessMocks(): void {
384388

385389
mockedClassifyFailoverReason.mockReset();
386390
mockedClassifyFailoverReason.mockReturnValue(null);
391+
mockedClassifyAssistantFailoverReason.mockReset();
392+
mockedClassifyAssistantFailoverReason.mockImplementation(
393+
(assistant?: { errorMessage?: string | null }): FailoverReason | null =>
394+
mockedClassifyFailoverReason(assistant?.errorMessage ?? ""),
395+
);
387396
mockedFormatBillingErrorMessage.mockReset();
388397
mockedFormatBillingErrorMessage.mockReturnValue("");
389398
mockedFormatAssistantErrorText.mockReset();
@@ -624,6 +633,7 @@ export async function loadRunOverflowCompactionHarness(): Promise<{
624633
vi.doMock("../embedded-agent-helpers.js", () => ({
625634
formatBillingErrorMessage: mockedFormatBillingErrorMessage,
626635
classifyFailoverReason: mockedClassifyFailoverReason,
636+
classifyAssistantFailoverReason: mockedClassifyAssistantFailoverReason,
627637
extractObservedOverflowTokenCount: mockedExtractObservedOverflowTokenCount,
628638
formatAssistantErrorText: mockedFormatAssistantErrorText,
629639
isAuthAssistantError: mockedIsAuthAssistantError,

0 commit comments

Comments
 (0)