Skip to content

Commit 53ba9fc

Browse files
nik13claude
andcommitted
fix(claude-agent-sdk): record the requested model on a synthetic API-error turn
On an API error the Claude Code CLI yields an assistant message it wrote itself, with message.model "<synthetic>" (seen live with 0.3.289 and an unknown model). That turn's LLM span carried gen_ai.request.model and claude_agent.model "<synthetic>", so Observe listed a model that does not exist. The turn now carries the requested model (options.model, else the init message's model) on both keys, and keeps the CLI's value on the new claude_agent.message.model. Real model replies are unchanged. The key lives in TraceAIAttributes, so the Python attribute parity list stays exact. Live: unknown model via CLIProxy -> turn model no-such-model-th8103, claude_agent.message.model "<synthetic>", status ERROR (was model "<synthetic>"). Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 4cf737f commit 53ba9fc

4 files changed

Lines changed: 62 additions & 3 deletions

File tree

‎typescript/packages/traceai_claude_agent_sdk/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ A foreground subagent span ends with its `tool.Agent` / `tool.Task` result. A su
7373

7474
| Field | Attributes | Source |
7575
|---|---|---|
76-
| Model | `gen_ai.request.model`, `claude_agent.model` | `options.model`, the init message, each assistant message |
76+
| Model | `gen_ai.request.model`, `claude_agent.model` | `options.model`, the init message, each assistant message. On an API error the CLI yields an assistant message it wrote itself, with model `<synthetic>`: that turn carries the requested model (`options.model`, else the init message's) and `claude_agent.message.model` = `<synthetic>` |
7777
| Provider | `gen_ai.provider.name` = `anthropic`, or `custom` when `ANTHROPIC_BASE_URL` (from `options.env`, else `process.env`) is not an `anthropic.com` host | |
7878
| Tokens | `gen_ai.usage.input_tokens`, `output_tokens`, `total_tokens`, `cache_read_tokens`, `cache_creation_tokens` | result `modelUsage` summed over every model, on the conversation span only. See "Usage and cost" |
7979
| Cost | `claude_agent.cost.total_usd` and `gen_ai.cost.total` | result `total_cost_usd`; never computed here. See "Usage and cost" |

‎typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -866,6 +866,42 @@ describe("wrapQuery", () => {
866866
});
867867
});
868868

869+
describe("synthetic API-error turn", () => {
870+
// On an API error (e.g. an unknown model) the CLI yields an assistant message it wrote itself:
871+
// message.model "<synthetic>", error "unknown", the API error as text (seen live with 0.3.289).
872+
function syntheticErrorJourney(initModel: string): SDKMessage[] {
873+
const turn = assistant("msg_synthetic", [text("API Error: 400 unknown provider for model x")], null, {
874+
error: "unknown",
875+
} as Partial<ReturnType<typeof assistant>>);
876+
(turn.message as { model: string }).model = "<synthetic>";
877+
return [init({ model: initModel }), turn, resultSuccess("API Error: 400 unknown provider for model x", { is_error: true } as never)];
878+
}
879+
880+
it("records the requested model on the turn and keeps <synthetic> in claude_agent.message.model", async () => {
881+
const { spans } = await run(syntheticErrorJourney("no-such-model"), { model: "no-such-model" });
882+
const turn = one(spans, "claude_agent.assistant_turn");
883+
expect(turn.attributes["gen_ai.request.model"]).toBe("no-such-model");
884+
expect(turn.attributes["claude_agent.model"]).toBe("no-such-model");
885+
expect(turn.attributes["claude_agent.message.model"]).toBe("<synthetic>");
886+
expect(turn.status.code).toBe(SpanStatusCode.ERROR);
887+
});
888+
889+
it("falls back to the init message's model when options.model is not set", async () => {
890+
const { spans } = await run(syntheticErrorJourney("claude-from-init"));
891+
const turn = one(spans, "claude_agent.assistant_turn");
892+
expect(turn.attributes["gen_ai.request.model"]).toBe("claude-from-init");
893+
expect(turn.attributes["claude_agent.message.model"]).toBe("<synthetic>");
894+
});
895+
896+
it("leaves a real model reply unchanged", async () => {
897+
const { spans } = await run(simpleToolJourney(), { model: "requested-alias" });
898+
for (const turn of byName(spans, "claude_agent.assistant_turn")) {
899+
expect(turn.attributes["gen_ai.request.model"]).toBe(MODEL);
900+
expect(turn.attributes["claude_agent.message.model"]).toBeUndefined();
901+
}
902+
});
903+
});
904+
869905
it("never copies FI_API_KEY / FI_SECRET_KEY onto spans", async () => {
870906
process.env.FI_API_KEY = "fi-api-key-PLACEHOLDER-123";
871907
process.env.FI_SECRET_KEY = "fi-secret-key-PLACEHOLDER-456";

‎typescript/packages/traceai_claude_agent_sdk/src/attributes.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,11 @@ export const TraceAIAttributes = {
156156
* known in this process, so no promoted usage or cost key was written.
157157
*/
158158
USAGE_BASELINE_UNKNOWN: "claude_agent.usage.baseline_unknown",
159+
/**
160+
* The model the CLI put on an assistant message it synthesized itself ("<synthetic>", on an
161+
* API error). The turn's model keys then carry the requested model instead.
162+
*/
163+
MESSAGE_MODEL: "claude_agent.message.model",
159164
} as const;
160165

161166
/**

‎typescript/packages/traceai_claude_agent_sdk/src/spans.ts‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@ export type FinishOutcome =
6060
| { kind: "error"; error: unknown }
6161
| { kind: "aborted"; reason?: unknown };
6262

63+
/** `message.model` on an assistant message the CLI synthesized itself (API errors). */
64+
const SYNTHETIC_MODEL = "<synthetic>";
6365
const PROMPT_MAX = 2000;
6466
const SYSTEM_PROMPT_MAX = 1000;
6567
const CONTENT_MAX = 5000;
@@ -290,6 +292,8 @@ export class QueryTracer {
290292
/** A result arrived and the main loop has not started another turn since (see trackIdleAfterResult). */
291293
private idleAfterResult = false;
292294
private sessionId?: string;
295+
/** The model the CLI's init message reports for the main loop. */
296+
private initModel?: string;
293297
private finished = false;
294298
private abortListener?: () => void;
295299

@@ -650,6 +654,7 @@ export class QueryTracer {
650654
this.setSessionId(message.session_id);
651655
}
652656
if (typeof message.model === "string" && message.model) {
657+
this.initModel = message.model;
653658
conversation.setAttribute(A.AGENT_MODEL, message.model);
654659
conversation.setAttribute(T.GEN_AI_REQUEST_MODEL, message.model);
655660
}
@@ -754,8 +759,15 @@ export class QueryTracer {
754759
span.setAttribute(T.GEN_AI_PROVIDER_NAME, this.providerName);
755760
const model = message.message?.model;
756761
if (typeof model === "string" && model) {
757-
span.setAttribute(A.AGENT_MODEL, model);
758-
span.setAttribute(T.GEN_AI_REQUEST_MODEL, model);
762+
// The CLI writes its own assistant message for an API error, with model "<synthetic>".
763+
// Record the model that was asked for instead; keep the CLI's value on its own key.
764+
const synthetic = model === SYNTHETIC_MODEL;
765+
if (synthetic) span.setAttribute(T.MESSAGE_MODEL, model);
766+
const recorded = synthetic ? this.requestedModel() : model;
767+
if (recorded) {
768+
span.setAttribute(A.AGENT_MODEL, recorded);
769+
span.setAttribute(T.GEN_AI_REQUEST_MODEL, recorded);
770+
}
759771
}
760772
if (scope !== null) {
761773
span.setAttribute(A.PARENT_TOOL_USE_ID, scope);
@@ -1111,6 +1123,12 @@ export class QueryTracer {
11111123
}
11121124
}
11131125

1126+
/** The model this query asked for: options.model, else the init message's model. */
1127+
private requestedModel(): string | undefined {
1128+
const model = this.options?.model;
1129+
return typeof model === "string" && model ? model : this.initModel;
1130+
}
1131+
11141132
private conversationStartAttributes(): Attributes {
11151133
const options = this.options ?? {};
11161134
const attributes: Record<string, AttributeValue> = {

0 commit comments

Comments
 (0)