Skip to content

Commit 66337a2

Browse files
steipetePick-cat
steipete
and
Pick-cat
committed
fix(anthropic): normalize legacy thinking budgets
Co-authored-by: Pick-cat <[email protected]>
1 parent 8c7fe0a commit 66337a2

10 files changed

Lines changed: 180 additions & 213 deletions

File tree

‎extensions/amazon-bedrock-mantle/mantle-anthropic.runtime.test.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,26 @@ describe("createMantleAnthropicStreamFn", () => {
214214
expect(streamOptions.effort).toBe("low");
215215
});
216216

217+
it("disables legacy thinking when the adjusted budget is below 1024", () => {
218+
const model = createTestModel({
219+
id: "anthropic.claude-haiku-4-5",
220+
name: "Claude Haiku 4.5",
221+
reasoning: true,
222+
maxTokens: 1500,
223+
});
224+
const deps = createTestDeps();
225+
deps.stream.mockReturnValue({ kind: "anthropic-stream" } as never);
226+
227+
void createMantleAnthropicStreamFn(deps)(
228+
model,
229+
{ messages: [] },
230+
{ apiKey: "bedrock-bearer-token", reasoning: "low" },
231+
);
232+
233+
expect(firstStreamOptions(deps)).toMatchObject({ maxTokens: 1500, thinkingEnabled: false });
234+
expect(firstStreamOptions(deps)).not.toHaveProperty("thinkingBudgetTokens");
235+
});
236+
217237
it.each([
218238
{ reasoning: undefined, effort: "high" },
219239
{ reasoning: "off" as const, effort: "low" },

‎extensions/amazon-bedrock-mantle/mantle-anthropic.runtime.ts‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -210,13 +210,18 @@ export function createMantleAnthropicStreamFn(deps?: {
210210
reasoning,
211211
options?.thinkingBudgets,
212212
);
213+
const adaptiveThinking = requiresClaudeMythosAdaptiveThinking(model);
214+
const thinkingEnabled = adaptiveThinking || adjusted.thinkingBudget >= 1024;
213215
return streamFn(model as Model<"anthropic-messages">, context, {
214216
...base,
215217
client: streamClient,
216218
maxTokens: adjusted.maxTokens,
217-
thinkingEnabled: true,
218-
...(requiresClaudeMythosAdaptiveThinking(model) ? { effort: reasoning } : {}),
219-
thinkingBudgetTokens: adjusted.thinkingBudget,
219+
thinkingEnabled,
220+
...(adaptiveThinking
221+
? { effort: reasoning }
222+
: thinkingEnabled
223+
? { thinkingBudgetTokens: adjusted.thinkingBudget }
224+
: {}),
220225
});
221226
};
222227
}

‎extensions/amazon-bedrock/stream.runtime.test.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,24 @@ describe("Bedrock thinking effort mapping", () => {
204204
expect(testing.buildAdditionalModelRequestFields(model, options)).toBeUndefined();
205205
});
206206

207+
it.each([
208+
{ reasoning: "minimal" as const, maxTokens: 1024 },
209+
{ reasoning: "low" as const, maxTokens: 1500 },
210+
])(
211+
"disables legacy thinking when $reasoning exceeds the $maxTokens token cap",
212+
({ reasoning, maxTokens }) => {
213+
const model = bedrockModel({
214+
id: "anthropic.claude-haiku-4-5-v1:0",
215+
name: "Claude Haiku 4.5",
216+
maxTokens,
217+
});
218+
const options = testing.resolveSimpleBedrockOptions(model, { reasoning });
219+
220+
expect(options).toMatchObject({ maxTokens, reasoning: "off" });
221+
expect(testing.buildAdditionalModelRequestFields(model, options)).toBeUndefined();
222+
},
223+
);
224+
207225
it("uses the model maxTokens cap for adaptive Claude thinking requests", () => {
208226
const model = bedrockModel({
209227
id: "us.anthropic.claude-opus-4-8",

‎extensions/amazon-bedrock/stream.runtime.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -438,6 +438,14 @@ function resolveSimpleBedrockOptions(
438438
options.thinkingBudgets,
439439
);
440440

441+
if (adjusted.thinkingBudget < 1024) {
442+
return {
443+
...base,
444+
maxTokens: adjusted.maxTokens,
445+
reasoning: "off",
446+
} satisfies BedrockOptions;
447+
}
448+
441449
return {
442450
...base,
443451
maxTokens: adjusted.maxTokens,

‎extensions/anthropic-vertex/stream-runtime.test.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -371,6 +371,25 @@ describe("createAnthropicVertexStreamFn", () => {
371371
expect(transportOptions.effort).toBe("max");
372372
});
373373

374+
it("disables manual thinking when the configured budget is below 1024", () => {
375+
const { deps, streamAnthropicMock } = createStreamDeps();
376+
const streamFn = createAnthropicVertexStreamFn("vertex-project", "us-east5", undefined, deps);
377+
const model = makeModel({ id: "claude-haiku-4-5", maxTokens: 8192 });
378+
379+
void streamFn(
380+
model,
381+
{ messages: [] },
382+
{
383+
reasoning: "low",
384+
thinkingBudgets: { low: 512 },
385+
},
386+
);
387+
388+
const transportOptions = streamTransportOptions(streamAnthropicMock);
389+
expect(transportOptions.thinkingEnabled).toBe(false);
390+
expect(transportOptions).not.toHaveProperty("thinkingBudgetTokens");
391+
});
392+
374393
it("preserves native max reasoning for Sonnet 4.6", () => {
375394
const { deps, streamAnthropicMock } = createStreamDeps();
376395
const streamFn = createAnthropicVertexStreamFn("vertex-project", "us-east5", undefined, deps);

‎extensions/anthropic-vertex/stream-runtime.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -200,12 +200,17 @@ export function createAnthropicVertexStreamFn(
200200
contractModelId,
201201
) as AnthropicVertexEffort;
202202
} else {
203-
opts.thinkingEnabled = true;
204203
const budgets = options?.thinkingBudgets;
205-
opts.thinkingBudgetTokens =
204+
const thinkingBudgetTokens =
206205
(budgets && reasoning in budgets
207206
? budgets[reasoning as keyof typeof budgets]
208207
: undefined) ?? 10000;
208+
const requestMaxTokens = opts.maxTokens ?? transportModel.maxTokens;
209+
opts.thinkingEnabled =
210+
thinkingBudgetTokens >= 1024 && thinkingBudgetTokens < requestMaxTokens;
211+
if (opts.thinkingEnabled) {
212+
opts.thinkingBudgetTokens = thinkingBudgetTokens;
213+
}
209214
}
210215
} else if (mandatoryAdaptiveThinking) {
211216
opts.thinkingEnabled = true;

‎packages/ai/src/providers/anthropic.test.ts‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1698,6 +1698,48 @@ describe("Anthropic provider", () => {
16981698
expect((capturedPayload as { thinking?: unknown }).thinking).toEqual({ type: "disabled" });
16991699
});
17001700

1701+
it.each([
1702+
{ budgetTokens: 512, maxTokens: 8192 },
1703+
{ budgetTokens: 1024, maxTokens: 1024 },
1704+
])(
1705+
"normalizes raw manual thinking budget $budgetTokens below max $maxTokens",
1706+
async ({ budgetTokens, maxTokens }) => {
1707+
const model = makeAnthropicModel({
1708+
id: "claude-haiku-4-5",
1709+
name: "Claude Haiku 4.5",
1710+
maxTokens: 8192,
1711+
});
1712+
let capturedPayload: unknown;
1713+
const stream = streamAnthropic(
1714+
model,
1715+
{
1716+
messages: [{ role: "user", content: "hello", timestamp: 0 }],
1717+
tools: [{ name: "lookup", description: "Lookup", parameters: { type: "object" } }],
1718+
},
1719+
{
1720+
apiKey: "sk-ant-provider",
1721+
maxTokens,
1722+
temperature: 0.2,
1723+
thinkingEnabled: true,
1724+
thinkingBudgetTokens: budgetTokens,
1725+
toolChoice: "any",
1726+
onPayload: (payload) => {
1727+
capturedPayload = payload;
1728+
throw new Error("stop before network");
1729+
},
1730+
},
1731+
);
1732+
1733+
await stream.result();
1734+
1735+
expect(capturedPayload).toMatchObject({
1736+
thinking: { type: "disabled" },
1737+
temperature: 0.2,
1738+
tool_choice: { type: "any" },
1739+
});
1740+
},
1741+
);
1742+
17011743
it.each(["claude-opus-4-8", "claude-mythos-preview"])(
17021744
"restores default sampling for %s after payload hooks",
17031745
async (modelId) => {

‎packages/ai/src/providers/anthropic.ts‎

Lines changed: 54 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,7 @@ export type AnthropicThinkingDisplay = "summarized" | "omitted";
231231

232232
const FINE_GRAINED_TOOL_STREAMING_BETA = "fine-grained-tool-streaming-2025-05-14";
233233
const INTERLEAVED_THINKING_BETA = "interleaved-thinking-2025-05-14";
234+
const ANTHROPIC_MIN_THINKING_BUDGET_TOKENS = 1024;
234235

235236
function getAnthropicCompat(model: Model<"anthropic-messages">): Required<AnthropicMessagesCompat> {
236237
// Auto-detect session affinity and cache control support from provider
@@ -504,6 +505,7 @@ export const streamAnthropic: StreamFunction<"anthropic-messages", AnthropicOpti
504505
) => {
505506
const stream = new AssistantMessageEventStream();
506507
const requestContext = prepareClaudeSonnet5RequestContext(model, context);
508+
const requestOptions = normalizeAnthropicThinkingOptions(model, options);
507509

508510
void (async () => {
509511
const output: AssistantMessage = {
@@ -527,7 +529,7 @@ export const streamAnthropic: StreamFunction<"anthropic-messages", AnthropicOpti
527529
// to expose until the terminal stop reason is known.
528530
const refusalBuffer = usesClaudeStreamingRefusalContract(model)
529531
? createDeferredEventBuffer<AssistantMessageEvent>(stream, () =>
530-
notifyLlmRequestActivity(options?.signal),
532+
notifyLlmRequestActivity(requestOptions?.signal),
531533
)
532534
: undefined;
533535
const eventSink = refusalBuffer ?? stream;
@@ -544,11 +546,11 @@ export const streamAnthropic: StreamFunction<"anthropic-messages", AnthropicOpti
544546
// caller-owned headers.
545547
let serverSideFallback = false;
546548

547-
if (options?.client) {
548-
client = options.client;
549+
if (requestOptions?.client) {
550+
client = requestOptions.client;
549551
isOAuth = false;
550552
} else {
551-
const apiKey = options?.apiKey ?? getEnvApiKey(model.provider) ?? "";
553+
const apiKey = requestOptions?.apiKey ?? getEnvApiKey(model.provider) ?? "";
552554

553555
let copilotDynamicHeaders: Record<string, string> | undefined;
554556
if (model.provider === "github-copilot") {
@@ -559,40 +561,48 @@ export const streamAnthropic: StreamFunction<"anthropic-messages", AnthropicOpti
559561
});
560562
}
561563

562-
const cacheRetention = options?.cacheRetention ?? resolveCacheRetention();
563-
const cacheSessionId = cacheRetention === "none" ? undefined : options?.sessionId;
564+
const cacheRetention = requestOptions?.cacheRetention ?? resolveCacheRetention();
565+
const cacheSessionId = cacheRetention === "none" ? undefined : requestOptions?.sessionId;
564566

565567
const created = createClient(
566568
model,
567569
apiKey,
568-
options?.thinkingEnabled === true,
569-
options?.interleavedThinking ?? true,
570+
requestOptions?.thinkingEnabled === true,
571+
requestOptions?.interleavedThinking ?? true,
570572
shouldUseFineGrainedToolStreamingBeta(model, requestContext),
571-
options?.headers,
573+
requestOptions?.headers,
572574
copilotDynamicHeaders,
573575
cacheSessionId,
574576
);
575577
client = created.client;
576578
isOAuth = created.isOAuthToken;
577579
serverSideFallback = created.serverSideFallback;
578580
}
579-
const builtParams = buildParams(model, requestContext, isOAuth, options, serverSideFallback);
581+
const builtParams = buildParams(
582+
model,
583+
requestContext,
584+
isOAuth,
585+
requestOptions,
586+
serverSideFallback,
587+
);
580588
let params = builtParams.params;
581589
const toolProjection = builtParams.toolProjection;
582-
const nextParams = await options?.onPayload?.(params, model);
590+
const nextParams = await requestOptions?.onPayload?.(params, model);
583591
if (nextParams !== undefined) {
584592
params = nextParams as MessageCreateParamsStreaming;
585593
}
586594
applyClaudeRequestContract(params as unknown as Record<string, unknown>, model);
587-
const requestOptions = {
588-
...(options?.signal ? { signal: options.signal } : {}),
589-
...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}),
590-
...(options?.maxRetries !== undefined ? { maxRetries: options.maxRetries } : {}),
595+
const sdkRequestOptions = {
596+
...(requestOptions?.signal ? { signal: requestOptions.signal } : {}),
597+
...(requestOptions?.timeoutMs !== undefined ? { timeout: requestOptions.timeoutMs } : {}),
598+
...(requestOptions?.maxRetries !== undefined
599+
? { maxRetries: requestOptions.maxRetries }
600+
: {}),
591601
};
592602
const response = await client.messages
593-
.create({ ...params, stream: true }, requestOptions)
603+
.create({ ...params, stream: true }, sdkRequestOptions)
594604
.asResponse();
595-
await options?.onResponse?.(
605+
await requestOptions?.onResponse?.(
596606
{ status: response.status, headers: headersToRecord(response.headers) },
597607
model,
598608
);
@@ -605,7 +615,7 @@ export const streamAnthropic: StreamFunction<"anthropic-messages", AnthropicOpti
605615

606616
for await (const event of iterateAnthropicEvents(
607617
response,
608-
options?.signal,
618+
requestOptions?.signal,
609619
refusalBuffer !== undefined,
610620
)) {
611621
if (event.type === "message_start") {
@@ -904,7 +914,7 @@ export const streamAnthropic: StreamFunction<"anthropic-messages", AnthropicOpti
904914
}
905915
}
906916

907-
if (options?.signal?.aborted) {
917+
if (requestOptions?.signal?.aborted) {
908918
throw new Error("Request was aborted");
909919
}
910920

@@ -925,7 +935,7 @@ export const streamAnthropic: StreamFunction<"anthropic-messages", AnthropicOpti
925935
refusalBuffer.discard();
926936
output.content = [];
927937
}
928-
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
938+
output.stopReason = requestOptions?.signal?.aborted ? "aborted" : "error";
929939
output.errorMessage = error instanceof Error ? error.message : JSON.stringify(error);
930940
stream.push({ type: "error", reason: output.stopReason, error: output });
931941
stream.end();
@@ -955,6 +965,25 @@ function supportsAdaptiveThinking(model: Model<"anthropic-messages">): boolean {
955965
return supportsClaudeAdaptiveThinking(model);
956966
}
957967

968+
function normalizeAnthropicThinkingOptions(
969+
model: Model<"anthropic-messages">,
970+
options: AnthropicOptions | undefined,
971+
): AnthropicOptions | undefined {
972+
if (options?.thinkingEnabled !== true || supportsAdaptiveThinking(model)) {
973+
return options;
974+
}
975+
976+
const budgetTokens = options.thinkingBudgetTokens ?? ANTHROPIC_MIN_THINKING_BUDGET_TOKENS;
977+
const maxTokens = options.maxTokens ?? model.maxTokens;
978+
if (budgetTokens >= ANTHROPIC_MIN_THINKING_BUDGET_TOKENS && budgetTokens < maxTokens) {
979+
return options;
980+
}
981+
982+
// Manual thinking is one request-wide mode: replay, sampling, tool choice,
983+
// headers, and payload construction must all observe the disabled state.
984+
return { ...options, thinkingEnabled: false, thinkingBudgetTokens: undefined };
985+
}
986+
958987
function supportsNativeXhighEffort(model: Model<"anthropic-messages">): boolean {
959988
return supportsClaudeNativeXhighEffort(model);
960989
}
@@ -1353,19 +1382,11 @@ function buildParams(
13531382
}
13541383
} else {
13551384
// Budget-based thinking for older models.
1356-
// Anthropic SDK requires budget_tokens >= 1024. Sub-minimum budgets
1357-
// (including explicit zero) skip the thinking block instead of
1358-
// producing a request the API would reject. Option resolution already
1359-
// normalizes these for simple/transport paths; this guard protects
1360-
// direct streamAnthropic and bundled-plugin callers.
1361-
const budgetTokens = options?.thinkingBudgetTokens ?? 1024;
1362-
if (budgetTokens >= 1024) {
1363-
params.thinking = {
1364-
type: "enabled",
1365-
budget_tokens: budgetTokens,
1366-
display,
1367-
};
1368-
}
1385+
params.thinking = {
1386+
type: "enabled",
1387+
budget_tokens: options?.thinkingBudgetTokens ?? ANTHROPIC_MIN_THINKING_BUDGET_TOKENS,
1388+
display,
1389+
};
13691390
}
13701391
} else if (options?.thinkingEnabled === false) {
13711392
params.thinking = { type: "disabled" };

0 commit comments

Comments
 (0)