Skip to content

Commit 4a6deef

Browse files
committed
Merge branch 'stable/1.106.x' into litellm_backport_p0_batch_translation_cost_stable_1_106_x
2 parents 0aefd1b + 25c3121 commit 4a6deef

3 files changed

Lines changed: 174 additions & 3 deletions

File tree

‎litellm/llms/bedrock/count_tokens/bedrock_token_counter.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
RUNTIME_TOKENIZER_TYPE: Final = "bedrock_api"
2222
MANTLE_TOKENIZER_TYPE: Final = "bedrock_mantle_api"
23+
_RUNTIME_FALLBACK_STATUSES: Final = frozenset({400, 404})
2324

2425

2526
class _CountTokensReply(LiteLLMPydanticObjectBase):
@@ -46,14 +47,14 @@ class CountTokensFailure:
4647
def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model: str) -> bool:
4748
return (
4849
isinstance(outcome, CountTokensFailure)
49-
and outcome.status_code == 400
50+
and outcome.status_code in _RUNTIME_FALLBACK_STATUSES
5051
and is_mantle_claude_model(resolved_model)
5152
)
5253

5354

5455
class BedrockTokenCounter(BaseTokenCounter):
5556
"""Token counter for AWS Bedrock: bedrock-runtime CountTokens first, and Anthropic's count_tokens
56-
on bedrock-mantle for the Claude models bedrock-runtime answers 400 for"""
57+
on bedrock-mantle for Claude models bedrock-runtime answers 400 or 404 for"""
5758

5859
def __init__(
5960
self,

‎tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -457,6 +457,140 @@ def test_gemini_count_tokens_route_reports_the_runtime_total_for_a_model_bedrock
457457
assert mantle.drain() == ()
458458

459459

460+
def test_runtime_404_on_claude_falls_back_to_mantle(counting_proxy: OwnedProxy, mantle_port: int) -> None:
461+
gateway: Final = counting_proxy.gateway
462+
system: Final = f"{_SYSTEM} status=404"
463+
with (
464+
wire_server(_runtime) as runtime,
465+
wire_server(_mantle(), port=mantle_port) as mantle,
466+
gateway.scenario() as scenario,
467+
):
468+
model: Final = _deployment(scenario, runtime.url)
469+
response: Final = _count(gateway, {**_bare(model), "system": system, "tools": _TOOLS})
470+
assert _payload(response) == {"input_tokens": _MANTLE_COUNT}, response.text
471+
assert len(_runtime_count_targets(runtime)) == 1
472+
assert _mantle_bodies(mantle) == (_mantle_body(system=system, tools=_TOOLS),)
473+
474+
475+
def test_runtime_404_and_mantle_404_surface_provider_error(counting_proxy: OwnedProxy, mantle_port: int) -> None:
476+
gateway: Final = counting_proxy.gateway
477+
system: Final = f"{_SYSTEM} status=404"
478+
with (
479+
wire_server(_runtime) as runtime,
480+
wire_server(_mantle(_rejecting(404)), port=mantle_port) as mantle,
481+
gateway.scenario() as scenario,
482+
):
483+
model: Final = _deployment(scenario, runtime.url)
484+
response: Final = _count(gateway, {**_bare(model), "system": system, "tools": _TOOLS})
485+
assert response.status_code == 404, response.text
486+
assert _REJECTION in response.text
487+
assert len(_runtime_count_targets(runtime)) == 1
488+
assert _mantle_bodies(mantle) == (_mantle_body(system=system, tools=_TOOLS),)
489+
490+
491+
def test_runtime_404_on_nova_does_not_call_mantle(counting_proxy: OwnedProxy, mantle_port: int) -> None:
492+
gateway: Final = counting_proxy.gateway
493+
with (
494+
wire_server(_runtime) as runtime,
495+
wire_server(_mantle(), port=mantle_port) as mantle,
496+
gateway.scenario() as scenario,
497+
):
498+
model: Final = _deployment(scenario, runtime.url, _NOVA)
499+
body: Final = {**_bare(model), "messages": [{"role": "user", "content": "status=404"}]}
500+
response: Final = _count(gateway, body)
501+
assert response.status_code == 404, response.text
502+
assert len(_runtime_count_targets(runtime)) == 1
503+
assert mantle.drain() == ()
504+
505+
506+
def test_google_count_tokens_forwards_system_and_tools_to_mantle(
507+
counting_proxy: OwnedProxy, mantle_port: int
508+
) -> None:
509+
gateway: Final = counting_proxy.gateway
510+
system_text: Final = "Use the provided tool"
511+
tool_schema: Final = {"type": "object", "properties": {"location": {"type": "string"}}}
512+
body: Final[dict[str, JsonValue]] = {
513+
"contents": [{"role": "user", "parts": [{"text": "status=404"}]}],
514+
"systemInstruction": {"parts": [{"text": system_text}]},
515+
"tools": [
516+
{
517+
"functionDeclarations": [
518+
{"name": "get_weather", "description": "Look up weather", "parameters": tool_schema}
519+
]
520+
}
521+
],
522+
}
523+
with (
524+
wire_server(_runtime) as runtime,
525+
wire_server(_mantle(), port=mantle_port) as mantle,
526+
gateway.scenario() as scenario,
527+
):
528+
model: Final = _deployment(scenario, runtime.url)
529+
response: Final = gateway.request("POST", f"/v1beta/models/{model}:countTokens", body)
530+
assert _payload(response) == {"totalTokens": _MANTLE_COUNT, "promptTokensDetails": []}, response.text
531+
assert len(_runtime_count_targets(runtime)) == 1
532+
assert _mantle_bodies(mantle) == (
533+
_mantle_body(
534+
messages=[{"role": "user", "content": "status=404"}],
535+
system=[{"type": "text", "text": system_text}],
536+
tools=[
537+
{
538+
"name": "get_weather",
539+
"description": "Look up weather",
540+
"input_schema": tool_schema,
541+
}
542+
],
543+
),
544+
)
545+
546+
547+
def test_google_count_tokens_normalizes_uppercase_parameters_for_mantle(
548+
counting_proxy: OwnedProxy, mantle_port: int
549+
) -> None:
550+
gateway: Final = counting_proxy.gateway
551+
body: Final[dict[str, JsonValue]] = {
552+
"contents": [{"role": "user", "parts": [{"text": "status=404"}]}],
553+
"tools": [
554+
{
555+
"functionDeclarations": [
556+
{
557+
"name": "get_weather",
558+
"parameters": {
559+
"type": "OBJECT",
560+
"properties": {"location": {"type": "STRING"}},
561+
"required": ["location"],
562+
},
563+
}
564+
]
565+
}
566+
],
567+
}
568+
with (
569+
wire_server(_runtime) as runtime,
570+
wire_server(_mantle(), port=mantle_port) as mantle,
571+
gateway.scenario() as scenario,
572+
):
573+
model: Final = _deployment(scenario, runtime.url)
574+
response: Final = gateway.request("POST", f"/v1beta/models/{model}:countTokens", body)
575+
assert _payload(response) == {"totalTokens": _MANTLE_COUNT, "promptTokensDetails": []}, response.text
576+
assert len(_runtime_count_targets(runtime)) == 1
577+
assert _mantle_bodies(mantle) == (
578+
_mantle_body(
579+
messages=[{"role": "user", "content": "status=404"}],
580+
tools=[
581+
{
582+
"name": "get_weather",
583+
"input_schema": {
584+
"type": "object",
585+
"properties": {"location": {"type": "string"}},
586+
"required": ["location"],
587+
},
588+
}
589+
],
590+
),
591+
)
592+
593+
460594
def test_responses_input_tokens_counts_through_mantle(counting_proxy: OwnedProxy, mantle_port: int) -> None:
461595
gateway: Final = counting_proxy.gateway
462596
with (

‎tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,8 +93,10 @@ async def test_bedrock_runtime_count_is_kept_when_it_answers() -> None:
9393
@pytest.mark.parametrize(
9494
("model_to_use", "runtime"),
9595
(
96+
("global.anthropic.claude-opus-4-8", (401, {"Message": "not authorized to perform: bedrock:CountTokens"})),
9697
("global.anthropic.claude-opus-4-8", (403, {"Message": "not authorized to perform: bedrock:CountTokens"})),
9798
("amazon.nova-pro-v1:0", (400, UNSUPPORTED)),
99+
("amazon.nova-pro-v1:0", (404, UNSUPPORTED)),
98100
),
99101
)
100102
async def test_other_bedrock_runtime_failures_are_not_retried_on_mantle(
@@ -120,7 +122,7 @@ async def test_other_bedrock_runtime_failures_are_not_retried_on_mantle(
120122
@pytest.mark.asyncio
121123
async def test_mantle_failure_is_reported_with_its_status() -> None:
122124
bedrock: Final = _Bedrock(
123-
runtime=(400, UNSUPPORTED),
125+
runtime=(404, UNSUPPORTED),
124126
mantle=(404, {"type": "error", "error": {"type": "not_found_error", "message": "does not exist"}}),
125127
)
126128

@@ -138,3 +140,37 @@ async def test_mantle_failure_is_reported_with_its_status() -> None:
138140
assert result.tokenizer_type == "bedrock_mantle_api"
139141
assert "does not exist" in (result.error_message or "")
140142
assert bedrock.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST)
143+
144+
145+
@pytest.mark.asyncio
146+
async def test_claude_runtime_404_retries_with_system_and_tools_on_mantle() -> None:
147+
bedrock: Final = _Bedrock(runtime=(404, UNSUPPORTED), mantle=(200, {"input_tokens": 2177}))
148+
tools: Final = [
149+
{
150+
"name": "lookup",
151+
"description": "Find a record",
152+
"input_schema": {"type": "object", "properties": {"query": {"type": "string"}}},
153+
}
154+
]
155+
156+
result: Final = await _counter(bedrock).count_tokens(
157+
model_to_use="global.anthropic.claude-opus-4-8",
158+
messages=MESSAGES,
159+
contents=None,
160+
deployment=DEPLOYMENT,
161+
request_model="claude-opus-4-8",
162+
system="You are a terse assistant.",
163+
tools=tools,
164+
)
165+
166+
assert result is not None
167+
assert result.error is False
168+
assert result.total_tokens == 2177
169+
assert bedrock.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST)
170+
mantle_body: Final = _JSON_BODY.validate_json(bedrock.requests[1].content)
171+
assert mantle_body == {
172+
"model": "anthropic.claude-opus-4-8",
173+
"messages": MESSAGES,
174+
"system": "You are a terse assistant.",
175+
"tools": tools,
176+
}

0 commit comments

Comments
 (0)