Repository navigation
Expand file tree
/
Copy pathlitellm_chat_target.py
More file actions
738 lines (643 loc) · 33.4 KB
/
Copy pathlitellm_chat_target.py
File metadata and controls
738 lines (643 loc) · 33.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.
import itertools
import logging
import os
from collections.abc import Awaitable, Callable, Mapping, MutableSequence, Sequence
from types import MappingProxyType
from typing import Any, ClassVar, NoReturn, cast
from pyrit.auth import ensure_async_token_provider
from pyrit.exceptions import (
EmptyResponseException,
PyritException,
RateLimitException,
get_retry_max_num_attempts,
handle_bad_request_exception,
)
from pyrit.models import (
ComponentIdentifier,
JsonResponseConfig,
Message,
MessagePiece,
PromptDataType,
)
from pyrit.prompt_target.common.chat_completions_message_builder import (
build_multimodal_chat_messages_async,
build_response_format,
build_text_chat_messages,
is_text_only_conversation,
should_skip_audio_piece,
validate_chat_tool_message,
)
from pyrit.prompt_target.common.chat_completions_response_parser import (
build_content_filter_message,
build_response_pieces_async,
capture_usage_and_finish_reason,
extract_partial_content,
get_finish_reason,
is_content_filter_response,
validate_chat_completion_response,
)
from pyrit.prompt_target.common.prompt_target import PromptTarget
from pyrit.prompt_target.common.target_capabilities import (
TargetCapabilities,
get_known_capabilities,
)
from pyrit.prompt_target.common.target_configuration import TargetConfiguration
from pyrit.prompt_target.common.target_trace_config import TargetTraceConfig, request_trace_headers
from pyrit.prompt_target.common.tool_call_history import TOOL_CALL_INPUT_MODALITIES
from pyrit.prompt_target.common.utils import (
build_empty_truncated_response,
limit_requests_per_minute,
validate_frequency_penalty,
validate_presence_penalty,
validate_temperature,
validate_top_p,
warn_truncated_response,
)
from pyrit.prompt_target.openai.openai_chat_audio_config import OpenAIChatAudioConfig
logger = logging.getLogger(__name__)
# Conservative capability profile used when litellm metadata can't tell us more (unknown
# model, or litellm not importable at construction). Deliberately text-only with no JSON so
# we never advertise a capability we can't honor.
_TEXT_INPUT: frozenset[frozenset[PromptDataType]] = cast(
"frozenset[frozenset[PromptDataType]]",
frozenset({frozenset({"text"})}),
)
def _build_input_modalities(*, image: bool, audio: bool) -> frozenset[frozenset[PromptDataType]]:
"""
Build the set of supported input-modality combinations from capability flags.
Always includes text. Enumerates every non-empty combination of the present modalities
(text, image, audio) so callers can advertise mixed-modality messages.
Args:
image (bool): Whether image input is supported.
audio (bool): Whether audio input is supported.
Returns:
frozenset[frozenset[PromptDataType]]: The supported input-modality combinations.
"""
present: list[PromptDataType] = ["text"]
if image:
present.append("image_path")
if audio:
present.append("audio_path")
combos = [
frozenset(combo) for size in range(1, len(present) + 1) for combo in itertools.combinations(present, size)
]
return frozenset(combos)
def _build_output_modalities(*, audio: bool) -> frozenset[frozenset[PromptDataType]]:
"""
Build the set of supported output-modality combinations from capability flags.
Args:
audio (bool): Whether audio output is supported.
Returns:
frozenset[frozenset[PromptDataType]]: The supported output-modality combinations.
"""
output: list[frozenset[PromptDataType]] = [cast("frozenset[PromptDataType]", frozenset({"text"}))]
if audio:
output.append(cast("frozenset[PromptDataType]", frozenset({"audio_path"})))
output.append(cast("frozenset[PromptDataType]", frozenset({"text", "audio_path"})))
return frozenset(output)
def _provider_specific_headers(provider_specific_header: Any) -> dict[str, str]:
"""
Collect the headers from every ``provider_specific_header`` entry.
LiteLLM accepts one entry or a sequence of entries and merges the ``extra_headers`` of each
entry scoped to the resolved provider. Every entry is collected so a conflict check does
not depend on provider resolution.
Args:
provider_specific_header (Any): The ``provider_specific_header`` request value.
Returns:
dict[str, str]: The headers of all entries.
"""
if isinstance(provider_specific_header, Mapping):
entries: Sequence[Any] = (provider_specific_header,)
elif isinstance(provider_specific_header, Sequence):
entries = provider_specific_header
else:
return {}
headers: dict[str, str] = {}
for entry in entries:
if isinstance(entry, Mapping):
headers.update(entry.get("extra_headers") or {})
return headers
class LiteLLMChatTarget(PromptTarget):
"""
Chat target that uses the LiteLLM SDK to access 100+ LLM providers.
Unlike ``OpenAIChatTarget`` (which uses the OpenAI SDK directly), this target calls
``litellm.acompletion()`` so it can route to any provider LiteLLM supports (Anthropic,
AWS Bedrock, Google Vertex, Cohere, etc.) without requiring a separate proxy server.
LiteLLM speaks the OpenAI *Chat Completions* wire format, so this target shares its
request-building and response-parsing logic with ``OpenAIChatTarget`` via the helpers in
``pyrit.prompt_target.common.chat_completions_message_builder`` and
``pyrit.prompt_target.common.chat_completions_response_parser``.
LiteLLM reads provider API keys from environment variables automatically
(e.g. ``ANTHROPIC_API_KEY``, ``AWS_ACCESS_KEY_ID``). You can also pass ``api_key``
explicitly, or a callable/token-provider for Entra-style auth.
Install the optional dependency with ``pip install pyrit[litellm]`` (or ``pip install
pyrit[all]``).
Args:
model_name: LiteLLM model string (e.g. ``"anthropic/claude-sonnet-4-6"``,
``"bedrock/anthropic.claude-v2"``, ``"vertex_ai/gemini-pro"``). Falls back to the
``LITELLM_MODEL`` environment variable.
api_key: Optional API key, or a callable that returns an access token (sync or async).
When omitted, falls back to the ``LITELLM_API_KEY`` environment variable and then
to LiteLLM's own provider-specific environment variable lookup.
endpoint: Optional base URL override (e.g. for a self-hosted proxy or LiteLLM gateway).
Falls back to the ``LITELLM_ENDPOINT`` environment variable.
headers: Optional extra HTTP headers forwarded to the provider (``extra_headers``).
temperature: Sampling temperature (0-2).
top_p: Nucleus sampling probability (0-1).
max_tokens: Maximum number of tokens to generate. This is the single token-limit knob:
LiteLLM normalizes it to the parameter each model/provider expects (for example, it
maps to ``max_completion_tokens`` for OpenAI reasoning and gpt-5 models). To send a
provider-specific token parameter directly instead, use ``extra_body_parameters``.
frequency_penalty: Penalize frequently generated tokens; a finite value in [-2, 2], or None.
presence_penalty: Penalize tokens already present in the conversation; a finite value in [-2, 2], or None.
seed: Best-effort deterministic sampling seed.
n: Number of completions to generate.
stop: Stop sequence(s).
audio_response_config: Optional audio-output configuration (voice + format). When set,
audio modality is enabled on the request (``modalities``/``audio``) for models that
support it (e.g. ``gpt-4o-audio-preview``), and audio responses are saved as
``audio_path`` pieces alongside their transcript.
drop_unsupported_params: When True (the default), LiteLLM silently drops any request
parameter the resolved provider does not support instead of raising. This is a core
LiteLLM behavior (it maps to LiteLLM's ``drop_params``) that lets a single target
send the full OpenAI parameter set across many providers. Set to False for strict
validation, where an unsupported parameter raises instead of being dropped.
extra_body_parameters: Additional provider parameters merged into the request body
(passthrough). These may also override the target's defaults (including
``drop_params``, e.g. pass ``{"drop_params": False}`` to force strict validation for
a single request, or ``{"timeout": 30}`` to set a per-request timeout).
underlying_model: The underlying model name (e.g. ``"gpt-4o"``) used for capability
lookup and identification when the provider/model string differs from a known model.
max_requests_per_minute: Client-side request cap.
custom_configuration: Override the derived target configuration.
trace_config: Request tracing configuration. Tracing is disabled by default because a
provider or gateway is not known to accept W3C trace context. Pass
``TargetTraceConfig(enabled=True)`` for an instrumented endpoint; each request then
sends a fresh ``traceparent`` in ``extra_headers``.
"""
server_fallback_parameters: ClassVar[Mapping[str, str | None]] = MappingProxyType({"api_key": None})
# Fallback only. The real per-instance configuration is normally derived from LiteLLM's
# model metadata at construction time (see ``_derive_capabilities_from_litellm``).
_SUPPORTS_TOOL_CALL_HISTORY = True
_DEFAULT_CONFIGURATION: TargetConfiguration = TargetConfiguration(
capabilities=TargetCapabilities(
supports_multi_turn=True,
supports_multi_message_pieces=True,
supports_editable_history=True,
supports_prefill=True,
supports_system_prompt=True,
input_modalities=_TEXT_INPUT,
)
)
def __init__(
self,
*,
model_name: str | None = None,
api_key: str | Callable[[], str | Awaitable[str]] | None = None,
endpoint: str | None = None,
headers: dict[str, str] | None = None,
temperature: float | None = None,
top_p: float | None = None,
max_tokens: int | None = None,
frequency_penalty: float | None = None,
presence_penalty: float | None = None,
seed: int | None = None,
n: int | None = None,
stop: str | list[str] | None = None,
audio_response_config: OpenAIChatAudioConfig | None = None,
drop_unsupported_params: bool = True,
extra_body_parameters: dict[str, Any] | None = None,
underlying_model: str | None = None,
max_requests_per_minute: int | None = None,
custom_configuration: TargetConfiguration | None = None,
trace_config: TargetTraceConfig | None = None,
) -> None:
"""
Initialize a LiteLLMChatTarget.
Raises:
ValueError: If model_name is not provided and LITELLM_MODEL env var is not set.
PyritException: If a generation parameter is out of bounds or a penalty is non-finite.
"""
resolved_model = model_name or os.environ.get("LITELLM_MODEL", "")
if not resolved_model:
raise ValueError("model_name is required. Pass it directly or set the LITELLM_MODEL environment variable.")
validate_temperature(temperature)
validate_top_p(top_p)
validate_presence_penalty(presence_penalty)
validate_frequency_penalty(frequency_penalty)
super().__init__(
model_name=resolved_model,
underlying_model=underlying_model,
max_requests_per_minute=max_requests_per_minute,
custom_configuration=custom_configuration,
trace_config=trace_config,
)
# Resolve api_key: explicit value/callable > LITELLM_API_KEY env var > None (LiteLLM
# then reads provider-specific env vars itself). ``ensure_async_token_provider`` wraps a
# sync token provider so we can uniformly await it at request time.
if api_key is None:
api_key = os.environ.get("LITELLM_API_KEY")
self._api_key = ensure_async_token_provider(api_key)
self._endpoint = endpoint or os.environ.get("LITELLM_ENDPOINT")
self._headers = headers
self._temperature = temperature
self._top_p = top_p
self._max_tokens = max_tokens
self._frequency_penalty = frequency_penalty
self._presence_penalty = presence_penalty
self._seed = seed
self._n = n
self._stop = stop
self._audio_response_config = audio_response_config
self._drop_unsupported_params = drop_unsupported_params
# Merge audio-output config into the passthrough body (modalities + audio params), so it
# rides the same passthrough OpenAIChatTarget uses.
if audio_response_config:
audio_params = audio_response_config.to_extra_body_parameters()
extra_body_parameters = {**audio_params, **extra_body_parameters} if extra_body_parameters else audio_params
self._extra_body_parameters = extra_body_parameters
# Delegate transient/rate-limit retry to LiteLLM (provider-aware: honors ``Retry-After``
# and per-provider rate-limit semantics), rather than stacking PyRIT's
# ``pyrit_target_retry`` (which would double-retry). Derive the count from PyRIT's global
# ``RETRY_MAX_NUM_ATTEMPTS`` convention; ``num_retries`` is a retry count so it is
# attempts minus one. Per-request timeout is left to LiteLLM's own default; advanced
# callers can override it via ``extra_body_parameters={"timeout": ...}``.
self._num_retries = max(get_retry_max_num_attempts() - 1, 0)
# Capability precedence: custom_configuration > known underlying_model profile >
# LiteLLM-derived default > conservative fallback. The base __init__ already applied the
# first two; only derive from LiteLLM metadata when neither was supplied.
if custom_configuration is None:
known = get_known_capabilities(underlying_model) if underlying_model else None
if known is None:
derived = self._derive_capabilities_from_litellm(resolved_model)
if derived is not None:
self._configuration = TargetConfiguration(capabilities=derived)
@staticmethod
def _import_litellm() -> Any:
try:
import litellm
except ImportError as e:
raise ImportError(
"The litellm package is required for LiteLLMChatTarget. Install it with `pip install pyrit[litellm]`."
) from e
return litellm
def _derive_capabilities_from_litellm(self, model: str) -> TargetCapabilities | None:
"""
Best-effort capability derivation from LiteLLM's model metadata.
Uses LiteLLM's own model-capability helpers (``supports_vision``,
``supports_response_schema``, ``get_supported_openai_params``) rather than reinventing a
per-provider capability table. Returns None if LiteLLM is unavailable so the caller falls
back to ``_DEFAULT_CONFIGURATION``.
Args:
model (str): The LiteLLM model string to inspect.
Returns:
TargetCapabilities | None: The derived capabilities, or None if LiteLLM is
unavailable.
"""
try:
litellm = self._import_litellm()
except ImportError:
return None
def _supports(attr: str) -> bool:
fn = getattr(litellm, attr, None)
if fn is None:
return False
try:
return bool(fn(model))
except Exception:
return False
supports_vision = _supports("supports_vision")
supports_json_schema = _supports("supports_response_schema")
supports_audio_input = _supports("supports_audio_input")
supports_audio_output = _supports("supports_audio_output")
try:
supported_params = litellm.get_supported_openai_params(model=model) or []
except Exception:
supported_params = []
supports_json_output = supports_json_schema or ("response_format" in supported_params)
return TargetCapabilities(
supports_multi_turn=True,
supports_multi_message_pieces=True,
supports_editable_history=True,
supports_prefill=self._DEFAULT_CONFIGURATION.capabilities.supports_prefill,
supports_system_prompt=True,
supports_json_output=supports_json_output,
supports_json_schema=supports_json_schema,
input_modalities=_build_input_modalities(image=supports_vision, audio=supports_audio_input)
| (TOOL_CALL_INPUT_MODALITIES if _supports("supports_function_calling") else frozenset()),
output_modalities=_build_output_modalities(audio=supports_audio_output),
)
def validate_tool_history(self, messages: Sequence[Message]) -> None:
"""Check stored tool history and Chat Completions tool-message constraints."""
super().validate_tool_history(messages)
for message in messages:
validate_chat_tool_message(message)
def _build_identifier(self) -> ComponentIdentifier:
"""
Build the identifier with LiteLLM-specific behavioral parameters.
The API key is intentionally excluded.
Returns:
ComponentIdentifier: The identifier for this target instance.
"""
return self._create_identifier(
params={
"endpoint": self._endpoint,
"temperature": self._temperature,
"top_p": self._top_p,
"max_tokens": self._max_tokens,
"frequency_penalty": self._frequency_penalty,
"presence_penalty": self._presence_penalty,
"seed": self._seed,
"n": self._n,
"stop": self._stop,
"extra_body_parameters": self._extra_body_parameters,
},
)
def is_json_response_supported(self) -> bool:
"""
Whether this target honors a JSON ``response_format`` request.
Returns:
bool: True if the target advertises JSON output support.
"""
return self.capabilities.supports_json_output
# Not decorated with ``pyrit_target_retry``: LiteLLM owns transient/rate-limit retry via
# ``num_retries`` (see ``__init__``). Stacking both would multiply the retry count.
@limit_requests_per_minute
async def _send_prompt_to_target_async(self, *, normalized_conversation: list[Message]) -> list[Message]:
litellm = self._import_litellm()
message = normalized_conversation[-1]
request_piece: MessagePiece = message.message_pieces[0]
logger.info(f"Sending prompt to LiteLLM target ({self._model_name}): {message}")
json_config = self._get_json_response_config(message_piece=request_piece)
messages = await self._build_chat_messages_async(normalized_conversation)
api_key = await self._resolve_api_key_async()
body = self._construct_request_body(messages=messages, json_config=json_config, api_key=api_key)
# Applied after the passthrough merge so ``extra_body_parameters`` cannot drop the context.
# LiteLLM merges ``headers`` with ``extra_headers`` and then the provider-specific headers,
# so all three are checked for manual values.
trace_headers = request_trace_headers(
request=message,
headers=body.get("extra_headers") or {},
default_headers={
**(body.get("headers") or {}),
**_provider_specific_headers(body.get("provider_specific_header")),
},
)
if trace_headers:
body["extra_headers"] = trace_headers
try:
response = await litellm.acompletion(**body)
except Exception as exc:
return self._handle_litellm_exception(exc=exc, request=request_piece)
# Content filtering is red-team critical: surface it as an error Message (not an
# exception) so attacks can continue and blocked-content scorers can still score.
if is_content_filter_response(response):
logger.warning("Output content filtered by content policy.")
filter_message = build_content_filter_message(
response=response,
request=request_piece,
partial_content=extract_partial_content(response),
)
# A filtered response still reports the tokens (and cost) it consumed, which would
# otherwise be lost precisely where a red teamer most wants to see it.
capture_usage_and_finish_reason(pieces=filter_message.message_pieces, response=response)
self._capture_response_cost(pieces=filter_message.message_pieces, response=response)
return [filter_message]
# A response cut off at the token limit may legitimately carry a partial answer (or no
# content at all), so skip the strict empty-response validation for it — mirroring
# OpenAIChatTarget. Truncation is flagged on the piece by the construct step below.
if self._is_truncated_response(response):
warn_truncated_response(signal="finish_reason='length'", limit_parameter="max_tokens")
else:
validate_chat_completion_response(response=response)
return [await self._construct_message_from_response_async(response=response, request=request_piece)]
async def _resolve_api_key_async(self) -> str | None:
"""
Resolve the api_key to a concrete string, awaiting async token providers.
Returns:
str | None: The resolved API key, or None when no key is configured.
"""
api_key = self._api_key
if api_key is None or isinstance(api_key, str):
return api_key
result = api_key()
if isinstance(result, Awaitable):
result = await result
return result
async def _build_chat_messages_async(self, conversation: MutableSequence[Message]) -> list[dict[str, Any]]:
# Text-only conversations use the simpler {"role", "content": str} form, which the widest
# set of OpenAI-"compatible" providers accept.
if is_text_only_conversation(conversation):
return build_text_chat_messages(conversation)
prefer_transcript_for_history = bool(
self._audio_response_config and self._audio_response_config.prefer_transcript_for_history
)
return await build_multimodal_chat_messages_async(
conversation, prefer_transcript_for_history=prefer_transcript_for_history
)
def _get_history_piece_data_type(
self, *, message_piece: MessagePiece, has_text_piece: bool
) -> PromptDataType | None:
"""
Match the shared Chat Completions builder's historical wire modalities.
Args:
message_piece: Historical message piece to check.
has_text_piece: Whether its message contains a text piece.
Returns:
PromptDataType | None: The wire modality, or None for omitted audio.
"""
prefer_transcript_for_history = bool(
self._audio_response_config and self._audio_response_config.prefer_transcript_for_history
)
if should_skip_audio_piece(
message_piece=message_piece,
is_last_message=False,
has_text_piece=has_text_piece,
prefer_transcript_for_history=prefer_transcript_for_history,
):
return None
if message_piece.is_blocked_error():
return "text"
return message_piece.converted_value_data_type
def _construct_request_body(
self,
*,
messages: list[dict[str, Any]],
json_config: JsonResponseConfig,
api_key: str | None = None,
) -> dict[str, Any]:
body: dict[str, Any] = {
"model": self._model_name,
"messages": messages,
# Drop provider-unsupported params (LiteLLM's ``drop_params``). As a cross-provider
# target we send the full OpenAI param set, but providers support different subsets;
# with this enabled LiteLLM drops what a provider does not accept instead of raising.
# Controlled by the ``drop_unsupported_params`` constructor arg; advanced callers can
# still override per request via ``extra_body_parameters={"drop_params": ...}``.
"drop_params": self._drop_unsupported_params,
"api_key": api_key,
"api_base": self._endpoint,
"extra_headers": self._headers,
"temperature": self._temperature,
"top_p": self._top_p,
"max_tokens": self._max_tokens,
"frequency_penalty": self._frequency_penalty,
"presence_penalty": self._presence_penalty,
"seed": self._seed,
"n": self._n,
"stop": self._stop,
"num_retries": self._num_retries,
"response_format": build_response_format(json_config=json_config),
}
# Passthrough for arbitrary provider params (may override defaults above, e.g. drop_params).
if self._extra_body_parameters:
body.update(self._extra_body_parameters)
return {k: v for k, v in body.items() if v is not None}
@staticmethod
def _is_truncated_response(response: Any) -> bool:
"""
Whether the response was cut off at the output-token limit.
Args:
response (Any): The LiteLLM completion response object.
Returns:
bool: True when the first choice's ``finish_reason`` is ``"length"``.
"""
return get_finish_reason(response=response) == "length"
async def _construct_message_from_response_async(self, *, response: Any, request: MessagePiece) -> Message:
"""
Construct a Message from a LiteLLM completion response.
Args:
response (Any): The LiteLLM completion response object.
request (MessagePiece): The originating request piece.
Returns:
Message: Constructed message with one or more MessagePiece entries.
Raises:
EmptyResponseException: If a non-truncated response contains no content, audio, or tool
calls. A truncated (``finish_reason == "length"``) response with no content instead
yields a graceful empty piece so the run continues. Truncated responses are flagged
via ``MessagePiece.mark_as_truncated`` on the first piece.
"""
audio_format = self._audio_response_config.audio_format if self._audio_response_config else "wav"
truncated = self._is_truncated_response(response)
pieces = await build_response_pieces_async(response=response, request=request, audio_format=audio_format)
if not pieces:
# A truncated (finish_reason == "length") response may legitimately produce no content;
# return a graceful empty piece so the run continues. Validation already raised for
# genuinely empty (non-truncated) responses.
if truncated:
empty_message = build_empty_truncated_response(request=request)
capture_usage_and_finish_reason(pieces=empty_message.message_pieces, response=response)
self._capture_response_cost(pieces=empty_message.message_pieces, response=response)
empty_message.message_pieces[0].mark_as_truncated()
return empty_message
raise EmptyResponseException(message="Failed to extract any response content from LiteLLM.")
capture_usage_and_finish_reason(pieces=pieces, response=response)
self._capture_response_cost(pieces=pieces, response=response)
if truncated:
pieces[0].mark_as_truncated()
return Message(message_pieces=pieces)
def _capture_response_cost(self, *, pieces: list[MessagePiece], response: Any) -> None:
"""
Record LiteLLM's computed per-call dollar cost into the first piece's metadata.
LiteLLM attaches the spend for a completion at ``response._hidden_params["response_cost"]``
and, failing that, can recompute it via ``litellm.completion_cost``. Cost is provider- and
model-aware and is unique to LiteLLM (the raw OpenAI SDK response carries no cost), so this
mirrors ``capture_token_usage`` and writes ``token_usage_cost`` alongside the token counts.
The value is stored as a string to honor the ``prompt_metadata`` value contract, and any
failure is swallowed so cost accounting never breaks the response path. This has to run
after usage capture, which clears the whole ``token_usage_`` prefix and would otherwise
drop the cost again.
Args:
pieces (list[MessagePiece]): The constructed response pieces.
response (Any): The LiteLLM completion response object.
"""
if not pieces:
return
cost = self._extract_response_cost(response=response)
if cost is None:
return
pieces[0].prompt_metadata["token_usage_cost"] = str(cost)
@staticmethod
def _extract_response_cost(*, response: Any) -> float | None:
"""
Pull the per-call cost from a LiteLLM response, or None when it cannot be determined.
Prefers LiteLLM's authoritative post-call ``_hidden_params["response_cost"]`` (which may be
``0.0`` for free/local models) and falls back to recomputing via ``litellm.completion_cost``.
Args:
response (Any): The LiteLLM completion response object.
Returns:
float | None: The cost in dollars, or None if unavailable.
"""
hidden_params = getattr(response, "_hidden_params", None)
if isinstance(hidden_params, dict) and hidden_params.get("response_cost") is not None:
try:
return float(hidden_params["response_cost"])
except (TypeError, ValueError):
return None
try:
litellm = LiteLLMChatTarget._import_litellm()
cost = litellm.completion_cost(completion_response=response)
return float(cost) if cost else None
except Exception:
return None
def _handle_litellm_exception(self, *, exc: Exception, request: MessagePiece) -> list[Message]:
"""
Translate a LiteLLM exception into either a blocked-content error Message or a PyRIT
exception. LiteLLM re-exports the OpenAI SDK exception classes, so we match on those
types (``isinstance``) rather than fragile string/qualname comparisons.
Args:
exc (Exception): The exception raised by ``litellm.acompletion``.
request (MessagePiece): The originating request piece.
Returns:
list[Message]: A single error Message when the failure is a content-policy block.
Raises:
RateLimitException: For rate-limit and transient provider errors.
PyritException: For authentication and all other errors.
"""
litellm = self._import_litellm()
exceptions = litellm.exceptions
# Content policy violations are surfaced as an error Message (not raised) so attacks
# continue and blocked-content scorers can score the refusal.
if self._is_content_policy_error(exc=exc, exceptions=exceptions):
status_code = getattr(exc, "status_code", 400) or 400
return [
handle_bad_request_exception(
response_text=str(exc),
request=request,
error_code=status_code,
is_content_filter=True,
)
]
return self._raise_translated_exception(exc=exc, exceptions=exceptions)
@staticmethod
def _is_content_policy_error(*, exc: Exception, exceptions: Any) -> bool:
content_policy_error = getattr(exceptions, "ContentPolicyViolationError", None)
if content_policy_error is not None and isinstance(exc, content_policy_error):
return True
text = str(exc).lower()
return "content_filter" in text or "content policy" in text or "content_policy" in text
@staticmethod
def _raise_translated_exception(*, exc: Exception, exceptions: Any) -> NoReturn:
rate_limit_error = getattr(exceptions, "RateLimitError", ())
transient_errors = tuple(
err
for err in (
getattr(exceptions, "APIConnectionError", None),
getattr(exceptions, "Timeout", None),
getattr(exceptions, "InternalServerError", None),
getattr(exceptions, "ServiceUnavailableError", None),
)
if err is not None
)
authentication_error = getattr(exceptions, "AuthenticationError", ())
if isinstance(exc, rate_limit_error):
raise RateLimitException(status_code=429, message=f"Rate limited by provider: {exc}") from exc
if transient_errors and isinstance(exc, transient_errors):
status_code = getattr(exc, "status_code", 503) or 503
raise RateLimitException(status_code=status_code, message=f"Transient provider error: {exc}") from exc
if isinstance(exc, authentication_error):
raise PyritException(message=f"Authentication failed: {exc}") from exc
status_code = getattr(exc, "status_code", None) or 500
raise PyritException(status_code=status_code, message=f"LiteLLM error: {exc}") from exc