Skip to content

Commit 78c56ae

Browse files
nik13claude
andcommitted
fix(python): string-only error description cleanup, hide raw.input/raw.output, error.type fallback
- clean_error_description no longer builds a regex from exception messages. It compiled and ran one on every ERROR set_status in default mode (100 KB message: 0.83 s, 1 MB: 6.4 s) and left each message in the global re cache. It now strips the known "<Type>: " line prefixes with str.find/startswith (1 MB: ~1 ms). - Content-hiding fix: hide_inputs also redacts raw.input and the prompt template variables; hide_outputs also redacts raw.output (same keys as @traceai/fi-core). - error.type: ERROR with no exception to name gets "_OTHER"; a recovered exception the status does not mention is no longer used; the in-flight exception, an exception recorded as escaped, or a recorded one the status description mentions still are. - Correct the litellm comment: its exceptions carry no code/type. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent c845102 commit 78c56ae

6 files changed

Lines changed: 276 additions & 36 deletions

File tree

‎python/CHANGELOG.md‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,15 @@
1414
`"<Type>: <message>"` form built by OpenTelemetry's `use_span` and most instrumentors
1515
becomes `<message>` (the type is in `error.type`), also when one such line is joined
1616
per exception (llama-index), and a Python traceback is never kept in the description.
17+
The cleanup uses plain string operations, so a multi-megabyte exception message adds
18+
about a millisecond to `set_status` (no regular expression is built from message text).
19+
20+
- **Content-hiding fix:** `hide_inputs` (`FI_HIDE_INPUTS`) now also redacts `raw.input`
21+
(the raw request body, messages included) and the prompt template variables
22+
(`llm.prompt_template.variables`, `gen_ai.prompt.template.variables`), and
23+
`hide_outputs` (`FI_HIDE_OUTPUTS`) also redacts `raw.output`. Before, an instrumentation
24+
that set these kept the prompt in the stored row while the hide flag was on. Same keys
25+
as `@traceai/fi-core` 1.1.0.
1726

1827
### Feature
1928
- New `TraceConfig(hide_error_details=...)` / `FI_HIDE_ERROR_DETAILS`. Unset, it is
@@ -22,12 +31,16 @@
2231
unless it was set explicitly.
2332
- Spans that end with status `ERROR` get `error.type` (an exception recorded on a span
2433
that still ends OK, such as a recovered retry, does not set it). It is the provider
25-
error code for exceptions from known provider SDKs: openai, groq and litellm
34+
error code for exceptions from known provider SDKs: openai and groq
2635
`code`/`type`, anthropic `error.type`, botocore/boto3 `Error.Code`, google-genai
2736
`status`, google-api-core gRPC status, mistralai body `type`, and the HTTP status for
2837
httpx `HTTPStatusError`. Every other exception gets its fully qualified class name
2938
(for example `sqlalchemy.exc.OperationalError`, never its `.code`). An instrumentor's
30-
own `error.type` is not overwritten.
39+
own `error.type` is not overwritten. A failed span whose status is not about a
40+
named exception (for example `set_status(ERROR, "guardrail rejected")` after a retried
41+
timeout, outside an `except` block) gets `_OTHER` (OpenTelemetry semantic conventions)
42+
instead of a stale recovered exception. litellm exceptions carry no `code`/`type`, so
43+
they get their class name (for example `litellm.exceptions.NotFoundError`).
3144

3245
### Fixed
3346
- `with tracer.start_span(...) as span:` now yields the FI span wrapper instead of the

‎python/fi_instrumentation/fi_types.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,15 @@ class SpanAttributes:
199199
"""Prompts provided to a completions API."""
200200

201201
# Input/Output values (OpenInference compat)
202+
RAW_INPUT = "raw.input"
203+
"""The raw request body (e.g. the full provider request, messages included)"""
204+
205+
RAW_OUTPUT = "raw.output"
206+
"""The raw response body"""
207+
208+
LLM_PROMPT_TEMPLATE_VARIABLES = "llm.prompt_template.variables"
209+
"""JSON of the values filled into a prompt template"""
210+
202211
OUTPUT_VALUE = "output.value"
203212
OUTPUT_MIME_TYPE = "output.mime_type"
204213
INPUT_VALUE = "input.value"

‎python/fi_instrumentation/instrumentation/_errors.py‎

Lines changed: 43 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010
import json
1111
import re
12-
from typing import Callable, Dict, Iterable, Mapping, NamedTuple, Optional, Tuple, Union
12+
from typing import Callable, Dict, Iterable, List, Mapping, NamedTuple, Optional, Tuple, Union
1313

1414
from opentelemetry.util.types import AttributeValue
1515

@@ -20,6 +20,8 @@
2020
EXCEPTION_ESCAPED = "exception.escaped"
2121
ERROR_TYPE = "error.type"
2222
ERROR_MESSAGE = "error.message"
23+
# error.type when the failure has no exception to name (OpenTelemetry semconv).
24+
ERROR_TYPE_OTHER = "_OTHER"
2325

2426
# Attribute keys that carry error text. They are dropped when
2527
# ``TraceConfig.hide_error_details`` is on.
@@ -85,8 +87,10 @@ def _code_from_error_body(body: object) -> Optional[str]:
8587

8688

8789
def _stainless_code(exception: BaseException) -> Optional[str]:
88-
# openai (and litellm, whose errors subclass openai's): `code` and `type`
89-
# from the error object, e.g. "model_not_found" / "invalid_request_error".
90+
# openai: `code` and `type` from the error object, e.g. "model_not_found" /
91+
# "invalid_request_error". litellm's errors subclass openai's but are built
92+
# with code=None, body=None and type=None, so they fall back to the class
93+
# name (e.g. "litellm.exceptions.NotFoundError").
9094
if code := _as_code(getattr(exception, "code", None)):
9195
return code
9296
# groq: body = {"error": {"code": "model_not_found", ...}}; older anthropic:
@@ -221,20 +225,36 @@ def _cut_traceback(description: str) -> Optional[str]:
221225
return description[:index].rstrip() or None
222226

223227

224-
def _type_prefix_pattern(summaries: Iterable[ExceptionSummary]) -> Optional[str]:
228+
def _prefix_cuts(
229+
description: str, summaries: Iterable[ExceptionSummary]
230+
) -> List[Tuple[int, int]]:
225231
# "<Type>: " at the start of a line, when the exception's own message
226-
# follows it up to the end of that line (or of the description).
227-
alternatives = []
232+
# follows it up to the end of that line (or of the description). Plain string
233+
# operations only: exception messages can be megabytes and quote the prompt,
234+
# so they never go into a regular expression (slow to compile and match, and
235+
# kept in the global `re` cache after the span ends).
236+
size = len(description)
237+
cuts: Dict[int, int] = {}
228238
for summary in summaries:
229-
names = "|".join(re.escape(name) for name in summary.type_names)
230-
if summary.message:
231-
follows = re.escape(summary.message) + r"(?:\n|\Z)"
232-
alternatives.append(f"(?:{names}): (?={follows})")
233-
else:
234-
alternatives.append(rf"(?:{names}): ?(?=\n|\Z)")
235-
if not alternatives:
236-
return None
237-
return "^(?:" + "|".join(alternatives) + ")"
239+
for name in summary.type_names:
240+
prefix = f"{name}:" if not summary.message else f"{name}: "
241+
start = 0
242+
while (pos := description.find(prefix, start)) != -1:
243+
start = pos + 1
244+
if pos and description[pos - 1] != "\n":
245+
continue
246+
end = pos + len(prefix)
247+
if summary.message:
248+
if not description.startswith(summary.message, end):
249+
continue
250+
tail = end + len(summary.message)
251+
else:
252+
tail = end + 1 if description.startswith(" ", end) else end
253+
end = tail
254+
if tail != size and description[tail] != "\n":
255+
continue
256+
cuts.setdefault(pos, end)
257+
return sorted(cuts.items())
238258

239259

240260
def clean_error_description(
@@ -259,11 +279,15 @@ def clean_error_description(
259279
)
260280
)
261281
cleaned = description
262-
if pattern := _type_prefix_pattern(summaries):
282+
if cuts := _prefix_cuts(description, summaries):
263283
# One pass, so a message that itself starts with "<Type>: " is kept.
264-
cleaned = re.sub(pattern, "", description, flags=re.MULTILINE)
265-
if cleaned != description:
266-
cleaned = cleaned.strip("\n")
284+
parts = []
285+
last = 0
286+
for pos, end in cuts:
287+
parts.append(description[last:pos])
288+
last = end
289+
parts.append(description[last:])
290+
cleaned = "".join(parts).strip("\n")
267291
if _TRACEBACK_MARKER in cleaned:
268292
if summaries:
269293
message = summaries[0].message

‎python/fi_instrumentation/instrumentation/_spans.py‎

Lines changed: 65 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import sys
22
from types import TracebackType
3-
from typing import Any, Callable, Dict, List, Optional, Type, Union, cast
3+
from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union, cast
44

55
import wrapt # type: ignore[import-untyped]
66
from fi_instrumentation.fi_types import FiSpanKindValues, SpanAttributes
@@ -14,8 +14,10 @@
1414
)
1515
from ._errors import (
1616
ERROR_TYPE,
17+
ERROR_TYPE_OTHER,
1718
EXCEPTION_ESCAPED,
1819
EXCEPTION_EVENT_NAME,
20+
EXCEPTION_MESSAGE,
1921
EXCEPTION_TYPE,
2022
ExceptionSummary,
2123
clean_error_description,
@@ -43,8 +45,9 @@ class FiSpan(wrapt.ObjectProxy): # type: ignore[misc]
4345
4446
- ``error.type`` is set when the span ends with status ERROR, from the
4547
exception that failed it (provider error code or fully qualified class
46-
name). An exception recorded on a span that still ends OK, such as a
47-
recovered retry, does not set it.
48+
name), else ``_OTHER``. An exception recorded on a span that still ends
49+
OK, such as a recovered retry, does not set it, and neither does a
50+
recovered exception set it on a later unrelated ERROR status.
4851
- An ERROR status description is reduced to the exception message; a Python
4952
traceback never ends up in it.
5053
- With ``hide_error_details`` on, the ERROR status has no description and
@@ -58,9 +61,15 @@ def __init__(self, wrapped: Span, config: TraceConfig) -> None:
5861
# Only summaries (class names, message) of recorded exceptions are kept,
5962
# never the exceptions, so their traceback frames and locals are freed.
6063
self._self_exception_summaries: List[ExceptionSummary] = []
61-
# error.type of the last recorded exception, and of the exception that
62-
# was being handled when the status was set to ERROR.
63-
self._self_recorded_error_type: Optional[str] = None
64+
# error.type of the last exception recorded as escaped, and of the
65+
# exception that was being handled when the status was set to ERROR.
66+
self._self_escaped_error_type: Optional[str] = None
67+
# Recently recorded exceptions (error.type, summary) and the description
68+
# of the last ERROR status, to tell which recorded exception a status
69+
# set outside an `except` block is about (LangChain records the error in
70+
# a callback and sets the status later).
71+
self._self_recorded: List[Tuple[str, ExceptionSummary]] = []
72+
self._self_status_description: Optional[str] = None
6473
self._self_status_error_type: Optional[str] = None
6574
self._self_status_code: Optional[StatusCode] = None
6675

@@ -88,6 +97,10 @@ def __exit__(
8897
)
8998
self.end()
9099

100+
def _remember_recorded(self, error_type: str, summary: ExceptionSummary) -> None:
101+
self._self_recorded.append((error_type, summary))
102+
del self._self_recorded[:-_MAX_EXCEPTION_SUMMARIES]
103+
91104
def record_exception(
92105
self,
93106
exception: BaseException,
@@ -98,7 +111,10 @@ def record_exception(
98111
span = cast(Span, self.__wrapped__)
99112
if not span.is_recording():
100113
return
101-
self._self_recorded_error_type = get_error_type(exception)
114+
error_type = get_error_type(exception)
115+
self._remember_recorded(error_type, summarize_exception(exception))
116+
if escaped:
117+
self._self_escaped_error_type = error_type
102118
if not self._self_config.hide_error_details:
103119
self._self_exception_summaries.append(summarize_exception(exception))
104120
del self._self_exception_summaries[:-_MAX_EXCEPTION_SUMMARIES]
@@ -124,8 +140,17 @@ def add_event(
124140
span = cast(Span, self.__wrapped__)
125141
if name == EXCEPTION_EVENT_NAME and attributes:
126142
exception_type = attributes.get(EXCEPTION_TYPE)
143+
escaped = attributes.get(EXCEPTION_ESCAPED)
127144
if isinstance(exception_type, str) and exception_type:
128-
self._self_recorded_error_type = exception_type
145+
message = attributes.get(EXCEPTION_MESSAGE)
146+
self._remember_recorded(
147+
exception_type,
148+
ExceptionSummary(
149+
(exception_type,), message if isinstance(message, str) else None
150+
),
151+
)
152+
if str(escaped).lower() == "true":
153+
self._self_escaped_error_type = exception_type
129154
if self._self_config.hide_error_details:
130155
attributes = strip_error_details(attributes)
131156
span.add_event(name, attributes=attributes, timestamp=timestamp)
@@ -151,8 +176,10 @@ def set_status(
151176
# Called from an `except` block (use_span, most instrumentors), the
152177
# handled exception is the one this status is about.
153178
in_flight = sys.exc_info()[1]
154-
if in_flight is not None:
155-
self._self_status_error_type = get_error_type(in_flight)
179+
self._self_status_error_type = (
180+
get_error_type(in_flight) if in_flight is not None else None
181+
)
182+
self._self_status_description = status_description
156183
if self._self_config.hide_error_details:
157184
status_description = None
158185
else:
@@ -169,13 +196,36 @@ def _ended_with_error(self) -> bool:
169196
status_code = self._self_status_code
170197
return status_code is StatusCode.ERROR
171198

199+
def _recorded_error_type_for_status(self) -> Optional[str]:
200+
# The newest recorded exception the ERROR status is about: its message
201+
# or class name is in the status description, or the status has no
202+
# description. A recovered exception the status does not mention (a
203+
# timeout retried successfully, then a guardrail rejection) is skipped.
204+
description = self._self_status_description
205+
for error_type, summary in reversed(self._self_recorded):
206+
if not description:
207+
return error_type
208+
if summary.message and summary.message in description:
209+
return error_type
210+
if any(name in description for name in summary.type_names):
211+
return error_type
212+
return None
213+
172214
def _set_error_type(self) -> None:
173215
# Only a span that ends with status ERROR gets error.type: from the
174-
# exception handled when the status was set, else the last recorded
175-
# one. An instrumentor that set error.type itself is not overwritten.
176-
error_type = self._self_status_error_type or self._self_recorded_error_type
177-
if error_type is None or not self._ended_with_error():
216+
# exception handled when the status was set, else the last exception
217+
# recorded as escaped, else "_OTHER" (OpenTelemetry semantic
218+
# conventions). A recovered exception recorded earlier is not used: the
219+
# status may be about something else. An instrumentor that set
220+
# error.type itself is not overwritten.
221+
if not self._ended_with_error():
178222
return
223+
error_type = (
224+
self._self_status_error_type
225+
or self._self_escaped_error_type
226+
or self._recorded_error_type_for_status()
227+
or ERROR_TYPE_OTHER
228+
)
179229
span = cast(Span, self.__wrapped__)
180230
existing = getattr(span, "attributes", None) or {}
181231
if ERROR_TYPE not in existing:
@@ -206,6 +256,7 @@ def end(self, end_time: Optional[int] = None) -> None:
206256
self._set_error_type()
207257
span.end(end_time)
208258
self._self_exception_summaries.clear()
259+
self._self_recorded.clear()
209260

210261
def set_input(
211262
self,

‎python/fi_instrumentation/instrumentation/config.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,16 @@ def __aexit__(
8383
# Hides error messages, stacktraces and the ERROR status description.
8484
# Unset: on when any input/output hide flag is on.
8585
REDACTED_VALUE = "__REDACTED__"
86+
87+
# Besides input.value: the raw request body and the prompt template's values.
88+
# ``gen_ai.prompt.template.variables`` is what the context helper writes.
89+
_HIDDEN_WITH_INPUTS = frozenset(
90+
{
91+
SpanAttributes.RAW_INPUT,
92+
SpanAttributes.LLM_PROMPT_TEMPLATE_VARIABLES,
93+
SpanAttributes.GEN_AI_PROMPT_TEMPLATE_VARIABLES,
94+
}
95+
)
8696
# When a value is hidden, it will be replaced by this redacted value
8797

8898
DEFAULT_HIDE_LLM_INVOCATION_PARAMETERS = False
@@ -294,10 +304,16 @@ def mask(
294304
value = REDACTED_VALUE
295305
elif self.hide_inputs and key == SpanAttributes.INPUT_MIME_TYPE:
296306
return None
307+
elif self.hide_inputs and key in _HIDDEN_WITH_INPUTS:
308+
# The raw request body (messages included) and the values filled
309+
# into a prompt template are inputs too.
310+
value = REDACTED_VALUE
297311
elif self.hide_outputs and key == SpanAttributes.OUTPUT_VALUE:
298312
value = REDACTED_VALUE
299313
elif self.hide_outputs and key == SpanAttributes.OUTPUT_MIME_TYPE:
300314
return None
315+
elif self.hide_outputs and key == SpanAttributes.RAW_OUTPUT:
316+
value = REDACTED_VALUE
301317
elif (
302318
self.hide_inputs or self.hide_input_messages
303319
) and SpanAttributes.GEN_AI_INPUT_MESSAGES in key:

0 commit comments

Comments
 (0)