Skip to content

Commit de0aa85

Browse files
authored
fix(tracing): omit reasoning from default trace exports (#5108)
1 parent 91f69fd commit de0aa85

3 files changed

Lines changed: 349 additions & 3 deletions

File tree

‎src/agents/tracing/processors.py‎

Lines changed: 49 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
import random
88
import threading
99
import time
10-
from collections.abc import Callable
10+
from collections.abc import Callable, Mapping, Sequence
1111
from functools import cached_property
1212
from typing import Any, cast
1313

@@ -318,7 +318,10 @@ def _should_sanitize_for_openai_tracing_api(self) -> bool:
318318
return self.endpoint.rstrip("/") == self._OPENAI_TRACING_INGEST_ENDPOINT.rstrip("/")
319319

320320
def _sanitize_for_openai_tracing_api(self, payload_item: dict[str, Any]) -> dict[str, Any]:
321-
"""Drop or truncate span fields known to be rejected by traces ingest."""
321+
"""Omit reasoning from generation data and enforce traces ingest field limits.
322+
323+
Original spans remain available to custom processors and exporters.
324+
"""
322325
span_data = payload_item.get("span_data")
323326
if not isinstance(span_data, dict):
324327
return payload_item
@@ -329,7 +332,10 @@ def _sanitize_for_openai_tracing_api(self, payload_item: dict[str, Any]) -> dict
329332
for field_name in ("input", "output"):
330333
if field_name not in span_data:
331334
continue
332-
sanitized_field = self._truncate_span_field_value(span_data[field_name])
335+
field_value = span_data[field_name]
336+
if span_data.get("type") == "generation":
337+
field_value = self._omit_generation_reasoning(field_value)
338+
sanitized_field = self._truncate_span_field_value(field_value)
333339
if sanitized_field is span_data[field_name]:
334340
continue
335341
if not did_mutate:
@@ -377,6 +383,46 @@ def _sanitize_for_openai_tracing_api(self, payload_item: dict[str, Any]) -> dict
377383
sanitized_payload_item["span_data"] = sanitized_span_data
378384
return sanitized_payload_item
379385

386+
def _omit_generation_reasoning(self, value: Any) -> Any:
387+
"""Filter SDK generation message shapes without traversing arbitrary user data."""
388+
if not isinstance(value, Sequence) or isinstance(value, str | bytes | bytearray):
389+
return value
390+
391+
def omit_message_reasoning(item: Any) -> Any:
392+
if not isinstance(item, Mapping):
393+
return item
394+
item = dict(item)
395+
if item.get("role") == "assistant":
396+
for key in ("reasoning", "reasoning_content", "thinking_blocks", "thinking"):
397+
item.pop(key, None)
398+
content = item.get("content")
399+
if isinstance(content, list):
400+
# Legacy thinking replay uses inline Anthropic content blocks.
401+
item["content"] = [
402+
part
403+
for part in content
404+
if not isinstance(part, dict)
405+
or part.get("type") not in ("thinking", "redacted_thinking")
406+
]
407+
return item
408+
409+
def omit_items(items: Sequence[Any]) -> list[Any]:
410+
# Third-party reasoning_content is also normalized into summary text.
411+
return [
412+
omit_message_reasoning(item)
413+
for item in items
414+
if not isinstance(item, Mapping) or item.get("type") != "reasoning"
415+
]
416+
417+
return [
418+
{**item, "output": omit_items(item["output"])}
419+
if isinstance(item, Mapping)
420+
and item.get("object") == "response"
421+
and isinstance(item.get("output"), list)
422+
else item
423+
for item in omit_items(value)
424+
]
425+
380426
def _value_json_size_bytes(self, value: Any) -> int:
381427
try:
382428
serialized = json.dumps(value, ensure_ascii=False, separators=(",", ":"))

‎tests/models/test_any_llm_model.py‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
11
from __future__ import annotations
22

33
import asyncio
4+
import copy
45
import importlib
6+
import json
57
import sys
68
import types as pytypes
79
from collections.abc import AsyncIterator
810
from typing import Any, Literal, cast
911

12+
import httpx2
1013
import pytest
1114
from openai.types.chat import (
1215
ChatCompletion,
@@ -47,6 +50,8 @@
4750
ModelBehaviorError,
4851
ModelSettings,
4952
ModelTracing,
53+
RunConfig,
54+
Runner,
5055
Tool,
5156
TResponseInputItem,
5257
__version__,
@@ -57,6 +62,8 @@
5762
from agents.exceptions import UserError
5863
from agents.models.chatcmpl_helpers import HEADERS_OVERRIDE
5964
from agents.models.fake_id import FAKE_RESPONSES_ID
65+
from agents.tracing.processors import BackendSpanExporter
66+
from tests.testing_processor import fetch_ordered_spans
6067

6168

6269
class FakeAnyLLMProvider:
@@ -1589,6 +1596,50 @@ def test_any_llm_stream_flattens_reasoning_object_when_reasoning_content_is_empt
15891596
assert normalized.choices[0].delta.reasoning == "Plaintext reasoning"
15901597

15911598

1599+
@pytest.mark.allow_call_model_methods
1600+
async def test_any_llm_thinking_is_omitted_from_default_trace_export(monkeypatch) -> None:
1601+
secret = "PRIVATE_ANY_LLM_THINKING_SENTINEL"
1602+
completion = _chat_completion("Visible answer")
1603+
completion.choices[0].message = ChatCompletionMessage.model_validate(
1604+
{"role": "assistant", "content": "Visible answer", "thinking": secret}
1605+
)
1606+
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=completion)
1607+
module, _ = _import_any_llm_module(monkeypatch, provider)
1608+
model = module.AnyLLMModel(model="openrouter/test-model", api="chat_completions")
1609+
result = await Runner.run(
1610+
Agent(name="Test", model=model),
1611+
"Question",
1612+
run_config=RunConfig(trace_include_sensitive_data=True),
1613+
)
1614+
generation = next(span for span in fetch_ordered_spans() if span.span_data.type == "generation")
1615+
original = copy.deepcopy(generation.export())
1616+
assert original is not None
1617+
assert original["span_data"]["output"][0]["thinking"] == secret
1618+
replay = result.to_input_list()
1619+
assert secret in json.dumps(replay)
1620+
assert result.final_output == "Visible answer"
1621+
1622+
payloads: list[dict[str, Any]] = []
1623+
1624+
def post(self, url, **kwargs):
1625+
payloads.append(copy.deepcopy(kwargs["json"]))
1626+
return httpx2.Response(200)
1627+
1628+
monkeypatch.setattr(httpx2.Client, "post", post)
1629+
exporter = BackendSpanExporter(api_key="test-key", max_retries=0)
1630+
try:
1631+
exporter.export([generation])
1632+
sent = payloads[-1]["data"][0]["span_data"]
1633+
assert secret not in json.dumps(sent)
1634+
assert sent["output"][0]["content"] == "Visible answer"
1635+
assert sent["usage"]["output_tokens"] == 5
1636+
assert generation.export() == original
1637+
assert result.to_input_list() == replay
1638+
finally:
1639+
exporter.close()
1640+
await model.close()
1641+
1642+
15921643
def test_any_llm_nonstream_preserves_native_reasoning_content_field(monkeypatch) -> None:
15931644
provider = FakeAnyLLMProvider(supports_responses=False)
15941645
module, _create_calls = _import_any_llm_module(monkeypatch, provider)

0 commit comments

Comments
 (0)