Skip to content

Commit 88b722d

Browse files
authored
fix(core/extensions): respect sensitive trace capture for model metadata (#5129)
1 parent f010d18 commit 88b722d

6 files changed

Lines changed: 172 additions & 2 deletions

File tree

‎src/agents/extensions/models/any_llm_model.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -585,6 +585,7 @@ async def _get_response_via_chat(
585585
model=str(self.model),
586586
model_config=model_config_for_trace(
587587
model_settings,
588+
tracing,
588589
base_url=self.base_url or "",
589590
extra_config={"provider": self._provider_name, "model_impl": "any-llm"},
590591
),
@@ -721,6 +722,7 @@ async def _stream_response_via_chat(
721722
model=str(self.model),
722723
model_config=model_config_for_trace(
723724
model_settings,
725+
tracing,
724726
base_url=self.base_url or "",
725727
extra_config={"provider": self._provider_name, "model_impl": "any-llm"},
726728
),

‎src/agents/extensions/models/litellm_model.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,7 @@ async def get_response(
224224
model=str(self.model),
225225
model_config=model_config_for_trace(
226226
model_settings,
227+
tracing,
227228
base_url=self.base_url or "",
228229
extra_config={"model_impl": "litellm"},
229230
),
@@ -397,6 +398,7 @@ async def stream_response(
397398
model=str(self.model),
398399
model_config=model_config_for_trace(
399400
model_settings,
401+
tracing,
400402
base_url=self.base_url or "",
401403
extra_config={"model_impl": "litellm"},
402404
),

‎src/agents/models/_trace.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,14 @@ def sanitize_url_for_trace(url: object) -> str:
2727

2828
def model_config_for_trace(
2929
model_settings: ModelSettings,
30+
tracing: ModelTracing,
3031
*,
3132
base_url: object | None = None,
3233
extra_config: dict[str, Any] | None = None,
3334
) -> dict[str, Any]:
3435
config = model_settings.to_traceable_dict()
36+
if not tracing.include_data():
37+
config.pop("metadata", None)
3538
if base_url is not None:
3639
config["base_url"] = sanitize_url_for_trace(base_url)
3740
if extra_config:

‎src/agents/models/openai_chatcompletions.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,9 @@ async def get_response(
235235
with (
236236
generation_span(
237237
model=str(self.model),
238-
model_config=model_config_for_trace(model_settings, base_url=self._client.base_url),
238+
model_config=model_config_for_trace(
239+
model_settings, tracing, base_url=self._client.base_url
240+
),
239241
disabled=tracing.is_disabled(),
240242
) as span_generation,
241243
model_span_errors(
@@ -447,7 +449,9 @@ async def stream_response(
447449
with (
448450
generation_span(
449451
model=str(self.model),
450-
model_config=model_config_for_trace(model_settings, base_url=self._client.base_url),
452+
model_config=model_config_for_trace(
453+
model_settings, tracing, base_url=self._client.base_url
454+
),
451455
disabled=tracing.is_disabled(),
452456
) as span_generation,
453457
model_span_errors(
Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
from __future__ import annotations
2+
3+
import json
4+
from collections.abc import AsyncIterator
5+
from types import SimpleNamespace
6+
from typing import Any, cast
7+
8+
import litellm
9+
import pytest
10+
from openai import AsyncOpenAI
11+
from openai.types.chat import ChatCompletion, ChatCompletionChunk
12+
13+
from agents import Agent, ModelSettings, OpenAIChatCompletionsModel, RunConfig, Runner, trace
14+
from agents.extensions.models.litellm_model import LitellmModel
15+
from agents.models.interface import Model
16+
from tests.testing_processor import fetch_ordered_spans, fetch_traces
17+
18+
19+
@pytest.mark.allow_call_model_methods
20+
@pytest.mark.asyncio
21+
@pytest.mark.parametrize("adapter", ["openai", "litellm", "any-llm"])
22+
@pytest.mark.parametrize("stream", [False, True])
23+
@pytest.mark.parametrize("include_data", [False, True])
24+
async def test_generation_metadata_respects_sensitive_capture(
25+
monkeypatch: pytest.MonkeyPatch, adapter: str, stream: bool, include_data: bool
26+
) -> None:
27+
metadata = {"request_label": "synthetic-request-metadata-sentinel"}
28+
settings = ModelSettings(temperature=0.25, metadata=metadata)
29+
request_calls: list[dict[str, Any]] = []
30+
31+
async def chunks() -> AsyncIterator[ChatCompletionChunk]:
32+
yield ChatCompletionChunk.model_validate(
33+
{
34+
"id": "chatcmpl_test",
35+
"created": 0,
36+
"model": "test-model",
37+
"object": "chat.completion.chunk",
38+
"choices": [
39+
{
40+
"index": 0,
41+
"delta": {"role": "assistant", "content": "ok"},
42+
"finish_reason": "stop",
43+
}
44+
],
45+
}
46+
)
47+
48+
async def completion(**kwargs: Any) -> Any:
49+
request_calls.append(kwargs)
50+
if kwargs["stream"]:
51+
return chunks()
52+
payload = {
53+
"id": "chatcmpl_test",
54+
"created": 0,
55+
"model": "test-model",
56+
"object": "chat.completion",
57+
"choices": [
58+
{
59+
"index": 0,
60+
"message": {"role": "assistant", "content": "ok"},
61+
"finish_reason": "stop",
62+
}
63+
],
64+
}
65+
if adapter == "litellm":
66+
return litellm.ModelResponse(**payload)
67+
return ChatCompletion.model_validate(payload)
68+
69+
model: Model
70+
if adapter == "openai":
71+
client = SimpleNamespace(
72+
base_url="https://example.com/v1",
73+
chat=SimpleNamespace(completions=SimpleNamespace(create=completion)),
74+
)
75+
model = OpenAIChatCompletionsModel("test-model", cast(AsyncOpenAI, client))
76+
elif adapter == "litellm":
77+
monkeypatch.setattr(litellm, "acompletion", completion)
78+
model = LitellmModel("test-model")
79+
else:
80+
pytest.importorskip("any_llm", reason="any-llm-sdk requires Python 3.11+.")
81+
from agents.extensions.models.any_llm_model import AnyLLM, AnyLLMModel
82+
83+
provider = SimpleNamespace(SUPPORTS_RESPONSES=False, acompletion=completion)
84+
monkeypatch.setattr(AnyLLM, "create", lambda *args, **kwargs: provider)
85+
model = AnyLLMModel("openai/test-model")
86+
87+
agent = Agent(name="test", model=model, model_settings=settings)
88+
trace_metadata = {"metadata": "intentional-trace-label"}
89+
config = RunConfig(trace_include_sensitive_data=include_data, trace_metadata=trace_metadata)
90+
if stream:
91+
result = Runner.run_streamed(agent, "hello", run_config=config)
92+
async for _ in result.stream_events():
93+
pass
94+
else:
95+
result = await Runner.run(agent, "hello", run_config=config)
96+
assert result.final_output == "ok"
97+
98+
assert len(request_calls) == 1
99+
assert request_calls[0]["metadata"] == metadata
100+
assert settings.metadata == metadata
101+
assert metadata == {"request_label": "synthetic-request-metadata-sentinel"}
102+
assert trace_metadata == {"metadata": "intentional-trace-label"}
103+
traces = fetch_traces()
104+
assert len(traces) == 1
105+
exported_trace = traces[0].export()
106+
assert exported_trace is not None
107+
assert exported_trace["metadata"] == trace_metadata
108+
109+
spans = fetch_ordered_spans()
110+
generations = [span for span in spans if span.span_data.type == "generation"]
111+
assert len(generations) == 1
112+
exported = generations[0].export()
113+
assert exported is not None
114+
config_data = exported["span_data"]["model_config"]
115+
assert config_data["temperature"] == 0.25
116+
if include_data:
117+
assert config_data["metadata"] == metadata
118+
else:
119+
assert "metadata" not in config_data
120+
assert "synthetic-request-metadata-sentinel" not in json.dumps(
121+
[span.export() for span in spans]
122+
)
123+
124+
125+
@pytest.mark.allow_call_model_methods
126+
@pytest.mark.asyncio
127+
async def test_caller_authored_trace_metadata_remains_explicit() -> None:
128+
async def completion(**kwargs: Any) -> ChatCompletion:
129+
return ChatCompletion.model_validate(
130+
{
131+
"id": "chatcmpl_test",
132+
"created": 0,
133+
"model": "test-model",
134+
"object": "chat.completion",
135+
"choices": [
136+
{
137+
"index": 0,
138+
"message": {"role": "assistant", "content": "ok"},
139+
"finish_reason": "stop",
140+
}
141+
],
142+
}
143+
)
144+
145+
client = SimpleNamespace(
146+
base_url="https://example.com/v1",
147+
chat=SimpleNamespace(completions=SimpleNamespace(create=completion)),
148+
)
149+
agent = Agent(
150+
name="test", model=OpenAIChatCompletionsModel("test-model", cast(AsyncOpenAI, client))
151+
)
152+
metadata = {"metadata": "caller-owned-label"}
153+
with trace("caller-owned", metadata=metadata):
154+
await Runner.run(agent, "hello", run_config=RunConfig(trace_include_sensitive_data=False))
155+
exported = fetch_traces()[0].export()
156+
assert exported is not None
157+
assert exported["metadata"] == metadata == {"metadata": "caller-owned-label"}

‎tests/models/test_trace_config.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
from agents import ModelTracing
12
from agents.model_settings import ModelSettings
23
from agents.models._trace import model_config_for_trace, sanitize_url_for_trace
34

@@ -19,6 +20,7 @@ def test_model_config_for_trace_sanitizes_base_url_and_omits_request_extras() ->
1920
extra_body={"secret": "body-token"},
2021
extra_args={"api_key": "arg-token"},
2122
),
23+
ModelTracing.ENABLED,
2224
base_url="https://user:[email protected]/v1?api-key=secret#fragment",
2325
extra_config={"model_impl": "test-model"},
2426
)

0 commit comments

Comments
 (0)