Skip to content

Commit 10e9dd7

Browse files
committed
fix(ag2-classic): keep content dropping after uninstrument
uninstrument() removed AG2ClassicSpanProcessor and shut it down, but upstream instrument_agent patches each agent once with no undo. Agents instrumented earlier kept emitting spans on the same provider, now unfiltered: gen_ai.input/output.messages were exported on invoke_agent and conversation spans, and conversation spans regained the chat-wide gen_ai.usage.* tokens (double count). The same happened to a second handle sharing the processor when the first handle uninstrumented. uninstrument() now restores OpenAIWrapper.create only. The processor stays installed, shared by every handle on the provider, and shuts down with the provider. approved: Nikhil 2026-10-03 blanket Refs: TH-8237
1 parent 96ef5b3 commit 10e9dd7

3 files changed

Lines changed: 82 additions & 15 deletions

File tree

‎python/frameworks/ag2-classic/README.md‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,8 +89,11 @@ the default; pass `transport=Transport.GRPC` (from `fi_instrumentation.otel`) to
8989
also install the `[tracing]` extra's gRPC exporter as a global provider, or
9090
spans are duplicated.
9191

92-
`tracing.uninstrument()` removes the processor and restores
93-
`OpenAIWrapper.create`. Agents stay instrumented; upstream has no per-agent undo.
92+
`tracing.uninstrument()` restores `OpenAIWrapper.create`, so no new LLM spans.
93+
Agents stay instrumented; upstream has no per-agent undo. The processor stays on
94+
the provider, so spans from those agents still have content removed and
95+
aggregate usage moved off non-LLM spans. It is shared by every handle on that
96+
provider and shuts down with `trace_provider.shutdown()`.
9497

9598
## Conformance
9699

‎python/frameworks/ag2-classic/tests/test_setup.py‎

Lines changed: 68 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -182,9 +182,75 @@ def test_setup_is_idempotent_per_provider_and_uninstrument_restores(pipeline):
182182
assert second.owns_llm_wrapper is False # upstream wraps once (llm_wrapper.py:64-65)
183183
first.uninstrument()
184184
assert OpenAIWrapper.create is original
185-
assert not any(
186-
isinstance(p, AG2ClassicSpanProcessor) for p in provider._active_span_processor._span_processors
185+
# Upstream has no per-agent undo, so agents instrumented earlier keep
186+
# emitting spans on this provider. The processor stays installed and live
187+
# so those spans are still filtered; the second handle keeps working too.
188+
ours = [p for p in provider._active_span_processor._span_processors if isinstance(p, AG2ClassicSpanProcessor)]
189+
assert ours == [first.processor]
190+
assert provider._active_span_processor._span_processors[0] is first.processor
191+
assert first.processor._disabled is False
192+
193+
194+
PROMOTED_USAGE_KEYS = (
195+
"gen_ai.usage.input_tokens",
196+
"gen_ai.usage.output_tokens",
197+
"gen_ai.usage.total_tokens",
198+
"gen_ai.cost.total",
199+
"llm.cost.total",
200+
)
201+
202+
203+
def _assert_no_content_and_no_promoted_usage_off_llm(spans):
204+
assert spans, "no spans were exported"
205+
for span in spans:
206+
attrs = dict(span.attributes)
207+
for key in CONTENT_KEYS:
208+
assert key not in attrs, (key, span.name)
209+
blob = json.dumps(attrs, default=str)
210+
assert SECRET_PROMPT not in blob and TOOL_SECRET_CITY not in blob, span.name
211+
if attrs.get("gen_ai.span.kind") != "LLM":
212+
leaked = [k for k in attrs if k in PROMOTED_USAGE_KEYS]
213+
assert not leaked, (span.name, leaked)
214+
215+
216+
def test_previously_instrumented_agents_stay_filtered_after_uninstrument(pipeline, fake):
217+
from autogen import ConversableAgent
218+
219+
_provider, exporter, make = pipeline
220+
tracing = make()
221+
assistant = ConversableAgent(
222+
"assistant",
223+
llm_config=fake.llm_config(),
224+
human_input_mode="NEVER",
225+
is_termination_msg=lambda m: "TERMINATE" in str(m.get("content") or ""),
187226
)
227+
user = ConversableAgent("user", llm_config=False, human_input_mode="NEVER", max_consecutive_auto_reply=1)
228+
tracing.instrument_agent(assistant)
229+
tracing.instrument_agent(user)
230+
tracing.uninstrument()
231+
exporter.clear()
232+
233+
user.initiate_chat(assistant, message=SECRET_PROMPT, max_turns=1, silent=True)
234+
spans = exporter.get_finished_spans()
235+
names = {s.name for s in spans}
236+
# Upstream patched these agents for good; their spans still arrive.
237+
assert {"conversation user", "invoke_agent assistant"} <= names
238+
_assert_no_content_and_no_promoted_usage_off_llm(spans)
239+
conversation = next(s for s in spans if s.name == "conversation user").attributes
240+
assert conversation["gen_ai.span.kind"] == "CHAIN"
241+
assert conversation["ag2.usage.input_tokens"] == PROMPT_TOKENS
242+
243+
244+
def test_second_handle_keeps_filtering_after_first_handle_uninstruments(pipeline, fake):
245+
_provider, exporter, make = pipeline
246+
first = make()
247+
second = make()
248+
first.uninstrument()
249+
exporter.clear()
250+
two_agent_tool_chat(second, fake)
251+
spans = exporter.get_finished_spans()
252+
assert any(s.name == "conversation user" for s in spans)
253+
_assert_no_content_and_no_promoted_usage_off_llm(spans)
188254

189255

190256
def test_setup_rejects_a_non_sdk_provider():

‎python/frameworks/ag2-classic/traceai_ag2_classic/_setup.py‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -168,10 +168,15 @@ def instrument_a2a_server(self, server: Any) -> Any:
168168
# Teardown ---------------------------------------------------------------
169169

170170
def uninstrument(self) -> None:
171-
"""Remove the processor and restore ``OpenAIWrapper.create`` if we patched it.
172-
173-
Upstream patches agents per instance and offers no undo, so agents
174-
instrumented through this handle stay instrumented.
171+
"""Restore ``OpenAIWrapper.create`` if this handle patched it.
172+
173+
The span processor stays on the provider. Upstream patches agents per
174+
instance and offers no undo, so agents instrumented earlier keep
175+
emitting spans on this provider; the processor keeps dropping their
176+
content and moving their aggregate usage off non-LLM spans. Every
177+
handle on a provider shares that one processor, so removing it here
178+
would also unfilter the other handles. It is shut down with the
179+
provider (``provider.shutdown()``).
175180
"""
176181
if self.owns_llm_wrapper and self._original_create is not None:
177182
from autogen.oai import client as oai_client_module
@@ -181,13 +186,6 @@ def uninstrument(self) -> None:
181186
oai_client_module.OpenAIWrapper.create = self._original_create
182187
self.owns_llm_wrapper = False
183188
self._original_create = None
184-
active = getattr(self.tracer_provider, "_active_span_processor", None)
185-
if active is not None and hasattr(active, "_span_processors"):
186-
with getattr(active, "_lock", _NullLock()):
187-
active._span_processors = tuple(
188-
p for p in active._span_processors if p is not self.processor
189-
)
190-
self.processor.shutdown()
191189

192190

193191
def setup(

0 commit comments

Comments
 (0)