Skip to content

Commit 6dd3095

Browse files
authored
Python: Fix background agent telemetry context error (#6764)
* Fix issue when using background agents with telemetry * Address PR comments * Fix issue when using background agents with telemetry * Address PR comments * Fix uv.lock * Remove unecessary comments and commit hook reformatted code
1 parent c89a539 commit 6dd3095

4 files changed

Lines changed: 265 additions & 20 deletions

File tree

‎python/packages/core/agent_framework/__init__.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,6 @@
168168
chat_middleware,
169169
function_middleware,
170170
)
171-
172171
from ._sessions import (
173172
AgentSession,
174173
ContextProvider,

‎python/packages/core/agent_framework/observability.py‎

Lines changed: 48 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1778,12 +1778,17 @@ def _trace_agent_invocation(
17781778
**merged_client_kwargs,
17791779
)
17801780

1781-
inner_response_telemetry_captured_fields: set[str] = set()
1782-
inner_response_telemetry_captured_fields_token = INNER_RESPONSE_TELEMETRY_CAPTURED_FIELDS.set(
1783-
inner_response_telemetry_captured_fields
1784-
)
1785-
inner_accumulated_usage_token = INNER_ACCUMULATED_USAGE.set({})
17861781
if stream:
1782+
# Do NOT set the inner-telemetry context vars here: this synchronous run() body executes
1783+
# in the CALLER's context, but the ResponseStream may be consumed in a different context
1784+
# (e.g. ``stream = agent.run(stream=True)`` then ``await asyncio.create_task(consume(stream))``).
1785+
# The cleanup-hook reset (in _finalize_stream) runs in the consuming context, so a token
1786+
# created here would raise ``ValueError: <Token ...> was created in a different Context``.
1787+
# Instead the tokens are set lazily on the first pull (see _inner_telemetry_pull_context
1788+
# below), so set and reset both happen in the consumer's context.
1789+
inner_response_telemetry_captured_fields: set[str] = set()
1790+
inner_response_telemetry_captured_fields_token: contextvars.Token[set[str] | None] | None = None
1791+
inner_accumulated_usage_token: contextvars.Token[UsageDetails | None] | None = None
17871792
span = _start_streaming_span(attributes, OtelAttr.AGENT_NAME)
17881793

17891794
if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED and messages and span.is_recording():
@@ -1827,8 +1832,6 @@ def _record_duration() -> None:
18271832
raise RuntimeError("Streaming telemetry requires a ResponseStream result.")
18281833
except Exception as exception:
18291834
capture_exception(span=span, exception=exception, timestamp=time_ns())
1830-
INNER_RESPONSE_TELEMETRY_CAPTURED_FIELDS.reset(inner_response_telemetry_captured_fields_token)
1831-
INNER_ACCUMULATED_USAGE.reset(inner_accumulated_usage_token)
18321835
_close_span()
18331836
raise
18341837

@@ -1868,26 +1871,59 @@ async def _finalize_stream() -> None:
18681871
except Exception as exception:
18691872
capture_exception(span=span, exception=exception, timestamp=time_ns())
18701873
finally:
1871-
INNER_RESPONSE_TELEMETRY_CAPTURED_FIELDS.reset(inner_response_telemetry_captured_fields_token)
1872-
INNER_ACCUMULATED_USAGE.reset(inner_accumulated_usage_token)
1874+
# Reset only if the lazy set actually ran (it may not have if the stream was
1875+
# never pulled). These run in the consuming context — the same context the
1876+
# pull-context factory below set the tokens in — so the reset is cross-context safe.
1877+
if inner_response_telemetry_captured_fields_token is not None:
1878+
INNER_RESPONSE_TELEMETRY_CAPTURED_FIELDS.reset(inner_response_telemetry_captured_fields_token)
1879+
if inner_accumulated_usage_token is not None:
1880+
INNER_ACCUMULATED_USAGE.reset(inner_accumulated_usage_token)
18731881
_close_span()
18741882

1883+
def _inner_telemetry_pull_context() -> contextlib.AbstractContextManager[Any]:
1884+
# Invoked at the start of every pull (and during stream resolution), in the
1885+
# consuming context. On the first pull it sets the inner-telemetry context vars so
1886+
# that set and the reset in _finalize_stream both run in the consumer's context,
1887+
# avoiding the cross-context Token reset failure. Setting happens before the
1888+
# underlying iterator is pulled, so inner chat completion spans created during the
1889+
# pull can still accumulate usage / mark captured fields.
1890+
nonlocal inner_response_telemetry_captured_fields_token, inner_accumulated_usage_token
1891+
if inner_response_telemetry_captured_fields_token is None:
1892+
inner_response_telemetry_captured_fields_token = INNER_RESPONSE_TELEMETRY_CAPTURED_FIELDS.set(
1893+
inner_response_telemetry_captured_fields
1894+
)
1895+
inner_accumulated_usage_token = INNER_ACCUMULATED_USAGE.set({})
1896+
return _activate_span(span)
1897+
18751898
# The pull context manager attaches the span around each underlying iterator pull so
18761899
# that child spans created during the pull (e.g. inner chat completion spans from the
18771900
# underlying ChatTelemetryLayer) are parented under this agent invoke span. Attach and
18781901
# detach happen in the same async context as the pull, avoiding cross-context cleanup
1879-
# issues. The weakref finalizer ensures the span is closed even if the stream is
1880-
# garbage collected without being consumed.
1902+
# issues. It also lazily sets the inner-telemetry context vars on the first pull (see
1903+
# _inner_telemetry_pull_context). The weakref finalizer ensures the span is closed even
1904+
# if the stream is garbage collected without being consumed.
18811905
wrapped_stream: ResponseStream[AgentResponseUpdate, AgentResponse[Any]] = (
18821906
result_stream
18831907
.with_cleanup_hook(_record_duration)
18841908
.with_cleanup_hook(_finalize_stream)
1885-
.with_pull_context_manager(lambda: _activate_span(span))
1909+
.with_pull_context_manager(_inner_telemetry_pull_context)
18861910
)
18871911
weakref.finalize(wrapped_stream, _close_span)
18881912
return wrapped_stream
18891913

18901914
async def _run() -> AgentResponse[Any]:
1915+
# Set the inner-telemetry context vars inside the coroutine so the set and the
1916+
# reset in `finally` always happen in the same execution context. `run()` is a sync
1917+
# method that returns this coroutine, which may be awaited in a different context than
1918+
# the one that called `run()` (e.g. `asyncio.create_task(agent.run(...))`, as used by
1919+
# BackgroundAgentsProvider). A contextvars.Token can only be reset in the context that
1920+
# created it, so setting eagerly in `run()`/`_trace_agent_invocation` and resetting
1921+
# here would raise "Token was created in a different Context".
1922+
inner_response_telemetry_captured_fields: set[str] = set()
1923+
inner_response_telemetry_captured_fields_token = INNER_RESPONSE_TELEMETRY_CAPTURED_FIELDS.set(
1924+
inner_response_telemetry_captured_fields
1925+
)
1926+
inner_accumulated_usage_token = INNER_ACCUMULATED_USAGE.set({})
18911927
try:
18921928
with _get_span(attributes=attributes, span_name_attribute=OtelAttr.AGENT_NAME) as span:
18931929
try:

‎python/packages/core/agent_framework/security.py‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3267,19 +3267,23 @@ def __init__(
32673267
if url is not None:
32683268
from httpx import AsyncClient, Timeout
32693269

3270-
from ._mcp import MCPStreamableHTTPTool, MCP_DEFAULT_TIMEOUT, MCP_DEFAULT_SSE_READ_TIMEOUT
3270+
from ._mcp import MCP_DEFAULT_SSE_READ_TIMEOUT, MCP_DEFAULT_TIMEOUT, MCPStreamableHTTPTool
32713271

32723272
static_headers = dict(headers or {})
32733273
# Pass headers via an AsyncClient so they are included on ALL requests
32743274
# (including session.initialize()), not just tool calls. Using
32753275
# header_provider alone only sets headers via a ContextVar that is
32763276
# populated during call_tool() and would be empty during initialization,
32773277
# causing 401s that silently manifest as anyio cancel-scope errors.
3278-
http_client = AsyncClient(
3279-
headers=static_headers,
3280-
follow_redirects=True,
3281-
timeout=Timeout(MCP_DEFAULT_TIMEOUT, read=MCP_DEFAULT_SSE_READ_TIMEOUT),
3282-
) if static_headers else None
3278+
http_client = (
3279+
AsyncClient(
3280+
headers=static_headers,
3281+
follow_redirects=True,
3282+
timeout=Timeout(MCP_DEFAULT_TIMEOUT, read=MCP_DEFAULT_SSE_READ_TIMEOUT),
3283+
)
3284+
if static_headers
3285+
else None
3286+
)
32833287
mcp_tool = MCPStreamableHTTPTool(
32843288
name=name or "mcp",
32853289
url=url,

‎python/packages/core/tests/core/test_observability.py‎

Lines changed: 207 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# Copyright (c) Microsoft. All rights reserved.
22

3+
import asyncio
34
import logging
45
from collections.abc import AsyncIterable, Awaitable, Mapping, Sequence
56
from typing import Any, cast
@@ -5249,7 +5250,212 @@ class FailingExecuteAgent(AgentTelemetryLayer, _FailingExecuteAgent): # type: i
52495250
assert agent_spans[0].status.status_code == StatusCode.ERROR
52505251

52515252

5252-
# region Test heavy operations skipped when span is not recording
5253+
@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True)
5254+
async def test_agent_run_contextvars_safe_when_awaited_in_different_context(
5255+
span_exporter: InMemorySpanExporter, enable_sensitive_data
5256+
):
5257+
"""``run()`` is a sync method that returns an awaitable; the telemetry contextvar set and reset
5258+
must happen in the same execution context so the returned coroutine can be awaited in a different
5259+
context.
5260+
5261+
Regression for background agents (``BackgroundAgentsProvider``), which do
5262+
``asyncio.create_task(agent.run(...))``: ``run()`` executes synchronously in the parent context
5263+
while the returned coroutine is awaited in a fresh copied context. If the contextvar token were
5264+
created eagerly in the parent context but reset inside the coroutine, this raised
5265+
``ValueError: <Token ...> was created in a different Context``.
5266+
"""
5267+
5268+
class _SimpleAgent:
5269+
AGENT_PROVIDER_NAME = "test_provider"
5270+
5271+
def __init__(self):
5272+
self._id = "simple"
5273+
self._name = "Simple"
5274+
self._description = "Agent that returns a response without raising"
5275+
self._default_options: dict[str, Any] = {}
5276+
5277+
@property
5278+
def id(self):
5279+
return self._id
5280+
5281+
@property
5282+
def name(self):
5283+
return self._name
5284+
5285+
@property
5286+
def description(self):
5287+
return self._description
5288+
5289+
@property
5290+
def default_options(self):
5291+
return self._default_options
5292+
5293+
def run(self, messages=None, *, stream: bool = False, session=None, **kwargs):
5294+
async def _inner() -> AgentResponse:
5295+
return AgentResponse(messages=[Message(role="assistant", contents=["hi"])])
5296+
5297+
return _inner()
5298+
5299+
class SimpleAgent(AgentTelemetryLayer, _SimpleAgent): # type: ignore[misc]
5300+
pass
5301+
5302+
agent = SimpleAgent()
5303+
span_exporter.clear()
5304+
5305+
# Mimic BackgroundAgentsProvider: invoke run() synchronously in this context, then await the
5306+
# returned coroutine inside a separate task (a different/copied context).
5307+
awaitable = agent.run(messages="Hello", stream=False)
5308+
5309+
async def _runner(aw):
5310+
return await aw
5311+
5312+
result = await asyncio.create_task(_runner(awaitable))
5313+
assert isinstance(result, AgentResponse)
5314+
5315+
spans = span_exporter.get_finished_spans()
5316+
agent_spans = [s for s in spans if s.attributes.get(OtelAttr.OPERATION.value) == OtelAttr.AGENT_INVOKE_OPERATION] # type: ignore[union-attr] # ty: ignore[unresolved-attribute]
5317+
assert len(agent_spans) == 1
5318+
assert agent_spans[0].status.status_code != StatusCode.ERROR
5319+
5320+
5321+
@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True)
5322+
async def test_agent_run_error_path_contextvars_safe_when_awaited_in_different_context(
5323+
span_exporter: InMemorySpanExporter, enable_sensitive_data
5324+
):
5325+
"""Error-path variant: the coroutine returned by ``run()`` raises, and is awaited in a different
5326+
context via ``asyncio.create_task``. The telemetry contextvars are set and reset inside the
5327+
coroutine (its ``finally``), so the reset on the exception path must not raise
5328+
``ValueError: <Token ...> was created in a different Context``; the original error must surface.
5329+
"""
5330+
5331+
class _FailingRunAgent:
5332+
AGENT_PROVIDER_NAME = "test_provider"
5333+
5334+
def __init__(self):
5335+
self._id = "failing_run"
5336+
self._name = "Failing Run"
5337+
self._description = "Agent whose run coroutine raises"
5338+
self._default_options: dict[str, Any] = {}
5339+
5340+
@property
5341+
def id(self):
5342+
return self._id
5343+
5344+
@property
5345+
def name(self):
5346+
return self._name
5347+
5348+
@property
5349+
def description(self):
5350+
return self._description
5351+
5352+
@property
5353+
def default_options(self):
5354+
return self._default_options
5355+
5356+
def run(self, messages=None, *, stream: bool = False, session=None, **kwargs):
5357+
async def _inner() -> AgentResponse:
5358+
raise RuntimeError("run failed")
5359+
5360+
return _inner()
5361+
5362+
class FailingRunAgent(AgentTelemetryLayer, _FailingRunAgent): # type: ignore[misc]
5363+
pass
5364+
5365+
agent = FailingRunAgent()
5366+
span_exporter.clear()
5367+
5368+
awaitable = agent.run(messages="Hello", stream=False)
5369+
5370+
async def _runner(aw):
5371+
return await aw
5372+
5373+
# The original RuntimeError must propagate unchanged — not a cross-context ValueError from the
5374+
# contextvar reset in the coroutine's finally block.
5375+
with pytest.raises(RuntimeError, match="run failed"):
5376+
await asyncio.create_task(_runner(awaitable))
5377+
5378+
spans = span_exporter.get_finished_spans()
5379+
agent_spans = [s for s in spans if s.attributes.get(OtelAttr.OPERATION.value) == OtelAttr.AGENT_INVOKE_OPERATION] # type: ignore[union-attr] # ty: ignore[unresolved-attribute]
5380+
assert len(agent_spans) == 1
5381+
assert agent_spans[0].status.status_code == StatusCode.ERROR
5382+
5383+
5384+
@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True)
5385+
async def test_agent_streaming_contextvars_safe_when_consumed_in_different_context(
5386+
span_exporter: InMemorySpanExporter, enable_sensitive_data
5387+
):
5388+
"""``run(stream=True)`` returns a ``ResponseStream`` synchronously, but its cleanup hooks (which
5389+
reset the telemetry contextvars) run when the stream is *consumed* — possibly in a different
5390+
context (e.g. ``stream = agent.run(stream=True)`` then ``await asyncio.create_task(consume(stream))``).
5391+
5392+
The contextvars are therefore set lazily on the first pull, in the consuming context, so the set
5393+
and the reset both run there. Otherwise this raised
5394+
``ValueError: <Token ...> was created in a different Context``.
5395+
"""
5396+
from agent_framework import AgentResponseUpdate
5397+
5398+
class _StreamingAgent:
5399+
AGENT_PROVIDER_NAME = "test_provider"
5400+
5401+
def __init__(self):
5402+
self._id = "streaming_xctx"
5403+
self._name = "Streaming XCtx"
5404+
self._description = "Streaming agent for cross-context consumption"
5405+
self._default_options: dict[str, Any] = {}
5406+
5407+
@property
5408+
def id(self):
5409+
return self._id
5410+
5411+
@property
5412+
def name(self):
5413+
return self._name
5414+
5415+
@property
5416+
def description(self):
5417+
return self._description
5418+
5419+
@property
5420+
def default_options(self):
5421+
return self._default_options
5422+
5423+
def run(self, messages=None, *, stream: bool = False, session=None, **kwargs):
5424+
if stream:
5425+
5426+
async def _stream():
5427+
yield AgentResponseUpdate(contents=[Content.from_text("Hello ")], role="assistant")
5428+
yield AgentResponseUpdate(contents=[Content.from_text("World")], role="assistant")
5429+
5430+
return ResponseStream(_stream(), finalizer=AgentResponse.from_updates)
5431+
raise NotImplementedError
5432+
5433+
class StreamingAgent(AgentTelemetryLayer, _StreamingAgent): # type: ignore[misc]
5434+
pass
5435+
5436+
agent = StreamingAgent()
5437+
span_exporter.clear()
5438+
5439+
# Create the stream synchronously in this context, then consume it inside a separate task (a
5440+
# different/copied context) — mirroring how a caller might hand the stream off to be drained.
5441+
stream = agent.run(messages="Hello", stream=True)
5442+
5443+
async def _consume(s):
5444+
collected = []
5445+
async for update in s:
5446+
collected.append(update)
5447+
await s.get_final_response()
5448+
return collected
5449+
5450+
updates = await asyncio.create_task(_consume(stream))
5451+
assert len(updates) == 2
5452+
5453+
spans = span_exporter.get_finished_spans()
5454+
agent_spans = [s for s in spans if s.attributes.get(OtelAttr.OPERATION.value) == OtelAttr.AGENT_INVOKE_OPERATION] # type: ignore[union-attr] # ty: ignore[unresolved-attribute]
5455+
assert len(agent_spans) == 1
5456+
assert agent_spans[0].status.status_code != StatusCode.ERROR
5457+
5458+
52535459
#
52545460
# When ``ENABLE_INSTRUMENTATION`` is on (the default) but no OpenTelemetry
52555461
# tracer provider has been configured, the global provider is the

0 commit comments

Comments
 (0)