Skip to content

Commit 10c694f

Browse files
Yuhan Gaocopybara-github
authored andcommitted
fix(telemetry): stop holding OTel context across async generator yields
Co-authored-by: Yuhan Gao <[email protected]> PiperOrigin-RevId: 998239754
1 parent 75aef1e commit 10c694f

12 files changed

Lines changed: 1195 additions & 740 deletions

File tree

‎src/google/adk/agents/base_agent.py‎

Lines changed: 49 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,6 @@
4949
from ..utils._callback_pipeline import _normalize_callbacks
5050
from ..utils._callback_pipeline import _run_callbacks
5151
from ..utils._callback_pipeline import _stop_on_truthy
52-
from ..utils._runner_utils import _with_caller_context
5352
from ..utils.context_utils import Aclosing
5453
from ..workflow import BaseNode
5554
from .base_agent_config import BaseAgentConfig as BaseAgentConfig
@@ -405,58 +404,61 @@ async def _run_with_lifecycle(
405404
impl_fn: Callable[[InvocationContext], AsyncGenerator[Event, None]],
406405
) -> AsyncGenerator[Event, None]:
407406
"""Runs an agent implementation generator with full callback and trace lifecycle."""
408-
from opentelemetry import context
409-
410407
from ..telemetry import _instrumentation
411408

412-
caller_ctx = context.get_current()
409+
ctx = self._create_invocation_context(parent_context)
413410

414411
async def _run() -> AsyncGenerator[Event, None]:
415-
ctx = self._create_invocation_context(parent_context)
416-
async with _instrumentation.record_agent_invocation(ctx, self):
417-
before_callback_completed = False
418-
after_callback_called = False
419-
try:
420-
event = await self._handle_before_agent_callback(ctx)
421-
before_callback_completed = True
422-
if event:
412+
before_callback_completed = False
413+
after_callback_called = False
414+
try:
415+
event = await self._handle_before_agent_callback(ctx)
416+
before_callback_completed = True
417+
if event:
418+
yield event
419+
if ctx.end_invocation:
420+
return
421+
422+
async with Aclosing(impl_fn(ctx)) as agen:
423+
async for event in agen:
423424
yield event
424-
if ctx.end_invocation:
425-
return
426-
427-
async with Aclosing(impl_fn(ctx)) as agen:
428-
async for event in agen:
429-
yield event
430-
431-
if ctx.end_invocation:
432-
return
433425

434-
after_callback_called = True
435-
if event := await self._handle_after_agent_callback(ctx):
436-
yield event
437-
except asyncio.CancelledError:
438-
if (
439-
before_callback_completed
440-
and not after_callback_called
441-
and not ctx.end_invocation
442-
):
443-
try:
444-
await self._handle_after_agent_callback(ctx)
445-
except asyncio.CancelledError:
446-
raise
447-
except Exception: # pylint: disable=broad-except
448-
logger.exception(
449-
'after_agent_callback raised on cancellation;'
450-
' suppressing so original cancellation propagates.'
451-
)
452-
raise
453-
except Exception as e:
454-
await self._handle_agent_error_callback(ctx, e)
455-
raise
456-
457-
async with Aclosing(_with_caller_context(_run(), caller_ctx)) as agen:
458-
async for event in agen:
459-
yield event
426+
if ctx.end_invocation:
427+
return
428+
429+
after_callback_called = True
430+
if event := await self._handle_after_agent_callback(ctx):
431+
yield event
432+
except asyncio.CancelledError:
433+
if (
434+
before_callback_completed
435+
and not after_callback_called
436+
and not ctx.end_invocation
437+
):
438+
try:
439+
await self._handle_after_agent_callback(ctx)
440+
except asyncio.CancelledError:
441+
raise
442+
except Exception: # pylint: disable=broad-except
443+
logger.exception(
444+
'after_agent_callback raised on cancellation;'
445+
' suppressing so original cancellation propagates.'
446+
)
447+
raise
448+
except Exception as e:
449+
await self._handle_agent_error_callback(ctx, e)
450+
raise
451+
452+
# The scope is entered out here rather than inside `_run`, so that its
453+
# span is not opened and closed by whoever happens to drive the generator,
454+
# and `stream_in_scope` attaches it only while `_run` is actually running
455+
# (b/485899016).
456+
async with _instrumentation.record_agent_invocation(ctx, self) as scope:
457+
async with Aclosing(
458+
_instrumentation.stream_in_scope(scope.otel_context, _run())
459+
) as agen:
460+
async for event in agen:
461+
yield event
460462

461463
async def _run_async_impl(
462464
self, ctx: InvocationContext

‎src/google/adk/flows/llm_flows/core/_model_call.py‎

Lines changed: 129 additions & 113 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@
3535
from ....telemetry import _instrumentation
3636
from ....telemetry.tracing import trace_call_llm
3737
from ....telemetry.tracing import tracer
38-
from ....utils._runner_utils import _with_caller_context
3938
from ....utils.context_utils import Aclosing
4039
from ._finalizer import has_meaningful_content
4140
from ._utils import as_llm_agent as _as_llm_agent
@@ -182,129 +181,146 @@ async def call_llm_async(
182181
# caller's post-processing -- tool calls, agent transfers -- is traced as
183182
# a child of the model call instead of a sibling of it.
184183
caller_context = otel_context.get_current()
184+
# The span is opened here and made current by `stream_in_scope` around
185+
# each drive of the generator below, rather than by a `with` block inside
186+
# it: a token attached inside a generator and held across its `yield` is
187+
# detached by whoever closes the stream, in a context that never attached it.
188+
span = tracer.start_span('call_llm', context=caller_context)
189+
model_call_context = trace.set_span_in_context(span, caller_context)
185190

186191
async def _call_llm_with_tracing() -> AsyncGenerator[LlmResponse, None]:
187-
with tracer.start_as_current_span('call_llm') as span:
188-
# Runs before_model_callback inside the call_llm span so
189-
# plugins observe the same span as after/error callbacks.
190-
if response := await flow._handle_before_model_callback(
191-
invocation_context, llm_request, model_response_event
192-
):
193-
# The model was never called, but the span still has to carry its
194-
# attributes: trace consumers key off the event id attribute and
195-
# drop spans that lack it.
196-
trace_call_llm(
197-
invocation_context,
198-
model_response_event.id,
199-
llm_request,
200-
response,
201-
span,
202-
)
203-
_instrumentation.record_response_source(
204-
span, 'before_model_callback', invocation_context
205-
)
206-
yield response
207-
return
208-
209-
llm_request.config = llm_request.config or types.GenerateContentConfig()
210-
llm_request.config.labels = llm_request.config.labels or {}
211-
212-
# Add agent name as a label to the llm_request. This will help
213-
# with slicing billing reports on a per-agent basis.
214-
if ADK_AGENT_NAME_LABEL_KEY not in llm_request.config.labels:
215-
llm_request.config.labels[ADK_AGENT_NAME_LABEL_KEY] = agent.name
216-
217-
# Calls the LLM.
218-
llm = await flow._get_llm(invocation_context)
219-
220-
# Check if we can make this llm call or not. If the current
221-
# call pushes the counter beyond the max set value, then the
222-
# execution is stopped right here, and exception is thrown.
223-
invocation_context.increment_llm_call_count()
224-
225-
if run_config.support_cfc:
226-
if invocation_context.live_request_queue is None:
227-
invocation_context.live_request_queue = LiveRequestQueue()
228-
async with Aclosing(
229-
flow._run_and_handle_error(
230-
flow.run_live(invocation_context),
192+
# Runs before_model_callback inside the call_llm span so
193+
# plugins observe the same span as after/error callbacks.
194+
if response := await flow._handle_before_model_callback(
195+
invocation_context, llm_request, model_response_event
196+
):
197+
# The model was never called, but the span still has to carry its
198+
# attributes: trace consumers key off the event id attribute and
199+
# drop spans that lack it.
200+
trace_call_llm(
201+
invocation_context,
202+
model_response_event.id,
203+
llm_request,
204+
response,
205+
span,
206+
)
207+
_instrumentation.record_response_source(
208+
span, 'before_model_callback', invocation_context
209+
)
210+
yield response
211+
return
212+
213+
llm_request.config = llm_request.config or types.GenerateContentConfig()
214+
llm_request.config.labels = llm_request.config.labels or {}
215+
216+
# Add agent name as a label to the llm_request. This will help
217+
# with slicing billing reports on a per-agent basis.
218+
if ADK_AGENT_NAME_LABEL_KEY not in llm_request.config.labels:
219+
llm_request.config.labels[ADK_AGENT_NAME_LABEL_KEY] = agent.name
220+
221+
# Calls the LLM.
222+
llm = await flow._get_llm(invocation_context)
223+
224+
# Check if we can make this llm call or not. If the current
225+
# call pushes the counter beyond the max set value, then the
226+
# execution is stopped right here, and exception is thrown.
227+
invocation_context.increment_llm_call_count()
228+
229+
if run_config.support_cfc:
230+
if invocation_context.live_request_queue is None:
231+
invocation_context.live_request_queue = LiveRequestQueue()
232+
async with Aclosing(
233+
flow._run_and_handle_error(
234+
flow.run_live(invocation_context),
235+
invocation_context,
236+
llm_request,
237+
model_response_event,
238+
call_llm_span=span,
239+
)
240+
) as agen:
241+
async for event in agen:
242+
# Rebind to call_llm span for after_model_callback.
243+
with trace.use_span(span, end_on_exit=False):
244+
if altered := await flow._handle_after_model_callback(
231245
invocation_context,
232-
llm_request,
246+
event,
233247
model_response_event,
234-
call_llm_span=span,
235-
)
236-
) as agen:
237-
async for event in agen:
248+
):
249+
event = altered
250+
# only yield partial response in SSE streaming mode
251+
if (
252+
run_config.streaming_mode == StreamingMode.SSE
253+
or not event.partial
254+
):
255+
yield event
256+
if event.turn_complete:
257+
queue = invocation_context.live_request_queue
258+
assert queue is not None
259+
queue.close()
260+
else:
261+
responses_generator = llm.generate_content_async(
262+
llm_request,
263+
stream=run_config.streaming_mode == StreamingMode.SSE,
264+
)
265+
async with Aclosing(
266+
flow._run_and_handle_error(
267+
responses_generator,
268+
invocation_context,
269+
llm_request,
270+
model_response_event,
271+
call_llm_span=span,
272+
)
273+
) as agen:
274+
# Partials overwrite each other on the span; trace only the last one.
275+
deferred_partial: tuple[str, LlmResponse] | None = None
276+
try:
277+
async for llm_response in agen:
278+
if llm_response.partial:
279+
deferred_partial = (model_response_event.id, llm_response)
280+
else:
281+
deferred_partial = None
282+
trace_call_llm(
283+
invocation_context,
284+
model_response_event.id,
285+
llm_request,
286+
llm_response,
287+
span,
288+
)
238289
# Rebind to call_llm span for after_model_callback.
239290
with trace.use_span(span, end_on_exit=False):
240291
if altered := await flow._handle_after_model_callback(
241292
invocation_context,
242-
event,
293+
llm_response,
243294
model_response_event,
244295
):
245-
event = altered
246-
# only yield partial response in SSE streaming mode
247-
if (
248-
run_config.streaming_mode == StreamingMode.SSE
249-
or not event.partial
250-
):
251-
yield event
252-
if event.turn_complete:
253-
queue = invocation_context.live_request_queue
254-
assert queue is not None
255-
queue.close()
256-
else:
257-
responses_generator = llm.generate_content_async(
258-
llm_request,
259-
stream=run_config.streaming_mode == StreamingMode.SSE,
260-
)
261-
async with Aclosing(
262-
flow._run_and_handle_error(
263-
responses_generator,
296+
llm_response = altered
297+
298+
yield llm_response
299+
finally:
300+
if deferred_partial is not None:
301+
deferred_event_id, partial_response = deferred_partial
302+
trace_call_llm(
264303
invocation_context,
304+
deferred_event_id,
265305
llm_request,
266-
model_response_event,
267-
call_llm_span=span,
306+
partial_response,
307+
span,
268308
)
269-
) as agen:
270-
# Partials overwrite each other on the span; trace only the last one.
271-
deferred_partial: tuple[str, LlmResponse] | None = None
272-
try:
273-
async for llm_response in agen:
274-
if llm_response.partial:
275-
deferred_partial = (model_response_event.id, llm_response)
276-
else:
277-
deferred_partial = None
278-
trace_call_llm(
279-
invocation_context,
280-
model_response_event.id,
281-
llm_request,
282-
llm_response,
283-
span,
284-
)
285-
# Rebind to call_llm span for after_model_callback.
286-
with trace.use_span(span, end_on_exit=False):
287-
if altered := await flow._handle_after_model_callback(
288-
invocation_context,
289-
llm_response,
290-
model_response_event,
291-
):
292-
llm_response = altered
293-
294-
yield llm_response
295-
finally:
296-
if deferred_partial is not None:
297-
deferred_event_id, partial_response = deferred_partial
298-
trace_call_llm(
299-
invocation_context,
300-
deferred_event_id,
301-
llm_request,
302-
partial_response,
303-
span,
304-
)
305309

306-
async with Aclosing(
307-
_with_caller_context(_call_llm_with_tracing(), caller_context)
308-
) as agen:
309-
async for event in agen:
310-
yield event
310+
try:
311+
async with Aclosing(
312+
_instrumentation.stream_in_scope(
313+
model_call_context, _call_llm_with_tracing()
314+
)
315+
) as agen:
316+
async for event in agen:
317+
yield event
318+
except Exception as e: # pylint: disable=broad-except
319+
if span.is_recording():
320+
span.record_exception(e)
321+
span.set_status(
322+
trace.Status(trace.StatusCode.ERROR, f'{type(e).__name__}: {e}')
323+
)
324+
raise
325+
finally:
326+
span.end()

0 commit comments

Comments
 (0)