|
35 | 35 | from ....telemetry import _instrumentation |
36 | 36 | from ....telemetry.tracing import trace_call_llm |
37 | 37 | from ....telemetry.tracing import tracer |
38 | | -from ....utils._runner_utils import _with_caller_context |
39 | 38 | from ....utils.context_utils import Aclosing |
40 | 39 | from ._finalizer import has_meaningful_content |
41 | 40 | from ._utils import as_llm_agent as _as_llm_agent |
@@ -182,129 +181,146 @@ async def call_llm_async( |
182 | 181 | # caller's post-processing -- tool calls, agent transfers -- is traced as |
183 | 182 | # a child of the model call instead of a sibling of it. |
184 | 183 | caller_context = otel_context.get_current() |
| 184 | + # The span is opened here and made current by `stream_in_scope` around |
| 185 | + # each drive of the generator below, rather than by a `with` block inside |
| 186 | + # it: a token attached inside a generator and held across its `yield` is |
| 187 | + # detached by whoever closes the stream, in a context that never attached it. |
| 188 | + span = tracer.start_span('call_llm', context=caller_context) |
| 189 | + model_call_context = trace.set_span_in_context(span, caller_context) |
185 | 190 |
|
186 | 191 | async def _call_llm_with_tracing() -> AsyncGenerator[LlmResponse, None]: |
187 | | - with tracer.start_as_current_span('call_llm') as span: |
188 | | - # Runs before_model_callback inside the call_llm span so |
189 | | - # plugins observe the same span as after/error callbacks. |
190 | | - if response := await flow._handle_before_model_callback( |
191 | | - invocation_context, llm_request, model_response_event |
192 | | - ): |
193 | | - # The model was never called, but the span still has to carry its |
194 | | - # attributes: trace consumers key off the event id attribute and |
195 | | - # drop spans that lack it. |
196 | | - trace_call_llm( |
197 | | - invocation_context, |
198 | | - model_response_event.id, |
199 | | - llm_request, |
200 | | - response, |
201 | | - span, |
202 | | - ) |
203 | | - _instrumentation.record_response_source( |
204 | | - span, 'before_model_callback', invocation_context |
205 | | - ) |
206 | | - yield response |
207 | | - return |
208 | | - |
209 | | - llm_request.config = llm_request.config or types.GenerateContentConfig() |
210 | | - llm_request.config.labels = llm_request.config.labels or {} |
211 | | - |
212 | | - # Add agent name as a label to the llm_request. This will help |
213 | | - # with slicing billing reports on a per-agent basis. |
214 | | - if ADK_AGENT_NAME_LABEL_KEY not in llm_request.config.labels: |
215 | | - llm_request.config.labels[ADK_AGENT_NAME_LABEL_KEY] = agent.name |
216 | | - |
217 | | - # Calls the LLM. |
218 | | - llm = await flow._get_llm(invocation_context) |
219 | | - |
220 | | - # Check if we can make this llm call or not. If the current |
221 | | - # call pushes the counter beyond the max set value, then the |
222 | | - # execution is stopped right here, and exception is thrown. |
223 | | - invocation_context.increment_llm_call_count() |
224 | | - |
225 | | - if run_config.support_cfc: |
226 | | - if invocation_context.live_request_queue is None: |
227 | | - invocation_context.live_request_queue = LiveRequestQueue() |
228 | | - async with Aclosing( |
229 | | - flow._run_and_handle_error( |
230 | | - flow.run_live(invocation_context), |
| 192 | + # Runs before_model_callback inside the call_llm span so |
| 193 | + # plugins observe the same span as after/error callbacks. |
| 194 | + if response := await flow._handle_before_model_callback( |
| 195 | + invocation_context, llm_request, model_response_event |
| 196 | + ): |
| 197 | + # The model was never called, but the span still has to carry its |
| 198 | + # attributes: trace consumers key off the event id attribute and |
| 199 | + # drop spans that lack it. |
| 200 | + trace_call_llm( |
| 201 | + invocation_context, |
| 202 | + model_response_event.id, |
| 203 | + llm_request, |
| 204 | + response, |
| 205 | + span, |
| 206 | + ) |
| 207 | + _instrumentation.record_response_source( |
| 208 | + span, 'before_model_callback', invocation_context |
| 209 | + ) |
| 210 | + yield response |
| 211 | + return |
| 212 | + |
| 213 | + llm_request.config = llm_request.config or types.GenerateContentConfig() |
| 214 | + llm_request.config.labels = llm_request.config.labels or {} |
| 215 | + |
| 216 | + # Add agent name as a label to the llm_request. This will help |
| 217 | + # with slicing billing reports on a per-agent basis. |
| 218 | + if ADK_AGENT_NAME_LABEL_KEY not in llm_request.config.labels: |
| 219 | + llm_request.config.labels[ADK_AGENT_NAME_LABEL_KEY] = agent.name |
| 220 | + |
| 221 | + # Calls the LLM. |
| 222 | + llm = await flow._get_llm(invocation_context) |
| 223 | + |
| 224 | + # Check if we can make this llm call or not. If the current |
| 225 | + # call pushes the counter beyond the max set value, then the |
| 226 | + # execution is stopped right here, and exception is thrown. |
| 227 | + invocation_context.increment_llm_call_count() |
| 228 | + |
| 229 | + if run_config.support_cfc: |
| 230 | + if invocation_context.live_request_queue is None: |
| 231 | + invocation_context.live_request_queue = LiveRequestQueue() |
| 232 | + async with Aclosing( |
| 233 | + flow._run_and_handle_error( |
| 234 | + flow.run_live(invocation_context), |
| 235 | + invocation_context, |
| 236 | + llm_request, |
| 237 | + model_response_event, |
| 238 | + call_llm_span=span, |
| 239 | + ) |
| 240 | + ) as agen: |
| 241 | + async for event in agen: |
| 242 | + # Rebind to call_llm span for after_model_callback. |
| 243 | + with trace.use_span(span, end_on_exit=False): |
| 244 | + if altered := await flow._handle_after_model_callback( |
231 | 245 | invocation_context, |
232 | | - llm_request, |
| 246 | + event, |
233 | 247 | model_response_event, |
234 | | - call_llm_span=span, |
235 | | - ) |
236 | | - ) as agen: |
237 | | - async for event in agen: |
| 248 | + ): |
| 249 | + event = altered |
| 250 | + # only yield partial response in SSE streaming mode |
| 251 | + if ( |
| 252 | + run_config.streaming_mode == StreamingMode.SSE |
| 253 | + or not event.partial |
| 254 | + ): |
| 255 | + yield event |
| 256 | + if event.turn_complete: |
| 257 | + queue = invocation_context.live_request_queue |
| 258 | + assert queue is not None |
| 259 | + queue.close() |
| 260 | + else: |
| 261 | + responses_generator = llm.generate_content_async( |
| 262 | + llm_request, |
| 263 | + stream=run_config.streaming_mode == StreamingMode.SSE, |
| 264 | + ) |
| 265 | + async with Aclosing( |
| 266 | + flow._run_and_handle_error( |
| 267 | + responses_generator, |
| 268 | + invocation_context, |
| 269 | + llm_request, |
| 270 | + model_response_event, |
| 271 | + call_llm_span=span, |
| 272 | + ) |
| 273 | + ) as agen: |
| 274 | + # Partials overwrite each other on the span; trace only the last one. |
| 275 | + deferred_partial: tuple[str, LlmResponse] | None = None |
| 276 | + try: |
| 277 | + async for llm_response in agen: |
| 278 | + if llm_response.partial: |
| 279 | + deferred_partial = (model_response_event.id, llm_response) |
| 280 | + else: |
| 281 | + deferred_partial = None |
| 282 | + trace_call_llm( |
| 283 | + invocation_context, |
| 284 | + model_response_event.id, |
| 285 | + llm_request, |
| 286 | + llm_response, |
| 287 | + span, |
| 288 | + ) |
238 | 289 | # Rebind to call_llm span for after_model_callback. |
239 | 290 | with trace.use_span(span, end_on_exit=False): |
240 | 291 | if altered := await flow._handle_after_model_callback( |
241 | 292 | invocation_context, |
242 | | - event, |
| 293 | + llm_response, |
243 | 294 | model_response_event, |
244 | 295 | ): |
245 | | - event = altered |
246 | | - # only yield partial response in SSE streaming mode |
247 | | - if ( |
248 | | - run_config.streaming_mode == StreamingMode.SSE |
249 | | - or not event.partial |
250 | | - ): |
251 | | - yield event |
252 | | - if event.turn_complete: |
253 | | - queue = invocation_context.live_request_queue |
254 | | - assert queue is not None |
255 | | - queue.close() |
256 | | - else: |
257 | | - responses_generator = llm.generate_content_async( |
258 | | - llm_request, |
259 | | - stream=run_config.streaming_mode == StreamingMode.SSE, |
260 | | - ) |
261 | | - async with Aclosing( |
262 | | - flow._run_and_handle_error( |
263 | | - responses_generator, |
| 296 | + llm_response = altered |
| 297 | + |
| 298 | + yield llm_response |
| 299 | + finally: |
| 300 | + if deferred_partial is not None: |
| 301 | + deferred_event_id, partial_response = deferred_partial |
| 302 | + trace_call_llm( |
264 | 303 | invocation_context, |
| 304 | + deferred_event_id, |
265 | 305 | llm_request, |
266 | | - model_response_event, |
267 | | - call_llm_span=span, |
| 306 | + partial_response, |
| 307 | + span, |
268 | 308 | ) |
269 | | - ) as agen: |
270 | | - # Partials overwrite each other on the span; trace only the last one. |
271 | | - deferred_partial: tuple[str, LlmResponse] | None = None |
272 | | - try: |
273 | | - async for llm_response in agen: |
274 | | - if llm_response.partial: |
275 | | - deferred_partial = (model_response_event.id, llm_response) |
276 | | - else: |
277 | | - deferred_partial = None |
278 | | - trace_call_llm( |
279 | | - invocation_context, |
280 | | - model_response_event.id, |
281 | | - llm_request, |
282 | | - llm_response, |
283 | | - span, |
284 | | - ) |
285 | | - # Rebind to call_llm span for after_model_callback. |
286 | | - with trace.use_span(span, end_on_exit=False): |
287 | | - if altered := await flow._handle_after_model_callback( |
288 | | - invocation_context, |
289 | | - llm_response, |
290 | | - model_response_event, |
291 | | - ): |
292 | | - llm_response = altered |
293 | | - |
294 | | - yield llm_response |
295 | | - finally: |
296 | | - if deferred_partial is not None: |
297 | | - deferred_event_id, partial_response = deferred_partial |
298 | | - trace_call_llm( |
299 | | - invocation_context, |
300 | | - deferred_event_id, |
301 | | - llm_request, |
302 | | - partial_response, |
303 | | - span, |
304 | | - ) |
305 | 309 |
|
306 | | - async with Aclosing( |
307 | | - _with_caller_context(_call_llm_with_tracing(), caller_context) |
308 | | - ) as agen: |
309 | | - async for event in agen: |
310 | | - yield event |
| 310 | + try: |
| 311 | + async with Aclosing( |
| 312 | + _instrumentation.stream_in_scope( |
| 313 | + model_call_context, _call_llm_with_tracing() |
| 314 | + ) |
| 315 | + ) as agen: |
| 316 | + async for event in agen: |
| 317 | + yield event |
| 318 | + except Exception as e: # pylint: disable=broad-except |
| 319 | + if span.is_recording(): |
| 320 | + span.record_exception(e) |
| 321 | + span.set_status( |
| 322 | + trace.Status(trace.StatusCode.ERROR, f'{type(e).__name__}: {e}') |
| 323 | + ) |
| 324 | + raise |
| 325 | + finally: |
| 326 | + span.end() |
0 commit comments