Skip to content

Commit 462e1b6

Browse files
caohy1988claude
andcommitted
fix: trace_id continuity and o11y alignment for BQ plugin (google#4645)
Fix trace_id fracture between INVOCATION_STARTING and AGENT_STARTING when no ambient OTel span exists (e.g. Agent Engine, custom runners). Also align BQ rows with Cloud Trace span IDs when o11y is active. Changes: - TraceManager.ensure_invocation_span(): ensures a root span is on the plugin stack before any events fire, preventing early events from falling back to invocation_id while later events get OTel hex IDs. - TraceManager.push_span(): create child spans under existing stack parent via trace.set_span_in_context() so all spans in an invocation share the same trace_id (google#4645). - _resolve_ids() replaces _resolve_span_ids(): 3-layer priority for trace_id/span_id/parent_span_id resolution: 1. EventData overrides (post-pop callbacks) 2. Ambient OTel span (aligns with Cloud Trace when o11y is active) 3. Plugin's internal span stack (fallback for no-ambient paths) - EventData.trace_id_override: captured before pop in after_run_callback so INVOCATION_COMPLETED shares the same trace_id as earlier events. - on_user_message_callback / before_run_callback: call ensure_invocation_span() before logging. - after_run_callback: capture trace_id before pop, pass as override. Tests: - TestTraceIdContinuity: 5 tests covering no-ambient, ambient, cross-turn isolation, completion-event continuity, and full callback-level integration. - TestResolveIds: 6 tests replacing TestResolveSpanIds, adding ambient priority and override-beats-ambient coverage. Co-Authored-By: Claude Opus 4.6 <[email protected]>
1 parent b82b263 commit 462e1b6

2 files changed

Lines changed: 493 additions & 60 deletions

File tree

‎src/google/adk/plugins/bigquery_agent_analytics_plugin.py‎

Lines changed: 94 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -604,7 +604,16 @@ def push_span(
604604
# Create the span without attaching it to the ambient context.
605605
# This avoids re-parenting framework spans like ``call_llm``
606606
# or ``execute_tool``. See #4561.
607-
span = tracer.start_span(span_name)
607+
#
608+
# If the internal stack already has a span, create the new span
609+
# as a child so it shares the same trace_id. Without this, each
610+
# ``start_span`` would be an independent root with its own
611+
# trace_id — causing trace_id fracture (see #4645).
612+
records = TraceManager._get_records()
613+
parent_ctx = None
614+
if records and records[-1].span.get_span_context().is_valid:
615+
parent_ctx = trace.set_span_in_context(records[-1].span)
616+
span = tracer.start_span(span_name, context=parent_ctx)
608617

609618
if span.get_span_context().is_valid:
610619
span_id_str = format(span.get_span_context().span_id, "016x")
@@ -618,7 +627,6 @@ def push_span(
618627
start_time_ns=time.time_ns(),
619628
)
620629

621-
records = TraceManager._get_records()
622630
new_records = list(records) + [record]
623631
_span_records_ctx.set(new_records)
624632

@@ -655,6 +663,32 @@ def attach_current_span(
655663

656664
return span_id_str
657665

666+
@staticmethod
667+
def ensure_invocation_span(
668+
callback_context: CallbackContext,
669+
) -> None:
670+
"""Ensures a root span exists on the plugin stack for this invocation.
671+
672+
Must be called before any events are logged so that every event in
673+
the invocation shares the same trace_id.
674+
675+
* If the stack already has entries → no-op (already initialised).
676+
* If the ambient OTel span is valid → ``attach_current_span`` (reuse
677+
the runner's span without owning it).
678+
* Otherwise → ``push_span("invocation")`` (create a new root span
679+
that will be popped in ``after_run_callback``).
680+
"""
681+
records = _span_records_ctx.get()
682+
if records:
683+
return # Already initialised for this invocation.
684+
685+
# Check for a valid ambient span (e.g. the Runner's invocation span).
686+
ambient = trace.get_current_span()
687+
if ambient.get_span_context().is_valid:
688+
TraceManager.attach_current_span(callback_context)
689+
else:
690+
TraceManager.push_span(callback_context, "invocation")
691+
658692
@staticmethod
659693
def pop_span() -> tuple[Optional[str], Optional[int]]:
660694
"""Ends the current span and pops it from the stack.
@@ -1709,6 +1743,7 @@ class _LoopState:
17091743
class EventData:
17101744
"""Typed container for structured fields passed to _log_event."""
17111745

1746+
trace_id_override: Optional[str] = None
17121747
span_id_override: Optional[str] = None
17131748
parent_span_id_override: Optional[str] = None
17141749
latency_ms: Optional[int] = None
@@ -2256,27 +2291,50 @@ async def _ensure_started(self, **kwargs) -> None:
22562291
logger.error("Failed to initialize BigQuery Plugin: %s", e)
22572292

22582293
@staticmethod
2259-
def _resolve_span_ids(
2294+
def _resolve_ids(
22602295
event_data: EventData,
2261-
) -> tuple[str, str]:
2262-
"""Reads span/parent overrides from EventData, falling back to TraceManager.
2296+
callback_context: CallbackContext,
2297+
) -> tuple[Optional[str], Optional[str], Optional[str]]:
2298+
"""Resolves trace_id, span_id, and parent_span_id for a log row.
2299+
2300+
Priority order (highest first):
2301+
1. Explicit ``EventData`` overrides (needed for post-pop callbacks).
2302+
2. Ambient OTel span (the framework's ``start_as_current_span``).
2303+
When present this aligns BQ rows with Cloud Trace / o11y.
2304+
3. Plugin's internal span stack (``TraceManager``).
2305+
4. ``invocation_id`` fallback for trace_id.
22632306
22642307
Returns:
2265-
(span_id, parent_span_id)
2308+
(trace_id, span_id, parent_span_id)
22662309
"""
2267-
current_span_id, current_parent_span_id = (
2310+
# --- Layer 3: plugin stack baseline ---
2311+
trace_id = TraceManager.get_trace_id(callback_context)
2312+
plugin_span_id, plugin_parent_span_id = (
22682313
TraceManager.get_current_span_and_parent()
22692314
)
2270-
2271-
span_id = current_span_id
2315+
span_id = plugin_span_id
2316+
parent_span_id = plugin_parent_span_id
2317+
2318+
# --- Layer 2: ambient OTel span ---
2319+
ambient = trace.get_current_span()
2320+
ambient_ctx = ambient.get_span_context()
2321+
if ambient_ctx.is_valid:
2322+
trace_id = format(ambient_ctx.trace_id, "032x")
2323+
span_id = format(ambient_ctx.span_id, "016x")
2324+
# SDK spans expose .parent; non-recording spans do not.
2325+
parent_ctx = getattr(ambient, "parent", None)
2326+
if parent_ctx is not None and parent_ctx.span_id:
2327+
parent_span_id = format(parent_ctx.span_id, "016x")
2328+
2329+
# --- Layer 1: explicit EventData overrides ---
2330+
if event_data.trace_id_override is not None:
2331+
trace_id = event_data.trace_id_override
22722332
if event_data.span_id_override is not None:
22732333
span_id = event_data.span_id_override
2274-
2275-
parent_span_id = current_parent_span_id
22762334
if event_data.parent_span_id_override is not None:
22772335
parent_span_id = event_data.parent_span_id_override
22782336

2279-
return span_id, parent_span_id
2337+
return trace_id, span_id, parent_span_id
22802338

22812339
@staticmethod
22822340
def _extract_latency(
@@ -2389,8 +2447,9 @@ async def _log_event(
23892447
except Exception as e:
23902448
logger.warning("Content formatter failed: %s", e)
23912449

2392-
trace_id = TraceManager.get_trace_id(callback_context)
2393-
span_id, parent_span_id = self._resolve_span_ids(event_data)
2450+
trace_id, span_id, parent_span_id = self._resolve_ids(
2451+
event_data, callback_context
2452+
)
23942453

23952454
if not self.parser:
23962455
logger.warning("Parser not initialized; skipping event %s.", event_type)
@@ -2457,6 +2516,7 @@ async def on_user_message_callback(
24572516
user_message: The message content received from the user.
24582517
"""
24592518
callback_ctx = CallbackContext(invocation_context)
2519+
TraceManager.ensure_invocation_span(callback_ctx)
24602520
await self._log_event(
24612521
"USER_MESSAGE_RECEIVED",
24622522
callback_ctx,
@@ -2591,9 +2651,11 @@ async def before_run_callback(
25912651
invocation_context: The context of the current invocation.
25922652
"""
25932653
await self._ensure_started()
2654+
callback_ctx = CallbackContext(invocation_context)
2655+
TraceManager.ensure_invocation_span(callback_ctx)
25942656
await self._log_event(
25952657
"INVOCATION_STARTING",
2596-
CallbackContext(invocation_context),
2658+
callback_ctx,
25972659
)
25982660

25992661
@_safe_callback
@@ -2605,9 +2667,25 @@ async def after_run_callback(
26052667
Args:
26062668
invocation_context: The context of the current invocation.
26072669
"""
2670+
# Capture trace_id BEFORE popping the invocation-root span so that
2671+
# INVOCATION_COMPLETED shares the same trace_id as all earlier events
2672+
# in this invocation (fixes #4645 completion-event fracture).
2673+
callback_ctx = CallbackContext(invocation_context)
2674+
trace_id = TraceManager.get_trace_id(callback_ctx)
2675+
2676+
# Pop the invocation-root span pushed by ensure_invocation_span().
2677+
span_id, duration = TraceManager.pop_span()
2678+
parent_span_id = TraceManager.get_current_span_id()
2679+
26082680
await self._log_event(
26092681
"INVOCATION_COMPLETED",
2610-
CallbackContext(invocation_context),
2682+
callback_ctx,
2683+
event_data=EventData(
2684+
trace_id_override=trace_id,
2685+
latency_ms=duration,
2686+
span_id_override=span_id,
2687+
parent_span_id_override=parent_span_id,
2688+
),
26112689
)
26122690
# Ensure all logs are flushed before the agent returns
26132691
await self.flush()

0 commit comments

Comments
 (0)