Skip to content

Commit 671abac

Browse files
committed
fix(plugins): address BigQuery analytics review findings
Preserve ambiguous committed-stream outcomes across retries, reject first-attempt offset conflicts, and rotate without waiting on optional finalization. Record streaming termination metadata only on final responses, distinguish model termination from node failures, and preserve structured node outputs. Document stream quota and loss boundaries and add focused regression coverage. The committed-stream design follows the public discussions in google#6465 and @addenergyx's proposal in google#6466; this implementation was developed independently.
1 parent 0380dfe commit 671abac

2 files changed

Lines changed: 491 additions & 85 deletions

File tree

‎src/google/adk/plugins/bigquery_agent_analytics_plugin.py‎

Lines changed: 101 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,12 @@ async def wrapper(
217217
_GRPC_INTERNAL = 13
218218
_GRPC_UNAVAILABLE = 14
219219

220+
_LLM_RESPONSE_ERROR_CODES = frozenset(
221+
reason.value
222+
for reason_type in (types.FinishReason, types.BlockedReason)
223+
for reason in reason_type
224+
)
225+
220226

221227
# --- Helper Formatters ---
222228
def _format_content(
@@ -1711,7 +1717,14 @@ class BigQueryLoggerConfig:
17111717
batch_flush_interval: Max time to wait before flushing a batch.
17121718
shutdown_timeout: Max time to wait for shutdown.
17131719
queue_max_size: Max size of the in-memory queue.
1714-
exactly_once_delivery: Opt into committed streams with explicit offsets.
1720+
exactly_once_delivery: Use one committed stream per event loop and
1721+
explicit offsets to prevent ambiguous retries from producing duplicate
1722+
rows. Stream rotation can consume additional ``CreateWriteStream``
1723+
quota. This mode is not lossless: batches are dropped after retry
1724+
exhaustion, offset conflicts, or replacement-stream failures, and
1725+
events arriving during the 30-second rotation backoff are also dropped.
1726+
The unconditional ``event_id`` column remains the deduplication key for
1727+
default-mode writes.
17151728
content_formatter: Optional custom formatter for content.
17161729
gcs_bucket_name: GCS bucket for offloading large content.
17171730
connection_id: BigQuery connection ID for ObjectRef columns.
@@ -1771,6 +1784,8 @@ class BigQueryLoggerConfig:
17711784
batch_flush_interval: float = 1.0
17721785
shutdown_timeout: float = 10.0
17731786
queue_max_size: int = 10000
1787+
# Opt-in duplicate prevention for retries within a live processor. See the
1788+
# class docstring for stream-quota and data-loss boundaries.
17741789
exactly_once_delivery: bool = False
17751790
content_formatter: Optional[Callable[[Any, str], Any]] = None
17761791
# If provided, large content (images, audio, video, large text) will be offloaded to this GCS bucket.
@@ -2200,7 +2215,9 @@ def __init__(
22002215
retry_config: Retry configuration.
22012216
queue_max_size: Max size of the in-memory queue.
22022217
shutdown_timeout: Max time to wait for shutdown.
2203-
exactly_once_delivery: Whether to use committed-stream offsets.
2218+
exactly_once_delivery: Whether to use committed-stream offsets to
2219+
prevent retry duplicates. Replacement streams consume stream-creation
2220+
quota, and unrecoverable/ambiguous batches may still be dropped.
22042221
create_stream: Async factory for replacement committed streams.
22052222
"""
22062223
self.write_client = write_client
@@ -2490,6 +2507,25 @@ def _confirm_committed_delivery(
24902507
return
24912508
self._next_offset = offset + row_count
24922509

2510+
def _handle_already_exists(
2511+
self,
2512+
offset: Optional[int],
2513+
row_count: int,
2514+
*,
2515+
had_ambiguous_send: bool,
2516+
) -> None:
2517+
"""Confirms this batch's retry or rejects an occupied foreign offset."""
2518+
if had_ambiguous_send:
2519+
self._confirm_committed_delivery(offset, row_count)
2520+
return
2521+
logger.warning(
2522+
"BigQuery committed stream reported an occupied offset %s before"
2523+
" this batch had an ambiguous append; rotating the stream.",
2524+
offset,
2525+
)
2526+
self._desync_stream()
2527+
self._dropped["offset_conflict"] += row_count
2528+
24932529
async def _ensure_writable_stream(self, row_count: int) -> bool:
24942530
"""Rotates a desynchronized committed stream before another append."""
24952531
if not self.exactly_once_delivery or not self._offset_desynced:
@@ -2500,9 +2536,10 @@ async def _ensure_writable_stream(self, row_count: int) -> bool:
25002536
return False
25012537

25022538
old_stream = self.write_stream
2503-
await self._finalize_stream()
2504-
if not self._stream_finalized:
2505-
self._pending_finalize_streams.add(old_stream)
2539+
# Finalization is optional for committed streams and may block on the
2540+
# network. Preserve the old name for bounded shutdown cleanup, but do not
2541+
# stall the single batch writer before creating its replacement.
2542+
self._pending_finalize_streams.add(old_stream)
25062543
try:
25072544
new_stream = await self._create_stream()
25082545
except asyncio.CancelledError:
@@ -2538,6 +2575,7 @@ async def _write_rows_with_retry(self, rows: list[dict[str, Any]]) -> None:
25382575
attempt = 0
25392576
delay = self.retry_config.initial_delay
25402577
offset_for_batch = self._next_offset if self.exactly_once_delivery else None
2578+
had_ambiguous_send = False
25412579

25422580
try:
25432581
arrow_batch = self._prepare_arrow_batch(rows)
@@ -2609,7 +2647,11 @@ async def perform_write() -> None:
26092647
definitive_rejection = True
26102648
if self.exactly_once_delivery:
26112649
if error_code == _GRPC_ALREADY_EXISTS:
2612-
self._confirm_committed_delivery(offset_for_batch, len(rows))
2650+
self._handle_already_exists(
2651+
offset_for_batch,
2652+
len(rows),
2653+
had_ambiguous_send=had_ambiguous_send,
2654+
)
26132655
return
26142656
if error_code in (_GRPC_NOT_FOUND, _GRPC_OUT_OF_RANGE):
26152657
self._desync_stream()
@@ -2653,7 +2695,11 @@ async def perform_write() -> None:
26532695

26542696
except api_exceptions.AlreadyExists as e:
26552697
if self.exactly_once_delivery:
2656-
self._confirm_committed_delivery(offset_for_batch, len(rows))
2698+
self._handle_already_exists(
2699+
offset_for_batch,
2700+
len(rows),
2701+
had_ambiguous_send=had_ambiguous_send,
2702+
)
26572703
return
26582704
self._dropped["unexpected_error"] += len(rows)
26592705
logger.error("Unexpected BigQuery Write API error: %s", e)
@@ -2677,14 +2723,12 @@ async def perform_write() -> None:
26772723
InternalServerError,
26782724
asyncio.TimeoutError,
26792725
) as e:
2726+
if request_sent and not definitive_rejection:
2727+
had_ambiguous_send = True
26802728
attempt += 1
26812729
if attempt > self.retry_config.max_retries:
26822730
self._dropped["retry_exhausted"] += len(rows)
2683-
if (
2684-
self.exactly_once_delivery
2685-
and request_sent
2686-
and not definitive_rejection
2687-
):
2731+
if self.exactly_once_delivery and had_ambiguous_send:
26882732
self._desync_stream()
26892733
logger.error(
26902734
"BigQuery Batch Dropped after %s attempts. Last error: %s."
@@ -2709,11 +2753,9 @@ async def perform_write() -> None:
27092753
delay *= self.retry_config.multiplier
27102754
except Exception as e:
27112755
self._dropped["unexpected_error"] += len(rows)
2712-
if (
2713-
self.exactly_once_delivery
2714-
and request_sent
2715-
and not definitive_rejection
2716-
):
2756+
if request_sent and not definitive_rejection:
2757+
had_ambiguous_send = True
2758+
if self.exactly_once_delivery and had_ambiguous_send:
27172759
self._desync_stream()
27182760
logger.error(
27192761
"Unexpected BigQuery Write API error (Dropping batch): %s."
@@ -3712,7 +3754,10 @@ def _get_events_schema() -> list[bigquery.SchemaField]:
37123754
"error_message",
37133755
"STRING",
37143756
mode="NULLABLE",
3715-
description="Detailed error message if the status is 'ERROR'.",
3757+
description=(
3758+
"Diagnostic message for errors and model termination details;"
3759+
" may be populated on LLM_RESPONSE rows whose status is 'OK'."
3760+
),
37163761
),
37173762
bigquery.SchemaField(
37183763
"is_truncated",
@@ -6537,30 +6582,34 @@ async def on_event_callback(
65376582
),
65386583
)
65396584

6540-
node_path = getattr(getattr(event, "node_info", None), "path", "")
6541-
if node_path and event.error_code:
6542-
await self._log_event(
6543-
"NODE_ERROR",
6544-
callback_ctx,
6545-
raw_content={"error_code": event.error_code},
6546-
event_data=EventData(
6547-
source_event=event,
6548-
status="ERROR",
6549-
error_message=event.error_message,
6550-
),
6551-
)
6552-
elif (
6553-
node_path
6554-
and event.partial is not True
6555-
and event.content is None
6556-
and event.output is not None
6557-
):
6558-
await self._log_event(
6559-
"NODE_OUTPUT",
6560-
callback_ctx,
6561-
raw_content=event.output,
6562-
event_data=EventData(source_event=event),
6563-
)
6585+
node_info = getattr(event, "node_info", None)
6586+
node_path = getattr(node_info, "path", "")
6587+
if node_path and event.partial is not True:
6588+
if event.error_code and event.error_code not in _LLM_RESPONSE_ERROR_CODES:
6589+
await self._log_event(
6590+
"NODE_ERROR",
6591+
callback_ctx,
6592+
raw_content={"error_code": event.error_code},
6593+
event_data=EventData(
6594+
source_event=event,
6595+
status="ERROR",
6596+
error_message=event.error_message,
6597+
),
6598+
)
6599+
if (
6600+
event.output is not None
6601+
and getattr(node_info, "message_as_output", None) is not True
6602+
):
6603+
node_output, output_truncated = _recursive_smart_truncate(
6604+
event.output, self.config.max_content_length
6605+
)
6606+
await self._log_event(
6607+
"NODE_OUTPUT",
6608+
callback_ctx,
6609+
raw_content=node_output,
6610+
is_truncated=output_truncated,
6611+
event_data=EventData(source_event=event),
6612+
)
65646613

65656614
# --- AGENT_TRANSFER ---
65666615
# actions.transfer_to_agent stores the *target* agent only
@@ -7013,6 +7062,7 @@ async def after_model_callback(
70137062
callback_context: The callback context.
70147063
llm_response: The LLM response object.
70157064
"""
7065+
is_partial = getattr(llm_response, "partial", None) is True
70167066
content_dict = {}
70177067
is_truncated = False
70187068
if llm_response.content:
@@ -7048,7 +7098,7 @@ async def after_model_callback(
70487098
duration = 0
70497099
tfft = None
70507100

7051-
if hasattr(llm_response, "partial") and llm_response.partial:
7101+
if is_partial:
70527102
# Streaming chunk - do NOT pop span yet
70537103
if span_id:
70547104
TraceManager.record_first_token(span_id)
@@ -7094,16 +7144,21 @@ async def after_model_callback(
70947144
usage_metadata=llm_response.usage_metadata,
70957145
cache_metadata=getattr(llm_response, "cache_metadata", None),
70967146
finish_reason=(
7097-
finish_reason.name
7098-
if (
7147+
getattr(finish_reason, "name", str(finish_reason))
7148+
if not is_partial
7149+
and (
70997150
finish_reason := getattr(
71007151
llm_response, "finish_reason", None
71017152
)
71027153
)
71037154
is not None
71047155
else None
71057156
),
7106-
error_message=getattr(llm_response, "error_message", None),
7157+
error_message=(
7158+
None
7159+
if is_partial
7160+
else getattr(llm_response, "error_message", None)
7161+
),
71077162
span_id_override=span_id if is_popped else None,
71087163
parent_span_id_override=(parent_span_id if is_popped else None),
71097164
),

0 commit comments

Comments
 (0)