@@ -217,6 +217,12 @@ async def wrapper(
217217_GRPC_INTERNAL = 13
218218_GRPC_UNAVAILABLE = 14
219219
220+ _LLM_RESPONSE_ERROR_CODES = frozenset (
221+ reason .value
222+ for reason_type in (types .FinishReason , types .BlockedReason )
223+ for reason in reason_type
224+ )
225+
220226
221227# --- Helper Formatters ---
222228def _format_content (
@@ -1711,7 +1717,14 @@ class BigQueryLoggerConfig:
17111717 batch_flush_interval: Max time to wait before flushing a batch.
17121718 shutdown_timeout: Max time to wait for shutdown.
17131719 queue_max_size: Max size of the in-memory queue.
1714- exactly_once_delivery: Opt into committed streams with explicit offsets.
1720+ exactly_once_delivery: Use one committed stream per event loop and
1721+ explicit offsets to prevent ambiguous retries from producing duplicate
1722+ rows. Stream rotation can consume additional ``CreateWriteStream``
1723+ quota. This mode is not lossless: batches are dropped after retry
1724+ exhaustion, offset conflicts, or replacement-stream failures, and
1725+ events arriving during the 30-second rotation backoff are also dropped.
1726+ The unconditional ``event_id`` column remains the deduplication key for
1727+ default-mode writes.
17151728 content_formatter: Optional custom formatter for content.
17161729 gcs_bucket_name: GCS bucket for offloading large content.
17171730 connection_id: BigQuery connection ID for ObjectRef columns.
@@ -1771,6 +1784,8 @@ class BigQueryLoggerConfig:
17711784 batch_flush_interval : float = 1.0
17721785 shutdown_timeout : float = 10.0
17731786 queue_max_size : int = 10000
1787+ # Opt-in duplicate prevention for retries within a live processor. See the
1788+ # class docstring for stream-quota and data-loss boundaries.
17741789 exactly_once_delivery : bool = False
17751790 content_formatter : Optional [Callable [[Any , str ], Any ]] = None
17761791 # If provided, large content (images, audio, video, large text) will be offloaded to this GCS bucket.
@@ -2200,7 +2215,9 @@ def __init__(
22002215 retry_config: Retry configuration.
22012216 queue_max_size: Max size of the in-memory queue.
22022217 shutdown_timeout: Max time to wait for shutdown.
2203- exactly_once_delivery: Whether to use committed-stream offsets.
2218+ exactly_once_delivery: Whether to use committed-stream offsets to
2219+ prevent retry duplicates. Replacement streams consume stream-creation
2220+ quota, and unrecoverable/ambiguous batches may still be dropped.
22042221 create_stream: Async factory for replacement committed streams.
22052222 """
22062223 self .write_client = write_client
@@ -2490,6 +2507,25 @@ def _confirm_committed_delivery(
24902507 return
24912508 self ._next_offset = offset + row_count
24922509
2510+ def _handle_already_exists (
2511+ self ,
2512+ offset : Optional [int ],
2513+ row_count : int ,
2514+ * ,
2515+ had_ambiguous_send : bool ,
2516+ ) -> None :
2517+ """Confirms this batch's retry or rejects an occupied foreign offset."""
2518+ if had_ambiguous_send :
2519+ self ._confirm_committed_delivery (offset , row_count )
2520+ return
2521+ logger .warning (
2522+ "BigQuery committed stream reported an occupied offset %s before"
2523+ " this batch had an ambiguous append; rotating the stream." ,
2524+ offset ,
2525+ )
2526+ self ._desync_stream ()
2527+ self ._dropped ["offset_conflict" ] += row_count
2528+
24932529 async def _ensure_writable_stream (self , row_count : int ) -> bool :
24942530 """Rotates a desynchronized committed stream before another append."""
24952531 if not self .exactly_once_delivery or not self ._offset_desynced :
@@ -2500,9 +2536,10 @@ async def _ensure_writable_stream(self, row_count: int) -> bool:
25002536 return False
25012537
25022538 old_stream = self .write_stream
2503- await self ._finalize_stream ()
2504- if not self ._stream_finalized :
2505- self ._pending_finalize_streams .add (old_stream )
2539+ # Finalization is optional for committed streams and may block on the
2540+ # network. Preserve the old name for bounded shutdown cleanup, but do not
2541+ # stall the single batch writer before creating its replacement.
2542+ self ._pending_finalize_streams .add (old_stream )
25062543 try :
25072544 new_stream = await self ._create_stream ()
25082545 except asyncio .CancelledError :
@@ -2538,6 +2575,7 @@ async def _write_rows_with_retry(self, rows: list[dict[str, Any]]) -> None:
25382575 attempt = 0
25392576 delay = self .retry_config .initial_delay
25402577 offset_for_batch = self ._next_offset if self .exactly_once_delivery else None
2578+ had_ambiguous_send = False
25412579
25422580 try :
25432581 arrow_batch = self ._prepare_arrow_batch (rows )
@@ -2609,7 +2647,11 @@ async def perform_write() -> None:
26092647 definitive_rejection = True
26102648 if self .exactly_once_delivery :
26112649 if error_code == _GRPC_ALREADY_EXISTS :
2612- self ._confirm_committed_delivery (offset_for_batch , len (rows ))
2650+ self ._handle_already_exists (
2651+ offset_for_batch ,
2652+ len (rows ),
2653+ had_ambiguous_send = had_ambiguous_send ,
2654+ )
26132655 return
26142656 if error_code in (_GRPC_NOT_FOUND , _GRPC_OUT_OF_RANGE ):
26152657 self ._desync_stream ()
@@ -2653,7 +2695,11 @@ async def perform_write() -> None:
26532695
26542696 except api_exceptions .AlreadyExists as e :
26552697 if self .exactly_once_delivery :
2656- self ._confirm_committed_delivery (offset_for_batch , len (rows ))
2698+ self ._handle_already_exists (
2699+ offset_for_batch ,
2700+ len (rows ),
2701+ had_ambiguous_send = had_ambiguous_send ,
2702+ )
26572703 return
26582704 self ._dropped ["unexpected_error" ] += len (rows )
26592705 logger .error ("Unexpected BigQuery Write API error: %s" , e )
@@ -2677,14 +2723,12 @@ async def perform_write() -> None:
26772723 InternalServerError ,
26782724 asyncio .TimeoutError ,
26792725 ) as e :
2726+ if request_sent and not definitive_rejection :
2727+ had_ambiguous_send = True
26802728 attempt += 1
26812729 if attempt > self .retry_config .max_retries :
26822730 self ._dropped ["retry_exhausted" ] += len (rows )
2683- if (
2684- self .exactly_once_delivery
2685- and request_sent
2686- and not definitive_rejection
2687- ):
2731+ if self .exactly_once_delivery and had_ambiguous_send :
26882732 self ._desync_stream ()
26892733 logger .error (
26902734 "BigQuery Batch Dropped after %s attempts. Last error: %s."
@@ -2709,11 +2753,9 @@ async def perform_write() -> None:
27092753 delay *= self .retry_config .multiplier
27102754 except Exception as e :
27112755 self ._dropped ["unexpected_error" ] += len (rows )
2712- if (
2713- self .exactly_once_delivery
2714- and request_sent
2715- and not definitive_rejection
2716- ):
2756+ if request_sent and not definitive_rejection :
2757+ had_ambiguous_send = True
2758+ if self .exactly_once_delivery and had_ambiguous_send :
27172759 self ._desync_stream ()
27182760 logger .error (
27192761 "Unexpected BigQuery Write API error (Dropping batch): %s."
@@ -3712,7 +3754,10 @@ def _get_events_schema() -> list[bigquery.SchemaField]:
37123754 "error_message" ,
37133755 "STRING" ,
37143756 mode = "NULLABLE" ,
3715- description = "Detailed error message if the status is 'ERROR'." ,
3757+ description = (
3758+ "Diagnostic message for errors and model termination details;"
3759+ " may be populated on LLM_RESPONSE rows whose status is 'OK'."
3760+ ),
37163761 ),
37173762 bigquery .SchemaField (
37183763 "is_truncated" ,
@@ -6537,30 +6582,34 @@ async def on_event_callback(
65376582 ),
65386583 )
65396584
6540- node_path = getattr (getattr (event , "node_info" , None ), "path" , "" )
6541- if node_path and event .error_code :
6542- await self ._log_event (
6543- "NODE_ERROR" ,
6544- callback_ctx ,
6545- raw_content = {"error_code" : event .error_code },
6546- event_data = EventData (
6547- source_event = event ,
6548- status = "ERROR" ,
6549- error_message = event .error_message ,
6550- ),
6551- )
6552- elif (
6553- node_path
6554- and event .partial is not True
6555- and event .content is None
6556- and event .output is not None
6557- ):
6558- await self ._log_event (
6559- "NODE_OUTPUT" ,
6560- callback_ctx ,
6561- raw_content = event .output ,
6562- event_data = EventData (source_event = event ),
6563- )
6585+ node_info = getattr (event , "node_info" , None )
6586+ node_path = getattr (node_info , "path" , "" )
6587+ if node_path and event .partial is not True :
6588+ if event .error_code and event .error_code not in _LLM_RESPONSE_ERROR_CODES :
6589+ await self ._log_event (
6590+ "NODE_ERROR" ,
6591+ callback_ctx ,
6592+ raw_content = {"error_code" : event .error_code },
6593+ event_data = EventData (
6594+ source_event = event ,
6595+ status = "ERROR" ,
6596+ error_message = event .error_message ,
6597+ ),
6598+ )
6599+ if (
6600+ event .output is not None
6601+ and getattr (node_info , "message_as_output" , None ) is not True
6602+ ):
6603+ node_output , output_truncated = _recursive_smart_truncate (
6604+ event .output , self .config .max_content_length
6605+ )
6606+ await self ._log_event (
6607+ "NODE_OUTPUT" ,
6608+ callback_ctx ,
6609+ raw_content = node_output ,
6610+ is_truncated = output_truncated ,
6611+ event_data = EventData (source_event = event ),
6612+ )
65646613
65656614 # --- AGENT_TRANSFER ---
65666615 # actions.transfer_to_agent stores the *target* agent only
@@ -7013,6 +7062,7 @@ async def after_model_callback(
70137062 callback_context: The callback context.
70147063 llm_response: The LLM response object.
70157064 """
7065+ is_partial = getattr (llm_response , "partial" , None ) is True
70167066 content_dict = {}
70177067 is_truncated = False
70187068 if llm_response .content :
@@ -7048,7 +7098,7 @@ async def after_model_callback(
70487098 duration = 0
70497099 tfft = None
70507100
7051- if hasattr ( llm_response , "partial" ) and llm_response . partial :
7101+ if is_partial :
70527102 # Streaming chunk - do NOT pop span yet
70537103 if span_id :
70547104 TraceManager .record_first_token (span_id )
@@ -7094,16 +7144,21 @@ async def after_model_callback(
70947144 usage_metadata = llm_response .usage_metadata ,
70957145 cache_metadata = getattr (llm_response , "cache_metadata" , None ),
70967146 finish_reason = (
7097- finish_reason .name
7098- if (
7147+ getattr (finish_reason , "name" , str (finish_reason ))
7148+ if not is_partial
7149+ and (
70997150 finish_reason := getattr (
71007151 llm_response , "finish_reason" , None
71017152 )
71027153 )
71037154 is not None
71047155 else None
71057156 ),
7106- error_message = getattr (llm_response , "error_message" , None ),
7157+ error_message = (
7158+ None
7159+ if is_partial
7160+ else getattr (llm_response , "error_message" , None )
7161+ ),
71077162 span_id_override = span_id if is_popped else None ,
71087163 parent_span_id_override = (parent_span_id if is_popped else None ),
71097164 ),
0 commit comments