Skip to content

Commit 4aa1a81

Browse files
committed
fix(plugins): Make BigQuery retry duplicates identifiable
Assign each analytics row a stable event_id before enqueue so Storage Write API retries preserve a queryable deduplication key. Existing tables receive the nullable column through additive schema upgrade. Refs google#6465
1 parent 0c79d1a commit 4aa1a81

2 files changed

Lines changed: 107 additions & 1 deletion

File tree

‎src/google/adk/plugins/bigquery_agent_analytics_plugin.py‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@
104104

105105
# Bumped when the schema changes (1 → 2 → 3 …). Used as a table
106106
# label for governance and to decide whether auto-upgrade should run.
107-
_SCHEMA_VERSION = "1"
107+
_SCHEMA_VERSION = "2"
108108
_SCHEMA_VERSION_LABEL_KEY = "adk_schema_version"
109109

110110
# ADK 2.0 envelope version. Stamped onto every ADK-enriched row as
@@ -3284,6 +3284,16 @@ def _get_events_schema() -> list[bigquery.SchemaField]:
32843284
" events within a session."
32853285
),
32863286
),
3287+
bigquery.SchemaField(
3288+
"event_id",
3289+
"STRING",
3290+
mode="NULLABLE",
3291+
description=(
3292+
"A unique identifier assigned before enqueue. Storage Write API"
3293+
" retries preserve this value so duplicate rows can be"
3294+
" identified reliably."
3295+
),
3296+
),
32873297
bigquery.SchemaField(
32883298
"event_type",
32893299
"STRING",
@@ -3583,6 +3593,7 @@ def _parse_custom_metadata_allowlist(
35833593
# Columns included in every per-event-type view.
35843594
_VIEW_COMMON_COLUMNS = (
35853595
"timestamp",
3596+
"event_id",
35863597
"event_type",
35873598
"agent",
35883599
"session_id",
@@ -6096,6 +6107,7 @@ async def _log_event(
60966107

60976108
row = {
60986109
"timestamp": timestamp,
6110+
"event_id": uuid.uuid4().hex,
60996111
"event_type": event_type,
61006112
"agent": self._resolve_agent_label(
61016113
callback_context, event_data.source_event

‎tests/unittests/plugins/test_bigquery_agent_analytics_plugin.py‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@ async def fake_append_rows(requests, **kwargs):
161161
def dummy_arrow_schema():
162162
return pa.schema([
163163
pa.field("timestamp", pa.timestamp("us", tz="UTC"), nullable=False),
164+
pa.field("event_id", pa.string(), nullable=True),
164165
pa.field("root_agent_name", pa.string(), nullable=True),
165166
pa.field("event_type", pa.string(), nullable=True),
166167
pa.field("agent", pa.string(), nullable=True),
@@ -5155,6 +5156,98 @@ async def test_multi_turn_multi_subagent_full_sequence(
51555156
assert row["session_id"] == "session-multi"
51565157

51575158

5159+
class TestEventId:
5160+
"""Rows carry a stable identifier for query-time retry deduplication."""
5161+
5162+
def test_schema_and_views_expose_event_id(self):
5163+
"""The physical schema and every typed view expose the row identifier."""
5164+
schema_fields = {
5165+
field.name: field
5166+
for field in bigquery_agent_analytics_plugin._get_events_schema()
5167+
}
5168+
5169+
assert schema_fields["event_id"].field_type == "STRING"
5170+
assert schema_fields["event_id"].mode == "NULLABLE"
5171+
assert "event_id" in bigquery_agent_analytics_plugin._VIEW_COMMON_COLUMNS
5172+
5173+
@pytest.mark.asyncio
5174+
async def test_each_emitted_row_has_a_distinct_hex_event_id(
5175+
self,
5176+
bq_plugin_inst,
5177+
mock_write_client,
5178+
invocation_context,
5179+
dummy_arrow_schema,
5180+
):
5181+
"""Separate plugin rows receive distinct UUID-derived identifiers."""
5182+
user_message = types.Content(parts=[types.Part(text="hello")])
5183+
5184+
await bq_plugin_inst.on_user_message_callback(
5185+
invocation_context=invocation_context,
5186+
user_message=user_message,
5187+
)
5188+
await bq_plugin_inst.on_user_message_callback(
5189+
invocation_context=invocation_context,
5190+
user_message=user_message,
5191+
)
5192+
await bq_plugin_inst.flush()
5193+
5194+
rows = await _get_captured_rows_async(mock_write_client, dummy_arrow_schema)
5195+
event_ids = [row["event_id"] for row in rows]
5196+
assert len(event_ids) == 2
5197+
assert len(set(event_ids)) == 2
5198+
for event_id in event_ids:
5199+
assert len(event_id) == 32
5200+
assert event_id == event_id.lower()
5201+
assert int(event_id, 16) >= 0
5202+
5203+
@pytest.mark.asyncio
5204+
async def test_bigquery_retry_reuses_the_same_event_id(
5205+
self,
5206+
bq_plugin_inst,
5207+
mock_write_client,
5208+
invocation_context,
5209+
dummy_arrow_schema,
5210+
):
5211+
"""A transport retry resends the original row identifier unchanged."""
5212+
state = next(iter(bq_plugin_inst._loop_state_by_loop.values()))
5213+
state.batch_processor.retry_config = (
5214+
bigquery_agent_analytics_plugin.RetryConfig(
5215+
max_retries=1,
5216+
initial_delay=0,
5217+
multiplier=1,
5218+
max_delay=0,
5219+
)
5220+
)
5221+
event_ids = []
5222+
5223+
async def append_then_lose_ack(requests, **kwargs):
5224+
del kwargs
5225+
request = [request async for request in requests][0]
5226+
batch = pa.ipc.read_record_batch(
5227+
pa.py_buffer(request.arrow_rows.rows.serialized_record_batch),
5228+
dummy_arrow_schema,
5229+
)
5230+
event_ids.append(batch.to_pylist()[0]["event_id"])
5231+
if len(event_ids) == 1:
5232+
raise bigquery_agent_analytics_plugin.ServiceUnavailable("ack lost")
5233+
response = mock.MagicMock()
5234+
response.error.code = 0
5235+
response.row_errors = []
5236+
return _async_gen(response)
5237+
5238+
mock_write_client.append_rows.side_effect = append_then_lose_ack
5239+
5240+
await bq_plugin_inst.on_user_message_callback(
5241+
invocation_context=invocation_context,
5242+
user_message=types.Content(parts=[types.Part(text="hello")]),
5243+
)
5244+
await bq_plugin_inst.flush()
5245+
5246+
assert len(event_ids) == 2
5247+
assert event_ids[0] is not None
5248+
assert event_ids[0] == event_ids[1]
5249+
5250+
51585251
class TestSchemaAutoUpgrade:
51595252
"""Tests for _ensure_schema_exists with auto_schema_upgrade."""
51605253

@@ -5212,6 +5305,7 @@ def test_upgrade_adds_missing_columns(self):
52125305
updated_table = plugin.client.update_table.call_args[0][0]
52135306
updated_names = {f.name for f in updated_table.schema}
52145307
assert "event_type" in updated_names
5308+
assert "event_id" in updated_names
52155309
assert "agent" in updated_names
52165310
assert "content" in updated_names
52175311
assert (

0 commit comments

Comments
 (0)