Skip to content

Commit 2198289

Browse files
committed
feat(plugins): project LlmResponse finish_reason to BigQuery analytics
Projects `finish_reason` from `LlmResponse` to the `attributes` JSON column of `LLM_RESPONSE` events written to BigQuery by `BigQueryAgentAnalyticsPlugin`, and exposes it as a typed column in the generated per-event SQL view. Without this, classifying model failure modes (MAX_TOKENS, SAFETY, MALFORMED_FUNCTION_CALL, RECITATION, etc.) via SQL against the analytics table is impossible — the signal lives only in unstructured Cloud Logging output that is not joinable to the structured event stream. Changes: - `EventData` gets a `finish_reason: Optional[str]` field. - `after_model_callback` reads `llm_response.finish_reason.name` via a defensive getattr (same back-compat pattern as `cache_metadata`, covering legacy response shapes). - `_enrich_attributes` writes `attrs["finish_reason"]` only when set, keeping existing consumers and partial streaming events unaffected. - `_EVENT_VIEW_DEFS["LLM_RESPONSE"]` projects `JSON_VALUE(attributes, '$.finish_reason')` as `finish_reason`. - Schema-field description for `attributes` mentions the new key with example values. Tests: - `test_after_model_callback_projects_finish_reason` — MAX_TOKENS round-trips end-to-end into the BQ row's attributes. - `test_after_model_callback_omits_finish_reason_when_absent` — key is omitted when `LlmResponse.finish_reason` is None. - `TestEnrichAttributes::test_finish_reason_included_when_set` and `..._omitted_when_none` — direct `_enrich_attributes` coverage. - Existing 229 plugin tests still pass; 5 skipped (pre-existing). Fixes google#5644
1 parent c35a579 commit 2198289

2 files changed

Lines changed: 93 additions & 2 deletions

File tree

‎src/google/adk/plugins/bigquery_agent_analytics_plugin.py‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1746,8 +1746,10 @@ def _get_events_schema() -> list[bigquery.SchemaField]:
17461746
"A JSON object containing arbitrary key-value pairs for"
17471747
" additional event metadata. Includes enrichment fields like"
17481748
" 'root_agent_name' (turn orchestration), 'model' (request"
1749-
" model), 'model_version' (response version), and"
1750-
" 'usage_metadata' (detailed token counts)."
1749+
" model), 'model_version' (response version),"
1750+
" 'usage_metadata' (detailed token counts), and"
1751+
" 'finish_reason' (LLM_RESPONSE termination reason, e.g."
1752+
" 'STOP', 'MAX_TOKENS', 'SAFETY', 'MALFORMED_FUNCTION_CALL')."
17511753
),
17521754
),
17531755
bigquery.SchemaField(
@@ -1847,6 +1849,7 @@ def _get_events_schema() -> list[bigquery.SchemaField]:
18471849
"JSON_VALUE(attributes, '$.model_version') AS model_version",
18481850
"JSON_QUERY(attributes, '$.usage_metadata') AS usage_metadata",
18491851
"JSON_QUERY(attributes, '$.cache_metadata') AS cache_metadata",
1852+
"JSON_VALUE(attributes, '$.finish_reason') AS finish_reason",
18501853
],
18511854
"LLM_ERROR": [
18521855
"CAST(JSON_VALUE(latency_ms, '$.total_ms') AS INT64) AS total_ms",
@@ -1958,6 +1961,7 @@ class EventData:
19581961
model_version: Optional[str] = None
19591962
usage_metadata: Any = None
19601963
cache_metadata: Any = None
1964+
finish_reason: Optional[str] = None
19611965
status: str = "OK"
19621966
error_message: Optional[str] = None
19631967
extra_attributes: dict[str, Any] = field(default_factory=dict)
@@ -2821,6 +2825,9 @@ def _enrich_attributes(
28212825
else:
28222826
attrs["cache_metadata"] = event_data.cache_metadata
28232827

2828+
if event_data.finish_reason is not None:
2829+
attrs["finish_reason"] = event_data.finish_reason
2830+
28242831
if self.config.log_session_metadata:
28252832
try:
28262833
session = callback_context._invocation_context.session
@@ -3417,6 +3424,11 @@ async def after_model_callback(
34173424
# Otherwise log_event will fetch current stack (which is parent).
34183425
span_id = popped_span_id or span_id
34193426

3427+
raw_finish_reason = getattr(llm_response, "finish_reason", None)
3428+
finish_reason = (
3429+
raw_finish_reason.name if raw_finish_reason is not None else None
3430+
)
3431+
34203432
await self._log_event(
34213433
"LLM_RESPONSE",
34223434
callback_context,
@@ -3428,6 +3440,7 @@ async def after_model_callback(
34283440
model_version=llm_response.model_version,
34293441
usage_metadata=llm_response.usage_metadata,
34303442
cache_metadata=getattr(llm_response, "cache_metadata", None),
3443+
finish_reason=finish_reason,
34313444
span_id_override=span_id if is_popped else None,
34323445
parent_span_id_override=(parent_span_id if is_popped else None),
34333446
),

‎tests/unittests/plugins/test_bigquery_agent_analytics_plugin.py‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1530,6 +1530,60 @@ async def test_after_model_callback_text_response(
15301530
# In this test we didn't pass it in kwargs in the updated call above, so it might be missing unless we add it back to kwargs.
15311531
# The original test passed it as kwarg.
15321532

1533+
@pytest.mark.asyncio
1534+
async def test_after_model_callback_projects_finish_reason(
1535+
self,
1536+
bq_plugin_inst,
1537+
mock_write_client,
1538+
callback_context,
1539+
dummy_arrow_schema,
1540+
):
1541+
"""finish_reason from LlmResponse is projected to attributes JSON."""
1542+
llm_response = llm_response_lib.LlmResponse(
1543+
content=types.Content(parts=[types.Part(text="Truncated")]),
1544+
usage_metadata=types.UsageMetadata(
1545+
prompt_token_count=10, total_token_count=15
1546+
),
1547+
finish_reason=types.FinishReason.MAX_TOKENS,
1548+
)
1549+
bigquery_agent_analytics_plugin.TraceManager.push_span(callback_context)
1550+
await bq_plugin_inst.after_model_callback(
1551+
callback_context=callback_context,
1552+
llm_response=llm_response,
1553+
)
1554+
await asyncio.sleep(0.01)
1555+
log_entry = await _get_captured_event_dict_async(
1556+
mock_write_client, dummy_arrow_schema
1557+
)
1558+
_assert_common_fields(log_entry, "LLM_RESPONSE")
1559+
attrs = json.loads(log_entry["attributes"])
1560+
assert attrs["finish_reason"] == "MAX_TOKENS"
1561+
1562+
@pytest.mark.asyncio
1563+
async def test_after_model_callback_omits_finish_reason_when_absent(
1564+
self,
1565+
bq_plugin_inst,
1566+
mock_write_client,
1567+
callback_context,
1568+
dummy_arrow_schema,
1569+
):
1570+
"""When LlmResponse.finish_reason is None, the key is not emitted."""
1571+
llm_response = llm_response_lib.LlmResponse(
1572+
content=types.Content(parts=[types.Part(text="ok")]),
1573+
)
1574+
bigquery_agent_analytics_plugin.TraceManager.push_span(callback_context)
1575+
await bq_plugin_inst.after_model_callback(
1576+
callback_context=callback_context,
1577+
llm_response=llm_response,
1578+
)
1579+
await asyncio.sleep(0.01)
1580+
log_entry = await _get_captured_event_dict_async(
1581+
mock_write_client, dummy_arrow_schema
1582+
)
1583+
_assert_common_fields(log_entry, "LLM_RESPONSE")
1584+
attrs = json.loads(log_entry["attributes"])
1585+
assert "finish_reason" not in attrs
1586+
15331587
@pytest.mark.asyncio
15341588
async def test_after_model_callback_tool_call(
15351589
self,
@@ -3755,6 +3809,30 @@ def test_usage_metadata_truncated(self):
37553809
"output_tokens": 50,
37563810
}
37573811

3812+
def test_finish_reason_included_when_set(self):
3813+
"""Should include finish_reason in attributes when set on EventData."""
3814+
plugin = self._make_plugin()
3815+
ed = bigquery_agent_analytics_plugin.EventData(finish_reason="MAX_TOKENS")
3816+
with mock.patch.object(
3817+
bigquery_agent_analytics_plugin.TraceManager,
3818+
"get_root_agent_name",
3819+
return_value="agent",
3820+
):
3821+
attrs = plugin._enrich_attributes(ed, self._make_callback_context())
3822+
assert attrs["finish_reason"] == "MAX_TOKENS"
3823+
3824+
def test_finish_reason_omitted_when_none(self):
3825+
"""Should not add finish_reason key when EventData.finish_reason is None."""
3826+
plugin = self._make_plugin()
3827+
ed = bigquery_agent_analytics_plugin.EventData()
3828+
with mock.patch.object(
3829+
bigquery_agent_analytics_plugin.TraceManager,
3830+
"get_root_agent_name",
3831+
return_value="agent",
3832+
):
3833+
attrs = plugin._enrich_attributes(ed, self._make_callback_context())
3834+
assert "finish_reason" not in attrs
3835+
37583836

37593837
class TestMultiSubagentToolLogging:
37603838
"""Tests that tool events from different subagents are attributed correctly.

0 commit comments

Comments
 (0)