Skip to content

Commit 64baec5

Browse files
committed
feat(plugins): Expose BigQuery LLM termination details
Project finish_reason into LLM response attributes and typed views, and route model response diagnostics through the existing sanitized error_message column without changing status semantics. The finish_reason projection follows the approach proposed by @roanny in google#5704. Refs google#5644
1 parent 4aa1a81 commit 64baec5

2 files changed

Lines changed: 112 additions & 0 deletions

File tree

‎src/google/adk/plugins/bigquery_agent_analytics_plugin.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3661,6 +3661,7 @@ def _parse_custom_metadata_allowlist(
36613661
"JSON_VALUE(attributes, '$.model_version') AS model_version",
36623662
"JSON_QUERY(attributes, '$.usage_metadata') AS usage_metadata",
36633663
"JSON_QUERY(attributes, '$.cache_metadata') AS cache_metadata",
3664+
"JSON_VALUE(attributes, '$.finish_reason') AS finish_reason",
36643665
],
36653666
"LLM_ERROR": [
36663667
"CAST(JSON_VALUE(latency_ms, '$.total_ms') AS INT64) AS total_ms",
@@ -3830,6 +3831,7 @@ class EventData:
38303831
model_version: Optional[str] = None
38313832
usage_metadata: Any = None
38323833
cache_metadata: Any = None
3834+
finish_reason: Optional[str] = None
38333835
status: str = "OK"
38343836
error_message: Optional[str] = None
38353837
extra_attributes: dict[str, Any] = field(default_factory=dict)
@@ -5794,6 +5796,9 @@ def _enrich_attributes(
57945796
else:
57955797
attrs["cache_metadata"] = event_data.cache_metadata
57965798

5799+
if event_data.finish_reason is not None:
5800+
attrs["finish_reason"] = event_data.finish_reason
5801+
57975802
if self.config.log_session_metadata:
57985803
try:
57995804
session = callback_context._invocation_context.session
@@ -6812,6 +6817,12 @@ async def after_model_callback(
68126817
model_version=llm_response.model_version,
68136818
usage_metadata=llm_response.usage_metadata,
68146819
cache_metadata=getattr(llm_response, "cache_metadata", None),
6820+
finish_reason=(
6821+
llm_response.finish_reason.name
6822+
if llm_response.finish_reason is not None
6823+
else None
6824+
),
6825+
error_message=llm_response.error_message,
68156826
span_id_override=span_id if is_popped else None,
68166827
parent_span_id_override=(parent_span_id if is_popped else None),
68176828
),

‎tests/unittests/plugins/test_bigquery_agent_analytics_plugin.py‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1852,6 +1852,99 @@ async def test_after_model_callback_text_response(
18521852
# In this test we didn't pass it in kwargs in the updated call above, so it might be missing unless we add it back to kwargs.
18531853
# The original test passed it as kwarg.
18541854

1855+
@pytest.mark.asyncio
1856+
@pytest.mark.parametrize(
1857+
"finish_reason",
1858+
[
1859+
types.FinishReason.STOP,
1860+
types.FinishReason.MAX_TOKENS,
1861+
types.FinishReason.SAFETY,
1862+
types.FinishReason.MALFORMED_FUNCTION_CALL,
1863+
],
1864+
)
1865+
async def test_after_model_callback_projects_finish_reason(
1866+
self,
1867+
finish_reason,
1868+
bq_plugin_inst,
1869+
mock_write_client,
1870+
callback_context,
1871+
dummy_arrow_schema,
1872+
):
1873+
"""LLM termination reasons are queryable in response attributes."""
1874+
response = llm_response_lib.LlmResponse(
1875+
content=types.Content(parts=[types.Part(text="response")]),
1876+
finish_reason=finish_reason,
1877+
)
1878+
bigquery_agent_analytics_plugin.TraceManager.push_span(
1879+
callback_context, "llm_request"
1880+
)
1881+
1882+
await bq_plugin_inst.after_model_callback(
1883+
callback_context=callback_context, llm_response=response
1884+
)
1885+
await bq_plugin_inst.flush()
1886+
1887+
row = await _get_captured_event_dict_async(
1888+
mock_write_client, dummy_arrow_schema
1889+
)
1890+
assert json.loads(row["attributes"])["finish_reason"] == finish_reason.name
1891+
1892+
@pytest.mark.asyncio
1893+
async def test_streaming_partial_omits_missing_finish_reason(
1894+
self,
1895+
bq_plugin_inst,
1896+
mock_write_client,
1897+
callback_context,
1898+
dummy_arrow_schema,
1899+
):
1900+
"""Streaming chunks without a termination reason omit the JSON key."""
1901+
response = llm_response_lib.LlmResponse(
1902+
content=types.Content(parts=[types.Part(text="chunk")]), partial=True
1903+
)
1904+
bigquery_agent_analytics_plugin.TraceManager.push_span(
1905+
callback_context, "llm_request"
1906+
)
1907+
1908+
await bq_plugin_inst.after_model_callback(
1909+
callback_context=callback_context, llm_response=response
1910+
)
1911+
await bq_plugin_inst.flush()
1912+
1913+
row = await _get_captured_event_dict_async(
1914+
mock_write_client, dummy_arrow_schema
1915+
)
1916+
assert "finish_reason" not in json.loads(row["attributes"])
1917+
1918+
@pytest.mark.asyncio
1919+
async def test_after_model_callback_sanitizes_error_message_without_error_status(
1920+
self,
1921+
bq_plugin_inst,
1922+
mock_write_client,
1923+
callback_context,
1924+
dummy_arrow_schema,
1925+
):
1926+
"""Response diagnostics use the safe error column without changing status."""
1927+
response = llm_response_lib.LlmResponse(
1928+
error_message="Authorization: Bearer MODEL-SECRET",
1929+
finish_reason=types.FinishReason.SAFETY,
1930+
)
1931+
bigquery_agent_analytics_plugin.TraceManager.push_span(
1932+
callback_context, "llm_request"
1933+
)
1934+
1935+
await bq_plugin_inst.after_model_callback(
1936+
callback_context=callback_context, llm_response=response
1937+
)
1938+
await bq_plugin_inst.flush()
1939+
1940+
row = await _get_captured_event_dict_async(
1941+
mock_write_client, dummy_arrow_schema
1942+
)
1943+
assert row["error_message"] == "Authorization: [REDACTED]"
1944+
assert row["status"] == "OK"
1945+
assert row["is_truncated"] is True
1946+
assert "MODEL-SECRET" not in json.dumps(row, default=str)
1947+
18551948
@pytest.mark.asyncio
18561949
async def test_after_model_callback_tool_call(
18571950
self,
@@ -6678,6 +6771,14 @@ def test_llm_response_view_exposes_token_usage_columns(self):
66786771
assert "$.usage_metadata.thoughts_token_count" in all_sql
66796772
assert "$.usage_metadata.tool_use_prompt_token_count" in all_sql
66806773

6774+
def test_llm_response_view_exposes_finish_reason(self):
6775+
"""LLM_RESPONSE views expose the termination reason as a typed column."""
6776+
columns = bigquery_agent_analytics_plugin._EVENT_VIEW_DEFS["LLM_RESPONSE"]
6777+
6778+
assert (
6779+
"JSON_VALUE(attributes, '$.finish_reason') AS finish_reason" in columns
6780+
)
6781+
66816782
def test_config_create_views_default_true(self):
66826783
"""Config create_views defaults to True."""
66836784
config = bigquery_agent_analytics_plugin.BigQueryLoggerConfig()

0 commit comments

Comments
 (0)