Skip to content

Commit d62419a

Browse files
jsondaicopybara-github
jsondai
authored and
copybara-github
committed
fix: GenAI Client(evals) - Handle agent traces that end without a text event in run_inference
PiperOrigin-RevId: 1013358423
1 parent 84df3a7 commit d62419a

3 files changed

Lines changed: 180 additions & 6 deletions

File tree

‎agentplatform/_genai/_evals_common.py‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3238,6 +3238,31 @@ def _process_multi_turn_agent_response(
32383238
).model_dump(exclude_unset=True)
32393239

32403240

3241+
def _extract_final_response_text(
3242+
events: list[dict[str, Any]],
3243+
) -> tuple[Optional[int], str]:
3244+
"""Returns the index and text of the last final response in the events.
3245+
3246+
As in ADK's `Event.is_final_response()`, events with function calls or
3247+
responses are not final responses, so they stay in the intermediate events.
3248+
Thought parts are not part of the response text.
3249+
"""
3250+
for index in range(len(events) - 1, -1, -1):
3251+
parts = (events[index].get("content") or {}).get("parts") or []
3252+
if any(
3253+
part.get("function_call") or part.get("function_response") for part in parts
3254+
):
3255+
continue
3256+
texts = [
3257+
part["text"]
3258+
for part in parts
3259+
if part.get("text") and not part.get("thought")
3260+
]
3261+
if texts:
3262+
return index, "".join(texts)
3263+
return None, ""
3264+
3265+
32413266
def _process_single_turn_agent_response(
32423267
resp_item: Any,
32433268
agent_data_agents: Optional[dict[str, Any]],
@@ -3253,8 +3278,10 @@ def _process_single_turn_agent_response(
32533278

32543279
if isinstance(resp_item, list):
32553280
try:
3256-
response_row = resp_item[-1]["content"]["parts"][0]["text"]
3257-
for intermediate_event in resp_item[:-1]:
3281+
response_index, response_row = _extract_final_response_text(resp_item)
3282+
for index, intermediate_event in enumerate(resp_item):
3283+
if index == response_index:
3284+
continue
32583285
intermediate_events_row.append(
32593286
{
32603287
"event_id": intermediate_event.get("id"),
@@ -3263,7 +3290,7 @@ def _process_single_turn_agent_response(
32633290
"author": intermediate_event.get("author"),
32643291
}
32653292
)
3266-
# Construct AgentData natively for single-turn runs
3293+
# Construct AgentData natively for single-turn runs
32673294
agent_events = []
32683295
for event_dict in resp_item:
32693296
content_dict = event_dict.get("content")
@@ -3288,6 +3315,7 @@ def _process_single_turn_agent_response(
32883315
agents=agent_data_agents,
32893316
).model_dump(exclude_unset=True)
32903317
except Exception as e: # pylint: disable=broad-exception-caught
3318+
intermediate_events_row = []
32913319
error_payload = {
32923320
"error": (
32933321
f"Failed to parse agent run response {str(resp_item)} to "

‎tests/unit/agentplatform/genai/test_evals.py‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5347,6 +5347,124 @@ def test_run_agent_internal_malformed_event(self, mock_run_agent):
53475347
assert "Failed to parse agent run response" in response_content
53485348
assert not result_df["intermediate_events"][0]
53495349

5350+
def _policy_agent_events(self, *events):
5351+
return [
5352+
{
5353+
"id": "1",
5354+
"author": "orchestrator",
5355+
"content": {
5356+
"role": "model",
5357+
"parts": [
5358+
{"text": "Transferring you to the policy agent."},
5359+
{
5360+
"function_call": {
5361+
"name": "transfer_to_agent",
5362+
"args": {"agent_name": "policy_agent"},
5363+
}
5364+
},
5365+
],
5366+
},
5367+
},
5368+
{
5369+
"id": "2",
5370+
"author": "orchestrator",
5371+
"content": {
5372+
"role": "user",
5373+
"parts": [
5374+
{
5375+
"function_response": {
5376+
"name": "transfer_to_agent",
5377+
"response": {},
5378+
}
5379+
}
5380+
],
5381+
},
5382+
},
5383+
{
5384+
"id": "3",
5385+
"author": "policy_agent",
5386+
"content": {
5387+
"role": "model",
5388+
"parts": [
5389+
{"text": "I should look up the policy.", "thought": True},
5390+
{"function_call": {"name": "lookup_policy", "args": {}}},
5391+
],
5392+
},
5393+
},
5394+
{
5395+
"id": "4",
5396+
"author": "policy_agent",
5397+
"content": {
5398+
"role": "user",
5399+
"parts": [
5400+
{
5401+
"function_response": {
5402+
"name": "lookup_policy",
5403+
"response": {"covered": True},
5404+
}
5405+
}
5406+
],
5407+
},
5408+
},
5409+
*events,
5410+
]
5411+
5412+
@mock.patch.object(_evals_common, "_run_agent")
5413+
def test_run_agent_internal_multi_agent_trace_without_final_text(
5414+
self, mock_run_agent
5415+
):
5416+
mock_run_agent.return_value = [self._policy_agent_events()]
5417+
result_df = _evals_common._run_agent_internal(
5418+
api_client=mock.Mock(),
5419+
runtime=mock.Mock(),
5420+
agent=None,
5421+
prompt_dataset=pd.DataFrame({"prompt": ["prompt1"]}),
5422+
)
5423+
5424+
assert result_df["response"][0] == ""
5425+
assert [e["event_id"] for e in result_df["intermediate_events"][0]] == [
5426+
"1",
5427+
"2",
5428+
"3",
5429+
"4",
5430+
]
5431+
assert len(result_df["agent_data"][0]["turns"][0]["events"]) == 4
5432+
5433+
@mock.patch.object(_evals_common, "_run_agent")
5434+
def test_run_agent_internal_multi_agent_trace_with_final_text(self, mock_run_agent):
5435+
mock_run_agent.return_value = [
5436+
self._policy_agent_events(
5437+
{
5438+
"id": "5",
5439+
"author": "policy_agent",
5440+
"content": {
5441+
"role": "model",
5442+
"parts": [
5443+
{"text": "The policy covers it.", "thought": True},
5444+
{"text": "Your policy covers this."},
5445+
],
5446+
},
5447+
},
5448+
{"id": "6", "author": None, "content": {"parts": []}},
5449+
)
5450+
]
5451+
result_df = _evals_common._run_agent_internal(
5452+
api_client=mock.Mock(),
5453+
runtime=mock.Mock(),
5454+
agent=None,
5455+
prompt_dataset=pd.DataFrame({"prompt": ["prompt1"]}),
5456+
)
5457+
5458+
assert result_df["response"][0] == "Your policy covers this."
5459+
assert [e["event_id"] for e in result_df["intermediate_events"][0]] == [
5460+
"1",
5461+
"2",
5462+
"3",
5463+
"4",
5464+
"6",
5465+
]
5466+
assert len(result_df["agent_data"][0]["turns"][0]["events"]) == 6
5467+
53505468

53515469
class TestIsMultiTurnAgentSimulation:
53525470
"""Unit tests for the _is_multi_turn_agent_simulation function."""

‎vertexai/_genai/_evals_common.py‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2037,6 +2037,31 @@ def _process_multi_turn_agent_response(
20372037
).model_dump(exclude_unset=True)
20382038

20392039

2040+
def _extract_final_response_text(
2041+
events: list[dict[str, Any]],
2042+
) -> tuple[Optional[int], str]:
2043+
"""Returns the index and text of the last final response in the events.
2044+
2045+
As in ADK's `Event.is_final_response()`, events with function calls or
2046+
responses are not final responses, so they stay in the intermediate events.
2047+
Thought parts are not part of the response text.
2048+
"""
2049+
for index in range(len(events) - 1, -1, -1):
2050+
parts = (events[index].get("content") or {}).get("parts") or []
2051+
if any(
2052+
part.get("function_call") or part.get("function_response") for part in parts
2053+
):
2054+
continue
2055+
texts = [
2056+
part["text"]
2057+
for part in parts
2058+
if part.get("text") and not part.get("thought")
2059+
]
2060+
if texts:
2061+
return index, "".join(texts)
2062+
return None, ""
2063+
2064+
20402065
def _process_single_turn_agent_response(
20412066
resp_item: Any,
20422067
agent_data_agents: Optional[dict[str, Any]],
@@ -2052,8 +2077,10 @@ def _process_single_turn_agent_response(
20522077

20532078
if isinstance(resp_item, list):
20542079
try:
2055-
response_row = resp_item[-1]["content"]["parts"][0]["text"]
2056-
for intermediate_event in resp_item[:-1]:
2080+
response_index, response_row = _extract_final_response_text(resp_item)
2081+
for index, intermediate_event in enumerate(resp_item):
2082+
if index == response_index:
2083+
continue
20572084
intermediate_events_row.append(
20582085
{
20592086
"event_id": intermediate_event.get("id"),
@@ -2062,7 +2089,7 @@ def _process_single_turn_agent_response(
20622089
"author": intermediate_event.get("author"),
20632090
}
20642091
)
2065-
# Construct AgentData natively for single-turn runs
2092+
# Construct AgentData natively for single-turn runs
20662093
agent_events = []
20672094
for event_dict in resp_item:
20682095
content_dict = event_dict.get("content")
@@ -2087,6 +2114,7 @@ def _process_single_turn_agent_response(
20872114
agents=agent_data_agents,
20882115
).model_dump(exclude_unset=True)
20892116
except Exception as e: # pylint: disable=broad-exception-caught
2117+
intermediate_events_row = []
20902118
error_payload = {
20912119
"error": (
20922120
f"Failed to parse agent run response {str(resp_item)} to "

0 commit comments

Comments
 (0)