Skip to content

Commit 39c8cfa

Browse files
ftnextharanrk
authored and
copybara-github
committed
feat(cli): Support adk eval --num_runs N as AgentEvaluator
Merge #4411 **Problem:** `adk eval` did not support repeated runs, so users had to use external loops to reduce nondeterminism. Also, when adding multi-run support, reporting needs to stay consistent between summary and detailed output. **Solution:** Added `--num_runs` to `adk eval` (default: `1`, min: `1`) and wired it into inference execution so each eval set can be run multiple times in a single CLI invocation. For reporting: - Summary is aggregated per eval case (`eval_set_id` + `eval_id`) across runs. - Detailed output (`--print_detailed_results`) keeps per-run details (each run's `EvalCaseResult`) to avoid mismatches between aggregated scores and invocation-level debugging info. Also added unit tests for: - Aggregation behavior across runs. - CLI validation for invalid `--num_runs` values. - Multi-run summary aggregation and detailed per-run printing behavior. Related: #4410 Co-authored-by: Haran Rajkumar <[email protected]> PiperOrigin-RevId: 959841339
1 parent a5864a0 commit 39c8cfa

4 files changed

Lines changed: 355 additions & 2 deletions

File tree

‎src/google/adk/cli/cli_eval.py‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import importlib.util
1818
import logging
1919
import os
20+
import statistics
2021
import sys
2122
from types import ModuleType
2223
from typing import Any
@@ -36,7 +37,10 @@
3637
from ..evaluation.constants import MISSING_EVAL_DEPENDENCIES_MESSAGE
3738
from ..evaluation.eval_case import get_all_tool_calls
3839
from ..evaluation.eval_case import IntermediateDataType
40+
from ..evaluation.eval_metrics import _get_metric_threshold
3941
from ..evaluation.eval_metrics import EvalMetric
42+
from ..evaluation.eval_metrics import EvalMetricResult
43+
from ..evaluation.eval_metrics import EvalStatus
4044
from ..evaluation.eval_metrics import RubricsBasedCriterion
4145
from ..evaluation.eval_result import EvalCaseResult
4246
from ..evaluation.eval_sets_manager import EvalSetsManager
@@ -164,6 +168,96 @@ def parse_and_get_evals_to_run(
164168
return eval_set_to_evals
165169

166170

171+
def _generate_final_eval_status(
172+
overall_eval_metric_results: list[EvalMetricResult],
173+
) -> EvalStatus:
174+
"""Returns final eval status for a case from overall metric results."""
175+
final_eval_status = EvalStatus.NOT_EVALUATED
176+
for overall_eval_metric_result in overall_eval_metric_results:
177+
overall_eval_status = overall_eval_metric_result.eval_status
178+
if overall_eval_status == EvalStatus.PASSED:
179+
final_eval_status = EvalStatus.PASSED
180+
elif overall_eval_status == EvalStatus.NOT_EVALUATED:
181+
continue
182+
elif overall_eval_status == EvalStatus.FAILED:
183+
final_eval_status = EvalStatus.FAILED
184+
break
185+
else:
186+
raise ValueError(f"Unknown eval status: {overall_eval_status}.")
187+
return final_eval_status
188+
189+
190+
def _aggregate_metric_results(
191+
metric_results: list[EvalMetricResult],
192+
) -> EvalMetricResult:
193+
"""Aggregates results of the same metric across runs."""
194+
if not metric_results:
195+
raise ValueError("`metric_results` should not be empty.")
196+
197+
aggregate_metric_result = metric_results[0].model_copy(deep=True)
198+
scores = [m.score for m in metric_results if m.score is not None]
199+
if scores:
200+
threshold = _get_metric_threshold(aggregate_metric_result)
201+
aggregate_metric_result.score = statistics.mean(scores)
202+
aggregate_metric_result.eval_status = (
203+
EvalStatus.PASSED
204+
if aggregate_metric_result.score >= threshold
205+
else EvalStatus.FAILED
206+
)
207+
else:
208+
aggregate_metric_result.score = None
209+
aggregate_metric_result.eval_status = EvalStatus.NOT_EVALUATED
210+
211+
return aggregate_metric_result
212+
213+
214+
def aggregate_eval_case_results(
215+
eval_results: list[EvalCaseResult],
216+
) -> list[EvalCaseResult]:
217+
"""Aggregates EvalCaseResults with the same eval_set_id and eval_id."""
218+
eval_results_by_case_id: dict[tuple[str, str], list[EvalCaseResult]] = {}
219+
for eval_result in eval_results:
220+
key = (eval_result.eval_set_id, eval_result.eval_id)
221+
if key not in eval_results_by_case_id:
222+
eval_results_by_case_id[key] = []
223+
eval_results_by_case_id[key].append(eval_result)
224+
225+
aggregate_results: list[EvalCaseResult] = []
226+
for _, per_case_results in eval_results_by_case_id.items():
227+
aggregate_result = per_case_results[0].model_copy(deep=True)
228+
metric_results_by_name: dict[str, list[EvalMetricResult]] = {}
229+
for per_case_result in per_case_results:
230+
for metric_result in per_case_result.overall_eval_metric_results:
231+
metric_name = metric_result.metric_name
232+
if metric_name not in metric_results_by_name:
233+
metric_results_by_name[metric_name] = []
234+
metric_results_by_name[metric_name].append(metric_result)
235+
236+
metric_names_in_order = [
237+
metric_result.metric_name
238+
for metric_result in aggregate_result.overall_eval_metric_results
239+
]
240+
missing_metric_names = sorted(
241+
set(metric_results_by_name.keys()) - set(metric_names_in_order)
242+
)
243+
metric_names_in_order.extend(missing_metric_names)
244+
245+
aggregate_overall_eval_metric_results: list[EvalMetricResult] = []
246+
for metric_name in metric_names_in_order:
247+
aggregate_overall_eval_metric_results.append(
248+
_aggregate_metric_results(metric_results_by_name[metric_name])
249+
)
250+
aggregate_result.overall_eval_metric_results = (
251+
aggregate_overall_eval_metric_results
252+
)
253+
aggregate_result.final_eval_status = _generate_final_eval_status(
254+
aggregate_overall_eval_metric_results
255+
)
256+
aggregate_results.append(aggregate_result)
257+
258+
return sorted(aggregate_results, key=lambda x: (x.eval_set_id, x.eval_id))
259+
260+
167261
async def _collect_inferences(
168262
inference_requests: list[InferenceRequest],
169263
eval_service: BaseEvalService,

‎src/google/adk/cli/cli_tools_click.py‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1110,6 +1110,13 @@ def wrapper(*args, **kwargs):
11101110
)
11111111
@click.argument("eval_set_file_path_or_id", nargs=-1)
11121112
@click.option("--config_file_path", help="Optional. The path to config file.")
1113+
@click.option(
1114+
"--num_runs",
1115+
type=click.IntRange(min=1),
1116+
default=1,
1117+
show_default=True,
1118+
help="Optional. Number of times each eval set should be run.",
1119+
)
11131120
@click.option(
11141121
"--print_detailed_results",
11151122
is_flag=True,
@@ -1125,6 +1132,7 @@ def cli_eval(
11251132
print_detailed_results: bool,
11261133
eval_storage_uri: str | None = None,
11271134
log_level: str = "INFO",
1135+
num_runs: int = 1,
11281136
):
11291137
"""Evaluates an agent given the eval sets.
11301138
@@ -1178,6 +1186,8 @@ def cli_eval(
11781186
11791187
CONFIG_FILE_PATH: The path to config file.
11801188
1189+
NUM_RUNS: Number of times each eval set should be run.
1190+
11811191
PRINT_DETAILED_RESULTS: Prints detailed results on the console.
11821192
"""
11831193
envs.load_dotenv_for_agent(agent_module_file_path, ".")
@@ -1200,6 +1210,7 @@ def cli_eval(
12001210
from ..evaluation.simulation.user_simulator_provider import UserSimulatorProvider
12011211
from .cli_eval import _collect_eval_results
12021212
from .cli_eval import _collect_inferences
1213+
from .cli_eval import aggregate_eval_case_results
12031214
from .cli_eval import get_app_or_root_agent
12041215
from .cli_eval import parse_and_get_evals_to_run
12051216
from .cli_eval import pretty_print_eval_result
@@ -1315,9 +1326,14 @@ def cli_eval(
13151326
app=app,
13161327
)
13171328

1329+
repeated_inference_requests = [
1330+
inference_request.model_copy(deep=True)
1331+
for inference_request in inference_requests
1332+
] * num_runs
13181333
inference_results = asyncio.run(
13191334
_collect_inferences(
1320-
inference_requests=inference_requests, eval_service=eval_service
1335+
inference_requests=repeated_inference_requests,
1336+
eval_service=eval_service,
13211337
)
13221338
)
13231339
eval_results = asyncio.run(
@@ -1327,6 +1343,7 @@ def cli_eval(
13271343
eval_metrics=eval_metrics,
13281344
)
13291345
)
1346+
aggregate_eval_results = aggregate_eval_case_results(eval_results)
13301347
except ModuleNotFoundError as mnf:
13311348
raise click.ClickException(_missing_eval_dependencies_message()) from mnf
13321349

@@ -1335,7 +1352,7 @@ def cli_eval(
13351352
)
13361353
eval_run_summary = {}
13371354

1338-
for eval_result in eval_results:
1355+
for eval_result in aggregate_eval_results:
13391356
if eval_result.eval_set_id not in eval_run_summary:
13401357
eval_run_summary[eval_result.eval_set_id] = [0, 0]
13411358

‎tests/unittests/cli/utils/test_cli_eval.py‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,11 @@
2121

2222
from google.adk.agents.base_agent import BaseAgent
2323
from google.adk.apps.app import App
24+
from google.adk.cli.cli_eval import aggregate_eval_case_results
2425
from google.adk.cli.cli_eval import get_root_agent
26+
from google.adk.evaluation.eval_metrics import EvalMetricResult
27+
from google.adk.evaluation.eval_metrics import EvalStatus
28+
from google.adk.evaluation.eval_result import EvalCaseResult
2529
import pytest
2630

2731

@@ -261,3 +265,50 @@ async def test_get_root_agent_back_compat(monkeypatch):
261265
)
262266

263267
assert await get_root_agent("some/path") is root_agent
268+
269+
270+
def test_aggregate_eval_case_results_across_runs():
271+
eval_results = [
272+
EvalCaseResult(
273+
eval_set_id="set1",
274+
eval_id="case1",
275+
final_eval_status=EvalStatus.PASSED,
276+
overall_eval_metric_results=[
277+
EvalMetricResult(
278+
metric_name="response_match_score",
279+
threshold=0.8,
280+
score=1.0,
281+
eval_status=EvalStatus.PASSED,
282+
)
283+
],
284+
eval_metric_result_per_invocation=[],
285+
session_id="s1",
286+
),
287+
EvalCaseResult(
288+
eval_set_id="set1",
289+
eval_id="case1",
290+
final_eval_status=EvalStatus.FAILED,
291+
overall_eval_metric_results=[
292+
EvalMetricResult(
293+
metric_name="response_match_score",
294+
threshold=0.8,
295+
score=0.6,
296+
eval_status=EvalStatus.FAILED,
297+
)
298+
],
299+
eval_metric_result_per_invocation=[],
300+
session_id="s2",
301+
),
302+
]
303+
304+
aggregate_results = aggregate_eval_case_results(eval_results)
305+
306+
assert len(aggregate_results) == 1
307+
assert aggregate_results[0].eval_set_id == "set1"
308+
assert aggregate_results[0].eval_id == "case1"
309+
assert aggregate_results[0].overall_eval_metric_results[0].score == 0.8
310+
assert (
311+
aggregate_results[0].overall_eval_metric_results[0].eval_status
312+
== EvalStatus.PASSED
313+
)
314+
assert aggregate_results[0].final_eval_status == EvalStatus.PASSED

0 commit comments

Comments
 (0)