Skip to content

Commit 0477e57

Browse files
ftnextharanrk
authored and
copybara-github
committed
feat(cli): auto-discover test_config.json for single eval file in adk eval
Merge #4412 **Problem:** `adk eval` behavior was inconsistent with the expected config discovery flow. When `--config_file_path` was omitted, CLI always fell back to default criteria, instead of using `test_config.json` located next to an eval file. This differs from `AgentEvaluator.evaluate`, which already discovers a `test_config.json` sitting next to each test file. **Solution:** Added config path resolution in `adk eval`: - If `--config_file_path` is provided, use it as-is. - If omitted and input is a single eval file, look for `<eval_file_dir>/test_config.json`. - If omitted and input is multiple eval files or eval set IDs, do not auto-discover and keep default criteria behavior. If no adjacent `test_config.json` is found, behavior is unchanged and the built-in default evaluation criteria are used. This keeps behavior explicit for mixed-directory multi-file runs while enabling convenient per-file config discovery for single-file usage. Auto-discovery is intentionally limited to single-file input to avoid ambiguous behavior when multiple eval files are provided from different directories. Towards #4410 Co-authored-by: Haran Rajkumar <[email protected]> PiperOrigin-RevId: 961863867
1 parent 4f58306 commit 0477e57

2 files changed

Lines changed: 101 additions & 14 deletions

File tree

‎src/google/adk/cli/cli_tools_click.py‎

Lines changed: 44 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1100,6 +1100,33 @@ def wrapper(*args, **kwargs):
11001100
return decorator
11011101

11021102

1103+
def _resolve_eval_config_file_path(
1104+
config_file_path: Optional[str],
1105+
eval_set_file_or_id_to_evals: dict[str, list[str]],
1106+
) -> Optional[str]:
1107+
"""Returns config file path for eval command.
1108+
1109+
If `config_file_path` is provided, it is used as-is. If omitted and evals are
1110+
loaded from a single file, this returns
1111+
`<eval_set_file_dir>/test_config.json`. Otherwise, returns None.
1112+
"""
1113+
if config_file_path:
1114+
return config_file_path
1115+
1116+
if not eval_set_file_or_id_to_evals:
1117+
return None
1118+
1119+
if len(eval_set_file_or_id_to_evals) != 1:
1120+
return None
1121+
1122+
first_eval_set = next(iter(eval_set_file_or_id_to_evals))
1123+
if os.path.exists(first_eval_set):
1124+
eval_set_dir = os.path.dirname(first_eval_set)
1125+
return os.path.join(eval_set_dir, "test_config.json")
1126+
1127+
return None
1128+
1129+
11031130
@main.command("eval", cls=HelpfulCommand)
11041131
@feature_options()
11051132
@click.argument(
@@ -1206,20 +1233,6 @@ def cli_eval(
12061233
except ModuleNotFoundError as mnf:
12071234
raise click.ClickException(_missing_eval_dependencies_message()) from mnf
12081235

1209-
eval_config = get_evaluation_criteria_or_default(config_file_path)
1210-
print(f"Using evaluation criteria: {eval_config}")
1211-
eval_metrics = get_eval_metrics_from_config(eval_config)
1212-
1213-
# Live mode is resolved from the eval config, consistent with how
1214-
# `user_simulator_config` and other eval settings are sourced.
1215-
if eval_config.live_model_config:
1216-
inference_config = InferenceConfig(
1217-
use_live=True,
1218-
live_timeout_seconds=eval_config.live_model_config.timeout_seconds,
1219-
)
1220-
else:
1221-
inference_config = InferenceConfig(use_live=False)
1222-
12231236
app, root_agent = asyncio.run(get_app_or_root_agent(agent_module_file_path))
12241237
app_name = os.path.basename(agent_module_file_path)
12251238
agents_dir = os.path.dirname(agent_module_file_path)
@@ -1241,6 +1254,23 @@ def cli_eval(
12411254
eval_set_file_or_id_to_evals = parse_and_get_evals_to_run(
12421255
eval_set_file_path_or_id
12431256
)
1257+
resolved_config_file_path = _resolve_eval_config_file_path(
1258+
config_file_path=config_file_path,
1259+
eval_set_file_or_id_to_evals=eval_set_file_or_id_to_evals,
1260+
)
1261+
eval_config = get_evaluation_criteria_or_default(resolved_config_file_path)
1262+
print(f"Using evaluation criteria: {eval_config}")
1263+
eval_metrics = get_eval_metrics_from_config(eval_config)
1264+
1265+
# Live mode is resolved from the eval config, consistent with how
1266+
# `user_simulator_config` and other eval settings are sourced.
1267+
if eval_config.live_model_config:
1268+
inference_config = InferenceConfig(
1269+
use_live=True,
1270+
live_timeout_seconds=eval_config.live_model_config.timeout_seconds,
1271+
)
1272+
else:
1273+
inference_config = InferenceConfig(use_live=False)
12441274

12451275
# Check if the first entry is a file that exists, if it does then we assume
12461276
# rest of the entries are also files. We enforce this assumption in the if

‎tests/unittests/cli/utils/test_cli_tools_click.py‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,63 @@ def test_validate_exclusive_blocks_multiple() -> None:
125125
cli_tools_click.validate_exclusive(ctx, param2, "resume.json")
126126

127127

128+
def test_resolve_eval_config_file_path_prefers_explicit_path(
129+
tmp_path: Path,
130+
) -> None:
131+
eval_set_file = tmp_path / "sample.test.json"
132+
eval_set_file.touch()
133+
explicit_config = tmp_path / "explicit_config.json"
134+
135+
resolved_path = cli_tools_click._resolve_eval_config_file_path(
136+
config_file_path=str(explicit_config),
137+
eval_set_file_or_id_to_evals={str(eval_set_file): []},
138+
)
139+
140+
assert resolved_path == str(explicit_config)
141+
142+
143+
def test_resolve_eval_config_file_path_uses_test_config_next_to_eval_file(
144+
tmp_path: Path,
145+
) -> None:
146+
eval_set_file = tmp_path / "sample.test.json"
147+
eval_set_file.touch()
148+
149+
resolved_path = cli_tools_click._resolve_eval_config_file_path(
150+
config_file_path=None,
151+
eval_set_file_or_id_to_evals={str(eval_set_file): []},
152+
)
153+
154+
assert resolved_path == str(tmp_path / "test_config.json")
155+
156+
157+
def test_resolve_eval_config_file_path_returns_none_for_eval_set_id() -> None:
158+
resolved_path = cli_tools_click._resolve_eval_config_file_path(
159+
config_file_path=None,
160+
eval_set_file_or_id_to_evals={"eval_set_id": []},
161+
)
162+
163+
assert resolved_path is None
164+
165+
166+
def test_resolve_eval_config_file_path_returns_none_for_multiple_eval_files(
167+
tmp_path: Path,
168+
) -> None:
169+
eval_set_file_1 = tmp_path / "sample_1.test.json"
170+
eval_set_file_2 = tmp_path / "sample_2.test.json"
171+
eval_set_file_1.touch()
172+
eval_set_file_2.touch()
173+
174+
resolved_path = cli_tools_click._resolve_eval_config_file_path(
175+
config_file_path=None,
176+
eval_set_file_or_id_to_evals={
177+
str(eval_set_file_1): [],
178+
str(eval_set_file_2): [],
179+
},
180+
)
181+
182+
assert resolved_path is None
183+
184+
128185
# cli create
129186
def test_cli_create_cmd_invokes_run_cmd(
130187
tmp_path: Path, monkeypatch: pytest.MonkeyPatch

0 commit comments

Comments
 (0)