Skip to content

Commit 46f4e55

Browse files
vertex-sdk-botcopybara-github
authored andcommitted
fix: Resolve agentplatform metric objects in vertexai evals
PiperOrigin-RevId: 1008431830
1 parent dd39926 commit 46f4e55

3 files changed

Lines changed: 167 additions & 0 deletions

File tree

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
# -*- coding: utf-8 -*-
2+
3+
# Copyright 2026 Google LLC
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
"""Tests that `vertexai.types` metrics resolve in the `vertexai` evals client.
18+
19+
`vertexai.types` resolves to the agentplatform classes, so these objects reach
20+
`vertexai._genai._evals_common` as agentplatform instances.
21+
"""
22+
23+
from unittest import mock
24+
25+
import vertexai
26+
from vertexai._genai import _evals_common
27+
from vertexai._genai import _evals_metric_loaders
28+
from vertexai._genai import types as legacy_types
29+
from google.genai import types as genai_types
30+
import pytest
31+
32+
33+
@pytest.fixture(autouse=True)
34+
def gcs_utils():
35+
with mock.patch.object(_evals_metric_loaders._gcs_utils, "GcsUtils") as gcs:
36+
yield gcs
37+
38+
39+
def test_rubric_metric_resolves_to_predefined_spec_in_evaluation_run(gcs_utils):
40+
resolved = _evals_common._resolve_evaluation_run_metrics(
41+
[vertexai.types.RubricMetric.TOOL_USE_QUALITY(version="v2")], mock.Mock()
42+
)
43+
44+
assert len(resolved) == 1
45+
assert isinstance(resolved[0], legacy_types.EvaluationRunMetric)
46+
assert resolved[0].metric == "tool_use_quality_v2"
47+
assert (
48+
resolved[0].metric_config.predefined_metric_spec.metric_spec_name
49+
== "tool_use_quality_v2"
50+
)
51+
gcs_utils.assert_not_called()
52+
53+
54+
def test_rubric_metric_keeps_requested_version(gcs_utils):
55+
resolved = _evals_common._resolve_evaluation_run_metrics(
56+
[vertexai.types.RubricMetric.TOOL_USE_QUALITY(version="v1")], mock.Mock()
57+
)
58+
59+
assert resolved[0].metric == "tool_use_quality_v1"
60+
gcs_utils.assert_not_called()
61+
62+
63+
def test_rubric_metric_resolves_in_evaluate(gcs_utils):
64+
resolved = _evals_common._resolve_metrics(
65+
[vertexai.types.RubricMetric.TOOL_USE_QUALITY(version="v2")], mock.Mock()
66+
)
67+
68+
assert len(resolved) == 1
69+
assert isinstance(resolved[0], legacy_types.Metric)
70+
assert resolved[0].name == "tool_use_quality_v2"
71+
gcs_utils.assert_not_called()
72+
73+
74+
def test_llm_metric_resolves_in_evaluation_run():
75+
resolved = _evals_common._resolve_evaluation_run_metrics(
76+
[vertexai.types.LLMMetric(name="my_metric", prompt_template="Rate {response}")],
77+
mock.Mock(),
78+
)
79+
80+
assert resolved[0].metric == "my_metric"
81+
assert resolved[0].metric_config is not None
82+
83+
84+
def test_llm_metric_converts_to_legacy_class_in_evaluate():
85+
metric = vertexai.types.LLMMetric(
86+
name="my_metric", prompt_template="Rate {response}"
87+
)
88+
89+
resolved = _evals_common._resolve_metrics([metric], mock.Mock())
90+
91+
assert type(resolved[0]) is legacy_types.LLMMetric
92+
assert resolved[0].name == "my_metric"
93+
assert resolved[0].prompt_template == "Rate {response}"
94+
95+
96+
def test_evaluation_run_metric_passes_through():
97+
metric = vertexai.types.EvaluationRunMetric(
98+
metric="my_metric",
99+
metric_config=vertexai.types.UnifiedMetric(
100+
predefined_metric_spec=genai_types.PredefinedMetricSpec(
101+
metric_spec_name="my_metric"
102+
)
103+
),
104+
)
105+
106+
resolved = _evals_common._resolve_evaluation_run_metrics([metric], mock.Mock())
107+
108+
assert type(resolved[0]) is legacy_types.EvaluationRunMetric
109+
assert resolved[0].metric == "my_metric"
110+
111+
112+
@pytest.mark.parametrize(
113+
"resolve",
114+
[
115+
_evals_common._resolve_evaluation_run_metrics,
116+
_evals_common._resolve_metrics,
117+
],
118+
)
119+
def test_unsupported_metric_type_raises_type_error(resolve, gcs_utils):
120+
with pytest.raises(TypeError, match="Unsupported metric type"):
121+
resolve([object()], mock.Mock())
122+
gcs_utils.assert_not_called()

‎vertexai/_genai/_evals_common.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,12 @@
3030

3131
from google.api_core import exceptions as api_exceptions
3232
import vertexai
33+
from agentplatform._genai import (
34+
_evals_metric_loaders as _agentplatform_metric_loaders,
35+
)
36+
from agentplatform._genai import (
37+
types as _agentplatform_types,
38+
)
3339
from google.genai import errors as genai_errors
3440
from google.genai import types as genai_types
3541
from google.genai._api_client import BaseApiClient
@@ -1692,6 +1698,36 @@ def _resolve_dataset_inputs(
16921698
return processed_eval_dataset, num_response_candidates
16931699

16941700

1701+
def _to_vertexai_metric(metric: Any) -> Any:
1702+
"""Converts an agentplatform metric object to its vertexai._genai class.
1703+
1704+
`vertexai.types` resolves to the agentplatform classes, which fail
1705+
isinstance checks against the vertexai._genai classes used in this module.
1706+
1707+
Args:
1708+
metric: A metric object, metric name, or any other value.
1709+
1710+
Returns:
1711+
The vertexai._genai equivalent of an agentplatform metric, otherwise
1712+
`metric` unchanged.
1713+
"""
1714+
if isinstance(metric, _agentplatform_metric_loaders.LazyLoadedPrebuiltMetric):
1715+
return _evals_metric_loaders.LazyLoadedPrebuiltMetric(
1716+
name=metric.name, version=metric.version, **metric.metric_kwargs
1717+
)
1718+
if isinstance(
1719+
metric,
1720+
(_agentplatform_types.Metric, _agentplatform_types.EvaluationRunMetric),
1721+
):
1722+
converted = getattr(types, type(metric).__name__).model_validate(
1723+
metric, from_attributes=True
1724+
)
1725+
if metric.__pydantic_private__:
1726+
converted.__pydantic_private__.update(metric.__pydantic_private__)
1727+
return converted
1728+
return metric
1729+
1730+
16951731
def _resolve_evaluation_run_metrics(
16961732
metrics: Union[list[types.EvaluationRunMetric], list[types.Metric]], api_client: Any
16971733
) -> list[types.EvaluationRunMetric]:
@@ -1700,6 +1736,7 @@ def _resolve_evaluation_run_metrics(
17001736
return []
17011737
resolved_metrics_list = []
17021738
for metric_instance in metrics:
1739+
metric_instance = _to_vertexai_metric(metric_instance)
17031740
if isinstance(metric_instance, types.EvaluationRunMetric):
17041741
resolved_metrics_list.append(metric_instance)
17051742
elif isinstance(
@@ -1736,6 +1773,8 @@ def _resolve_evaluation_run_metrics(
17361773
metric_resource_name=res_name,
17371774
)
17381775
)
1776+
elif not isinstance(metric_instance, str):
1777+
raise TypeError(f"Unsupported metric type: {type(metric_instance)}")
17391778
else:
17401779
try:
17411780
metric_name_str = str(metric_instance)
@@ -1777,6 +1816,7 @@ def _resolve_metrics(
17771816
"""Resolves a list of metric instances, loading RubricMetric if necessary."""
17781817
resolved_metrics_list = []
17791818
for metric_instance in metrics:
1819+
metric_instance = _to_vertexai_metric(metric_instance)
17801820
if isinstance(metric_instance, _evals_metric_loaders.LazyLoadedPrebuiltMetric):
17811821
try:
17821822
resolved_metrics_list.append(
@@ -1792,6 +1832,8 @@ def _resolve_metrics(
17921832
raise
17931833
elif isinstance(metric_instance, types.Metric):
17941834
resolved_metrics_list.append(metric_instance)
1835+
elif not isinstance(metric_instance, str):
1836+
raise TypeError(f"Unsupported metric type: {type(metric_instance)}")
17951837
else:
17961838
try:
17971839
metric_name_str = str(metric_instance)

‎vertexai/_genai/mypy.ini‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,8 @@ disable_error_code = import-not-found, import-untyped, arg-type
66
[mypy-vertexai.*]
77
ignore_errors = True
88

9+
[mypy-agentplatform.*]
10+
ignore_errors = True
11+
912
[mypy-vertexai._genai.*]
1013
ignore_errors = False

0 commit comments

Comments
 (0)