Skip to content

Commit 93c52a6

Browse files
jsondaicopybara-github
authored andcommitted
fix: GenAI Client(evals) - Default missing rubric verdicts to False
PiperOrigin-RevId: 1000529827
1 parent 1d93955 commit 93c52a6

3 files changed

Lines changed: 46 additions & 2 deletions

File tree

‎agentplatform/_genai/types/evals.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -948,7 +948,7 @@ class RubricVerdict(_common.BaseModel):
948948
generated.""",
949949
)
950950
verdict: Optional[bool] = Field(
951-
default=None,
951+
default=False,
952952
description="""Required. Outcome of the evaluation against the rubric, represented as a
953953
boolean. `true` indicates a "Pass", `false` indicates a "Fail".""",
954954
)

‎tests/unit/agentplatform/genai/test_evals.py‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
from vertexai._genai import (
4949
_evals_data_converters as vertexai_evals_data_converters,
5050
)
51+
from vertexai._genai import evals as vertexai_evals
5152
from vertexai._genai import types as vertexai_genai_types
5253
from google.genai import client
5354
from google.genai import errors as genai_errors
@@ -2361,6 +2362,49 @@ def test_eval_evaluate_with_agent_info(self, mock_execute_evaluation):
23612362
assert kwargs["agent_info"] == agent_info
23622363

23632364

2365+
class TestRubricVerdict:
2366+
"""Unit tests for parsing RubricVerdict from API responses."""
2367+
2368+
@pytest.mark.parametrize(
2369+
"evals_module", [evals, vertexai_evals], ids=["agent_platform", "vertexai"]
2370+
)
2371+
def test_evaluate_instances_missing_verdict_is_false(self, evals_module):
2372+
api_client = mock.MagicMock()
2373+
api_client.vertexai = True
2374+
api_client.request.return_value.body = json.dumps(
2375+
{
2376+
"metricResults": [
2377+
{
2378+
"score": 0.5,
2379+
"rubricVerdicts": [
2380+
{
2381+
"evaluatedRubric": {
2382+
"content": {
2383+
"property": {"description": "In English."}
2384+
}
2385+
},
2386+
"verdict": True,
2387+
},
2388+
{
2389+
"evaluatedRubric": {
2390+
"content": {
2391+
"property": {"description": "One sentence."}
2392+
}
2393+
},
2394+
"reasoning": "The response has two sentences.",
2395+
},
2396+
],
2397+
}
2398+
]
2399+
}
2400+
)
2401+
2402+
response = evals_module.Evals(api_client_=api_client)._evaluate_instances()
2403+
2404+
verdicts = response.metric_results[0].rubric_verdicts
2405+
assert [verdict.verdict for verdict in verdicts] == [True, False]
2406+
2407+
23642408
class TestEvalsVisualization:
23652409
# fmt: off
23662410
@mock.patch(

‎vertexai/_genai/types/evals.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -948,7 +948,7 @@ class RubricVerdict(_common.BaseModel):
948948
generated.""",
949949
)
950950
verdict: Optional[bool] = Field(
951-
default=None,
951+
default=False,
952952
description="""Required. Outcome of the evaluation against the rubric, represented as a
953953
boolean. `true` indicates a "Pass", `false` indicates a "Fail".""",
954954
)

0 commit comments

Comments
 (0)