Skip to content

Commit c0a8de2

Browse files
vertex-sdk-botcopybara-github
authored andcommitted
feat: Add extra_context to EvaluationItemRequest
PiperOrigin-RevId: 1002414538
1 parent a0705a3 commit c0a8de2

4 files changed

Lines changed: 212 additions & 0 deletions

File tree

‎agentplatform/_genai/_evals_common.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4120,6 +4120,36 @@ def _get_content(row: dict[str, Any], column: str) -> Optional[genai_types.Conte
41204120
)
41214121

41224122

4123+
# Columns mapped to EvaluationItemRequest fields; all others go to extra_context.
4124+
_EVALUATION_SET_STANDARD_COLUMNS = _evals_constant.COMMON_DATASET_COLUMNS | {
4125+
_evals_constant.RESPONSE,
4126+
"interaction",
4127+
"gemini_agent",
4128+
}
4129+
4130+
4131+
def _to_extra_context_content(value: Any) -> Optional[genai_types.Content]:
4132+
if isinstance(value, str):
4133+
return genai_types.Content(parts=[genai_types.Part(text=value)])
4134+
if isinstance(value, genai_types.Content):
4135+
return value
4136+
parts = []
4137+
for content in _evals_metric_handlers._value_to_content_list(value):
4138+
parts.extend(content.parts or [])
4139+
return genai_types.Content(parts=parts) if parts else None
4140+
4141+
4142+
def _get_extra_context(row: "pd.Series") -> Optional[dict[str, genai_types.Content]]:
4143+
extra_context = {}
4144+
for column, value in row.items():
4145+
if column in _EVALUATION_SET_STANDARD_COLUMNS or _is_missing(value):
4146+
continue
4147+
content = _to_extra_context_content(value)
4148+
if content:
4149+
extra_context[str(column)] = content
4150+
return extra_context or None
4151+
4152+
41234153
def _create_evaluation_set_from_dataframe(
41244154
api_client: BaseApiClient,
41254155
gcs_dest_prefix: str,
@@ -4244,6 +4274,7 @@ def _create_evaluation_set_from_dataframe(
42444274
candidate_responses=(
42454275
candidate_responses if candidate_responses else None
42464276
),
4277+
extra_context=_get_extra_context(row),
42474278
)
42484279
)
42494280
logger.info("Writing evaluation item requests to GCS.")

‎agentplatform/_genai/types/common.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1164,6 +1164,10 @@ class EvaluationItemRequest(_common.BaseModel):
11641164
default=None,
11651165
description="""Responses from model under test and other baseline models for comparison.""",
11661166
)
1167+
extra_context: Optional[dict[str, genai_types.Content]] = Field(
1168+
default=None,
1169+
description="""Additional named inputs for metrics, such as `golden_schema` alongside an agent trace. Each entry is passed to metrics under its key: LLM-based metrics reference it as `{key}` in the metric prompt template. Standard fields (prompt, response, reference) take precedence over a matching key; `extra_context` takes precedence over `prompt.prompt_template_data.values`.""",
1170+
)
11671171

11681172

11691173
class EvaluationItemRequestDict(TypedDict, total=False):
@@ -1181,6 +1185,9 @@ class EvaluationItemRequestDict(TypedDict, total=False):
11811185
candidate_responses: Optional[list[CandidateResponseDict]]
11821186
"""Responses from model under test and other baseline models for comparison."""
11831187

1188+
extra_context: Optional[dict[str, genai_types.Content]]
1189+
"""Additional named inputs for metrics, such as `golden_schema` alongside an agent trace. Each entry is passed to metrics under its key: LLM-based metrics reference it as `{key}` in the metric prompt template. Standard fields (prompt, response, reference) take precedence over a matching key; `extra_context` takes precedence over `prompt.prompt_template_data.values`."""
1190+
11841191

11851192
EvaluationItemRequestOrDict = Union[EvaluationItemRequest, EvaluationItemRequestDict]
11861193

‎tests/unit/agentplatform/genai/replays/test_create_evaluation_run.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
from tests.unit.agentplatform.genai.replays import pytest_helper
1818
from agentplatform import types
1919
from agentplatform._genai import _evals_common
20+
from agentplatform._genai import _gcs_utils
2021
from google.genai import types as genai_types
2122
import pandas as pd
2223
import pytest
@@ -353,6 +354,65 @@ def test_create_eval_run_with_metric_resource_name(
353354
assert evaluation_run.evaluation_config.metrics[0].metric == "my_custom_metric"
354355

355356

357+
# Recorded against staging in ucaip-e2e-test, whose bucket the staging service
358+
# agent can read; staging runs the extra_context backend changes.
359+
@mock.patch.object(
360+
_evals_common, "_local_timestamp", return_value="1/1/2026, 12:00:00 AM"
361+
)
362+
@mock.patch("uuid.uuid4")
363+
def test_create_eval_run_with_extra_context(
364+
mock_uuid4, unused_mock_local_timestamp, client
365+
):
366+
"""Tests that custom dataframe columns are sent as extra_context."""
367+
mock_uuid4.return_value = uuid.UUID("5f0e8a2c-6a1b-4c3d-9e7f-1a2b3c4d5e6f")
368+
client._api_client._http_options.api_version = "v1beta1"
369+
client._api_client._http_options.base_url = (
370+
"https://us-central1-staging-aiplatform.sandbox.googleapis.com/"
371+
)
372+
schema_metric = types.EvaluationRunMetric(
373+
metric="schema_check",
374+
metric_config=types.UnifiedMetric(
375+
llm_based_metric_spec=genai_types.LLMBasedMetricSpec(
376+
metric_prompt_template=(
377+
"Schema: {golden_schema}\nResponse: {response}\nScore 1 if the"
378+
" response only uses tables defined in the schema, else 0."
379+
' Reply with a JSON object with keys "score" and'
380+
' "explanation".'
381+
)
382+
)
383+
),
384+
)
385+
with mock.patch.object(
386+
_gcs_utils.GcsUtils,
387+
"upload_json_to_prefix",
388+
autospec=True,
389+
side_effect=_gcs_utils.GcsUtils.upload_json_to_prefix,
390+
) as mock_upload:
391+
evaluation_run = client.evals.create_evaluation_run(
392+
dataset=types.EvaluationDataset(
393+
eval_dataset_df=pd.DataFrame(
394+
{
395+
"prompt": ["List the ids of all users."],
396+
"response": ["SELECT id FROM users"],
397+
"golden_schema": ["CREATE TABLE users (id INT, name STRING)"],
398+
}
399+
)
400+
),
401+
metrics=[schema_metric],
402+
dest="gs://eval-task-e2e-test/sdk_extra_context",
403+
)
404+
uploaded_request = mock_upload.call_args.kwargs["data"]
405+
assert uploaded_request["extraContext"] == {
406+
"golden_schema": {
407+
"parts": [{"text": "CREATE TABLE users (id INT, name STRING)"}]
408+
}
409+
}
410+
assert isinstance(evaluation_run, types.EvaluationRun)
411+
assert evaluation_run.state == types.EvaluationRunState.PENDING
412+
assert evaluation_run.error is None
413+
assert evaluation_run.evaluation_config.metrics[0].metric == "schema_check"
414+
415+
356416
# Dataframe tests fail in replay mode because of UUID generation mismatch.
357417
# def test_create_eval_run_data_source_evaluation_dataset(client):
358418
# """Tests that create_evaluation_run() creates a correctly structured

‎tests/unit/agentplatform/genai/test_evals.py‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10537,6 +10537,58 @@ def test_convert_request_to_dataset_row_with_agent_data(self):
1053710537
assert result["intermediate_events"] == []
1053810538

1053910539

10540+
class TestEvaluationItemRequestExtraContext:
10541+
"""Unit tests for EvaluationItemRequest.extra_context."""
10542+
10543+
def test_extra_context_serializes_as_camel_case(self):
10544+
request = agentplatform_genai_types.EvaluationItemRequest(
10545+
prompt=agentplatform_genai_types.EvaluationPrompt(text="test prompt"),
10546+
extra_context={
10547+
"golden_schema": genai_types.Content(
10548+
parts=[genai_types.Part(text="CREATE TABLE t (id INT)")]
10549+
)
10550+
},
10551+
)
10552+
assert request.model_dump(mode="json", by_alias=True, exclude_none=True) == {
10553+
"prompt": {"text": "test prompt"},
10554+
"extraContext": {
10555+
"golden_schema": {"parts": [{"text": "CREATE TABLE t (id INT)"}]}
10556+
},
10557+
}
10558+
10559+
def test_extra_context_from_dict(self):
10560+
request = agentplatform_genai_types.EvaluationItemRequest.model_validate(
10561+
{"extra_context": {"golden_rows": {"parts": [{"text": "[1, 2]"}]}}}
10562+
)
10563+
assert request.extra_context == {
10564+
"golden_rows": genai_types.Content(parts=[genai_types.Part(text="[1, 2]")])
10565+
}
10566+
10567+
@mock.patch.object(_evals_common, "_gcs_utils")
10568+
def test_convert_gcs_to_evaluation_item_request_with_extra_context(
10569+
self, mock_gcs_utils
10570+
):
10571+
"""Tests that a camelCase request JSON from GCS keeps extra_context."""
10572+
mock_gcs_utils.GcsUtils.return_value.read_file_contents.return_value = (
10573+
json.dumps(
10574+
{
10575+
"prompt": {"text": "test prompt"},
10576+
"extraContext": {
10577+
"golden_schema": {"parts": [{"text": "CREATE TABLE t"}]}
10578+
},
10579+
}
10580+
)
10581+
)
10582+
request = _evals_common._convert_gcs_to_evaluation_item_request(
10583+
mock.Mock(), "gs://bucket/request.json"
10584+
)
10585+
assert request.extra_context == {
10586+
"golden_schema": genai_types.Content(
10587+
parts=[genai_types.Part(text="CREATE TABLE t")]
10588+
)
10589+
}
10590+
10591+
1054010592
class TestCreateEvaluationSetFromDataFrame:
1054110593
"""Unit tests for the _create_evaluation_set_from_dataframe function."""
1054210594

@@ -10545,6 +10597,68 @@ def setup_method(self):
1054510597
self.mock_api_client.project = "test-project"
1054610598
self.mock_api_client.location = "us-central1"
1054710599

10600+
def _uploaded_requests(self, eval_df):
10601+
"""Returns the request dicts uploaded to GCS for `eval_df`."""
10602+
with (
10603+
mock.patch.object(_evals_common, "evals"),
10604+
mock.patch.object(_evals_common, "_gcs_utils") as mock_gcs_utils,
10605+
):
10606+
_evals_common._create_evaluation_set_from_dataframe(
10607+
api_client=self.mock_api_client,
10608+
gcs_dest_prefix="gs://bucket/prefix",
10609+
eval_df=eval_df,
10610+
)
10611+
upload = mock_gcs_utils.GcsUtils.return_value.upload_json_to_prefix
10612+
return [c.kwargs["data"] for c in upload.call_args_list]
10613+
10614+
def test_custom_columns_go_to_extra_context(self):
10615+
"""Tests that only non-standard columns go to extra_context."""
10616+
eval_df = pd.DataFrame(
10617+
[
10618+
{
10619+
"prompt": "test prompt",
10620+
"response": "test response",
10621+
"reference": "test reference",
10622+
"context": "test context",
10623+
"golden_schema": "CREATE TABLE t (id INT)",
10624+
"golden_rows": [1, 2],
10625+
}
10626+
]
10627+
)
10628+
(request,) = self._uploaded_requests(eval_df)
10629+
assert request["extraContext"] == {
10630+
"golden_schema": {"parts": [{"text": "CREATE TABLE t (id INT)"}]},
10631+
"golden_rows": {"parts": [{"text": "[1, 2]"}]},
10632+
}
10633+
10634+
def test_custom_columns_kept_for_agent_data_only_row(self):
10635+
eval_df = pd.DataFrame(
10636+
[{"agent_data": {"turns": []}, "golden_schema": "CREATE TABLE t"}]
10637+
)
10638+
(request,) = self._uploaded_requests(eval_df)
10639+
assert "prompt" not in request
10640+
assert request["extraContext"] == {
10641+
"golden_schema": {"parts": [{"text": "CREATE TABLE t"}]}
10642+
}
10643+
10644+
def test_missing_custom_values_are_skipped(self):
10645+
eval_df = pd.DataFrame(
10646+
[
10647+
{"prompt": "p1", "golden_schema": "CREATE TABLE t"},
10648+
{"prompt": "p2", "golden_schema": float("nan")},
10649+
]
10650+
)
10651+
first, second = self._uploaded_requests(eval_df)
10652+
assert first["extraContext"] == {
10653+
"golden_schema": {"parts": [{"text": "CREATE TABLE t"}]}
10654+
}
10655+
assert "extraContext" not in second
10656+
10657+
def test_no_extra_context_without_custom_columns(self):
10658+
eval_df = pd.DataFrame([{"prompt": "test prompt", "response": "test response"}])
10659+
(request,) = self._uploaded_requests(eval_df)
10660+
assert "extraContext" not in request
10661+
1054810662
@mock.patch.object(_evals_common, "evals")
1054910663
@mock.patch.object(_evals_common, "_gcs_utils")
1055010664
def test_create_evaluation_set_with_intermediate_events(

0 commit comments

Comments
 (0)