Skip to content

Commit 9e5c721

Browse files
jsondaicopybara-github
authored andcommitted
fix: GenAI Client(evals) - Serialize DataFrame-backed evaluation datasets to JSON
PiperOrigin-RevId: 1000484727
1 parent 0edda68 commit 9e5c721

3 files changed

Lines changed: 189 additions & 0 deletions

File tree

‎agentplatform/_genai/types/common.py‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
)
3636
from google.genai import _common
3737
from google.genai import types as genai_types
38+
import pydantic
3839
from pydantic import (
3940
ConfigDict,
4041
Field,
@@ -3130,6 +3131,51 @@ def _check_pandas_installed(cls, data: Any) -> Any:
31303131
)
31313132
return data
31323133

3134+
@field_validator("eval_dataset_df", mode="before")
3135+
@classmethod
3136+
def _eval_dataset_df_from_records(cls, value: Any) -> Any:
3137+
if (
3138+
pd is not None
3139+
and isinstance(value, list)
3140+
and all(isinstance(row, dict) for row in value)
3141+
):
3142+
return pd.DataFrame(value)
3143+
return value
3144+
3145+
@pydantic.field_serializer("eval_dataset_df", when_used="json")
3146+
def _eval_dataset_df_to_records(self, value: Any) -> Any:
3147+
if pd is None or not isinstance(value, pd.DataFrame):
3148+
return value
3149+
labels = pd.Index([str(label) for label in value.columns])
3150+
duplicated = value.columns.duplicated() | labels.duplicated()
3151+
if duplicated.any():
3152+
duplicates = value.columns[duplicated].unique().tolist()
3153+
raise ValueError(
3154+
"eval_dataset_df needs unique column names to serialize to JSON,"
3155+
f" found duplicates: {duplicates}"
3156+
)
3157+
import numpy as np
3158+
3159+
def to_serializable(item: Any) -> Any:
3160+
if isinstance(item, np.ndarray) and item.ndim == 0:
3161+
return None if item is np.ma.masked else to_serializable(item[()])
3162+
if isinstance(item, dict):
3163+
return {
3164+
(
3165+
to_serializable(key) if isinstance(key, np.generic) else key
3166+
): to_serializable(val)
3167+
for key, val in item.items()
3168+
}
3169+
if isinstance(item, (list, tuple, np.ndarray, pd.Series)):
3170+
return [to_serializable(val) for val in item]
3171+
if isinstance(item, np.datetime64):
3172+
item = pd.Timestamp(item)
3173+
elif isinstance(item, np.generic):
3174+
return item.item()
3175+
return None if item is pd.NaT or item is pd.NA else item
3176+
3177+
return to_serializable(value.to_dict(orient="records"))
3178+
31333179
@classmethod
31343180
def load_from_observability_eval_cases(
31353181
cls, cases: list["ObservabilityEvalCase"]

‎tests/unit/agentplatform/genai/test_evals.py‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,11 @@
4545
types as agentplatform_genai_types,
4646
)
4747
from agentplatform._genai.types import common as common_types
48+
from vertexai._genai import types as vertexai_genai_types
4849
from google.genai import client
4950
from google.genai import errors as genai_errors
5051
from google.genai import types as genai_types
52+
import numpy as np
5153
import pandas as pd
5254
import pydantic
5355
import pytest
@@ -10067,6 +10069,101 @@ def read_file_contents_side_effect(src: str) -> str:
1006710069
),
1006810070
)
1006910071

10072+
@pytest.mark.parametrize(
10073+
"types_module",
10074+
[agentplatform_genai_types, vertexai_genai_types],
10075+
ids=["agent_platform", "vertexai"],
10076+
)
10077+
def test_model_dump_json_writes_dataframe_rows(self, types_module):
10078+
df = pd.DataFrame(
10079+
{
10080+
"tool_names": [
10081+
np.array(["search", "book"]),
10082+
np.array([], dtype=object),
10083+
],
10084+
"created_at": pd.to_datetime(["2026-10-02T10:00:00Z", None]),
10085+
"metadata": [
10086+
{
10087+
np.int64(1): np.int64(3),
10088+
"score": np.array(0.5),
10089+
"masked": np.ma.masked,
10090+
},
10091+
pd.NA,
10092+
],
10093+
"score": [0.1 + 0.2, float("nan")],
10094+
"response": [
10095+
genai_types.Content(parts=[genai_types.Part(text="r1")]),
10096+
None,
10097+
],
10098+
}
10099+
)
10100+
10101+
dataset_json = types_module.EvaluationDataset(
10102+
eval_dataset_df=df
10103+
).model_dump_json(exclude_none=True)
10104+
10105+
assert json.loads(dataset_json)["eval_dataset_df"] == [
10106+
{
10107+
"tool_names": ["search", "book"],
10108+
"created_at": "2026-10-02T10:00:00Z",
10109+
"metadata": {"1": 3, "score": 0.5, "masked": None},
10110+
"score": 0.30000000000000004,
10111+
"response": {"parts": [{"text": "r1"}]},
10112+
},
10113+
{
10114+
"tool_names": [],
10115+
"created_at": None,
10116+
"metadata": None,
10117+
"score": None,
10118+
"response": None,
10119+
},
10120+
]
10121+
10122+
@pytest.mark.parametrize(
10123+
"types_module",
10124+
[agentplatform_genai_types, vertexai_genai_types],
10125+
ids=["agent_platform", "vertexai"],
10126+
)
10127+
def test_json_round_trip_restores_dataframe(self, types_module):
10128+
df = pd.DataFrame([{"prompt": "p1", "response": "r1"}])
10129+
result = types_module.EvaluationResult(
10130+
evaluation_dataset=[types_module.EvaluationDataset(eval_dataset_df=df)]
10131+
)
10132+
10133+
restored = types_module.EvaluationResult.model_validate_json(
10134+
result.model_dump_json()
10135+
)
10136+
10137+
pd.testing.assert_frame_equal(
10138+
restored.evaluation_dataset[0].eval_dataset_df, df
10139+
)
10140+
assert isinstance(
10141+
result.model_dump()["evaluation_dataset"][0]["eval_dataset_df"],
10142+
pd.DataFrame,
10143+
)
10144+
10145+
@pytest.mark.parametrize(
10146+
"types_module",
10147+
[agentplatform_genai_types, vertexai_genai_types],
10148+
ids=["agent_platform", "vertexai"],
10149+
)
10150+
@pytest.mark.parametrize(
10151+
"columns, duplicates",
10152+
[
10153+
(["prompt", "prompt"], r"\['prompt'\]"),
10154+
([True, 1], r"\[1\]"),
10155+
([1, "1"], r"\['1'\]"),
10156+
],
10157+
ids=["same", "equal_in_python", "equal_as_strings"],
10158+
)
10159+
def test_model_dump_json_rejects_duplicate_column_names(
10160+
self, types_module, columns, duplicates
10161+
):
10162+
df = pd.DataFrame([["v1", "v2"]], columns=pd.Index(columns, dtype=object))
10163+
10164+
with pytest.raises(ValueError, match=f"found duplicates: {duplicates}"):
10165+
types_module.EvaluationDataset(eval_dataset_df=df).model_dump_json()
10166+
1007010167

1007110168
class TestEvalsGenerateConversationScenarios:
1007210169
"""Unit tests for the Evals generate_conversation_scenarios method."""

‎vertexai/_genai/types/common.py‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
)
3636
from google.genai import _common
3737
from google.genai import types as genai_types
38+
import pydantic
3839
from pydantic import (
3940
ConfigDict,
4041
Field,
@@ -3463,6 +3464,51 @@ def _check_pandas_installed(cls, data: Any) -> Any:
34633464
)
34643465
return data
34653466

3467+
@field_validator("eval_dataset_df", mode="before")
3468+
@classmethod
3469+
def _eval_dataset_df_from_records(cls, value: Any) -> Any:
3470+
if (
3471+
pd is not None
3472+
and isinstance(value, list)
3473+
and all(isinstance(row, dict) for row in value)
3474+
):
3475+
return pd.DataFrame(value)
3476+
return value
3477+
3478+
@pydantic.field_serializer("eval_dataset_df", when_used="json")
3479+
def _eval_dataset_df_to_records(self, value: Any) -> Any:
3480+
if pd is None or not isinstance(value, pd.DataFrame):
3481+
return value
3482+
labels = pd.Index([str(label) for label in value.columns])
3483+
duplicated = value.columns.duplicated() | labels.duplicated()
3484+
if duplicated.any():
3485+
duplicates = value.columns[duplicated].unique().tolist()
3486+
raise ValueError(
3487+
"eval_dataset_df needs unique column names to serialize to JSON,"
3488+
f" found duplicates: {duplicates}"
3489+
)
3490+
import numpy as np
3491+
3492+
def to_serializable(item: Any) -> Any:
3493+
if isinstance(item, np.ndarray) and item.ndim == 0:
3494+
return None if item is np.ma.masked else to_serializable(item[()])
3495+
if isinstance(item, dict):
3496+
return {
3497+
(
3498+
to_serializable(key) if isinstance(key, np.generic) else key
3499+
): to_serializable(val)
3500+
for key, val in item.items()
3501+
}
3502+
if isinstance(item, (list, tuple, np.ndarray, pd.Series)):
3503+
return [to_serializable(val) for val in item]
3504+
if isinstance(item, np.datetime64):
3505+
item = pd.Timestamp(item)
3506+
elif isinstance(item, np.generic):
3507+
return item.item()
3508+
return None if item is pd.NaT or item is pd.NA else item
3509+
3510+
return to_serializable(value.to_dict(orient="records"))
3511+
34663512
@classmethod
34673513
def load_from_observability_eval_cases(
34683514
cls, cases: list["ObservabilityEvalCase"]

0 commit comments

Comments
 (0)