Skip to content

Commit 317e9bb

Browse files
committed
Keep the declared field types when the JSON dump falls back
The fallback dumped raw attribute values by their own class, so a subclass in a field declared as its base sent its subclass-only fields. Escape the values JSON mode can't encode in a copy of the model and let Pydantic dump the copy instead.
1 parent 0a73949 commit 317e9bb

2 files changed

Lines changed: 57 additions & 63 deletions

File tree

‎python/packages/core/agent_framework/_tools.py‎

Lines changed: 32 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@
4848
from opentelemetry import trace
4949
from opentelemetry.metrics import Histogram, NoOpHistogram
5050
from opentelemetry.util.types import AttributeValue
51-
from pydantic import BaseModel, Field, PlainSerializer, TypeAdapter, ValidationError, WrapSerializer, create_model
51+
from pydantic import BaseModel, Field, TypeAdapter, ValidationError, create_model
5252
from pydantic_core import to_jsonable_python
5353

5454
from ._serialization import SerializationMixin
@@ -506,32 +506,31 @@ def _annotation_includes_function_invocation_context(annotation: Any) -> bool:
506506
)
507507

508508

509-
def _has_field_serializer(model_type: type[BaseModel], name: str) -> bool:
510-
"""Whether a model field has its own serializer, from a decorator or an annotation."""
511-
decorators = model_type.__pydantic_decorators__.field_serializers.values()
512-
if any(name in decorator.info.fields or "*" in decorator.info.fields for decorator in decorators):
513-
return True
514-
field = model_type.model_fields.get(name)
515-
return field is not None and any(
516-
isinstance(metadata, (PlainSerializer, WrapSerializer)) for metadata in field.metadata
517-
)
518-
509+
def _escape_unencodable(value: Any) -> Any:
510+
"""Copy a value with the parts JSON mode can't encode replaced, inside models, dicts and sequences too.
519511
520-
def _escape_invalid_utf8(value: Any) -> Any:
521-
"""Replace bytes that aren't valid UTF-8 with their str() form, inside models, dicts and sequences too."""
512+
Bytes that aren't valid UTF-8 become their str() form. On Pydantic versions without ``fallback``,
513+
values JSON mode doesn't know become str() too. Models are copied, not rebuilt, so their class stays
514+
the same and the parent's declared field types still decide how they are serialized.
515+
"""
522516
if isinstance(value, bytes):
523517
try:
524518
value.decode("utf-8")
525519
except UnicodeDecodeError:
526520
return str(value).encode()
527521
return value
528522
if isinstance(value, BaseModel):
529-
return value.model_copy(update={name: _escape_invalid_utf8(item) for name, item in value.__dict__.items()})
523+
return value.model_copy(update={name: _escape_unencodable(item) for name, item in value.__dict__.items()})
530524
if isinstance(value, dict):
531525
dict_value = cast(dict[object, object], value)
532-
return {key: _escape_invalid_utf8(item) for key, item in dict_value.items()}
526+
return {key: _escape_unencodable(item) for key, item in dict_value.items()}
533527
if type(value) in (list, tuple, set, frozenset):
534-
return type(value)(_escape_invalid_utf8(item) for item in value)
528+
return type(value)(_escape_unencodable(item) for item in value)
529+
if not _MODEL_DUMP_JSON_FALLBACK:
530+
try:
531+
to_jsonable_python(value)
532+
except ValueError:
533+
return str(value)
535534
return value
536535

537536

@@ -1414,71 +1413,41 @@ def _make_dumpable(value: Any) -> Any:
14141413

14151414
@staticmethod
14161415
def _dump_model(model: BaseModel) -> Any:
1417-
"""Dump a Pydantic model in JSON mode, falling back to python mode only for the values that need it.
1416+
"""Dump a Pydantic model in JSON mode.
14181417
14191418
JSON mode applies enum, datetime and UUID encoding and JSON-only field serializers. When it
14201419
fails (bytes that aren't valid UTF-8, or an arbitrary type on Pydantic versions without
1421-
``fallback``), each field is dumped in JSON mode on its own and only a failing field falls
1422-
back. Models inside a failing field, directly or in a list or dict, are dumped the same way,
1423-
so their JSON serializers and excluded fields still apply. A failing field with its own
1424-
serializer is dumped in JSON mode again with its invalid UTF-8 bytes escaped, so the
1425-
serializer still runs, and is left out if that fails too. Excluded fields stay out either
1426-
way, unlike the str(result) fallback in invoke.
1420+
``fallback``), those values are escaped in a copy of the model and Pydantic dumps the copy, so
1421+
the declared field types, serializers and exclusions still decide what is written. Computed
1422+
fields are dumped from the original model. A field that still can't be encoded is left out.
1423+
Excluded fields stay out either way, unlike the str(result) fallback in invoke.
14271424
"""
14281425
try:
14291426
return model.model_dump(mode="json", **_MODEL_DUMP_JSON_FALLBACK)
14301427
except ValueError:
14311428
pass
1429+
escaped = _escape_unencodable(model)
14321430
# The python-mode dump gives the serialized field names without the excluded ones. The class's
14331431
# model_computed_fields can't be used for that, it's a property on Pydantic 2.0.
14341432
python_dump = model.model_dump(by_alias=False)
14351433
if not isinstance(python_dump, dict):
1436-
return model.model_dump()
1437-
model_type = type(model)
1434+
try:
1435+
return escaped.model_dump(mode="json", **_MODEL_DUMP_JSON_FALLBACK)
1436+
except ValueError:
1437+
logger.warning("Leaving %s out of the tool result: it can't be encoded as JSON.", type(model).__name__)
1438+
return None
1439+
fields = type(model).model_fields
14381440
dumped: dict[str, Any] = {}
14391441
for name in python_dump:
1442+
source = escaped if name in fields else model
14401443
try:
1441-
dumped.update(model.model_dump(mode="json", include={name}, **_MODEL_DUMP_JSON_FALLBACK))
1442-
continue
1444+
dumped.update(source.model_dump(mode="json", include={name}, **_MODEL_DUMP_JSON_FALLBACK))
14431445
except ValueError:
1444-
pass
1445-
value = getattr(model, name, None)
1446-
if _has_field_serializer(model_type, name):
1447-
# The python-mode value would skip the field's serializer, and with it any redaction it does.
1448-
escaped = model.model_copy(update={name: _escape_invalid_utf8(value)})
1449-
try:
1450-
dumped.update(escaped.model_dump(mode="json", include={name}, **_MODEL_DUMP_JSON_FALLBACK))
1451-
except ValueError:
1452-
logger.warning(
1453-
"Leaving %s.%s out of the tool result: its serializer's output can't be encoded as JSON.",
1454-
model_type.__name__,
1455-
name,
1456-
)
1457-
continue
1458-
for key, python_value in model.model_dump(include={name}).items():
1459-
if isinstance(value, (BaseModel, dict, list, tuple, set, frozenset)):
1460-
dumped[key] = FunctionTool._dump_fallback_value(value)
1461-
else:
1462-
dumped[key] = python_value
1446+
logger.warning(
1447+
"Leaving %s.%s out of the tool result: it can't be encoded as JSON.", type(model).__name__, name
1448+
)
14631449
return dumped
14641450

1465-
@staticmethod
1466-
def _dump_fallback_value(value: Any) -> Any:
1467-
"""Dump a field value JSON mode could not encode, keeping JSON mode for what inside it can be."""
1468-
if isinstance(value, BaseModel):
1469-
return FunctionTool._dump_model(value)
1470-
if isinstance(value, dict):
1471-
dict_value = cast(dict[object, object], value)
1472-
return {key: FunctionTool._dump_fallback_value(item) for key, item in dict_value.items()}
1473-
if isinstance(value, (list, tuple, set, frozenset)):
1474-
items = cast(Iterable[object], value)
1475-
return [FunctionTool._dump_fallback_value(item) for item in items]
1476-
try:
1477-
return to_jsonable_python(value)
1478-
except ValueError:
1479-
# parse_result's json.dumps(default=str) turns it into a string
1480-
return value
1481-
14821451
@staticmethod
14831452
def parse_result(result: Any) -> list[Content]:
14841453
"""Convert a raw function return value to a list of Content items.

‎python/packages/core/tests/core/test_types.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4698,6 +4698,31 @@ def _redact(self, value: str) -> bytes:
46984698
assert json.loads(parsed[0].text) == {"created": "2024-01-01T12:00:00"} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type]
46994699

47004700

4701+
def test_parse_result_pydantic_fallback_keeps_declared_field_types():
4702+
"""A subclass in a field declared as its base keeps the subclass-only fields out when the fallback is used."""
4703+
4704+
class Public(BaseModel):
4705+
data: bytes
4706+
4707+
class Secret(Public):
4708+
api_key: str
4709+
4710+
class Envelope(BaseModel):
4711+
item: Public
4712+
items: list[Public]
4713+
by_name: dict[str, Public]
4714+
4715+
secret = Secret(data=b"\xff\xfe", api_key="key")
4716+
parsed = FunctionTool.parse_result(Envelope(item=secret, items=[secret], by_name={"a": secret}))
4717+
4718+
expected_item = {"data": str(b"\xff\xfe")}
4719+
assert json.loads(parsed[0].text) == { # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type]
4720+
"item": expected_item,
4721+
"items": [expected_item],
4722+
"by_name": {"a": expected_item},
4723+
}
4724+
4725+
47014726
# region FunctionTool.parse_result with MCP TextContent-like objects
47024727

47034728

0 commit comments

Comments
 (0)