Skip to content

Commit 027b5c7

Browse files
authored
Rebuild incomplete types during serialization inference (#13922)
1 parent 2706d1f commit 027b5c7

8 files changed

Lines changed: 159 additions & 43 deletions

File tree

‎pydantic-core/src/lookup_key.rs‎

Lines changed: 2 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ use std::convert::Infallible;
33
use std::fmt;
44

55
use pyo3::IntoPyObjectExt;
6-
use pyo3::exceptions::{PyAttributeError, PyTypeError, PyValueError};
6+
use pyo3::exceptions::{PyTypeError, PyValueError};
77
use pyo3::prelude::*;
88
use pyo3::pybacked::PyBackedStr;
99
use pyo3::types::{PyDict, PyList, PyMapping, PyString};
@@ -14,7 +14,7 @@ use smallvec::SmallVec;
1414
use crate::build_tools::py_schema_err;
1515
use crate::errors::{ErrorType, LocItem, Location, ValError, ValLineError, ValResult, py_err_string};
1616
use crate::input::StringMapping;
17-
use crate::tools::{mapping_get, py_err};
17+
use crate::tools::{mapping_get, py_err, py_get_attrs};
1818

1919
/// The possible choices for an alias value in Python
2020
#[derive(FromPyObject)]
@@ -454,20 +454,3 @@ impl LookupType {
454454
(self as u8 & other as u8) != 0
455455
}
456456
}
457-
458-
// TODO replace with `PyAnyMethods::getattr_opt` once https://github.com/PyO3/pyo3/pull/5985 is merged:
459-
fn py_get_attrs<'py, N>(obj: &Bound<'py, PyAny>, attr_name: N) -> PyResult<Option<Bound<'py, PyAny>>>
460-
where
461-
N: IntoPyObject<'py, Target = PyString>,
462-
{
463-
match obj.getattr(attr_name) {
464-
Ok(attr) => Ok(Some(attr)),
465-
Err(err) => {
466-
if err.get_type(obj.py()).is_subclass_of::<PyAttributeError>()? {
467-
Ok(None)
468-
} else {
469-
Err(err)
470-
}
471-
}
472-
}
473-
}

‎pydantic-core/src/serializers/infer.rs‎

Lines changed: 32 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ use crate::serializers::shared::serialize_to_python;
2727
use crate::serializers::type_serializers;
2828
use crate::serializers::type_serializers::any::AnySerializer;
2929
use crate::serializers::type_serializers::format::serialize_via_str;
30-
use crate::tools::{py_err, safe_repr};
30+
use crate::tools::{py_err, py_get_attrs, safe_repr};
3131

3232
use super::SchemaSerializer;
3333
use super::config::InfNanMode;
@@ -660,10 +660,33 @@ pub(crate) fn infer_json_key_known<'a, 'py>(
660660
}
661661
}
662662

663-
pub(crate) fn get_pydantic_serializer<'py>(value: &Bound<'py, PyAny>) -> PyResult<Bound<'py, SchemaSerializer>> {
663+
/// Get the `SchemaSerializer` stored as `__pydantic_serializer__` on `value`'s type, or `None`
664+
/// if there's no such attribute (e.g. for stdlib dataclasses).
665+
///
666+
/// If the type isn't fully built yet, pydantic stores a placeholder (`MockValSer`) that rebuilds the
667+
/// type lazily when one of its attributes is accessed from Python. Casting it from Rust bypasses that
668+
/// mechanism, so we explicitly call its `rebuild()` method, which rebuilds the whole type and either
669+
/// returns the built serializer or raises a meaningful error.
670+
pub(crate) fn get_pydantic_serializer<'py>(
671+
value: &Bound<'py, PyAny>,
672+
) -> PyResult<Option<Bound<'py, SchemaSerializer>>> {
664673
let py = value.py();
665-
let py_serializer = value.getattr(intern!(py, "__pydantic_serializer__"))?;
666-
py_serializer.cast_into_exact().map_err(Into::into)
674+
let Some(py_serializer) = py_get_attrs(value, intern!(py, "__pydantic_serializer__"))? else {
675+
return Ok(None);
676+
};
677+
if let Ok(serializer) = py_serializer.cast_exact::<SchemaSerializer>() {
678+
return Ok(Some(serializer.clone()));
679+
}
680+
if py_serializer.hasattr(intern!(py, "rebuild"))? {
681+
// The call stack is unrelated to where the type was defined, so don't use a parent frame namespace:
682+
let kwargs = PyDict::new(py);
683+
kwargs.set_item(intern!(py, "parent_namespace_depth"), 0)?;
684+
let rebuilt = py_serializer.call_method(intern!(py, "rebuild"), (), Some(&kwargs))?;
685+
if !rebuilt.is_none() {
686+
return Ok(Some(rebuilt.cast_into_exact()?));
687+
}
688+
}
689+
Ok(Some(py_serializer.cast_into_exact()?))
667690
}
668691

669692
/// Serialize `value` as if it had a `__pydantic_serializer__` attribute
@@ -674,9 +697,11 @@ fn serialize_pydantic_serializable<'py, S: DoSerialize>(
674697
state: &mut SerializationState<'py>,
675698
do_serialize: S,
676699
) -> Result<S::Ok, S::Error> {
677-
let py = value.py();
678-
let py_serializer = value.getattr(intern!(py, "__pydantic_serializer__"))?;
679-
call_pydantic_serializer(py_serializer.cast().map_err(Into::into)?, value, state, do_serialize)
700+
match get_pydantic_serializer(value)? {
701+
Some(serializer) => call_pydantic_serializer(&serializer, value, state, do_serialize),
702+
// `ObType::PydanticSerializable` was inferred from the attribute being present
703+
None => Err(unknown_type_error(value).into()),
704+
}
680705
}
681706

682707
pub(crate) fn call_pydantic_serializer<'py, S: DoSerialize>(

‎pydantic-core/src/serializers/polymorphism_trampoline.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ impl PolymorphismTrampoline {
5050
&& runtime_polymorphic.unwrap_or(self.enabled_from_config)
5151
&& self.is_subclass(value)?
5252
// stdlib dataclasses do not have a `__pydantic_serializer__`
53-
&& let Ok(serializer) = get_pydantic_serializer(value)
53+
&& let Some(serializer) = get_pydantic_serializer(value)?
5454
{
5555
call_pydantic_serializer(&serializer, value, state, do_serialize)
5656
} else {

‎pydantic-core/src/tools.rs‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ use core::fmt;
33
use hashbrown::HashTable;
44
use pyo3::PyTraverseError;
55
use pyo3::PyVisit;
6-
use pyo3::exceptions::PyKeyError;
6+
use pyo3::exceptions::{PyAttributeError, PyKeyError};
77
use pyo3::intern;
88
use pyo3::prelude::*;
99
use pyo3::types::{PyDict, PyMapping, PyString};
@@ -202,6 +202,23 @@ pub fn mapping_get<'py>(
202202
.map(|value| if value.is(undefined) { None } else { Some(value) })
203203
}
204204

205+
// TODO replace with `PyAnyMethods::getattr_opt` once https://github.com/PyO3/pyo3/pull/5985 is merged:
206+
pub(crate) fn py_get_attrs<'py, N>(obj: &Bound<'py, PyAny>, attr_name: N) -> PyResult<Option<Bound<'py, PyAny>>>
207+
where
208+
N: IntoPyObject<'py, Target = PyString>,
209+
{
210+
match obj.getattr(attr_name) {
211+
Ok(attr) => Ok(Some(attr)),
212+
Err(err) => {
213+
if err.get_type(obj.py()).is_subclass_of::<PyAttributeError>()? {
214+
Ok(None)
215+
} else {
216+
Err(err)
217+
}
218+
}
219+
}
220+
}
221+
205222
/// A hash table which uses (hashable) Python objects as keys
206223
#[derive(Debug)]
207224
pub struct PyHashTable<T>(HashTable<PyHashTableEntry<T>>);

‎pydantic/_internal/_mock_val_ser.py‎

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
from __future__ import annotations
22

33
from collections.abc import Callable, Iterator, Mapping
4-
from typing import TYPE_CHECKING, Any, Generic, Literal, TypeVar
4+
from typing import TYPE_CHECKING, Any, Generic, Literal, Protocol, TypeVar
55

66
from pydantic_core import CoreSchema, SchemaSerializer, SchemaValidator
77

@@ -16,6 +16,11 @@
1616

1717
ValSer = TypeVar('ValSer', bound=SchemaValidator | PluggableSchemaValidator | SchemaSerializer)
1818
T = TypeVar('T')
19+
T_co = TypeVar('T_co', covariant=True)
20+
21+
22+
class AttemptRebuild(Protocol[T_co]):
23+
def __call__(self, *, parent_namespace_depth: int = ...) -> T_co | None: ...
1924

2025

2126
class MockCoreSchema(Mapping[str, Any]):
@@ -30,7 +35,7 @@ def __init__(
3035
error_message: str,
3136
*,
3237
code: PydanticErrorCodes,
33-
attempt_rebuild: Callable[[], CoreSchema | None] | None = None,
38+
attempt_rebuild: AttemptRebuild[CoreSchema] | None = None,
3439
) -> None:
3540
self._error_message = error_message
3641
self._code: PydanticErrorCodes = code
@@ -81,7 +86,7 @@ def __init__(
8186
*,
8287
code: PydanticErrorCodes,
8388
val_or_ser: Literal['validator', 'serializer'],
84-
attempt_rebuild: Callable[[], ValSer | None] | None = None,
89+
attempt_rebuild: AttemptRebuild[ValSer] | None = None,
8590
) -> None:
8691
self._error_message = error_message
8792
self._val_or_ser = SchemaValidator if val_or_ser == 'validator' else SchemaSerializer
@@ -99,9 +104,9 @@ def __getattr__(self, item: str) -> None:
99104
getattr(self._val_or_ser, item)
100105
raise PydanticUserError(self._error_message, code=self._code)
101106

102-
def rebuild(self) -> ValSer | None:
107+
def rebuild(self, *, parent_namespace_depth: int = 5) -> ValSer | None:
103108
if self._attempt_rebuild:
104-
val_ser = self._attempt_rebuild()
109+
val_ser = self._attempt_rebuild(parent_namespace_depth=parent_namespace_depth)
105110
if val_ser is not None:
106111
return val_ser
107112
else:
@@ -121,9 +126,9 @@ def set_type_adapter_mocks(adapter: TypeAdapter) -> None:
121126
f' then call `.rebuild()` on the instance.'
122127
)
123128

124-
def attempt_rebuild_fn(attr_fn: Callable[[TypeAdapter], T]) -> Callable[[], T | None]:
125-
def handler() -> T | None:
126-
if adapter.rebuild(raise_errors=False, _parent_namespace_depth=5) is not False:
129+
def attempt_rebuild_fn(attr_fn: Callable[[TypeAdapter], T]) -> AttemptRebuild[T]:
130+
def handler(*, parent_namespace_depth: int = 5) -> T | None:
131+
if adapter.rebuild(raise_errors=False, _parent_namespace_depth=parent_namespace_depth) is not False:
127132
return attr_fn(adapter)
128133
return None
129134

@@ -162,9 +167,9 @@ def set_model_mocks(cls: type[BaseModel], undefined_name: str | None = None) ->
162167
f' then call `{cls.__name__}.model_rebuild()`.'
163168
)
164169

165-
def attempt_rebuild_fn(attr_fn: Callable[[type[BaseModel]], T]) -> Callable[[], T | None]:
166-
def handler() -> T | None:
167-
if cls.model_rebuild(raise_errors=False, _parent_namespace_depth=5) is not False:
170+
def attempt_rebuild_fn(attr_fn: Callable[[type[BaseModel]], T]) -> AttemptRebuild[T]:
171+
def handler(*, parent_namespace_depth: int = 5) -> T | None:
172+
if cls.model_rebuild(raise_errors=False, _parent_namespace_depth=parent_namespace_depth) is not False:
168173
return attr_fn(cls)
169174
return None
170175

@@ -205,9 +210,9 @@ def set_dataclass_mocks(cls: type[PydanticDataclass], undefined_name: str | None
205210
f' then call `pydantic.dataclasses.rebuild_dataclass({cls.__name__})`.'
206211
)
207212

208-
def attempt_rebuild_fn(attr_fn: Callable[[type[PydanticDataclass]], T]) -> Callable[[], T | None]:
209-
def handler() -> T | None:
210-
if rebuild_dataclass(cls, raise_errors=False, _parent_namespace_depth=5) is not False:
213+
def attempt_rebuild_fn(attr_fn: Callable[[type[PydanticDataclass]], T]) -> AttemptRebuild[T]:
214+
def handler(*, parent_namespace_depth: int = 5) -> T | None:
215+
if rebuild_dataclass(cls, raise_errors=False, _parent_namespace_depth=parent_namespace_depth) is not False:
211216
return attr_fn(cls)
212217
return None
213218

‎tests/test_construction.py‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import pickle
2-
from typing import Any
2+
from typing import Annotated, Any
33

44
import pytest
55
from pydantic_core import PydanticUndefined, ValidationError
@@ -669,3 +669,26 @@ class MyModel(BaseModel):
669669
assert MyModel.model_construct(a='a_value').a == 'a_value'
670670
assert MyModel.model_construct(aaa='a_value').a == 'a_value'
671671
assert MyModel.model_construct(AAA={'aaa': 'a_value'}).a == 'a_value'
672+
673+
674+
@pytest.mark.xfail(
675+
reason='`model_construct()` does not rebuild incomplete models, and relies on `FieldInfo` instances '
676+
'that may be incomplete (here, the `Field()` metadata is part of the unevaluated annotation).',
677+
)
678+
def test_model_construct_incomplete_model() -> None:
679+
# When making this test work, look at `test_string_annotation_union_type()` which assumes `model_construct()`
680+
# doesn't rebuild the model.
681+
class Model(BaseModel):
682+
a: 'Annotated[Later | None, Field(alias="A")]'
683+
b: 'Annotated[Later | None, Field(default=None)]'
684+
685+
class Later(BaseModel):
686+
pass
687+
688+
assert Model.__pydantic_complete__ is False
689+
690+
later = Later()
691+
m = Model.model_construct(A=later)
692+
assert m.a is later
693+
assert m.b is None
694+
assert m.model_fields_set == {'a'}

‎tests/test_forward_ref.py‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1731,3 +1731,44 @@ class Main(BaseModel):
17311731

17321732
assert typing_objects.is_union(get_origin(annotation))
17331733
assert get_args(annotation)[0] is Main
1734+
1735+
1736+
def test_serialization_inference_rebuilds_incomplete_model(create_module) -> None:
1737+
"""https://github.com/pydantic/pydantic/issues/7713"""
1738+
1739+
# Necessary to have classes defined at module level for forward ref
1740+
# to resolve when rebuilding model from pydantic-core:
1741+
@create_module
1742+
def module():
1743+
from typing import Optional
1744+
1745+
from pydantic import BaseModel
1746+
1747+
class B(BaseModel):
1748+
a: Optional['A'] = None
1749+
1750+
class A(BaseModel):
1751+
b: B
1752+
1753+
a = module.A.model_validate({'b': {}})
1754+
assert module.B.__pydantic_complete__ is False
1755+
1756+
assert TypeAdapter(Any).dump_python(a.b) == {'a': None}
1757+
assert module.B.__pydantic_complete__ is True
1758+
assert a.model_dump(serialize_as_any=True) == {'b': {'a': None}}
1759+
1760+
1761+
def test_serialization_inference_unbuildable_model() -> None:
1762+
"""When rebuilt from pydantic-core, the caller's frame namespace should not be used to resolve annotations."""
1763+
1764+
class Model(BaseModel):
1765+
x: 'Undefined' # noqa: F821
1766+
1767+
def serialize(m: Model) -> Any:
1768+
Undefined = int # noqa: F841
1769+
return TypeAdapter(Any).dump_python(m)
1770+
1771+
m = Model.model_construct(x=1)
1772+
with pytest.raises(PydanticUserError, match='`Model` is not fully defined; you should define `Undefined`'):
1773+
serialize(m)
1774+
assert Model.__pydantic_complete__ is False

‎tests/test_serialize_as_any.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,3 +209,25 @@ def test_serialize_as_any_annotation_with_incorrect_list_el_type() -> None:
209209
# notably, the warning is not raised when using the SerializeAsAny annotation
210210
ta = TypeAdapter(SerializeAsAny[list[int]])
211211
assert ta.dump_python(['a', 'b', 'c']) == ['a', 'b', 'c']
212+
213+
214+
@pytest.mark.parametrize('kwargs', [{'serialize_as_any': True}, {'polymorphic_serialization': True}])
215+
def test_serialize_as_any_deferred_subclass(kwargs) -> None:
216+
"""https://github.com/pydantic/pydantic/issues/13647"""
217+
218+
class Base(BaseModel):
219+
x: int = 1
220+
221+
class Sub(Base):
222+
model_config = ConfigDict(defer_build=True)
223+
y: int = 2
224+
225+
class Model(BaseModel):
226+
b: Base
227+
228+
# `model_construct()` currently doesn't trigger a rebuild of `Sub`:
229+
m = Model(b=Sub.model_construct(x=1, y=2))
230+
assert Sub.__pydantic_complete__ is False
231+
232+
assert m.model_dump(**kwargs) == {'b': {'x': 1, 'y': 2}}
233+
assert Sub.__pydantic_complete__ is True

0 commit comments

Comments
 (0)