Skip to content

Commit 70f3601

Browse files
authored
Fix handling of type var syntax and types.GenericAlias (#962)
* Fix handling of type-var syntax and types.GenericAlias * ensure info is cached * fix a typo * address review * add missing test cases. always gc.collect before refcount * fix multi-level inheritance with generic base * handle more oddities with manually constructed 'types.GenericAlias' * fix inspection for 3.10
1 parent 0cc800c commit 70f3601

3 files changed

Lines changed: 883 additions & 16 deletions

File tree

‎src/msgspec/_core.c‎

Lines changed: 57 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -519,6 +519,7 @@ typedef struct {
519519
PyObject *get_class_annotations;
520520
PyObject *get_typeddict_info;
521521
PyObject *get_dataclass_info;
522+
PyObject *convert_generic_alias;
522523
PyObject *rebuild;
523524
PyObject *types_uniontype;
524525
#if PY312_PLUS
@@ -4984,6 +4985,29 @@ is_dataclass_or_attrs_class(TypeNodeCollectState *state, PyObject *t) {
49844985
);
49854986
}
49864987

4988+
static MS_INLINE int
4989+
normalize_types_generic_alias(TypeNodeCollectState *state, PyObject **t, PyObject *origin, PyObject *args) {
4990+
// if '*t' is a 'types.GenericAlias', replace it (in place) with an equivalent
4991+
// 'typing._GenericAlias'. 'types.GenericAlias' has __slots__ and forwards
4992+
// attribute access to its origin, so we can't cache type info (as
4993+
// '__msgspec_cache__') on it; a 'typing._GenericAlias' can.
4994+
//
4995+
// only the branches that cache on the alias (struct/dataclass/TypedDict/
4996+
// NamedTuple/Literal) need to call this. a 'types.GenericAlias' here only arises
4997+
// when subclassing a builtin container generic (e.g. 'collections.abc.Mapping') or
4998+
// from a manually constructed 'types.GenericAlias' (e.g. wrapping 'typing.Literal').
4999+
//
5000+
// replace '*t' with a new reference on success (dropping the original), or
5001+
// return -1 with an exception set.
5002+
if (MS_LIKELY(Py_TYPE(*t) != &Py_GenericAliasType)) return 0;
5003+
PyObject *converted = PyObject_CallFunctionObjArgs(
5004+
state->mod->convert_generic_alias, origin, args, NULL
5005+
);
5006+
if (converted == NULL) return -1;
5007+
Py_SETREF(*t, converted);
5008+
return 0;
5009+
}
5010+
49875011
static int
49885012
typenode_collect_type(TypeNodeCollectState *state, PyObject *obj) {
49895013
int out = 0;
@@ -5065,7 +5089,12 @@ typenode_collect_type(TypeNodeCollectState *state, PyObject *obj) {
50655089
ms_is_struct_cls(t) ||
50665090
(origin != NULL && ms_is_struct_cls(origin))
50675091
) {
5068-
out = typenode_collect_struct(state, t);
5092+
if (normalize_types_generic_alias(state, &t, origin, args) < 0) {
5093+
out = -1;
5094+
}
5095+
else {
5096+
out = typenode_collect_struct(state, t);
5097+
}
50695098
}
50705099
else if (PyType_IsSubtype(Py_TYPE(t), state->mod->EnumMetaType)) {
50715100
out = typenode_collect_enum(state, t);
@@ -5155,25 +5184,45 @@ typenode_collect_type(TypeNodeCollectState *state, PyObject *obj) {
51555184
state->literals = PyList_New(0);
51565185
if (state->literals == NULL) goto done;
51575186
}
5158-
out = PyList_Append(state->literals, t);
5187+
if (normalize_types_generic_alias(state, &t, origin, args) < 0) {
5188+
out = -1;
5189+
}
5190+
else {
5191+
out = PyList_Append(state->literals, t);
5192+
}
51595193
}
51605194
else if (
51615195
is_typeddict_class(state, t) ||
51625196
(origin != NULL && is_typeddict_class(state, origin))
51635197
) {
5164-
out = typenode_collect_typeddict(state, t);
5198+
if (normalize_types_generic_alias(state, &t, origin, args) < 0) {
5199+
out = -1;
5200+
}
5201+
else {
5202+
out = typenode_collect_typeddict(state, t);
5203+
}
51655204
}
51665205
else if (
51675206
is_namedtuple_class(state, t) ||
51685207
(origin != NULL && is_namedtuple_class(state, origin))
51695208
) {
5170-
out = typenode_collect_namedtuple(state, t);
5209+
if (normalize_types_generic_alias(state, &t, origin, args) < 0) {
5210+
out = -1;
5211+
}
5212+
else {
5213+
out = typenode_collect_namedtuple(state, t);
5214+
}
51715215
}
51725216
else if (
51735217
is_dataclass_or_attrs_class(state, t) ||
51745218
(origin != NULL && is_dataclass_or_attrs_class(state, origin))
51755219
) {
5176-
out = typenode_collect_dataclass(state, t);
5220+
if (normalize_types_generic_alias(state, &t, origin, args) < 0) {
5221+
out = -1;
5222+
}
5223+
else {
5224+
out = typenode_collect_dataclass(state, t);
5225+
}
51775226
}
51785227
else {
51795228
if (origin != NULL) {
@@ -22428,6 +22477,7 @@ msgspec_clear(PyObject *m)
2242822477
Py_CLEAR(st->get_dataclass_info);
2242922478
Py_CLEAR(st->rebuild);
2243022479
Py_CLEAR(st->types_uniontype);
22480+
Py_CLEAR(st->convert_generic_alias);
2243122481
#if PY312_PLUS
2243222482
Py_CLEAR(st->typing_typealiastype);
2243322483
#endif
@@ -22501,6 +22551,7 @@ msgspec_traverse(PyObject *m, visitproc visit, void *arg)
2250122551
Py_VISIT(st->get_dataclass_info);
2250222552
Py_VISIT(st->rebuild);
2250322553
Py_VISIT(st->types_uniontype);
22554+
Py_VISIT(st->convert_generic_alias);
2250422555
#if PY312_PLUS
2250522556
Py_VISIT(st->typing_typealiastype);
2250622557
#endif
@@ -22702,6 +22753,7 @@ PyInit__core(void)
2270222753
SET_REF(get_dataclass_info, "get_dataclass_info");
2270322754
SET_REF(typing_annotated_alias, "_AnnotatedAlias");
2270422755
SET_REF(rebuild, "rebuild");
22756+
SET_REF(convert_generic_alias, "convert_generic_alias");
2270522757
Py_DECREF(temp_module);
2270622758

2270722759
temp_module = PyImport_ImportModule("types");

‎src/msgspec/_utils.py‎

Lines changed: 71 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
# type: ignore
22
import collections
33
import sys
4+
import types
45
import typing
5-
from typing import _AnnotatedAlias # noqa: F401
6+
from typing import _AnnotatedAlias, _GenericAlias # noqa: F401
67

78
try:
89
from typing_extensions import get_type_hints as _get_type_hints
@@ -22,6 +23,8 @@ def get_type_hints(obj):
2223
return _get_type_hints(obj, include_extras=True)
2324

2425

26+
PY_312PLUS = sys.version_info >= (3, 12)
27+
2528
# The `is_class` argument was new in 3.11, but was backported to 3.9 and 3.10.
2629
# It's _likely_ to be available for 3.9/3.10, but may not be. Easiest way to
2730
# check is to try it and see. This check can be removed when we drop support
@@ -79,23 +82,42 @@ def _apply_params(obj, mapping):
7982
def _get_class_mro_and_typevar_mappings(obj):
8083
mapping = {}
8184

82-
if isinstance(obj, type):
85+
# in Python 3.10 a natively produced 'types.GenericAlias' (e.g. 'list[int]', or the
86+
# 'Base[int]' produced when a 'Generic' subclass inherits a builtin's
87+
# '__class_getitem__') satisfies 'isinstance(_, type)', unlike on 3.11+. we still
88+
# want to treat those as parametrised aliases, not bare classes
89+
if isinstance(obj, type) and not isinstance(obj, types.GenericAlias):
8390
cls = obj
8491
else:
8592
cls = obj.__origin__
8693

8794
def inner(c, scope):
88-
if isinstance(c, type):
95+
if isinstance(c, type) and not isinstance(c, types.GenericAlias):
8996
cls = c
9097
new_scope = {}
9198
else:
92-
cls = getattr(c, "__origin__", None)
99+
cls = typing.get_origin(c)
93100
if cls in (None, object, typing.Generic) or cls in mapping:
94101
return
95-
params = cls.__parameters__
96-
args = tuple(_apply_params(a, scope) for a in c.__args__)
97-
assert len(params) == len(args)
98-
mapping[cls] = new_scope = dict(zip(params, args))
102+
103+
if hasattr(cls, "__parameters__"):
104+
# 'cls' carries its own type vars. This covers both ordinary
105+
# 'typing._GenericAlias' bases and the 'types.GenericAlias' that
106+
# get produced when a user-defined 'Generic' subclass inherits a
107+
# builtin's '__class_getitem__' (e.g. 'class Base(Mapping[str, T])',
108+
# whose 'Base[...]' is a 'types.GenericAlias'). Map cls's own
109+
# type vars onto the resolved args, applying '_apply_params' so any
110+
# outer bindings in 'scope' (e.g. 'U -> int') are propagated.
111+
params = cls.__parameters__
112+
args = tuple(_apply_params(a, scope) for a in typing.get_args(c))
113+
assert len(params) == len(args)
114+
new_scope = dict(zip(params, args))
115+
else:
116+
# a true built-in generic (e.g. 'collections.abc.Mapping[str, T]')
117+
# whose '__origin__' has no '__parameters__'; the unresolved type
118+
# vars and args live on the alias itself, not the origin.
119+
new_scope = dict(zip(c.__parameters__, typing.get_args(c)))
120+
mapping[cls] = new_scope
99121

100122
if issubclass(cls, typing.Generic):
101123
bases = getattr(cls, "__orig_bases__", cls.__bases__)
@@ -133,6 +155,11 @@ def get_class_annotations(obj):
133155

134156
mapping = typevar_mappings.get(cls)
135157
cls_locals = dict(vars(cls))
158+
159+
if PY_312PLUS:
160+
# resolve type parameters (e.g. class Foo[T]: pass)
161+
cls_locals.update({p.__name__: p for p in cls.__type_params__})
162+
136163
try:
137164
cls_module = cls.__module__
138165
except AttributeError:
@@ -315,3 +342,39 @@ def inner(obj):
315342
def rebuild(cls, kwargs):
316343
"""Used to unpickle Structs with keyword-only fields"""
317344
return cls(**kwargs)
345+
346+
347+
def convert_generic_alias(origin, args):
348+
# subscribed typing._GenericAlias instances are cached within the typing module
349+
# we make use of this fact, by storing a __msgspec_cache__ attribute on the
350+
# subscribed instance. only subscribed types are cached, so
351+
# 'typing._GenericAlias(list, int) is typing._GenericAlias(list, int)' would be
352+
# false.
353+
# to achieve the same behaviour when re-creating a typing._GenericAlias from a
354+
# types.GenericAlias, we first construct a temporary *unbound*
355+
# typing._GenericAlias, on which we then call __getattr__. effectively doing
356+
# typing._GenericAlias(list, T)[int], for which
357+
# 'typing._GenericAlias(list, T)[int] is typing._GenericAlias(list, T)[int]'
358+
# holds true
359+
try:
360+
params = origin.__parameters__
361+
except AttributeError:
362+
if not isinstance(origin, type):
363+
# a special form such as 'typing.Literal', whose args are values rather than
364+
# type parameters. Ordinary subscription yields the canonical, interned
365+
# alias of the right subclass (e.g. 'typing.Literal[...]' -> '_LiteralGenericAlias').
366+
return origin[args]
367+
368+
# a non-generic class with type arguments. only reachable for e.g.
369+
# manually-built 'types.GenericAlias' instances and is probably nonsense or at
370+
# least not somthing we can meaningfully represent, so complain about it here,
371+
# rather than silently dropping the arguments.
372+
raise TypeError(f"{origin.__name__!r} is not a generic type")
373+
374+
# a regularly-parametrised generic. Create a new typing._GenericAlias with the
375+
# origin's unbound type params (e.g. for a 'Mapping[str, int]' this is a
376+
# '_GenericAlias(Mapping, (~K, ~V))'), then bind it to the concrete args by
377+
# subscripting with the args *tuple* (i.e. 'alias[(int, str)]', not
378+
# 'alias[int, str]'), so generics with more than one type var work
379+
alias = _GenericAlias(origin, params)
380+
return alias[args]

0 commit comments

Comments
 (0)