Skip to content

Commit 2b73df8

Browse files
Merge branch 'main' into fix-timedelta-day-overflow
2 parents ab6b7b3 + eed27c3 commit 2b73df8

7 files changed

Lines changed: 173 additions & 11 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -256,4 +256,4 @@ jobs:
256256
path: dist
257257

258258
- name: Publish package distributions to PyPI
259-
uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
259+
uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3

‎docs/converters.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,9 @@ could add support by wrapping the standard library's `json` module as follows:
7171
ValidationError: Expected `int`, got `str` - at `$.x`
7272
7373
74+
When converting into a `dict`, `convert` also accepts mapping objects providing
75+
a ``keys()`` method and ``__getitem__`` for retrieving each key's value.
76+
7477
Since all protocols are different, `to_builtins` and `convert` have a few
7578
configuration options:
7679

‎src/msgspec/_core.c‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5000,7 +5000,11 @@ typenode_origin_args_metadata(
50005000
}
50015001
}
50025002

5003-
if (Py_TYPE(t) == (PyTypeObject *)(state->mod->types_uniontype)) {
5003+
/* Before Python 3.14, `types.UnionType` has no `__origin__`, so the
5004+
* `__origin__` lookup in the loop above leaves `args` unset and this
5005+
* fallback resolves the union. In 3.14+ that lookup already set `origin`
5006+
* and `args`; resolving them again here would leak both references. */
5007+
if (args == NULL && Py_TYPE(t) == (PyTypeObject *)(state->mod->types_uniontype)) {
50045008
/* Handle types.UnionType unions (`int | float | ...`) */
50055009
args = PyObject_GetAttr(t, state->mod->str___args__);
50065010
if (args == NULL) goto error;
@@ -22545,6 +22549,22 @@ getattr_then_getitem(PyObject *obj, PyObject *key) {
2254522549
return out;
2254622550
}
2254722551

22552+
static int
22553+
convert_has_keys(PyObject *obj) {
22554+
PyObject *keys_method = PyObject_GetAttrString(obj, "keys");
22555+
if (keys_method == NULL) {
22556+
if (PyErr_ExceptionMatches(PyExc_AttributeError)) {
22557+
PyErr_Clear();
22558+
return 0;
22559+
}
22560+
return -1;
22561+
}
22562+
22563+
Py_DECREF(keys_method);
22564+
22565+
return 1;
22566+
}
22567+
2254822568
static PyObject *
2254922569
convert_other(
2255022570
ConvertState *self, PyObject *obj, TypeNode *type, PathNode *path
@@ -22590,7 +22610,13 @@ convert_other(
2259022610

2259122611
/* Next try converting from a mapping or by attribute */
2259222612
bool is_mapping = PyMapping_Check(obj);
22593-
if (is_mapping) {
22613+
if (is_mapping && (type->types & MS_ANY_DICT)) {
22614+
int has_keys = convert_has_keys(obj);
22615+
if (has_keys < 0) return NULL;
22616+
if (!has_keys) {
22617+
return ms_validation_error(Py_TYPE(obj)->tp_name, type, path);
22618+
}
22619+
2259422620
if (type->types & MS_TYPE_DICT) {
2259522621
return convert_mapping_to_dict(self, obj, type, path);
2259622622
}

‎src/msgspec/toml.py‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -170,16 +170,16 @@ def decode(
170170
encode
171171
"""
172172
toml = _import_tomllib()
173-
if isinstance(buf, str):
174-
str_buf = buf
175-
elif isinstance(buf, (bytes, bytearray)):
176-
str_buf = buf.decode("utf-8")
177-
else:
178-
# call `memoryview` first, since `bytes(1)` is actually valid
179-
str_buf = bytes(memoryview(buf)).decode("utf-8")
180173
try:
174+
if isinstance(buf, str):
175+
str_buf = buf
176+
elif isinstance(buf, (bytes, bytearray)):
177+
str_buf = buf.decode("utf-8")
178+
else:
179+
# call `memoryview` first, since `bytes(1)` is actually valid
180+
str_buf = bytes(memoryview(buf)).decode("utf-8")
181181
obj = toml.loads(str_buf)
182-
except toml.TOMLDecodeError as exc:
182+
except (toml.TOMLDecodeError, UnicodeDecodeError) as exc:
183183
raise _DecodeError(str(exc)) from None
184184

185185
if type is Any:

‎tests/unit/test_common.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1187,6 +1187,27 @@ def test_310_union_types(self, proto):
11871187
assert dec.decode(proto.encode(1.5))
11881188

11891189

1190+
class _UnionLeakCustom:
1191+
pass
1192+
1193+
1194+
class _UnionLeakPoint(msgspec.Struct):
1195+
x: int
1196+
1197+
1198+
@pytest.mark.skipif(not hasattr(sys, "getrefcount"), reason="needs refcounts")
1199+
@pytest.mark.parametrize(
1200+
"member, other", [(_UnionLeakCustom, type(None)), (_UnionLeakPoint, int)]
1201+
)
1202+
def test_resolving_a_union_does_not_leak_its_args(member, other):
1203+
msgspec.json.Decoder(member | other)
1204+
before = sys.getrefcount(member)
1205+
for _ in range(10):
1206+
msgspec.json.Decoder(member | other)
1207+
msgspec.convert(1 if other is int else None, member | other)
1208+
assert sys.getrefcount(member) == before
1209+
1210+
11901211
class TestStructUnion:
11911212
def test_err_union_struct_mix_array_like(self, proto):
11921213
class Test1(Struct, tag=True, array_like=True):

‎tests/unit/test_convert.py‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import sys
77
import uuid
88
from base64 import b64encode
9+
from collections import UserDict
910
from collections.abc import MutableMapping
1011
from dataclasses import dataclass, field
1112
from typing import (
@@ -1219,6 +1220,84 @@ def test_dict_wrong_type(self):
12191220
with pytest.raises(ValidationError, match=r"Expected `object`, got `int`"):
12201221
assert convert(1, dict)
12211222

1223+
@pytest.mark.parametrize("strict", [False, True])
1224+
def test_dict_getitem_without_keys(self, strict):
1225+
class Table:
1226+
def __getitem__(self, key):
1227+
pytest.fail("Should not access non-mapping items")
1228+
1229+
class Ex(Struct):
1230+
table: dict[str, Any]
1231+
1232+
with pytest.raises(ValidationError) as rec:
1233+
convert({"table": Table()}, Ex, strict=strict)
1234+
assert str(rec.value) == "Expected `object`, got `Table` - at `$.table`"
1235+
1236+
@pytest.mark.parametrize(
1237+
"keys_type",
1238+
[
1239+
list,
1240+
tuple,
1241+
iter,
1242+
lambda keys: dict.fromkeys(keys).keys(),
1243+
lambda keys: UserDict.fromkeys(keys).keys(),
1244+
],
1245+
)
1246+
def test_dict_duck_mapping(self, keys_type):
1247+
class DuckMapping:
1248+
def keys(self):
1249+
return keys_type(["x", "y", "x"])
1250+
1251+
def __getitem__(self, key):
1252+
return {"x": 1, "y": 2}[key]
1253+
1254+
res = convert(DuckMapping(), dict[str, float])
1255+
assert res == {"x": 1.0, "y": 2.0}
1256+
1257+
def test_dict_mapping_keys_lookup_error(self):
1258+
class BadMapping:
1259+
@property
1260+
def keys(self):
1261+
raise RuntimeError("boom")
1262+
1263+
def __getitem__(self, key):
1264+
pytest.fail("Should not access items when keys lookup fails")
1265+
1266+
with pytest.raises(RuntimeError, match="boom"):
1267+
convert(BadMapping(), dict)
1268+
1269+
def test_dict_mapping_keys_call_error_propagates(self):
1270+
class BadMapping:
1271+
def keys(self):
1272+
raise RuntimeError("boom")
1273+
1274+
def __getitem__(self, key):
1275+
pytest.fail("Should not access items when keys call fails")
1276+
1277+
with pytest.raises(RuntimeError, match="boom"):
1278+
convert(BadMapping(), dict)
1279+
1280+
def test_dict_mapping_getitem_error_propagates(self):
1281+
class BadMapping:
1282+
def keys(self):
1283+
return ["x"]
1284+
1285+
def __getitem__(self, key):
1286+
raise RuntimeError("boom")
1287+
1288+
with pytest.raises(RuntimeError, match="boom"):
1289+
convert(BadMapping(), dict)
1290+
1291+
def test_dict_mapping_keys_not_callable(self):
1292+
class BadMapping:
1293+
keys = None
1294+
1295+
def __getitem__(self, key):
1296+
return 1
1297+
1298+
with pytest.raises(TypeError, match="not callable"):
1299+
convert(BadMapping(), dict)
1300+
12221301
def test_str_formatted_keys(self):
12231302
msg = {uuid.uuid4(): 1, uuid.uuid4(): 2}
12241303
res = convert(to_builtins(msg), dict[uuid.UUID, int])
@@ -1325,6 +1404,29 @@ class Ex(Struct):
13251404

13261405
@py315_or_later_only
13271406
class TestFrozenDict:
1407+
def test_getitem_without_keys(self):
1408+
class DuckMapping:
1409+
def __getitem__(self, key):
1410+
return 1
1411+
1412+
with pytest.raises(
1413+
ValidationError, match="Expected `object`, got `DuckMapping`"
1414+
):
1415+
convert(DuckMapping(), frozendict[str, float])
1416+
1417+
def test_duck_mapping(self):
1418+
class DuckMapping:
1419+
def keys(self):
1420+
return ["x"]
1421+
1422+
def __getitem__(self, key):
1423+
return 1
1424+
1425+
result = convert(DuckMapping(), frozendict[str, float])
1426+
assert type(result) is frozendict
1427+
assert result == frozendict({"x": 1.0})
1428+
assert type(result["x"]) is float
1429+
13281430
def check_frozen(self, val, expected):
13291431
assert isinstance(expected, frozendict)
13301432
assert type(val) is type(expected)

‎tests/unit/test_toml.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,16 @@ def test_decode_parse_error(msg):
208208
msgspec.toml.decode(msg)
209209

210210

211+
@needs_decode
212+
@pytest.mark.parametrize("buffer_type", [bytes, bytearray, memoryview])
213+
def test_decode_invalid_utf8(buffer_type):
214+
with pytest.raises(msgspec.DecodeError, match="utf-8"):
215+
msgspec.toml.decode(buffer_type(b"name = '\xff'"))
216+
assert msgspec.toml.decode(buffer_type(b"name = 'caf\xc3\xa9'")) == {
217+
"name": "caf\u00e9"
218+
}
219+
220+
211221
@needs_decode
212222
def test_decode_validation_error():
213223
with pytest.raises(msgspec.ValidationError, match="Expected `str`"):

0 commit comments

Comments
 (0)