Skip to content

Commit 2f0b50d

Browse files
authored
Correct the reported signatures of several public callables (#1197)
`inspect.signature` and `help()` are built from a text signature: the first line of a C-level callable's docstring, before the `--` separator. That line has to parse as a Python parameter list, and it has to match how the callable is really called. Fourteen of those lines in `_core.c` fail one or the other, and between them they cover sixteen callables. Three do not parse. `msgspec.Raw` declares `Raw(msg="", /)`, which is two adjacent C string literals concatenating to `Raw(msg=, /)`. `msgspec.StructMeta` declares `*, **struct_config`. `msgspec.structs.replace` declares `replace(struct, / **changes)`, missing a comma. For `Raw`, `inspect.signature` raises `ValueError: <class 'msgspec.Raw'> builtin has invalid signature` on every supported version, and the API docs render `class msgspec.Raw` with no parameters. `replace` behaves that way from Python 3.12 onwards: up to 3.11 the stray `/` was stripped along with the rest of the non-Python syntax, leaving something that parsed, and from 3.12 it is kept. `StructMeta` has a `__signature__` getset, so its string only reached `help()` and `docs/api.rst`, which hard-codes the same text. Four more parse, but describe the wrong calling convention. Keyword arguments are rejected at runtime by `msgspec.msgpack.Ext`, `msgspec.structs.asdict`, `msgspec.structs.astuple` and `msgspec.structs.force_setattr`, yet their parameters are marked as positional-or-keyword. The stubs carry the same omission, so `msgspec.structs.asdict(struct=p)` passes type checking and raises `TypeError` when run. The remaining seven begin with a bare `self` rather than CPython's `$self` marker, so the instance becomes a parameter of the bound method: `inspect.signature(encoder.encode)` gives `(self, obj, /)`, and binding a single argument to that fails although `encoder.encode(obj)` is the correct call. Nine bound methods are affected: `encode`, `encode_into` and `encode_lines` on `json.Encoder`, `encode` and `encode_into` on `msgpack.Encoder`, `decode` and `decode_lines` on `json.Decoder`, `decode` on `msgpack.Decoder`, and `Raw.copy`. In `Raw.copy` that `self` carries no trailing `/` either, so it was reported as keyword-addressable on top of being reported at all. Ten stub declarations gain the matching `/`. The unit tests assert the exact reported signature of fifteen callables against a literal, and compile the `StructMeta` text signature. Five negative checks in `tests/typing/basic_typing_examples.py` pin the tightened stubs in that file's existing idiom. `stubtest` couples four of the ten stub edits to the C side by itself; the `Ext.__init__` edit is pinned by nothing else, and the five `self` edits have no observable effect to pin. On current main, fifteen of the sixteen unit cases fail on Python 3.11 and all sixteen on 3.14, and mypy, pyright and pyrefly each flag all five negative checks as unused ignores. Not addressed here: four decoder strings give `type='Any'` as a default, where the real default is `typing.Any` rather than that string. A text signature cannot carry that value. A default written as a name is evaluated, and the result is kept only if it is one of a handful of scalar types, so `type=Any` and `type=typing.Any` alike make the whole signature invalid and `inspect.signature` then raises for the callable. Correcting this one means going outside the text signature. This continues #1116, which corrected nine such strings. --------- Co-authored-by: Tseluiko Aleksandr <[email protected]>
1 parent 8a25471 commit 2f0b50d

7 files changed

Lines changed: 95 additions & 25 deletions

File tree

‎docs/api.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ Structs
88

99
.. autoclass:: Struct
1010

11-
.. autoclass:: StructMeta(name, bases, namespace, /, *, **struct_config)
11+
.. autoclass:: StructMeta(name, bases, namespace, /, **struct_config)
1212

1313
.. autofunction:: field
1414

‎src/msgspec/__init__.pyi‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ _T = TypeVar("_T")
7676
@final
7777
class UnsetType(enum.Enum):
7878
UNSET = "UNSET"
79-
def __bool__(self) -> Literal[False]: ...
79+
def __bool__(self, /) -> Literal[False]: ...
8080

8181
UNSET: Final = UnsetType.UNSET
8282

@@ -126,8 +126,8 @@ class Struct(metaclass=StructMeta):
126126
dict: bool = False,
127127
cache_hash: bool = False,
128128
) -> None: ...
129-
def __rich_repr__(self) -> list[tuple[str, Any]]: ...
130-
def __replace__(self, **changes: Any) -> Self: ...
129+
def __rich_repr__(self, /) -> list[tuple[str, Any]]: ...
130+
def __replace__(self, /, **changes: Any) -> Self: ...
131131

132132
def defstruct(
133133
name: str,
@@ -165,8 +165,8 @@ class Raw(bytes):
165165
@overload
166166
def __new__(cls) -> "Raw": ...
167167
@overload
168-
def __new__(cls, msg: Buffer | str) -> "Raw": ...
169-
def copy(self) -> "Raw": ...
168+
def __new__(cls, msg: Buffer | str, /) -> "Raw": ...
169+
def copy(self, /) -> "Raw": ...
170170

171171
#: We can't represent this in types, only via a name:
172172
_NonNegativeInt: TypeAlias = int
@@ -258,7 +258,7 @@ class Meta:
258258
examples: Final[list[Any] | None]
259259
extra_json_schema: Final[dict[str, Any] | None]
260260
extra: Final[dict[str, Any] | None]
261-
def __rich_repr__(self) -> list[tuple[str, Any]]: ...
261+
def __rich_repr__(self, /) -> list[tuple[str, Any]]: ...
262262

263263
def to_builtins(
264264
obj: Any,

‎src/msgspec/_core.c‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1476,7 +1476,7 @@ Raw_New(PyObject *msg) {
14761476
}
14771477

14781478
PyDoc_STRVAR(Raw__doc__,
1479-
"Raw(msg="", /)\n"
1479+
"Raw(msg=b'', /)\n"
14801480
"--\n"
14811481
"\n"
14821482
"A buffer containing an encoded message.\n"
@@ -1621,7 +1621,7 @@ Raw_reduce(Raw *self, PyObject *unused)
16211621
}
16221622

16231623
PyDoc_STRVAR(Raw_copy__doc__,
1624-
"copy(self)\n"
1624+
"copy($self, /)\n"
16251625
"--\n"
16261626
"\n"
16271627
"Copy a Raw object.\n"
@@ -7523,7 +7523,7 @@ static PyGetSetDef StructMeta_getset[] = {
75237523
};
75247524

75257525
PyDoc_STRVAR(StructMeta__doc__,
7526-
"StructMeta(name, bases, namespace, /, *, **struct_config)\n"
7526+
"StructMeta(name, bases, namespace, /, **struct_config)\n"
75277527
"--\n"
75287528
"\n"
75297529
"The metaclass for creating `Struct` types. See its documentation for the\n"
@@ -8219,7 +8219,7 @@ AssocList_FromStruct(PyObject *obj) {
82198219
}
82208220

82218221
PyDoc_STRVAR(struct_replace__doc__,
8222-
"replace(struct, / **changes)\n"
8222+
"replace(struct, /, **changes)\n"
82238223
"--\n"
82248224
"\n"
82258225
"Create a new struct instance of the same type as ``struct``, replacing fields\n"
@@ -8265,7 +8265,7 @@ struct_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject
82658265
}
82668266

82678267
PyDoc_STRVAR(struct_asdict__doc__,
8268-
"asdict(struct)\n"
8268+
"asdict(struct, /)\n"
82698269
"--\n"
82708270
"\n"
82718271
"Convert a struct to a dict.\n"
@@ -8333,7 +8333,7 @@ struct_asdict(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
83338333
}
83348334

83358335
PyDoc_STRVAR(struct_astuple__doc__,
8336-
"astuple(struct)\n"
8336+
"astuple(struct, /)\n"
83378337
"--\n"
83388338
"\n"
83398339
"Convert a struct to a tuple.\n"
@@ -8400,7 +8400,7 @@ struct_astuple(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
84008400
}
84018401

84028402
PyDoc_STRVAR(struct_force_setattr__doc__,
8403-
"force_setattr(struct, name, value)\n"
8403+
"force_setattr(struct, name, value, /)\n"
84048404
"--\n"
84058405
"\n"
84068406
"Set an attribute on a struct, even if the struct is frozen.\n"
@@ -9267,7 +9267,7 @@ Ext_New(long code, PyObject *data) {
92679267
}
92689268

92699269
PyDoc_STRVAR(Ext__doc__,
9270-
"Ext(code, data)\n"
9270+
"Ext(code, data, /)\n"
92719271
"--\n"
92729272
"\n"
92739273
"A record representing a MessagePack Extension Type.\n"
@@ -9885,7 +9885,7 @@ Encoder_dealloc(Encoder *self)
98859885
}
98869886

98879887
PyDoc_STRVAR(Encoder_encode_into__doc__,
9888-
"encode_into(self, obj, buffer, offset=0, /)\n"
9888+
"encode_into($self, obj, buffer, offset=0, /)\n"
98899889
"--\n"
98909890
"\n"
98919891
"Serialize an object into an existing bytearray buffer.\n"
@@ -9967,7 +9967,7 @@ encoder_encode_into_common(
99679967
}
99689968

99699969
PyDoc_STRVAR(Encoder_encode__doc__,
9970-
"encode(self, obj, /)\n"
9970+
"encode($self, obj, /)\n"
99719971
"--\n"
99729972
"\n"
99739973
"Serialize an object to bytes.\n"
@@ -14942,7 +14942,7 @@ JSONEncoder_encode(Encoder *self, PyObject *const *args, Py_ssize_t nargs)
1494214942
}
1494314943

1494414944
PyDoc_STRVAR(JSONEncoder_encode_lines__doc__,
14945-
"encode_lines(self, items, /)\n"
14945+
"encode_lines($self, items, /)\n"
1494614946
"--\n"
1494714947
"\n"
1494814948
"Encode an iterable of items as newline-delimited JSON, one item per line.\n"
@@ -16822,7 +16822,7 @@ mpack_decode(
1682216822
}
1682316823

1682416824
PyDoc_STRVAR(Decoder_decode__doc__,
16825-
"decode(self, buf, /)\n"
16825+
"decode($self, buf, /)\n"
1682616826
"--\n"
1682716827
"\n"
1682816828
"Deserialize an object from MessagePack.\n"
@@ -19794,7 +19794,7 @@ msgspec_json_format(PyObject *self, PyObject *args, PyObject *kwargs)
1979419794

1979519795

1979619796
PyDoc_STRVAR(JSONDecoder_decode__doc__,
19797-
"decode(self, buf, /)\n"
19797+
"decode($self, buf, /)\n"
1979819798
"--\n"
1979919799
"\n"
1980019800
"Deserialize an object from JSON.\n"
@@ -19851,7 +19851,7 @@ JSONDecoder_decode(JSONDecoder *self, PyObject *const *args, Py_ssize_t nargs)
1985119851
}
1985219852

1985319853
PyDoc_STRVAR(JSONDecoder_decode_lines__doc__,
19854-
"decode_lines(self, buf, /)\n"
19854+
"decode_lines($self, buf, /)\n"
1985519855
"--\n"
1985619856
"\n"
1985719857
"Decode a list of items from newline-delimited JSON.\n"

‎src/msgspec/msgpack.pyi‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ _DecimalFormatSig: TypeAlias = (
2727
class Ext:
2828
code: int
2929
data: Buffer
30-
def __init__(self, code: int, data: Buffer) -> None: ...
30+
def __init__(self, code: int, data: Buffer, /) -> None: ...
3131

3232
@final
3333
class Decoder(Generic[_T]):

‎src/msgspec/structs.pyi‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,9 @@ __all__ = (
1515
_S = TypeVar("_S", bound=Struct)
1616

1717
def replace(struct: _S, /, **changes: Any) -> _S: ...
18-
def asdict(struct: Struct) -> dict[str, Any]: ...
19-
def astuple(struct: Struct) -> tuple[Any, ...]: ...
20-
def force_setattr(struct: Struct, name: str, value: Any) -> None: ...
18+
def asdict(struct: Struct, /) -> dict[str, Any]: ...
19+
def astuple(struct: Struct, /) -> tuple[Any, ...]: ...
20+
def force_setattr(struct: Struct, name: str, value: Any, /) -> None: ...
2121

2222
@final
2323
class StructConfig:

‎tests/typing/basic_typing_examples.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -478,6 +478,8 @@ class Test(msgspec.Struct):
478478
assert_type(o, dict[str, Any])
479479
assert_type(o["foo"], Any)
480480

481+
msgspec.structs.asdict(struct=x) # type: ignore[call-arg] # pyright: ignore[reportCallIssue] # pyrefly: ignore[unexpected-keyword]
482+
481483

482484
def check_astuple() -> None:
483485
class Test(msgspec.Struct):
@@ -489,6 +491,8 @@ class Test(msgspec.Struct):
489491
assert_type(o, tuple[Any, ...])
490492
assert_type(o[0], Any)
491493

494+
msgspec.structs.astuple(struct=x) # type: ignore[call-arg] # pyright: ignore[reportCallIssue] # pyrefly: ignore[unexpected-keyword]
495+
492496

493497
def check_force_setattr() -> None:
494498
class Point(msgspec.Struct, frozen=True):
@@ -498,6 +502,8 @@ class Point(msgspec.Struct, frozen=True):
498502
obj = Point(1, 2)
499503
msgspec.structs.force_setattr(obj, "x", 3)
500504

505+
msgspec.structs.force_setattr(struct=obj, name="x", value=3) # type: ignore[call-arg] # pyright: ignore[reportCallIssue] # pyrefly: ignore[unexpected-keyword]
506+
501507

502508
def check_fields() -> None:
503509
class Test(msgspec.Struct):
@@ -616,6 +622,8 @@ def check_raw_constructor() -> None:
616622
r4 = msgspec.Raw(memoryview(b"test"))
617623
r2 = msgspec.Raw("test")
618624

625+
msgspec.Raw(msg=b"test") # type: ignore[call-overload] # pyright: ignore[reportCallIssue] # pyrefly: ignore[no-matching-overload]
626+
619627

620628
def check_raw_copy() -> None:
621629
r = msgspec.Raw()
@@ -790,6 +798,8 @@ def check_msgpack_Ext() -> None:
790798
assert_type(msgspec.msgpack.Ext(1, memoryview(b'')), msgspec.msgpack.Ext)
791799
assert_type(msgspec.msgpack.Ext(1, array.array('i', [1, 2, 3])), msgspec.msgpack.Ext)
792800

801+
msgspec.msgpack.Ext(code=1, data=b"test") # type: ignore[call-arg] # pyright: ignore[reportCallIssue] # pyrefly: ignore[unexpected-keyword]
802+
793803
# Non buffers:
794804
msgspec.msgpack.Ext(1, {}) # type: ignore[arg-type] # pyright: ignore[reportArgumentType] # pyrefly: ignore[bad-argument-type]
795805

‎tests/unit/test_common.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import decimal
77
import enum
88
import gc
9+
import inspect
910
import sys
1011
import textwrap
1112
import types
@@ -5641,3 +5642,62 @@ def test_decode_frozendict_typed(self, proto):
56415642
msg = proto.encode(frozendict({"abc": "xyz"}))
56425643
with pytest.raises(ValidationError, match="Expected `int`, got `str`"):
56435644
dec.decode(msg)
5645+
5646+
5647+
@pytest.mark.parametrize(
5648+
"func, expected",
5649+
[
5650+
pytest.param(msgspec.Raw, "(msg=b'', /)", id="Raw"),
5651+
pytest.param(msgspec.msgpack.Ext, "(code, data, /)", id="Ext"),
5652+
pytest.param(msgspec.structs.replace, "(struct, /, **changes)", id="replace"),
5653+
pytest.param(msgspec.structs.asdict, "(struct, /)", id="asdict"),
5654+
pytest.param(msgspec.structs.astuple, "(struct, /)", id="astuple"),
5655+
pytest.param(
5656+
msgspec.structs.force_setattr,
5657+
"(struct, name, value, /)",
5658+
id="force_setattr",
5659+
),
5660+
pytest.param(msgspec.Raw(b"1").copy, "()", id="Raw.copy"),
5661+
pytest.param(
5662+
msgspec.json.Encoder().encode, "(obj, /)", id="json.Encoder.encode"
5663+
),
5664+
pytest.param(
5665+
msgspec.json.Encoder().encode_into,
5666+
"(obj, buffer, offset=0, /)",
5667+
id="json.Encoder.encode_into",
5668+
),
5669+
pytest.param(
5670+
msgspec.json.Encoder().encode_lines,
5671+
"(items, /)",
5672+
id="json.Encoder.encode_lines",
5673+
),
5674+
pytest.param(
5675+
msgspec.json.Decoder().decode, "(buf, /)", id="json.Decoder.decode"
5676+
),
5677+
pytest.param(
5678+
msgspec.json.Decoder().decode_lines,
5679+
"(buf, /)",
5680+
id="json.Decoder.decode_lines",
5681+
),
5682+
pytest.param(
5683+
msgspec.msgpack.Encoder().encode, "(obj, /)", id="msgpack.Encoder.encode"
5684+
),
5685+
pytest.param(
5686+
msgspec.msgpack.Encoder().encode_into,
5687+
"(obj, buffer, offset=0, /)",
5688+
id="msgpack.Encoder.encode_into",
5689+
),
5690+
pytest.param(
5691+
msgspec.msgpack.Decoder().decode, "(buf, /)", id="msgpack.Decoder.decode"
5692+
),
5693+
],
5694+
)
5695+
def test_reported_signature(func, expected):
5696+
"""The reported signature must match how the callable is really called"""
5697+
assert str(inspect.signature(func)) == expected
5698+
5699+
5700+
def test_structmeta_text_signature_is_valid():
5701+
"""`StructMeta` has a `__signature__` getset, so only the text form is checkable"""
5702+
text_signature = msgspec.StructMeta.__text_signature__
5703+
exec(compile(f"def _f{text_signature}: pass", "<signature>", "exec"), {})

0 commit comments

Comments
 (0)