Skip to content

Commit 35db69d

Browse files
committed
Add default callback to encode/Encoder
Adds a `default` callback kwarg to `Encoder` constructor and `encode` function. This takes a callable that's called for all unsupported types to optionally convert them into supported types (or error if unsupported). This is useful for lazily coercing possibly unsupported types into a supported representation. Also fixes a bug in the `Encoder` dealloc, and moves the `type` kwarg to `msgspec.decode` to a keyword-only kwarg.
1 parent 9e449e7 commit 35db69d

5 files changed

Lines changed: 234 additions & 39 deletions

File tree

‎msgspec/core.c‎

Lines changed: 100 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ typedef struct {
8585
PyObject *str__value2member_map_;
8686
PyObject *str_name;
8787
PyObject *str_type;
88+
PyObject *str_default;
8889
PyObject *typing_list;
8990
PyObject *typing_set;
9091
PyObject *typing_tuple;
@@ -1504,6 +1505,7 @@ PyDoc_STRVAR(Struct__doc__,
15041505
*************************************************************************/
15051506

15061507
typedef struct EncoderState {
1508+
PyObject *defaultfn; /* `default` callback */
15071509
Py_ssize_t write_buffer_size; /* Configured internal buffer size */
15081510

15091511
PyObject *output_buffer; /* Bytearray storing the output */
@@ -1518,27 +1520,33 @@ typedef struct Encoder {
15181520
} Encoder;
15191521

15201522
PyDoc_STRVAR(Encoder__doc__,
1521-
"Encoder(*, write_buffer_size=4096)\n"
1523+
"Encoder(*, default=None, write_buffer_size=4096)\n"
15221524
"--\n"
15231525
"\n"
15241526
"A MessagePack encoder.\n"
15251527
"\n"
15261528
"Parameters\n"
15271529
"----------\n"
1530+
"default : callable, optional\n"
1531+
" A callable to call for objects that aren't supported msgspec types. Takes the\n"
1532+
" unsupported object and should return a supported object, or raise a TypeError.\n"
15281533
"write_buffer_size : int, optional\n"
15291534
" The size of the internal static write buffer."
15301535
);
15311536
static int
15321537
Encoder_init(Encoder *self, PyObject *args, PyObject *kwds)
15331538
{
1534-
static char *kwlist[] = {"write_buffer_size", NULL};
1539+
static char *kwlist[] = {"default", "write_buffer_size", NULL};
15351540
Py_ssize_t write_buffer_size = 4096;
1541+
PyObject *defaultfn = NULL;
15361542

1537-
if (!PyArg_ParseTupleAndKeywords(args, kwds, "|$n", kwlist,
1538-
&write_buffer_size)) {
1543+
if (!PyArg_ParseTupleAndKeywords(args, kwds, "|$On", kwlist,
1544+
&defaultfn, &write_buffer_size)) {
15391545
return -1;
15401546
}
15411547

1548+
Py_XINCREF(defaultfn);
1549+
self->state.defaultfn = defaultfn;
15421550
self->state.write_buffer_size = Py_MAX(write_buffer_size, 32);
15431551
self->state.max_output_len = self->state.write_buffer_size;
15441552
self->state.output_len = 0;
@@ -1548,12 +1556,28 @@ Encoder_init(Encoder *self, PyObject *args, PyObject *kwds)
15481556
return 0;
15491557
}
15501558

1559+
static int
1560+
Encoder_clear(Encoder *self)
1561+
{
1562+
Py_CLEAR(self->state.output_buffer);
1563+
Py_CLEAR(self->state.defaultfn);
1564+
return 0;
1565+
}
1566+
15511567
static void
15521568
Encoder_dealloc(Encoder *self)
15531569
{
1570+
Encoder_clear(self);
15541571
Py_TYPE(self)->tp_free((PyObject *)self);
15551572
}
15561573

1574+
static int
1575+
Encoder_traverse(Encoder *self, visitproc visit, void *arg)
1576+
{
1577+
Py_VISIT(self->state.defaultfn);
1578+
return 0;
1579+
}
1580+
15571581
static PyObject*
15581582
Encoder_sizeof(Encoder *self)
15591583
{
@@ -2068,9 +2092,21 @@ mp_encode(EncoderState *self, PyObject *obj)
20682092
if (PyType_IsSubtype(type, st->EnumType)) {
20692093
return mp_encode_enum(self, obj);
20702094
}
2095+
if (self->defaultfn != NULL) {
2096+
int status = -1;
2097+
PyObject *temp;
2098+
temp = CALL_ONE_ARG(self->defaultfn, obj);
2099+
if (temp == NULL) return -1;
2100+
if (!Py_EnterRecursiveCall(" while serializing an object")) {
2101+
status = mp_encode(self, temp);
2102+
Py_LeaveRecursiveCall();
2103+
}
2104+
Py_DECREF(temp);
2105+
return status;
2106+
}
20712107
else {
20722108
PyErr_Format(PyExc_TypeError,
2073-
"Encoder.encode doesn't support objects of type %.200s",
2109+
"Encoding objects of type %.200s is unsupported",
20742110
type->tp_name);
20752111
return -1;
20762112
}
@@ -2137,6 +2173,20 @@ Encoder_encode(Encoder *self, PyObject *const *args, Py_ssize_t nargs)
21372173
return res;
21382174
}
21392175

2176+
static PyObject *
2177+
Encoder_default(PyObject *self, void *closure) {
2178+
PyObject *out = ((Encoder*)self)->state.defaultfn;
2179+
if (out == NULL)
2180+
out = Py_None;
2181+
Py_INCREF(out);
2182+
return out;
2183+
}
2184+
2185+
static PyGetSetDef Encoder_getset[] = {
2186+
{"default", (getter) Encoder_default, NULL, "`default` callback", NULL},
2187+
{NULL},
2188+
};
2189+
21402190
static struct PyMethodDef Encoder_methods[] = {
21412191
{
21422192
"encode", (PyCFunction) Encoder_encode, METH_FASTCALL,
@@ -2149,21 +2199,23 @@ static struct PyMethodDef Encoder_methods[] = {
21492199
{NULL, NULL} /* sentinel */
21502200
};
21512201

2152-
21532202
static PyTypeObject Encoder_Type = {
21542203
PyVarObject_HEAD_INIT(NULL, 0)
21552204
.tp_name = "msgspec.core.Encoder",
21562205
.tp_doc = Encoder__doc__,
21572206
.tp_basicsize = sizeof(Encoder),
21582207
.tp_dealloc = (destructor)Encoder_dealloc,
2159-
.tp_flags = Py_TPFLAGS_DEFAULT,
2208+
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,
2209+
.tp_traverse = (traverseproc)Encoder_traverse,
2210+
.tp_clear = (inquiry)Encoder_clear,
21602211
.tp_new = PyType_GenericNew,
21612212
.tp_init = (initproc)Encoder_init,
21622213
.tp_methods = Encoder_methods,
2214+
.tp_getset = Encoder_getset,
21632215
};
21642216

21652217
PyDoc_STRVAR(msgspec_encode__doc__,
2166-
"encode(obj)\n"
2218+
"encode(obj, *, default=None)\n"
21672219
"--\n"
21682220
"\n"
21692221
"Serialize an object to bytes.\n"
@@ -2172,6 +2224,9 @@ PyDoc_STRVAR(msgspec_encode__doc__,
21722224
"----------\n"
21732225
"obj : Any\n"
21742226
" The object to serialize.\n"
2227+
"default : callable, optional\n"
2228+
" A callable to call for objects that aren't supported msgspec types. Takes the\n"
2229+
" unsupported object and should return a supported object, or raise a TypeError.\n"
21752230
"\n"
21762231
"Returns\n"
21772232
"-------\n"
@@ -2183,18 +2238,44 @@ PyDoc_STRVAR(msgspec_encode__doc__,
21832238
"Encoder.encode"
21842239
);
21852240
static PyObject*
2186-
msgspec_encode(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
2241+
msgspec_encode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
21872242
{
21882243
int status;
2189-
PyObject *res = NULL;
2244+
PyObject *obj, *defaultfn = NULL, *res = NULL;
21902245
EncoderState state;
21912246

2192-
if (!check_positional_nargs(nargs, 1, 1)) {
2193-
return NULL;
2247+
/* Parse arguments */
2248+
if (!check_positional_nargs(nargs, 1, 1)) return NULL;
2249+
obj = args[0];
2250+
if (kwnames != NULL) {
2251+
Py_ssize_t nkwargs = PyTuple_GET_SIZE(kwnames);
2252+
if (nkwargs == 1) {
2253+
PyObject *key = PyTuple_GET_ITEM(kwnames, 0);
2254+
MsgspecState *st = msgspec_get_global_state();
2255+
if (st->str_default == key || _PyUnicode_EQ(st->str_default, key)) {
2256+
defaultfn = args[nargs];
2257+
}
2258+
else {
2259+
PyErr_Format(
2260+
PyExc_TypeError,
2261+
"Invalid keyword argument '%U'",
2262+
key
2263+
);
2264+
return NULL;
2265+
}
2266+
}
2267+
else if (nkwargs > 1) {
2268+
PyErr_SetString(
2269+
PyExc_TypeError,
2270+
"Extra keyword arguments provided"
2271+
);
2272+
return NULL;
2273+
}
21942274
}
21952275

21962276
/* use a smaller buffer size here to reduce chance of over allocating for one-off calls */
21972277
state.write_buffer_size = 64;
2278+
state.defaultfn = defaultfn;
21982279
state.max_output_len = state.write_buffer_size;
21992280
state.output_len = 0;
22002281
state.output_buffer = PyBytes_FromStringAndSize(NULL, state.max_output_len);
@@ -3422,24 +3503,14 @@ msgspec_decode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject
34223503
Py_buffer buffer;
34233504
MsgspecState *st = msgspec_get_global_state();
34243505

3425-
/* Parse keyword arguments */
3426-
if (!check_positional_nargs(nargs, 1, 2)) return NULL;
3506+
/* Parse arguments */
3507+
if (!check_positional_nargs(nargs, 1, 1)) return NULL;
34273508
buf = args[0];
3428-
if (nargs == 2) {
3429-
type = args[1];
3430-
}
34313509
if (kwnames != NULL) {
34323510
Py_ssize_t nkwargs = PyTuple_GET_SIZE(kwnames);
34333511
if (nkwargs == 1) {
34343512
PyObject *key = PyTuple_GET_ITEM(kwnames, 0);
34353513
if (st->str_type == key || _PyUnicode_EQ(st->str_type, key)) {
3436-
if (type != NULL) {
3437-
PyErr_SetString(
3438-
PyExc_TypeError,
3439-
"Argument 'type' given by name and position"
3440-
);
3441-
return NULL;
3442-
}
34433514
type = args[nargs];
34443515
}
34453516
else {
@@ -3495,7 +3566,7 @@ msgspec_decode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject
34953566

34963567
static struct PyMethodDef msgspec_methods[] = {
34973568
{
3498-
"encode", (PyCFunction) msgspec_encode, METH_FASTCALL,
3569+
"encode", (PyCFunction) msgspec_encode, METH_FASTCALL | METH_KEYWORDS,
34993570
msgspec_encode__doc__,
35003571
},
35013572
{
@@ -3518,6 +3589,7 @@ msgspec_clear(PyObject *m)
35183589
Py_CLEAR(st->str__value2member_map_);
35193590
Py_CLEAR(st->str_name);
35203591
Py_CLEAR(st->str_type);
3592+
Py_CLEAR(st->str_default);
35213593
Py_CLEAR(st->typing_dict);
35223594
Py_CLEAR(st->typing_list);
35233595
Py_CLEAR(st->typing_set);
@@ -3692,6 +3764,9 @@ PyInit_core(void)
36923764
st->str_type = PyUnicode_InternFromString("type");
36933765
if (st->str_type == NULL)
36943766
return NULL;
3767+
st->str_default = PyUnicode_InternFromString("default");
3768+
if (st->str_default == NULL)
3769+
return NULL;
36953770

36963771
return m;
36973772
}

‎msgspec/core.pyi‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from typing import Any, Type, TypeVar, Generic, overload
1+
from typing import Any, Type, TypeVar, Generic, Callable, overload
22

33
T = TypeVar("T")
44

@@ -10,11 +10,13 @@ class Decoder(Generic[T]):
1010
def decode(self, data: bytes) -> T: ...
1111

1212
class Encoder:
13-
def __init__(self, *, write_buffer_size: int = ...): ...
13+
def __init__(
14+
self, *, default: Callable[[Any], Any] = ..., write_buffer_size: int = ...
15+
): ...
1416
def encode(self, obj: Any) -> bytes: ...
1517

1618
@overload
1719
def decode(buf: bytes) -> Any: ...
1820
@overload
19-
def decode(buf: bytes, type: Type[T] = ...) -> T: ...
20-
def encode(obj: Any) -> bytes: ...
21+
def decode(buf: bytes, *, type: Type[T] = ...) -> T: ...
22+
def encode(obj: Any, *, default: Callable[[Any], Any] = ...) -> bytes: ...

‎tests/mypy_examples.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,14 @@ def check_decode_any() -> None:
5252

5353
def check_decode_typed() -> None:
5454
b = msgspec.encode([1, 2, 3])
55-
o = msgspec.decode(b, List[int])
55+
o = msgspec.decode(b, type=List[int])
5656

5757
reveal_type(o) # assert ("List" in typ or "list" in typ) and "int" in typ
58+
59+
60+
def check_encode_default() -> None:
61+
msgspec.encode(object(), default=lambda x: None)
62+
63+
64+
def check_Encoder_default() -> None:
65+
msgspec.Encoder(default=lambda x: None)

0 commit comments

Comments
 (0)