Skip to content

Commit c415ff8

Browse files
authored
Merge branch 'main' into issue-1104
2 parents ab26298 + 68f53a7 commit c415ff8

8 files changed

Lines changed: 252 additions & 33 deletions

File tree

‎.github/workflows/docs.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ jobs:
8787
path: site
8888

8989
- name: Restore link check cache
90-
uses: actions/cache@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v6.0.0
90+
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
9191
with:
9292
path: .lycheecache
9393
# Caches are immutable. Restore the newest cache in this trusted

‎docs/supported-types.rst‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -604,6 +604,36 @@ numbers by creating a ``Encoder`` and specifying ``decimal_format='number'``.
604604
>>> encoder.encode(x)
605605
b'1.2345'
606606
607+
For JSON and MessagePack you may also pass a callable to ``decimal_format`` to
608+
customize how decimals are encoded. The callable will be called with the
609+
`decimal.Decimal` instance as the only argument, and must return a JSON-serializable
610+
value (but **not** another `decimal.Decimal` or a nested structure containing
611+
`decimal.Decimal`). This is useful for custom rounding, formatting, or transforming
612+
decimal values before encoding.
613+
614+
.. code-block:: python
615+
616+
>>> import decimal
617+
618+
>>> class Price(msgspec.Struct):
619+
... amount: decimal.Decimal
620+
... currency: str
621+
622+
>>> enc = msgspec.json.Encoder(
623+
... decimal_format=lambda d: str(d.quantize(decimal.Decimal("0.01")))
624+
... )
625+
>>> enc.encode(Price(amount=decimal.Decimal("10.123456"), currency="USD"))
626+
b'{"amount":"10.12","currency":"USD"}'
627+
628+
.. warning::
629+
630+
The callable passed to ``decimal_format`` must **not** return a
631+
`decimal.Decimal` instance or any nested structure (list, dict, tuple, set)
632+
containing `decimal.Decimal`, as this would cause infinite recursion. The
633+
callable should return a `str`, `int`, `float`, or other JSON-serializable
634+
type. If the callable returns a value containing a `decimal.Decimal`, a
635+
`TypeError` will be raised.
636+
607637
This setting is not yet supported for YAML or TOML - if this option is
608638
important for you please `open an issue`_.
609639

‎src/msgspec/_core.c‎

Lines changed: 85 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -569,6 +569,13 @@ ms_err_truncated(void)
569569
return -1;
570570
}
571571

572+
static int
573+
ms_decimal_format_error(void)
574+
{
575+
PyErr_SetString(PyExc_TypeError, "callable returned a value containing a Decimal");
576+
return -1;
577+
}
578+
572579
/*************************************************************************
573580
* Utilities *
574581
*************************************************************************/
@@ -9613,6 +9620,7 @@ AssocList_FromObject(PyObject *obj) {
96139620
enum decimal_format {
96149621
DECIMAL_FORMAT_STRING = 0,
96159622
DECIMAL_FORMAT_NUMBER = 1,
9623+
DECIMAL_FORMAT_CALLABLE = 2,
96169624
};
96179625

96189626
enum uuid_format {
@@ -9624,10 +9632,12 @@ enum uuid_format {
96249632
typedef struct EncoderState {
96259633
MsgspecState *mod; /* module reference */
96269634
PyObject *enc_hook; /* `enc_hook` callback */
9635+
PyObject *decimal_callable; /* `decimal_format` callable */
96279636
enum decimal_format decimal_format;
96289637
enum uuid_format uuid_format;
96299638
enum order_mode order;
96309639
char* (*resize_buffer)(PyObject**, Py_ssize_t); /* callback for resizing buffer */
9640+
bool in_decimal_callable;
96319641

96329642
char *output_buffer_raw; /* raw pointer to output_buffer internal buffer */
96339643
Py_ssize_t output_len; /* Length of output_buffer */
@@ -9638,6 +9648,7 @@ typedef struct EncoderState {
96389648
typedef struct Encoder {
96399649
PyObject_HEAD
96409650
PyObject *enc_hook;
9651+
PyObject *decimal_callable;
96419652
MsgspecState *mod;
96429653
enum decimal_format decimal_format;
96439654
enum uuid_format uuid_format;
@@ -9709,9 +9720,15 @@ Encoder_init(Encoder *self, PyObject *args, PyObject *kwds)
97099720
}
97109721

97119722
/* Process decimal format */
9723+
self->decimal_callable = NULL;
97129724
if (decimal_format == NULL) {
97139725
self->decimal_format = DECIMAL_FORMAT_STRING;
97149726
}
9727+
else if (PyCallable_Check(decimal_format)) {
9728+
self->decimal_format = DECIMAL_FORMAT_CALLABLE;
9729+
Py_INCREF(decimal_format);
9730+
self->decimal_callable = decimal_format;
9731+
}
97159732
else {
97169733
bool ok = false;
97179734
if (PyUnicode_CheckExact(decimal_format)) {
@@ -9727,7 +9744,7 @@ Encoder_init(Encoder *self, PyObject *args, PyObject *kwds)
97279744
if (!ok) {
97289745
PyErr_Format(
97299746
PyExc_ValueError,
9730-
"`decimal_format` must be 'string' or 'number', got %R",
9747+
"`decimal_format` must be 'string', 'number', or a callable, got %R",
97319748
decimal_format
97329749
);
97339750
return -1;
@@ -9790,13 +9807,15 @@ static int
97909807
Encoder_traverse(Encoder *self, visitproc visit, void *arg)
97919808
{
97929809
Py_VISIT(self->enc_hook);
9810+
Py_VISIT(self->decimal_callable);
97939811
return 0;
97949812
}
97959813

97969814
static int
97979815
Encoder_clear(Encoder *self)
97989816
{
97999817
Py_CLEAR(self->enc_hook);
9818+
Py_CLEAR(self->decimal_callable);
98009819
return 0;
98019820
}
98029821

@@ -9871,6 +9890,8 @@ encoder_encode_into_common(
98719890
.mod = self->mod,
98729891
.enc_hook = self->enc_hook,
98739892
.decimal_format = self->decimal_format,
9893+
.decimal_callable = self->decimal_callable,
9894+
.in_decimal_callable = false,
98749895
.uuid_format = self->uuid_format,
98759896
.order = self->order,
98769897
.output_buffer = buf,
@@ -9918,6 +9939,8 @@ encoder_encode_common(
99189939
.mod = self->mod,
99199940
.enc_hook = self->enc_hook,
99209941
.decimal_format = self->decimal_format,
9942+
.decimal_callable = self->decimal_callable,
9943+
.in_decimal_callable = false,
99219944
.uuid_format = self->uuid_format,
99229945
.order = self->order,
99239946
.output_len = 0,
@@ -9975,6 +9998,8 @@ encode_common(
99759998
.mod = mod,
99769999
.enc_hook = enc_hook,
997710000
.decimal_format = DECIMAL_FORMAT_STRING,
10001+
.decimal_callable = NULL,
10002+
.in_decimal_callable = false,
997810003
.uuid_format = UUID_FORMAT_CANONICAL,
997910004
.output_len = 0,
998010005
.max_output_len = ENC_INIT_BUFSIZE,
@@ -10006,7 +10031,13 @@ Encoder_decimal_format(Encoder *self, void *closure) {
1000610031
if (self->decimal_format == DECIMAL_FORMAT_STRING) {
1000710032
return PyUnicode_InternFromString("string");
1000810033
}
10009-
return PyUnicode_InternFromString("number");
10034+
else if (self->decimal_format == DECIMAL_FORMAT_NUMBER) {
10035+
return PyUnicode_InternFromString("number");
10036+
}
10037+
else {
10038+
Py_INCREF(self->decimal_callable);
10039+
return self->decimal_callable;
10040+
}
1001010041
}
1001110042

1001210043
static PyObject*
@@ -12594,11 +12625,14 @@ PyDoc_STRVAR(Encoder__doc__,
1259412625
" A callable to call for objects that aren't supported msgspec types. Takes\n"
1259512626
" the unsupported object and should return a supported object, or raise a\n"
1259612627
" ``NotImplementedError`` if unsupported.\n"
12597-
"decimal_format : {'string', 'number'}, optional\n"
12628+
"decimal_format : {'string', 'number'} or callable, optional\n"
1259812629
" The format to use for encoding `decimal.Decimal` objects. If 'string'\n"
12599-
" they're encoded as strings, if 'number', they're encoded as floats.\n"
12600-
" Defaults to 'string', which is the recommended value since 'number'\n"
12601-
" may result in precision loss when decoding.\n"
12630+
" they're encoded as strings, if 'number', they're encoded as floats. If\n"
12631+
" a callable is provided, it will be called with the `decimal.Decimal`\n"
12632+
" object and must return a JSON-serializable value (but **not** another\n"
12633+
" `decimal.Decimal` or a nested structure containing `decimal.Decimal`).\n"
12634+
" Defaults to 'string', which is the recommended value since 'number' may\n"
12635+
" result in precision loss when decoding.\n"
1260212636
"uuid_format : {'canonical', 'hex', 'bytes'}, optional\n"
1260312637
" The format to use for encoding `uuid.UUID` objects. The 'canonical'\n"
1260412638
" and 'hex' formats encode them as strings with and without hyphens\n"
@@ -13483,11 +13517,21 @@ mpack_encode_decimal(EncoderState *self, PyObject *obj)
1348313517
if (temp == NULL) return -1;
1348413518
out = mpack_encode_str(self, temp);
1348513519
}
13486-
else {
13520+
else if (self->decimal_format == DECIMAL_FORMAT_NUMBER) {
1348713521
temp = PyNumber_Float(obj);
1348813522
if (temp == NULL) return -1;
1348913523
out = mpack_encode_float(self, temp);
1349013524
}
13525+
else {
13526+
if (self->in_decimal_callable) {
13527+
return ms_decimal_format_error();
13528+
}
13529+
temp = PyObject_CallFunctionObjArgs(self->decimal_callable, obj, NULL);
13530+
if (temp == NULL) return -1;
13531+
self->in_decimal_callable = true;
13532+
out = mpack_encode(self, temp);
13533+
self->in_decimal_callable = false;
13534+
}
1349113535
Py_DECREF(temp);
1349213536
return out;
1349313537
}
@@ -13817,12 +13861,14 @@ PyDoc_STRVAR(JSONEncoder__doc__,
1381713861
" A callable to call for objects that aren't supported msgspec types. Takes\n"
1381813862
" the unsupported object and should return a supported object, or raise a\n"
1381913863
" ``NotImplementedError`` if unsupported.\n"
13820-
"decimal_format : {'string', 'number'}, optional\n"
13864+
"decimal_format : {'string', 'number'} or callable, optional\n"
1382113865
" The format to use for encoding `decimal.Decimal` objects. If 'string'\n"
13822-
" they're encoded as strings, if 'number', they're encoded as floats.\n"
13823-
" Defaults to 'string', which is the recommended value since 'number'\n"
13824-
" may result in precision loss when decoding for some JSON library\n"
13825-
" implementations.\n"
13866+
" they're encoded as strings, if 'number', they're encoded as floats. If\n"
13867+
" a callable is provided, it will be called with the `decimal.Decimal`\n"
13868+
" object and must return a JSON-serializable value (but **not** another\n"
13869+
" `decimal.Decimal` or a nested structure containing `decimal.Decimal`).\n"
13870+
" Defaults to 'string', which is the recommended value since 'number' may\n"
13871+
" result in precision loss when decoding for some JSON library implementations.\n"
1382613872
"uuid_format : {'canonical', 'hex'}, optional\n"
1382713873
" The format to use for encoding `uuid.UUID` objects. The 'canonical'\n"
1382813874
" and 'hex' formats encode them as strings with and without hyphens\n"
@@ -14130,7 +14176,23 @@ json_encode_uuid(EncoderState *self, PyObject *obj)
1413014176
static int
1413114177
json_encode_decimal(EncoderState *self, PyObject *obj)
1413214178
{
14133-
PyObject *temp = PyObject_Str(obj);
14179+
PyObject *temp;
14180+
14181+
if (self->decimal_format == DECIMAL_FORMAT_CALLABLE) {
14182+
if (self->in_decimal_callable) {
14183+
return ms_decimal_format_error();
14184+
}
14185+
temp = PyObject_CallFunctionObjArgs(self->decimal_callable, obj, NULL);
14186+
if (temp == NULL) return -1;
14187+
14188+
self->in_decimal_callable = true;
14189+
int out = json_encode(self, temp);
14190+
self->in_decimal_callable = false;
14191+
Py_DECREF(temp);
14192+
return out;
14193+
}
14194+
14195+
temp = PyObject_Str(obj);
1413414196
if (temp == NULL) return -1;
1413514197

1413614198
Py_ssize_t size;
@@ -14142,14 +14204,11 @@ json_encode_decimal(EncoderState *self, PyObject *obj)
1414214204
Py_DECREF(temp);
1414314205
return -1;
1414414206
}
14145-
1414614207
char *p = self->output_buffer_raw + self->output_len;
1414714208
if (MS_LIKELY(decimal_as_string)) *p++ = '"';
1414814209
memcpy(p, buf, size);
1414914210
if (MS_LIKELY(decimal_as_string)) *(p + size) = '"';
14150-
1415114211
self->output_len += required;
14152-
1415314212
Py_DECREF(temp);
1415414213

1415514214
return 0;
@@ -14853,6 +14912,8 @@ JSONEncoder_encode_lines(Encoder *self, PyObject *const *args, Py_ssize_t nargs)
1485314912
.mod = self->mod,
1485414913
.enc_hook = self->enc_hook,
1485514914
.decimal_format = self->decimal_format,
14915+
.decimal_callable = self->decimal_callable,
14916+
.in_decimal_callable = false,
1485614917
.uuid_format = self->uuid_format,
1485714918
.order = self->order,
1485814919
.output_len = 0,
@@ -16486,7 +16547,10 @@ mpack_decode_ext(
1648616547
else if (type->types & MS_TYPE_EXT) {
1648716548
data = PyBytes_FromStringAndSize(data_buf, size);
1648816549
if (data == NULL) return NULL;
16489-
return Ext_New(code, data);
16550+
/* Ext_New doesn't steal the reference to data */
16551+
out = Ext_New(code, data);
16552+
Py_DECREF(data);
16553+
return out;
1649016554
}
1649116555
else if (!(type->types & MS_TYPE_ANY)) {
1649216556
return ms_validation_error("ext", type, path);
@@ -16503,7 +16567,10 @@ mpack_decode_ext(
1650316567
else if (self->ext_hook == NULL) {
1650416568
data = PyBytes_FromStringAndSize(data_buf, size);
1650516569
if (data == NULL) return NULL;
16506-
return Ext_New(code, data);
16570+
/* Ext_New doesn't steal the reference to data */
16571+
out = Ext_New(code, data);
16572+
Py_DECREF(data);
16573+
return out;
1650716574
}
1650816575
else {
1650916576
pycode = PyLong_FromLong(code);

‎src/msgspec/json.pyi‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import decimal
12
from collections.abc import Callable, Iterable
23
from types import GenericAlias
34
from typing import (
@@ -19,19 +20,22 @@ _EncHookSig: TypeAlias = Callable[[Any], Any] | None
1920
_DecHookSig: TypeAlias = Callable[[type, Any], Any] | None
2021
_FloatHookSig: TypeAlias = Callable[[str], Any] | None
2122
_SchemaHookSig: TypeAlias = Callable[[type], dict[str, Any]] | None
23+
_DecimalFormatSig: TypeAlias = (
24+
Callable[[decimal.Decimal], Any] | Literal["string", "number"]
25+
)
2226

2327
@final
2428
class Encoder:
2529
enc_hook: _EncHookSig
26-
decimal_format: Literal["string", "number"]
30+
decimal_format: _DecimalFormatSig
2731
uuid_format: Literal["canonical", "hex"]
2832
order: Literal["deterministic", "sorted"] | None
2933

3034
def __init__(
3135
self,
3236
*,
3337
enc_hook: _EncHookSig = None,
34-
decimal_format: Literal["string", "number"] = "string",
38+
decimal_format: _DecimalFormatSig = "string",
3539
uuid_format: Literal["canonical", "hex"] = "canonical",
3640
order: Literal["deterministic", "sorted"] | None = None,
3741
): ...

‎src/msgspec/msgpack.pyi‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import decimal
12
from collections.abc import Callable
23
from types import GenericAlias
34
from typing import (
@@ -18,6 +19,9 @@ _T = TypeVar("_T")
1819
_EncHookSig: TypeAlias = Callable[[Any], Any] | None
1920
_ExtHookSig: TypeAlias = Callable[[int, memoryview], Any] | None
2021
_DecHookSig: TypeAlias = Callable[[type[Any], Any], Any] | None
22+
_DecimalFormatSig: TypeAlias = (
23+
Callable[[decimal.Decimal], Any] | Literal["string", "number"]
24+
)
2125

2226
@final
2327
class Ext:
@@ -55,14 +59,14 @@ class Decoder(Generic[_T]):
5559
@final
5660
class Encoder:
5761
enc_hook: _EncHookSig
58-
decimal_format: Literal["string", "number"]
62+
decimal_format: _DecimalFormatSig
5963
uuid_format: Literal["canonical", "hex", "bytes"]
6064
order: Literal["deterministic", "sorted"] | None
6165
def __init__(
6266
self,
6367
*,
6468
enc_hook: _EncHookSig = None,
65-
decimal_format: Literal["string", "number"] = "string",
69+
decimal_format: _DecimalFormatSig = "string",
6670
uuid_format: Literal["canonical", "hex", "bytes"] = "canonical",
6771
order: Literal["deterministic", "sorted"] | None = None,
6872
) -> None: ...

‎tests/typing/basic_typing_examples.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# fmt: off
22
from __future__ import annotations
33
import array
4+
from collections.abc import Callable
45
import decimal
56
import pickle
67
from typing import Annotated, Any, Final, Literal
@@ -697,7 +698,9 @@ def check_msgpack_order() -> None:
697698
def check_msgpack_Encoder_decimal_format() -> None:
698699
enc = msgspec.msgpack.Encoder(decimal_format="string")
699700
msgspec.msgpack.Encoder(decimal_format="number")
700-
assert_type(enc.decimal_format, Literal['string', 'number'])
701+
msgspec.msgpack.Encoder(decimal_format=lambda x: str(x))
702+
assert_type(enc.decimal_format, Literal['string', 'number'] | Callable[[decimal.Decimal], Any])
703+
701704

702705

703706
def check_msgpack_Encoder_uuid_format() -> None:
@@ -885,7 +888,8 @@ def check_json_order() -> None:
885888
def check_json_Encoder_decimal_format() -> None:
886889
enc = msgspec.json.Encoder(decimal_format="string")
887890
msgspec.json.Encoder(decimal_format="number")
888-
assert_type(enc.decimal_format, Literal['string', 'number'])
891+
msgspec.json.Encoder(decimal_format=lambda x: str(x))
892+
assert_type(enc.decimal_format, Literal['string', 'number'] | Callable[[decimal.Decimal], Any])
889893

890894

891895
def check_json_Encoder_uuid_format() -> None:

0 commit comments

Comments
 (0)