Skip to content

Commit e709f3b

Browse files
committed
Support large integer stringification thresholds in both encoders
1 parent fbc057f commit e709f3b

7 files changed

Lines changed: 94 additions & 16 deletions

File tree

‎AGENTS.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -240,3 +240,17 @@ per-module state in 4.0 was driven by free-threading and
240240
subinterpreter isolation, not by abi3 readiness. If you're adding a
241241
new feature, don't feel obligated to stay limited-API-compatible —
242242
performance-sensitive code wins.
243+
244+
## Integer stringification thresholds
245+
246+
Both encoders accept positive `int_as_string_bitcount` values beyond native
247+
integer widths. Keep the C min/max comparison path for small thresholds;
248+
large thresholds compare magnitude bit lengths without allocating `2**n`.
249+
`large_int_bitcount` is an owned reference in `JSON_ENCODER_OBJECT_FIELDS`,
250+
so traversal and cleanup must include it. The boundary and huge-threshold tests
251+
in `test_bitsize_int_as_string.py` run under both paths via `_cibw_runner`.
252+
253+
When creating a release venv, verify `sysconfig.get_config_var("Py_DEBUG")`
254+
and the resolved interpreter path. A version-only uv request can select an
255+
already-installed debug interpreter. Use an explicit release executable path
256+
when checking release-only compiler flags, rather than repeating the debug suite.

‎CHANGES.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
Unreleased
2+
3+
* Support int_as_string_bitcount thresholds of 64 bits and above in the C
4+
encoder, and avoid allocating power-of-two boundaries for large thresholds.
5+
https://github.com/simplejson/simplejson/issues/306
6+
17
Version 4.1.2 released 2026-08-26
28

39
* Fix control character error position when content precedes the control char

‎index.rst‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,10 @@ Basic Usage
310310
:class:`int` ``2**n`` and higher or lower than ``-2**n`` will be encoded as strings. This is to
311311
avoid the rounding that happens in Javascript otherwise. Note that this
312312
option loses type information, so use with extreme caution.
313+
Thresholds of 64 bits and above are supported by both encoders, including
314+
128-bit integers. Very large thresholds do not allocate a power-of-two
315+
boundary.
316+
313317
See also *bigint_as_string* (which is equivalent to `int_as_string_bitcount=53`).
314318

315319
.. note::

‎simplejson/_speedups.c‎

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -301,6 +301,7 @@ typedef struct _PyEncoderObject {
301301
int iterable_as_array;
302302
PyObject *max_long_size;
303303
PyObject *min_long_size;
304+
PyObject *large_int_bitcount;
304305
PyObject *item_sort_key;
305306
PyObject *item_sort_kw;
306307
int for_json;
@@ -327,6 +328,7 @@ typedef struct _PyEncoderObject {
327328
X(item_sort_key) \
328329
X(max_long_size) \
329330
X(min_long_size) \
331+
X(large_int_bitcount) \
330332
X(Decimal)
331333

332334
static PyMemberDef encoder_members[] = {
@@ -708,6 +710,36 @@ maybe_quote_bigint(PyEncoderObject* s, PyObject *encoded, PyObject *obj)
708710
int ge, le;
709711
PyObject *quoted;
710712

713+
if (s->large_int_bitcount != NULL) {
714+
size_t nbits;
715+
PyObject *integer, *bits;
716+
integer = PyNumber_Long(obj);
717+
if (integer == NULL) {
718+
Py_DECREF(encoded);
719+
return NULL;
720+
}
721+
nbits = _PyLong_NumBits(integer);
722+
Py_DECREF(integer);
723+
if (nbits == (size_t)-1 && PyErr_Occurred()) {
724+
Py_DECREF(encoded);
725+
return NULL;
726+
}
727+
bits = PyLong_FromSize_t(nbits);
728+
if (bits == NULL) {
729+
Py_DECREF(encoded);
730+
return NULL;
731+
}
732+
ge = PyObject_RichCompareBool(bits, s->large_int_bitcount, Py_GT);
733+
Py_DECREF(bits);
734+
if (ge < 0) {
735+
Py_DECREF(encoded);
736+
return NULL;
737+
}
738+
if (!ge)
739+
return encoded;
740+
goto quote;
741+
}
742+
711743
/* int_as_string_bitcount is not set: fast path, return as-is. */
712744
if (s->max_long_size == Py_None || s->min_long_size == Py_None)
713745
return encoded;
@@ -725,6 +757,7 @@ maybe_quote_bigint(PyEncoderObject* s, PyObject *encoded, PyObject *obj)
725757
if (!(ge || le))
726758
return encoded;
727759

760+
quote:
728761
#if PY_MAJOR_VERSION >= 3
729762
quoted = PyUnicode_FromFormat("\"%U\"", encoded);
730763
#else
@@ -2653,10 +2686,11 @@ encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
26532686
goto bail;
26542687
if (PyInt_Check(int_as_string_bitcount) || PyLong_Check(int_as_string_bitcount)) {
26552688
static const unsigned long long_long_bitsize = sizeof(long long) * CHAR_BIT;
2656-
long int_as_string_bitcount_val = PyLong_AsLong(int_as_string_bitcount);
2689+
int overflow = 0;
2690+
long int_as_string_bitcount_val = PyLong_AsLongAndOverflow(int_as_string_bitcount, &overflow);
26572691
if (int_as_string_bitcount_val == -1 && PyErr_Occurred())
26582692
goto bail;
2659-
if (int_as_string_bitcount_val > 0 && int_as_string_bitcount_val < (long)long_long_bitsize) {
2693+
if (!overflow && int_as_string_bitcount_val > 0 && int_as_string_bitcount_val < (long)long_long_bitsize) {
26602694
int n = (int)int_as_string_bitcount_val;
26612695
/* Compute 2^n as unsigned (well-defined for n < 64) and
26622696
* -(2^n) as signed without UB. Naive "-1LL << n" is a
@@ -2672,10 +2706,19 @@ encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
26722706
goto bail;
26732707
}
26742708
}
2709+
else if (overflow > 0 || int_as_string_bitcount_val > 0) {
2710+
/* Compare magnitudes without allocating a 2**n boundary. */
2711+
s->large_int_bitcount = PyNumber_Long(int_as_string_bitcount);
2712+
if (s->large_int_bitcount == NULL)
2713+
goto bail;
2714+
Py_INCREF(Py_None);
2715+
s->max_long_size = Py_None;
2716+
Py_INCREF(Py_None);
2717+
s->min_long_size = Py_None;
2718+
}
26752719
else {
2676-
PyErr_Format(PyExc_TypeError,
2677-
"int_as_string_bitcount (%ld) must be greater than 0 and less than the number of bits of a `long long` type (%lu bits)",
2678-
int_as_string_bitcount_val, long_long_bitsize);
2720+
PyErr_SetString(PyExc_TypeError,
2721+
"int_as_string_bitcount must be a positive integer");
26792722
goto bail;
26802723
}
26812724
}

‎simplejson/encoder.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -516,9 +516,7 @@ def _encode_int(value):
516516
value = int(value)
517517
if (
518518
skip_quoting or
519-
(-1 << _int_as_string_bitcount)
520-
< value <
521-
(1 << _int_as_string_bitcount)
519+
value.bit_length() <= _int_as_string_bitcount
522520
):
523521
return str(value)
524522
return '"' + str(value) + '"'

‎simplejson/tests/test_bitsize_int_as_string.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,3 +118,23 @@ def test_boundary_at_max_bitcount(self):
118118
str(v),
119119
json.loads(json.dumps(v, int_as_string_bitcount=n)),
120120
"n=%d v=%d should be stringified" % (n, v))
121+
122+
123+
def test_large_bitcounts(self):
124+
for n in (64, 65, 127, 128, 256, 1024):
125+
boundary = 1 << n
126+
values = [0, boundary - 1, boundary, boundary + 1,
127+
-boundary + 1, -boundary, -boundary - 1]
128+
expected = [0, boundary - 1, str(boundary), str(boundary + 1),
129+
-boundary + 1, str(-boundary), str(-boundary - 1)]
130+
for indent in (None, 2):
131+
self.assertEqual(expected, json.loads(json.dumps(
132+
values, int_as_string_bitcount=n, indent=indent)))
133+
self.assertEqual({'value': expected}, json.loads(json.dumps(
134+
{'value': values}, int_as_string_bitcount=n, indent=indent)))
135+
136+
def test_huge_bitcount_does_not_build_boundary(self):
137+
# A threshold can exceed native integer sizes without allocating 2**n.
138+
for n in (2 ** 32 + 31, 2 ** 100):
139+
self.assertEqual('[0, 1, -1]', json.dumps(
140+
[0, 1, -1], int_as_string_bitcount=n))

‎simplejson/tests/test_speedups.py‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
import simplejson
66
from simplejson import encoder, decoder, scanner
7-
from simplejson.compat import PY3, long_type, b
7+
from simplejson.compat import PY3, b
88
from simplejson.tests._helpers import has_speedups, skip_if_speedups_missing
99

1010

@@ -82,13 +82,6 @@ def test(name):
8282
self.assertRaises(ZeroDivisionError, test, 'ignore_nan')
8383
self.assertRaises(ZeroDivisionError, test, 'iterable_as_array')
8484

85-
@skip_if_speedups_missing
86-
def test_int_as_string_bitcount_overflow(self):
87-
long_count = long_type(2)**32+31
88-
def test():
89-
encoder.JSONEncoder(int_as_string_bitcount=long_count).encode(0)
90-
self.assertRaises((TypeError, OverflowError), test)
91-
9285
if PY3:
9386
@skip_if_speedups_missing
9487
def test_bad_encoding(self):

0 commit comments

Comments
 (0)