Skip to content

Commit 8f2d0f1

Browse files
VaggelisDVaggelisD
authored
[mypyc] Add librt.strings.toupper and tolower codepoint primitives (#21553)
6th PR of #21418. This PR introduces two `i32 -> i32` case-conversion helpers, alongside the existing classifiers. **The constraint to flag**: A single i32 holds one codepoint, but some Unicode case mappings expand to multiple e.g `'ß'.upper()` becomes `'SS'`, `'fi'.upper()` becomes `'FI'` etc. For those inputs the primitive _returns the input unchanged_; This is the same split CPython makes between `Py_UNICODE_TOUPPER` (codepoint) and `str.upper()` (string), with the former returning the **first codepoint** of the expansion. Users needing full Unicode case conversion should call `s.upper()` / `s.lower()` on the string, for which we already have mypyc primitives (#20948). For ASCII benchmarks, the codepoint primitives are ~5x faster than their `str` counterparts, avoiding the 1-char allocation.
1 parent f59f649 commit 8f2d0f1

6 files changed

Lines changed: 169 additions & 0 deletions

File tree

‎mypy/typeshed/stubs/librt/librt/strings.pyi‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,3 +48,12 @@ def isdigit(c: i32, /) -> bool: ...
4848
def isalnum(c: i32, /) -> bool: ...
4949
def isalpha(c: i32, /) -> bool: ...
5050
def isidentifier(c: i32, /) -> bool: ...
51+
52+
# Codepoint case conversion. For the rare codepoints whose Unicode
53+
# uppercase / lowercase expands to multiple codepoints (e.g. U+00DF
54+
# uppercases to "SS", U+FB01 to "FI"), returns the input unchanged so
55+
# the signature stays i32 -> i32. Use str.upper() / str.lower() for full
56+
# Unicode case conversion when those cases matter. Negative inputs are
57+
# returned unchanged.
58+
def toupper(c: i32, /) -> i32: ...
59+
def tolower(c: i32, /) -> i32: ...

‎mypyc/lib-rt/strings/librt_strings.c‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1191,6 +1191,18 @@ DEFINE_CP_BOOL_WRAPPER(isalnum, LibRTStrings_IsAlnum)
11911191
DEFINE_CP_BOOL_WRAPPER(isalpha, LibRTStrings_IsAlpha)
11921192
DEFINE_CP_BOOL_WRAPPER(isidentifier, LibRTStrings_IsIdentifier)
11931193

1194+
#define DEFINE_CP_I32_WRAPPER(name, fn) \
1195+
static PyObject* \
1196+
cp_##name(PyObject *module, PyObject *arg) { \
1197+
int32_t c; \
1198+
if (cp_parse_i32(arg, &c) < 0) \
1199+
return NULL; \
1200+
return PyLong_FromLong((long) fn(c)); \
1201+
}
1202+
1203+
DEFINE_CP_I32_WRAPPER(toupper, LibRTStrings_ToUpper)
1204+
DEFINE_CP_I32_WRAPPER(tolower, LibRTStrings_ToLower)
1205+
11941206
static PyMethodDef librt_strings_module_methods[] = {
11951207
{"write_i16_le", (PyCFunction) write_i16_le, METH_FASTCALL,
11961208
PyDoc_STR("Write a 16-bit signed integer to BytesWriter in little-endian format")
@@ -1267,6 +1279,12 @@ static PyMethodDef librt_strings_module_methods[] = {
12671279
{"isidentifier", cp_isidentifier, METH_O,
12681280
PyDoc_STR("Test whether a codepoint (i32) is a valid identifier start (XID_Start).")
12691281
},
1282+
{"toupper", cp_toupper, METH_O,
1283+
PyDoc_STR("Single-codepoint uppercase mapping for a codepoint (i32). Returns the input unchanged if the Unicode uppercase expands to multiple codepoints (e.g. U+00DF uppercases to \"SS\"); use str.upper() for full Unicode case conversion.")
1284+
},
1285+
{"tolower", cp_tolower, METH_O,
1286+
PyDoc_STR("Single-codepoint lowercase mapping for a codepoint (i32). Returns the input unchanged if the Unicode lowercase expands to multiple codepoints; use str.lower() for full Unicode case conversion.")
1287+
},
12701288
{NULL, NULL, 0, NULL}
12711289
};
12721290

‎mypyc/lib-rt/strings/librt_strings.h‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,4 +73,48 @@ static inline bool LibRTStrings_IsIdentifier(int32_t c) {
7373
return r == 1;
7474
}
7575

76+
// Shared slow path for LibRTStrings_ToUpper / _ToLower. Round-trips the
77+
// codepoint through CPython's str.upper / str.lower on a 1-character
78+
// string. When the conversion expands to multiple codepoints (e.g.
79+
// 'ß'.upper() == 'SS') we return the input unchanged so the public
80+
// helpers stay i32 -> i32. Aborts via CPyError_OutOfMemory on allocation
81+
// failure.
82+
static inline int32_t LibRTStrings_ChangeCase_slow(int32_t c, const char *method) {
83+
PyObject *s = PyUnicode_FromOrdinal((int)c);
84+
if (s == NULL) {
85+
CPyError_OutOfMemory();
86+
}
87+
PyObject *u = PyObject_CallMethod(s, method, NULL);
88+
Py_DECREF(s);
89+
if (u == NULL) {
90+
CPyError_OutOfMemory();
91+
}
92+
int32_t result = c;
93+
if (PyUnicode_GET_LENGTH(u) == 1) {
94+
result = (int32_t)PyUnicode_READ_CHAR(u, 0);
95+
}
96+
Py_DECREF(u);
97+
return result;
98+
}
99+
100+
// Uppercase a codepoint. ASCII fast path is `a..z -> A..Z` (subtract 32);
101+
// non-ASCII delegates to str.upper on a 1-character string. Returns the
102+
// input unchanged when uppercasing expands to multiple codepoints.
103+
static inline int32_t LibRTStrings_ToUpper(int32_t c) {
104+
if (c < 0) return c;
105+
if (c >= 'a' && c <= 'z') return c - 32;
106+
if (c < 128) return c;
107+
return LibRTStrings_ChangeCase_slow(c, "upper");
108+
}
109+
110+
// Lowercase a codepoint. ASCII fast path is `A..Z -> a..z` (add 32);
111+
// non-ASCII delegates to str.lower on a 1-character string. Returns the
112+
// input unchanged when lowercasing expands to multiple codepoints.
113+
static inline int32_t LibRTStrings_ToLower(int32_t c) {
114+
if (c < 0) return c;
115+
if (c >= 'A' && c <= 'Z') return c + 32;
116+
if (c < 128) return c;
117+
return LibRTStrings_ChangeCase_slow(c, "lower");
118+
}
119+
76120
#endif // LIBRT_STRINGS_H

‎mypyc/primitives/librt_strings_ops.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -438,3 +438,26 @@
438438
error_kind=ERR_NEVER,
439439
dependencies=[LIBRT_STRINGS],
440440
)
441+
442+
# Codepoint case conversion. When the Unicode uppercase/lowercase of a
443+
# codepoint expands to multiple codepoints (e.g. U+00DF uppercases to "SS",
444+
# U+FB01 to "FI"), returns the input unchanged so the signature stays
445+
# i32 -> i32; callers needing full Unicode case conversion should use
446+
# str.upper() / .lower() instead. Negative inputs are returned unchanged.
447+
function_op(
448+
name="librt.strings.toupper",
449+
arg_types=[int32_rprimitive],
450+
return_type=int32_rprimitive,
451+
c_function_name="LibRTStrings_ToUpper",
452+
error_kind=ERR_NEVER,
453+
dependencies=[LIBRT_STRINGS],
454+
)
455+
456+
function_op(
457+
name="librt.strings.tolower",
458+
arg_types=[int32_rprimitive],
459+
return_type=int32_rprimitive,
460+
c_function_name="LibRTStrings_ToLower",
461+
error_kind=ERR_NEVER,
462+
dependencies=[LIBRT_STRINGS],
463+
)

‎mypyc/test-data/irbuild-librt-strings.test‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -401,3 +401,29 @@ def is_id(c):
401401
L0:
402402
r0 = LibRTStrings_IsIdentifier(c)
403403
return r0
404+
405+
[case testLibrtStringsToUpperIR]
406+
from librt.strings import toupper
407+
from mypy_extensions import i32
408+
409+
def up(c: i32) -> i32:
410+
return toupper(c)
411+
[out]
412+
def up(c):
413+
c, r0 :: i32
414+
L0:
415+
r0 = LibRTStrings_ToUpper(c)
416+
return r0
417+
418+
[case testLibrtStringsToLowerIR]
419+
from librt.strings import tolower
420+
from mypy_extensions import i32
421+
422+
def lo(c: i32) -> i32:
423+
return tolower(c)
424+
[out]
425+
def lo(c):
426+
c, r0 :: i32
427+
L0:
428+
r0 = LibRTStrings_ToLower(c)
429+
return r0

‎mypyc/test-data/run-librt-strings.test‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1490,3 +1490,52 @@ def test_codepoint_classifiers_via_any() -> None:
14901490
f(1 << 40)
14911491
with assertRaises(OverflowError, "codepoint out of i32 range"):
14921492
f(-(1 << 40))
1493+
1494+
[case testLibrtStringsCodepointCaseConversion_librt]
1495+
from typing import Any
1496+
from mypy_extensions import i32
1497+
from librt.strings import toupper, tolower
1498+
1499+
from testutil import assertRaises
1500+
1501+
1502+
def _expect(c: str, method: str) -> int:
1503+
# The contract: i32 -> i32 when conversion yields exactly one codepoint,
1504+
# else return the input unchanged.
1505+
converted = getattr(c, method)()
1506+
if len(converted) == 1:
1507+
return ord(converted)
1508+
return ord(c)
1509+
1510+
1511+
def test_codepoint_case_conversion() -> None:
1512+
# Negative inputs return unchanged.
1513+
for bad in (i32(-1), i32(-113)):
1514+
assert toupper(bad) == bad
1515+
assert tolower(bad) == bad
1516+
# Agree with str.upper / str.lower across the full Unicode range
1517+
# whenever the conversion is single-codepoint; otherwise return input.
1518+
for i in range(0x110000):
1519+
c = chr(i)
1520+
o = ord(c)
1521+
assert toupper(o) == _expect(c, "upper")
1522+
assert tolower(o) == _expect(c, "lower")
1523+
1524+
1525+
def test_codepoint_case_conversion_via_any() -> None:
1526+
# Iterate to force generic dispatch through the PyMethodDef wrapper.
1527+
for fn, in_cp, out_cp in (
1528+
(toupper, ord("a"), ord("A")),
1529+
(toupper, ord("A"), ord("A")),
1530+
(tolower, ord("Z"), ord("z")),
1531+
(tolower, ord("z"), ord("z")),
1532+
):
1533+
f: Any = fn
1534+
assert f(in_cp) == out_cp
1535+
# Negative values are valid i32, returned unchanged.
1536+
assert f(-1) == -1
1537+
# Inputs outside i32 range raise OverflowError through the wrapper.
1538+
with assertRaises(OverflowError, "codepoint out of i32 range"):
1539+
f(1 << 40)
1540+
with assertRaises(OverflowError, "codepoint out of i32 range"):
1541+
f(-(1 << 40))

0 commit comments

Comments
 (0)