Skip to content

Commit 965dd31

Browse files
VaggelisDVaggelisD
authored
[mypyc] Add librt.strings.isidentifier codepoint primitive (#21522)
5th PR of #21418
1 parent 47ca4c2 commit 965dd31

9 files changed

Lines changed: 90 additions & 58 deletions

File tree

‎mypy/typeshed/stubs/librt/librt/strings.pyi‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,3 +47,4 @@ def isspace(c: i32, /) -> bool: ...
4747
def isdigit(c: i32, /) -> bool: ...
4848
def isalnum(c: i32, /) -> bool: ...
4949
def isalpha(c: i32, /) -> bool: ...
50+
def isidentifier(c: i32, /) -> bool: ...

‎mypyc/ir/deps.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,5 +116,4 @@ def get_header(self) -> str:
116116
STRING_WRITER_EXTRA_OPS: Final = SourceDep("stringwriter_extra_ops.c")
117117
BYTEARRAY_EXTRA_OPS: Final = SourceDep("bytearray_extra_ops.c")
118118
STR_EXTRA_OPS: Final = SourceDep("str_extra_ops.c")
119-
CODEPOINT_EXTRA_OPS: Final = SourceDep("codepoint_extra_ops.c")
120119
VECS_EXTRA_OPS: Final = SourceDep("vecs_extra_ops.c")

‎mypyc/lib-rt/codepoint_extra_ops.c‎

Lines changed: 0 additions & 8 deletions
This file was deleted.

‎mypyc/lib-rt/codepoint_extra_ops.h‎

Lines changed: 0 additions & 28 deletions
This file was deleted.

‎mypyc/lib-rt/strings/librt_strings.c‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
#include <Python.h>
55
#include <stdint.h>
66
#include "CPy.h"
7-
#include "codepoint_extra_ops.h"
87
#include "librt_strings.h"
98

109
#define CPY_BOOL_ERROR 2
@@ -1154,15 +1153,11 @@ read_f64_be(PyObject *module, PyObject *const *args, size_t nargs) {
11541153
return PyFloat_FromDouble(CPyBytes_ReadF64BEUnsafe(data + index));
11551154
}
11561155

1157-
// Codepoint classification helpers exposed to interpreted callers.
1158-
// The C-side names are prefixed `cp_` to avoid colliding with libc's
1159-
// <ctype.h> isspace / isdigit / etc. Compiled callers go through the
1160-
// LibRTStrings_* static inlines in codepoint_extra_ops.h instead.
1161-
//
1162-
// All wrappers parse a single int argument as i32 (codepoint) and
1163-
// dispatch to the corresponding LibRTStrings_* function. The parse
1164-
// step accepts any int but rejects values outside the i32 range with
1165-
// OverflowError, matching the input domain of the compiled fast path.
1156+
// Python-level wrappers (`cp_*`) for interpreted callers. The C-side names
1157+
// are prefixed `cp_` to avoid colliding with libc's <ctype.h> isspace etc.
1158+
// The LibRTStrings_Is* helpers themselves are static inline in librt_strings.h
1159+
// so they compile directly into mypyc-emitted code with no capsule
1160+
// indirection.
11661161

11671162
// Parse a Python int as i32 codepoint. Returns 0 on success and writes
11681163
// the value to *out; returns -1 on error with a Python exception set.
@@ -1194,6 +1189,7 @@ DEFINE_CP_BOOL_WRAPPER(isspace, LibRTStrings_IsSpace)
11941189
DEFINE_CP_BOOL_WRAPPER(isdigit, LibRTStrings_IsDigit)
11951190
DEFINE_CP_BOOL_WRAPPER(isalnum, LibRTStrings_IsAlnum)
11961191
DEFINE_CP_BOOL_WRAPPER(isalpha, LibRTStrings_IsAlpha)
1192+
DEFINE_CP_BOOL_WRAPPER(isidentifier, LibRTStrings_IsIdentifier)
11971193

11981194
static PyMethodDef librt_strings_module_methods[] = {
11991195
{"write_i16_le", (PyCFunction) write_i16_le, METH_FASTCALL,
@@ -1268,6 +1264,9 @@ static PyMethodDef librt_strings_module_methods[] = {
12681264
{"isalpha", cp_isalpha, METH_O,
12691265
PyDoc_STR("Test whether a codepoint (i32) is a Unicode letter.")
12701266
},
1267+
{"isidentifier", cp_isidentifier, METH_O,
1268+
PyDoc_STR("Test whether a codepoint (i32) is a valid identifier start (XID_Start).")
1269+
},
12711270
{NULL, NULL, 0, NULL}
12721271
};
12731272

‎mypyc/lib-rt/strings/librt_strings.h‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33

44
#include <Python.h>
55
#include <stdbool.h>
6+
#include <stdint.h>
7+
#include "CPy.h"
68
#include "librt_strings_common.h"
79

810
// ABI version -- only an exact match is compatible. This will only be changed in
@@ -28,4 +30,47 @@ typedef struct {
2830
char data[WRITER_EMBEDDED_BUF_LEN]; // Default buffer
2931
} StringWriterObject;
3032

33+
// Codepoint classification helpers. Inputs are signed i32 for compatibility
34+
// with mypyc's int32_rprimitive; negative values are non-codepoints and
35+
// return false. Defined `static inline` so they compile statically into
36+
// both the librt.strings module and any mypyc-compiled extension that
37+
// includes this header, avoiding the capsule indirection that would dwarf
38+
// the work of a single Py_UNICODE_IS* macro call.
39+
40+
static inline bool LibRTStrings_IsSpace(int32_t c) {
41+
return c >= 0 && Py_UNICODE_ISSPACE((Py_UCS4)c);
42+
}
43+
44+
static inline bool LibRTStrings_IsDigit(int32_t c) {
45+
return c >= 0 && Py_UNICODE_ISDIGIT((Py_UCS4)c);
46+
}
47+
48+
static inline bool LibRTStrings_IsAlnum(int32_t c) {
49+
return c >= 0 && Py_UNICODE_ISALNUM((Py_UCS4)c);
50+
}
51+
52+
static inline bool LibRTStrings_IsAlpha(int32_t c) {
53+
return c >= 0 && Py_UNICODE_ISALPHA((Py_UCS4)c);
54+
}
55+
56+
// True if c could start a valid identifier (XID_Start, per PEP 3131).
57+
// ASCII fast path covers `[A-Za-z_]`; non-ASCII delegates to CPython's
58+
// PyUnicode_IsIdentifier on a 1-character string. Aborts via
59+
// CPyError_OutOfMemory on allocation failure to keep this ERR_NEVER.
60+
static inline bool LibRTStrings_IsIdentifier(int32_t c) {
61+
if (c < 0) return false;
62+
if (c < 128) {
63+
return (c >= 'a' && c <= 'z')
64+
|| (c >= 'A' && c <= 'Z')
65+
|| c == '_';
66+
}
67+
PyObject *s = PyUnicode_FromOrdinal((int)c);
68+
if (s == NULL) {
69+
CPyError_OutOfMemory();
70+
}
71+
int r = PyUnicode_IsIdentifier(s);
72+
Py_DECREF(s);
73+
return r == 1;
74+
}
75+
3176
#endif // LIBRT_STRINGS_H

‎mypyc/primitives/librt_strings_ops.py‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,4 @@
1-
from mypyc.ir.deps import (
2-
BYTES_WRITER_EXTRA_OPS,
3-
CODEPOINT_EXTRA_OPS,
4-
LIBRT_STRINGS,
5-
STRING_WRITER_EXTRA_OPS,
6-
)
1+
from mypyc.ir.deps import BYTES_WRITER_EXTRA_OPS, LIBRT_STRINGS, STRING_WRITER_EXTRA_OPS
72
from mypyc.ir.ops import ERR_MAGIC, ERR_MAGIC_OVERLAPPING, ERR_NEVER
83
from mypyc.ir.rtypes import (
94
bool_rprimitive,
@@ -402,7 +397,7 @@
402397
return_type=bool_rprimitive,
403398
c_function_name="LibRTStrings_IsSpace",
404399
error_kind=ERR_NEVER,
405-
dependencies=[LIBRT_STRINGS, CODEPOINT_EXTRA_OPS],
400+
dependencies=[LIBRT_STRINGS],
406401
)
407402

408403
function_op(
@@ -411,7 +406,7 @@
411406
return_type=bool_rprimitive,
412407
c_function_name="LibRTStrings_IsDigit",
413408
error_kind=ERR_NEVER,
414-
dependencies=[LIBRT_STRINGS, CODEPOINT_EXTRA_OPS],
409+
dependencies=[LIBRT_STRINGS],
415410
)
416411

417412
function_op(
@@ -420,7 +415,7 @@
420415
return_type=bool_rprimitive,
421416
c_function_name="LibRTStrings_IsAlnum",
422417
error_kind=ERR_NEVER,
423-
dependencies=[LIBRT_STRINGS, CODEPOINT_EXTRA_OPS],
418+
dependencies=[LIBRT_STRINGS],
424419
)
425420

426421
function_op(
@@ -429,5 +424,17 @@
429424
return_type=bool_rprimitive,
430425
c_function_name="LibRTStrings_IsAlpha",
431426
error_kind=ERR_NEVER,
432-
dependencies=[LIBRT_STRINGS, CODEPOINT_EXTRA_OPS],
427+
dependencies=[LIBRT_STRINGS],
428+
)
429+
430+
# isidentifier checks XID_Start semantics for a single codepoint, matching
431+
# str.isidentifier() on a 1-character string. The non-ASCII path allocates
432+
# and aborts via CPyError_OutOfMemory on failure, so this stays ERR_NEVER.
433+
function_op(
434+
name="librt.strings.isidentifier",
435+
arg_types=[int32_rprimitive],
436+
return_type=bool_rprimitive,
437+
c_function_name="LibRTStrings_IsIdentifier",
438+
error_kind=ERR_NEVER,
439+
dependencies=[LIBRT_STRINGS],
433440
)

‎mypyc/test-data/irbuild-librt-strings.test‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -387,3 +387,17 @@ def is_a(c):
387387
L0:
388388
r0 = LibRTStrings_IsAlpha(c)
389389
return r0
390+
391+
[case testLibrtStringsIsIdentifierIR]
392+
from librt.strings import isidentifier
393+
from mypy_extensions import i32
394+
395+
def is_id(c: i32) -> bool:
396+
return isidentifier(c)
397+
[out]
398+
def is_id(c):
399+
c :: i32
400+
r0 :: bool
401+
L0:
402+
r0 = LibRTStrings_IsIdentifier(c)
403+
return r0

‎mypyc/test-data/run-librt-strings.test‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1443,7 +1443,7 @@ def test_new_without_init_is_usable() -> None:
14431443
[case testLibrtStringsCodepointClassifiers_librt]
14441444
from typing import Any
14451445
from mypy_extensions import i32
1446-
from librt.strings import isspace, isdigit, isalnum, isalpha
1446+
from librt.strings import isspace, isdigit, isalnum, isalpha, isidentifier
14471447

14481448
from testutil import assertRaises
14491449

@@ -1455,6 +1455,7 @@ def test_codepoint_classifiers() -> None:
14551455
assert not isdigit(bad)
14561456
assert not isalnum(bad)
14571457
assert not isalpha(bad)
1458+
assert not isidentifier(bad)
14581459
# Verify each codepoint primitive agrees with the matching str method
14591460
# across all Unicode codepoints, including the ord(chr(i)) round-trip.
14601461
# Any forces generic dispatch on the str side.
@@ -1466,6 +1467,7 @@ def test_codepoint_classifiers() -> None:
14661467
assert isdigit(o) == isdigit(i) == a.isdigit()
14671468
assert isalnum(o) == isalnum(i) == a.isalnum()
14681469
assert isalpha(o) == isalpha(i) == a.isalpha()
1470+
assert isidentifier(o) == isidentifier(i) == a.isidentifier()
14691471

14701472

14711473
def test_codepoint_classifiers_via_any() -> None:
@@ -1476,6 +1478,7 @@ def test_codepoint_classifiers_via_any() -> None:
14761478
(isdigit, "5", "a"),
14771479
(isalnum, "A", " "),
14781480
(isalpha, "A", " "),
1481+
(isidentifier, "A", "0"),
14791482
):
14801483
f: Any = fn
14811484
assert f(ord(true_input)) is True

0 commit comments

Comments
 (0)