Skip to content

Commit 2c23505

Browse files
vstinnervstinner
authored
[3.15] gh-158445: Detect invalid UCS4 characters in debug mode (#158502) (#159524)
* gh-158445: Detect invalid UCS4 characters in debug mode (#158502) In debug mode, PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND) and PyUnicodeWriter_WriteUCS4() now raise a SystemError if a character is not in range [U+0000; U+10ffff], instead of creating an invalid str object. In release mode, PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND) raises a SystemError if the size is 1. * In debug mode, unicode_char() no longer fails with an assertion error if the character is invalid. Instead, raise SystemError. * Add tests on invalid UCS-4 characters. * Add _testinternalcapi._Py_MAX_UNICODE. * Add unicode_invalid_character() helper function. (cherry picked from commit a14a4a6) * gh-158445: Skip invalid UCS-4 tests if Python is built with assertions (#158604) If Python is built in release mode with assertions, PyUnicode_FromKindAndData() and PyUnicodeWriter_Finish() fail with an assertion error on invalid characters. (cherry picked from commit 35bdbca)
1 parent bcbe9bc commit 2c23505

4 files changed

Lines changed: 142 additions & 30 deletions

File tree

‎Doc/c-api/unicode.rst‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -425,6 +425,10 @@ APIs:
425425
the UCS1 range, it will be transformed into UCS1
426426
(:c:macro:`PyUnicode_1BYTE_KIND`).
427427
428+
All characters must be in range [U+0000; U+10ffff]. If *kind* is
429+
:c:macro:`PyUnicode_4BYTE_KIND` and the string contains invalid characters,
430+
the behavior is undefined.
431+
428432
.. versionadded:: 3.3
429433
430434
@@ -1893,10 +1897,13 @@ object.
18931897
18941898
.. c:function:: int PyUnicodeWriter_WriteUCS4(PyUnicodeWriter *writer, const Py_UCS4 *str, Py_ssize_t size)
18951899
1896-
Writer the UCS4 string *str* into *writer*.
1900+
Write the UCS4 string *str* into *writer*.
18971901
18981902
*size* is a number of UCS4 characters.
18991903
1904+
All characters must be in range [U+0000; U+10ffff]. If the string contains
1905+
invalid characters, the behavior is undefined.
1906+
19001907
On success, return ``0``.
19011908
On error, set an exception, leave the writer unchanged, and return ``-1``.
19021909

‎Lib/test/test_capi/test_unicode.py‎

Lines changed: 71 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,11 @@
1717
from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, SIZEOF_WCHAR_T
1818

1919

20+
MAX_UNICODE = _testinternalcapi._Py_MAX_UNICODE
21+
# The first invalid character after MAX_UNICODE
22+
INVALID_CHAR = MAX_UNICODE + 1
23+
# Maximum invalid character which fits into 32-bit Py_UCS4
24+
MAX_INVALID_CHAR = 0xFFFF_FFFF
2025
NULL = None
2126

2227
class Str(str):
@@ -39,6 +44,12 @@ def __str__(self):
3944
SSTATE_INTERNED_IMMORTAL_STATIC = 3
4045

4146

47+
def assert_invalid_string(testcase, text):
48+
# Check that a Unicode string contains invalid characters:
49+
# not in range [U+0000; U+10ffff]
50+
testcase.assertRaises(SystemError, list, text)
51+
52+
4253
class CAPITest(unittest.TestCase):
4354

4455
def _test_check(self, check, *, exact):
@@ -71,14 +82,14 @@ def test_new(self):
7182
self.assertEqual(new(0, maxchar), '')
7283
self.assertEqual(new(5, maxchar), chr(maxchar)*5)
7384
self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX, maxchar)
74-
self.assertEqual(new(0, 0x110000), '')
85+
self.assertEqual(new(0, INVALID_CHAR), '')
7586
self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2, 0x4f60)
7687
self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2+1, 0x4f60)
7788
self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2, 0x1f600)
7889
self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2+1, 0x1f600)
7990
self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//4, 0x1f600)
8091
self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//4+1, 0x1f600)
81-
self.assertRaises(SystemError, new, 5, 0x110000)
92+
self.assertRaises(SystemError, new, 5, INVALID_CHAR)
8293
self.assertRaises(SystemError, new, -1, 0)
8394
self.assertRaises(SystemError, new, PY_SSIZE_T_MIN, 0)
8495

@@ -113,7 +124,7 @@ def test_fill(self):
113124
s = strings[0]
114125
self.assertRaises(IndexError, fill, s, -1, 0, 0x78)
115126
self.assertRaises(IndexError, fill, s, PY_SSIZE_T_MIN, 0, 0x78)
116-
self.assertRaises(ValueError, fill, s, 0, 0, 0x110000)
127+
self.assertRaises(ValueError, fill, s, 0, 0, INVALID_CHAR)
117128
self.assertRaises(SystemError, fill, b'abc', 0, 0, 0x78)
118129
self.assertRaises(SystemError, fill, [], 0, 0, 0x78)
119130
# CRASHES fill(s, 0, NULL, 0, 0)
@@ -127,7 +138,7 @@ def _test_writechar(self, writechar, *, check):
127138
'\U0001f600\U0001f601\U0001f602'
128139
]
129140
# one character for every kind + out of range code
130-
chars = [0x78, 0xa9, 0x20ac, 0x1f638, 0x110000]
141+
chars = [0x78, 0xa9, 0x20ac, 0x1f638, INVALID_CHAR]
131142
for i, s in enumerate(strings):
132143
for j, c in enumerate(chars):
133144
if j <= i:
@@ -299,7 +310,26 @@ def test_fromkindanddata(self):
299310
self.assertRaises(ValueError, fromkindanddata, 1, NULL, -1)
300311
self.assertRaises(ValueError, fromkindanddata, 1, NULL, PY_SSIZE_T_MIN)
301312
# CRASHES fromkindanddata(1, NULL, 1)
302-
# CRASHES fromkindanddata(4, b'\xff\xff\xff\xff')
313+
314+
# Test invalid UCS-4 string. Create an invalid string in release mode,
315+
# or raise SystemError in debug mode.
316+
built_with_assert = support.built_with_c_assertions()
317+
for invalid_char in (INVALID_CHAR, MAX_INVALID_CHAR):
318+
with self.subTest(invalid_char=invalid_char):
319+
# Test single character
320+
ucs4_char = invalid_char.to_bytes(4, byteorder=sys.byteorder)
321+
self.assertRaises(SystemError, fromkindanddata, 4, ucs4_char)
322+
323+
# Test multiple characters
324+
s = 'valid'.encode(enc4) + ucs4_char
325+
if support.Py_DEBUG:
326+
self.assertRaises(SystemError, fromkindanddata, 4, s)
327+
elif not built_with_assert :
328+
result = fromkindanddata(4, s)
329+
assert_invalid_string(self, result)
330+
else:
331+
# PyUnicode_FromKindAndData() fails with an assertion error
332+
pass
303333

304334
def test_substring(self):
305335
"""Test PyUnicode_Substring()"""
@@ -444,7 +474,7 @@ def check_format(expected, format, *args):
444474
check_format('\U0010ffff',
445475
b'%c', c_int(0x10ffff))
446476
with self.assertRaises(OverflowError):
447-
PyUnicode_FromFormat(b'%c', c_int(0x110000))
477+
PyUnicode_FromFormat(b'%c', c_int(INVALID_CHAR))
448478
# Issue #18183
449479
check_format('\U00010000\U00100000',
450480
b'%c%c', c_int(0x10000), c_int(0x100000))
@@ -1013,7 +1043,7 @@ def test_fromordinal(self):
10131043
self.assertEqual(fromordinal(0x20ac), '\u20ac')
10141044
self.assertEqual(fromordinal(0x1f600), '\U0001f600')
10151045

1016-
self.assertRaises(ValueError, fromordinal, 0x110000)
1046+
self.assertRaises(ValueError, fromordinal, INVALID_CHAR)
10171047
self.assertRaises(ValueError, fromordinal, -1)
10181048

10191049
def test_asutf8(self):
@@ -1365,8 +1395,8 @@ def test_findchar(self):
13651395
self.assertEqual(unicode_findchar(str, ord(ch), 0, len(str), -1), i)
13661396

13671397
str = "!>_<!"
1368-
self.assertEqual(unicode_findchar(str, 0x110000, 0, len(str), 1), -1)
1369-
self.assertEqual(unicode_findchar(str, 0x110000, 0, len(str), -1), -1)
1398+
self.assertEqual(unicode_findchar(str, INVALID_CHAR, 0, len(str), 1), -1)
1399+
self.assertEqual(unicode_findchar(str, INVALID_CHAR, 0, len(str), -1), -1)
13701400
# start < end
13711401
self.assertEqual(unicode_findchar(str, ord('!'), 1, len(str)+1, 1), 4)
13721402
self.assertEqual(unicode_findchar(str, ord('!'), 1, PY_SSIZE_T_MAX, 1), 4)
@@ -1858,8 +1888,8 @@ def test_write_char(self):
18581888
writer.write_char(ord('$'))
18591889
writer.write_char(0x20ac)
18601890
writer.write_char(0x10_ffff)
1861-
self.assertRaises(ValueError, writer.write_char, 0x11_0000)
1862-
self.assertRaises(ValueError, writer.write_char, 0xFFFF_FFFF)
1891+
self.assertRaises(ValueError, writer.write_char, INVALID_CHAR)
1892+
self.assertRaises(ValueError, writer.write_char, MAX_INVALID_CHAR)
18631893
self.assertEqual(writer.finish(),
18641894
"\0$\u20AC\U0010FFFF")
18651895

@@ -2024,15 +2054,41 @@ def test_ucs4(self):
20242054
writer.write_ucs4("pair\uD83D\uDC0D".encode(encoding, 'surrogatepass'))
20252055
writer.write_char(ord("-"))
20262056
writer.write_ucs4("null[\0]".encode(encoding), 7)
2027-
invalid = (b'\x00\x00\x11\x00' if sys.byteorder == 'little' else
2028-
b'\x00\x11\x00\x00')
2029-
# CRASHES writer.write_ucs4("invalid".encode(encoding) + invalid)
20302057
writer.write_ucs4(NULL, 0)
20312058
# CRASHES writer.write_ucs4(NULL, 1)
20322059
self.assertEqual(writer.finish(),
20332060
"lone\udc80-pair\ud83d\udc0d-null[\x00]")
20342061

2035-
# invalid size
2062+
# Invalid UCS-4 characters. Create an invalid string in release mode,
2063+
# or raise SystemError in debug mode.
2064+
built_with_assert = support.built_with_c_assertions()
2065+
writer = self.create_writer(0)
2066+
for invalid_char in (INVALID_CHAR, MAX_INVALID_CHAR):
2067+
with self.subTest(invalid_char=invalid_char):
2068+
# Test single character
2069+
ucs4_char = invalid_char.to_bytes(4, byteorder=sys.byteorder)
2070+
if support.Py_DEBUG:
2071+
self.assertRaises(SystemError, writer.write_ucs4, ucs4_char)
2072+
else:
2073+
writer.write_ucs4(ucs4_char)
2074+
2075+
# Test multiple characters
2076+
s = 'valid'.encode(encoding) + ucs4_char
2077+
if support.Py_DEBUG:
2078+
self.assertRaises(SystemError, writer.write_ucs4, s)
2079+
else:
2080+
writer.write_ucs4(s)
2081+
2082+
if support.Py_DEBUG:
2083+
self.assertEqual(writer.finish(), '')
2084+
elif not built_with_assert:
2085+
result = writer.finish()
2086+
assert_invalid_string(self, result)
2087+
else:
2088+
# PyUnicodeWriter_Finish() fails with an assertion error
2089+
pass
2090+
2091+
# Invalid size
20362092
writer = self.create_writer(0)
20372093
with self.assertRaises(ValueError):
20382094
writer.write_ucs4("text".encode(encoding), -1)

‎Modules/_testinternalcapi.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3500,6 +3500,10 @@ module_exec(PyObject *module)
35003500
}
35013501
PyModule_AddObject(module, "SelfInterruptingContextManager", (PyObject *)&SelfInterruptingContextManager_Type);
35023502

3503+
if (PyModule_AddIntMacro(module, _Py_MAX_UNICODE) < 0) {
3504+
return 1;
3505+
}
3506+
35033507
return 0;
35043508
}
35053509

‎Objects/unicodeobject.c‎

Lines changed: 59 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1583,6 +1583,16 @@ PyUnicode_CopyCharacters(PyObject *to, Py_ssize_t to_start,
15831583
return how_many;
15841584
}
15851585

1586+
1587+
static void
1588+
unicode_invalid_character(PyObject *exc, Py_UCS4 ch)
1589+
{
1590+
PyErr_Format(exc,
1591+
"character U+%x is not in range [U+0000; U+%x]",
1592+
ch, MAX_UNICODE);
1593+
}
1594+
1595+
15861596
/* Find the maximum code point and count the number of surrogate pairs so a
15871597
correct string length can be computed before converting a string to UCS4.
15881598
This function counts single surrogates as a character and not as a pair.
@@ -1618,9 +1628,7 @@ find_maxchar_surrogates(const wchar_t *begin, const wchar_t *end,
16181628
if (ch > *maxchar) {
16191629
*maxchar = ch;
16201630
if (*maxchar > MAX_UNICODE) {
1621-
PyErr_Format(PyExc_ValueError,
1622-
"character U+%x is not in range [U+0000; U+%x]",
1623-
ch, MAX_UNICODE);
1631+
unicode_invalid_character(PyExc_ValueError, ch);
16241632
return -1;
16251633
}
16261634
}
@@ -1822,17 +1830,15 @@ get_latin1_char(Py_UCS1 ch)
18221830
static PyObject*
18231831
unicode_char(Py_UCS4 ch)
18241832
{
1825-
PyObject *unicode;
1826-
1827-
assert(ch <= MAX_UNICODE);
1828-
18291833
if (ch < 256) {
18301834
return get_latin1_char(ch);
18311835
}
18321836

1833-
unicode = PyUnicode_New(1, ch);
1834-
if (unicode == NULL)
1837+
// Raise SystemError if the character is not in range [U+0000; MAX_UNICODE]
1838+
PyObject *unicode = PyUnicode_New(1, ch);
1839+
if (unicode == NULL) {
18351840
return NULL;
1841+
}
18361842

18371843
assert(PyUnicode_KIND(unicode) != PyUnicode_1BYTE_KIND);
18381844
if (PyUnicode_KIND(unicode) == PyUnicode_2BYTE_KIND) {
@@ -2211,15 +2217,36 @@ static PyObject*
22112217
_PyUnicode_FromUCS4(const Py_UCS4 *u, Py_ssize_t size)
22122218
{
22132219
PyObject *res;
2214-
Py_UCS4 max_char;
22152220

22162221
if (size == 0)
22172222
_Py_RETURN_UNICODE_EMPTY();
22182223
assert(size > 0);
2219-
if (size == 1)
2224+
2225+
if (size == 1) {
2226+
// Raise SystemError if the character is invalid
22202227
return unicode_char(u[0]);
2228+
}
2229+
2230+
#ifdef Py_DEBUG
2231+
// Check for invalid characters in debug mode
2232+
Py_UCS4 max_char = 127;
2233+
for (Py_ssize_t i = 0; i < size; i++) {
2234+
Py_UCS4 ch = u[i];
2235+
if (ch > max_char) {
2236+
if (ch > MAX_UNICODE) {
2237+
unicode_invalid_character(PyExc_SystemError, ch);
2238+
return NULL;
2239+
}
2240+
max_char = ch;
2241+
}
2242+
}
2243+
#else
2244+
// gh-158445: Return MAX_UNICODE even if the string contains invalid
2245+
// characters. Checking for invalid characters would make the function
2246+
// slower whereas it's unlikely in practice.
2247+
Py_UCS4 max_char = ucs4lib_find_max_char(u, u + size);
2248+
#endif
22212249

2222-
max_char = ucs4lib_find_max_char(u, u + size);
22232250
res = PyUnicode_New(size, max_char);
22242251
if (!res)
22252252
return NULL;
@@ -2253,9 +2280,27 @@ PyUnicodeWriter_WriteUCS4(PyUnicodeWriter *pub_writer,
22532280
return 0;
22542281
}
22552282

2256-
Py_UCS4 max_char = ucs4lib_find_max_char(str, str + size);
2283+
#ifdef Py_DEBUG
2284+
// Check for invalid characters in debug mode
2285+
Py_UCS4 maxchar = 127;
2286+
for (Py_ssize_t i = 0; i < size; i++) {
2287+
Py_UCS4 ch = str[i];
2288+
if (ch > maxchar) {
2289+
if (ch > MAX_UNICODE) {
2290+
unicode_invalid_character(PyExc_SystemError, ch);
2291+
return -1;
2292+
}
2293+
maxchar = ch;
2294+
}
2295+
}
2296+
#else
2297+
// gh-158445: Return MAX_UNICODE even if the string contains invalid
2298+
// characters. Checking for invalid characters would make the function
2299+
// slower whereas it's unlikely in practice.
2300+
Py_UCS4 maxchar = ucs4lib_find_max_char(str, str + size);
2301+
#endif
22572302

2258-
if (_PyUnicodeWriter_Prepare(writer, size, max_char) < 0) {
2303+
if (_PyUnicodeWriter_Prepare(writer, size, maxchar) < 0) {
22592304
return -1;
22602305
}
22612306

0 commit comments

Comments
 (0)