Skip to content

Commit 1d74d31

Browse files
encukoumcepl
encukou
authored and
mcepl
committed
[CVE-2026-17084] Don't consider Unicode codepoint attributes outside RFC 3454
Due to a bug, some Unicode codepoint attributes were considered for characters not yet defined in Unicode 3.2.0 or attributes which changed in later Unicode versions. RFC 3454 (StringPrep) requires using Unicode 3.2.0 strictly. Fixes: bsc#1276226 (CVE-2026-17084) Fixes: gh#python#155292 From-PR: gh#python/cpython!155293 Patch: CVE-2026-17084-stringprep-rfc3454.patch
1 parent 2b49601 commit 1d74d31

6 files changed

Lines changed: 437 additions & 186 deletions

File tree

‎Lib/stringprep.py‎

Lines changed: 315 additions & 148 deletions
Large diffs are not rendered by default.

‎Lib/test/test_codecs.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1652,6 +1652,17 @@ def test_builtin_encode(self):
16521652
self.assertEqual("pyth\xf6n.org".encode("idna"), b"xn--pythn-mua.org")
16531653
self.assertEqual("pyth\xf6n.org.".encode("idna"), b"xn--pythn-mua.org.")
16541654

1655+
def test_new_unicode_case_folding(self):
1656+
cases = [
1657+
('\N{CHEROKEE LETTER A}\N{CHEROKEE LETTER A}', b"xn--58da"),
1658+
('\N{GEORGIAN CAPITAL LETTER AN}.', b"xn--7md."),
1659+
('\N{CYRILLIC LETTER PALOCHKA}.example', b"xn--d5a.example"),
1660+
('\N{ROMAN NUMERAL REVERSED ONE HUNDRED}.example.', b"xn--q5g.example."),
1661+
]
1662+
for unicode, encoded in cases:
1663+
with self.subTest(unicode=unicode, encoded=encoded):
1664+
self.assertEqual(unicode.encode("idna"), encoded)
1665+
16551666
def test_builtin_decode_length_limit(self):
16561667
with self.assertRaisesRegex(UnicodeError, "too long"):
16571668
(b"xn--016c"+b"a"*1100).decode("idna")

‎Lib/test/test_unicodedata.py‎

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ def tearDown(self):
7777
del self.db
7878

7979
class UnicodeFunctionsTest(UnicodeDatabaseTest):
80+
old = False
8081

8182
# Update this if the database changes. Make sure to do a full rebuild
8283
# (e.g. 'make distclean && make') to get the correct checksum.
@@ -100,7 +101,8 @@ def test_function_checksum(self):
100101
]
101102
h.update(''.join(data).encode("ascii"))
102103
result = h.hexdigest()
103-
self.assertEqual(result, self.expectedchecksum)
104+
if not self.old:
105+
self.assertEqual(result, self.expectedchecksum)
104106

105107
def test_digit(self):
106108
self.assertEqual(self.db.digit('A', None), None)
@@ -119,9 +121,13 @@ def test_numeric(self):
119121
self.assertEqual(self.db.numeric('9'), 9)
120122
self.assertEqual(self.db.numeric('\u215b'), 0.125)
121123
self.assertEqual(self.db.numeric('\u2468'), 9.0)
122-
self.assertEqual(self.db.numeric('\ua627'), 7.0)
124+
# New in 5.1.0
125+
self.assertEqual(self.db.numeric('\ua627', None),
126+
None if self.old else 7.0)
123127
self.assertEqual(self.db.numeric('\U00020000', None), None)
124-
self.assertEqual(self.db.numeric('\U0001012A'), 9000)
128+
# New in 4.1.0
129+
self.assertEqual(self.db.numeric('\U0001012A', None),
130+
None if self.old else 9000)
125131

126132
self.assertRaises(TypeError, self.db.numeric)
127133
self.assertRaises(TypeError, self.db.numeric, 'xx')
@@ -144,7 +150,8 @@ def test_category(self):
144150
self.assertEqual(self.db.category('a'), 'Ll')
145151
self.assertEqual(self.db.category('A'), 'Lu')
146152
self.assertEqual(self.db.category('\U00020000'), 'Lo')
147-
self.assertEqual(self.db.category('\U0001012A'), 'No')
153+
self.assertEqual(self.db.category('\U0001012A'),
154+
'Cn' if self.old else 'No')
148155

149156
self.assertRaises(TypeError, self.db.category)
150157
self.assertRaises(TypeError, self.db.category, 'xx')
@@ -158,6 +165,15 @@ def test_bidirectional(self):
158165
self.assertRaises(TypeError, self.db.bidirectional)
159166
self.assertRaises(TypeError, self.db.bidirectional, 'xx')
160167

168+
def test_bidirectional_unassigned(self):
169+
self.assertEqual(self.db.bidirectional('\u0378'), '')
170+
self.assertEqual(self.db.bidirectional('\u077F'), '' if self.old else 'AL')
171+
self.assertEqual(self.db.bidirectional('\u20CF'), '')
172+
self.assertEqual(self.db.bidirectional('\u0590'), '')
173+
self.assertEqual(self.db.bidirectional('\uFFFF'), '')
174+
self.assertEqual(self.db.bidirectional('\U0001FFFE'), '')
175+
self.assertEqual(self.db.bidirectional('\U00010D01'), '')
176+
161177
def test_decomposition(self):
162178
self.assertEqual(self.db.decomposition('\uFFFE'),'')
163179
self.assertEqual(self.db.decomposition('\u00bc'), '<fraction> 0031 2044 0034')
@@ -264,8 +280,16 @@ def test_east_asian_width(self):
264280
self.assertEqual(eaw('\U00020000'), 'W')
265281

266282
def test_east_asian_width_9_0_changes(self):
267-
self.assertEqual(self.db.ucd_3_2_0.east_asian_width('\u231a'), 'N')
268-
self.assertEqual(self.db.east_asian_width('\u231a'), 'W')
283+
self.assertEqual(self.db.east_asian_width('\u231a'),
284+
'N' if self.old else 'W')
285+
286+
287+
class Unicode_3_2_0_FunctionsTest(UnicodeFunctionsTest):
288+
old = True
289+
def setUp(self):
290+
import unicodedata
291+
self.db = unicodedata.ucd_3_2_0
292+
269293

270294
class UnicodeMiscTest(UnicodeDatabaseTest):
271295

‎Misc/NEWS.d/next/Security/2026-08-06-11-43-20.gh-issue-155292.j4pHBO.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Change the :mod:`stringprep` module and :mod:`encodings.idna` codec to not
2+
consider Unicode codepoint attributes beyond those defined in :rfc:`3454`.

‎Tools/unicode/makeunicodedata.py‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,13 @@
2828
#
2929

3030
import os
31+
import subprocess
3132
import sys
3233
import zipfile
3334

3435
from textwrap import dedent
3536

36-
SCRIPT = sys.argv[0]
37+
SCRIPT = os.path.normpath(sys.argv[0])
3738
VERSION = "3.2"
3839

3940
# The Unicode Database
@@ -124,6 +125,7 @@ def maketables(trace=0):
124125
makeunicodename(unicode, trace)
125126
makeunicodedata(unicode, trace)
126127
makeunicodetype(unicode, trace)
128+
makestringprep()
127129

128130
# --------------------------------------------------------------------
129131
# unicode character properties
@@ -786,6 +788,19 @@ def word_key(a):
786788
fp.close()
787789

788790

791+
792+
def makestringprep():
793+
FILE = "Lib/stringprep.py"
794+
795+
print("--- Preparing", FILE, "...")
796+
797+
MKSTRINGPREP = "Tools/unicode/mkstringprep.py"
798+
799+
with open(FILE, "w") as f:
800+
f.truncate()
801+
subprocess.check_call([sys.executable, MKSTRINGPREP], stdout=f)
802+
803+
789804
def merge_old_version(version, new, old):
790805
# Changes to exclusion file not implemented yet
791806
if old.exclusions != new.exclusions:

‎Tools/unicode/mkstringprep.py‎

Lines changed: 63 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,22 @@
1-
import re, sys
2-
from unicodedata import ucd_3_2_0 as unicodedata
1+
import re, sys, os
2+
import unicodedata as unicodedata_current
3+
from unicodedata import ucd_3_2_0 as unicodedata_320
34

45
if sys.maxunicode == 65535:
56
raise RuntimeError("need UCS-4 Python")
67

8+
FILENAME = "Tools/unicode/data/rfc3454.txt"
9+
URL = "https://www.rfc-editor.org/rfc/rfc3454.txt"
10+
711
def gen_category(cats):
812
for i in range(0, 0x110000):
9-
if unicodedata.category(chr(i)) in cats:
10-
yield(i)
13+
if unicodedata_320.category(chr(i)) in cats:
14+
yield i
1115

1216
def gen_bidirectional(cats):
1317
for i in range(0, 0x110000):
14-
if unicodedata.bidirectional(chr(i)) in cats:
15-
yield(i)
18+
if unicodedata_320.bidirectional(chr(i)) in cats:
19+
yield i
1620

1721
def compact_set(l):
1822
single = []
@@ -50,8 +54,16 @@ def compact_set(l):
5054

5155
############## Read the tables in the RFC #######################
5256

53-
with open("rfc3454.txt") as f:
54-
data = f.readlines()
57+
try:
58+
data_file = open(FILENAME, encoding='utf-8')
59+
except FileNotFoundError:
60+
import urllib.request
61+
os.makedirs(os.path.dirname(FILENAME), exist_ok=True)
62+
urllib.request.urlretrieve(URL, filename=FILENAME)
63+
data_file = open(FILENAME, encoding='utf-8')
64+
65+
with data_file:
66+
data = data_file.readlines()
5567

5668
tables = []
5769
curname = None
@@ -119,10 +131,18 @@ def compact_set(l):
119131
and mappings, for which a mapping function is provided.
120132
\"\"\"
121133
122-
from unicodedata import ucd_3_2_0 as unicodedata
134+
# This check asserts that mkstringprep.py has been run
135+
# when unicodedata is modified to ensure conformant behavior.
136+
import unicodedata
137+
""")
138+
139+
print("assert unicodedata.unidata_version == %r" % (unicodedata_current.unidata_version,))
140+
141+
print("""
142+
from unicodedata import ucd_3_2_0 as unicodedata_320
123143
""")
124144

125-
print("assert unicodedata.unidata_version == %r" % (unicodedata.unidata_version,))
145+
print("assert unicodedata_320.unidata_version == %r" % (unicodedata_320.unidata_version,))
126146

127147
# A.1 is the table of unassigned characters
128148
# XXX Plane 15 PUA is listed as unassigned in Python.
@@ -142,7 +162,7 @@ def compact_set(l):
142162

143163
print("""
144164
def in_table_a1(code):
145-
if unicodedata.category(code) != 'Cn': return False
165+
if unicodedata_320.category(code) != 'Cn': return False
146166
c = ord(code)
147167
if 0xFDD0 <= c < 0xFDF0: return False
148168
return (c & 0xFFFF) not in (0xFFFE, 0xFFFF)
@@ -175,21 +195,33 @@ def in_table_b1(code):
175195

176196
# B.3 is mostly Python's .lower, except for a number
177197
# of special cases, e.g. considering canonical forms.
198+
# To enforce Unicode 3.2.0 behavior of .lower instead of
199+
# whatever Unicode version is included with Python we
200+
# add unassigned or newly case-folding codepoints to
201+
# the exception map, too.
178202

179203
b3_exceptions = {}
180204

181205
for k,v in table_b2.items():
182206
if list(map(ord, chr(k).lower())) != v:
183207
b3_exceptions[k] = "".join(map(chr,v))
208+
for cp in range(0x110000):
209+
ch = chr(cp)
210+
# Assigned in current Unicode version
211+
# and supports case folding, but not
212+
# explicitly in B.2 or B.3 tables.
213+
if (unicodedata_current.category(ch) != "Cn"
214+
and ch.lower() != ch
215+
and cp not in table_b2
216+
and cp not in table_b3):
217+
b3_exceptions[cp] = ch # Identity.
184218

185219
b3 = sorted(b3_exceptions.items())
186220

187221
print("""
188222
b3_exceptions = {""")
189223
for i, kv in enumerate(b3):
190-
print("0x%x:%a," % kv, end=' ')
191-
if i % 4 == 3:
192-
print()
224+
print("0x%x:%a," % kv, end='\n' if i % 4 == 3 else ' ')
193225
print("}")
194226

195227
print("""
@@ -210,9 +242,9 @@ def map_table_b3(code):
210242

211243
def map_table_b2(a):
212244
al = map_table_b3(a)
213-
b = unicodedata.normalize("NFKC", al)
245+
b = unicodedata_320.normalize("NFKC", al)
214246
bl = "".join([map_table_b3(ch) for ch in b])
215-
c = unicodedata.normalize("NFKC", bl)
247+
c = unicodedata_320.normalize("NFKC", bl)
216248
if b != c:
217249
return c
218250
else:
@@ -229,9 +261,9 @@ def map_table_b2(a):
229261
print("""
230262
def map_table_b2(a):
231263
al = map_table_b3(a)
232-
b = unicodedata.normalize("NFKC", al)
264+
b = unicodedata_320.normalize("NFKC", al)
233265
bl = "".join([map_table_b3(ch) for ch in b])
234-
c = unicodedata.normalize("NFKC", bl)
266+
c = unicodedata_320.normalize("NFKC", bl)
235267
if b != c:
236268
return c
237269
else:
@@ -254,16 +286,16 @@ def in_table_c11(code):
254286
del tables[0]
255287
assert name == "C.1.2"
256288

257-
# table = set(table.keys())
258-
# Zs = set(gen_category(["Zs"])) - {0x20}
259-
# assert Zs == table
289+
table = set(table.keys())
290+
Zs = set(gen_category(["Zs"])) - {0x20}
291+
assert Zs == table
260292

261293
print("""
262294
def in_table_c12(code):
263-
return unicodedata.category(code) == "Zs" and code != " "
295+
return unicodedata_320.category(code) == "Zs" and code != " "
264296
265297
def in_table_c11_c12(code):
266-
return unicodedata.category(code) == "Zs"
298+
return unicodedata_320.category(code) == "Zs"
267299
""")
268300

269301
# C.2.1 ASCII control characters
@@ -278,7 +310,7 @@ def in_table_c11_c12(code):
278310

279311
print("""
280312
def in_table_c21(code):
281-
return ord(code) < 128 and unicodedata.category(code) == "Cc"
313+
return ord(code) < 128 and unicodedata_320.category(code) == "Cc"
282314
""")
283315

284316
# C.2.2 Non-ASCII control characters. It also includes
@@ -298,11 +330,11 @@ def in_table_c21(code):
298330
def in_table_c22(code):
299331
c = ord(code)
300332
if c < 128: return False
301-
if unicodedata.category(code) == "Cc": return True
333+
if unicodedata_320.category(code) == "Cc": return True
302334
return c in c22_specials
303335
304336
def in_table_c21_c22(code):
305-
return unicodedata.category(code) == "Cc" or \\
337+
return unicodedata_320.category(code) == "Cc" or \\
306338
ord(code) in c22_specials
307339
""")
308340

@@ -316,7 +348,7 @@ def in_table_c21_c22(code):
316348

317349
print("""
318350
def in_table_c3(code):
319-
return unicodedata.category(code) == "Co"
351+
return unicodedata_320.category(code) == "Co"
320352
""")
321353

322354
# C.4 Non-character code points, xFFFE, xFFFF
@@ -349,7 +381,7 @@ def in_table_c4(code):
349381

350382
print("""
351383
def in_table_c5(code):
352-
return unicodedata.category(code) == "Cs"
384+
return unicodedata_320.category(code) == "Cs"
353385
""")
354386

355387
# C.6 Inappropriate for plain text
@@ -414,7 +446,7 @@ def in_table_c9(code):
414446

415447
print("""
416448
def in_table_d1(code):
417-
return unicodedata.bidirectional(code) in ("R","AL")
449+
return unicodedata_320.bidirectional(code) in ("R","AL")
418450
""")
419451

420452
# D.2 Characters with bidirectional property "L"
@@ -427,5 +459,5 @@ def in_table_d1(code):
427459

428460
print("""
429461
def in_table_d2(code):
430-
return unicodedata.bidirectional(code) == "L"
431-
""")
462+
return unicodedata_320.bidirectional(code) == "L"
463+
""", end="")

0 commit comments

Comments
 (0)