Skip to content

Commit f607706

Browse files
schuayv8-scoped@luci-project-accounts.iam.gserviceaccount.com
authored andcommitted
[regexp] Handle supplementary case equivalents of BMP characters
Unicode 18 adds U+1DF95, which simple-folds to U+00DF. This creates the first case-equivalence class spanning BMP and supplementary characters: {U+00DF, U+1E9E, U+1DF95}. Discard supplementary characters from the generator's non-unicode closures before calling Canonicalize, which requires UTF-16 code units and otherwise CHECK-fails with ICU 79. Apply the same filtering in runtime non-unicode closure. Generate SupplementaryEquivalentsSet and desugar its BMP characters into character classes under /iu and /iv, allowing literals to match surrogate pairs. Backreferences with differing UTF-16 character widths remain unsupported and are tracked in crbug.com/571793821. IgnoreSet is unchanged with ICU 78 and with ICU 78.2 patched to upstream Unicode 18 case data. With the patched data, the generator succeeds, SupplementaryEquivalentsSet is {U+00DF, U+1E9E}, and regexp tests pass in release and debug. Bug: 571232615, 571668686 Change-Id: I9eacb273a74c66e17f0cfb1b256e863f2718c130 Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/8548134 Reviewed-by: Patrick Thier <[email protected]> Commit-Queue: Jakob Linke <[email protected]> Cr-Commit-Position: refs/heads/main@{#110522}
1 parent 618b751 commit f607706

5 files changed

Lines changed: 143 additions & 13 deletions

File tree

‎src/regexp/gen-regexp-special-case.cc‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include "src/base/strings.h"
1111
#include "src/regexp/special-case.h"
1212
#include "unicode/locid.h"
13+
#include "unicode/uchar.h"
1314
#include "unicode/unistr.h"
1415

1516
namespace v8 {
@@ -52,6 +53,19 @@ UChar32 Canonicalize(UChar32 ch) {
5253
return cu;
5354
}
5455

56+
// Removes supplementary code points from |set| and returns whether any were
57+
// present. Since Unicode 18, the case closure of a BMP character can contain
58+
// supplementary code points (U+1DF95 LATIN SMALL LIGATURE LONG S WITH
59+
// DESCENDER S case-folds to U+00DF LATIN SMALL LETTER SHARP S). Non-unicode
60+
// ignoreCase matching compares UTF-16 code units, so a supplementary code
61+
// point never matches a BMP character and does not take part in Canonicalize.
62+
bool RemoveSupplementary(icu::UnicodeSet& set) {
63+
constexpr UChar32 kFirstSupplementary = static_cast<UChar32>(kNonBmpStart);
64+
if (!set.containsSome(kFirstSupplementary, UCHAR_MAX_VALUE)) return false;
65+
set.remove(kFirstSupplementary, UCHAR_MAX_VALUE);
66+
return true;
67+
}
68+
5569
// The following code generates "src/regexp/special-case.cc".
5670
void PrintSet(std::ofstream& out, const char* name,
5771
const icu::UnicodeSet& set) {
@@ -86,6 +100,7 @@ void PrintSpecial(std::ofstream& out) {
86100
icu::UnicodeSet current;
87101
icu::UnicodeSet special_add;
88102
icu::UnicodeSet ignore;
103+
icu::UnicodeSet supplementary_equivalents;
89104
UErrorCode status = U_ZERO_ERROR;
90105
icu::UnicodeSet upper("[\\p{Lu}]", status);
91106
CHECK(U_SUCCESS(status));
@@ -98,6 +113,7 @@ void PrintSpecial(std::ofstream& out) {
98113
}
99114
current.set(i, i);
100115
current.closeOver(USET_SIMPLE_CASE_INSENSITIVE);
116+
if (RemoveSupplementary(current)) supplementary_equivalents.add(i);
101117

102118
// Check to see if all characters in the case-folding equivalence
103119
// class as defined by UnicodeSet::closeOver all map to the same
@@ -147,6 +163,7 @@ void PrintSpecial(std::ofstream& out) {
147163
UChar32 canonical = Canonicalize(c);
148164
current.set(c, c);
149165
current.closeOver(USET_SIMPLE_CASE_INSENSITIVE);
166+
RemoveSupplementary(current);
150167
current.removeAll(ignore);
151168
for (int32_t j = 0; j < current.getRangeCount(); j++) {
152169
for (UChar32 c2 = current.getRangeStart(j);
@@ -158,6 +175,7 @@ void PrintSpecial(std::ofstream& out) {
158175
}
159176

160177
PrintSet(out, "IgnoreSet", ignore);
178+
PrintSet(out, "SupplementaryEquivalentsSet", supplementary_equivalents);
161179
}
162180

163181
void WriteHeader(const char* header_filename) {

‎src/regexp/regexp-parser.cc‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
#include "src/regexp/regexp-ast.h"
1313
#include "src/regexp/regexp-macro-assembler.h"
1414
#include "src/regexp/regexp.h"
15+
#include "src/regexp/special-case.h"
1516
#include "src/strings/char-predicates-inl.h"
1617
#include "src/utils/ostreams.h"
1718
#include "src/utils/utils.h"
@@ -135,6 +136,10 @@ void TextBuilder<mode>::FlushText() {
135136

136137
template <ParseMode mode>
137138
void TextBuilder<mode>::AddCharacter(base::uc16 c) {
139+
if (NeedsDesugaringForIgnoreCase(c)) {
140+
AddClassRangesForDesugaring(c);
141+
return;
142+
}
138143
if (characters_ == nullptr) {
139144
characters_ = zone()->template New<ZoneList<base::uc16>>(4, zone());
140145
}
@@ -222,16 +227,19 @@ bool TextBuilder<mode>::NeedsDesugaringForUnicode(ClassRanges* cc) {
222227
return false;
223228
}
224229

225-
// We only use this for characters made of surrogate pairs. All other
226-
// characters outside of character classes are made case independent in the
227-
// code generation.
230+
// Desugar supplementary characters with case equivalents and BMP characters
231+
// with supplementary equivalents. Other atoms are folded during code
232+
// generation.
228233
template <ParseMode mode>
229234
bool TextBuilder<mode>::NeedsDesugaringForIgnoreCase(base::uc32 c) {
230235
#ifdef V8_INTL_SUPPORT
231236
if (IsUnicodeMode() && ignore_case()) {
237+
if (c <= unibrow::Utf16::kMaxNonSurrogateCharCode) {
238+
return CaseFolding::HasSupplementaryEquivalents(
239+
static_cast<base::uc16>(c));
240+
}
232241
icu::UnicodeSet set(c, c);
233-
set.closeOver(USET_CASE_INSENSITIVE);
234-
set.removeAllStrings();
242+
CaseFolding::CloseOver(set, CaseFolding::Mode::kUnicode);
235243
return set.size() > 1;
236244
}
237245
// In the case where ICU is not included, we act as if the unicode flag is

‎src/regexp/special-case.h‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@
77

88
#ifdef V8_INTL_SUPPORT
99
#include "src/base/logging.h"
10+
#include "src/base/strings.h"
1011
#include "src/common/globals.h"
11-
1212
#include "unicode/uchar.h"
1313
#include "unicode/uniset.h"
1414

@@ -53,6 +53,12 @@ namespace regexp {
5353
// another character. Characters that match no other characters in
5454
// their equivalence class are added to IgnoreSet. The generator verifies
5555
// that each Unicode class has at most one non-trivial JS class.
56+
// Supplementary code points are ignored throughout: non-unicode matching
57+
// compares UTF-16 code units, so they can never match a BMP character.
58+
//
59+
// The generator additionally emits SupplementaryEquivalentsSet, the BMP
60+
// characters whose Unicode case closure contains supplementary code points.
61+
// This set is used for unicode-mode matching, see HasSupplementaryEquivalents.
5662

5763
class V8_EXPORT_PRIVATE CaseFolding final : public AllStatic {
5864
public:
@@ -75,9 +81,9 @@ class V8_EXPORT_PRIVATE CaseFolding final : public AllStatic {
7581
set.closeOver(USET_SIMPLE_CASE_INSENSITIVE);
7682
return;
7783
}
84+
DCHECK(!set.containsSome(0x10000, 0x10ffff));
7885
if (IgnoreSet().containsNone(set)) {
79-
set.closeOver(USET_SIMPLE_CASE_INSENSITIVE);
80-
set.removeAll(IgnoreSet());
86+
CloseOverNonUnicode(set);
8187
return;
8288
}
8389
if (IgnoreSet().containsAll(set)) return;
@@ -86,13 +92,25 @@ class V8_EXPORT_PRIVATE CaseFolding final : public AllStatic {
8692
icu::UnicodeSet ignored(set);
8793
ignored.retainAll(IgnoreSet());
8894
set.removeAll(IgnoreSet());
89-
set.closeOver(USET_SIMPLE_CASE_INSENSITIVE);
90-
set.removeAll(IgnoreSet());
95+
CloseOverNonUnicode(set);
9196
set.addAll(ignored);
9297
}
9398

99+
// Whether c has supplementary Unicode case equivalents.
100+
static bool HasSupplementaryEquivalents(base::uc16 c) {
101+
return SupplementaryEquivalentsSet().contains(c);
102+
}
103+
94104
private:
95105
static const icu::UnicodeSet& IgnoreSet();
106+
static const icu::UnicodeSet& SupplementaryEquivalentsSet();
107+
108+
// Requires a set containing no IgnoreSet characters.
109+
static void CloseOverNonUnicode(icu::UnicodeSet& set) {
110+
set.closeOver(USET_SIMPLE_CASE_INSENSITIVE);
111+
set.removeAll(IgnoreSet());
112+
set.remove(0x10000, 0x10ffff);
113+
}
96114
};
97115

98116
} // namespace regexp

‎test/intl/regexp-case-equivalence.js‎

Lines changed: 60 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,14 @@
22
// Use of this source code is governed by a BSD-style license that can be
33
// found in the LICENSE file.
44

5-
// Exercise literals, ranges, and backreferences with the same case pairs.
5+
// Unicode 18 assigns U+1DF95 LATIN SMALL LIGATURE LONG S WITH DESCENDER S,
6+
// the first supplementary character that case-folds to a BMP character
7+
// (U+00DF). It is thus case-equivalent to U+00DF and U+1E9E under /iu and
8+
// /iv, but never under /i, which compares UTF-16 code units.
9+
const hasUnicode18 = '\u{1DF95}'.toUpperCase() === 'SS';
10+
11+
// Exercise literals, ranges, backreferences, lookbehinds, and class strings
12+
// with the same case pairs.
613
const pairs = [
714
['a', 'A', true, true],
815
['k', '\u212a', false, true],
@@ -12,6 +19,8 @@ const pairs = [
1219
['i', '\u0130', false, false],
1320
['I', '\u0131', false, false],
1421
['\u{10400}', '\u{10428}', false, true],
22+
['\u00df', '\u{1DF95}', false, hasUnicode18],
23+
['\u1e9e', '\u{1DF95}', false, hasUnicode18],
1524
['\ud800', '\ud800', true, true],
1625
['\ud800', '\ud801', false, false],
1726
];
@@ -29,16 +38,64 @@ for (const [a, b, legacy, unicode] of pairs) {
2938
? new RegExp('^' + prefix + '[' + c1 + '-' + c1 + ']$', flags)
3039
: null;
3140
assertEquals(expected, literal.test(prefix + c2), literal.toString());
32-
assertEquals(expected, backref.test(prefix + c1 + c2),
33-
backref.toString());
41+
// TODO(crbug.com/571793821): Case-insensitive backreferences do not
42+
// yet match captures of a different UTF-16 length (U+00DF vs U+1DF95).
43+
if (c1.length === c2.length) {
44+
assertEquals(expected, backref.test(prefix + c1 + c2),
45+
backref.toString());
46+
}
3447
if (range !== null) {
3548
assertEquals(expected, range.test(prefix + c2), range.toString());
3649
}
50+
const lookbehind = new RegExp('(?<=^' + prefix + c1 + ')x$', flags);
51+
assertEquals(expected, lookbehind.test(prefix + c2 + 'x'),
52+
lookbehind.toString());
53+
if (flags === 'iv') {
54+
const classString = new RegExp(
55+
'^' + prefix + '[\\q{' + c1 + 'x}]$', flags);
56+
assertEquals(expected, classString.test(prefix + c2 + 'x'),
57+
classString.toString());
58+
}
3759
}
3860
}
3961
}
4062
}
4163

64+
// Supplementary case equivalents of BMP characters must be matched in
65+
// unicode mode both as plain pattern characters and inside character
66+
// classes, but never in legacy mode.
67+
if (hasUnicode18) {
68+
const S = '\u{1DF95}';
69+
const cases = [
70+
// [pattern, subject, expected]
71+
['^\u00df$', S, true],
72+
['^\u1e9e$', S, true],
73+
['^' + S + '$', '\u00df', true],
74+
['^' + S + '$', '\u1e9e', true],
75+
['^\\u00df$', S, true],
76+
['^\\u{df}$', S, true],
77+
['^a\u00dfb$', 'A' + S + 'B', true],
78+
['^(?:\u00df)+$', S + '\u00df\u1e9e' + S, true],
79+
['^\u00df{2}$', S + '\u1e9e', true],
80+
['^\u00df{2}$', S, false],
81+
['^(?:x|\u00df)$', S, true],
82+
['^[\u00df]$', S, true],
83+
['^[^\u00df]$', S, false],
84+
['^(\u00df)$', S, true],
85+
];
86+
for (const [pattern, subject, expected] of cases) {
87+
for (const flags of ['iu', 'iv']) {
88+
const re = new RegExp(pattern, flags);
89+
assertEquals(expected, re.test(subject), re.toString() + ' ' + subject);
90+
}
91+
// In legacy mode, the astral character consists of two code units,
92+
// neither of which has case equivalents.
93+
const re = new RegExp(pattern, 'i');
94+
assertFalse(re.test(subject), re.toString() + ' ' + subject);
95+
}
96+
assertEquals([S, S], /(\u00df)/iu.exec(S));
97+
}
98+
4299
for (const flags of ['i', 'iu', 'iv']) {
43100
const re = new RegExp('^[a-c\u017f\u00df]$', flags);
44101
for (const c of ['a', 'B', 'c', '\u017f', '\u00df']) {

‎test/unittests/regexp/regexp-unittest.cc‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1513,6 +1513,31 @@ TEST_F(RegExpTest, NonUnicodeCaseEquivalence) {
15131513
ASSERT_TRUE(expected == actual) << c;
15141514
}
15151515
}
1516+
1517+
TEST_F(RegExpTest, SupplementaryCaseEquivalents) {
1518+
using CaseFolding = regexp::CaseFolding;
1519+
icu::UnicodeSet expected;
1520+
for (UChar32 c = 0; c <= 0xffff; ++c) {
1521+
icu::UnicodeSet closure(c, c);
1522+
closure.closeOver(USET_SIMPLE_CASE_INSENSITIVE);
1523+
const bool has_supplementary = closure.containsSome(0x10000, 0x10ffff);
1524+
if (has_supplementary) expected.add(c);
1525+
const bool actual =
1526+
CaseFolding::HasSupplementaryEquivalents(static_cast<base::uc16>(c));
1527+
ASSERT_EQ(has_supplementary, actual) << c;
1528+
1529+
// Non-unicode closure must never yield supplementary code points.
1530+
icu::UnicodeSet non_unicode(c, c);
1531+
CaseFolding::CloseOver(non_unicode, CaseFolding::Mode::kNonUnicode);
1532+
ASSERT_FALSE(non_unicode.containsSome(0x10000, 0x10ffff)) << c;
1533+
}
1534+
// U+1DF95 LATIN SMALL LIGATURE LONG S WITH DESCENDER S (Unicode 18)
1535+
// case-folds to U+00DF LATIN SMALL LETTER SHARP S.
1536+
if (u_foldCase(0x1df95, U_FOLD_CASE_DEFAULT) == 0xdf) {
1537+
EXPECT_TRUE(expected.contains(0xdf));
1538+
EXPECT_TRUE(expected.contains(0x1e9e));
1539+
}
1540+
}
15161541
#endif // !DEBUG
15171542

15181543
TEST_F(RegExpTest, CaseClosureMixedSets) {
@@ -1525,6 +1550,10 @@ TEST_F(RegExpTest, CaseClosureMixedSets) {
15251550
expected.add('A', 'C').add('K');
15261551
if (mode == CaseFolding::Mode::kUnicode) {
15271552
expected.add('s').add('S').add(0x212a).add(0x1e9e);
1553+
// Since Unicode 18, U+1DF95 case-folds to U+00DF.
1554+
if (u_foldCase(0x1df95, U_FOLD_CASE_DEFAULT) == 0xdf) {
1555+
expected.add(0x1df95);
1556+
}
15281557
actual.add(0x10400);
15291558
expected.add(0x10400).add(0x10428);
15301559
}

0 commit comments

Comments
 (0)