Skip to content

Commit d93e32c

Browse files
Backport PR #71383 on branch 3.1.x (BUG: fix string regex operations with Python unicode escape sequence) (#71446)
Co-authored-by: Joris Van den Bossche <[email protected]>
1 parent bff0883 commit d93e32c

4 files changed

Lines changed: 107 additions & 8 deletions

File tree

‎doc/source/whatsnew/v3.1.0.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1002,6 +1002,7 @@ Strings
10021002
- Bug in :meth:`Series.str.split` with :class:`ArrowDtype` ``string`` not inferring regex for multi-character patterns when ``regex=None``, causing the pattern to be treated as a literal instead of a regular expression (:issue:`58321`)
10031003
- Bug in :meth:`Series.unique`, :meth:`Index.unique` and :func:`factorize` on object dtype returning incorrect results when the values were not UTF-8 encodable, e.g. lone surrogates (:issue:`34550`)
10041004
- Bug in :meth:`Series.unique`, :meth:`Index.unique`, :meth:`Series.nunique`, :func:`factorize` and ``groupby`` on object dtype, and on ``str``/``string`` dtype backed by python storage, collapsing distinct strings that are identical up to an embedded NUL byte, e.g. ``""`` and ``"\x00"``, into a single value (:issue:`34551`)
1005+
- Bug in string methods using a regex pattern with unicode escape sequences (``\u``/``\U``/``\N{}``) with PyArrow-backed string dtypes failing, now working again as in pandas 2.x (:issue:`63901`)
10051006

10061007
Interval
10071008
^^^^^^^^

‎pandas/core/arrays/_arrow_string_mixins.py‎

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,11 @@
3030
from pandas._typing import Scalar
3131

3232

33+
# Matches a `\uxxxx` (unicode escape sequence) with an odd number of preceding "\"
34+
# (otherwise the \\u is escaped itself). Also checks for `\U` and `\N{`.
35+
_escaped_unicode_char = re.compile(r"(?<!\\)(?:\\\\)*\\(?:[uU]|N{)")
36+
37+
3338
class ArrowStringArrayMixin:
3439
_pa_array: pa.ChunkedArray
3540

@@ -63,7 +68,7 @@ def _re_parser() -> ModuleType:
6368
return _parser
6469

6570
@staticmethod
66-
def _regex_nodes(pat: str | re.Pattern[str]) -> list[tuple[Any, Any]]:
71+
def _regex_nodes(pat: str) -> list[tuple[Any, Any]]:
6772
"""
6873
Parse a regex pattern into a flat list of its (op, arg) nodes.
6974
@@ -97,9 +102,8 @@ def walk(tokens):
97102
elif op_code in (_parser.ATOMIC_GROUP, _parser.IN):
98103
yield from walk(argument)
99104

100-
str_pat = pat.pattern if isinstance(pat, re.Pattern) else pat
101105
try:
102-
tokens = _parser.parse(str_pat)
106+
tokens = _parser.parse(pat)
103107
except re.error:
104108
# Pattern not valid for Python's re (e.g. RE2 syntax like \x{...} or \p)
105109
# Let the pyarrow backend handle it.
@@ -110,11 +114,19 @@ def _needs_python_regex(self, pat: str | re.Pattern[str]) -> bool:
110114
"""
111115
Whether a regex operation must use Python's ``re`` instead of pyarrow.
112116
113-
This is the case for features not supported by RE2 / pyarrow (lookaround
114-
assertions and backreferences), and for ``\\w``, ``\\d``, ``\\s`` and
115-
``\\b`` (or their negations) on non-ASCII data, since RE2 only matches
116-
them against ASCII characters while Python's ``re`` is Unicode-aware.
117+
This is the case for features not supported by RE2 / pyarrow or with
118+
different behaviour:
119+
- Not supported: lookaround assertions and backreferences, unicode escape
120+
sequences like ``\\u``, ``\\U`` and ``\\N{}``
121+
- ``\\w``, ``\\d``, ``\\s`` and ``\\b`` (or their negations) on non-ASCII data,
122+
since RE2 only matches them against ASCII characters while Python's ``re``
123+
is Unicode-aware
124+
117125
"""
126+
str_pat = pat.pattern if isinstance(pat, re.Pattern) else pat
127+
if re.search(_escaped_unicode_char, str_pat) is not None:
128+
return True
129+
118130
_parser = self._re_parser()
119131
unsupported = {_parser.ASSERT, _parser.ASSERT_NOT, _parser.GROUPREF}
120132
unicode_sensitive = {
@@ -128,7 +140,7 @@ def _needs_python_regex(self, pat: str | re.Pattern[str]) -> bool:
128140
(_parser.AT, _parser.AT_NON_BOUNDARY),
129141
}
130142

131-
nodes = self._regex_nodes(pat)
143+
nodes = self._regex_nodes(str_pat)
132144
if any(op_code in unsupported for op_code, _ in nodes):
133145
return True
134146
if not any(

‎pandas/tests/strings/test_find_replace.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -448,6 +448,32 @@ def test_contains_end_of_string_not_regex(any_string_dtype):
448448
tm.assert_series_equal(result, expected)
449449

450450

451+
def test_contains_unicode_char(any_string_dtype):
452+
# https://github.com/pandas-dev/pandas/issues/63901
453+
expected_dtype = (
454+
np.bool_ if is_object_or_nan_string_dtype(any_string_dtype) else "boolean"
455+
)
456+
457+
ser = pd.Series(["a", "ก"], dtype=any_string_dtype)
458+
result = ser.str.contains(r"[\u0e00-\u0e7f]")
459+
expected = pd.Series([False, True], dtype=expected_dtype)
460+
tm.assert_series_equal(result, expected)
461+
462+
# different unicode escape patterns (\u, \U, \N) should all work
463+
ser = pd.Series(["a", "€"], dtype=any_string_dtype)
464+
for pat in [r"\u20AC", r"\U000020AC", r"\N{EURO SIGN}"]:
465+
result = ser.str.contains(pat)
466+
expected = pd.Series([False, True], dtype=expected_dtype)
467+
tm.assert_series_equal(result, expected)
468+
469+
# invalid unicode escape gives clear error
470+
with pytest.raises(re.error, match="incomplete escape"):
471+
ser.str.contains(r"\u20G")
472+
473+
with pytest.raises(re.error, match=r"bad escape \\p"):
474+
ser.str.contains(r"\pL\u20AC")
475+
476+
451477
# --------------------------------------------------------------------------------------
452478
# str.startswith
453479
# --------------------------------------------------------------------------------------
@@ -1091,6 +1117,15 @@ def test_replace_end_of_string(any_string_dtype):
10911117
tm.assert_series_equal(result, expected)
10921118

10931119

1120+
@pytest.mark.parametrize("pat", [r"\u20AC", r"\U000020AC", r"\N{EURO SIGN}"])
1121+
def test_replace_unicode_escape_patterns(any_string_dtype, pat):
1122+
# https://github.com/pandas-dev/pandas/issues/63901
1123+
ser = pd.Series(["a", "€"], dtype=any_string_dtype)
1124+
result = ser.str.replace(pat, "X", regex=True)
1125+
expected = pd.Series(["a", "X"], dtype=any_string_dtype)
1126+
tm.assert_series_equal(result, expected)
1127+
1128+
10941129
def test_replace_empty_pattern(any_string_dtype):
10951130
# https://github.com/pandas-dev/pandas/issues/64941
10961131
ser = pd.Series(["abcd"], dtype=any_string_dtype)
@@ -1492,6 +1527,22 @@ def test_match_end_of_string(any_string_dtype):
14921527
tm.assert_series_equal(result, expected)
14931528

14941529

1530+
@pytest.mark.parametrize("pat", [r"\u20AC", r"\U000020AC", r"\N{EURO SIGN}"])
1531+
def test_match_unicode_escape_patterns(any_string_dtype, pat):
1532+
# https://github.com/pandas-dev/pandas/issues/63901
1533+
expected_dtype = (
1534+
np.bool_ if is_object_or_nan_string_dtype(any_string_dtype) else "boolean"
1535+
)
1536+
1537+
ser = pd.Series(["a", "€"], dtype=any_string_dtype)
1538+
result = ser.str.match(pat)
1539+
expected = pd.Series([False, True], dtype=expected_dtype)
1540+
tm.assert_series_equal(result, expected)
1541+
1542+
result = ser.str.fullmatch(pat)
1543+
tm.assert_series_equal(result, expected)
1544+
1545+
14951546
# --------------------------------------------------------------------------------------
14961547
# str.fullmatch
14971548
# --------------------------------------------------------------------------------------
@@ -1797,6 +1848,15 @@ def test_findall_end_of_string(any_string_dtype):
17971848
tm.assert_series_equal(result, expected)
17981849

17991850

1851+
@pytest.mark.parametrize("pat", [r"\u20AC", r"\U000020AC", r"\N{EURO SIGN}"])
1852+
def test_findall_unicode_escape_patterns(any_string_dtype, pat):
1853+
# https://github.com/pandas-dev/pandas/issues/63901
1854+
ser = pd.Series(["a", "10€", "10€ and 11€ and 12$"], dtype=any_string_dtype)
1855+
result = ser.str.findall(r"[0-9]*" + pat)
1856+
expected = pd.Series([[], ["10€"], ["10€", "11€"]], dtype=object)
1857+
tm.assert_series_equal(result, expected)
1858+
1859+
18001860
# --------------------------------------------------------------------------------------
18011861
# str.find
18021862
# --------------------------------------------------------------------------------------

‎pandas/tests/strings/test_strings.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,19 @@ def test_count_end_of_string(any_string_dtype):
113113
tm.assert_series_equal(result, expected)
114114

115115

116+
@pytest.mark.parametrize("pat", [r"\u20AC", r"\U000020AC", r"\N{EURO SIGN}"])
117+
def test_count_unicode_escape_patterns(any_string_dtype, pat):
118+
# https://github.com/pandas-dev/pandas/issues/63901
119+
expected_dtype = (
120+
"int64" if is_object_or_nan_string_dtype(any_string_dtype) else "Int64"
121+
)
122+
123+
ser = pd.Series(["a", "10€", "10€ and 11€ and 12$"], dtype=any_string_dtype)
124+
result = ser.str.count(r"[0-9]*" + pat)
125+
expected = pd.Series([0, 1, 2], dtype=expected_dtype)
126+
tm.assert_series_equal(result, expected)
127+
128+
116129
def test_count_compiled_pattern_with_flags(any_string_dtype):
117130
# GH#63705 flags on a compiled pattern must not be dropped
118131
expected_dtype = (
@@ -1023,6 +1036,19 @@ def test_setitem_with_different_string_storage():
10231036
(r"(abc)\1", True),
10241037
(r"\b(\w+)\s+\1\b", True),
10251038
(r"\b(?P<word>\w+)\s+(?P=word)\b", True),
1039+
# unicode escape sequences
1040+
(r"\u20AC", True),
1041+
(r"\U000020AC", True),
1042+
(r"\N{EURO SIGN}", True),
1043+
(r"\\u20AC", False),
1044+
(r"\\U000020AC", False),
1045+
(r"\\N{EURO SIGN}", False),
1046+
(r"\NEURO", False),
1047+
(r"\\\u20AC", True),
1048+
(r"\\\\u20AC", False),
1049+
(r"[\u0e00-\u0e7f]", True),
1050+
(r"\pL\u20AC", True), # also includes an invalid escape \p for Python
1051+
(r"\Q\u\E", True), # \Q..\E pattern supported by pyarrow, but prioritize \u
10261052
],
10271053
)
10281054
def test_needs_python_regex_unsupported_code(pat, expected):

0 commit comments

Comments
 (0)