Skip to content

Commit 55089f8

Browse files
Backport PR #71025 on branch 3.1.x (BUG: \s and \S in str regex methods missing Python whitespace characters for pyarrow-backed strings) (#71895)
Co-authored-by: Maha Rana Yadavalli <[email protected]>
1 parent eea23d0 commit 55089f8

4 files changed

Lines changed: 101 additions & 9 deletions

File tree

‎doc/source/whatsnew/v3.1.1.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ Bug fixes
2424
~~~~~~~~~
2525

2626
- Fixed ``AttributeError`` when using dateutil timezones such as ``"dateutil/US/Pacific"`` with python-dateutil 3.0 (:issue:`71459`)
27+
- Fixed bug in string methods using regex patterns on PyArrow-backed string dtypes where ``\s`` incorrectly did not match (and ``\S`` matched) whitespace such as ``"\x0b"`` and ``"\x1c"`` in all-ASCII data (:issue:`70991`)
2728

2829
.. ---------------------------------------------------------------------------
2930
.. _whatsnew_311.packaging.build:

‎pandas/core/arrays/_arrow_string_mixins.py‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -118,9 +118,11 @@ def _needs_python_regex(self, pat: str | re.Pattern[str]) -> bool:
118118
different behaviour:
119119
- Not supported: lookaround assertions and backreferences, unicode escape
120120
sequences like ``\\u``, ``\\U`` and ``\\N{}``
121-
- ``\\w``, ``\\d``, ``\\s`` and ``\\b`` (or their negations) on non-ASCII data,
122-
since RE2 only matches them against ASCII characters while Python's ``re``
123-
is Unicode-aware
121+
- ``\\s`` and ``\\S``, since RE2's ``\\s`` misses whitespace such as
122+
``"\\x0b"`` even in ASCII data
123+
- ``\\w``, ``\\d`` and ``\\b`` (or their negations) on non-ASCII data, since
124+
RE2 only matches them against ASCII characters while Python's ``re`` is
125+
Unicode-aware
124126
- At-end-of-string ``$`` assertion, which Python matches at the end or just
125127
before a trailing newline, while pyarrow only matches at the end
126128
(only falling back if newlines are present)
@@ -131,14 +133,17 @@ def _needs_python_regex(self, pat: str | re.Pattern[str]) -> bool:
131133

132134
_parser = self._re_parser()
133135
unsupported_ops = {_parser.ASSERT, _parser.ASSERT_NOT, _parser.GROUPREF}
136+
whitespace = {
137+
# RE2's \s misses whitespace such as "\x0b", even in ASCII data
138+
(_parser.CATEGORY, _parser.CATEGORY_SPACE),
139+
(_parser.CATEGORY, _parser.CATEGORY_NOT_SPACE),
140+
}
134141
unicode_sensitive = {
135142
# unicode-sensitive categories and assertions
136143
(_parser.CATEGORY, _parser.CATEGORY_WORD),
137144
(_parser.CATEGORY, _parser.CATEGORY_NOT_WORD),
138145
(_parser.CATEGORY, _parser.CATEGORY_DIGIT),
139146
(_parser.CATEGORY, _parser.CATEGORY_NOT_DIGIT),
140-
(_parser.CATEGORY, _parser.CATEGORY_SPACE),
141-
(_parser.CATEGORY, _parser.CATEGORY_NOT_SPACE),
142147
(_parser.AT, _parser.AT_BOUNDARY),
143148
(_parser.AT, _parser.AT_NON_BOUNDARY),
144149
}
@@ -150,6 +155,12 @@ def _needs_python_regex(self, pat: str | re.Pattern[str]) -> bool:
150155
nodes = self._regex_nodes(str_pat)
151156
if any(op_code in unsupported_ops for op_code, _ in nodes):
152157
return True
158+
if any(
159+
(op_code, argument) in whitespace
160+
for op_code, argument in nodes
161+
if op_code == _parser.CATEGORY
162+
):
163+
return True
153164
if any(
154165
(op_code, argument) in unicode_sensitive
155166
for op_code, argument in nodes

‎pandas/tests/strings/test_find_replace.py‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
from datetime import datetime
22
import re
3+
import sys
34

45
import numpy as np
56
import pytest
@@ -488,6 +489,79 @@ def test_contains_unicode_char(any_string_dtype):
488489
ser.str.contains(r"\pL\u20AC")
489490

490491

492+
@pytest.mark.parametrize("pat", [r"\s", r"\S"])
493+
def test_fullmatch_every_python_whitespace_char(any_string_dtype, pat):
494+
# GH#70991 RE2's \s only matches [\t\n\f\r ], Python's every char for which
495+
# str.isspace() is True
496+
expected_dtype = (
497+
np.bool_ if is_object_or_nan_string_dtype(any_string_dtype) else "boolean"
498+
)
499+
whitespace = [
500+
char for char in map(chr, range(sys.maxunicode + 1)) if char.isspace()
501+
]
502+
data = [*whitespace, "a", "\u200b"]
503+
ser = pd.Series(data, dtype=any_string_dtype)
504+
result = ser.str.fullmatch(pat)
505+
is_space = pat == r"\s"
506+
expected = pd.Series(
507+
[is_space] * len(whitespace) + [not is_space] * 2, dtype=expected_dtype
508+
)
509+
tm.assert_series_equal(result, expected)
510+
511+
512+
@pytest.mark.parametrize(
513+
"method, kwargs",
514+
[
515+
("contains", {}),
516+
("match", {}),
517+
("fullmatch", {}),
518+
("count", {}),
519+
("replace", {"repl": "_", "regex": True}),
520+
],
521+
)
522+
@pytest.mark.parametrize(
523+
"pat",
524+
[
525+
r"\s",
526+
r"\S",
527+
r"a\s+b",
528+
r"\S\s\S",
529+
r"[\s,]",
530+
r"[^\s]",
531+
r"[\S]",
532+
r"[\S\d]",
533+
r"[^\S,]",
534+
r"[\s\S]",
535+
r"[^\s\S]",
536+
r"[]\s]",
537+
r"[^^\S]",
538+
r"[^]\S]",
539+
r"\\s",
540+
],
541+
)
542+
def test_regex_python_whitespace(any_string_dtype, method, kwargs, pat):
543+
# GH#70991
544+
data = [
545+
"a\x0bb",
546+
"a\x1cb",
547+
"a\x1fb",
548+
"a b",
549+
"a,b",
550+
"a1b",
551+
"a]b",
552+
"a^b",
553+
r"a\sb",
554+
"ab",
555+
"\x0b",
556+
"x",
557+
"",
558+
]
559+
ser = pd.Series(data, dtype=any_string_dtype)
560+
result = getattr(ser.str, method)(pat, **kwargs)
561+
expected = getattr(pd.Series(data, dtype=object).str, method)(pat, **kwargs)
562+
assert result.tolist() == expected.tolist()
563+
564+
491565
# --------------------------------------------------------------------------------------
492566
# str.startswith
493567
# --------------------------------------------------------------------------------------

‎pandas/tests/strings/test_strings.py‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1062,6 +1062,14 @@ def test_setitem_with_different_string_storage():
10621062
(r"[\u0e00-\u0e7f]", True),
10631063
(r"\pL\u20AC", True), # also includes an invalid escape \p for Python
10641064
(r"\Q\u\E", True), # \Q..\E pattern supported by pyarrow, but prioritize \u
1065+
# RE2's \s misses ASCII whitespace such as "\x0b" (GH#70991)
1066+
(r"\s+", True),
1067+
(r"\S", True),
1068+
(r"[^\w\s]", True),
1069+
(r"(?:x\S)*", True),
1070+
(r"(a|\s)", True),
1071+
(r"\\s", False),
1072+
(r"[sS]", False),
10651073
],
10661074
)
10671075
def test_needs_python_regex_unsupported_code(pat, expected):
@@ -1078,13 +1086,11 @@ def test_needs_python_regex_unsupported_code(pat, expected):
10781086
(r"\W", True),
10791087
(r"\d{3}", True),
10801088
(r"\D", True),
1081-
(r"\s+", True),
1082-
(r"\S", True),
10831089
(r"\bword\b", True),
10841090
(r"\B", True),
1085-
(r"[^\w\s]", True),
1091+
(r"[^\w,]", True),
10861092
(r"[\w-]", True),
1087-
(r"(?:x\S)*", True),
1093+
(r"(?:x\D)*", True),
10881094
(r"(foo|bar\d)", True),
10891095
(r"(?>\w)", True),
10901096
(r"[a-z0-9_]+", False),

0 commit comments

Comments
 (0)