Skip to content

Commit 2126193

Browse files
Backport PR #71387 on branch 3.1.x (BUG: fix string regex operations with $ end indicator (always match Python behaviour)) (#71462)
Co-authored-by: Joris Van den Bossche <[email protected]>
1 parent 4700f5d commit 2126193

5 files changed

Lines changed: 94 additions & 38 deletions

File tree

‎doc/source/whatsnew/v3.1.0.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1018,6 +1018,7 @@ Strings
10181018
- Bug in :meth:`Series.str.split` with :class:`ArrowDtype` ``string`` not inferring regex for multi-character patterns when ``regex=None``, causing the pattern to be treated as a literal instead of a regular expression (:issue:`58321`)
10191019
- Bug in :meth:`Series.unique`, :meth:`Index.unique` and :func:`factorize` on object dtype returning incorrect results when the values were not UTF-8 encodable, e.g. lone surrogates (:issue:`34550`)
10201020
- Bug in :meth:`Series.unique`, :meth:`Index.unique`, :meth:`Series.nunique`, :func:`factorize` and ``groupby`` on object dtype, and on ``str``/``string`` dtype backed by python storage, collapsing distinct strings that are identical up to an embedded NUL byte, e.g. ``""`` and ``"\x00"``, into a single value (:issue:`34551`)
1021+
- Bug in string methods using a regex pattern with ``$`` to indicate the end of the string: when using PyArrow-backed string dtypes, for certain cases this no longer matched before or after an ending newline; now it is consistent again with Python's ``re`` behaviour in all cases (:issue:`63683`)
10211022
- Bug in string methods using a regex pattern with unicode escape sequences (``\u``/``\U``/``\N{}``) with PyArrow-backed string dtypes failing, now working again as in pandas 2.x (:issue:`63901`)
10221023

10231024
Interval

‎pandas/core/arrays/_arrow_string_mixins.py‎

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -121,15 +121,18 @@ def _needs_python_regex(self, pat: str | re.Pattern[str]) -> bool:
121121
- ``\\w``, ``\\d``, ``\\s`` and ``\\b`` (or their negations) on non-ASCII data,
122122
since RE2 only matches them against ASCII characters while Python's ``re``
123123
is Unicode-aware
124-
124+
- At-end-of-string ``$`` assertion, which Python matches at the end or just
125+
before a trailing newline, while pyarrow only matches at the end
126+
(only falling back if newlines are present)
125127
"""
126128
str_pat = pat.pattern if isinstance(pat, re.Pattern) else pat
127129
if re.search(_escaped_unicode_char, str_pat) is not None:
128130
return True
129131

130132
_parser = self._re_parser()
131-
unsupported = {_parser.ASSERT, _parser.ASSERT_NOT, _parser.GROUPREF}
133+
unsupported_ops = {_parser.ASSERT, _parser.ASSERT_NOT, _parser.GROUPREF}
132134
unicode_sensitive = {
135+
# unicode-sensitive categories and assertions
133136
(_parser.CATEGORY, _parser.CATEGORY_WORD),
134137
(_parser.CATEGORY, _parser.CATEGORY_NOT_WORD),
135138
(_parser.CATEGORY, _parser.CATEGORY_DIGIT),
@@ -139,18 +142,31 @@ def _needs_python_regex(self, pat: str | re.Pattern[str]) -> bool:
139142
(_parser.AT, _parser.AT_BOUNDARY),
140143
(_parser.AT, _parser.AT_NON_BOUNDARY),
141144
}
145+
newline_sensitive = {
146+
# newline-sensitive behaviour of $
147+
(_parser.AT, _parser.AT_END),
148+
}
142149

143150
nodes = self._regex_nodes(str_pat)
144-
if any(op_code in unsupported for op_code, _ in nodes):
151+
if any(op_code in unsupported_ops for op_code, _ in nodes):
145152
return True
146-
if not any(
153+
if any(
147154
(op_code, argument) in unicode_sensitive
148155
for op_code, argument in nodes
149156
if op_code in (_parser.CATEGORY, _parser.AT)
150157
):
151-
return False
152-
# all-ASCII data keeps the faster pyarrow path
153-
return self._has_non_ascii()
158+
# all-ASCII data keeps the faster pyarrow path
159+
if self._has_non_ascii():
160+
return True
161+
if any(
162+
(op_code, argument) in newline_sensitive
163+
for op_code, argument in nodes
164+
if op_code == _parser.AT
165+
):
166+
if pc.any(pc.match_substring(self._pa_array, "\n"), min_count=0).as_py():
167+
return True
168+
169+
return False
154170

155171
def _has_non_ascii(self) -> bool:
156172
# min_count=0 so that an empty or all-null array reports False

‎pandas/tests/strings/test_extract.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -784,7 +784,7 @@ def test_extract_end_of_string(any_string_dtype):
784784
# https://github.com/pandas-dev/pandas/pull/63613
785785
ser = pd.Series(["aa", "abc", "bb\n"], dtype=any_string_dtype)
786786

787-
# with dollar sign
787+
# with dollar sign (matches $ before or after an ending newline)
788788
result = ser.str.extract("([ab]+)$")
789789
expected = pd.Series(["aa", np.nan, "bb"], dtype=any_string_dtype).to_frame()
790790
tm.assert_frame_equal(result, expected)

‎pandas/tests/strings/test_find_replace.py‎

Lines changed: 28 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -376,14 +376,9 @@ def test_contains_end_of_string(any_string_dtype):
376376

377377
ser = pd.Series(["baz", "bar", "bars", "bar\n"], dtype=any_string_dtype)
378378

379-
# with dollar sign
379+
# with dollar sign (matches $ before or after an ending newline)
380380
result = ser.str.contains("bar$")
381-
if any_string_dtype == "string" and any_string_dtype.storage == "pyarrow":
382-
# pyarrow (RE2) only matches $ at the very end of the line
383-
expected = pd.Series([False, True, False, False], dtype=expected_dtype)
384-
else:
385-
# python matches $ before or after an ending newline
386-
expected = pd.Series([False, True, False, True], dtype=expected_dtype)
381+
expected = pd.Series([False, True, False, True], dtype=expected_dtype)
387382
tm.assert_series_equal(result, expected)
388383

389384
# with \Z (ensure this is translated to \z for pyarrow)
@@ -448,6 +443,25 @@ def test_contains_end_of_string_not_regex(any_string_dtype):
448443
tm.assert_series_equal(result, expected)
449444

450445

446+
def test_contains_end_of_line(any_string_dtype):
447+
expected_dtype = (
448+
np.bool_ if is_object_or_nan_string_dtype(any_string_dtype) else "boolean"
449+
)
450+
451+
ser = pd.Series(
452+
["foo1", "foo1\n", "foo1\nbar", "foo1\nfoo2", "foo1\nfoo2\n"],
453+
dtype=any_string_dtype,
454+
)
455+
456+
result = ser.str.contains(r"foo.$")
457+
expected = pd.Series([True, True, False, True, True], dtype=expected_dtype)
458+
tm.assert_series_equal(result, expected)
459+
460+
result = ser.str.contains(r"foo.$", flags=re.MULTILINE)
461+
expected = pd.Series([True, True, True, True, True], dtype=expected_dtype)
462+
tm.assert_series_equal(result, expected)
463+
464+
451465
def test_contains_unicode_char(any_string_dtype):
452466
# https://github.com/pandas-dev/pandas/issues/63901
453467
expected_dtype = (
@@ -1095,14 +1109,9 @@ def test_replace_end_of_string(any_string_dtype):
10951109
# https://github.com/pandas-dev/pandas/pull/63613
10961110
ser = pd.Series(["baz", "bar", "bars", "bar\n"], dtype=any_string_dtype)
10971111

1098-
# with dollar sign
1112+
# with dollar sign (matches $ before or after an ending newline)
10991113
result = ser.str.replace("bar$", "x", regex=True)
1100-
if any_string_dtype == "string" and any_string_dtype.storage == "pyarrow":
1101-
# pyarrow (RE2) only matches $ at the very end of the line
1102-
expected = pd.Series(["baz", "x", "bars", "bar\n"], dtype=any_string_dtype)
1103-
else:
1104-
# python matches $ before or after an ending newline
1105-
expected = pd.Series(["baz", "x", "bars", "x\n"], dtype=any_string_dtype)
1114+
expected = pd.Series(["baz", "x", "bars", "x\n"], dtype=any_string_dtype)
11061115
tm.assert_series_equal(result, expected)
11071116

11081117
# with \Z (ensure this is translated to \z for pyarrow)
@@ -1505,14 +1514,9 @@ def test_match_end_of_string(any_string_dtype):
15051514

15061515
ser = pd.Series(["baz", "bar", "bars", "bar\n"], dtype=any_string_dtype)
15071516

1508-
# with dollar sign
1517+
# with dollar sign (matches $ before or after an ending newline)
15091518
result = ser.str.match("bar$")
1510-
if any_string_dtype == "string" and any_string_dtype.storage == "pyarrow":
1511-
# pyarrow (RE2) only matches $ at the very end of the line
1512-
expected = pd.Series([False, True, False, False], dtype=expected_dtype)
1513-
else:
1514-
# python matches $ before or after an ending newline
1515-
expected = pd.Series([False, True, False, True], dtype=expected_dtype)
1519+
expected = pd.Series([False, True, False, True], dtype=expected_dtype)
15161520
tm.assert_series_equal(result, expected)
15171521

15181522
# with \Z (ensure this is translated to \z for pyarrow)
@@ -1739,7 +1743,8 @@ def test_fullmatch_end_of_string(any_string_dtype):
17391743

17401744
ser = pd.Series(["baz", "bar", "bars", "bar\n"], dtype=any_string_dtype)
17411745

1742-
# with dollar sign (for fullmatch, no difference between python and pyarrow)
1746+
# with dollar sign (for fullmatch, no difference between python and pyarrow
1747+
# because the the newline needs to be in the pattern as well)
17431748
result = ser.str.fullmatch("bar$")
17441749
expected = pd.Series([False, True, False, False], dtype=expected_dtype)
17451750
tm.assert_series_equal(result, expected)
@@ -1831,7 +1836,7 @@ def test_findall_end_of_string(any_string_dtype):
18311836
# https://github.com/pandas-dev/pandas/pull/63613
18321837
ser = pd.Series(["baz", "bar", "bars", "bar\n"], dtype=any_string_dtype)
18331838

1834-
# with dollar sign
1839+
# with dollar sign (matches $ before or after an ending newline)
18351840
result = ser.str.findall("bar$")
18361841
expected = pd.Series([[], ["bar"], [], ["bar"]], dtype=object)
18371842
tm.assert_series_equal(result, expected)

‎pandas/tests/strings/test_strings.py‎

Lines changed: 41 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -91,14 +91,9 @@ def test_count_end_of_string(any_string_dtype):
9191

9292
ser = pd.Series(["baz", "bar", "bars", "bar\n"], dtype=any_string_dtype)
9393

94-
# with dollar sign
94+
# with dollar sign (matches $ before or after an ending newline)
9595
result = ser.str.count("bar$")
96-
if any_string_dtype == "string" and any_string_dtype.storage == "pyarrow":
97-
# pyarrow (RE2) only matches $ at the very end of the line
98-
expected = pd.Series([0, 1, 0, 0], dtype=expected_dtype)
99-
else:
100-
# python matches $ before or after an ending newline
101-
expected = pd.Series([0, 1, 0, 1], dtype=expected_dtype)
96+
expected = pd.Series([0, 1, 0, 1], dtype=expected_dtype)
10297
tm.assert_series_equal(result, expected)
10398

10499
# with \Z (ensure this is translated to \z for pyarrow)
@@ -113,6 +108,24 @@ def test_count_end_of_string(any_string_dtype):
113108
tm.assert_series_equal(result, expected)
114109

115110

111+
def test_count_end_of_line(any_string_dtype):
112+
expected_dtype = (
113+
"int64" if is_object_or_nan_string_dtype(any_string_dtype) else "Int64"
114+
)
115+
116+
ser = pd.Series(
117+
["foo1", "foo1\n", "foo1\nbar", "foo1\nfoo2", "foo1\nfoo2\n"],
118+
dtype=any_string_dtype,
119+
)
120+
121+
result = ser.str.count(r"foo.$")
122+
expected = pd.Series([1, 1, 0, 1, 1], dtype=expected_dtype)
123+
tm.assert_series_equal(result, expected)
124+
125+
result = ser.str.count(r"foo.$", flags=re.MULTILINE)
126+
expected = pd.Series([1, 1, 1, 2, 2], dtype=expected_dtype)
127+
128+
116129
@pytest.mark.parametrize("pat", [r"\u20AC", r"\U000020AC", r"\N{EURO SIGN}"])
117130
def test_count_unicode_escape_patterns(any_string_dtype, pat):
118131
# https://github.com/pandas-dev/pandas/issues/63901
@@ -1088,3 +1101,24 @@ def test_needs_python_regex_unicode_classes(pat, expected):
10881101
# all-ASCII data keeps the pyarrow path
10891102
ascii_arr = pd.array(["e", None], dtype=pd.StringDtype("pyarrow"))
10901103
assert not ascii_arr._needs_python_regex(pat)
1104+
1105+
1106+
@pytest.mark.parametrize(
1107+
"pat, expected",
1108+
[
1109+
(r"test$", True),
1110+
(r"test\$", False),
1111+
(r"r[$]", False),
1112+
],
1113+
)
1114+
def test_needs_python_regex_newline_sensitive(pat, expected):
1115+
# https://github.com/pandas-dev/pandas/pull/71387
1116+
pytest.importorskip("pyarrow")
1117+
arr = pd.array(["test\n"], dtype=pd.StringDtype("pyarrow"))
1118+
assert arr._needs_python_regex(pat) == expected
1119+
arr = pd.array(["test\ntest", None], dtype=pd.StringDtype("pyarrow"))
1120+
assert arr._needs_python_regex(pat) == expected
1121+
1122+
# data without newlines keeps the pyarrow path
1123+
arr_no_newline = pd.array(["e", None], dtype=pd.StringDtype("pyarrow"))
1124+
assert not arr_no_newline._needs_python_regex(pat)

0 commit comments

Comments
 (0)