Skip to content

Commit e3df0e8

Browse files
[backport 3.0.x] BUG: full-slice setitem into a pyarrow-backed array shared memory with the value (#67990) (#68916)
Co-authored-by: jbrockmendel <[email protected]>
1 parent b7735f6 commit e3df0e8

4 files changed

Lines changed: 203 additions & 1 deletion

File tree

‎asv_bench/benchmarks/indexing.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -529,7 +529,9 @@ def time_setitem_object_dtype(self):
529529

530530

531531
class SeriesSetitem:
532-
params = ["str"]
532+
# "str" is exempt from the null-slice defensive copy, so a fixed-width
533+
# pyarrow dtype is needed to keep that cost on the dashboard (GH#67990)
534+
params = ["str", "double[pyarrow]"]
533535
param_names = ["dtype"]
534536

535537
def setup(self, dtype):

‎doc/source/whatsnew/v3.0.6.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ Fixed regressions
1717
- Bug in :meth:`DataFrame.plot.bar` and :meth:`DataFrame.plot.barh` raising ``AttributeError`` for a regularly-spaced :class:`DatetimeIndex` whose ``freq`` attribute is unset (:issue:`66771`)
1818
- Bug in :meth:`Series.str.split`, :meth:`Series.str.rsplit`, :meth:`Series.str.partition`, and :meth:`Series.str.rpartition` on :class:`CategoricalDtype` Series with ``expand=False`` returning stringified lists/tuples instead of actual lists/tuples (:issue:`66341`)
1919
- Bug in :meth:`Timestamp.tz_convert` and :meth:`Timestamp.tz_localize` and their :class:`Index` and :class:`Series` counterparts returning a UTC offset a few minutes off for dates before the specified timezone adopted a standard time (typically early 20th century); this only affected older systems whose ``tzdata`` was compiled with a toolchain from between 2013 and 2018, such as Amazon Linux 2 and RHEL 8 and its derivatives (:issue:`67066`)
20+
- Bug in full-slice assignment (``arr[:] = value``) into a PyArrow-backed array where the result shared memory with ``value``, so later modifying ``value`` also changed the assigned-to object (:issue:`67990`)
2021
- Fixed bug in :func:`read_csv` with the ``c`` engine and ``low_memory=True`` where a mixed-dtype column selected by ``usecols`` raised ``IndexError`` instead of emitting a :class:`DtypeWarning` unless it was the first column in the file, and where that warning named the wrong column when the file had an index column (:issue:`67375`)
2122
- Fixed bug in :func:`read_csv` with the ``c`` engine and ``low_memory=True`` where the number reported next to each column name in the mixed-type :class:`DtypeWarning` was a running count of the warned columns rather than the column's position in the file (:issue:`67375`)
2223
- Regression in :meth:`Series.interpolate` with ``method="linear"`` and a pyarrow-backed dtype leaving consecutive and trailing missing values unfilled, and truncating interpolated values for integer dtypes; the result now matches the equivalent masked (e.g. ``Int64``) dtype, including the upcast to ``float64[pyarrow]`` (:issue:`65345`)

‎pandas/core/arrays/arrow/array.py‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
from pandas.compat import (
3434
HAS_PYARROW,
3535
PYARROW_MIN_VERSION,
36+
pa_version_under16p0,
3637
pa_version_under21p0,
3738
)
3839
from pandas.errors import Pandas4Warning
@@ -247,6 +248,62 @@ def to_pyarrow_type(
247248
return None
248249

249250

251+
def _is_varbinary_type(pa_type: pa.DataType) -> bool:
252+
"""
253+
Whether this is one of string, large_string, binary and large_binary.
254+
255+
pc.if_else misreads a non-zero offset for exactly these four, silently
256+
truncating values (GH#64320, https://github.com/apache/arrow/issues/49410).
257+
Other offset-carrying layouts such as list and map are unaffected.
258+
"""
259+
return (
260+
pa.types.is_string(pa_type)
261+
or pa.types.is_large_string(pa_type)
262+
or pa.types.is_binary(pa_type)
263+
or pa.types.is_large_binary(pa_type)
264+
)
265+
266+
267+
def _is_string_or_binary_view(typ):
268+
return not pa_version_under16p0 and (
269+
pa.types.is_string_view(typ) or pa.types.is_binary_view(typ)
270+
)
271+
272+
273+
def _boxing_may_borrow_memory(pa_type: pa.DataType) -> bool:
274+
"""
275+
Whether ``pa.array`` on this type can return a view on caller-owned memory.
276+
277+
Zero-copy over numpy/masked arrays for fixed-width layouts; character
278+
layouts always repack, so copying those would cost a full copy of the
279+
character data for no safety gain. Nested types may have a zero-copy child.
280+
"""
281+
return not (_is_varbinary_type(pa_type) or _is_string_or_binary_view(pa_type))
282+
283+
284+
def _copy_pyarrow_buffers(
285+
pa_array: pa.Array | pa.ChunkedArray,
286+
) -> pa.Array | pa.ChunkedArray:
287+
"""
288+
Return an equal array that owns its buffers (GH#67990).
289+
290+
``pa.concat_arrays`` reuses the ``dictionary`` child rather than copying it,
291+
so dictionary types are rebuilt from copies of both halves.
292+
"""
293+
if isinstance(pa_array, pa.ChunkedArray):
294+
return pa.chunked_array(
295+
[_copy_pyarrow_buffers(chunk) for chunk in pa_array.chunks],
296+
type=pa_array.type,
297+
)
298+
if pa.types.is_dictionary(pa_array.type):
299+
return pa.DictionaryArray.from_arrays(
300+
_copy_pyarrow_buffers(pa_array.indices),
301+
_copy_pyarrow_buffers(pa_array.dictionary),
302+
ordered=pa_array.type.ordered,
303+
)
304+
return pa.concat_arrays([pa_array])
305+
306+
250307
@set_module("pandas.arrays")
251308
class ArrowExtensionArray(
252309
OpsMixin,
@@ -2260,6 +2317,10 @@ def __setitem__(self, key, value) -> None:
22602317
and value.type == self._pa_array.type
22612318
and len(value) == len(self)
22622319
):
2320+
# GH#67990 this adopts ``value`` as our backing array, so copy
2321+
# first if the caller may still own and mutate its buffers.
2322+
if _boxing_may_borrow_memory(value.type):
2323+
value = _copy_pyarrow_buffers(value)
22632324
data = value
22642325
else:
22652326
data = self._if_else(True, value, self._pa_array)

‎pandas/tests/extension/test_arrow.py‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1673,6 +1673,144 @@ def test_setitem_null_slice(data):
16731673
tm.assert_extension_array_equal(result, expected)
16741674

16751675

1676+
@pytest.mark.parametrize(
1677+
"dtype, np_dtype",
1678+
[
1679+
("int64[pyarrow]", "int64"),
1680+
("uint8[pyarrow]", "uint8"),
1681+
("double[pyarrow]", "float64"),
1682+
("timestamp[ns][pyarrow]", "M8[ns]"),
1683+
("duration[ns][pyarrow]", "m8[ns]"),
1684+
],
1685+
)
1686+
def test_setitem_null_slice_no_alias(dtype, np_dtype):
1687+
# GH#67990 the null-slice fast path must not adopt a buffer the caller owns
1688+
expected = pd.array(np.array([10, 20, 30]).astype(np_dtype), dtype=dtype)
1689+
1690+
np_values = np.array([10, 20, 30]).astype(np_dtype)
1691+
arr = pd.array([None] * 3, dtype=dtype)
1692+
arr[:] = np_values
1693+
np_values[0] = np_values[1]
1694+
tm.assert_extension_array_equal(arr, expected)
1695+
1696+
ser = pd.Series(np.array([10, 20, 30]).astype(np_dtype))
1697+
arr = pd.array([None] * 3, dtype=dtype)
1698+
arr[:] = ser
1699+
ser.iloc[0] = ser.iloc[1]
1700+
tm.assert_extension_array_equal(arr, expected)
1701+
1702+
1703+
def test_setitem_null_slice_no_alias_masked():
1704+
# GH#67990 masked arrays are zero-copy through __arrow_array__
1705+
arr = pd.array([None] * 3, dtype="int64[pyarrow]")
1706+
values = pd.array([10, 20, 30], dtype="Int64")
1707+
arr[:] = values
1708+
values[0] = -1
1709+
expected = pd.array([10, 20, 30], dtype="int64[pyarrow]")
1710+
tm.assert_extension_array_equal(arr, expected)
1711+
1712+
1713+
@pytest.mark.parametrize(
1714+
"wrap",
1715+
[
1716+
lambda np_values: pa.array(np_values),
1717+
lambda np_values: pa.chunked_array([pa.array(np_values)]),
1718+
lambda np_values: pa.chunked_array(
1719+
[pa.array(np_values[:1]), pa.array(np_values[1:])]
1720+
),
1721+
lambda np_values: ArrowExtensionArray(pa.array(np_values)),
1722+
lambda np_values: pd.Series(ArrowExtensionArray(pa.array(np_values))),
1723+
],
1724+
ids=["array", "chunked", "chunked_multi", "extension_array", "series"],
1725+
)
1726+
def test_setitem_null_slice_no_alias_pyarrow(wrap):
1727+
# GH#67990 a pyarrow array is immutable, but its buffers can still be
1728+
# zero-copy over a numpy array the caller owns
1729+
np_values = np.array([10, 20, 30], dtype="int64")
1730+
arr = pd.array([None] * 3, dtype="int64[pyarrow]")
1731+
value = wrap(np_values)
1732+
arr[:] = value
1733+
np_values[0] = -1
1734+
expected = pd.array([10, 20, 30], dtype="int64[pyarrow]")
1735+
tm.assert_extension_array_equal(arr, expected)
1736+
# assert_extension_array_equal ignores chunking, so pin it separately
1737+
assert arr._pa_array.num_chunks == getattr(value, "num_chunks", 1)
1738+
1739+
1740+
@pytest.mark.parametrize(
1741+
"dtype",
1742+
[
1743+
pd.StringDtype("pyarrow", na_value=np.nan),
1744+
"string[pyarrow]",
1745+
"binary[pyarrow]",
1746+
ArrowDtype(pa.large_string()),
1747+
ArrowDtype(pa.large_binary()),
1748+
],
1749+
)
1750+
def test_setitem_null_slice_string_stays_zero_copy(dtype):
1751+
# GH#67990 the aliasing fix must not undo GH#64529/GH#64530: pa.array never
1752+
# packs character data into caller-owned memory, so it is not copied here
1753+
is_binary = "binary" in str(dtype)
1754+
values = pd.array(
1755+
[b"a", b"bb", b"ccc"] if is_binary else ["a", "bb", "ccc"], dtype=dtype
1756+
)
1757+
arr = pd.array([None] * 3, dtype=dtype)
1758+
arr[:] = values
1759+
# the character data is the last buffer for every one of these layouts
1760+
assert (
1761+
arr._pa_array.chunks[0].buffers()[-1].address
1762+
== values._pa_array.chunks[0].buffers()[-1].address
1763+
)
1764+
1765+
1766+
def test_setitem_null_slice_no_alias_dictionary():
1767+
# GH#67990 pa.concat_arrays reuses the dictionary child, so the values half
1768+
# of a dictionary type needs copying too
1769+
dtype = ArrowDtype(pa.dictionary(pa.int32(), pa.int64()))
1770+
np_values = np.array([100, 200], dtype="int64")
1771+
indices = pa.array(np.array([0, 1, 0], dtype="int32"))
1772+
value = pa.DictionaryArray.from_arrays(indices, pa.array(np_values))
1773+
1774+
arr = pd.array(value, dtype=dtype)
1775+
arr[:] = value
1776+
np_values[0] = -1
1777+
expected = pd.array(
1778+
pa.DictionaryArray.from_arrays(indices, pa.array([100, 200], type=pa.int64())),
1779+
dtype=dtype,
1780+
)
1781+
tm.assert_extension_array_equal(arr, expected)
1782+
1783+
1784+
@pytest.mark.parametrize("kind", ["list", "struct"])
1785+
def test_setitem_null_slice_no_alias_nested(kind):
1786+
# GH#67990 pa.concat_arrays does copy a nested type's children, which is why
1787+
# only dictionary needs the special case above
1788+
np_values = np.array([1, 2, 3, 4], dtype="int64")
1789+
child = pa.array(np_values)
1790+
if kind == "list":
1791+
value = pa.ListArray.from_arrays(pa.array([0, 2, 4], type=pa.int32()), child)
1792+
expected_data = [[1, 2], [3, 4]]
1793+
else:
1794+
value = pa.StructArray.from_arrays([child], names=["x"])
1795+
expected_data = [{"x": 1}, {"x": 2}, {"x": 3}, {"x": 4}]
1796+
dtype = ArrowDtype(value.type)
1797+
1798+
arr = pd.array(value, dtype=dtype)
1799+
arr[:] = value
1800+
np_values[0] = -1
1801+
assert arr.tolist() == expected_data
1802+
1803+
1804+
def test_setitem_null_slice_cow():
1805+
# GH#67990 full-slice assignment must not tie the two frames together
1806+
df = pd.DataFrame({"a": pd.array([1, 2, 3], dtype="int64[pyarrow]")})
1807+
other = pd.DataFrame({"b": [10, 20, 30]})
1808+
df.loc[:, "a"] = other["b"]
1809+
other.iloc[0, 0] = -777
1810+
expected = pd.DataFrame({"a": pd.array([10, 20, 30], dtype="int64[pyarrow]")})
1811+
tm.assert_frame_equal(df, expected)
1812+
1813+
16761814
def test_setitem_invalid_dtype(data):
16771815
# GH50248
16781816
pa_type = data._pa_array.type

0 commit comments

Comments
 (0)