Skip to content

Commit 49dca6e

Browse files
kirilkleinclaude
andcommitted
apacheGH-26685: [Python][C++] Trim buffers when pickling a sliced array
Pickling a sliced array previously serialized the array's entire parent buffers instead of just the referenced slice. Add arrow::internal::TrimArrayDataBuffers, which compacts a sliced array (offset != 0) to its referenced range via Concatenate, and call it from Array.__reduce__ so pickling only serializes referenced bytes. Unsliced arrays are returned untouched, preserving zero-copy / protocol-5 out-of-band pickling. ChunkedArray/RecordBatch/Table inherit the fix. Co-Authored-By: Claude Opus 4.8 <[email protected]>
1 parent 3c5c227 commit 49dca6e

5 files changed

Lines changed: 92 additions & 1 deletion

File tree

‎cpp/src/arrow/array/util.cc‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1000,5 +1000,21 @@ std::vector<ArrayVector> RechunkArraysConsistently(
10001000
return rechunked_groups;
10011001
}
10021002

1003+
Result<std::shared_ptr<ArrayData>> TrimArrayDataBuffers(
1004+
const std::shared_ptr<ArrayData>& data, MemoryPool* pool) {
1005+
if (data->offset == 0) {
1006+
// An array with no offset owns its buffers from the start (any excess is
1007+
// just allocator padding), so there is nothing worth trimming. Returning it
1008+
// unchanged keeps pickling zero-copy / protocol-5 out-of-band friendly.
1009+
return data;
1010+
}
1011+
// A sliced array (offset != 0) shares its parent's buffers; compact it so only
1012+
// the referenced range is serialized. Concatenating the single array yields an
1013+
// equivalent array whose buffers start at offset 0, correctly handling every
1014+
// nested, variable-length and dictionary type.
1015+
ARROW_ASSIGN_OR_RAISE(auto trimmed, Concatenate({MakeArray(data)}, pool));
1016+
return trimmed->data();
1017+
}
1018+
10031019
} // namespace internal
10041020
} // namespace arrow

‎cpp/src/arrow/array/util.h‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,5 +92,21 @@ Result<std::shared_ptr<ArrayData>> SwapEndianArrayData(
9292
ARROW_EXPORT
9393
std::vector<ArrayVector> RechunkArraysConsistently(const std::vector<ArrayVector>&);
9494

95+
/// \brief Return an equivalent ArrayData whose buffers are trimmed to the range
96+
/// the array actually references (its offset/length window), recursively for
97+
/// child and dictionary data.
98+
///
99+
/// A sliced array shares its parent's (potentially large) buffers and only
100+
/// references a window of them. Serializing the raw buffers -- e.g. pickling in
101+
/// PyArrow (GH-26685) -- would otherwise copy the whole parent buffers. If the
102+
/// buffers are already minimal, \p data is returned unchanged without copying.
103+
///
104+
/// \param[in] data the array contents
105+
/// \param[in] pool the memory pool to allocate trimmed buffers from
106+
/// \return the trimmed ArrayData (or \p data itself if already minimal)
107+
ARROW_EXPORT
108+
Result<std::shared_ptr<ArrayData>> TrimArrayDataBuffers(
109+
const std::shared_ptr<ArrayData>& data, MemoryPool* pool = default_memory_pool());
110+
95111
} // namespace internal
96112
} // namespace arrow

‎python/pyarrow/array.pxi‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1324,8 +1324,13 @@ cdef class Array(_PandasConvertible):
13241324

13251325
def __reduce__(self):
13261326
self._assert_cpu()
1327+
# Trim the buffers to the range this (possibly sliced) array references
1328+
# so we don't serialize the whole parent buffers (GH-26685).
13271329
return _restore_array, \
1328-
(_reduce_array_data(self.sp_array.get().data().get()),)
1330+
(_reduce_array_data(
1331+
GetResultValue(TrimArrayDataBuffers(
1332+
self.sp_array.get().data(),
1333+
c_default_memory_pool())).get()),)
13291334

13301335
@staticmethod
13311336
def from_buffers(DataType type, length, buffers, null_count=-1, offset=0,

‎python/pyarrow/includes/libarrow.pxd‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3318,3 +3318,7 @@ cdef extern from "arrow/python/udf.h" namespace "arrow::py" nogil:
33183318

33193319
cdef extern from "arrow/compute/cast.h" namespace "arrow::compute":
33203320
CResult[CDatum] Cast(const CDatum& value, const CCastOptions& options)
3321+
3322+
cdef extern from "arrow/array/util.h" namespace "arrow::internal" nogil:
3323+
CResult[shared_ptr[CArrayData]] TrimArrayDataBuffers(
3324+
const shared_ptr[CArrayData]& data, CMemoryPool* pool)

‎python/pyarrow/tests/test_array.py‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import hypothesis as h
2222
import hypothesis.strategies as st
2323
import itertools
24+
import pickle
2425
import pytest
2526
import struct
2627
import subprocess
@@ -4612,3 +4613,52 @@ def test_dictionary_uint64_index_to_pandas():
46124613
result = arr.to_pandas()
46134614
assert list(result.cat.categories) == ["a", "b"]
46144615
assert result.cat.codes.tolist() == [0, 1, -1, 0]
4616+
4617+
4618+
@pytest.mark.parametrize("typ", [
4619+
pa.int64(),
4620+
pa.float64(),
4621+
pa.bool_(),
4622+
pa.string(),
4623+
pa.large_string(),
4624+
pa.list_(pa.int64()),
4625+
])
4626+
def test_pickle_sliced_array_does_not_copy_parent_buffers(typ):
4627+
# GH-26685: pickling a small slice of a large array must not serialize the
4628+
# whole parent buffers.
4629+
n = 100_000
4630+
if pa.types.is_boolean(typ):
4631+
arr = pa.array([i % 2 == 0 for i in range(n)])
4632+
elif pa.types.is_string(typ) or pa.types.is_large_string(typ):
4633+
arr = pa.array(["x%d" % i for i in range(n)], type=typ)
4634+
elif pa.types.is_list(typ):
4635+
arr = pa.array([[i] for i in range(n)], type=typ)
4636+
else:
4637+
arr = pa.array(range(n), type=typ)
4638+
4639+
sliced = arr.slice(10, 5)
4640+
for protocol in (4, 5):
4641+
data = pickle.dumps(sliced, protocol=protocol)
4642+
# A 5-element slice serializes to a few hundred bytes; copying the
4643+
# parent buffers would be orders of magnitude larger.
4644+
assert len(data) < 2000
4645+
assert pickle.loads(data).equals(sliced)
4646+
4647+
4648+
def test_pickle_sliced_array_preserves_out_of_band_buffers():
4649+
# GH-26685: protocol-5 out-of-band buffers still work and carry only the
4650+
# referenced bytes, not the whole parent buffer.
4651+
arr = pa.array(range(100_000), type=pa.int64())
4652+
sliced = arr.slice(10, 5)
4653+
collected = []
4654+
data = pickle.dumps(sliced, protocol=5, buffer_callback=collected.append)
4655+
oob = sum(memoryview(b.raw()).nbytes for b in collected)
4656+
assert oob < arr.nbytes / 100
4657+
restored = pickle.loads(data, buffers=[b.raw() for b in collected])
4658+
assert restored.equals(sliced)
4659+
4660+
4661+
def test_pickle_unsliced_array_roundtrips():
4662+
# Regression: a non-sliced array is unaffected.
4663+
arr = pa.array(range(1000), type=pa.int64())
4664+
assert pickle.loads(pickle.dumps(arr)).equals(arr)

0 commit comments

Comments
 (0)