|
21 | 21 | import hypothesis as h |
22 | 22 | import hypothesis.strategies as st |
23 | 23 | import itertools |
| 24 | +import pickle |
24 | 25 | import pytest |
25 | 26 | import struct |
26 | 27 | import subprocess |
@@ -4612,3 +4613,52 @@ def test_dictionary_uint64_index_to_pandas(): |
4612 | 4613 | result = arr.to_pandas() |
4613 | 4614 | assert list(result.cat.categories) == ["a", "b"] |
4614 | 4615 | assert result.cat.codes.tolist() == [0, 1, -1, 0] |
| 4616 | + |
| 4617 | + |
| 4618 | +@pytest.mark.parametrize("typ", [ |
| 4619 | + pa.int64(), |
| 4620 | + pa.float64(), |
| 4621 | + pa.bool_(), |
| 4622 | + pa.string(), |
| 4623 | + pa.large_string(), |
| 4624 | + pa.list_(pa.int64()), |
| 4625 | +]) |
| 4626 | +def test_pickle_sliced_array_does_not_copy_parent_buffers(typ): |
| 4627 | + # GH-26685: pickling a small slice of a large array must not serialize the |
| 4628 | + # whole parent buffers. |
| 4629 | + n = 100_000 |
| 4630 | + if pa.types.is_boolean(typ): |
| 4631 | + arr = pa.array([i % 2 == 0 for i in range(n)]) |
| 4632 | + elif pa.types.is_string(typ) or pa.types.is_large_string(typ): |
| 4633 | + arr = pa.array(["x%d" % i for i in range(n)], type=typ) |
| 4634 | + elif pa.types.is_list(typ): |
| 4635 | + arr = pa.array([[i] for i in range(n)], type=typ) |
| 4636 | + else: |
| 4637 | + arr = pa.array(range(n), type=typ) |
| 4638 | + |
| 4639 | + sliced = arr.slice(10, 5) |
| 4640 | + for protocol in (4, 5): |
| 4641 | + data = pickle.dumps(sliced, protocol=protocol) |
| 4642 | + # A 5-element slice serializes to a few hundred bytes; copying the |
| 4643 | + # parent buffers would be orders of magnitude larger. |
| 4644 | + assert len(data) < 2000 |
| 4645 | + assert pickle.loads(data).equals(sliced) |
| 4646 | + |
| 4647 | + |
| 4648 | +def test_pickle_sliced_array_preserves_out_of_band_buffers(): |
| 4649 | + # GH-26685: protocol-5 out-of-band buffers still work and carry only the |
| 4650 | + # referenced bytes, not the whole parent buffer. |
| 4651 | + arr = pa.array(range(100_000), type=pa.int64()) |
| 4652 | + sliced = arr.slice(10, 5) |
| 4653 | + collected = [] |
| 4654 | + data = pickle.dumps(sliced, protocol=5, buffer_callback=collected.append) |
| 4655 | + oob = sum(memoryview(b.raw()).nbytes for b in collected) |
| 4656 | + assert oob < arr.nbytes / 100 |
| 4657 | + restored = pickle.loads(data, buffers=[b.raw() for b in collected]) |
| 4658 | + assert restored.equals(sliced) |
| 4659 | + |
| 4660 | + |
| 4661 | +def test_pickle_unsliced_array_roundtrips(): |
| 4662 | + # Regression: a non-sliced array is unaffected. |
| 4663 | + arr = pa.array(range(1000), type=pa.int64()) |
| 4664 | + assert pickle.loads(pickle.dumps(arr)).equals(arr) |
0 commit comments