Skip to content

Commit 76381d5

Browse files
authored
Add combine_chunks option to Python Arrow export (#10279)
Pyarrow export can combine chunks on rust side instead of exporting them as pa.chunked_array Signed-off-by: Robert Kruszewski <[email protected]>
1 parent 4435d5b commit 76381d5

4 files changed

Lines changed: 68 additions & 7 deletions

File tree

‎vortex-python/python/vortex/_lib/arrays.pyi‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,9 @@ class Array:
2525
) -> Array: ...
2626
@staticmethod
2727
def from_range(obj: range, *, dtype: DType | None = None) -> Array: ...
28-
def to_arrow_array(self, *, arrow_type: pa.DataType | None = None) -> pa.Array[pa.Scalar[pa.DataType]]: ...
28+
def to_arrow_array(
29+
self, *, arrow_type: pa.DataType | None = None, combine_chunks: bool = False
30+
) -> pa.Array[pa.Scalar[pa.DataType]]: ...
2931
def __vortex_array_metadata__(self) -> tuple[str, bytes, int, bytes, list[object], list[object]]: ...
3032
@property
3133
def id(self) -> str: ...
@@ -38,7 +40,7 @@ class Array:
3840
def take(self, indices: Array) -> Array: ...
3941
def slice(self, start: int, end: int) -> Array: ...
4042
def display_tree(self) -> str: ...
41-
def to_arrow_table(self) -> pa.Table: ...
43+
def to_arrow_table(self, *, combine_chunks: bool = False) -> pa.Table: ...
4244
def to_numpy(self, *, zero_copy_only: bool = True) -> np.ndarray: ...
4345
def to_pandas(self) -> pd.DataFrame: ...
4446
def to_polars_dataframe(self) -> pl.DataFrame: ...

‎vortex-python/python/vortex/arrays.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ def arrow_table_from_struct_array(
6262
return pyarrow.Table.from_struct_array(array)
6363

6464

65-
def _Array_to_arrow_table(self: _arrays.Array) -> pyarrow.Table:
65+
def _Array_to_arrow_table(self: _arrays.Array, *, combine_chunks: bool = False) -> pyarrow.Table:
6666
"""Construct an Arrow table from this Vortex array.
6767
6868
.. seealso::
@@ -73,6 +73,12 @@ def _Array_to_arrow_table(self: _arrays.Array) -> pyarrow.Table:
7373
7474
Only struct-typed arrays can be converted to Arrow tables.
7575
76+
Parameters
77+
----------
78+
combine_chunks : :class:`bool`, optional
79+
If ``True``, a chunked Vortex array is concatenated natively during export so that every
80+
column of the resulting table has a single chunk. Defaults to ``False``.
81+
7682
Returns
7783
-------
7884
@@ -96,7 +102,7 @@ def _Array_to_arrow_table(self: _arrays.Array) -> pyarrow.Table:
96102
age: [[25,31,33,57]]
97103
98104
"""
99-
array = self.to_arrow_array()
105+
array = self.to_arrow_array(combine_chunks=combine_chunks)
100106
assert isinstance(array, pyarrow.StructArray | pyarrow.ChunkedArray)
101107
return arrow_table_from_struct_array(array)
102108

‎vortex-python/src/arrays/mod.rs‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -437,10 +437,17 @@ impl PyArray {
437437
/// arrow_type : :class:`pyarrow.DataType`, optional
438438
/// The Arrow type to return. By default, UTF-8 data returns a ``StringViewArray`` and
439439
/// binary data returns a ``BinaryViewArray``.
440+
/// combine_chunks : :class:`bool`, optional
441+
/// If ``True``, a chunked Vortex array is exported as a single contiguous
442+
/// :class:`pyarrow.Array` instead of a :class:`pyarrow.ChunkedArray`. The chunks are
443+
/// concatenated natively during export, which is faster than calling
444+
/// :meth:`pyarrow.ChunkedArray.combine_chunks` on the result. Defaults to ``False``.
440445
///
441446
/// Returns
442447
/// -------
443-
/// :class:`pyarrow.Array`
448+
/// :class:`pyarrow.Array` or :class:`pyarrow.ChunkedArray`
449+
/// A :class:`pyarrow.ChunkedArray` is returned only for chunked arrays when
450+
/// ``combine_chunks`` is ``False``.
444451
///
445452
/// Examples
446453
/// --------
@@ -469,10 +476,26 @@ impl PyArray {
469476
/// "world"
470477
/// ]
471478
/// ```
472-
#[pyo3(signature = (*, arrow_type = None))]
479+
///
480+
/// Export a chunked array as a single contiguous Arrow array:
481+
///
482+
/// ```python
483+
/// >>> import pyarrow
484+
/// >>> import vortex as vx
485+
/// >>> chunked = vx.array(pyarrow.chunked_array([[1, 2], [3]]))
486+
/// >>> chunked.to_arrow_array(combine_chunks=True)
487+
/// <pyarrow.lib.Int64Array object at ...>
488+
/// [
489+
/// 1,
490+
/// 2,
491+
/// 3
492+
/// ]
493+
/// ```
494+
#[pyo3(signature = (*, arrow_type = None, combine_chunks = false))]
473495
fn to_arrow_array<'py>(
474496
self_: &'py Bound<'py, Self>,
475497
arrow_type: Option<&Bound<'py, PyAny>>,
498+
combine_chunks: bool,
476499
) -> PyVortexResult<Bound<'py, PyAny>> {
477500
// NOTE(ngates): for struct arrays, we could also return a RecordBatchStreamReader.
478501
let array = PyArrayRef::extract(self_.as_any().as_borrowed())?.into_inner();
@@ -482,7 +505,9 @@ impl PyArray {
482505
.transpose()?
483506
.map(|data_type| Field::new("", data_type, array.dtype().is_nullable()));
484507

485-
if let Some(chunked_array) = array.as_opt::<Chunked>() {
508+
if let Some(chunked_array) = array.as_opt::<Chunked>()
509+
&& !combine_chunks
510+
{
486511
// We figure out a single Arrow Data Type to convert all chunks into, otherwise
487512
// the preferred type of each chunk may be different.
488513
let inferred_field;

‎vortex-python/test/test_array.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,3 +85,31 @@ def test_unsupported_arrow_type_raises_value_error(arrow_type: pa.DataType) -> N
8585
table = pa.table({"c0": pa.array([], type=arrow_type)})
8686
with pytest.raises(ValueError):
8787
_ = vortex.array(table)
88+
89+
90+
@pytest.mark.parametrize(
91+
("chunks", "arrow_type"),
92+
[
93+
([[1, 2], [3, None]], None),
94+
([["a", "b"], ["c"]], pa.string()),
95+
([[{"x": 1}, {"x": 2}], [{"x": 3}]], None),
96+
],
97+
)
98+
def test_chunked_array_combine_chunks(chunks: list[list[object]], arrow_type: pa.DataType | None) -> None:
99+
arr = vortex.array(pa.chunked_array(chunks))
100+
assert isinstance(arr, vortex.ChunkedArray)
101+
102+
chunked = arr.to_arrow_array(arrow_type=arrow_type)
103+
assert isinstance(chunked, pa.ChunkedArray)
104+
assert chunked.num_chunks == len(chunks)
105+
106+
combined = arr.to_arrow_array(arrow_type=arrow_type, combine_chunks=True)
107+
assert isinstance(combined, pa.Array)
108+
assert combined.equals(chunked.combine_chunks())
109+
110+
111+
def test_chunked_struct_to_arrow_table_combine_chunks() -> None:
112+
arr = vortex.array(pa.chunked_array([[{"x": 1}], [{"x": 2}, {"x": 3}]]))
113+
table = arr.to_arrow_table(combine_chunks=True)
114+
assert table.column("x").num_chunks == 1
115+
assert table.column("x").to_pylist() == [1, 2, 3]

0 commit comments

Comments
 (0)