Skip to content

Commit 8ff4b4a

Browse files
authored
fix(python): preserve nullability when converting a chunked array with nulls (#10283)
## What Converting a `pyarrow.ChunkedArray` whose chunks contain nulls raised an error, even though `ChunkedArray` is a documented input and the same data converts fine as a single `pyarrow.Array` or a `pyarrow.Table`. ## Fix The chunked branch hardcoded non-nullable when encoding each chunk and when building the dtype, unlike the single-array branch which derives nullability. Derive one nullability for the whole chunked array — so every chunk shares a dtype — and use it for both. ## AI assistance Prepared with AI assistance; I have reviewed and verified the change. --------- Signed-off-by: jackylee-ch <[email protected]>
1 parent 2cc04c4 commit 8ff4b4a

2 files changed

Lines changed: 21 additions & 4 deletions

File tree

‎vortex-python/src/arrays/from_arrow.rs‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,22 @@ pub(super) fn from_arrow(obj: &Borrowed<'_, '_, PyAny>) -> PyVortexResult<PyArra
4141
Ok(PyArrayRef::from(enc_array))
4242
} else if obj.is_instance(chunked_array)? {
4343
let chunks: Vec<Bound<PyAny>> = obj.getattr(intern!(py, "chunks"))?.extract()?;
44-
let encoded_chunks = chunks
44+
let arrow_chunks = chunks
4545
.iter()
4646
.map(|a| {
47-
let arrow_array = ArrowArrayData::from_pyarrow(&a.as_borrowed()).map(make_array)?;
47+
ArrowArrayData::from_pyarrow(&a.as_borrowed())
48+
.map(make_array)
49+
.map_err(PyVortexError::from)
50+
})
51+
.collect::<PyVortexResult<Vec<_>>>()?;
52+
// One nullability for the whole array so every chunk shares a dtype.
53+
let is_nullable = arrow_chunks.iter().any(|a| a.is_nullable());
54+
let encoded_chunks = arrow_chunks
55+
.into_iter()
56+
.map(|arrow_array| {
4857
session()
4958
.arrow()
50-
.from_arrow_array(arrow_array, false)
59+
.from_arrow_array(arrow_array, is_nullable)
5160
.map_err(PyVortexError::from)
5261
})
5362
.collect::<PyVortexResult<Vec<_>>>()?;
@@ -56,7 +65,7 @@ pub(super) fn from_arrow(obj: &Borrowed<'_, '_, PyAny>) -> PyVortexResult<PyArra
5665
.and_then(|v| DataType::from_pyarrow(&v.as_borrowed()))?;
5766
let dtype = session()
5867
.arrow()
59-
.from_arrow_field(&Field::new("_", arrow_dtype, false))
68+
.from_arrow_field(&Field::new("_", arrow_dtype, is_nullable))
6069
.map_err(|e| PyValueError::new_err(e.to_string()))?;
6170
Ok(PyArrayRef::from(
6271
ChunkedArray::try_new(encoded_chunks, dtype)?.into_array(),

‎vortex-python/test/test_array.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,14 @@ def test_array_with_nulls() -> None:
1919
assert arr.to_arrow_array() == a
2020

2121

22+
def test_chunked_array_with_nulls_round_trip() -> None:
23+
a = pa.chunked_array([[1, None, 2], [3, None]])
24+
arr = vortex.array(a)
25+
assert isinstance(arr, vortex.ChunkedArray)
26+
assert len(arr) == 5
27+
assert arr.to_arrow_array().null_count == 2
28+
29+
2230
def test_varbin_array_round_trip() -> None:
2331
a = pa.array(["a", "b", "c"], type=pa.string_view())
2432
arr = vortex.array(a)

0 commit comments

Comments
 (0)