Skip to content

Commit 6df3d03

Browse files
committed
fix(io): keep null lists of structs when projecting Arrow data
The projection visitor rebuilds a list of structs from its offsets alone, which cannot represent a null list, so every null list<struct<...>> was turned into an empty list. On the write path this is persisted into the Parquet file and the null is lost for every reader. Rebuild the list from the original validity and offsets buffers instead, which also keeps the original array offset so sliced batches stay correct. Closes apache#3833
1 parent 1310a19 commit 6df3d03

3 files changed

Lines changed: 46 additions & 6 deletions

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2183,7 +2183,15 @@ def list(self, list_type: ListType, list_array: pa.Array | None, value_array: pa
21832183
if isinstance(value_array, pa.StructArray):
21842184
# This can be removed once this has been fixed:
21852185
# https://github.com/apache/arrow/issues/38809
2186-
list_array = pa.LargeListArray.from_arrays(list_array.offsets, value_array)
2186+
# Keep the validity and offsets buffers of the original array, otherwise null lists become empty lists
2187+
list_array = pa.Array.from_buffers(
2188+
list_initializer(value_array.type),
2189+
len(list_array),
2190+
list_array.buffers()[:2],
2191+
list_array.null_count,
2192+
list_array.offset,
2193+
[value_array],
2194+
)
21872195
value_array = self._cast_if_needed(list_type.element_field, value_array)
21882196
arrow_field = list_initializer(self._construct_field(list_type.element_field, value_array.type))
21892197
return list_array.cast(arrow_field)

‎tests/integration/test_reads.py‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1001,10 +1001,7 @@ def test_null_list_and_map(catalog: Catalog) -> None:
10011001
arrow_table = table_test_empty_list_and_map.scan().to_arrow()
10021002
assert arrow_table["col_list"].to_pylist() == [None, []]
10031003
assert arrow_table["col_map"].to_pylist() == [None, []]
1004-
# This should be:
1005-
# assert arrow_table["col_list_with_struct"].to_pylist() == [None, [{'test': 1}]]
1006-
# Once https://github.com/apache/arrow/issues/38809 has been fixed
1007-
assert arrow_table["col_list_with_struct"].to_pylist() == [[], [{"test": 1}]]
1004+
assert arrow_table["col_list_with_struct"].to_pylist() == [None, [{"test": 1}]]
10081005

10091006

10101007
@pytest.mark.integration

‎tests/io/test_pyarrow.py‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
import uuid
2525
import warnings
2626
import zlib
27-
from collections.abc import Iterator
27+
from collections.abc import Callable, Iterator
2828
from dataclasses import replace
2929
from datetime import UTC, date, datetime
3030
from pathlib import Path
@@ -3813,6 +3813,41 @@ def test__to_requested_schema_float_promotion(
38133813
assert result.column(0).to_pylist() == [1.5, 2.25, 3.0, None]
38143814

38153815

3816+
@pytest.mark.parametrize("list_type", [pa.list_, pa.large_list])
3817+
def test__to_requested_schema_null_list_of_structs(list_type: Callable[[pa.DataType], pa.DataType]) -> None:
3818+
requested_schema = Schema(
3819+
NestedField(
3820+
1,
3821+
"col_list_with_struct",
3822+
ListType(11, StructType(NestedField(111, "test", IntegerType(), required=False)), element_required=False),
3823+
required=False,
3824+
),
3825+
NestedField(2, "col_list", ListType(21, IntegerType(), element_required=False), required=False),
3826+
)
3827+
arrow_schema = pa.schema(
3828+
[
3829+
pa.field("col_list_with_struct", list_type(pa.struct([pa.field("test", pa.int32())]))),
3830+
pa.field("col_list", list_type(pa.int32())),
3831+
]
3832+
)
3833+
batch = pa.RecordBatch.from_arrays(
3834+
[
3835+
pa.array([[{"test": 1}], [], None, [{"test": 2}, None]], type=arrow_schema.field(0).type),
3836+
pa.array([[1], [], None, [2, None]], type=arrow_schema.field(1).type),
3837+
],
3838+
schema=arrow_schema,
3839+
)
3840+
3841+
result = _to_requested_schema(requested_schema, requested_schema, batch)
3842+
assert result.column(0).to_pylist() == [[{"test": 1}], [], None, [{"test": 2}, None]]
3843+
assert result.column(1).to_pylist() == [[1], [], None, [2, None]]
3844+
3845+
# A sliced batch has to keep the nulls as well
3846+
result = _to_requested_schema(requested_schema, requested_schema, batch.slice(1, 3))
3847+
assert result.column(0).to_pylist() == [[], None, [{"test": 2}, None]]
3848+
assert result.column(1).to_pylist() == [[], None, [2, None]]
3849+
3850+
38163851
@skip_if_pyarrow_too_old
38173852
@pytest.mark.parametrize("scheme", ["abfs", "abfss", "wasb", "wasbs"])
38183853
def test_adls_account_name_from_location(scheme: str) -> None:

0 commit comments

Comments
 (0)