Skip to content

Commit c4cd513

Browse files
authored
fix(io): keep null lists of structs when projecting Arrow data (apache#4065)
The projection visitor rebuilds a list of structs from its offsets alone, which cannot represent a null list, so every null list<struct<...>> was turned into an empty list. On the write path this is persisted into the Parquet file and the null is lost for every reader. Rebuild the list from the original validity and offsets buffers instead, which also keeps the original array offset so sliced batches stay correct. Closes apache#3833
1 parent 03e80a7 commit c4cd513

3 files changed

Lines changed: 46 additions & 6 deletions

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2198,7 +2198,15 @@ def list(self, list_type: ListType, list_array: pa.Array | None, value_array: pa
21982198
if isinstance(value_array, pa.StructArray):
21992199
# This can be removed once this has been fixed:
22002200
# https://github.com/apache/arrow/issues/38809
2201-
list_array = pa.LargeListArray.from_arrays(list_array.offsets, value_array)
2201+
# Keep the validity and offsets buffers of the original array, otherwise null lists become empty lists
2202+
list_array = pa.Array.from_buffers(
2203+
list_initializer(value_array.type),
2204+
len(list_array),
2205+
list_array.buffers()[:2],
2206+
list_array.null_count,
2207+
list_array.offset,
2208+
[value_array],
2209+
)
22022210
value_array = self._cast_if_needed(list_type.element_field, value_array)
22032211
arrow_field = list_initializer(self._construct_field(list_type.element_field, value_array.type))
22042212
return list_array.cast(arrow_field)

‎tests/integration/test_reads.py‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1002,10 +1002,7 @@ def test_null_list_and_map(catalog: Catalog) -> None:
10021002
arrow_table = table_test_empty_list_and_map.scan().to_arrow()
10031003
assert arrow_table["col_list"].to_pylist() == [None, []]
10041004
assert arrow_table["col_map"].to_pylist() == [None, []]
1005-
# This should be:
1006-
# assert arrow_table["col_list_with_struct"].to_pylist() == [None, [{'test': 1}]]
1007-
# Once https://github.com/apache/arrow/issues/38809 has been fixed
1008-
assert arrow_table["col_list_with_struct"].to_pylist() == [[], [{"test": 1}]]
1005+
assert arrow_table["col_list_with_struct"].to_pylist() == [None, [{"test": 1}]]
10091006

10101007

10111008
@pytest.mark.integration

‎tests/io/test_pyarrow.py‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
import uuid
2525
import warnings
2626
import zlib
27-
from collections.abc import Iterator
27+
from collections.abc import Callable, Iterator
2828
from dataclasses import replace
2929
from datetime import UTC, date, datetime
3030
from decimal import Decimal
@@ -3946,6 +3946,41 @@ def test__to_requested_schema_float_promotion(
39463946
assert result.column(0).to_pylist() == [1.5, 2.25, 3.0, None]
39473947

39483948

3949+
@pytest.mark.parametrize("list_type", [pa.list_, pa.large_list])
3950+
def test__to_requested_schema_null_list_of_structs(list_type: Callable[[pa.DataType], pa.DataType]) -> None:
3951+
requested_schema = Schema(
3952+
NestedField(
3953+
1,
3954+
"col_list_with_struct",
3955+
ListType(11, StructType(NestedField(111, "test", IntegerType(), required=False)), element_required=False),
3956+
required=False,
3957+
),
3958+
NestedField(2, "col_list", ListType(21, IntegerType(), element_required=False), required=False),
3959+
)
3960+
arrow_schema = pa.schema(
3961+
[
3962+
pa.field("col_list_with_struct", list_type(pa.struct([pa.field("test", pa.int32())]))),
3963+
pa.field("col_list", list_type(pa.int32())),
3964+
]
3965+
)
3966+
batch = pa.RecordBatch.from_arrays(
3967+
[
3968+
pa.array([[{"test": 1}], [], None, [{"test": 2}, None]], type=arrow_schema.field(0).type),
3969+
pa.array([[1], [], None, [2, None]], type=arrow_schema.field(1).type),
3970+
],
3971+
schema=arrow_schema,
3972+
)
3973+
3974+
result = _to_requested_schema(requested_schema, requested_schema, batch)
3975+
assert result.column(0).to_pylist() == [[{"test": 1}], [], None, [{"test": 2}, None]]
3976+
assert result.column(1).to_pylist() == [[1], [], None, [2, None]]
3977+
3978+
# A sliced batch has to keep the nulls as well
3979+
result = _to_requested_schema(requested_schema, requested_schema, batch.slice(1, 3))
3980+
assert result.column(0).to_pylist() == [[], None, [{"test": 2}, None]]
3981+
assert result.column(1).to_pylist() == [[], None, [2, None]]
3982+
3983+
39493984
@skip_if_pyarrow_too_old
39503985
@pytest.mark.parametrize("scheme", ["abfs", "abfss", "wasb", "wasbs"])
39513986
def test_adls_account_name_from_location(scheme: str) -> None:

0 commit comments

Comments
 (0)