Skip to content

Commit 1ee2f40

Browse files
committed
Keep the list rebuild for renamed fields, not for arrow#38809
The comment claimed the branch could go once apache/arrow#38809 was fixed. That is fixed in pyarrow 19, but the branch must stay: Iceberg renames by field-id while cast matches element fields by name, so dropping it nulls out any field renamed inside a list element. Verified: the new test passes, and fails with the rebuild removed.
1 parent 69af7ae commit 1ee2f40

2 files changed

Lines changed: 39 additions & 5 deletions

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2073,11 +2073,11 @@ def list(self, list_type: ListType, list_array: pa.Array | None, value_array: pa
20732073
if isinstance(list_array, (pa.ListArray, pa.LargeListArray, pa.FixedSizeListArray)) and value_array is not None:
20742074
list_initializer = pa.large_list if isinstance(list_array, pa.LargeListArray) else pa.list_
20752075
if isinstance(value_array, pa.StructArray):
2076-
# This can be removed once this has been fixed:
2077-
# https://github.com/apache/arrow/issues/38809
2078-
# The mask must be carried over explicitly: from_arrays() takes the offsets
2079-
# buffer alone, which cannot express a null list, so without it every null
2080-
# list is rebuilt as an empty one.
2076+
# Rebuilding is what applies the projection to the element struct. A plain
2077+
# cast would not do: Iceberg renames by field-id, while cast matches the
2078+
# element fields by name and would silently null out any renamed one. The
2079+
# mask has to be carried over because the offsets buffer alone cannot
2080+
# express a null list, only an empty one.
20812081
list_array = pa.LargeListArray.from_arrays(list_array.offsets, value_array, mask=list_array.is_null())
20822082
value_array = self._cast_if_needed(list_type.element_field, value_array)
20832083
arrow_field = list_initializer(self._construct_field(list_type.element_field, value_array.type))

‎tests/io/test_pyarrow.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3247,6 +3247,40 @@ def test__to_requested_schema_null_list_of_structs() -> None:
32473247
assert result.column(1).to_pylist() == [[1], [], None]
32483248

32493249

3250+
def test__to_requested_schema_renamed_field_in_list_of_structs() -> None:
3251+
"""Test that a field renamed inside a list element keeps its values."""
3252+
file_schema = Schema(
3253+
NestedField(
3254+
1,
3255+
"col",
3256+
ListType(11, StructType(NestedField(111, "before", IntegerType(), required=False)), element_required=False),
3257+
required=False,
3258+
),
3259+
)
3260+
requested_schema = Schema(
3261+
NestedField(
3262+
1,
3263+
"col",
3264+
ListType(11, StructType(NestedField(111, "after", IntegerType(), required=False)), element_required=False),
3265+
required=False,
3266+
),
3267+
)
3268+
3269+
arrow_schema = pa.schema([pa.field("col", pa.list_(pa.struct([pa.field("before", pa.int32())])))])
3270+
batch = pa.RecordBatch.from_arrays(
3271+
[pa.array([[{"before": 1}], [], None, [{"before": 3}]], type=arrow_schema.field(0).type)],
3272+
schema=arrow_schema,
3273+
)
3274+
3275+
result = _to_requested_schema(
3276+
requested_schema, file_schema, batch, downcast_ns_timestamp_to_us=False, include_field_ids=False
3277+
)
3278+
3279+
# Rebuilding the list from the projected element array is what carries the rename.
3280+
# Casting instead would match the element fields by name and yield nulls throughout.
3281+
assert result.column(0).to_pylist() == [[{"after": 1}], [], None, [{"after": 3}]]
3282+
3283+
32503284
def test_pyarrow_file_io_fs_by_scheme_cache() -> None:
32513285
# It's better to set up multi-region minio servers for an integration test once `endpoint_url` argument
32523286
# becomes available for `resolve_s3_region`

0 commit comments

Comments
 (0)