Skip to content

Commit 3406cf2

Browse files
committed
test(arrow): pin struct validity for id-less struct fields
An id-less struct keeps the file's validity and reads its fields as null, as Spark's reader does.
1 parent f31c686 commit 3406cf2

1 file changed

Lines changed: 7 additions & 5 deletions

File tree

‎crates/iceberg/src/arrow/record_batch_transformer.rs‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1696,23 +1696,25 @@ mod test {
16961696

16971697
#[test]
16981698
fn promote_struct_without_source_field_ids_reads_null() {
1699+
let nulls = NullBuffer::from(vec![true, true, false]);
16991700
let source = Arc::new(StructArray::new(
17001701
Fields::from(vec![Field::new("x", DataType::Int32, true)]),
1701-
vec![Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef],
1702-
None,
1702+
vec![Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef],
1703+
Some(nulls.clone()),
17031704
)) as ArrayRef;
17041705

1706+
// Fields read as null and the struct keeps the file's validity, as Spark's reader does.
17051707
let out = promote(&source, &evolved_struct_type(), &empty_schema()).unwrap();
17061708
let expected = StructArray::new(
17071709
Fields::from(vec![
17081710
field_with_id("x", DataType::Int32, true, 5),
17091711
field_with_id("y", DataType::Int32, true, 6),
17101712
]),
17111713
vec![
1712-
new_null_array(&DataType::Int32, 2),
1713-
new_null_array(&DataType::Int32, 2),
1714+
new_null_array(&DataType::Int32, 3),
1715+
new_null_array(&DataType::Int32, 3),
17141716
],
1715-
None,
1717+
Some(nulls),
17161718
);
17171719
assert_eq!(out.as_struct(), &expected);
17181720
}

0 commit comments

Comments
 (0)