Skip to content

Commit 390d210

Browse files
authored
Preserve list element metadata when rebuilding scalar values (#26092)
## Summary `ScalarValue::cast_to` reconstructs LIST values through `SingleRowListArrayBuilder::with_field`. That builder copied the child field name and nullability but dropped its metadata. Casting a LIST to a target field containing nested metadata (for example an Iceberg `PARQUET:field_id`) therefore produced a scalar with a different type from the requested target, and subsequent batch expansion could fail schema validation. Copy the field metadata as well. This only runs when constructing a scalar LIST; columnar cast and batch fast paths are unchanged. ## Validation - New regression test covers LIST scalar cast to a metadata-bearing element field and batch expansion. - `cargo +1.95.0 test --offline -p datafusion-common --lib`: 624 passed. - `cargo +1.95.0 fmt --all --check`: passed. - Independent read-only review: approved, no blocking findings.
1 parent 7d4893c commit 390d210

1 file changed

Lines changed: 26 additions & 6 deletions

File tree

  • datafusion/common/src/utils

‎datafusion/common/src/utils/mod.rs‎

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ use arrow::compute::kernels::cmp::eq;
4343
use arrow::compute::kernels::length::length;
4444
use arrow::compute::{SortColumn, SortOptions, nullif, partition};
4545
use arrow::datatypes::{
46-
ArrowNativeType, DataType, Field, Int32Type, Int64Type, SchemaRef,
46+
ArrowNativeType, DataType, Field, Int32Type, Int64Type, Metadata, SchemaRef,
4747
};
4848
#[cfg(feature = "sql")]
4949
use sqlparser::{ast::Ident, dialect::GenericDialect, parser::Parser};
@@ -561,6 +561,7 @@ pub struct SingleRowListArrayBuilder {
561561
/// Specify the field name for the resulting array. Defaults to value used in
562562
/// [`Field::new_list_field`]
563563
field_name: Option<String>,
564+
field_metadata: Metadata,
564565
}
565566

566567
impl SingleRowListArrayBuilder {
@@ -570,6 +571,7 @@ impl SingleRowListArrayBuilder {
570571
arr,
571572
nullable: true,
572573
field_name: None,
574+
field_metadata: Metadata::default(),
573575
}
574576
}
575577

@@ -585,10 +587,12 @@ impl SingleRowListArrayBuilder {
585587
self
586588
}
587589

588-
/// Copies field name and nullable from the specified field
589-
pub fn with_field(self, field: &Field) -> Self {
590-
self.with_field_name(Some(field.name().to_owned()))
591-
.with_nullable(field.is_nullable())
590+
/// Copies field name, nullability, and metadata from the specified field
591+
pub fn with_field(mut self, field: &Field) -> Self {
592+
self.field_name = Some(field.name().to_owned());
593+
self.nullable = field.is_nullable();
594+
self.field_metadata = field.metadata().clone();
595+
self
592596
}
593597

594598
/// Build a single element [`ListArray`]
@@ -661,13 +665,14 @@ impl SingleRowListArrayBuilder {
661665
arr,
662666
nullable,
663667
field_name,
668+
field_metadata,
664669
} = self;
665670
let data_type = arr.data_type().to_owned();
666671
let field = match field_name {
667672
Some(name) => Field::new(name, data_type, nullable),
668673
None => Field::new_list_field(data_type, nullable),
669674
};
670-
(Arc::new(field), arr)
675+
(Arc::new(field.with_metadata(field_metadata)), arr)
671676
}
672677
}
673678

@@ -1610,6 +1615,7 @@ pub fn apply_parent_nulls(
16101615

16111616
#[cfg(test)]
16121617
mod tests {
1618+
use std::collections::HashMap;
16131619
use std::sync::Arc;
16141620

16151621
use super::*;
@@ -1621,6 +1627,20 @@ mod tests {
16211627
#[cfg(feature = "sql")]
16221628
use sqlparser::ast::Ident;
16231629

1630+
#[test]
1631+
fn test_list_scalar_cast_preserves_target_element_metadata() -> Result<()> {
1632+
let input =
1633+
ListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![Some(1)])]);
1634+
let item = Field::new("item", DataType::Int32, true)
1635+
.with_metadata(HashMap::from([("PARQUET:field_id".into(), "2".into())]));
1636+
let target = DataType::List(Arc::new(item));
1637+
1638+
let casted = ScalarValue::List(Arc::new(input)).cast_to(&target)?;
1639+
assert_eq!(casted.data_type(), target);
1640+
assert_eq!(casted.to_array_of_size(2)?.data_type(), &target);
1641+
Ok(())
1642+
}
1643+
16241644
#[test]
16251645
fn test_apply_parent_nulls_sliced() -> Result<()> {
16261646
let child = Arc::new(

0 commit comments

Comments
 (0)