Skip to content

Commit 699e5ac

Browse files
adriangbclaude
andcommitted
fix: propagate inner-field metadata through composite-type constructors (apache#21982)
`make_array`, `array_agg`, `array_repeat`, `arrays_zip`, `map`, `range` / `generate_series`, and `struct` only overrode `return_type` and synthesized fresh inner fields at runtime, so each one silently dropped the input field's metadata when wrapping it into a `List`/`Struct`/`Map`. In practice this broke Arrow extension types (`ARROW:extension:name` / `ARROW:extension:metadata`) round-tripping through SQL list/struct/map constructors. Add two helpers in `datafusion-common::utils` — `list_inner_field_from` and `struct_inner_fields_from` — that wrap a source `Field` into the inner field of a list/struct while preserving its metadata, and extend `SingleRowListArrayBuilder::with_field` to copy metadata too. Use these helpers consistently from each affected function's `return_field_from_args` / `return_field`, and thread the resulting metadata-bearing inner field into the runtime construction paths (including the `array_agg` accumulators, which now carry a `FieldRef` instead of a bare `DataType`). Adds Rust unit tests for each affected function plus an end-to-end `array_metadata_propagation.slt` that asserts metadata survives every constructor by string-matching the rendered data type. Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
1 parent 9d92944 commit 699e5ac

11 files changed

Lines changed: 1030 additions & 194 deletions

File tree

‎datafusion/common/src/utils/mod.rs‎

Lines changed: 67 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -40,13 +40,13 @@ use arrow::compute::kernels::cmp::eq;
4040
use arrow::compute::kernels::length::length;
4141
use arrow::compute::{SortColumn, SortOptions, partition};
4242
use arrow::datatypes::{
43-
ArrowNativeType, DataType, Field, Int32Type, Int64Type, SchemaRef,
43+
ArrowNativeType, DataType, Field, FieldRef, Fields, Int32Type, Int64Type, SchemaRef,
4444
};
4545
#[cfg(feature = "sql")]
4646
use sqlparser::{ast::Ident, dialect::GenericDialect, parser::Parser};
4747
use std::borrow::{Borrow, Cow};
4848
use std::cmp::{Ordering, min};
49-
use std::collections::HashSet;
49+
use std::collections::{HashMap, HashSet};
5050
use std::iter::repeat_n;
5151
use std::num::NonZero;
5252
use std::ops::Range;
@@ -425,6 +425,8 @@ pub struct SingleRowListArrayBuilder {
425425
/// Specify the field name for the resulting array. Defaults to value used in
426426
/// [`Field::new_list_field`]
427427
field_name: Option<String>,
428+
/// Optional Arrow field metadata to attach to the resulting list's inner field.
429+
field_metadata: Option<HashMap<String, String>>,
428430
}
429431

430432
impl SingleRowListArrayBuilder {
@@ -434,6 +436,7 @@ impl SingleRowListArrayBuilder {
434436
arr,
435437
nullable: true,
436438
field_name: None,
439+
field_metadata: None,
437440
}
438441
}
439442

@@ -449,10 +452,17 @@ impl SingleRowListArrayBuilder {
449452
self
450453
}
451454

452-
/// Copies field name and nullable from the specified field
453-
pub fn with_field(self, field: &Field) -> Self {
454-
self.with_field_name(Some(field.name().to_owned()))
455-
.with_nullable(field.is_nullable())
455+
/// Copies field name, nullable, and metadata from the specified field.
456+
///
457+
/// Propagating metadata is required for Arrow extension types
458+
/// (`ARROW:extension:name` / `ARROW:extension:metadata`) to round-trip
459+
/// through SQL list constructors (e.g. `make_array`, `array_agg`).
460+
pub fn with_field(mut self, field: &Field) -> Self {
461+
self.field_name = Some(field.name().to_owned());
462+
self.nullable = field.is_nullable();
463+
let metadata = field.metadata();
464+
self.field_metadata = (!metadata.is_empty()).then(|| metadata.clone());
465+
self
456466
}
457467

458468
/// Build a single element [`ListArray`]
@@ -524,16 +534,66 @@ impl SingleRowListArrayBuilder {
524534
arr,
525535
nullable,
526536
field_name,
537+
field_metadata,
527538
} = self;
528539
let data_type = arr.data_type().to_owned();
529-
let field = match field_name {
540+
let mut field = match field_name {
530541
Some(name) => Field::new(name, data_type, nullable),
531542
None => Field::new_list_field(data_type, nullable),
532543
};
544+
if let Some(metadata) = field_metadata {
545+
field = field.with_metadata(metadata);
546+
}
533547
(Arc::new(field), arr)
534548
}
535549
}
536550

551+
/// Build the inner field for a list-of-`inner` (`List`/`LargeList`/
552+
/// `FixedSizeList`/`ListView`).
553+
///
554+
/// The returned field uses [`Field::LIST_FIELD_DEFAULT_NAME`] as the name,
555+
/// preserves `inner`'s data type and metadata, and is always nullable.
556+
///
557+
/// Preserving metadata is what lets Arrow extension types
558+
/// (`ARROW:extension:name` / `ARROW:extension:metadata`) round-trip through
559+
/// SQL list constructors (e.g. `make_array`, `array_agg`, `repeat`).
560+
pub fn list_inner_field_from(inner: &Field) -> FieldRef {
561+
let mut field = Field::new(
562+
Field::LIST_FIELD_DEFAULT_NAME,
563+
inner.data_type().clone(),
564+
true,
565+
);
566+
let metadata = inner.metadata();
567+
if !metadata.is_empty() {
568+
field = field.with_metadata(metadata.clone());
569+
}
570+
Arc::new(field)
571+
}
572+
573+
/// Build a struct field (the inner-field shape used by `arrays_zip`'s
574+
/// element struct, `struct(...)`, etc.) from a sequence of `(name, inner)`
575+
/// pairs, preserving each inner field's metadata.
576+
///
577+
/// The output fields are constructed with the supplied names, the data type
578+
/// and metadata of each `inner`, and `nullable = true`.
579+
pub fn struct_inner_fields_from<'a, I, S>(named_inners: I) -> Fields
580+
where
581+
I: IntoIterator<Item = (S, &'a Field)>,
582+
S: Into<String>,
583+
{
584+
named_inners
585+
.into_iter()
586+
.map(|(name, inner)| {
587+
let mut f = Field::new(name.into(), inner.data_type().clone(), true);
588+
let metadata = inner.metadata();
589+
if !metadata.is_empty() {
590+
f = f.with_metadata(metadata.clone());
591+
}
592+
Arc::new(f)
593+
})
594+
.collect()
595+
}
596+
537597
/// Wrap arrays into a single element `ListArray`.
538598
///
539599
/// Example:

‎datafusion/functions-aggregate/benches/array_agg.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ use std::sync::Arc;
2121
use arrow::array::{
2222
Array, ArrayRef, ArrowPrimitiveType, AsArray, ListArray, NullBufferBuilder,
2323
};
24-
use arrow::datatypes::{Field, Int64Type};
24+
use arrow::datatypes::{Field, FieldRef, Int64Type};
2525
use criterion::{Criterion, criterion_group, criterion_main};
2626
use datafusion_expr::Accumulator;
2727
use datafusion_functions_aggregate::array_agg::ArrayAggAccumulator;
@@ -41,11 +41,12 @@ pub fn seedable_rng() -> StdRng {
4141
#[expect(clippy::needless_pass_by_value)]
4242
fn merge_batch_bench(c: &mut Criterion, name: &str, values: ArrayRef) {
4343
let list_item_data_type = values.as_list::<i32>().values().data_type().clone();
44+
let list_item_field: FieldRef = Arc::new(Field::new("v", list_item_data_type, true));
4445
c.bench_function(name, |b| {
4546
b.iter(|| {
4647
#[expect(clippy::unit_arg)]
4748
black_box(
48-
ArrayAggAccumulator::try_new(&list_item_data_type, false)
49+
ArrayAggAccumulator::try_new(&list_item_field, false)
4950
.unwrap()
5051
.merge_batch(std::slice::from_ref(&values))
5152
.unwrap(),

0 commit comments

Comments
 (0)