Skip to content

Commit 79650ec

Browse files
quwinalambblaginin
authored
fix: UNNEST now preserves item fields in expression inference and logical-plan schemas (#26433)
## Which issue does this PR close? - Closes #26331. ## Rationale for this change In SedonaDB, unnesting a list of GeoArrow geometries currently returns plain binary values, losing the geometry type and CRS metadata needed by downstream spatial operations; metadata-aware SQL functions can also reject the field during planning. ## What changes are included in this PR? - Preserve list-item fields in expression inference and logical-plan schemas while retaining output naming and nullability behavior. ## What is the testing strategy for this PR? - DataFrame regressions cover all five list variants, nested unnesting, aliases, NULL/empty lists, and metadata-sensitive UDF planning. The targeted regression run passed all 7 cases. - The `logical_unnest_plus_1000_columns` benchmark in the optimized `release-nonlto` profile measured 47.58 ms on base `0d79252f` and 48.75 ms on this branch (+2.5% median). This is within the noise threshold. - The full extended workspace suite and all-features Clippy were not rerun; the existing Clippy issue #26181 remains noted. ## Are there any user-facing changes? UNNEST retains list-item metadata, including Arrow extension metadata, in inferred fields and result schemas. No public Rust API changes. --------- Co-authored-by: Andrew Lamb <[email protected]> Co-authored-by: Dmitrii Blaginin <[email protected]>
1 parent 39e2304 commit 79650ec

4 files changed

Lines changed: 259 additions & 33 deletions

File tree

‎datafusion/core/tests/dataframe/mod.rs‎

Lines changed: 204 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4483,6 +4483,210 @@ async fn unnest_dict_encoded_columns() -> Result<()> {
44834483
Ok(())
44844484
}
44854485

4486+
#[rstest]
4487+
#[case::list(DataType::List)]
4488+
#[case::large_list(DataType::LargeList)]
4489+
#[case::fixed_size_list(|field| DataType::FixedSizeList(field, 2))]
4490+
#[case::list_view(DataType::ListView)]
4491+
#[case::large_list_view(DataType::LargeListView)]
4492+
#[tokio::test]
4493+
async fn unnest_list_item_metadata(
4494+
#[case] list_type: fn(Arc<Field>) -> DataType,
4495+
) -> Result<()> {
4496+
use arrow::array::BinaryArray;
4497+
use arrow::buffer::OffsetBuffer;
4498+
use arrow::compute::cast;
4499+
use datafusion_common::RecursionUnnestOption;
4500+
4501+
let item = Arc::new(Field::new("wkb", DataType::Binary, false).with_metadata(
4502+
HashMap::from([
4503+
("ARROW:extension:name".into(), "geoarrow.wkb".into()),
4504+
(
4505+
"ARROW:extension:metadata".into(),
4506+
r#"{"crs":"EPSG:4326"}"#.into(),
4507+
),
4508+
("custom".into(), "item".into()),
4509+
]),
4510+
));
4511+
let values: ArrayRef =
4512+
Arc::new(BinaryArray::from_iter_values([b"a", b"b", b"c", b"d"]));
4513+
let lists = ListArray::new(
4514+
Arc::clone(&item),
4515+
OffsetBuffer::new(vec![0, 2, 4].into()),
4516+
Arc::clone(&values),
4517+
None,
4518+
);
4519+
let lists = cast(&lists, &list_type(Arc::clone(&item)))?;
4520+
let nested_item = Arc::new(
4521+
Field::new("lists", lists.data_type().clone(), false)
4522+
.with_metadata(HashMap::from([("custom".into(), "nested item".into())])),
4523+
);
4524+
let nested = ListArray::new(
4525+
Arc::clone(&nested_item),
4526+
OffsetBuffer::new(vec![0, 2].into()),
4527+
Arc::clone(&lists),
4528+
None,
4529+
);
4530+
let nested = cast(&nested, &list_type(Arc::clone(&nested_item)))?;
4531+
let schema = Arc::new(Schema::new(vec![
4532+
Field::new("input", nested.data_type().clone(), false)
4533+
.with_metadata(HashMap::from([("custom".into(), "container".into())])),
4534+
]));
4535+
let ctx = SessionContext::new();
4536+
let batch = RecordBatch::try_new(schema, vec![nested])?;
4537+
ctx.register_batch("t", batch.clone())?;
4538+
let df = ctx.read_batch(batch)?;
4539+
4540+
// Each recursion depth must use that element's metadata, rather than the
4541+
// outer list's metadata. Renaming the result must retain it as well.
4542+
for (depth, field, expected_values) in
4543+
[(1, &nested_item, &lists), (2, &item, &values)]
4544+
{
4545+
let options = UnnestOptions::new().with_recursions(RecursionUnnestOption {
4546+
input_column: "input".into(),
4547+
output_column: "expanded".into(),
4548+
depth,
4549+
});
4550+
let unnested = df
4551+
.clone()
4552+
.unnest_columns_with_options(&["input"], options)?
4553+
.select(vec![col("expanded").alias("output")])?;
4554+
let expected_field = field
4555+
.as_ref()
4556+
.clone()
4557+
.with_name("output")
4558+
.with_nullable(true);
4559+
let sql = if depth == 1 {
4560+
"SELECT unnest(input) AS output FROM t"
4561+
} else {
4562+
"SELECT unnest(unnest(input)) AS output FROM t"
4563+
};
4564+
for unnested in [unnested, ctx.sql(sql).await?] {
4565+
assert_eq!(unnested.schema().field(0).as_ref(), &expected_field);
4566+
let batches = unnested.collect().await?;
4567+
assert_eq!(batches.len(), 1);
4568+
assert_eq!(batches[0].schema().field(0), &expected_field);
4569+
assert_eq!(batches[0].column(0), expected_values);
4570+
}
4571+
}
4572+
Ok(())
4573+
}
4574+
4575+
#[tokio::test]
4576+
async fn unnest_list_item_metadata_in_udf() -> Result<()> {
4577+
use arrow::buffer::OffsetBuffer;
4578+
use datafusion_expr::{
4579+
ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature,
4580+
};
4581+
4582+
#[derive(Debug, PartialEq, Eq, Hash)]
4583+
struct RequireItemMetadata(Signature);
4584+
4585+
impl ScalarUDFImpl for RequireItemMetadata {
4586+
fn name(&self) -> &str {
4587+
"require_item_metadata"
4588+
}
4589+
4590+
fn signature(&self) -> &Signature {
4591+
&self.0
4592+
}
4593+
4594+
fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
4595+
Ok(arg_types[0].clone())
4596+
}
4597+
4598+
fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<Arc<Field>> {
4599+
let field = &args.arg_fields[0];
4600+
if field.metadata().get("custom").map(String::as_str) != Some("item") {
4601+
return datafusion_common::plan_err!("Missing item metadata");
4602+
}
4603+
Ok(Arc::clone(field))
4604+
}
4605+
4606+
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
4607+
Ok(args.args[0].clone())
4608+
}
4609+
}
4610+
4611+
let item = Arc::new(
4612+
Field::new("item", DataType::Int32, false)
4613+
.with_metadata(HashMap::from([("custom".into(), "item".into())])),
4614+
);
4615+
let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2]));
4616+
let list = ListArray::new(
4617+
Arc::clone(&item),
4618+
OffsetBuffer::new(vec![0, 2].into()),
4619+
Arc::clone(&values),
4620+
None,
4621+
);
4622+
let schema = Arc::new(Schema::new(vec![Field::new(
4623+
"input",
4624+
list.data_type().clone(),
4625+
false,
4626+
)]));
4627+
let ctx = SessionContext::new();
4628+
ctx.register_batch("t", RecordBatch::try_new(schema, vec![Arc::new(list)])?)?;
4629+
ctx.register_udf(ScalarUDF::from(RequireItemMetadata(Signature::any(
4630+
1,
4631+
Volatility::Immutable,
4632+
))));
4633+
let expected_field = item
4634+
.as_ref()
4635+
.clone()
4636+
.with_name("output")
4637+
.with_nullable(true);
4638+
4639+
// A UDF can inspect argument metadata before SQL rewrites UNNEST into a plan.
4640+
for sql in [
4641+
"SELECT require_item_metadata(unnest(input)) AS output FROM t",
4642+
"SELECT require_item_metadata(output) AS output \
4643+
FROM (SELECT unnest(input) AS output FROM t)",
4644+
] {
4645+
let df = ctx.sql(sql).await?;
4646+
assert_eq!(df.schema().field(0).as_ref(), &expected_field);
4647+
let batches = df.collect().await?;
4648+
assert_eq!(batches.len(), 1);
4649+
assert_eq!(batches[0].schema().field(0), &expected_field);
4650+
assert_eq!(batches[0].column(0), &values);
4651+
}
4652+
Ok(())
4653+
}
4654+
4655+
#[tokio::test]
4656+
async fn unnest_list_item_metadata_with_nulls() -> Result<()> {
4657+
let item = Arc::new(
4658+
Field::new("item", DataType::Int32, false)
4659+
.with_metadata(HashMap::from([("custom".into(), "item".into())])),
4660+
);
4661+
let mut builder = ListBuilder::new(Int32Builder::new()).with_field(Arc::clone(&item));
4662+
builder.values().append_slice(&[1, 2]);
4663+
builder.append(true);
4664+
builder.append(true); // Empty list
4665+
builder.append(false); // NULL list
4666+
let list = builder.finish();
4667+
let schema = Arc::new(Schema::new(vec![Field::new(
4668+
"input",
4669+
list.data_type().clone(),
4670+
true,
4671+
)]));
4672+
let ctx = SessionContext::new();
4673+
let options = UnnestOptions::new()
4674+
.with_null_handling(datafusion_common::NullHandling::PreserveAndExpandEmpty);
4675+
let df = ctx
4676+
.read_batch(RecordBatch::try_new(schema, vec![Arc::new(list)])?)?
4677+
.unnest_columns_with_options(&["input"], options)?;
4678+
let expected_field = item.as_ref().clone().with_name("input").with_nullable(true);
4679+
assert_eq!(df.schema().field(0).as_ref(), &expected_field);
4680+
let batches = df.collect().await?;
4681+
assert_eq!(batches.len(), 1);
4682+
assert_eq!(batches[0].schema().field(0), &expected_field);
4683+
assert_eq!(
4684+
batches[0].column(0).as_ref(),
4685+
&Int32Array::from(vec![Some(1), Some(2), None, None]) as &dyn Array
4686+
);
4687+
Ok(())
4688+
}
4689+
44864690
#[tokio::test]
44874691
async fn unnest_column_nulls() -> Result<()> {
44884692
let df = table_with_lists_and_nulls().await?;

‎datafusion/expr/src/expr_schema.rs‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -517,6 +517,7 @@ impl ExprSchemable for Expr {
517517
/// - **Column references**: Preserve original field metadata from input schema
518518
/// - **Literals**: Use explicitly provided metadata, otherwise empty
519519
/// - **Aliases**: Merge underlying expr metadata with alias-specific metadata, preferring the alias metadata
520+
/// - **List unnesting**: Preserve the element field metadata
520521
/// - **Binary expressions**: field metadata is empty
521522
/// - **Boolean expressions**: field metadata is empty
522523
/// - **Cast expressions**: Type-only casts pass through source metadata (stripping extension
@@ -678,6 +679,27 @@ impl ExprSchemable for Expr {
678679
Expr::LambdaVariable(LambdaVariable {
679680
field: Some(field), ..
680681
}) => Ok(Arc::clone(field).renamed(&schema_name)),
682+
Expr::Unnest(Unnest { expr, .. }) => {
683+
let nullable = self.nullable(schema)?;
684+
match expr.get_type(schema)? {
685+
DataType::List(field)
686+
| DataType::LargeList(field)
687+
| DataType::FixedSizeList(field, _)
688+
| DataType::ListView(field)
689+
| DataType::LargeListView(field) => Ok(Arc::new(
690+
field
691+
.as_ref()
692+
.clone()
693+
.with_name(&schema_name)
694+
.with_nullable(nullable),
695+
)),
696+
_ => Ok(Arc::new(Field::new(
697+
&schema_name,
698+
self.get_type(schema)?,
699+
nullable,
700+
))),
701+
}
702+
}
681703
Expr::Like(_)
682704
| Expr::SimilarTo(_)
683705
| Expr::Not(_)
@@ -689,7 +711,6 @@ impl ExprSchemable for Expr {
689711
| Expr::Wildcard { .. }
690712
| Expr::GroupingSet(_)
691713
| Expr::Placeholder(_)
692-
| Expr::Unnest(_)
693714
| Expr::Lambda(_)
694715
| Expr::LambdaVariable(_) => Ok(Arc::new(Field::new(
695716
&schema_name,

‎datafusion/expr/src/logical_plan/plan.rs‎

Lines changed: 11 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,8 @@ use crate::logical_plan::{DmlStatement, Statement, WriteOp};
4444
use crate::utils::{
4545
check_aggregate_and_window_nesting, check_no_window_functions,
4646
enumerate_grouping_sets, expr_to_columns, exprlist_to_fields,
47-
find_out_reference_exprs, grouping_set_expr_count, grouping_set_to_exprlist,
48-
merge_schema, passthrough_field_index, split_conjunction,
47+
find_out_reference_exprs, get_unnested_list_field, grouping_set_expr_count,
48+
grouping_set_to_exprlist, merge_schema, passthrough_field_index, split_conjunction,
4949
};
5050
use crate::{
5151
BinaryExpr, CreateMemoryTable, CreateView, Execute, Expr, ExprSchemable, GroupingSet,
@@ -5338,13 +5338,15 @@ fn get_unnested_columns(
53385338
| DataType::LargeList(_)
53395339
| DataType::ListView(_)
53405340
| DataType::LargeListView(_) => {
5341-
let data_type = get_unnested_list_datatype_recursive(data_type, depth)?;
5342-
let new_field = Arc::new(Field::new(
5343-
col_name, data_type,
5344-
// Unnesting may produce NULLs even if the list is not null.
5345-
// For example: unnest([1], []) -> 1, null
5346-
true,
5347-
));
5341+
let field = get_unnested_list_field(data_type, depth)?;
5342+
let new_field = Arc::new(
5343+
field
5344+
.clone()
5345+
.with_name(col_name)
5346+
// Unnesting may produce NULLs even if the list is not null.
5347+
// For example: unnest([1], []) -> 1, null
5348+
.with_nullable(true),
5349+
);
53485350
let column = Column::from_name(col_name);
53495351
// let column = Column::from((None, &new_field));
53505352
qualified_columns.push((column, new_field));
@@ -5369,29 +5371,6 @@ fn get_unnested_columns(
53695371
Ok(qualified_columns)
53705372
}
53715373

5372-
// Get the data type of a multi-dimensional type after unnesting it
5373-
// with a given depth
5374-
fn get_unnested_list_datatype_recursive(
5375-
data_type: &DataType,
5376-
depth: usize,
5377-
) -> Result<DataType> {
5378-
match data_type {
5379-
DataType::List(field)
5380-
| DataType::FixedSizeList(field, _)
5381-
| DataType::LargeList(field)
5382-
| DataType::ListView(field)
5383-
| DataType::LargeListView(field) => {
5384-
if depth == 1 {
5385-
return Ok(field.data_type().clone());
5386-
}
5387-
return get_unnested_list_datatype_recursive(field.data_type(), depth - 1);
5388-
}
5389-
_ => {}
5390-
}
5391-
5392-
internal_err!("trying to unnest on invalid data type {data_type}")
5393-
}
5394-
53955374
#[cfg(test)]
53965375
mod tests {
53975376
use super::*;

‎datafusion/expr/src/utils.rs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -919,6 +919,28 @@ where
919919
err
920920
}
921921

922+
/// Returns the element field at the given list unnesting depth.
923+
pub(crate) fn get_unnested_list_field(
924+
data_type: &DataType,
925+
depth: usize,
926+
) -> Result<&Field> {
927+
match data_type {
928+
DataType::List(field)
929+
| DataType::FixedSizeList(field, _)
930+
| DataType::LargeList(field)
931+
| DataType::ListView(field)
932+
| DataType::LargeListView(field) => {
933+
if depth == 1 {
934+
return Ok(field.as_ref());
935+
}
936+
return get_unnested_list_field(field.data_type(), depth - 1);
937+
}
938+
_ => {}
939+
}
940+
941+
internal_err!("trying to unnest on invalid data type {data_type}")
942+
}
943+
922944
/// Create schema fields from an expression list, for use in result set schema construction
923945
///
924946
/// This function converts a list of expressions into a list of complete schema fields,

0 commit comments

Comments
 (0)