Skip to content

Commit a4d71ff

Browse files
authored
fix: Unify schema metadata merge logic in unions to "last one wins" (apache#26881)
## Which issue does this PR close? - Closes apache#26880 ## Rationale for this change There's currently some drift between how different planning stages and layers merge field metadata for unions, which can cause errors at multiple stages. ## What changes are included in this PR? The main change here is unifying the metadata merge logic in `datafusion-common` and using it in both the logical and physical layer. I've also ran the existing benchmarks in `union_schema` and made sure they keep the gains from apache#26790. ## What is the testing strategy for this PR? Added a couple of tests from the original issue and unit tests. ## Are there any user-facing changes? Deprecates an existing helper that does intersection between field metadata, but existing SLT cases still seem to hold.
1 parent de6f3e7 commit a4d71ff

6 files changed

Lines changed: 120 additions & 68 deletions

File tree

‎datafusion/common/src/metadata.rs‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,39 @@ pub fn format_type_and_metadata(
122122
}
123123
}
124124

125+
/// Merges the field or schema metadata of the inputs of a `UNION`.
126+
///
127+
/// Inputs are merged in order: keys present in only some inputs are kept, and
128+
/// when inputs disagree on a key's value, the last input wins. Inputs without
129+
/// metadata (e.g. `NULL` literals) contribute nothing.
130+
///
131+
/// # Example
132+
/// ```
133+
/// # use arrow::datatypes::Metadata;
134+
/// # use datafusion_common::metadata::merge_metadata_for_union;
135+
/// let first = Metadata::new().with("id", "1").with("only_first", "a");
136+
/// let last = Metadata::new().with("id", "2");
137+
/// assert_eq!(
138+
/// merge_metadata_for_union([&first, &last]),
139+
/// Metadata::new().with("id", "2").with("only_first", "a"),
140+
/// );
141+
/// ```
142+
pub fn merge_metadata_for_union<'a>(
143+
metadatas: impl IntoIterator<Item = &'a Metadata>,
144+
) -> Metadata {
145+
metadatas
146+
.into_iter()
147+
.fold(Metadata::new(), |mut merged, metadata| {
148+
if merged.is_empty() {
149+
// Shares the underlying map rather than copying it
150+
metadata.clone()
151+
} else {
152+
merged.extend(metadata);
153+
merged
154+
}
155+
})
156+
}
157+
125158
/// Literal metadata
126159
///
127160
/// Stores metadata associated with a literal expressions

‎datafusion/expr/src/expr.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -633,6 +633,10 @@ pub type SchemaFieldMetadata = Metadata;
633633
/// # Returns
634634
///
635635
/// A new `SchemaFieldMetadata` containing only the intersected metadata
636+
#[deprecated(
637+
since = "56.0.0",
638+
note = "UNION now merges metadata with `datafusion_common::metadata::merge_metadata_for_union`"
639+
)]
636640
pub fn intersect_metadata_for_union<'a>(
637641
metadatas: impl IntoIterator<Item = &'a SchemaFieldMetadata>,
638642
) -> SchemaFieldMetadata {
@@ -4670,6 +4674,7 @@ mod test {
46704674
}
46714675
}
46724676

4677+
#[expect(deprecated)]
46734678
mod intersect_metadata_tests {
46744679
use super::super::intersect_metadata_for_union;
46754680
use arrow::datatypes::Metadata;

‎datafusion/expr/src/logical_plan/plan.rs‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ use super::invariants::{
1919
use crate::builder::{unique_field_aliases, unnest_with_options};
2020
use crate::expr::{
2121
Alias, Placeholder, Sort as SortExpr, WindowFunction, WindowFunctionParams,
22-
intersect_metadata_for_union,
2322
};
2423
use crate::expr_rewriter::{
2524
NamePreserver, create_col_from_scalar_expr, normalize_cols, normalize_sorts,
@@ -48,7 +47,9 @@ use arrow::compute::SortOptions;
4847
use arrow::datatypes::{DataType, Field, FieldRef, Metadata, Schema, SchemaRef};
4948
use datafusion_common::cse::{NormalizeEq, Normalizeable};
5049
use datafusion_common::format::{ExplainAnalyzeCategories, ExplainFormat, MetricType};
51-
use datafusion_common::metadata::check_metadata_with_storage_equal;
50+
use datafusion_common::metadata::{
51+
check_metadata_with_storage_equal, merge_metadata_for_union,
52+
};
5253
use datafusion_common::tree_node::{
5354
Transformed, TreeNode, TreeNodeContainer, TreeNodeRecursion,
5455
};
@@ -3663,14 +3664,14 @@ impl Union {
36633664

36643665
let mut field =
36653666
Field::new(name, data_type.clone(), final_is_nullable);
3666-
field.set_metadata(intersect_metadata_for_union(unmerged_metadata));
3667+
field.set_metadata(merge_metadata_for_union(unmerged_metadata));
36673668

36683669
(None, Arc::new(field))
36693670
},
36703671
)
36713672
.collect::<Vec<(Option<TableReference>, _)>>();
36723673

3673-
let union_schema_metadata = intersect_metadata_for_union(
3674+
let union_schema_metadata = merge_metadata_for_union(
36743675
inputs.iter().map(|input| input.schema().metadata()),
36753676
);
36763677

@@ -3741,14 +3742,13 @@ impl Union {
37413742
};
37423743

37433744
let mut field = Field::new(&name, data_type.clone(), nullable);
3744-
let field_metadata = intersect_metadata_for_union(
3745-
fields.iter().map(|field| field.metadata()),
3746-
);
3745+
let field_metadata =
3746+
merge_metadata_for_union(fields.iter().map(|field| field.metadata()));
37473747
field.set_metadata(field_metadata);
37483748
Ok((None, Arc::new(field)))
37493749
})
37503750
.collect::<Result<_>>()?;
3751-
let union_schema_metadata = intersect_metadata_for_union(
3751+
let union_schema_metadata = merge_metadata_for_union(
37523752
inputs.iter().map(|input| input.schema().metadata()),
37533753
);
37543754

‎datafusion/optimizer/src/analyzer/type_coercion.rs‎

Lines changed: 15 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ use crate::utils::{NamePreserver, merge_into_schema};
1515
use arrow::datatypes::{DataType, Field, IntervalUnit, Schema, TimeUnit};
1616
use arrow::temporal_conversions::SECONDS_IN_DAY;
1717
use datafusion_common::config::ConfigOptions;
18+
use datafusion_common::metadata::merge_metadata_for_union;
1819
use datafusion_common::tree_node::{
1920
Transformed, TreeNode, TreeNodeRecursion, TreeNodeRewriter,
2021
};
@@ -1690,17 +1691,9 @@ fn coerce_union_schema_with_schema(
16901691
.iter()
16911692
.map(|f| f.is_nullable())
16921693
.collect::<Vec<_>>();
1693-
let mut union_field_meta = base_schema
1694-
.fields()
1695-
.iter()
1696-
.map(|f| f.metadata().clone())
1697-
.collect::<Vec<_>>();
1698-
1699-
let mut metadata = base_schema.metadata().clone();
17001694

17011695
for (i, plan) in inputs.iter().enumerate() {
17021696
let plan_schema = plan.schema();
1703-
metadata.extend(plan_schema.metadata().clone());
17041697

17051698
if plan_schema.fields().len() != base_schema.fields().len() {
17061699
return plan_err!(
@@ -1713,10 +1706,9 @@ fn coerce_union_schema_with_schema(
17131706
}
17141707

17151708
// coerce data type and nullability for each field
1716-
for (union_datatype, union_nullable, union_field_map, plan_field) in izip!(
1709+
for (union_datatype, union_nullable, plan_field) in izip!(
17171710
union_datatypes.iter_mut(),
17181711
union_nullabilities.iter_mut(),
1719-
union_field_meta.iter_mut(),
17201712
plan_schema.fields().iter()
17211713
) {
17221714
let coerced_type =
@@ -1734,22 +1726,29 @@ fn coerce_union_schema_with_schema(
17341726

17351727
*union_datatype = coerced_type;
17361728
*union_nullable = *union_nullable || plan_field.is_nullable();
1737-
union_field_map.extend(plan_field.metadata().clone());
17381729
}
17391730
}
1731+
1732+
let all_schemas = std::iter::once(base_schema)
1733+
.chain(inputs.iter().map(|p| p.schema()))
1734+
.collect::<Vec<_>>();
1735+
17401736
let union_qualified_fields = izip!(
1741-
base_schema.fields(),
1737+
base_schema.fields().iter().enumerate(),
17421738
union_datatypes.into_iter(),
1743-
union_nullabilities,
1744-
union_field_meta.into_iter()
1739+
union_nullabilities
17451740
)
1746-
.map(|(field, datatype, nullable, metadata)| {
1741+
.map(|((i, field), datatype, nullable)| {
17471742
let mut field = Field::new(field.name().clone(), datatype, nullable);
1748-
field.set_metadata(metadata);
1743+
field.set_metadata(merge_metadata_for_union(
1744+
all_schemas.iter().map(|schema| schema.field(i).metadata()),
1745+
));
17491746
(None, field.into())
17501747
})
17511748
.collect::<Vec<_>>();
17521749

1750+
let metadata =
1751+
merge_metadata_for_union(all_schemas.iter().map(|schema| schema.metadata()));
17531752
DFSchema::new_with_metadata(union_qualified_fields, metadata)
17541753
}
17551754

‎datafusion/physical-plan/src/union.rs‎

Lines changed: 28 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,10 @@ use crate::statistics::{ChildStats, StatisticsArgs};
3333
use crate::stream::ObservedStream;
3434
use crate::{ChildrenPropertiesMode, ReplaceChildrenOptions, validate_child_count};
3535

36-
use arrow::datatypes::{Metadata, Schema, SchemaRef};
36+
use arrow::datatypes::{Schema, SchemaRef};
3737
use arrow::record_batch::RecordBatch;
3838
use datafusion_common::config::ConfigOptions;
39+
use datafusion_common::metadata::merge_metadata_for_union;
3940
use datafusion_common::stats::NdvFallback;
4041
use datafusion_common::tree_node::TreeNodeRecursion;
4142
use datafusion_common::{
@@ -997,47 +998,34 @@ fn union_schema(inputs: &[Arc<dyn ExecutionPlan>]) -> Result<SchemaRef> {
997998
}
998999
}
9991000

1001+
let schemas: Vec<SchemaRef> = inputs.iter().map(|input| input.schema()).collect();
10001002
let fields = (0..first_field_count)
10011003
.map(|i| {
10021004
// We take the name from the left side of the union to match how names are coerced during logical planning,
10031005
// which also uses the left side names.
10041006
let base_field = first_schema.field(i).clone();
10051007

1006-
// Select the first nullable field, or the first field if none are
1007-
// nullable, before merging metadata. Otherwise, each non-nullable
1008-
// candidate would repeat the full metadata merge.
1009-
let (input_idx, input) = inputs
1008+
// Take the first nullable field, or the first field if none are
1009+
// nullable.
1010+
let field = schemas
10101011
.iter()
1011-
.enumerate()
1012-
.find_or_first(|(_, input)| input.schema().field(i).is_nullable())
1012+
.find_or_first(|schema| schema.field(i).is_nullable())
10131013
// Empty inputs are rejected above.
1014-
.unwrap();
1015-
let field = input.schema().field(i).clone();
1016-
let mut metadata = field.metadata().clone();
1014+
.unwrap()
1015+
.field(i)
1016+
.clone();
10171017

1018-
// Preserve precedence: the selected field's metadata comes first,
1019-
// followed by the other inputs in their original order.
1020-
let other_metadatas = inputs
1021-
.iter()
1022-
.enumerate()
1023-
.filter(|(other_idx, _)| *other_idx != input_idx)
1024-
.flat_map(|(_, other_input)| {
1025-
other_input.schema().field(i).metadata().clone().into_iter()
1026-
});
1027-
metadata.extend(other_metadatas);
1018+
let metadata = merge_metadata_for_union(
1019+
schemas.iter().map(|schema| schema.field(i).metadata()),
1020+
);
10281021
field.with_metadata(metadata).with_name(base_field.name())
10291022
})
10301023
.collect::<Vec<_>>();
10311024

1032-
let all_metadata_merged: Metadata = inputs
1033-
.iter()
1034-
.flat_map(|i| i.schema().metadata().clone().into_iter())
1035-
.collect();
1025+
let metadata =
1026+
merge_metadata_for_union(schemas.iter().map(|schema| schema.metadata()));
10361027

1037-
Ok(Arc::new(Schema::new_with_metadata(
1038-
fields,
1039-
all_metadata_merged,
1040-
)))
1028+
Ok(Arc::new(Schema::new_with_metadata(fields, metadata)))
10411029
}
10421030

10431031
/// CombinedRecordBatchStream can be used to combine a Vec of SendableRecordBatchStreams into one
@@ -1117,7 +1105,7 @@ mod tests {
11171105
use crate::test::{self, TestMemoryExec};
11181106

11191107
use arrow::compute::SortOptions;
1120-
use arrow::datatypes::{DataType, Field};
1108+
use arrow::datatypes::{DataType, Field, Metadata};
11211109
use datafusion_common::SplitPoint;
11221110
use datafusion_common::stats::Precision;
11231111
use datafusion_common::{ColumnStatistics, ScalarValue};
@@ -1727,14 +1715,14 @@ mod tests {
17271715
async fn test_union_schema_metadata_and_nullability() -> Result<()> {
17281716
use arrow::array::Int32Array;
17291717

1730-
// Field metadata from the selected field is merged first, followed by
1731-
// the other inputs in order. Schema metadata always follows input order.
1732-
for (nullability, expected_field_value) in [
1733-
([false, false, false], "input2"),
1734-
([true, false, false], "input2"),
1735-
([false, true, false], "input2"),
1736-
([false, false, true], "input1"),
1737-
([true, true, true], "input2"),
1718+
// Field and schema metadata are merged in input order, so the last
1719+
// input wins regardless of which field supplies the nullability.
1720+
for nullability in [
1721+
[false, false, false],
1722+
[true, false, false],
1723+
[false, true, false],
1724+
[false, false, true],
1725+
[true, true, true],
17381726
] {
17391727
let inputs = nullability
17401728
.iter()
@@ -1760,19 +1748,17 @@ mod tests {
17601748
.collect::<Result<Vec<_>>>()?;
17611749

17621750
let expected_metadata = Metadata::from([
1763-
("shared".to_owned(), expected_field_value.to_owned()),
1751+
("shared".to_owned(), "input2".to_owned()),
17641752
("only0".to_owned(), "kept".to_owned()),
17651753
("only1".to_owned(), "kept".to_owned()),
17661754
("only2".to_owned(), "kept".to_owned()),
17671755
]);
1768-
let mut expected_schema_metadata = expected_metadata.clone();
1769-
expected_schema_metadata.insert("shared".to_owned(), "input2".to_owned());
17701756
let expected_schema = Arc::new(Schema::new_with_metadata(
17711757
vec![
17721758
Field::new("value0", DataType::Int32, nullability.contains(&true))
1773-
.with_metadata(expected_metadata),
1759+
.with_metadata(expected_metadata.clone()),
17741760
],
1775-
expected_schema_metadata,
1761+
expected_metadata,
17761762
));
17771763

17781764
let union = UnionExec::try_new(inputs)?;

‎datafusion/sqllogictest/test_files/metadata.slt‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -127,8 +127,9 @@ NULL NULL l_bar
127127
# Regression test: UNION ALL + optimize_projections pruning unused columns causes
128128
# metadata loss when one branch has NULL literals (empty metadata) and the other
129129
# has field metadata. The optimizer prunes unused columns, triggering recompute_schema
130-
# which rebuilds the Union schema via intersect_metadata_for_union. Previously, this
131-
# intersection would drop metadata when any branch had empty metadata (from NULL literals).
130+
# which rebuilds the Union schema via merge_metadata_for_union. Previously, the
131+
# metadata intersection used there dropped metadata when any branch had empty metadata
132+
# (from NULL literals).
132133
# See https://github.com/apache/datafusion/issues/19049
133134
query T
134135
SELECT name FROM (
@@ -170,6 +171,34 @@ no_baz
170171
no_foo
171172
NULL
172173

174+
# Regression test: conflicting field metadata across UNION legs with different
175+
# nullability. The logical and physical planners must agree on which value wins.
176+
# See https://github.com/apache/datafusion/issues/26880
177+
query I rowsort
178+
SELECT with_metadata(1, 'id', '1') UNION SELECT with_metadata(NULL, 'id', '2');
179+
----
180+
1
181+
NULL
182+
183+
# Same, with the nullable leg first
184+
query I rowsort
185+
SELECT with_metadata(NULL, 'id', '2') UNION SELECT with_metadata(1, 'id', '1');
186+
----
187+
1
188+
NULL
189+
190+
# Same, but the unused column `y` is pruned, so recompute_schema rebuilds the
191+
# Union schema with Union::try_new
192+
query I rowsort
193+
SELECT x FROM (
194+
SELECT with_metadata(1, 'id', '1') AS x, 1 AS y
195+
UNION ALL
196+
SELECT with_metadata(NULL, 'id', '2') AS x, 2 AS y
197+
) GROUP BY x;
198+
----
199+
1
200+
NULL
201+
173202
query P rowsort
174203
SELECT ts
175204
FROM ((

0 commit comments

Comments
 (0)