Skip to content

Commit 1504f92

Browse files
committed
fix: skip empty metadata in intersect_metadata_for_union to prevent schema mismatch
When a UNION ALL combines columns from sources with field metadata (e.g. Parquet) and NULL literals (which have no metadata), the intersect_metadata_for_union function was dropping all metadata because intersecting anything with an empty set yields an empty set. After optimize_projections prunes unused columns and recompute_schema rebuilds the Union via Union::try_new, the logical schema ends up with empty metadata while the physical schema retains the original field metadata from Parquet, causing a physical/logical schema mismatch error. The fix treats empty metadata as a non-vote in the intersection: branches with no metadata (NULL literals, computed expressions) are skipped, so only branches with actual metadata participate. When non-empty branches conflict, their metadata is still correctly intersected as before. Closes apache#19049
1 parent 646d183 commit 1504f92

1 file changed

Lines changed: 79 additions & 7 deletions

File tree

‎datafusion/expr/src/expr.rs‎

Lines changed: 79 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -512,17 +512,26 @@ pub type SchemaFieldMetadata = std::collections::HashMap<String, String>;
512512
pub fn intersect_metadata_for_union<'a>(
513513
metadatas: impl IntoIterator<Item = &'a SchemaFieldMetadata>,
514514
) -> SchemaFieldMetadata {
515-
let mut metadatas = metadatas.into_iter();
516-
let Some(mut intersected) = metadatas.next().cloned() else {
517-
return Default::default();
518-
};
515+
let mut intersected: Option<SchemaFieldMetadata> = None;
519516

520517
for metadata in metadatas {
521-
// Only keep keys that exist in both with the same value
522-
intersected.retain(|k, v| metadata.get(k) == Some(v));
518+
// Skip empty metadata (e.g. from NULL literals or computed expressions)
519+
// to avoid dropping metadata from branches that have it.
520+
if metadata.is_empty() {
521+
continue;
522+
}
523+
match &mut intersected {
524+
None => {
525+
intersected = Some(metadata.clone());
526+
}
527+
Some(current) => {
528+
// Only keep keys that exist in both with the same value
529+
current.retain(|k, v| metadata.get(k) == Some(v));
530+
}
531+
}
523532
}
524533

525-
intersected
534+
intersected.unwrap_or_default()
526535
}
527536

528537
/// UNNEST expression.
@@ -4127,4 +4136,67 @@ mod test {
41274136
}
41284137
}
41294138
}
4139+
4140+
mod intersect_metadata_tests {
4141+
use super::super::intersect_metadata_for_union;
4142+
use std::collections::HashMap;
4143+
4144+
#[test]
4145+
fn all_branches_same_metadata() {
4146+
let m1 = HashMap::from([("key".into(), "val".into())]);
4147+
let m2 = HashMap::from([("key".into(), "val".into())]);
4148+
let result = intersect_metadata_for_union([&m1, &m2]);
4149+
assert_eq!(result, HashMap::from([("key".into(), "val".into())]));
4150+
}
4151+
4152+
#[test]
4153+
fn conflicting_metadata_dropped() {
4154+
let m1 = HashMap::from([("key".into(), "a".into())]);
4155+
let m2 = HashMap::from([("key".into(), "b".into())]);
4156+
let result = intersect_metadata_for_union([&m1, &m2]);
4157+
assert!(result.is_empty());
4158+
}
4159+
4160+
#[test]
4161+
fn empty_metadata_branch_skipped() {
4162+
let m1 = HashMap::from([("key".into(), "val".into())]);
4163+
let m2 = HashMap::new(); // e.g. NULL literal
4164+
let result = intersect_metadata_for_union([&m1, &m2]);
4165+
assert_eq!(result, HashMap::from([("key".into(), "val".into())]));
4166+
}
4167+
4168+
#[test]
4169+
fn empty_metadata_first_branch_skipped() {
4170+
let m1 = HashMap::new();
4171+
let m2 = HashMap::from([("key".into(), "val".into())]);
4172+
let result = intersect_metadata_for_union([&m1, &m2]);
4173+
assert_eq!(result, HashMap::from([("key".into(), "val".into())]));
4174+
}
4175+
4176+
#[test]
4177+
fn all_branches_empty_metadata() {
4178+
let m1: HashMap<String, String> = HashMap::new();
4179+
let m2: HashMap<String, String> = HashMap::new();
4180+
let result = intersect_metadata_for_union([&m1, &m2]);
4181+
assert!(result.is_empty());
4182+
}
4183+
4184+
#[test]
4185+
fn mixed_empty_and_conflicting() {
4186+
let m1 = HashMap::from([("key".into(), "a".into())]);
4187+
let m2 = HashMap::new();
4188+
let m3 = HashMap::from([("key".into(), "b".into())]);
4189+
let result = intersect_metadata_for_union([&m1, &m2, &m3]);
4190+
// m2 is skipped; m1 and m3 conflict → dropped
4191+
assert!(result.is_empty());
4192+
}
4193+
4194+
#[test]
4195+
fn no_inputs() {
4196+
let result = intersect_metadata_for_union(
4197+
std::iter::empty::<&HashMap<String, String>>(),
4198+
);
4199+
assert!(result.is_empty());
4200+
}
4201+
}
41304202
}

0 commit comments

Comments
 (0)