@@ -33,9 +33,10 @@ use crate::statistics::{ChildStats, StatisticsArgs};
3333use crate :: stream:: ObservedStream ;
3434use crate :: { ChildrenPropertiesMode , ReplaceChildrenOptions , validate_child_count} ;
3535
36- use arrow:: datatypes:: { Metadata , Schema , SchemaRef } ;
36+ use arrow:: datatypes:: { Schema , SchemaRef } ;
3737use arrow:: record_batch:: RecordBatch ;
3838use datafusion_common:: config:: ConfigOptions ;
39+ use datafusion_common:: metadata:: merge_metadata_for_union;
3940use datafusion_common:: stats:: NdvFallback ;
4041use datafusion_common:: tree_node:: TreeNodeRecursion ;
4142use datafusion_common:: {
@@ -997,47 +998,34 @@ fn union_schema(inputs: &[Arc<dyn ExecutionPlan>]) -> Result<SchemaRef> {
997998 }
998999 }
9991000
1001+ let schemas: Vec < SchemaRef > = inputs. iter ( ) . map ( |input| input. schema ( ) ) . collect ( ) ;
10001002 let fields = ( 0 ..first_field_count)
10011003 . map ( |i| {
10021004 // We take the name from the left side of the union to match how names are coerced during logical planning,
10031005 // which also uses the left side names.
10041006 let base_field = first_schema. field ( i) . clone ( ) ;
10051007
1006- // Select the first nullable field, or the first field if none are
1007- // nullable, before merging metadata. Otherwise, each non-nullable
1008- // candidate would repeat the full metadata merge.
1009- let ( input_idx, input) = inputs
1008+ // Take the first nullable field, or the first field if none are
1009+ // nullable.
1010+ let field = schemas
10101011 . iter ( )
1011- . enumerate ( )
1012- . find_or_first ( |( _, input) | input. schema ( ) . field ( i) . is_nullable ( ) )
1012+ . find_or_first ( |schema| schema. field ( i) . is_nullable ( ) )
10131013 // Empty inputs are rejected above.
1014- . unwrap ( ) ;
1015- let field = input . schema ( ) . field ( i) . clone ( ) ;
1016- let mut metadata = field . metadata ( ) . clone ( ) ;
1014+ . unwrap ( )
1015+ . field ( i)
1016+ . clone ( ) ;
10171017
1018- // Preserve precedence: the selected field's metadata comes first,
1019- // followed by the other inputs in their original order.
1020- let other_metadatas = inputs
1021- . iter ( )
1022- . enumerate ( )
1023- . filter ( |( other_idx, _) | * other_idx != input_idx)
1024- . flat_map ( |( _, other_input) | {
1025- other_input. schema ( ) . field ( i) . metadata ( ) . clone ( ) . into_iter ( )
1026- } ) ;
1027- metadata. extend ( other_metadatas) ;
1018+ let metadata = merge_metadata_for_union (
1019+ schemas. iter ( ) . map ( |schema| schema. field ( i) . metadata ( ) ) ,
1020+ ) ;
10281021 field. with_metadata ( metadata) . with_name ( base_field. name ( ) )
10291022 } )
10301023 . collect :: < Vec < _ > > ( ) ;
10311024
1032- let all_metadata_merged: Metadata = inputs
1033- . iter ( )
1034- . flat_map ( |i| i. schema ( ) . metadata ( ) . clone ( ) . into_iter ( ) )
1035- . collect ( ) ;
1025+ let metadata =
1026+ merge_metadata_for_union ( schemas. iter ( ) . map ( |schema| schema. metadata ( ) ) ) ;
10361027
1037- Ok ( Arc :: new ( Schema :: new_with_metadata (
1038- fields,
1039- all_metadata_merged,
1040- ) ) )
1028+ Ok ( Arc :: new ( Schema :: new_with_metadata ( fields, metadata) ) )
10411029}
10421030
10431031/// CombinedRecordBatchStream can be used to combine a Vec of SendableRecordBatchStreams into one
@@ -1117,7 +1105,7 @@ mod tests {
11171105 use crate :: test:: { self , TestMemoryExec } ;
11181106
11191107 use arrow:: compute:: SortOptions ;
1120- use arrow:: datatypes:: { DataType , Field } ;
1108+ use arrow:: datatypes:: { DataType , Field , Metadata } ;
11211109 use datafusion_common:: SplitPoint ;
11221110 use datafusion_common:: stats:: Precision ;
11231111 use datafusion_common:: { ColumnStatistics , ScalarValue } ;
@@ -1727,14 +1715,14 @@ mod tests {
17271715 async fn test_union_schema_metadata_and_nullability ( ) -> Result < ( ) > {
17281716 use arrow:: array:: Int32Array ;
17291717
1730- // Field metadata from the selected field is merged first, followed by
1731- // the other inputs in order. Schema metadata always follows input order .
1732- for ( nullability, expected_field_value ) in [
1733- ( [ false , false , false ] , "input2" ) ,
1734- ( [ true , false , false ] , "input2" ) ,
1735- ( [ false , true , false ] , "input2" ) ,
1736- ( [ false , false , true ] , "input1" ) ,
1737- ( [ true , true , true ] , "input2" ) ,
1718+ // Field and schema metadata are merged in input order, so the last
1719+ // input wins regardless of which field supplies the nullability .
1720+ for nullability in [
1721+ [ false , false , false ] ,
1722+ [ true , false , false ] ,
1723+ [ false , true , false ] ,
1724+ [ false , false , true ] ,
1725+ [ true , true , true ] ,
17381726 ] {
17391727 let inputs = nullability
17401728 . iter ( )
@@ -1760,19 +1748,17 @@ mod tests {
17601748 . collect :: < Result < Vec < _ > > > ( ) ?;
17611749
17621750 let expected_metadata = Metadata :: from ( [
1763- ( "shared" . to_owned ( ) , expected_field_value . to_owned ( ) ) ,
1751+ ( "shared" . to_owned ( ) , "input2" . to_owned ( ) ) ,
17641752 ( "only0" . to_owned ( ) , "kept" . to_owned ( ) ) ,
17651753 ( "only1" . to_owned ( ) , "kept" . to_owned ( ) ) ,
17661754 ( "only2" . to_owned ( ) , "kept" . to_owned ( ) ) ,
17671755 ] ) ;
1768- let mut expected_schema_metadata = expected_metadata. clone ( ) ;
1769- expected_schema_metadata. insert ( "shared" . to_owned ( ) , "input2" . to_owned ( ) ) ;
17701756 let expected_schema = Arc :: new ( Schema :: new_with_metadata (
17711757 vec ! [
17721758 Field :: new( "value0" , DataType :: Int32 , nullability. contains( & true ) )
1773- . with_metadata( expected_metadata) ,
1759+ . with_metadata( expected_metadata. clone ( ) ) ,
17741760 ] ,
1775- expected_schema_metadata ,
1761+ expected_metadata ,
17761762 ) ) ;
17771763
17781764 let union = UnionExec :: try_new ( inputs) ?;
0 commit comments