@@ -86,10 +86,7 @@ pub fn create_if_expr(
8686) -> Result < Arc < dyn PhysicalExpr > > {
8787 let true_type = true_expr. data_type ( input_schema) ?;
8888 let false_type = false_expr. data_type ( input_schema) ?;
89- // The coercion that `get_coerce_type_for_case_expression` folds over the branches of a CASE
90- // WHEN, starting from the ELSE branch. Here the THEN branch goes first, so the common type
91- // takes its struct field names, as Spark's `If.dataType` does.
92- let Some ( common_type) = type_union_coercion ( & true_type, & false_type) else {
89+ let Some ( common_type) = if_common_type ( & true_type, & false_type) else {
9390 return Ok ( Arc :: new ( IfExpr :: new ( if_expr, true_expr, false_expr) ) ) ;
9491 } ;
9592 Ok ( Arc :: new ( IfExpr :: new (
@@ -99,6 +96,47 @@ pub fn create_if_expr(
9996 ) ) )
10097}
10198
99+ /// Reconciles Spark IF branches positionally, retaining THEN names and merging nullability.
100+ /// Spark has already coerced the branches to the same SQL type. DataFusion's struct union may
101+ /// instead match by name, pairing different positions when names differ only in case.
102+ fn if_common_type ( then_type : & DataType , else_type : & DataType ) -> Option < DataType > {
103+ use arrow:: datatypes:: FieldRef ;
104+
105+ fn field ( then_field : & FieldRef , else_field : & FieldRef ) -> Option < FieldRef > {
106+ Some ( Arc :: new (
107+ then_field
108+ . as_ref ( )
109+ . clone ( )
110+ . with_data_type ( if_common_type (
111+ then_field. data_type ( ) ,
112+ else_field. data_type ( ) ,
113+ ) ?)
114+ . with_nullable ( then_field. is_nullable ( ) || else_field. is_nullable ( ) ) ,
115+ ) )
116+ }
117+
118+ match ( then_type, else_type) {
119+ ( DataType :: Struct ( then_fields) , DataType :: Struct ( else_fields) ) => {
120+ if then_fields. len ( ) != else_fields. len ( ) {
121+ return None ;
122+ }
123+ Some ( DataType :: Struct (
124+ then_fields
125+ . iter ( )
126+ . zip ( else_fields)
127+ . map ( |( t, e) | field ( t, e) )
128+ . collect :: < Option < Vec < _ > > > ( ) ?
129+ . into ( ) ,
130+ ) )
131+ }
132+ ( DataType :: List ( t) , DataType :: List ( e) ) => Some ( DataType :: List ( field ( t, e) ?) ) ,
133+ ( DataType :: Map ( t, t_sorted) , DataType :: Map ( e, e_sorted) ) => {
134+ Some ( DataType :: Map ( field ( t, e) ?, * t_sorted && * e_sorted) )
135+ }
136+ _ => type_union_coercion ( then_type, else_type) ,
137+ }
138+ }
139+
102140/// Casts a CASE WHEN or IF branch whose type is `data_type` to the branches' `common_type`.
103141///
104142/// The branches share a Spark type, so any difference is in the Arrow representation. For a
@@ -1320,6 +1358,73 @@ mod tests {
13201358 expr. evaluate ( batch) . unwrap ( ) . into_array ( 2 ) . unwrap ( )
13211359 }
13221360
1361+ /// Spark matches struct fields by position even when their case-distinct names are reordered.
1362+ /// Name-based union pairs the first INT with the second DOUBLE and silently widens it.
1363+ #[ test]
1364+ fn if_reconciles_case_variant_fields_positionally ( ) {
1365+ use arrow:: array:: Float64Array ;
1366+
1367+ let then_fields: arrow:: datatypes:: Fields = vec ! [
1368+ Field :: new( "x" , DataType :: Int32 , true ) ,
1369+ Field :: new( "X" , DataType :: Float64 , false ) ,
1370+ ]
1371+ . into ( ) ;
1372+ let else_fields: arrow:: datatypes:: Fields = vec ! [
1373+ Field :: new( "X" , DataType :: Int32 , false ) ,
1374+ Field :: new( "x" , DataType :: Float64 , true ) ,
1375+ ]
1376+ . into ( ) ;
1377+ let expected_fields: arrow:: datatypes:: Fields = vec ! [
1378+ Field :: new( "x" , DataType :: Int32 , true ) ,
1379+ Field :: new( "X" , DataType :: Float64 , true ) ,
1380+ ]
1381+ . into ( ) ;
1382+ let schema = Schema :: new ( vec ! [
1383+ Field :: new( "b" , DataType :: Boolean , false ) ,
1384+ Field :: new( "t" , DataType :: Struct ( then_fields. clone( ) ) , false ) ,
1385+ Field :: new( "e" , DataType :: Struct ( else_fields. clone( ) ) , false ) ,
1386+ ] ) ;
1387+ let batch = RecordBatch :: try_new (
1388+ Arc :: new ( schema. clone ( ) ) ,
1389+ vec ! [
1390+ Arc :: new( BooleanArray :: from( vec![ true , false ] ) ) ,
1391+ Arc :: new( StructArray :: new(
1392+ then_fields,
1393+ vec![
1394+ Arc :: new( Int32Array :: from( vec![ Some ( 7 ) , None ] ) ) ,
1395+ Arc :: new( Float64Array :: from( vec![ 5.5 , 5.5 ] ) ) ,
1396+ ] ,
1397+ None ,
1398+ ) ) ,
1399+ Arc :: new( StructArray :: new(
1400+ else_fields,
1401+ vec![
1402+ Arc :: new( Int32Array :: from( vec![ 0 , 0 ] ) ) ,
1403+ Arc :: new( Float64Array :: from( vec![ Some ( 9.5 ) , None ] ) ) ,
1404+ ] ,
1405+ None ,
1406+ ) ) ,
1407+ ] ,
1408+ )
1409+ . unwrap ( ) ;
1410+ let c = |name : & str | col ( name, & schema) . unwrap ( ) ;
1411+ let expr = create_if_expr ( c ( "b" ) , c ( "t" ) , c ( "e" ) , & schema) . unwrap ( ) ;
1412+ assert_eq ! (
1413+ expr. data_type( & schema) . unwrap( ) ,
1414+ DataType :: Struct ( expected_fields. clone( ) )
1415+ ) ;
1416+ let result = evaluate_if ( & expr, & batch) ;
1417+ let expected = StructArray :: new (
1418+ expected_fields,
1419+ vec ! [
1420+ Arc :: new( Int32Array :: from( vec![ 7 , 0 ] ) ) ,
1421+ Arc :: new( Float64Array :: from( vec![ Some ( 5.5 ) , None ] ) ) ,
1422+ ] ,
1423+ None ,
1424+ ) ;
1425+ assert_eq ! ( result. as_ref( ) , & expected) ;
1426+ }
1427+
13231428 /// IF branches that share a Spark timestamp type but carry different Arrow timezone labels are
13241429 /// reconciled by relabelling one of them, as CASE branches are. The result has the THEN
13251430 /// branch's label, which `IfExpr` reported before the ELSE branch was cast, unless it has none.
0 commit comments