@@ -41,10 +41,7 @@ pub fn create_if_expr(
4141) -> Result < Arc < dyn PhysicalExpr > > {
4242 let true_type = true_expr. data_type ( input_schema) ?;
4343 let false_type = false_expr. data_type ( input_schema) ?;
44- // The coercion that `get_coerce_type_for_case_expression` folds over the branches of a CASE
45- // WHEN, starting from the ELSE branch. Here the THEN branch goes first, so the common type
46- // takes its struct field names, as Spark's `If.dataType` does.
47- let Some ( common_type) = type_union_coercion ( & true_type, & false_type) else {
44+ let Some ( common_type) = if_common_type ( & true_type, & false_type) else {
4845 return Ok ( Arc :: new ( IfExpr :: new ( if_expr, true_expr, false_expr) ) ) ;
4946 } ;
5047 Ok ( Arc :: new ( IfExpr :: new (
@@ -54,6 +51,47 @@ pub fn create_if_expr(
5451 ) ) )
5552}
5653
54+ /// Reconciles Spark IF branches positionally, retaining THEN names and merging nullability.
55+ /// Spark has already coerced the branches to the same SQL type. DataFusion's struct union may
56+ /// instead match by name, pairing different positions when names differ only in case.
57+ fn if_common_type ( then_type : & DataType , else_type : & DataType ) -> Option < DataType > {
58+ use arrow:: datatypes:: FieldRef ;
59+
60+ fn field ( then_field : & FieldRef , else_field : & FieldRef ) -> Option < FieldRef > {
61+ Some ( Arc :: new (
62+ then_field
63+ . as_ref ( )
64+ . clone ( )
65+ . with_data_type ( if_common_type (
66+ then_field. data_type ( ) ,
67+ else_field. data_type ( ) ,
68+ ) ?)
69+ . with_nullable ( then_field. is_nullable ( ) || else_field. is_nullable ( ) ) ,
70+ ) )
71+ }
72+
73+ match ( then_type, else_type) {
74+ ( DataType :: Struct ( then_fields) , DataType :: Struct ( else_fields) ) => {
75+ if then_fields. len ( ) != else_fields. len ( ) {
76+ return None ;
77+ }
78+ Some ( DataType :: Struct (
79+ then_fields
80+ . iter ( )
81+ . zip ( else_fields)
82+ . map ( |( t, e) | field ( t, e) )
83+ . collect :: < Option < Vec < _ > > > ( ) ?
84+ . into ( ) ,
85+ ) )
86+ }
87+ ( DataType :: List ( t) , DataType :: List ( e) ) => Some ( DataType :: List ( field ( t, e) ?) ) ,
88+ ( DataType :: Map ( t, t_sorted) , DataType :: Map ( e, e_sorted) ) => {
89+ Some ( DataType :: Map ( field ( t, e) ?, * t_sorted && * e_sorted) )
90+ }
91+ _ => type_union_coercion ( then_type, else_type) ,
92+ }
93+ }
94+
5795/// Casts an IF branch whose type is `data_type` to the branches' `common_type`.
5896///
5997/// The branches share a Spark type, so any difference is in the Arrow representation. For a
@@ -203,6 +241,73 @@ mod tests {
203241 expr. evaluate ( batch) . unwrap ( ) . into_array ( 2 ) . unwrap ( )
204242 }
205243
244+ /// Spark matches struct fields by position even when their case-distinct names are reordered.
245+ /// Name-based union pairs the first INT with the second DOUBLE and silently widens it.
246+ #[ test]
247+ fn if_reconciles_case_variant_fields_positionally ( ) {
248+ use arrow:: array:: Float64Array ;
249+
250+ let then_fields: arrow:: datatypes:: Fields = vec ! [
251+ Field :: new( "x" , DataType :: Int32 , true ) ,
252+ Field :: new( "X" , DataType :: Float64 , false ) ,
253+ ]
254+ . into ( ) ;
255+ let else_fields: arrow:: datatypes:: Fields = vec ! [
256+ Field :: new( "X" , DataType :: Int32 , false ) ,
257+ Field :: new( "x" , DataType :: Float64 , true ) ,
258+ ]
259+ . into ( ) ;
260+ let expected_fields: arrow:: datatypes:: Fields = vec ! [
261+ Field :: new( "x" , DataType :: Int32 , true ) ,
262+ Field :: new( "X" , DataType :: Float64 , true ) ,
263+ ]
264+ . into ( ) ;
265+ let schema = Schema :: new ( vec ! [
266+ Field :: new( "b" , DataType :: Boolean , false ) ,
267+ Field :: new( "t" , DataType :: Struct ( then_fields. clone( ) ) , false ) ,
268+ Field :: new( "e" , DataType :: Struct ( else_fields. clone( ) ) , false ) ,
269+ ] ) ;
270+ let batch = RecordBatch :: try_new (
271+ Arc :: new ( schema. clone ( ) ) ,
272+ vec ! [
273+ Arc :: new( BooleanArray :: from( vec![ true , false ] ) ) ,
274+ Arc :: new( StructArray :: new(
275+ then_fields,
276+ vec![
277+ Arc :: new( Int32Array :: from( vec![ Some ( 7 ) , None ] ) ) ,
278+ Arc :: new( Float64Array :: from( vec![ 5.5 , 5.5 ] ) ) ,
279+ ] ,
280+ None ,
281+ ) ) ,
282+ Arc :: new( StructArray :: new(
283+ else_fields,
284+ vec![
285+ Arc :: new( Int32Array :: from( vec![ 0 , 0 ] ) ) ,
286+ Arc :: new( Float64Array :: from( vec![ Some ( 9.5 ) , None ] ) ) ,
287+ ] ,
288+ None ,
289+ ) ) ,
290+ ] ,
291+ )
292+ . unwrap ( ) ;
293+ let c = |name : & str | col ( name, & schema) . unwrap ( ) ;
294+ let expr = create_if_expr ( c ( "b" ) , c ( "t" ) , c ( "e" ) , & schema) . unwrap ( ) ;
295+ assert_eq ! (
296+ expr. data_type( & schema) . unwrap( ) ,
297+ DataType :: Struct ( expected_fields. clone( ) )
298+ ) ;
299+ let result = evaluate_if ( & expr, & batch) ;
300+ let expected = StructArray :: new (
301+ expected_fields,
302+ vec ! [
303+ Arc :: new( Int32Array :: from( vec![ 7 , 0 ] ) ) ,
304+ Arc :: new( Float64Array :: from( vec![ Some ( 5.5 ) , None ] ) ) ,
305+ ] ,
306+ None ,
307+ ) ;
308+ assert_eq ! ( result. as_ref( ) , & expected) ;
309+ }
310+
206311 /// IF branches that share a Spark timestamp type but carry different Arrow timezone labels are
207312 /// reconciled by relabelling one of them, as CASE branches are. The result has the THEN
208313 /// branch's label, which `IfExpr` reported before the ELSE branch was cast, unless it has none.
0 commit comments