@@ -30,6 +30,7 @@ use datafusion::common::cast::as_boolean_array;
3030use datafusion:: common:: {
3131 exec_err, internal_datafusion_err, internal_err, DataFusionError , Result , ScalarValue ,
3232} ;
33+ use datafusion:: logical_expr:: type_coercion:: binary:: type_union_coercion;
3334use datafusion:: logical_expr:: type_coercion:: other:: get_coerce_type_for_case_expression;
3435use datafusion:: logical_expr:: { ColumnarValue , Operator } ;
3536use datafusion:: physical_expr:: expressions:: {
@@ -63,34 +64,66 @@ pub fn create_case_when(
6364 else {
6465 return Ok ( Arc :: new ( CaseWhenExpr :: try_new ( when_then, else_expr) ?) ) ;
6566 } ;
66- // The branches share a Spark type, so any difference is in the Arrow representation. For a
67- // timestamp that is the timezone label, and the cast only relabels it, but Comet's cast still
68- // needs a timezone. Every `TimestampType` value in a native plan is labelled UTC.
69- let cast_options = SparkCastOptions :: new ( EvalMode :: Legacy , "UTC" , false ) ;
70- // A branch that already has the common type is not wrapped in a cast, which would do nothing
71- // but hide what the branch is from the evaluation.
72- let coerce = |expr : Arc < dyn PhysicalExpr > , data_type : & DataType | -> Arc < dyn PhysicalExpr > {
73- if data_type == & coerce_type {
74- expr
75- } else {
76- Arc :: new ( Cast :: new (
77- expr,
78- coerce_type. clone ( ) ,
79- cast_options. clone ( ) ,
80- None ,
81- None ,
82- ) )
83- }
84- } ;
8567 let when_then = when_then
8668 . into_iter ( )
8769 . zip ( & then_types)
88- . map ( |( ( when, then) , then_type) | ( when, coerce ( then, then_type) ) )
70+ . map ( |( ( when, then) , then_type) | ( when, coerce_branch ( then, then_type, & coerce_type ) ) )
8971 . collect ( ) ;
90- let else_expr = else_expr. map ( |e| coerce ( e, & else_type) ) ;
72+ let else_expr = else_expr. map ( |e| coerce_branch ( e, & else_type, & coerce_type ) ) ;
9173 Ok ( Arc :: new ( CaseWhenExpr :: try_new ( when_then, else_expr) ?) )
9274}
9375
76+ /// Creates an `IF`, casting a branch whose type differs from the other's to their common type.
77+ ///
78+ /// Spark adds no cast when the branches differ only in whether a nested field can be NULL, or in
79+ /// the case of a struct field name. But [`IfExpr`] reports the THEN branch's type, and returns the
80+ /// ELSE branch's array unchanged when no row of a batch takes the THEN branch.
81+ pub fn create_if_expr (
82+ if_expr : Arc < dyn PhysicalExpr > ,
83+ true_expr : Arc < dyn PhysicalExpr > ,
84+ false_expr : Arc < dyn PhysicalExpr > ,
85+ input_schema : & Schema ,
86+ ) -> Result < Arc < dyn PhysicalExpr > > {
87+ let true_type = true_expr. data_type ( input_schema) ?;
88+ let false_type = false_expr. data_type ( input_schema) ?;
89+ // The coercion that `get_coerce_type_for_case_expression` folds over the branches of a CASE
90+ // WHEN, starting from the ELSE branch. Here the THEN branch goes first, so the common type
91+ // takes its struct field names, as Spark's `If.dataType` does.
92+ let Some ( common_type) = type_union_coercion ( & true_type, & false_type) else {
93+ return Ok ( Arc :: new ( IfExpr :: new ( if_expr, true_expr, false_expr) ) ) ;
94+ } ;
95+ Ok ( Arc :: new ( IfExpr :: new (
96+ if_expr,
97+ coerce_branch ( true_expr, & true_type, & common_type) ,
98+ coerce_branch ( false_expr, & false_type, & common_type) ,
99+ ) ) )
100+ }
101+
102+ /// Casts a CASE WHEN or IF branch whose type is `data_type` to the branches' `common_type`.
103+ ///
104+ /// The branches share a Spark type, so any difference is in the Arrow representation. For a
105+ /// timestamp that is the timezone label, and the cast only relabels it, but Comet's cast still
106+ /// needs a timezone. Every `TimestampType` value in a native plan is labelled UTC.
107+ fn coerce_branch (
108+ expr : Arc < dyn PhysicalExpr > ,
109+ data_type : & DataType ,
110+ common_type : & DataType ,
111+ ) -> Arc < dyn PhysicalExpr > {
112+ // A branch that already has the common type is not wrapped in a cast, which would do nothing
113+ // but hide what the branch is from the evaluation.
114+ if data_type == common_type {
115+ return expr;
116+ }
117+ let cast_options = SparkCastOptions :: new ( EvalMode :: Legacy , "UTC" , false ) ;
118+ Arc :: new ( Cast :: new (
119+ expr,
120+ common_type. clone ( ) ,
121+ cast_options,
122+ None ,
123+ None ,
124+ ) )
125+ }
126+
94127/// Spark's `CASE WHEN`, which Comet also uses for `IF` and `COALESCE`.
95128///
96129/// Spark evaluates a WHEN only for the rows that no earlier WHEN matched, and a branch's value
@@ -766,11 +799,11 @@ fn merge_nulls(num_rows: usize, branches: &[Branch]) -> Option<NullBuffer> {
766799mod tests {
767800 use super :: * ;
768801 use arrow:: array:: {
769- BinaryArray , Int32Array , Int64Array , LargeStringArray , StringArray ,
802+ BinaryArray , Int32Array , Int64Array , LargeStringArray , StringArray , StructArray ,
770803 TimestampMicrosecondArray ,
771804 } ;
772805 use arrow:: compute:: cast;
773- use arrow:: datatypes:: { Field , TimeUnit , TimestampMicrosecondType } ;
806+ use arrow:: datatypes:: { Field , FieldRef , TimeUnit , TimestampMicrosecondType } ;
774807 use datafusion:: physical_expr:: expressions:: { col, lit} ;
775808 use rand:: rngs:: StdRng ;
776809 use rand:: { RngExt , SeedableRng } ;
@@ -1269,6 +1302,126 @@ mod tests {
12691302 }
12701303 }
12711304
1305+ /// Evaluates an IF over `batch`, whose first row takes the THEN branch and second row the ELSE
1306+ /// branch, and over each row alone, and checks that every result has the type the IF reports.
1307+ /// Returns the result for the whole batch.
1308+ fn evaluate_if ( expr : & Arc < dyn PhysicalExpr > , batch : & RecordBatch ) -> ArrayRef {
1309+ let data_type = expr. data_type ( & batch. schema ( ) ) . unwrap ( ) ;
1310+ // Both branches, only THEN, and only ELSE, whose array `IfExpr` returns unchanged
1311+ for ( offset, len) in [ ( 0 , 2 ) , ( 0 , 1 ) , ( 1 , 1 ) ] {
1312+ let result = expr. evaluate ( & batch. slice ( offset, len) ) . unwrap ( ) ;
1313+ let result = result. into_array ( len) . unwrap ( ) ;
1314+ assert_eq ! (
1315+ result. data_type( ) ,
1316+ & data_type,
1317+ "{expr} over rows {offset}..+{len}"
1318+ ) ;
1319+ }
1320+ expr. evaluate ( batch) . unwrap ( ) . into_array ( 2 ) . unwrap ( )
1321+ }
1322+
1323+ /// IF branches that share a Spark timestamp type but carry different Arrow timezone labels are
1324+ /// reconciled by relabelling one of them, as CASE branches are. The result has the THEN
1325+ /// branch's label, which `IfExpr` reported before the ELSE branch was cast, unless it has none.
1326+ #[ test]
1327+ fn if_reconciles_timestamp_timezone_labels ( ) {
1328+ let timestamp =
1329+ |label : Option < & str > | DataType :: Timestamp ( TimeUnit :: Microsecond , label. map ( Into :: into) ) ;
1330+ // The THEN branch's label, the ELSE branch's label, and the result's
1331+ for ( then_label, else_label, label) in [
1332+ ( Some ( "Etc/UTC" ) , Some ( "UTC" ) , Some ( "Etc/UTC" ) ) ,
1333+ ( None , Some ( "UTC" ) , Some ( "UTC" ) ) ,
1334+ ( Some ( "UTC" ) , Some ( "Etc/UTC" ) , Some ( "UTC" ) ) ,
1335+ ( Some ( "UTC" ) , None , Some ( "UTC" ) ) ,
1336+ ] {
1337+ let schema = Schema :: new ( vec ! [
1338+ Field :: new( "b" , DataType :: Boolean , true ) ,
1339+ Field :: new( "t" , timestamp( then_label) , true ) ,
1340+ Field :: new( "e" , timestamp( else_label) , true ) ,
1341+ ] ) ;
1342+ let batch = RecordBatch :: try_new (
1343+ Arc :: new ( schema. clone ( ) ) ,
1344+ vec ! [
1345+ Arc :: new( BooleanArray :: from( vec![ true , false ] ) ) ,
1346+ Arc :: new(
1347+ TimestampMicrosecondArray :: from( vec![ 1 , 2 ] ) . with_timezone_opt( then_label) ,
1348+ ) ,
1349+ Arc :: new(
1350+ TimestampMicrosecondArray :: from( vec![ 10 , 20 ] ) . with_timezone_opt( else_label) ,
1351+ ) ,
1352+ ] ,
1353+ )
1354+ . unwrap ( ) ;
1355+ let c = |name : & str | col ( name, & schema) . unwrap ( ) ;
1356+ let expr = create_if_expr ( c ( "b" ) , c ( "t" ) , c ( "e" ) , & schema) . unwrap ( ) ;
1357+ let labels = format ! ( "{then_label:?} {else_label:?}" ) ;
1358+ assert_eq ! (
1359+ expr. data_type( & schema) . unwrap( ) ,
1360+ timestamp( label) ,
1361+ "{labels}"
1362+ ) ;
1363+ let result = evaluate_if ( & expr, & batch) ;
1364+ let values = result. as_primitive :: < TimestampMicrosecondType > ( ) ;
1365+ assert_eq ! ( values. values( ) . to_vec( ) , vec![ 1 , 20 ] , "{labels}" ) ;
1366+ }
1367+ }
1368+
1369+ /// IF branches whose struct field differs only in whether it can be NULL, or in the case of its
1370+ /// name, which Spark adds no cast for. The result's field can be NULL if either branch's can,
1371+ /// and has the THEN branch's name, as in Spark's `If.dataType`.
1372+ #[ test]
1373+ fn if_reconciles_struct_field_nullability_and_names ( ) {
1374+ let field =
1375+ |name : & str , nullable : bool | Arc :: new ( Field :: new ( name, DataType :: Int32 , nullable) ) ;
1376+ let struct_type = |field : & FieldRef | DataType :: Struct ( vec ! [ Arc :: clone( field) ] . into ( ) ) ;
1377+ let column = |field : & FieldRef , values : Vec < Option < i32 > > | -> ArrayRef {
1378+ Arc :: new ( StructArray :: new (
1379+ vec ! [ Arc :: clone( field) ] . into ( ) ,
1380+ vec ! [ Arc :: new( Int32Array :: from( values) ) as ArrayRef ] ,
1381+ None ,
1382+ ) )
1383+ } ;
1384+ // The THEN branch's field, the ELSE branch's field, and the result's
1385+ for ( then_field, else_field, result_field) in [
1386+ ( field ( "x" , false ) , field ( "x" , true ) , field ( "x" , true ) ) ,
1387+ ( field ( "x" , true ) , field ( "x" , false ) , field ( "x" , true ) ) ,
1388+ ( field ( "x" , true ) , field ( "X" , true ) , field ( "x" , true ) ) ,
1389+ ( field ( "X" , false ) , field ( "x" , true ) , field ( "X" , true ) ) ,
1390+ ] {
1391+ let schema = Schema :: new ( vec ! [
1392+ Field :: new( "b" , DataType :: Boolean , true ) ,
1393+ Field :: new( "t" , struct_type( & then_field) , true ) ,
1394+ Field :: new( "e" , struct_type( & else_field) , true ) ,
1395+ ] ) ;
1396+ // A NULL field in the row that takes the ELSE branch, where it can be NULL
1397+ let else_values = if else_field. is_nullable ( ) {
1398+ vec ! [ Some ( 10 ) , None ]
1399+ } else {
1400+ vec ! [ Some ( 10 ) , Some ( 20 ) ]
1401+ } ;
1402+ let batch = RecordBatch :: try_new (
1403+ Arc :: new ( schema. clone ( ) ) ,
1404+ vec ! [
1405+ Arc :: new( BooleanArray :: from( vec![ true , false ] ) ) ,
1406+ column( & then_field, vec![ Some ( 1 ) , Some ( 2 ) ] ) ,
1407+ column( & else_field, else_values. clone( ) ) ,
1408+ ] ,
1409+ )
1410+ . unwrap ( ) ;
1411+ let c = |name : & str | col ( name, & schema) . unwrap ( ) ;
1412+ let expr = create_if_expr ( c ( "b" ) , c ( "t" ) , c ( "e" ) , & schema) . unwrap ( ) ;
1413+ let fields = format ! ( "{then_field:?} {else_field:?}" ) ;
1414+ assert_eq ! (
1415+ expr. data_type( & schema) . unwrap( ) ,
1416+ struct_type( & result_field) ,
1417+ "{fields}"
1418+ ) ;
1419+ let result = evaluate_if ( & expr, & batch) ;
1420+ let expected = column ( & result_field, vec ! [ Some ( 1 ) , else_values[ 1 ] ] ) ;
1421+ assert_eq ! ( result. as_ref( ) , expected. as_ref( ) , "{fields}" ) ;
1422+ }
1423+ }
1424+
12721425 #[ test]
12731426 fn many_branches ( ) {
12741427 let len = 1000 ;
0 commit comments