Skip to content

Commit 171e50e

Browse files
committed
refactor: plan native IF through create_if_expr, sharing CASE WHEN's branch cast
Move the IF branch coercion out of the planner into `create_if_expr`, next to `create_case_when`. It handles the no-common-type case up front, and the cast of a differing branch is now one helper that both use. IF still takes the common type from `type_union_coercion` with the THEN branch first, so struct field names come from the THEN branch, as in Spark's `If.dataType`. `create_case_when` folds from the ELSE branch, so IF can't reuse it as is. Add unit tests that plan an IF whose timestamp branches have different timezone labels, and one whose struct field differs in nullability or in the case of its name. They check the result type over a batch that mixes the branches, one where every row takes THEN, and one where every row takes ELSE.
1 parent 7e46bd9 commit 171e50e

3 files changed

Lines changed: 184 additions & 59 deletions

File tree

‎native/core/src/execution/planner.rs‎

Lines changed: 7 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,6 @@ use datafusion::common::{
106106
JoinType as DFJoinType, NullEquality, ScalarValue,
107107
};
108108
use datafusion::datasource::listing::PartitionedFile;
109-
use datafusion::logical_expr::type_coercion::binary::type_union_coercion;
110109
use datafusion::logical_expr::type_coercion::functions::fields_with_udf;
111110
use datafusion::logical_expr::{
112111
AggregateUDF, ReturnFieldArgs, ScalarUDF, TypeSignature, WindowFrame, WindowFrameBound,
@@ -147,12 +146,12 @@ use datafusion_comet_proto::{
147146
spark_partitioning::{partitioning::PartitioningStruct, Partitioning as SparkPartitioning},
148147
};
149148
use datafusion_comet_spark_expr::{
150-
create_case_when, jvm_udf::JvmScalarUdfExpr, normalize_floats, spark_in_list, ApproxPercentile,
151-
ArrayInsert, Avg, AvgDecimal, Cast, CheckOverflow, Correlation, Covariance, CreateNamedStruct,
152-
DecimalRescaleCheckOverflow, GetArrayStructFields, GetStructField, HllPlusPlus, HllSketchAgg,
153-
HllUnionAgg, IfExpr, ListExtract, MaxMinBy, Mode, NormalizeNaNAndZero, Regr, RegrType,
154-
SparkCastOptions, Stddev, SumDecimal, ToJson, UnboundColumn, Variance, WideDecimalBinaryExpr,
155-
WideDecimalOp,
149+
create_case_when, create_if_expr, jvm_udf::JvmScalarUdfExpr, normalize_floats, spark_in_list,
150+
ApproxPercentile, ArrayInsert, Avg, AvgDecimal, Cast, CheckOverflow, Correlation, Covariance,
151+
CreateNamedStruct, DecimalRescaleCheckOverflow, GetArrayStructFields, GetStructField,
152+
HllPlusPlus, HllSketchAgg, HllUnionAgg, IfExpr, ListExtract, MaxMinBy, Mode,
153+
NormalizeNaNAndZero, Regr, RegrType, SparkCastOptions, Stddev, SumDecimal, ToJson,
154+
UnboundColumn, Variance, WideDecimalBinaryExpr, WideDecimalOp,
156155
};
157156
use itertools::Itertools;
158157
use jni::objects::{Global, JObject};
@@ -803,34 +802,7 @@ impl PhysicalPlanner {
803802
self.create_expr(expr.true_expr.as_ref().unwrap(), Arc::clone(&input_schema))?;
804803
let false_expr =
805804
self.create_expr(expr.false_expr.as_ref().unwrap(), Arc::clone(&input_schema))?;
806-
// Spark adds no cast when the branches differ only in whether a nested field can
807-
// be NULL, but `IfExpr` reports the THEN branch's type and returns the ELSE
808-
// branch's array unchanged when no row of a batch takes the THEN branch. So cast a
809-
// branch whose type differs from the common type, as `create_case_expr` does for
810-
// CASE WHEN. The THEN branch goes first so that the common type keeps its field
811-
// names, as Spark's `If` does.
812-
let true_type = true_expr.data_type(&input_schema)?;
813-
let false_type = false_expr.data_type(&input_schema)?;
814-
let common_type = type_union_coercion(&true_type, &false_type);
815-
let coerce = |branch, data_type: &DataType| -> Arc<dyn PhysicalExpr> {
816-
match &common_type {
817-
// The branches share a Spark type, so a timestamp can only differ in its
818-
// label, and every TimestampType in a native plan is labelled UTC
819-
Some(common_type) if data_type != common_type => Arc::new(Cast::new(
820-
branch,
821-
common_type.clone(),
822-
SparkCastOptions::new(EvalMode::Legacy, "UTC", false),
823-
None,
824-
None,
825-
)),
826-
_ => branch,
827-
}
828-
};
829-
Ok(Arc::new(IfExpr::new(
830-
if_expr,
831-
coerce(true_expr, &true_type),
832-
coerce(false_expr, &false_type),
833-
)))
805+
create_if_expr(if_expr, true_expr, false_expr, &input_schema).map_err(|e| e.into())
834806
}
835807
ExprStruct::NormalizeNanAndZero(expr) => {
836808
let child = self.create_expr(expr.child.as_ref().unwrap(), input_schema)?;

‎native/spark-expr/src/conditional_funcs/case_when.rs‎

Lines changed: 176 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ use datafusion::common::cast::as_boolean_array;
3030
use datafusion::common::{
3131
exec_err, internal_datafusion_err, internal_err, DataFusionError, Result, ScalarValue,
3232
};
33+
use datafusion::logical_expr::type_coercion::binary::type_union_coercion;
3334
use datafusion::logical_expr::type_coercion::other::get_coerce_type_for_case_expression;
3435
use datafusion::logical_expr::{ColumnarValue, Operator};
3536
use datafusion::physical_expr::expressions::{
@@ -63,34 +64,66 @@ pub fn create_case_when(
6364
else {
6465
return Ok(Arc::new(CaseWhenExpr::try_new(when_then, else_expr)?));
6566
};
66-
// The branches share a Spark type, so any difference is in the Arrow representation. For a
67-
// timestamp that is the timezone label, and the cast only relabels it, but Comet's cast still
68-
// needs a timezone. Every `TimestampType` value in a native plan is labelled UTC.
69-
let cast_options = SparkCastOptions::new(EvalMode::Legacy, "UTC", false);
70-
// A branch that already has the common type is not wrapped in a cast, which would do nothing
71-
// but hide what the branch is from the evaluation.
72-
let coerce = |expr: Arc<dyn PhysicalExpr>, data_type: &DataType| -> Arc<dyn PhysicalExpr> {
73-
if data_type == &coerce_type {
74-
expr
75-
} else {
76-
Arc::new(Cast::new(
77-
expr,
78-
coerce_type.clone(),
79-
cast_options.clone(),
80-
None,
81-
None,
82-
))
83-
}
84-
};
8567
let when_then = when_then
8668
.into_iter()
8769
.zip(&then_types)
88-
.map(|((when, then), then_type)| (when, coerce(then, then_type)))
70+
.map(|((when, then), then_type)| (when, coerce_branch(then, then_type, &coerce_type)))
8971
.collect();
90-
let else_expr = else_expr.map(|e| coerce(e, &else_type));
72+
let else_expr = else_expr.map(|e| coerce_branch(e, &else_type, &coerce_type));
9173
Ok(Arc::new(CaseWhenExpr::try_new(when_then, else_expr)?))
9274
}
9375

76+
/// Creates an `IF`, casting a branch whose type differs from the other's to their common type.
77+
///
78+
/// Spark adds no cast when the branches differ only in whether a nested field can be NULL, or in
79+
/// the case of a struct field name. But [`IfExpr`] reports the THEN branch's type, and returns the
80+
/// ELSE branch's array unchanged when no row of a batch takes the THEN branch.
81+
pub fn create_if_expr(
82+
if_expr: Arc<dyn PhysicalExpr>,
83+
true_expr: Arc<dyn PhysicalExpr>,
84+
false_expr: Arc<dyn PhysicalExpr>,
85+
input_schema: &Schema,
86+
) -> Result<Arc<dyn PhysicalExpr>> {
87+
let true_type = true_expr.data_type(input_schema)?;
88+
let false_type = false_expr.data_type(input_schema)?;
89+
// The coercion that `get_coerce_type_for_case_expression` folds over the branches of a CASE
90+
// WHEN, starting from the ELSE branch. Here the THEN branch goes first, so the common type
91+
// takes its struct field names, as Spark's `If.dataType` does.
92+
let Some(common_type) = type_union_coercion(&true_type, &false_type) else {
93+
return Ok(Arc::new(IfExpr::new(if_expr, true_expr, false_expr)));
94+
};
95+
Ok(Arc::new(IfExpr::new(
96+
if_expr,
97+
coerce_branch(true_expr, &true_type, &common_type),
98+
coerce_branch(false_expr, &false_type, &common_type),
99+
)))
100+
}
101+
102+
/// Casts a CASE WHEN or IF branch whose type is `data_type` to the branches' `common_type`.
103+
///
104+
/// The branches share a Spark type, so any difference is in the Arrow representation. For a
105+
/// timestamp that is the timezone label, and the cast only relabels it, but Comet's cast still
106+
/// needs a timezone. Every `TimestampType` value in a native plan is labelled UTC.
107+
fn coerce_branch(
108+
expr: Arc<dyn PhysicalExpr>,
109+
data_type: &DataType,
110+
common_type: &DataType,
111+
) -> Arc<dyn PhysicalExpr> {
112+
// A branch that already has the common type is not wrapped in a cast, which would do nothing
113+
// but hide what the branch is from the evaluation.
114+
if data_type == common_type {
115+
return expr;
116+
}
117+
let cast_options = SparkCastOptions::new(EvalMode::Legacy, "UTC", false);
118+
Arc::new(Cast::new(
119+
expr,
120+
common_type.clone(),
121+
cast_options,
122+
None,
123+
None,
124+
))
125+
}
126+
94127
/// Spark's `CASE WHEN`, which Comet also uses for `IF` and `COALESCE`.
95128
///
96129
/// Spark evaluates a WHEN only for the rows that no earlier WHEN matched, and a branch's value
@@ -766,11 +799,11 @@ fn merge_nulls(num_rows: usize, branches: &[Branch]) -> Option<NullBuffer> {
766799
mod tests {
767800
use super::*;
768801
use arrow::array::{
769-
BinaryArray, Int32Array, Int64Array, LargeStringArray, StringArray,
802+
BinaryArray, Int32Array, Int64Array, LargeStringArray, StringArray, StructArray,
770803
TimestampMicrosecondArray,
771804
};
772805
use arrow::compute::cast;
773-
use arrow::datatypes::{Field, TimeUnit, TimestampMicrosecondType};
806+
use arrow::datatypes::{Field, FieldRef, TimeUnit, TimestampMicrosecondType};
774807
use datafusion::physical_expr::expressions::{col, lit};
775808
use rand::rngs::StdRng;
776809
use rand::{RngExt, SeedableRng};
@@ -1269,6 +1302,126 @@ mod tests {
12691302
}
12701303
}
12711304

1305+
/// Evaluates an IF over `batch`, whose first row takes the THEN branch and second row the ELSE
1306+
/// branch, and over each row alone, and checks that every result has the type the IF reports.
1307+
/// Returns the result for the whole batch.
1308+
fn evaluate_if(expr: &Arc<dyn PhysicalExpr>, batch: &RecordBatch) -> ArrayRef {
1309+
let data_type = expr.data_type(&batch.schema()).unwrap();
1310+
// Both branches, only THEN, and only ELSE, whose array `IfExpr` returns unchanged
1311+
for (offset, len) in [(0, 2), (0, 1), (1, 1)] {
1312+
let result = expr.evaluate(&batch.slice(offset, len)).unwrap();
1313+
let result = result.into_array(len).unwrap();
1314+
assert_eq!(
1315+
result.data_type(),
1316+
&data_type,
1317+
"{expr} over rows {offset}..+{len}"
1318+
);
1319+
}
1320+
expr.evaluate(batch).unwrap().into_array(2).unwrap()
1321+
}
1322+
1323+
/// IF branches that share a Spark timestamp type but carry different Arrow timezone labels are
1324+
/// reconciled by relabelling one of them, as CASE branches are. The result has the THEN
1325+
/// branch's label, which `IfExpr` reported before the ELSE branch was cast, unless it has none.
1326+
#[test]
1327+
fn if_reconciles_timestamp_timezone_labels() {
1328+
let timestamp =
1329+
|label: Option<&str>| DataType::Timestamp(TimeUnit::Microsecond, label.map(Into::into));
1330+
// The THEN branch's label, the ELSE branch's label, and the result's
1331+
for (then_label, else_label, label) in [
1332+
(Some("Etc/UTC"), Some("UTC"), Some("Etc/UTC")),
1333+
(None, Some("UTC"), Some("UTC")),
1334+
(Some("UTC"), Some("Etc/UTC"), Some("UTC")),
1335+
(Some("UTC"), None, Some("UTC")),
1336+
] {
1337+
let schema = Schema::new(vec![
1338+
Field::new("b", DataType::Boolean, true),
1339+
Field::new("t", timestamp(then_label), true),
1340+
Field::new("e", timestamp(else_label), true),
1341+
]);
1342+
let batch = RecordBatch::try_new(
1343+
Arc::new(schema.clone()),
1344+
vec![
1345+
Arc::new(BooleanArray::from(vec![true, false])),
1346+
Arc::new(
1347+
TimestampMicrosecondArray::from(vec![1, 2]).with_timezone_opt(then_label),
1348+
),
1349+
Arc::new(
1350+
TimestampMicrosecondArray::from(vec![10, 20]).with_timezone_opt(else_label),
1351+
),
1352+
],
1353+
)
1354+
.unwrap();
1355+
let c = |name: &str| col(name, &schema).unwrap();
1356+
let expr = create_if_expr(c("b"), c("t"), c("e"), &schema).unwrap();
1357+
let labels = format!("{then_label:?} {else_label:?}");
1358+
assert_eq!(
1359+
expr.data_type(&schema).unwrap(),
1360+
timestamp(label),
1361+
"{labels}"
1362+
);
1363+
let result = evaluate_if(&expr, &batch);
1364+
let values = result.as_primitive::<TimestampMicrosecondType>();
1365+
assert_eq!(values.values().to_vec(), vec![1, 20], "{labels}");
1366+
}
1367+
}
1368+
1369+
/// IF branches whose struct field differs only in whether it can be NULL, or in the case of its
1370+
/// name, which Spark adds no cast for. The result's field can be NULL if either branch's can,
1371+
/// and has the THEN branch's name, as in Spark's `If.dataType`.
1372+
#[test]
1373+
fn if_reconciles_struct_field_nullability_and_names() {
1374+
let field =
1375+
|name: &str, nullable: bool| Arc::new(Field::new(name, DataType::Int32, nullable));
1376+
let struct_type = |field: &FieldRef| DataType::Struct(vec![Arc::clone(field)].into());
1377+
let column = |field: &FieldRef, values: Vec<Option<i32>>| -> ArrayRef {
1378+
Arc::new(StructArray::new(
1379+
vec![Arc::clone(field)].into(),
1380+
vec![Arc::new(Int32Array::from(values)) as ArrayRef],
1381+
None,
1382+
))
1383+
};
1384+
// The THEN branch's field, the ELSE branch's field, and the result's
1385+
for (then_field, else_field, result_field) in [
1386+
(field("x", false), field("x", true), field("x", true)),
1387+
(field("x", true), field("x", false), field("x", true)),
1388+
(field("x", true), field("X", true), field("x", true)),
1389+
(field("X", false), field("x", true), field("X", true)),
1390+
] {
1391+
let schema = Schema::new(vec![
1392+
Field::new("b", DataType::Boolean, true),
1393+
Field::new("t", struct_type(&then_field), true),
1394+
Field::new("e", struct_type(&else_field), true),
1395+
]);
1396+
// A NULL field in the row that takes the ELSE branch, where it can be NULL
1397+
let else_values = if else_field.is_nullable() {
1398+
vec![Some(10), None]
1399+
} else {
1400+
vec![Some(10), Some(20)]
1401+
};
1402+
let batch = RecordBatch::try_new(
1403+
Arc::new(schema.clone()),
1404+
vec![
1405+
Arc::new(BooleanArray::from(vec![true, false])),
1406+
column(&then_field, vec![Some(1), Some(2)]),
1407+
column(&else_field, else_values.clone()),
1408+
],
1409+
)
1410+
.unwrap();
1411+
let c = |name: &str| col(name, &schema).unwrap();
1412+
let expr = create_if_expr(c("b"), c("t"), c("e"), &schema).unwrap();
1413+
let fields = format!("{then_field:?} {else_field:?}");
1414+
assert_eq!(
1415+
expr.data_type(&schema).unwrap(),
1416+
struct_type(&result_field),
1417+
"{fields}"
1418+
);
1419+
let result = evaluate_if(&expr, &batch);
1420+
let expected = column(&result_field, vec![Some(1), else_values[1]]);
1421+
assert_eq!(result.as_ref(), expected.as_ref(), "{fields}");
1422+
}
1423+
}
1424+
12721425
#[test]
12731426
fn many_branches() {
12741427
let len = 1000;

‎native/spark-expr/src/conditional_funcs/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,5 +18,5 @@
1818
mod case_when;
1919
mod if_expr;
2020

21-
pub use case_when::{create_case_when, CaseWhenExpr};
21+
pub use case_when::{create_case_when, create_if_expr, CaseWhenExpr};
2222
pub use if_expr::IfExpr;

0 commit comments

Comments
 (0)