Skip to content

Commit e918603

Browse files
committed
fix: skip CASE specialization for fallible casts
Do not treat every CastExpr as safe to evaluate on the full batch. Only specialize when a cast is provably infallible (safe-mode, TRY_CAST, same-type, or a widening numeric cast). Fallible narrowing casts fall back to filtered CASE so excluded rows cannot change short-circuit semantics.
1 parent cb893d7 commit e918603

2 files changed

Lines changed: 194 additions & 8 deletions

File tree

  • datafusion

‎datafusion/physical-expr/src/expressions/case.rs‎

Lines changed: 178 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -337,8 +337,12 @@ fn is_cheap_and_infallible(expr: &Arc<dyn PhysicalExpr>) -> bool {
337337
/// True when `when` / `then` is the divide-by-zero protection pattern
338338
/// `CASE WHEN y {>, !=, <} 0 THEN x / y`.
339339
///
340-
/// Both divide operands must be cheap (column, literal, or cast of those) so
341-
/// evaluating them on the full batch cannot introduce errors on skipped rows.
340+
/// Both divide operands must be cheap (column, literal, or cast of those).
341+
/// [`CastExpr`]s that are provably fallible (for example a narrowing
342+
/// `Int32` → `Int8` of a literal) are rejected so CASE short-circuit
343+
/// semantics are preserved. Column casts whose source type is not known
344+
/// until evaluation are re-checked with the batch schema in
345+
/// [`CaseExpr::divide_by_zero_protection`].
342346
fn is_divide_by_zero_protection(
343347
when_expr: &Arc<dyn PhysicalExpr>,
344348
then_expr: &Arc<dyn PhysicalExpr>,
@@ -350,8 +354,10 @@ fn is_divide_by_zero_protection(
350354
return false;
351355
};
352356
unwrap_casts(divisor).eq(unwrap_casts(checked))
353-
&& is_cheap_for_full_batch(numerator)
354-
&& is_cheap_for_full_batch(divisor)
357+
&& is_column_literal_or_cast(numerator)
358+
&& is_column_literal_or_cast(divisor)
359+
&& !has_provably_fallible_cast(numerator)
360+
&& !has_provably_fallible_cast(divisor)
355361
}
356362

357363
/// Operand being tested for a non-zero (or strictly positive / negative) value.
@@ -412,13 +418,92 @@ fn is_literal_zero(expr: &dyn PhysicalExpr) -> bool {
412418
}
413419
}
414420

415-
fn is_cheap_for_full_batch(expr: &Arc<dyn PhysicalExpr>) -> bool {
421+
/// Column, literal, or (possibly nested) `Cast` / `TryCast` of those.
422+
///
423+
/// This only describes the *shape* of the tree: evaluating it on a full
424+
/// batch is cheap. It does **not** mean a `CastExpr` is safe — use
425+
/// [`is_cheap_for_full_batch`] for that.
426+
fn is_column_literal_or_cast(expr: &Arc<dyn PhysicalExpr>) -> bool {
416427
if expr.is::<Column>() || expr.is::<Literal>() {
417428
true
418429
} else if let Some(cast) = expr.downcast_ref::<CastExpr>() {
419-
is_cheap_for_full_batch(cast.expr())
430+
is_column_literal_or_cast(cast.expr())
431+
} else if let Some(try_cast) = expr.downcast_ref::<TryCastExpr>() {
432+
is_column_literal_or_cast(try_cast.expr())
433+
} else {
434+
false
435+
}
436+
}
437+
438+
/// True when `expr` is cheap **and** safe to evaluate on every row of a
439+
/// batch, including rows that CASE would otherwise skip.
440+
///
441+
/// `CastExpr` is allowed only when it cannot fail: safe-mode casts,
442+
/// [`TryCastExpr`] (returns NULL on failure), same-type casts, and
443+
/// widening numeric casts identified by [`CastExpr::check_bigger_cast`].
444+
/// Narrowing casts such as `Int32` → `Int8` return false.
445+
///
446+
/// When `schema` is `None`, column types are unknown, so a `CastExpr` over
447+
/// a column cannot be proven infallible and is rejected.
448+
fn is_cheap_for_full_batch(
449+
expr: &Arc<dyn PhysicalExpr>,
450+
schema: Option<&Schema>,
451+
) -> bool {
452+
if expr.is::<Column>() || expr.is::<Literal>() {
453+
true
454+
} else if let Some(cast) = expr.downcast_ref::<CastExpr>() {
455+
is_infallible_cast(cast, schema) && is_cheap_for_full_batch(cast.expr(), schema)
456+
} else if let Some(try_cast) = expr.downcast_ref::<TryCastExpr>() {
457+
// TRY_CAST never errors (it yields NULL on failure).
458+
is_cheap_for_full_batch(try_cast.expr(), schema)
459+
} else {
460+
false
461+
}
462+
}
463+
464+
/// True when a `CastExpr` is known to be unable to error.
465+
fn is_infallible_cast(cast: &CastExpr, schema: Option<&Schema>) -> bool {
466+
// Safe-mode CAST returns NULL on failure rather than erroring.
467+
if cast.cast_options().safe {
468+
return true;
469+
}
470+
let Some(source_type) = expr_data_type(cast.expr(), schema) else {
471+
return false;
472+
};
473+
CastExpr::check_bigger_cast(cast.cast_type(), &source_type)
474+
}
475+
476+
/// Best-effort data type of `expr`. With a schema this uses
477+
/// [`PhysicalExpr::data_type`]; without one, only literals and casts
478+
/// (which know their output type) can be resolved.
479+
fn expr_data_type(
480+
expr: &Arc<dyn PhysicalExpr>,
481+
schema: Option<&Schema>,
482+
) -> Option<DataType> {
483+
if let Some(schema) = schema {
484+
return expr.data_type(schema).ok();
485+
}
486+
if let Some(lit) = expr.downcast_ref::<Literal>() {
487+
return Some(lit.value().data_type());
488+
}
489+
if let Some(cast) = expr.downcast_ref::<CastExpr>() {
490+
return Some(cast.cast_type().clone());
491+
}
492+
expr.downcast_ref::<TryCastExpr>()
493+
.map(|try_cast| try_cast.cast_type().clone())
494+
}
495+
496+
/// Walk a column / literal / cast tree and report whether any `CastExpr`
497+
/// is known to be fallible without a schema (typically a narrowing cast
498+
/// of a literal or of another cast whose output type is known).
499+
fn has_provably_fallible_cast(expr: &Arc<dyn PhysicalExpr>) -> bool {
500+
if let Some(cast) = expr.downcast_ref::<CastExpr>() {
501+
let known_fallible = !cast.cast_options().safe
502+
&& expr_data_type(cast.expr(), None)
503+
.is_some_and(|src| !CastExpr::check_bigger_cast(cast.cast_type(), &src));
504+
known_fallible || has_provably_fallible_cast(cast.expr())
420505
} else if let Some(try_cast) = expr.downcast_ref::<TryCastExpr>() {
421-
is_cheap_for_full_batch(try_cast.expr())
506+
has_provably_fallible_cast(try_cast.expr())
422507
} else {
423508
false
424509
}
@@ -876,6 +961,18 @@ impl CaseExpr {
876961
)
877962
})?;
878963

964+
// Mixed WHEN: the fast path evaluates the divisor on the full batch.
965+
// Fallible casts (e.g. Int32 → Int8) can error on excluded rows and
966+
// must use filtered CASE evaluation instead. Widening casts such as
967+
// Int32 → Float64 stay on the fast path.
968+
let schema = batch.schema();
969+
if !is_cheap_for_full_batch(binary.left(), Some(schema.as_ref()))
970+
|| !is_cheap_for_full_batch(binary.right(), Some(schema.as_ref()))
971+
{
972+
let projected = self.body.project()?;
973+
return self.expr_or_expr(batch, &projected);
974+
}
975+
879976
let num_rows = batch.num_rows();
880977
let numerator = binary.left().evaluate(batch)?.into_array(num_rows)?;
881978
let divisor = binary.right().evaluate(batch)?.into_array(num_rows)?;
@@ -1746,7 +1843,7 @@ mod tests {
17461843
use arrow::buffer::Buffer;
17471844
use arrow::datatypes::DataType::Float64;
17481845
use arrow::datatypes::Field;
1749-
use datafusion_common::cast::{as_float64_array, as_int32_array};
1846+
use datafusion_common::cast::{as_float64_array, as_int8_array, as_int32_array};
17501847
use datafusion_common::plan_err;
17511848
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
17521849
use datafusion_expr::type_coercion::binary::type_union_coercion;
@@ -2809,6 +2906,79 @@ mod tests {
28092906
Ok(())
28102907
}
28112908

2909+
#[test]
2910+
fn test_divide_by_zero_protection_skips_fallible_cast() -> Result<()> {
2911+
// CASE WHEN d < 0 THEN CAST(10 AS TINYINT) / CAST(d AS TINYINT) END
2912+
// over (-1), (1000), (0) => -10, NULL, NULL
2913+
//
2914+
// CAST(1000 AS TINYINT) would error, but 1000 is excluded by WHEN so
2915+
// CASE short-circuiting must yield NULL instead of failing.
2916+
let schema = Arc::new(Schema::new(vec![Field::new("d", DataType::Int32, true)]));
2917+
let batch = RecordBatch::try_new(
2918+
Arc::clone(&schema),
2919+
vec![Arc::new(Int32Array::from(vec![
2920+
Some(-1),
2921+
Some(1000),
2922+
Some(0),
2923+
]))],
2924+
)?;
2925+
2926+
let when = binary(col("d", &schema)?, Operator::Lt, lit(0i32), &schema)?;
2927+
let then = binary(
2928+
cast(lit(10i32), &schema, DataType::Int8)?,
2929+
Operator::Divide,
2930+
cast(col("d", &schema)?, &schema, DataType::Int8)?,
2931+
&schema,
2932+
)?;
2933+
2934+
let expr = CaseExpr::try_new(None, vec![(when, then)], None)?;
2935+
assert!(
2936+
!matches!(expr.eval_method, EvalMethod::DivideByZeroProtection),
2937+
"fallible CAST must not specialize, got {:?}",
2938+
expr.eval_method
2939+
);
2940+
2941+
let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;
2942+
let result = as_int8_array(&result)?;
2943+
let expected = &Int8Array::from(vec![Some(-10), None, None]);
2944+
assert_eq!(expected, result);
2945+
2946+
Ok(())
2947+
}
2948+
2949+
#[test]
2950+
fn test_divide_by_zero_protection_fallible_column_cast_does_not_error() -> Result<()>
2951+
{
2952+
// Numerator is already Int8, so construction cannot prove the
2953+
// CAST(d AS TINYINT) narrowing is fallible. Evaluation must still
2954+
// fall back to filtered CASE and not error on the excluded 1000.
2955+
let schema = Arc::new(Schema::new(vec![Field::new("d", DataType::Int32, true)]));
2956+
let batch = RecordBatch::try_new(
2957+
Arc::clone(&schema),
2958+
vec![Arc::new(Int32Array::from(vec![
2959+
Some(-1),
2960+
Some(1000),
2961+
Some(0),
2962+
]))],
2963+
)?;
2964+
2965+
let when = binary(col("d", &schema)?, Operator::Lt, lit(0i32), &schema)?;
2966+
let then = binary(
2967+
lit(10i8),
2968+
Operator::Divide,
2969+
cast(col("d", &schema)?, &schema, DataType::Int8)?,
2970+
&schema,
2971+
)?;
2972+
2973+
let expr = CaseExpr::try_new(None, vec![(when, then)], None)?;
2974+
let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;
2975+
let result = as_int8_array(&result)?;
2976+
let expected = &Int8Array::from(vec![Some(-10), None, None]);
2977+
assert_eq!(expected, result);
2978+
2979+
Ok(())
2980+
}
2981+
28122982
fn make_col(name: &str, index: usize) -> Arc<dyn PhysicalExpr> {
28132983
Arc::new(Column::new(name, index))
28142984
}

‎datafusion/sqllogictest/test_files/case.slt‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -663,6 +663,22 @@ SELECT CASE WHEN 0 < d THEN n / d ELSE NULL END FROM (VALUES (1, 1), (1, 0), (1,
663663
NULL
664664
NULL
665665

666+
# Swapped comparison form: 0 > d (same as d < 0)
667+
query I
668+
SELECT CASE WHEN 0 > d THEN n / d ELSE NULL END FROM (VALUES (1, 1), (1, 0), (1, -1)) t(n,d)
669+
----
670+
NULL
671+
NULL
672+
-1
673+
674+
# Fallible CAST on a row excluded by WHEN must not error (CASE short-circuit)
675+
query I
676+
SELECT CASE WHEN d < 0 THEN CAST(10 AS TINYINT) / CAST(d AS TINYINT) END FROM (VALUES (-1), (1000), (0)) t(d)
677+
----
678+
-10
679+
NULL
680+
NULL
681+
666682
# Float divide-by-zero protection
667683
query R
668684
SELECT CASE WHEN d != 0 THEN n / d ELSE NULL END FROM (VALUES (1.0, 2.0), (1.0, 0.0), (1.0, -2.0)) t(n,d)

0 commit comments

Comments
 (0)