Skip to content

Commit 48b72e4

Browse files
fix(physical-expr): guard CASE projection for custom leaves
Signed-off-by: Hasnaat Hussain <[email protected]>
1 parent 76bda35 commit 48b72e4

1 file changed

Lines changed: 132 additions & 25 deletions

File tree

  • datafusion/physical-expr/src/expressions

‎datafusion/physical-expr/src/expressions/case.rs‎

Lines changed: 132 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
mod literal_lookup_table;
1919

2020
use super::{Column, Literal};
21-
use crate::PhysicalExpr;
21+
use crate::{PhysicalExpr, ScalarFunctionExpr};
2222
use crate::expressions::{
2323
CastExpr, LambdaVariable, NegativeExpr, NotExpr, lit, try_cast,
2424
};
@@ -143,6 +143,7 @@ impl CaseBody {
143143
{
144144
used_column_indices.insert(lambda_variable.index());
145145
} else if expr.downcast_ref::<Literal>().is_none()
146+
&& expr.downcast_ref::<ScalarFunctionExpr>().is_none()
146147
&& expr.children().is_empty()
147148
{
148149
// Unknown leaves may read input columns without exposing a Column child.
@@ -254,13 +255,34 @@ impl CaseBody {
254255
///
255256
/// The projection vector and the rewritten expression (which only differs from the original in
256257
/// column reference indices) are held in a `ProjectedCaseBody`.
258+
///
259+
/// When `supports_projection` is false, evaluation uses the original body and full batch because
260+
/// the expression may depend on input columns that are not visible in its children.
257261
#[derive(Debug, Hash, PartialEq, Eq)]
258262
struct ProjectedCaseBody {
259263
projection: Vec<usize>,
260264
body: CaseBody,
265+
/// Whether `body` is safe to evaluate against a projected batch.
266+
///
267+
/// When false, `projection` and the rewritten `body` must not be used because an
268+
/// expression may read input columns without exposing them through its children.
261269
supports_projection: bool,
262270
}
263271

272+
impl ProjectedCaseBody {
273+
/// Returns a projection when evaluating the derived body can avoid unused input columns.
274+
fn projection_for(&self, batch: &RecordBatch) -> Option<Vec<usize>> {
275+
let projection = self
276+
.projection
277+
.iter()
278+
.copied()
279+
.filter(|index| *index < batch.num_columns())
280+
.collect::<Vec<_>>();
281+
(self.supports_projection && projection.len() < batch.num_columns())
282+
.then_some(projection)
283+
}
284+
}
285+
264286
/// The CASE expression is similar to a series of nested if/else and there are two forms that
265287
/// can be used. The first form consists of a series of boolean "when" expressions with
266288
/// corresponding "then" expressions, and an optional "else" expression.
@@ -1057,14 +1079,7 @@ impl CaseExpr {
10571079
projected: &ProjectedCaseBody,
10581080
) -> Result<ColumnarValue> {
10591081
let return_type = self.data_type(&batch.schema())?;
1060-
// projected.projection may include indexes of lambda variables not available on this batch
1061-
let projection = projected
1062-
.projection
1063-
.iter()
1064-
.copied()
1065-
.filter(|index| *index < batch.num_columns())
1066-
.collect::<Vec<_>>();
1067-
if projected.supports_projection && projection.len() < batch.num_columns() {
1082+
if let Some(projection) = projected.projection_for(batch) {
10681083
let projected_batch = batch.project(&projection)?;
10691084
projected
10701085
.body
@@ -1087,14 +1102,7 @@ impl CaseExpr {
10871102
projected: &ProjectedCaseBody,
10881103
) -> Result<ColumnarValue> {
10891104
let return_type = self.data_type(&batch.schema())?;
1090-
// projected.projection may include indexes of lambda variables not available on this batch
1091-
let projection = projected
1092-
.projection
1093-
.iter()
1094-
.copied()
1095-
.filter(|index| *index < batch.num_columns())
1096-
.collect::<Vec<_>>();
1097-
if projected.supports_projection && projection.len() < batch.num_columns() {
1105+
if let Some(projection) = projected.projection_for(batch) {
10981106
let projected_batch = batch.project(&projection)?;
10991107
projected
11001108
.body
@@ -1213,14 +1221,7 @@ impl CaseExpr {
12131221
}
12141222
}
12151223
} else {
1216-
// projected.projection may include indexes of lambda variables not available on this batch
1217-
let projection = projected
1218-
.projection
1219-
.iter()
1220-
.copied()
1221-
.filter(|index| *index < batch.num_columns())
1222-
.collect::<Vec<_>>();
1223-
if projected.supports_projection && projection.len() < batch.num_columns() {
1224+
if let Some(projection) = projected.projection_for(batch) {
12241225
// The case expressions do not use all the columns of the input batch.
12251226
// Project first to reduce time spent filtering.
12261227
let projected_batch = batch.project(&projection)?;
@@ -1584,6 +1585,7 @@ mod tests {
15841585
use arrow::datatypes::DataType::Float64;
15851586
use arrow::datatypes::Field;
15861587
use datafusion_common::cast::{as_float64_array, as_int32_array};
1588+
use datafusion_common::config::ConfigOptions;
15871589
use datafusion_common::plan_err;
15881590
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
15891591
use datafusion_expr::type_coercion::binary::type_union_coercion;
@@ -1967,6 +1969,95 @@ mod tests {
19671969
Ok(())
19681970
}
19691971

1972+
#[test]
1973+
fn case_with_expr_with_custom_column() -> Result<()> {
1974+
let batch = case_test_batch_with_extra_columns()?;
1975+
let schema = batch.schema();
1976+
1977+
let expr = CaseExpr::try_new(
1978+
Some(Arc::new(CustomColumn::new("a", 0))),
1979+
vec![(lit("foo"), col("b", &schema)?)],
1980+
Some(col("c", &schema)?),
1981+
)?;
1982+
1983+
match &expr.eval_method {
1984+
EvalMethod::WithExpression(projected) => {
1985+
assert!(!projected.supports_projection);
1986+
}
1987+
method => panic!("expected WithExpression, got {method:?}"),
1988+
}
1989+
1990+
let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;
1991+
let result = as_int32_array(&result)?;
1992+
let expected = Int32Array::from(vec![Some(1), Some(20), Some(30), Some(4)]);
1993+
1994+
assert_eq!(&expected, result);
1995+
Ok(())
1996+
}
1997+
1998+
#[test]
1999+
fn case_expr_or_expr_with_custom_column() -> Result<()> {
2000+
let batch = case_test_batch_with_extra_columns()?;
2001+
let schema = batch.schema();
2002+
let when = binary(
2003+
Arc::new(CustomColumn::new("a", 0)),
2004+
Operator::Eq,
2005+
lit("foo"),
2006+
&schema,
2007+
)?;
2008+
2009+
let expr = CaseExpr::try_new(
2010+
None,
2011+
vec![(when, col("b", &schema)?)],
2012+
Some(col("c", &schema)?),
2013+
)?;
2014+
2015+
match &expr.eval_method {
2016+
EvalMethod::ExpressionOrExpression(projected) => {
2017+
assert!(!projected.supports_projection);
2018+
}
2019+
method => panic!("expected ExpressionOrExpression, got {method:?}"),
2020+
}
2021+
2022+
let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;
2023+
let result = as_int32_array(&result)?;
2024+
let expected = Int32Array::from(vec![Some(1), Some(20), Some(30), Some(4)]);
2025+
2026+
assert_eq!(&expected, result);
2027+
Ok(())
2028+
}
2029+
2030+
#[test]
2031+
fn case_expr_or_expr_keeps_nullary_scalar_functions_projectable() -> Result<()> {
2032+
let batch = case_test_batch1()?;
2033+
let schema = batch.schema();
2034+
let random: Arc<dyn PhysicalExpr> = Arc::new(ScalarFunctionExpr::try_new(
2035+
datafusion_functions::math::random(),
2036+
vec![],
2037+
&schema,
2038+
Arc::new(ConfigOptions::default()),
2039+
)?);
2040+
let when = binary(random, Operator::Gt, lit(0.5f64), &schema)?;
2041+
2042+
let expr = CaseExpr::try_new(
2043+
None,
2044+
vec![(when, col("b", &schema)?)],
2045+
Some(col("c", &schema)?),
2046+
)?;
2047+
2048+
match &expr.eval_method {
2049+
EvalMethod::ExpressionOrExpression(projected) => {
2050+
assert!(projected.supports_projection);
2051+
assert_eq!(projected.projection, vec![1, 2]);
2052+
}
2053+
method => panic!("expected ExpressionOrExpression, got {method:?}"),
2054+
}
2055+
2056+
let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;
2057+
assert_eq!(result.len(), batch.num_rows());
2058+
Ok(())
2059+
}
2060+
19702061
#[test]
19712062
fn case_with_expr_when_null() -> Result<()> {
19722063
let batch = case_test_batch()?;
@@ -2034,6 +2125,22 @@ mod tests {
20342125
Ok(())
20352126
}
20362127

2128+
fn case_test_batch_with_extra_columns() -> Result<RecordBatch> {
2129+
let schema = Schema::new(vec![
2130+
Field::new("a", DataType::Utf8, true),
2131+
Field::new("b", DataType::Int32, true),
2132+
Field::new("c", DataType::Int32, true),
2133+
]);
2134+
let a = StringArray::from(vec![Some("foo"), Some("baz"), None, Some("bar")]);
2135+
let b = Int32Array::from(vec![Some(1), Some(2), Some(3), Some(4)]);
2136+
let c = Int32Array::from(vec![Some(10), Some(20), Some(30), Some(40)]);
2137+
RecordBatch::try_new(
2138+
Arc::new(schema),
2139+
vec![Arc::new(a), Arc::new(b), Arc::new(c)],
2140+
)
2141+
.map_err(Into::into)
2142+
}
2143+
20372144
fn case_test_batch1() -> Result<RecordBatch> {
20382145
let schema = Schema::new(vec![
20392146
Field::new("a", DataType::Int32, true),

0 commit comments

Comments
 (0)