1818mod literal_lookup_table;
1919
2020use super :: { Column , Literal } ;
21- use crate :: PhysicalExpr ;
21+ use crate :: { PhysicalExpr , ScalarFunctionExpr } ;
2222use crate :: expressions:: {
2323 CastExpr , LambdaVariable , NegativeExpr , NotExpr , lit, try_cast,
2424} ;
@@ -143,6 +143,7 @@ impl CaseBody {
143143 {
144144 used_column_indices. insert ( lambda_variable. index ( ) ) ;
145145 } else if expr. downcast_ref :: < Literal > ( ) . is_none ( )
146+ && expr. downcast_ref :: < ScalarFunctionExpr > ( ) . is_none ( )
146147 && expr. children ( ) . is_empty ( )
147148 {
148149 // Unknown leaves may read input columns without exposing a Column child.
@@ -254,13 +255,34 @@ impl CaseBody {
254255///
255256/// The projection vector and the rewritten expression (which only differs from the original in
256257/// column reference indices) are held in a `ProjectedCaseBody`.
258+ ///
259+ /// When `supports_projection` is false, evaluation uses the original body and full batch because
260+ /// the expression may depend on input columns that are not visible in its children.
257261#[ derive( Debug , Hash , PartialEq , Eq ) ]
258262struct ProjectedCaseBody {
259263 projection : Vec < usize > ,
260264 body : CaseBody ,
265+ /// Whether `body` is safe to evaluate against a projected batch.
266+ ///
267+ /// When false, `projection` and the rewritten `body` must not be used because an
268+ /// expression may read input columns without exposing them through its children.
261269 supports_projection : bool ,
262270}
263271
272+ impl ProjectedCaseBody {
273+ /// Returns a projection when evaluating the derived body can avoid unused input columns.
274+ fn projection_for ( & self , batch : & RecordBatch ) -> Option < Vec < usize > > {
275+ let projection = self
276+ . projection
277+ . iter ( )
278+ . copied ( )
279+ . filter ( |index| * index < batch. num_columns ( ) )
280+ . collect :: < Vec < _ > > ( ) ;
281+ ( self . supports_projection && projection. len ( ) < batch. num_columns ( ) )
282+ . then_some ( projection)
283+ }
284+ }
285+
264286/// The CASE expression is similar to a series of nested if/else and there are two forms that
265287/// can be used. The first form consists of a series of boolean "when" expressions with
266288/// corresponding "then" expressions, and an optional "else" expression.
@@ -1057,14 +1079,7 @@ impl CaseExpr {
10571079 projected : & ProjectedCaseBody ,
10581080 ) -> Result < ColumnarValue > {
10591081 let return_type = self . data_type ( & batch. schema ( ) ) ?;
1060- // projected.projection may include indexes of lambda variables not available on this batch
1061- let projection = projected
1062- . projection
1063- . iter ( )
1064- . copied ( )
1065- . filter ( |index| * index < batch. num_columns ( ) )
1066- . collect :: < Vec < _ > > ( ) ;
1067- if projected. supports_projection && projection. len ( ) < batch. num_columns ( ) {
1082+ if let Some ( projection) = projected. projection_for ( batch) {
10681083 let projected_batch = batch. project ( & projection) ?;
10691084 projected
10701085 . body
@@ -1087,14 +1102,7 @@ impl CaseExpr {
10871102 projected : & ProjectedCaseBody ,
10881103 ) -> Result < ColumnarValue > {
10891104 let return_type = self . data_type ( & batch. schema ( ) ) ?;
1090- // projected.projection may include indexes of lambda variables not available on this batch
1091- let projection = projected
1092- . projection
1093- . iter ( )
1094- . copied ( )
1095- . filter ( |index| * index < batch. num_columns ( ) )
1096- . collect :: < Vec < _ > > ( ) ;
1097- if projected. supports_projection && projection. len ( ) < batch. num_columns ( ) {
1105+ if let Some ( projection) = projected. projection_for ( batch) {
10981106 let projected_batch = batch. project ( & projection) ?;
10991107 projected
11001108 . body
@@ -1213,14 +1221,7 @@ impl CaseExpr {
12131221 }
12141222 }
12151223 } else {
1216- // projected.projection may include indexes of lambda variables not available on this batch
1217- let projection = projected
1218- . projection
1219- . iter ( )
1220- . copied ( )
1221- . filter ( |index| * index < batch. num_columns ( ) )
1222- . collect :: < Vec < _ > > ( ) ;
1223- if projected. supports_projection && projection. len ( ) < batch. num_columns ( ) {
1224+ if let Some ( projection) = projected. projection_for ( batch) {
12241225 // The case expressions do not use all the columns of the input batch.
12251226 // Project first to reduce time spent filtering.
12261227 let projected_batch = batch. project ( & projection) ?;
@@ -1584,6 +1585,7 @@ mod tests {
15841585 use arrow:: datatypes:: DataType :: Float64 ;
15851586 use arrow:: datatypes:: Field ;
15861587 use datafusion_common:: cast:: { as_float64_array, as_int32_array} ;
1588+ use datafusion_common:: config:: ConfigOptions ;
15871589 use datafusion_common:: plan_err;
15881590 use datafusion_common:: tree_node:: { Transformed , TransformedResult , TreeNode } ;
15891591 use datafusion_expr:: type_coercion:: binary:: type_union_coercion;
@@ -1967,6 +1969,95 @@ mod tests {
19671969 Ok ( ( ) )
19681970 }
19691971
1972+ #[ test]
1973+ fn case_with_expr_with_custom_column ( ) -> Result < ( ) > {
1974+ let batch = case_test_batch_with_extra_columns ( ) ?;
1975+ let schema = batch. schema ( ) ;
1976+
1977+ let expr = CaseExpr :: try_new (
1978+ Some ( Arc :: new ( CustomColumn :: new ( "a" , 0 ) ) ) ,
1979+ vec ! [ ( lit( "foo" ) , col( "b" , & schema) ?) ] ,
1980+ Some ( col ( "c" , & schema) ?) ,
1981+ ) ?;
1982+
1983+ match & expr. eval_method {
1984+ EvalMethod :: WithExpression ( projected) => {
1985+ assert ! ( !projected. supports_projection) ;
1986+ }
1987+ method => panic ! ( "expected WithExpression, got {method:?}" ) ,
1988+ }
1989+
1990+ let result = expr. evaluate ( & batch) ?. into_array ( batch. num_rows ( ) ) ?;
1991+ let result = as_int32_array ( & result) ?;
1992+ let expected = Int32Array :: from ( vec ! [ Some ( 1 ) , Some ( 20 ) , Some ( 30 ) , Some ( 4 ) ] ) ;
1993+
1994+ assert_eq ! ( & expected, result) ;
1995+ Ok ( ( ) )
1996+ }
1997+
1998+ #[ test]
1999+ fn case_expr_or_expr_with_custom_column ( ) -> Result < ( ) > {
2000+ let batch = case_test_batch_with_extra_columns ( ) ?;
2001+ let schema = batch. schema ( ) ;
2002+ let when = binary (
2003+ Arc :: new ( CustomColumn :: new ( "a" , 0 ) ) ,
2004+ Operator :: Eq ,
2005+ lit ( "foo" ) ,
2006+ & schema,
2007+ ) ?;
2008+
2009+ let expr = CaseExpr :: try_new (
2010+ None ,
2011+ vec ! [ ( when, col( "b" , & schema) ?) ] ,
2012+ Some ( col ( "c" , & schema) ?) ,
2013+ ) ?;
2014+
2015+ match & expr. eval_method {
2016+ EvalMethod :: ExpressionOrExpression ( projected) => {
2017+ assert ! ( !projected. supports_projection) ;
2018+ }
2019+ method => panic ! ( "expected ExpressionOrExpression, got {method:?}" ) ,
2020+ }
2021+
2022+ let result = expr. evaluate ( & batch) ?. into_array ( batch. num_rows ( ) ) ?;
2023+ let result = as_int32_array ( & result) ?;
2024+ let expected = Int32Array :: from ( vec ! [ Some ( 1 ) , Some ( 20 ) , Some ( 30 ) , Some ( 4 ) ] ) ;
2025+
2026+ assert_eq ! ( & expected, result) ;
2027+ Ok ( ( ) )
2028+ }
2029+
2030+ #[ test]
2031+ fn case_expr_or_expr_keeps_nullary_scalar_functions_projectable ( ) -> Result < ( ) > {
2032+ let batch = case_test_batch1 ( ) ?;
2033+ let schema = batch. schema ( ) ;
2034+ let random: Arc < dyn PhysicalExpr > = Arc :: new ( ScalarFunctionExpr :: try_new (
2035+ datafusion_functions:: math:: random ( ) ,
2036+ vec ! [ ] ,
2037+ & schema,
2038+ Arc :: new ( ConfigOptions :: default ( ) ) ,
2039+ ) ?) ;
2040+ let when = binary ( random, Operator :: Gt , lit ( 0.5f64 ) , & schema) ?;
2041+
2042+ let expr = CaseExpr :: try_new (
2043+ None ,
2044+ vec ! [ ( when, col( "b" , & schema) ?) ] ,
2045+ Some ( col ( "c" , & schema) ?) ,
2046+ ) ?;
2047+
2048+ match & expr. eval_method {
2049+ EvalMethod :: ExpressionOrExpression ( projected) => {
2050+ assert ! ( projected. supports_projection) ;
2051+ assert_eq ! ( projected. projection, vec![ 1 , 2 ] ) ;
2052+ }
2053+ method => panic ! ( "expected ExpressionOrExpression, got {method:?}" ) ,
2054+ }
2055+
2056+ let result = expr. evaluate ( & batch) ?. into_array ( batch. num_rows ( ) ) ?;
2057+ assert_eq ! ( result. len( ) , batch. num_rows( ) ) ;
2058+ Ok ( ( ) )
2059+ }
2060+
19702061 #[ test]
19712062 fn case_with_expr_when_null ( ) -> Result < ( ) > {
19722063 let batch = case_test_batch ( ) ?;
@@ -2034,6 +2125,22 @@ mod tests {
20342125 Ok ( ( ) )
20352126 }
20362127
2128+ fn case_test_batch_with_extra_columns ( ) -> Result < RecordBatch > {
2129+ let schema = Schema :: new ( vec ! [
2130+ Field :: new( "a" , DataType :: Utf8 , true ) ,
2131+ Field :: new( "b" , DataType :: Int32 , true ) ,
2132+ Field :: new( "c" , DataType :: Int32 , true ) ,
2133+ ] ) ;
2134+ let a = StringArray :: from ( vec ! [ Some ( "foo" ) , Some ( "baz" ) , None , Some ( "bar" ) ] ) ;
2135+ let b = Int32Array :: from ( vec ! [ Some ( 1 ) , Some ( 2 ) , Some ( 3 ) , Some ( 4 ) ] ) ;
2136+ let c = Int32Array :: from ( vec ! [ Some ( 10 ) , Some ( 20 ) , Some ( 30 ) , Some ( 40 ) ] ) ;
2137+ RecordBatch :: try_new (
2138+ Arc :: new ( schema) ,
2139+ vec ! [ Arc :: new( a) , Arc :: new( b) , Arc :: new( c) ] ,
2140+ )
2141+ . map_err ( Into :: into)
2142+ }
2143+
20372144 fn case_test_batch1 ( ) -> Result < RecordBatch > {
20382145 let schema = Schema :: new ( vec ! [
20392146 Field :: new( "a" , DataType :: Int32 , true ) ,
0 commit comments