@@ -2080,54 +2080,46 @@ impl PhysicalPlanner {
20802080 // unnesting. See https://github.com/apache/datafusion/issues/19053. Once
20812081 // Comet moves to a DataFusion release carrying
20822082 // https://github.com/apache/datafusion/pull/22100, `ListEmptyToNullExpr`
2083- // and the pre-projection below can be removed in favor of
2084- // `NullHandling::PreserveAndExpandEmpty`. See
2083+ // can be removed in favor of `NullHandling::PreserveAndExpandEmpty`. See
20852084 // https://github.com/apache/datafusion-comet/issues/5210.
2086- //
2087- // For `posexplode_outer` the wrapped array is materialized in a
2088- // pre-projection so `ListPositionsExpr` and the array passthrough share
2089- // a single evaluation of `ListEmptyToNullExpr` instead of re-running it
2090- // per branch. Plain `explode_outer` references the wrapped array exactly
2091- // once, so no pre-projection is needed there.
2092- let ( child_expr, child_native_plan) : ( Arc < dyn PhysicalExpr > , _ ) =
2093- match ( explode. outer , explode. position ) {
2094- ( true , true ) => {
2095- let wrapped: Arc < dyn PhysicalExpr > =
2096- Arc :: new ( ListEmptyToNullExpr :: new ( raw_child_expr) ) ;
2097- let reserved_name =
2098- format ! ( "__comet_explode_outer_{}" , child_field_name) ;
2099-
2100- let mut pre_exprs: Vec < ( Arc < dyn PhysicalExpr > , String ) > = child_schema
2101- . fields ( )
2102- . iter ( )
2103- . enumerate ( )
2104- . map ( |( i, f) | {
2105- (
2106- Arc :: new ( Column :: new ( f. name ( ) , i) ) as Arc < dyn PhysicalExpr > ,
2107- f. name ( ) . to_string ( ) ,
2108- )
2109- } )
2110- . collect ( ) ;
2111- let wrapped_idx = pre_exprs. len ( ) ;
2112- pre_exprs. push ( ( wrapped, reserved_name. clone ( ) ) ) ;
2085+ let child_expr: Arc < dyn PhysicalExpr > = if explode. outer {
2086+ Arc :: new ( ListEmptyToNullExpr :: new ( raw_child_expr) )
2087+ } else {
2088+ raw_child_expr
2089+ } ;
21132090
2114- let pre_exec = Arc :: new ( ProjectionExec :: try_new (
2115- pre_exprs,
2116- Arc :: clone ( & child. native_plan ) ,
2117- ) ?) ;
2091+ // Both posexplode variants reference the array twice: once for positions
2092+ // and once for values. Materialize computed arrays so both references
2093+ // share one evaluation. A plain Column is already materialized.
2094+ let ( child_expr, child_native_plan) = if explode. position
2095+ && !child_expr. is :: < Column > ( )
2096+ {
2097+ let reserved_name = format ! ( "__comet_explode_{}" , child_field_name) ;
2098+ let mut pre_exprs: Vec < ( Arc < dyn PhysicalExpr > , String ) > = child_schema
2099+ . fields ( )
2100+ . iter ( )
2101+ . enumerate ( )
2102+ . map ( |( i, f) | {
21182103 (
2119- Arc :: new ( Column :: new ( & reserved_name, wrapped_idx) )
2120- as Arc < dyn PhysicalExpr > ,
2121- pre_exec as Arc < dyn ExecutionPlan > ,
2104+ Arc :: new ( Column :: new ( f. name ( ) , i) ) as Arc < dyn PhysicalExpr > ,
2105+ f. name ( ) . to_string ( ) ,
21222106 )
2123- }
2124- ( true , false ) => (
2125- Arc :: new ( ListEmptyToNullExpr :: new ( raw_child_expr) )
2126- as Arc < dyn PhysicalExpr > ,
2127- Arc :: clone ( & child. native_plan ) ,
2128- ) ,
2129- ( false , _) => ( raw_child_expr, Arc :: clone ( & child. native_plan ) ) ,
2130- } ;
2107+ } )
2108+ . collect ( ) ;
2109+ let child_idx = pre_exprs. len ( ) ;
2110+ pre_exprs. push ( ( child_expr, reserved_name. clone ( ) ) ) ;
2111+
2112+ let pre_exec = Arc :: new ( ProjectionExec :: try_new (
2113+ pre_exprs,
2114+ Arc :: clone ( & child. native_plan ) ,
2115+ ) ?) ;
2116+ (
2117+ Arc :: new ( Column :: new ( & reserved_name, child_idx) ) as Arc < dyn PhysicalExpr > ,
2118+ pre_exec as Arc < dyn ExecutionPlan > ,
2119+ )
2120+ } else {
2121+ ( child_expr, Arc :: clone ( & child. native_plan ) )
2122+ } ;
21312123
21322124 // Create projection expressions for other columns
21332125 let projections: Vec < Arc < dyn PhysicalExpr > > = explode
@@ -6147,6 +6139,162 @@ mod tests {
61476139 }
61486140 }
61496141
6142+ #[ tokio:: test]
6143+ async fn explode_evaluates_array_once_per_batch ( ) {
6144+ use arrow:: datatypes:: Int32Type ;
6145+ use datafusion:: common:: tree_node:: { Transformed , TreeNode } ;
6146+ use datafusion:: logical_expr:: { create_udf, Volatility } ;
6147+ use datafusion:: physical_plan:: projection:: ProjectionExec ;
6148+ use spark_expression:: data_type:: { data_type_info:: DatatypeStruct , DataTypeInfo , ListInfo } ;
6149+
6150+ let array_type = spark_expression:: DataType {
6151+ type_id : 14 ,
6152+ type_info : Some ( Box :: new ( DataTypeInfo {
6153+ datatype_struct : Some ( DatatypeStruct :: List ( Box :: new ( ListInfo {
6154+ element_type : Some ( Box :: new ( create_proto_datatype ( ) ) ) ,
6155+ contains_null : true ,
6156+ element_field_id : None ,
6157+ } ) ) ) ,
6158+ } ) ) ,
6159+ } ;
6160+ let arrays = Arc :: new ( ListArray :: from_iter_primitive :: < Int32Type , _ , _ > ( vec ! [
6161+ Some ( vec![ Some ( 10 ) , None ] ) ,
6162+ Some ( vec![ ] ) ,
6163+ None ,
6164+ Some ( vec![ Some ( 20 ) ] ) ,
6165+ ] ) ) as ArrayRef ;
6166+
6167+ for outer in [ false , true ] {
6168+ for position in [ false , true ] {
6169+ for computed in [ false , true ] {
6170+ let calls = Arc :: new ( AtomicUsize :: new ( 0 ) ) ;
6171+ let counter = Arc :: clone ( & calls) ;
6172+ let ctx = SessionContext :: new ( ) ;
6173+ ctx. register_udf ( create_udf (
6174+ "counted_array" ,
6175+ vec ! [ arrays. data_type( ) . clone( ) ] ,
6176+ arrays. data_type ( ) . clone ( ) ,
6177+ Volatility :: Immutable ,
6178+ Arc :: new ( move |args| {
6179+ counter. fetch_add ( 1 , Ordering :: Relaxed ) ;
6180+ Ok ( args[ 0 ] . clone ( ) )
6181+ } ) ,
6182+ ) ) ;
6183+ let task_ctx = ctx. task_ctx ( ) ;
6184+ let planner = PhysicalPlanner :: new ( Arc :: new ( ctx) , 0 ) ;
6185+ let bound = Expr {
6186+ expr_struct : Some ( Bound ( spark_expression:: BoundReference {
6187+ index : 0 ,
6188+ datatype : Some ( array_type. clone ( ) ) ,
6189+ } ) ) ,
6190+ ..Default :: default ( )
6191+ } ;
6192+ let child_expr = if computed {
6193+ Expr {
6194+ expr_struct : Some ( ScalarFunc ( spark_expression:: ScalarFunc {
6195+ func : "counted_array" . to_string ( ) ,
6196+ args : vec ! [ bound. clone( ) ] ,
6197+ ..Default :: default ( )
6198+ } ) ) ,
6199+ ..Default :: default ( )
6200+ }
6201+ } else {
6202+ bound. clone ( )
6203+ } ;
6204+ let op = Operator {
6205+ children : vec ! [ Operator {
6206+ op_struct: Some ( OpStruct :: Scan ( spark_operator:: Scan {
6207+ fields: vec![ array_type. clone( ) ] ,
6208+ source: String :: new( ) ,
6209+ } ) ) ,
6210+ ..Default :: default ( )
6211+ } ] ,
6212+ op_struct : Some ( OpStruct :: Explode ( spark_operator:: Explode {
6213+ child : Some ( child_expr) ,
6214+ outer,
6215+ position,
6216+ project_list : vec ! [ bound] ,
6217+ } ) ) ,
6218+ ..Default :: default ( )
6219+ } ;
6220+ let ( _, _, plan) = planner. create_plan ( & op, & mut vec ! [ ] , 1 ) . unwrap ( ) ;
6221+ let schema = plan. children [ 0 ] . schema ( ) ;
6222+ let batch = RecordBatch :: try_new ( schema, vec ! [ Arc :: clone( & arrays) ] ) . unwrap ( ) ;
6223+ let input: Arc < dyn ExecutionPlan > = MemorySourceConfig :: try_new_exec (
6224+ & [ vec ! [ batch. clone( ) , batch. clone( ) ] ] ,
6225+ batch. schema ( ) ,
6226+ None ,
6227+ )
6228+ . unwrap ( ) ;
6229+ let mut projections = 0 ;
6230+ // Replace only the JNI-fed scan, keeping the planner's generated operators.
6231+ let native_plan = Arc :: clone ( & plan. native_plan )
6232+ . transform_up ( |node| {
6233+ projections += usize:: from ( node. is :: < ProjectionExec > ( ) ) ;
6234+ Ok ( if node. name ( ) == "ScanExec" {
6235+ Transformed :: yes ( Arc :: clone ( & input) )
6236+ } else {
6237+ Transformed :: no ( node)
6238+ } )
6239+ } )
6240+ . unwrap ( )
6241+ . data ;
6242+ let results = collect ( native_plan. execute ( 0 , task_ctx) . unwrap ( ) )
6243+ . await
6244+ . unwrap ( ) ;
6245+ let context =
6246+ format ! ( "outer={outer}, position={position}, computed={computed}" ) ;
6247+ assert_eq ! (
6248+ calls. load( Ordering :: Relaxed ) ,
6249+ if computed { 2 } else { 0 } ,
6250+ "{context}"
6251+ ) ;
6252+ assert_eq ! (
6253+ projections,
6254+ 1 + usize :: from( position && ( outer || computed) ) ,
6255+ "{context}"
6256+ ) ;
6257+ let expected_values = if outer {
6258+ vec ! [ Some ( 10 ) , None , None , None , Some ( 20 ) ]
6259+ } else {
6260+ vec ! [ Some ( 10 ) , None , Some ( 20 ) ]
6261+ } ;
6262+ let values: Vec < _ > = results
6263+ . iter ( )
6264+ . flat_map ( |batch| {
6265+ batch
6266+ . column ( batch. num_columns ( ) - 1 )
6267+ . as_any ( )
6268+ . downcast_ref :: < Int32Array > ( )
6269+ . unwrap ( )
6270+ . iter ( )
6271+ } )
6272+ . collect ( ) ;
6273+ assert_eq ! ( values, expected_values. repeat( 2 ) , "{context}" ) ;
6274+ if position {
6275+ let expected_positions = if outer {
6276+ vec ! [ Some ( 0 ) , Some ( 1 ) , None , None , Some ( 0 ) ]
6277+ } else {
6278+ vec ! [ Some ( 0 ) , Some ( 1 ) , Some ( 0 ) ]
6279+ } ;
6280+ let positions: Vec < _ > = results
6281+ . iter ( )
6282+ . flat_map ( |batch| {
6283+ batch
6284+ . column ( 1 )
6285+ . as_any ( )
6286+ . downcast_ref :: < Int32Array > ( )
6287+ . unwrap ( )
6288+ . iter ( )
6289+ } )
6290+ . collect ( ) ;
6291+ assert_eq ! ( positions, expected_positions. repeat( 2 ) , "{context}" ) ;
6292+ }
6293+ }
6294+ }
6295+ }
6296+ }
6297+
61506298 #[ test]
61516299 fn test_create_array ( ) {
61526300 let session_ctx = SessionContext :: new ( ) ;
0 commit comments