Skip to content

Commit cbe3b39

Browse files
authored
perf: evaluate posexplode array expressions once per batch (#5737)
1 parent 31a69ee commit cbe3b39

2 files changed

Lines changed: 206 additions & 44 deletions

File tree

‎native/core/src/execution/planner.rs‎

Lines changed: 192 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -2080,54 +2080,46 @@ impl PhysicalPlanner {
20802080
// unnesting. See https://github.com/apache/datafusion/issues/19053. Once
20812081
// Comet moves to a DataFusion release carrying
20822082
// https://github.com/apache/datafusion/pull/22100, `ListEmptyToNullExpr`
2083-
// and the pre-projection below can be removed in favor of
2084-
// `NullHandling::PreserveAndExpandEmpty`. See
2083+
// can be removed in favor of `NullHandling::PreserveAndExpandEmpty`. See
20852084
// https://github.com/apache/datafusion-comet/issues/5210.
2086-
//
2087-
// For `posexplode_outer` the wrapped array is materialized in a
2088-
// pre-projection so `ListPositionsExpr` and the array passthrough share
2089-
// a single evaluation of `ListEmptyToNullExpr` instead of re-running it
2090-
// per branch. Plain `explode_outer` references the wrapped array exactly
2091-
// once, so no pre-projection is needed there.
2092-
let (child_expr, child_native_plan): (Arc<dyn PhysicalExpr>, _) =
2093-
match (explode.outer, explode.position) {
2094-
(true, true) => {
2095-
let wrapped: Arc<dyn PhysicalExpr> =
2096-
Arc::new(ListEmptyToNullExpr::new(raw_child_expr));
2097-
let reserved_name =
2098-
format!("__comet_explode_outer_{}", child_field_name);
2099-
2100-
let mut pre_exprs: Vec<(Arc<dyn PhysicalExpr>, String)> = child_schema
2101-
.fields()
2102-
.iter()
2103-
.enumerate()
2104-
.map(|(i, f)| {
2105-
(
2106-
Arc::new(Column::new(f.name(), i)) as Arc<dyn PhysicalExpr>,
2107-
f.name().to_string(),
2108-
)
2109-
})
2110-
.collect();
2111-
let wrapped_idx = pre_exprs.len();
2112-
pre_exprs.push((wrapped, reserved_name.clone()));
2085+
let child_expr: Arc<dyn PhysicalExpr> = if explode.outer {
2086+
Arc::new(ListEmptyToNullExpr::new(raw_child_expr))
2087+
} else {
2088+
raw_child_expr
2089+
};
21132090

2114-
let pre_exec = Arc::new(ProjectionExec::try_new(
2115-
pre_exprs,
2116-
Arc::clone(&child.native_plan),
2117-
)?);
2091+
// Both posexplode variants reference the array twice: once for positions
2092+
// and once for values. Materialize computed arrays so both references
2093+
// share one evaluation. A plain Column is already materialized.
2094+
let (child_expr, child_native_plan) = if explode.position
2095+
&& !child_expr.is::<Column>()
2096+
{
2097+
let reserved_name = format!("__comet_explode_{}", child_field_name);
2098+
let mut pre_exprs: Vec<(Arc<dyn PhysicalExpr>, String)> = child_schema
2099+
.fields()
2100+
.iter()
2101+
.enumerate()
2102+
.map(|(i, f)| {
21182103
(
2119-
Arc::new(Column::new(&reserved_name, wrapped_idx))
2120-
as Arc<dyn PhysicalExpr>,
2121-
pre_exec as Arc<dyn ExecutionPlan>,
2104+
Arc::new(Column::new(f.name(), i)) as Arc<dyn PhysicalExpr>,
2105+
f.name().to_string(),
21222106
)
2123-
}
2124-
(true, false) => (
2125-
Arc::new(ListEmptyToNullExpr::new(raw_child_expr))
2126-
as Arc<dyn PhysicalExpr>,
2127-
Arc::clone(&child.native_plan),
2128-
),
2129-
(false, _) => (raw_child_expr, Arc::clone(&child.native_plan)),
2130-
};
2107+
})
2108+
.collect();
2109+
let child_idx = pre_exprs.len();
2110+
pre_exprs.push((child_expr, reserved_name.clone()));
2111+
2112+
let pre_exec = Arc::new(ProjectionExec::try_new(
2113+
pre_exprs,
2114+
Arc::clone(&child.native_plan),
2115+
)?);
2116+
(
2117+
Arc::new(Column::new(&reserved_name, child_idx)) as Arc<dyn PhysicalExpr>,
2118+
pre_exec as Arc<dyn ExecutionPlan>,
2119+
)
2120+
} else {
2121+
(child_expr, Arc::clone(&child.native_plan))
2122+
};
21312123

21322124
// Create projection expressions for other columns
21332125
let projections: Vec<Arc<dyn PhysicalExpr>> = explode
@@ -6147,6 +6139,162 @@ mod tests {
61476139
}
61486140
}
61496141

6142+
#[tokio::test]
6143+
async fn explode_evaluates_array_once_per_batch() {
6144+
use arrow::datatypes::Int32Type;
6145+
use datafusion::common::tree_node::{Transformed, TreeNode};
6146+
use datafusion::logical_expr::{create_udf, Volatility};
6147+
use datafusion::physical_plan::projection::ProjectionExec;
6148+
use spark_expression::data_type::{data_type_info::DatatypeStruct, DataTypeInfo, ListInfo};
6149+
6150+
let array_type = spark_expression::DataType {
6151+
type_id: 14,
6152+
type_info: Some(Box::new(DataTypeInfo {
6153+
datatype_struct: Some(DatatypeStruct::List(Box::new(ListInfo {
6154+
element_type: Some(Box::new(create_proto_datatype())),
6155+
contains_null: true,
6156+
element_field_id: None,
6157+
}))),
6158+
})),
6159+
};
6160+
let arrays = Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
6161+
Some(vec![Some(10), None]),
6162+
Some(vec![]),
6163+
None,
6164+
Some(vec![Some(20)]),
6165+
])) as ArrayRef;
6166+
6167+
for outer in [false, true] {
6168+
for position in [false, true] {
6169+
for computed in [false, true] {
6170+
let calls = Arc::new(AtomicUsize::new(0));
6171+
let counter = Arc::clone(&calls);
6172+
let ctx = SessionContext::new();
6173+
ctx.register_udf(create_udf(
6174+
"counted_array",
6175+
vec![arrays.data_type().clone()],
6176+
arrays.data_type().clone(),
6177+
Volatility::Immutable,
6178+
Arc::new(move |args| {
6179+
counter.fetch_add(1, Ordering::Relaxed);
6180+
Ok(args[0].clone())
6181+
}),
6182+
));
6183+
let task_ctx = ctx.task_ctx();
6184+
let planner = PhysicalPlanner::new(Arc::new(ctx), 0);
6185+
let bound = Expr {
6186+
expr_struct: Some(Bound(spark_expression::BoundReference {
6187+
index: 0,
6188+
datatype: Some(array_type.clone()),
6189+
})),
6190+
..Default::default()
6191+
};
6192+
let child_expr = if computed {
6193+
Expr {
6194+
expr_struct: Some(ScalarFunc(spark_expression::ScalarFunc {
6195+
func: "counted_array".to_string(),
6196+
args: vec![bound.clone()],
6197+
..Default::default()
6198+
})),
6199+
..Default::default()
6200+
}
6201+
} else {
6202+
bound.clone()
6203+
};
6204+
let op = Operator {
6205+
children: vec![Operator {
6206+
op_struct: Some(OpStruct::Scan(spark_operator::Scan {
6207+
fields: vec![array_type.clone()],
6208+
source: String::new(),
6209+
})),
6210+
..Default::default()
6211+
}],
6212+
op_struct: Some(OpStruct::Explode(spark_operator::Explode {
6213+
child: Some(child_expr),
6214+
outer,
6215+
position,
6216+
project_list: vec![bound],
6217+
})),
6218+
..Default::default()
6219+
};
6220+
let (_, _, plan) = planner.create_plan(&op, &mut vec![], 1).unwrap();
6221+
let schema = plan.children[0].schema();
6222+
let batch = RecordBatch::try_new(schema, vec![Arc::clone(&arrays)]).unwrap();
6223+
let input: Arc<dyn ExecutionPlan> = MemorySourceConfig::try_new_exec(
6224+
&[vec![batch.clone(), batch.clone()]],
6225+
batch.schema(),
6226+
None,
6227+
)
6228+
.unwrap();
6229+
let mut projections = 0;
6230+
// Replace only the JNI-fed scan, keeping the planner's generated operators.
6231+
let native_plan = Arc::clone(&plan.native_plan)
6232+
.transform_up(|node| {
6233+
projections += usize::from(node.is::<ProjectionExec>());
6234+
Ok(if node.name() == "ScanExec" {
6235+
Transformed::yes(Arc::clone(&input))
6236+
} else {
6237+
Transformed::no(node)
6238+
})
6239+
})
6240+
.unwrap()
6241+
.data;
6242+
let results = collect(native_plan.execute(0, task_ctx).unwrap())
6243+
.await
6244+
.unwrap();
6245+
let context =
6246+
format!("outer={outer}, position={position}, computed={computed}");
6247+
assert_eq!(
6248+
calls.load(Ordering::Relaxed),
6249+
if computed { 2 } else { 0 },
6250+
"{context}"
6251+
);
6252+
assert_eq!(
6253+
projections,
6254+
1 + usize::from(position && (outer || computed)),
6255+
"{context}"
6256+
);
6257+
let expected_values = if outer {
6258+
vec![Some(10), None, None, None, Some(20)]
6259+
} else {
6260+
vec![Some(10), None, Some(20)]
6261+
};
6262+
let values: Vec<_> = results
6263+
.iter()
6264+
.flat_map(|batch| {
6265+
batch
6266+
.column(batch.num_columns() - 1)
6267+
.as_any()
6268+
.downcast_ref::<Int32Array>()
6269+
.unwrap()
6270+
.iter()
6271+
})
6272+
.collect();
6273+
assert_eq!(values, expected_values.repeat(2), "{context}");
6274+
if position {
6275+
let expected_positions = if outer {
6276+
vec![Some(0), Some(1), None, None, Some(0)]
6277+
} else {
6278+
vec![Some(0), Some(1), Some(0)]
6279+
};
6280+
let positions: Vec<_> = results
6281+
.iter()
6282+
.flat_map(|batch| {
6283+
batch
6284+
.column(1)
6285+
.as_any()
6286+
.downcast_ref::<Int32Array>()
6287+
.unwrap()
6288+
.iter()
6289+
})
6290+
.collect();
6291+
assert_eq!(positions, expected_positions.repeat(2), "{context}");
6292+
}
6293+
}
6294+
}
6295+
}
6296+
}
6297+
61506298
#[test]
61516299
fn test_create_array() {
61526300
let session_ctx = SessionContext::new();

‎spark/src/test/scala/org/apache/comet/exec/CometGenerateExecSuite.scala‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,20 @@ class CometGenerateExecSuite extends CometTestBase {
2828

2929
import testImplicits._
3030

31+
test("posexplode with a computed array from Parquet") {
32+
withSQLConf(CometConf.COMET_EXEC_EXPLODE_ENABLED.key -> "true") {
33+
val input = Seq((1, "axb"), (2, ""), (3, null), (4, "xxc"))
34+
withParquetDataFrame(input) { parquet =>
35+
withParquetTable(parquet.toDF("id", "s"), "t") {
36+
for (generator <- Seq("posexplode", "posexplode_outer")) {
37+
val df = sql(s"SELECT id, $generator(split(s, 'x')) AS (pos, value) FROM t")
38+
checkSparkAnswerAndOperator(df)
39+
}
40+
}
41+
}
42+
}
43+
}
44+
3145
test("explode with simple array") {
3246
withSQLConf(
3347
CometConf.COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED.key -> "true",

0 commit comments

Comments
 (0)