Skip to content

Commit 33f4f88

Browse files
authored
feat: run global aggregates with no grouping keys or functions natively (apache#6899)
* feat: run global aggregates with no grouping keys or functions natively Spark plans `Aggregate(Nil, Nil, child)` when nothing reads a global aggregate's output, as in `df.agg(sum($"a")).count()`. Comet fell back to Spark for it with "No group by or aggregation". DataFusion's AggregateStream can't build a batch with no columns (apache/datafusion#25782), so the native planner plans the shape as a count(1) aggregate under an empty projection. That needs no DataFusion change. Closes apache#6001. * test: cover a Spark partial aggregate under a native final one Also write the fixture's multi-file table with an explicit partition count, so it doesn't depend on the harness's parallelism. * test: say why the fixture's five files become five partitions * refactor: build the count with the planner's aggregate helper Also drop test assertions that another assertion already implies, and shorten the serde comment that repeated the native planner's. * docs: say newer DataFusion rejects an aggregate with no keys or functions apache/datafusion#25786 makes AggregateExec::try_new reject that shape, so the count plan is how Comet runs it from now on, not a stopgap. * perf: pass a NULL literal to the throwaway count The aggregate expands its argument to each input batch's length. Int64(1) filled an Int64Array per batch only for the projection to drop the count, while NULL comes back as a cached NullArray with no buffers. Also shorten the planner comment. * test: cover HAVING without GROUP BY over an empty global aggregate The parser plans it as a global aggregate that outputs a literal. Also note that CollapseProject already gives the SELECT 1 queries a result projection.
1 parent 8a4029c commit 33f4f88

5 files changed

Lines changed: 217 additions & 8 deletions

File tree

‎native/core/src/execution/planner.rs‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1750,6 +1750,45 @@ impl PhysicalPlanner {
17501750
let (scans, shuffle_scans, child) =
17511751
self.create_plan(&children[0], inputs, partition_count)?;
17521752

1753+
// For `df.agg(sum($"a")).count()`, Spark plans a global aggregate with neither
1754+
// keys nor functions that emits one row with no columns per partition. DataFusion
1755+
// can't run that (apache/datafusion#25786), so plan a throwaway `count(NULL)` and
1756+
// project it away. The count still drains the input, so errors below it surface.
1757+
// Both Spark stages take this path, since no count passes between them.
1758+
if agg.grouping_exprs.is_empty() && agg.agg_exprs.is_empty() {
1759+
let schema = child.schema();
1760+
let count = Self::create_aggr_func_expr(
1761+
"count",
1762+
Arc::clone(&schema),
1763+
vec![Arc::new(Literal::new(ScalarValue::Null))],
1764+
Arc::unwrap_or_clone(count_udaf()),
1765+
)?;
1766+
let aggregate: Arc<dyn ExecutionPlan> = Arc::new(
1767+
datafusion::physical_plan::aggregates::AggregateExec::try_new(
1768+
DFAggregateMode::Single,
1769+
PhysicalGroupBy::default(),
1770+
vec![Arc::new(count)],
1771+
vec![None],
1772+
Arc::clone(&child.native_plan),
1773+
schema,
1774+
)?,
1775+
);
1776+
let projection: Arc<dyn ExecutionPlan> = Arc::new(ProjectionExec::try_new(
1777+
Vec::<(Arc<dyn PhysicalExpr>, String)>::new(),
1778+
Arc::clone(&aggregate),
1779+
)?);
1780+
return Ok((
1781+
scans,
1782+
shuffle_scans,
1783+
Arc::new(SparkPlan::new_with_additional(
1784+
spark_plan.plan_id,
1785+
projection,
1786+
vec![child],
1787+
vec![aggregate],
1788+
)),
1789+
));
1790+
}
1791+
17531792
let group_exprs: PhyExprResult = agg
17541793
.grouping_exprs
17551794
.iter()
@@ -6351,6 +6390,76 @@ mod tests {
63516390
assert_eq!("ScanExec", projection_exec.children[0].native_plan.name());
63526391
}
63536392

6393+
#[tokio::test]
6394+
async fn keyless_aggregate_without_functions_emits_one_empty_row() {
6395+
use arrow::array::RecordBatchOptions;
6396+
use datafusion::common::tree_node::{Transformed, TreeNode};
6397+
6398+
let scan = Operator {
6399+
op_struct: Some(OpStruct::Scan(spark_operator::Scan {
6400+
fields: vec![],
6401+
source: String::new(),
6402+
})),
6403+
..Default::default()
6404+
};
6405+
let schema = Arc::new(Schema::empty());
6406+
let rows = |num_rows| {
6407+
RecordBatch::try_new_with_options(
6408+
Arc::clone(&schema),
6409+
vec![],
6410+
&RecordBatchOptions::new().with_row_count(Some(num_rows)),
6411+
)
6412+
.unwrap()
6413+
};
6414+
let ctx = SessionContext::new();
6415+
let task_ctx = ctx.task_ctx();
6416+
let planner = PhysicalPlanner::new(Arc::new(ctx), 0);
6417+
6418+
// Spark serializes the stage below its exchange as Partial and the one above as Final.
6419+
// Both emit one row with no columns per partition, even for an empty input.
6420+
for mode in [
6421+
spark_operator::AggregateMode::Partial,
6422+
spark_operator::AggregateMode::Final,
6423+
] {
6424+
let aggregate = Operator {
6425+
plan_id: 1,
6426+
children: vec![scan.clone()],
6427+
op_struct: Some(OpStruct::HashAgg(spark_operator::HashAggregate {
6428+
mode: mode as i32,
6429+
..Default::default()
6430+
})),
6431+
..Default::default()
6432+
};
6433+
let (_, _, planned) = planner.create_plan(&aggregate, &mut vec![], 1).unwrap();
6434+
assert_eq!("ProjectionExec", planned.native_plan.name());
6435+
assert_eq!(0, planned.schema().fields().len());
6436+
assert_eq!(1, planned.additional_native_plans.len());
6437+
assert_eq!("AggregateExec", planned.additional_native_plans[0].name());
6438+
assert_eq!(1, planned.children.len());
6439+
assert_eq!("ScanExec", planned.children[0].native_plan.name());
6440+
6441+
for batches in [vec![rows(3), rows(2)], vec![]] {
6442+
let input: Arc<dyn ExecutionPlan> =
6443+
MemorySourceConfig::try_new_exec(&[batches], Arc::clone(&schema), None)
6444+
.unwrap();
6445+
let native_plan = Arc::clone(&planned.native_plan)
6446+
.transform_up(|node| {
6447+
Ok(if node.name() == "ScanExec" {
6448+
Transformed::yes(Arc::clone(&input))
6449+
} else {
6450+
Transformed::no(node)
6451+
})
6452+
})
6453+
.unwrap()
6454+
.data;
6455+
let results = collect(native_plan.execute(0, Arc::clone(&task_ctx)).unwrap())
6456+
.await
6457+
.unwrap();
6458+
assert_eq!(1, results.iter().map(RecordBatch::num_rows).sum::<usize>());
6459+
}
6460+
}
6461+
}
6462+
63546463
#[tokio::test]
63556464
async fn sort_aggregate_output_is_ordered_by_grouping_keys() {
63566465
use arrow::datatypes::Int32Type;

‎spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1725,8 +1725,6 @@ case class CometExecRule(session: SparkSession, queryStagePrep: Boolean = false)
17251725
val aggregateExpressions = agg.aggregateExpressions
17261726
val groupingExpressions = agg.groupingExpressions
17271727

1728-
if (groupingExpressions.isEmpty && aggregateExpressions.isEmpty) return false
1729-
17301728
if (groupingExpressions.exists(e =>
17311729
SupportLevel.containsType(e.dataType, classOf[MapType]))) {
17321730
return false

‎spark/src/main/scala/org/apache/spark/sql/comet/operators.scala‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2265,11 +2265,6 @@ trait CometBaseAggregate[T <: BaseAggregateExec] extends CometOperatorSerde[T] {
22652265
val resultExpressions = aggregate.resultExpressions
22662266
val child = aggregate.child
22672267

2268-
if (groupingExpressions.isEmpty && aggregateExpressions.isEmpty) {
2269-
withFallbackReason(aggregate, "No group by or aggregation")
2270-
return None
2271-
}
2272-
22732268
if (groupingExpressions.exists(expr =>
22742269
SupportLevel.containsType(expr.dataType, classOf[MapType]))) {
22752270
withFallbackReason(aggregate, "Grouping on map-containing types is not supported")
@@ -2314,6 +2309,10 @@ trait CometBaseAggregate[T <: BaseAggregateExec] extends CometOperatorSerde[T] {
23142309
// +- FileScan parquet spark_catalog.default.test[col1#6, col2#7] ......
23152310
// If the aggregateExpressions is empty, we only want to build groupingExpressions,
23162311
// and skip processing of aggregateExpressions.
2312+
//
2313+
// The groupingExpressions can be empty too, for the `Aggregate(Nil, Nil, child)` that
2314+
// ColumnPruning leaves when nothing reads a global aggregate's output. The native planner
2315+
// handles that shape.
23172316
if (aggregateExpressions.isEmpty) {
23182317
val hashAggBuilder = OperatorOuterClass.HashAggregate.newBuilder()
23192318
hashAggBuilder.addAllGroupingExprs(groupingExprs.map(_.get).asJava)
Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
-- Licensed to the Apache Software Foundation (ASF) under one
2+
-- or more contributor license agreements. See the NOTICE file
3+
-- distributed with this work for additional information
4+
-- regarding copyright ownership. The ASF licenses this file
5+
-- to you under the Apache License, Version 2.0 (the
6+
-- "License"); you may not use this file except in compliance
7+
-- with the License. You may obtain a copy of the License at
8+
--
9+
-- http://www.apache.org/licenses/LICENSE-2.0
10+
--
11+
-- Unless required by applicable law or agreed to in writing,
12+
-- software distributed under the License is distributed on an
13+
-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
-- KIND, either express or implied. See the License for the
15+
-- specific language governing permissions and limitations
16+
-- under the License.
17+
18+
-- ColumnPruning leaves a global aggregate with neither keys nor functions when nothing reads its
19+
-- output. Each stage of the partial/final pair emits one row with no columns per partition, even
20+
-- for an empty input.
21+
-- Config: spark.comet.exec.localTableScan.enabled=true
22+
23+
-- Five files. With the test session's parallelism of 5, the scan reads them as five partitions, so
24+
-- the final stage collapses five rows into one.
25+
statement
26+
CREATE TABLE empty_agg_multi(a int) USING parquet
27+
28+
statement
29+
INSERT INTO empty_agg_multi SELECT CAST(id AS INT) FROM range(0, 10, 1, 5)
30+
31+
-- No files at all.
32+
statement
33+
CREATE TABLE empty_agg_none(a int) USING parquet
34+
35+
statement
36+
CREATE TABLE empty_agg_division(n int, d int) USING parquet
37+
38+
statement
39+
INSERT INTO empty_agg_division SELECT 1, 0
40+
41+
-- CollapseProject folds the outer SELECT into the aggregate, so the final stage projects the
42+
-- literal over the row with no columns.
43+
query
44+
SELECT 1 FROM (SELECT count(*) FROM empty_agg_multi)
45+
46+
query
47+
SELECT 1 FROM (SELECT count(*) FROM empty_agg_multi WHERE a > 100)
48+
49+
query
50+
SELECT 1 FROM (SELECT count(*) FROM empty_agg_none)
51+
52+
-- Dataset.count() over an aggregated DataFrame plans this shape.
53+
query
54+
SELECT count(*) FROM (SELECT sum(a) FROM empty_agg_multi)
55+
56+
query
57+
SELECT count(*) FROM (SELECT sum(a) FROM empty_agg_none)
58+
59+
query
60+
SELECT 1 FROM (SELECT count(*) FROM VALUES (1) AS v(a))
61+
62+
query
63+
SELECT 1 FROM (SELECT count(*) FROM VALUES (1) AS v(a) WHERE a > 5)
64+
65+
-- The parser plans HAVING without GROUP BY as a global aggregate, here one that outputs a literal.
66+
query
67+
SELECT 1 AS x FROM empty_agg_multi HAVING true
68+
69+
query
70+
SELECT 1 AS x FROM empty_agg_none HAVING true
71+
72+
-- The aggregate still reads its whole input, so an error below it surfaces as it does in Spark.
73+
statement
74+
SET spark.sql.ansi.enabled=true
75+
76+
query expect_error_class(DIVIDE_BY_ZERO)
77+
SELECT 1 FROM (SELECT count(*) FROM empty_agg_division WHERE n / d > 0)

‎spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ import org.apache.spark.sql.comet.execution.shuffle.CometShuffleExchangeExec
3636
import org.apache.spark.sql.execution.LocalTableScanExec
3737
import org.apache.spark.sql.execution.SQLExecution
3838
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AdaptiveSparkPlanHelper, ShuffleQueryStageExec}
39-
import org.apache.spark.sql.execution.aggregate.{BaseAggregateExec, ObjectHashAggregateExec, SortAggregateExec}
39+
import org.apache.spark.sql.execution.aggregate.{BaseAggregateExec, HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec}
4040
import org.apache.spark.sql.execution.exchange.{ReusedExchangeExec, ShuffleExchangeExec}
4141
import org.apache.spark.sql.functions.{avg, col, collect_list, collect_set, count_distinct, expr, sort_array, sum}
4242
import org.apache.spark.sql.internal.SQLConf
@@ -4129,4 +4129,30 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
41294129
}
41304130
}
41314131

4132+
// global_empty_aggregate.sql covers the SQL shapes. A SQL query always projects a column, so
4133+
// only a DataFrame hands the zero-column row of `Aggregate(Nil, Nil, child)` to the JVM, where
4134+
// losing its row count would return no rows.
4135+
test("global aggregate with no keys and no functions returns one empty row") {
4136+
withParquetTable((0 until 10).map(Tuple1(_)), "tbl") {
4137+
Seq(spark.table("tbl"), spark.table("tbl").where($"_1" > 100)).foreach { input =>
4138+
checkSparkAnswerAndOperator(input.agg(sum($"_1")).select())
4139+
}
4140+
}
4141+
}
4142+
4143+
// Local table scans run in Spark by default, so the partial aggregate stays in Spark and the
4144+
// native final aggregate reads its rows through a JVM columnar shuffle.
4145+
test("global aggregate with no keys and no functions over a Spark partial aggregate") {
4146+
withSQLConf(CometConf.COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED.key -> "false") {
4147+
val (_, cometPlan) = checkSparkAnswerAndOperator(
4148+
sql("SELECT * FROM VALUES (1), (2) AS v(a)").agg(sum($"a")).select(),
4149+
Seq(classOf[CometHashAggregateExec]),
4150+
classOf[HashAggregateExec],
4151+
classOf[LocalTableScanExec])
4152+
assert(
4153+
collect(cometPlan) { case agg: HashAggregateExec => agg }.nonEmpty,
4154+
s"Expected a Spark partial aggregate; plan:\n$cometPlan")
4155+
}
4156+
}
4157+
41324158
}

0 commit comments

Comments
 (0)