Skip to content

Commit 46ad7ec

Browse files
committed
fix: block unsafe native AVG partials before Spark final
1 parent f1e5868 commit 46ad7ec

3 files changed

Lines changed: 90 additions & 11 deletions

File tree

‎spark/src/main/scala/org/apache/comet/serde/aggregates.scala‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,11 @@ object CometCount extends CometAggregateExpressionSerde[Count] {
133133

134134
object CometAverage extends CometAggregateExpressionSerde[Average] {
135135

136+
// A never-updated non-decimal native AVG partial emits (null, 0), but Spark's merge needs
137+
// (0.0, 0). Keep this direction disabled until #5420 repairs the emitted state, including
138+
// aggregate nodes that also contain a buffer-compatible function such as COUNT.
139+
override def supportsNativePartialToSparkFinal(fn: Average): Boolean = false
140+
136141
override def supportsSparkPartialToNativeFinal(fn: Average): Boolean =
137142
// Non-decimal AVG has a (sum: double, count: long) buffer matching Spark. Decimal AVG is
138143
// deferred (overflow nulls count differently) and stays unsafe for mixed execution.

‎spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala‎

Lines changed: 81 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,74 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
369369
}
370370

371371
for (adaptive <- Seq(false, true)) {
372+
test(s"COUNT and AVG fall back together across a Spark shuffle (AQE=$adaptive)") {
373+
withTempDir { dir =>
374+
val path = s"${dir.getAbsolutePath}/data"
375+
withSQLConf(CometConf.COMET_ENABLED.key -> "false") {
376+
spark
377+
.range(0L, 8L, 1L, 4)
378+
.selectExpr("id", "CAST(1 AS BIGINT) AS v", "CAST(NULL AS BIGINT) AS n")
379+
.write
380+
.parquet(path)
381+
}
382+
383+
withSQLConf(
384+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> adaptive.toString,
385+
SQLConf.FILES_MAX_PARTITION_BYTES.key -> "1048576",
386+
SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED.key -> "false",
387+
CometConf.COMET_NATIVE_SCAN_ENABLED.key -> "false",
388+
CometConf.COMET_CONVERT_FROM_PARQUET_ENABLED.key -> "true",
389+
CometConf.COMET_SHUFFLE_ENABLED.key -> "false") {
390+
withParquetTable(path, "count_avg_fallback") {
391+
assert(sql("SELECT * FROM count_avg_fallback").rdd.getNumPartitions == 4)
392+
for (finalEnabled <- Seq(false, true)) {
393+
withSQLConf(
394+
CometConf.COMET_ENABLE_FINAL_HASH_AGGREGATE.key -> finalEnabled.toString) {
395+
// A safe COUNT buffer must not admit an unsafe AVG buffer in the same Partial.
396+
// Three partitions have no surviving rows, so AVG has no update_batch call and
397+
// its old native state is (null, 0), which poisons Spark Final's sum. Disabling
398+
// Final tests early tagging; leaving it enabled tests post-conversion repair.
399+
for ((selection, expected) <- Seq(
400+
"AVG(v) FROM count_avg_fallback WHERE id = 1" -> Row(1L, 1.0),
401+
"AVG(n) FROM count_avg_fallback WHERE id = 1" -> Row(1L, null),
402+
"AVG(v) FROM count_avg_fallback WHERE id < 0" -> Row(0L, null))) {
403+
val df = sql(s"SELECT COUNT(*), $selection")
404+
val initialPlan = stripAQEPlan(df.queryExecution.executedPlan)
405+
checkAnswer(df, Seq(expected))
406+
for (plan <- Seq(initialPlan, df.queryExecution.executedPlan)) {
407+
assert(collect(plan) { case agg: CometHashAggregateExec => agg }.isEmpty)
408+
val partials = collect(plan) {
409+
case agg: BaseAggregateExec
410+
if agg.aggregateExpressions.map(_.mode).distinct == Seq(Partial) =>
411+
agg
412+
}
413+
assert(partials.size == 1)
414+
assert(
415+
partials.head.getTagValue(CometExecRule.COMET_UNSAFE_PARTIAL).isDefined)
416+
assert(collect(plan) { case filter: CometFilterExec => filter }.nonEmpty)
417+
}
418+
}
419+
}
420+
}
421+
withSQLConf(
422+
CometConf.COMET_SHUFFLE_ENABLED.key -> "true",
423+
CometConf.COMET_SHUFFLE_MODE.key -> "native",
424+
CometConf.COMET_ENABLE_FINAL_HASH_AGGREGATE.key -> "true") {
425+
// The native Final can consume its own empty AVG buffers; only the engine split
426+
// is unsafe. Keep the fully native aggregate path enabled.
427+
val native =
428+
sql("SELECT COUNT(*), AVG(v) FROM count_avg_fallback WHERE id = 1")
429+
val initialNativePlan = stripAQEPlan(native.queryExecution.executedPlan)
430+
checkAnswer(native, Seq(Row(1L, 1.0)))
431+
for (plan <- Seq(initialNativePlan, native.queryExecution.executedPlan)) {
432+
assert(collect(plan) { case agg: CometHashAggregateExec => agg }.size == 2)
433+
}
434+
}
435+
}
436+
}
437+
}
438+
}
439+
372440
test(s"COUNT preserves safe native partials across a Spark shuffle (AQE=$adaptive)") {
373441
val data = Seq((0, None), (0, None), (1, Some(3)), (1, None), (1, Some(4)))
374442
withParquetTable(data, "count_fallback", false) {
@@ -515,15 +583,16 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
515583
}
516584
}
517585

518-
test("mixed engine sum/avg: Comet partial + Spark final matches Spark") {
586+
test("mixed engine sum/avg falls back when Spark Final would consume native AVG") {
519587
val data = (0 until 100).map(i => (i, i.toLong, i.toDouble, i % 7))
520588
withParquetTable(data, "tbl") {
521589
withSQLConf(
522590
CometConf.COMET_ENABLE_FINAL_HASH_AGGREGATE.key -> "false",
523591
CometConf.COMET_SHUFFLE_ENABLED.key -> "true",
524592
CometConf.COMET_SHUFFLE_MODE.key -> "jvm") {
525-
checkSparkAnswer(
526-
"SELECT _4, SUM(_1), SUM(_2), SUM(_3), AVG(_1), AVG(_2), AVG(_3) FROM tbl GROUP BY _4")
593+
checkSparkAnswerAndNumOfAggregates(
594+
"SELECT _4, SUM(_1), SUM(_2), SUM(_3), AVG(_1), AVG(_2), AVG(_3) FROM tbl GROUP BY _4",
595+
0)
527596
}
528597
}
529598
}
@@ -793,7 +862,10 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
793862
checkSparkAnswerAndNumOfAggregates("SELECT _2, COUNT(_1) FROM tbl GROUP BY _2", n)
794863
checkSparkAnswerAndNumOfAggregates("SELECT _2, MIN(_1) FROM tbl GROUP BY _2", n)
795864
checkSparkAnswerAndNumOfAggregates("SELECT _2, MAX(_1) FROM tbl GROUP BY _2", n)
796-
checkSparkAnswerAndNumOfAggregates("SELECT _2, AVG(_1) FROM tbl GROUP BY _2", n)
865+
val avgStages = if (nativeShuffleEnabled) 2 else 0
866+
checkSparkAnswerAndNumOfAggregates(
867+
"SELECT _2, AVG(_1) FROM tbl GROUP BY _2",
868+
avgStages)
797869
}
798870
}
799871
}
@@ -1561,14 +1633,14 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
15611633
}
15621634
}
15631635

1564-
test("test partial avg") {
1636+
test("AVG stays in Spark across a Spark shuffle") {
15651637
Seq(true, false).foreach { dictionaryEnabled =>
15661638
withParquetTable(
15671639
(0 until 5).map(i => (i.toDouble, i.toDouble % 2)),
15681640
"tbl",
15691641
dictionaryEnabled) {
15701642
withSQLConf(CometConf.COMET_SHUFFLE_ENABLED.key -> "false") {
1571-
checkSparkAnswerAndNumOfAggregates("SELECT _2 , AVG(_1) FROM tbl GROUP BY _2", 1)
1643+
checkSparkAnswerAndNumOfAggregates("SELECT _2 , AVG(_1) FROM tbl GROUP BY _2", 0)
15721644
}
15731645
}
15741646
}
@@ -1605,8 +1677,9 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
16051677
val path = new Path(dir.toURI.toString, "test")
16061678
makeParquetFile(path, 1000, 20, dictionaryEnabled)
16071679
withParquetTable(path.toUri.toString, "tbl") {
1608-
// Only _7 is rewritten to a mixed-safe Long AVG by Spark's decimal optimizer.
1609-
val expectedNumOfCometAggregates = if (nativeShuffleEnabled) 2 else 1
1680+
// Spark rewrites _7 to Long AVG, whose empty native buffer is also unsafe for a
1681+
// Spark Final until #5420. Keep all AVG partials in Spark across this boundary.
1682+
val expectedNumOfCometAggregates = if (nativeShuffleEnabled) 2 else 0
16101683
val expectedNumOfDecimalAggregates = if (nativeShuffleEnabled) 2 else 0
16111684

16121685
checkSparkAnswerAndNumOfAggregates(

‎spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -362,7 +362,7 @@ class CometExecRuleSuite extends CometTestBase {
362362
}
363363
}
364364

365-
test("CometExecRule should allow AVG mixed Comet partial and Spark final") {
365+
test("CometExecRule should not allow AVG Comet partial and Spark final before buffer repair") {
366366
withTempView("test_data") {
367367
createTestDataFrame.createOrReplaceTempView("test_data")
368368
val sparkPlan =
@@ -372,8 +372,9 @@ class CometExecRuleSuite extends CometTestBase {
372372
CometConf.COMET_ENABLE_FINAL_HASH_AGGREGATE.key -> "false",
373373
CometConf.COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED.key -> "true") {
374374
val transformedPlan = applyCometExecRule(sparkPlan)
375-
assert(countOperators(transformedPlan, classOf[HashAggregateExec]) == 1) // final
376-
assert(countOperators(transformedPlan, classOf[CometHashAggregateExec]) == 1) // partial
375+
// Matching field types do not make native AVG's empty (null, 0) state safe for Spark.
376+
assert(countOperators(transformedPlan, classOf[HashAggregateExec]) == 2)
377+
assert(countOperators(transformedPlan, classOf[CometHashAggregateExec]) == 0)
377378
}
378379
}
379380
}

0 commit comments

Comments
 (0)