@@ -369,6 +369,74 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
369369 }
370370
371371 for (adaptive <- Seq (false , true )) {
372+ test(s " COUNT and AVG fall back together across a Spark shuffle (AQE= $adaptive) " ) {
373+ withTempDir { dir =>
374+ val path = s " ${dir.getAbsolutePath}/data "
375+ withSQLConf(CometConf .COMET_ENABLED .key -> " false" ) {
376+ spark
377+ .range(0L , 8L , 1L , 4 )
378+ .selectExpr(" id" , " CAST(1 AS BIGINT) AS v" , " CAST(NULL AS BIGINT) AS n" )
379+ .write
380+ .parquet(path)
381+ }
382+
383+ withSQLConf(
384+ SQLConf .ADAPTIVE_EXECUTION_ENABLED .key -> adaptive.toString,
385+ SQLConf .FILES_MAX_PARTITION_BYTES .key -> " 1048576" ,
386+ SQLConf .PARQUET_FILTER_PUSHDOWN_ENABLED .key -> " false" ,
387+ CometConf .COMET_NATIVE_SCAN_ENABLED .key -> " false" ,
388+ CometConf .COMET_CONVERT_FROM_PARQUET_ENABLED .key -> " true" ,
389+ CometConf .COMET_SHUFFLE_ENABLED .key -> " false" ) {
390+ withParquetTable(path, " count_avg_fallback" ) {
391+ assert(sql(" SELECT * FROM count_avg_fallback" ).rdd.getNumPartitions == 4 )
392+ for (finalEnabled <- Seq (false , true )) {
393+ withSQLConf(
394+ CometConf .COMET_ENABLE_FINAL_HASH_AGGREGATE .key -> finalEnabled.toString) {
395+ // A safe COUNT buffer must not admit an unsafe AVG buffer in the same Partial.
396+ // Three partitions have no surviving rows, so AVG has no update_batch call and
397+ // its old native state is (null, 0), which poisons Spark Final's sum. Disabling
398+ // Final tests early tagging; leaving it enabled tests post-conversion repair.
399+ for ((selection, expected) <- Seq (
400+ " AVG(v) FROM count_avg_fallback WHERE id = 1" -> Row (1L , 1.0 ),
401+ " AVG(n) FROM count_avg_fallback WHERE id = 1" -> Row (1L , null ),
402+ " AVG(v) FROM count_avg_fallback WHERE id < 0" -> Row (0L , null ))) {
403+ val df = sql(s " SELECT COUNT(*), $selection" )
404+ val initialPlan = stripAQEPlan(df.queryExecution.executedPlan)
405+ checkAnswer(df, Seq (expected))
406+ for (plan <- Seq (initialPlan, df.queryExecution.executedPlan)) {
407+ assert(collect(plan) { case agg : CometHashAggregateExec => agg }.isEmpty)
408+ val partials = collect(plan) {
409+ case agg : BaseAggregateExec
410+ if agg.aggregateExpressions.map(_.mode).distinct == Seq (Partial ) =>
411+ agg
412+ }
413+ assert(partials.size == 1 )
414+ assert(
415+ partials.head.getTagValue(CometExecRule .COMET_UNSAFE_PARTIAL ).isDefined)
416+ assert(collect(plan) { case filter : CometFilterExec => filter }.nonEmpty)
417+ }
418+ }
419+ }
420+ }
421+ withSQLConf(
422+ CometConf .COMET_SHUFFLE_ENABLED .key -> " true" ,
423+ CometConf .COMET_SHUFFLE_MODE .key -> " native" ,
424+ CometConf .COMET_ENABLE_FINAL_HASH_AGGREGATE .key -> " true" ) {
425+ // The native Final can consume its own empty AVG buffers; only the engine split
426+ // is unsafe. Keep the fully native aggregate path enabled.
427+ val native =
428+ sql(" SELECT COUNT(*), AVG(v) FROM count_avg_fallback WHERE id = 1" )
429+ val initialNativePlan = stripAQEPlan(native.queryExecution.executedPlan)
430+ checkAnswer(native, Seq (Row (1L , 1.0 )))
431+ for (plan <- Seq (initialNativePlan, native.queryExecution.executedPlan)) {
432+ assert(collect(plan) { case agg : CometHashAggregateExec => agg }.size == 2 )
433+ }
434+ }
435+ }
436+ }
437+ }
438+ }
439+
372440 test(s " COUNT preserves safe native partials across a Spark shuffle (AQE= $adaptive) " ) {
373441 val data = Seq ((0 , None ), (0 , None ), (1 , Some (3 )), (1 , None ), (1 , Some (4 )))
374442 withParquetTable(data, " count_fallback" , false ) {
@@ -515,15 +583,16 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
515583 }
516584 }
517585
518- test(" mixed engine sum/avg: Comet partial + Spark final matches Spark " ) {
586+ test(" mixed engine sum/avg falls back when Spark Final would consume native AVG " ) {
519587 val data = (0 until 100 ).map(i => (i, i.toLong, i.toDouble, i % 7 ))
520588 withParquetTable(data, " tbl" ) {
521589 withSQLConf(
522590 CometConf .COMET_ENABLE_FINAL_HASH_AGGREGATE .key -> " false" ,
523591 CometConf .COMET_SHUFFLE_ENABLED .key -> " true" ,
524592 CometConf .COMET_SHUFFLE_MODE .key -> " jvm" ) {
525- checkSparkAnswer(
526- " SELECT _4, SUM(_1), SUM(_2), SUM(_3), AVG(_1), AVG(_2), AVG(_3) FROM tbl GROUP BY _4" )
593+ checkSparkAnswerAndNumOfAggregates(
594+ " SELECT _4, SUM(_1), SUM(_2), SUM(_3), AVG(_1), AVG(_2), AVG(_3) FROM tbl GROUP BY _4" ,
595+ 0 )
527596 }
528597 }
529598 }
@@ -793,7 +862,10 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
793862 checkSparkAnswerAndNumOfAggregates(" SELECT _2, COUNT(_1) FROM tbl GROUP BY _2" , n)
794863 checkSparkAnswerAndNumOfAggregates(" SELECT _2, MIN(_1) FROM tbl GROUP BY _2" , n)
795864 checkSparkAnswerAndNumOfAggregates(" SELECT _2, MAX(_1) FROM tbl GROUP BY _2" , n)
796- checkSparkAnswerAndNumOfAggregates(" SELECT _2, AVG(_1) FROM tbl GROUP BY _2" , n)
865+ val avgStages = if (nativeShuffleEnabled) 2 else 0
866+ checkSparkAnswerAndNumOfAggregates(
867+ " SELECT _2, AVG(_1) FROM tbl GROUP BY _2" ,
868+ avgStages)
797869 }
798870 }
799871 }
@@ -1561,14 +1633,14 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
15611633 }
15621634 }
15631635
1564- test(" test partial avg " ) {
1636+ test(" AVG stays in Spark across a Spark shuffle " ) {
15651637 Seq (true , false ).foreach { dictionaryEnabled =>
15661638 withParquetTable(
15671639 (0 until 5 ).map(i => (i.toDouble, i.toDouble % 2 )),
15681640 " tbl" ,
15691641 dictionaryEnabled) {
15701642 withSQLConf(CometConf .COMET_SHUFFLE_ENABLED .key -> " false" ) {
1571- checkSparkAnswerAndNumOfAggregates(" SELECT _2 , AVG(_1) FROM tbl GROUP BY _2" , 1 )
1643+ checkSparkAnswerAndNumOfAggregates(" SELECT _2 , AVG(_1) FROM tbl GROUP BY _2" , 0 )
15721644 }
15731645 }
15741646 }
@@ -1605,8 +1677,9 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
16051677 val path = new Path (dir.toURI.toString, " test" )
16061678 makeParquetFile(path, 1000 , 20 , dictionaryEnabled)
16071679 withParquetTable(path.toUri.toString, " tbl" ) {
1608- // Only _7 is rewritten to a mixed-safe Long AVG by Spark's decimal optimizer.
1609- val expectedNumOfCometAggregates = if (nativeShuffleEnabled) 2 else 1
1680+ // Spark rewrites _7 to Long AVG, whose empty native buffer is also unsafe for a
1681+ // Spark Final until #5420. Keep all AVG partials in Spark across this boundary.
1682+ val expectedNumOfCometAggregates = if (nativeShuffleEnabled) 2 else 0
16101683 val expectedNumOfDecimalAggregates = if (nativeShuffleEnabled) 2 else 0
16111684
16121685 checkSparkAnswerAndNumOfAggregates(
0 commit comments