@@ -368,6 +368,78 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
368368 }
369369 }
370370
371+ for (adaptive <- Seq (false , true )) {
372+ test(s " COUNT preserves safe native partials across a Spark shuffle (AQE= $adaptive) " ) {
373+ val data = Seq ((0 , None ), (0 , None ), (1 , Some (3 )), (1 , None ), (1 , Some (4 )))
374+ withParquetTable(data, " count_fallback" , false ) {
375+ for (finalEnabled <- Seq (false , true )) {
376+ withSQLConf(
377+ SQLConf .ADAPTIVE_EXECUTION_ENABLED .key -> adaptive.toString,
378+ SQLConf .PARQUET_FILTER_PUSHDOWN_ENABLED .key -> " false" ,
379+ CometConf .COMET_SHUFFLE_ENABLED .key -> " false" ,
380+ CometConf .COMET_ENABLE_FINAL_HASH_AGGREGATE .key -> finalEnabled.toString) {
381+ for (query <- Seq (
382+ " SELECT _1, COUNT(_2), COUNT(*) FROM count_fallback GROUP BY _1" ,
383+ " SELECT COUNT(_2), COUNT(*) FROM count_fallback WHERE _1 = 0" ,
384+ " SELECT COUNT(_2), COUNT(*) FROM count_fallback WHERE _1 < 0" )) {
385+ val df = sql(query)
386+ val initialPlan = stripAQEPlan(df.queryExecution.executedPlan)
387+ assert(collect(initialPlan) {
388+ case agg : CometHashAggregateExec if agg.modes == Seq (Partial ) => agg
389+ }.size == 1 )
390+ assert(collect(initialPlan) {
391+ case agg : BaseAggregateExec
392+ if agg.aggregateExpressions.map(_.mode).distinct == Seq (Final ) =>
393+ agg
394+ }.size == 1 )
395+ checkSparkAnswer(df)
396+ }
397+ }
398+ }
399+ }
400+ }
401+
402+ for (fn <- Seq (" collect_list" , " collect_set" )) {
403+ test(s " $fn falls back when enabled native shuffle is ineligible (AQE= $adaptive) " ) {
404+ val data = (0 until 30 ).map(i => (i % 3 , if (i % 7 == 0 ) None else Some (i % 5 )))
405+ withParquetTable(data, " collect_fallback" , false ) {
406+ withSQLConf(
407+ SQLConf .ADAPTIVE_EXECUTION_ENABLED .key -> adaptive.toString,
408+ SQLConf .PARQUET_FILTER_PUSHDOWN_ENABLED .key -> " false" ,
409+ SQLConf .USE_OBJECT_HASH_AGG .key -> " true" ,
410+ CometConf .COMET_SHUFFLE_ENABLED .key -> " true" ,
411+ CometConf .COMET_SHUFFLE_MODE .key -> " native" ,
412+ CometConf .COMET_SHUFFLE_NATIVE_HASH_PARTITIONING_ENABLED .key -> " false" ) {
413+ // Integer keys isolate this from the wide-decimal shuffle restriction in #5420.
414+ // The native Partial emits an Array buffer, but Spark's Final expects Binary.
415+ val query = s " SELECT _1, sort_array( $fn(_2)), COUNT(*) " +
416+ " FROM collect_fallback WHERE _1 >= 0 GROUP BY _1"
417+ val df = sql(query)
418+ val initialPlan = stripAQEPlan(df.queryExecution.executedPlan)
419+ checkSparkAnswer(df)
420+ for (plan <- Seq (initialPlan, df.queryExecution.executedPlan)) {
421+ assert(collect(plan) { case agg : CometHashAggregateExec => agg }.isEmpty)
422+ val partials = collect(plan) {
423+ case agg : BaseAggregateExec
424+ if agg.aggregateExpressions.map(_.mode).distinct == Seq (Partial ) =>
425+ agg
426+ }
427+ assert(partials.size == 1 )
428+ assert(partials.head.getTagValue(CometExecRule .COMET_UNSAFE_PARTIAL ).isDefined)
429+ assert(collect(plan) { case filter : CometFilterExec => filter }.nonEmpty)
430+ }
431+ // A fully native producer/consumer pair can still use its native buffer format.
432+ withSQLConf(CometConf .COMET_SHUFFLE_NATIVE_HASH_PARTITIONING_ENABLED .key -> " true" ) {
433+ val native = sql(query)
434+ checkSparkAnswer(native)
435+ assert(getNumCometHashAggregate(native) == 2 )
436+ }
437+ }
438+ }
439+ }
440+ }
441+ }
442+
371443 test(" stddev_pop should return NaN for some cases" ) {
372444 withSQLConf(CometConf .COMET_SHUFFLE_ENABLED .key -> " true" ) {
373445 Seq (true , false ).foreach { nullOnDivideByZero =>
@@ -718,10 +790,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
718790 dictionaryEnabled) {
719791 val n = if (nativeShuffleEnabled) 2 else 1
720792 checkSparkAnswerAndNumOfAggregates(" SELECT _2, SUM(_1) FROM tbl GROUP BY _2" , n)
721- // COUNT is not declared safe for mixed execution, unlike the other aggregates here.
722- checkSparkAnswerAndNumOfAggregates(
723- " SELECT _2, COUNT(_1) FROM tbl GROUP BY _2" ,
724- if (nativeShuffleEnabled) 2 else 0 )
793+ checkSparkAnswerAndNumOfAggregates(" SELECT _2, COUNT(_1) FROM tbl GROUP BY _2" , n)
725794 checkSparkAnswerAndNumOfAggregates(" SELECT _2, MIN(_1) FROM tbl GROUP BY _2" , n)
726795 checkSparkAnswerAndNumOfAggregates(" SELECT _2, MAX(_1) FROM tbl GROUP BY _2" , n)
727796 checkSparkAnswerAndNumOfAggregates(" SELECT _2, AVG(_1) FROM tbl GROUP BY _2" , n)
0 commit comments