@@ -32,11 +32,13 @@ import org.apache.spark.sql.execution._
3232import org .apache .spark .sql .execution .adaptive .{AQEShuffleReadExec , QueryStageExec , ShuffleQueryStageExec }
3333import org .apache .spark .sql .execution .aggregate .{HashAggregateExec , ObjectHashAggregateExec }
3434import org .apache .spark .sql .execution .exchange .{BroadcastExchangeExec , ReusedExchangeExec , ShuffleExchangeExec }
35+ import org .apache .spark .sql .internal .SQLConf
3536import org .apache .spark .sql .types .{DataTypes , StructField , StructType }
3637
3738import org .apache .comet .{CometConf , CometExplainInfo , ExtendedExplainInfo }
3839import org .apache .comet .CometSparkSessionExtensions .{isSpark35Plus , isSpark40Plus , isSpark42Plus }
3940import org .apache .comet .CometSparkSessionExtensions .withFallbackReason
41+ import org .apache .comet .serde .QueryPlanSerde
4042import org .apache .comet .testing .{DataGenOptions , FuzzDataGenerator }
4143
4244/**
@@ -474,6 +476,129 @@ class CometExecRuleSuite extends CometTestBase {
474476 }
475477 }
476478
479+ // Inspect fresh, unmaterialized global/grouped one-DISTINCT plans across shuffle modes and
480+ // refusal states. Each case checks both initial conversion and re-entry without running Spark
481+ // jobs. Native-only fallback must repair the whole chain; selective auto/JVM fallback pins the
482+ // narrower grouped-AVG boundary that this Final-only pass does not repair. Scoped configuration
483+ // and temporary-view helpers restore session state, and assertion failures identify the plan.
484+ for (grouped <- Seq (false , true )) {
485+ test(s " nondecimal AVG distinct stages follow actual shuffle conversion (grouped= $grouped) " ) {
486+ withSQLConf(
487+ SQLConf .ADAPTIVE_EXECUTION_ENABLED .key -> " false" ,
488+ // Comet's real rule phase precedes whole-stage wrappers. Apply the rule to the same
489+ // phase here so codegen wrappers do not create artificial non-native children.
490+ SQLConf .WHOLESTAGE_CODEGEN_ENABLED .key -> " false" ,
491+ SQLConf .SHUFFLE_PARTITIONS .key -> " 4" ,
492+ CometConf .COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED .key -> " true" ,
493+ CometConf .COMET_SHUFFLE_ENABLED .key -> " true" ,
494+ CometConf .COMET_SHUFFLE_CONVERT_FROM_SPARK_PLAN_ENABLED .key -> " true" ,
495+ CometConf .COMET_ENABLE_PARTIAL_HASH_AGGREGATE .key -> " true" ,
496+ CometConf .COMET_ENABLE_FINAL_HASH_AGGREGATE .key -> " true" ) {
497+ withTempView(" test_data" ) {
498+ createTestDataFrame.createOrReplaceTempView(" test_data" )
499+ val groupBy = if (grouped) " GROUP BY (id % 3)" else " "
500+ val query = s " SELECT AVG(id), SUM(DISTINCT id) FROM test_data $groupBy"
501+ val expectedModes =
502+ Seq (Seq (Final ), Seq (PartialMerge , Partial ), Seq (PartialMerge ), Seq (Partial ))
503+
504+ for {
505+ shuffleMode <- Seq (" native" , " auto" , " jvm" )
506+ fallback <- Seq (" none" , " lower-only sticky" , " disabled hash partitioning" )
507+ } {
508+ withSQLConf(
509+ CometConf .COMET_SHUFFLE_MODE .key -> shuffleMode,
510+ CometConf .COMET_SHUFFLE_NATIVE_HASH_PARTITIONING_ENABLED .key ->
511+ (fallback != " disabled hash partitioning" ).toString) {
512+ val input = createSparkPlan(spark, query)
513+ assert(input.collect { case agg : HashAggregateExec =>
514+ agg.aggregateExpressions.map(_.mode).distinct
515+ } == expectedModes)
516+
517+ if (fallback == " lower-only sticky" ) {
518+ val lower = input.collect {
519+ case exchange : ShuffleExchangeExec
520+ if exchange.child.isInstanceOf [HashAggregateExec ] &&
521+ exchange.child
522+ .asInstanceOf [HashAggregateExec ]
523+ .aggregateExpressions
524+ .map(_.mode)
525+ .distinct == Seq (Partial ) =>
526+ exchange
527+ }
528+ assert(lower.size == 1 )
529+ // Simulate one prior refusal without tagging the upper exchange. This isolates
530+ // native execution restarting above Spark merge stages; it does not claim an
531+ // ordinary SQL query currently causes this selective refusal by itself.
532+ withFallbackReason(lower.head, " test lower-only prior shuffle fallback" )
533+ }
534+
535+ val allSpark = shuffleMode == " native" && fallback != " none"
536+ val split = shuffleMode != " native" && fallback == " lower-only sticky"
537+ val expectedNative =
538+ if (allSpark) Seq (false , false , false , false )
539+ else if (split) Seq (true , false , false , true )
540+ else Seq (true , true , true , true )
541+ val transformed = applyCometExecRule(input)
542+ for (plan <- Seq (transformed, applyCometExecRule(transformed))) {
543+ withClue(s " shuffle= $shuffleMode fallback= $fallback\n ${plan.treeString}" ) {
544+ val stages = plan.collect {
545+ case agg : CometHashAggregateExec => (true , agg.modes)
546+ case agg : HashAggregateExec =>
547+ (false , agg.aggregateExpressions.map(_.mode).distinct)
548+ }
549+ assert(stages.map(_._1) == expectedNative)
550+ assert(stages.map(_._2) == expectedModes)
551+ assert(plan.collect { case project : CometProjectExec => project }.nonEmpty)
552+
553+ if (allSpark) {
554+ val partial = plan.collectFirst {
555+ case agg : HashAggregateExec
556+ if agg.aggregateExpressions.map(_.mode).distinct == Seq (Partial ) =>
557+ agg
558+ }.get
559+ assert(partial.getTagValue(CometExecRule .COMET_UNSAFE_PARTIAL ).isDefined)
560+ } else if (split) {
561+ val partial = plan.collectFirst {
562+ case agg : CometHashAggregateExec if agg.modes == Seq (Partial ) => agg
563+ }.get
564+ val merge = plan.collectFirst {
565+ case agg : HashAggregateExec
566+ if agg.aggregateExpressions.map(_.mode).distinct == Seq (PartialMerge ) =>
567+ agg
568+ }.get
569+ val nativeFinal = plan.collectFirst {
570+ case agg : CometHashAggregateExec if agg.modes == Seq (Final ) => agg
571+ }.get
572+ val lower = merge.child.asInstanceOf [ShuffleExchangeExec ]
573+ assert(lower.child eq partial)
574+ assert(
575+ lower
576+ .getTagValue(CometExplainInfo .FALLBACK_REASONS )
577+ .exists(_.contains(" test lower-only prior shuffle fallback" )))
578+ assert(partial.getTagValue(CometExecRule .COMET_UNSAFE_PARTIAL ).isEmpty)
579+ assert(
580+ ! QueryPlanSerde .allAggsSupportNativePartialToSparkFinal(
581+ partial.aggregateExpressions))
582+ assert(
583+ QueryPlanSerde
584+ .aggsNotSupportingSparkPartialToNativeFinal(
585+ nativeFinal.aggregateExpressions)
586+ .isEmpty)
587+ // Even the global query's bottom Partial groups by the DISTINCT key.
588+ // Grouped AVG exports non-null sum/count states: (0.0, 0) for all-null
589+ // groups, and no row for an empty partition. This is not the ungrouped,
590+ // never-updated AVG (null, 0) producer.
591+ assert(partial.groupingExpressions.size == (if (grouped) 2 else 1 ))
592+ }
593+ }
594+ }
595+ }
596+ }
597+ }
598+ }
599+ }
600+ }
601+
477602 test(" unrepaired aggregate buffers warn without rewriting stages or unrelated producers" ) {
478603 // Synthetic wrappers exercise traversal drift, not a claim that Spark currently plans these
479604 // unsafe boundaries. The diagnostic must see through passthrough wrappers but stop at an
0 commit comments