Skip to content

Commit cb22f68

Browse files
committed
test: cover DISTINCT aggregate fallback boundaries
Cover nondecimal AVG with SUM(DISTINCT) across global and grouped plans, shuffle modes, and selective lower-exchange fallback. Document why native-only fallback reaches the Final repair and qualify the auto/JVM boundary without changing the production predicate.
1 parent 197ca77 commit cb22f68

2 files changed

Lines changed: 133 additions & 0 deletions

File tree

‎spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1174,6 +1174,14 @@ case class CometExecRule(session: SparkSession)
11741174
* Restore only the feeding aggregate/exchange chain; keep native work below its Partial. If a
11751175
* remaining native buffer producer cannot be restored, warn and annotate the Spark Final; do
11761176
* not rewrite materialized stages or assume unknown operators can safely be reconstructed.
1177+
*
1178+
* In native-only shuffle mode, a failed lower exchange in a one-distinct chain leaves its
1179+
* PartialMerge consumers in Spark. Their Spark output also prevents the upper exchange from
1180+
* becoming native, so the Final remains Spark and triggers this repair, including with
1181+
* SUM(DISTINCT). In auto/jvm mode an upper columnar shuffle can instead bridge Spark merge
1182+
* buffers into a compatible native Final; this Final-only trigger does not inspect that
1183+
* separate boundary. The nondecimal AVG distinct-stage tests pin both paths without treating a
1184+
* grouped AVG producer as the scalar AVG whose untouched state is (null, 0).
11771185
*/
11781186
private[rules] def revertUnsafePartialAggregates(plan: SparkPlan): SparkPlan = {
11791187
def revertChain(node: SparkPlan): Option[SparkPlan] = node match {

‎spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,13 @@ import org.apache.spark.sql.execution._
3232
import org.apache.spark.sql.execution.adaptive.{AQEShuffleReadExec, QueryStageExec, ShuffleQueryStageExec}
3333
import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec}
3434
import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ReusedExchangeExec, ShuffleExchangeExec}
35+
import org.apache.spark.sql.internal.SQLConf
3536
import org.apache.spark.sql.types.{DataTypes, StructField, StructType}
3637

3738
import org.apache.comet.{CometConf, CometExplainInfo, ExtendedExplainInfo}
3839
import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus, isSpark42Plus}
3940
import org.apache.comet.CometSparkSessionExtensions.withFallbackReason
41+
import org.apache.comet.serde.QueryPlanSerde
4042
import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator}
4143

4244
/**
@@ -474,6 +476,129 @@ class CometExecRuleSuite extends CometTestBase {
474476
}
475477
}
476478

479+
// Inspect fresh, unmaterialized global/grouped one-DISTINCT plans across shuffle modes and
480+
// refusal states. Each case checks both initial conversion and re-entry without running Spark
481+
// jobs. Native-only fallback must repair the whole chain; selective auto/JVM fallback pins the
482+
// narrower grouped-AVG boundary that this Final-only pass does not repair. Scoped configuration
483+
// and temporary-view helpers restore session state, and assertion failures identify the plan.
484+
for (grouped <- Seq(false, true)) {
485+
test(s"nondecimal AVG distinct stages follow actual shuffle conversion (grouped=$grouped)") {
486+
withSQLConf(
487+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
488+
// Comet's real rule phase precedes whole-stage wrappers. Apply the rule to the same
489+
// phase here so codegen wrappers do not create artificial non-native children.
490+
SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false",
491+
SQLConf.SHUFFLE_PARTITIONS.key -> "4",
492+
CometConf.COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED.key -> "true",
493+
CometConf.COMET_SHUFFLE_ENABLED.key -> "true",
494+
CometConf.COMET_SHUFFLE_CONVERT_FROM_SPARK_PLAN_ENABLED.key -> "true",
495+
CometConf.COMET_ENABLE_PARTIAL_HASH_AGGREGATE.key -> "true",
496+
CometConf.COMET_ENABLE_FINAL_HASH_AGGREGATE.key -> "true") {
497+
withTempView("test_data") {
498+
createTestDataFrame.createOrReplaceTempView("test_data")
499+
val groupBy = if (grouped) " GROUP BY (id % 3)" else ""
500+
val query = s"SELECT AVG(id), SUM(DISTINCT id) FROM test_data$groupBy"
501+
val expectedModes =
502+
Seq(Seq(Final), Seq(PartialMerge, Partial), Seq(PartialMerge), Seq(Partial))
503+
504+
for {
505+
shuffleMode <- Seq("native", "auto", "jvm")
506+
fallback <- Seq("none", "lower-only sticky", "disabled hash partitioning")
507+
} {
508+
withSQLConf(
509+
CometConf.COMET_SHUFFLE_MODE.key -> shuffleMode,
510+
CometConf.COMET_SHUFFLE_NATIVE_HASH_PARTITIONING_ENABLED.key ->
511+
(fallback != "disabled hash partitioning").toString) {
512+
val input = createSparkPlan(spark, query)
513+
assert(input.collect { case agg: HashAggregateExec =>
514+
agg.aggregateExpressions.map(_.mode).distinct
515+
} == expectedModes)
516+
517+
if (fallback == "lower-only sticky") {
518+
val lower = input.collect {
519+
case exchange: ShuffleExchangeExec
520+
if exchange.child.isInstanceOf[HashAggregateExec] &&
521+
exchange.child
522+
.asInstanceOf[HashAggregateExec]
523+
.aggregateExpressions
524+
.map(_.mode)
525+
.distinct == Seq(Partial) =>
526+
exchange
527+
}
528+
assert(lower.size == 1)
529+
// Simulate one prior refusal without tagging the upper exchange. This isolates
530+
// native execution restarting above Spark merge stages; it does not claim an
531+
// ordinary SQL query currently causes this selective refusal by itself.
532+
withFallbackReason(lower.head, "test lower-only prior shuffle fallback")
533+
}
534+
535+
val allSpark = shuffleMode == "native" && fallback != "none"
536+
val split = shuffleMode != "native" && fallback == "lower-only sticky"
537+
val expectedNative =
538+
if (allSpark) Seq(false, false, false, false)
539+
else if (split) Seq(true, false, false, true)
540+
else Seq(true, true, true, true)
541+
val transformed = applyCometExecRule(input)
542+
for (plan <- Seq(transformed, applyCometExecRule(transformed))) {
543+
withClue(s"shuffle=$shuffleMode fallback=$fallback\n${plan.treeString}") {
544+
val stages = plan.collect {
545+
case agg: CometHashAggregateExec => (true, agg.modes)
546+
case agg: HashAggregateExec =>
547+
(false, agg.aggregateExpressions.map(_.mode).distinct)
548+
}
549+
assert(stages.map(_._1) == expectedNative)
550+
assert(stages.map(_._2) == expectedModes)
551+
assert(plan.collect { case project: CometProjectExec => project }.nonEmpty)
552+
553+
if (allSpark) {
554+
val partial = plan.collectFirst {
555+
case agg: HashAggregateExec
556+
if agg.aggregateExpressions.map(_.mode).distinct == Seq(Partial) =>
557+
agg
558+
}.get
559+
assert(partial.getTagValue(CometExecRule.COMET_UNSAFE_PARTIAL).isDefined)
560+
} else if (split) {
561+
val partial = plan.collectFirst {
562+
case agg: CometHashAggregateExec if agg.modes == Seq(Partial) => agg
563+
}.get
564+
val merge = plan.collectFirst {
565+
case agg: HashAggregateExec
566+
if agg.aggregateExpressions.map(_.mode).distinct == Seq(PartialMerge) =>
567+
agg
568+
}.get
569+
val nativeFinal = plan.collectFirst {
570+
case agg: CometHashAggregateExec if agg.modes == Seq(Final) => agg
571+
}.get
572+
val lower = merge.child.asInstanceOf[ShuffleExchangeExec]
573+
assert(lower.child eq partial)
574+
assert(
575+
lower
576+
.getTagValue(CometExplainInfo.FALLBACK_REASONS)
577+
.exists(_.contains("test lower-only prior shuffle fallback")))
578+
assert(partial.getTagValue(CometExecRule.COMET_UNSAFE_PARTIAL).isEmpty)
579+
assert(
580+
!QueryPlanSerde.allAggsSupportNativePartialToSparkFinal(
581+
partial.aggregateExpressions))
582+
assert(
583+
QueryPlanSerde
584+
.aggsNotSupportingSparkPartialToNativeFinal(
585+
nativeFinal.aggregateExpressions)
586+
.isEmpty)
587+
// Even the global query's bottom Partial groups by the DISTINCT key.
588+
// Grouped AVG exports non-null sum/count states: (0.0, 0) for all-null
589+
// groups, and no row for an empty partition. This is not the ungrouped,
590+
// never-updated AVG (null, 0) producer.
591+
assert(partial.groupingExpressions.size == (if (grouped) 2 else 1))
592+
}
593+
}
594+
}
595+
}
596+
}
597+
}
598+
}
599+
}
600+
}
601+
477602
test("unrepaired aggregate buffers warn without rewriting stages or unrelated producers") {
478603
// Synthetic wrappers exercise traversal drift, not a claim that Spark currently plans these
479604
// unsafe boundaries. The diagnostic must see through passthrough wrappers but stop at an

0 commit comments

Comments
 (0)