Skip to content

Commit 6dca7f1

Browse files
committed
fix: preserve safe AVG interop and measure wide decimal shuffle cost
1 parent e392eec commit 6dca7f1

2 files changed

Lines changed: 99 additions & 0 deletions

File tree

‎spark/src/main/scala/org/apache/comet/serde/aggregates.scala‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,11 @@ object CometCount extends CometAggregateExpressionSerde[Count] {
133133

134134
object CometAverage extends CometAggregateExpressionSerde[Average] {
135135

136+
// Native AVG exports (0.0, 0) for empty partials after #5420, so Spark Final can merge its
137+
// non-decimal buffers. This replaces #5421's conservative policy for the older native state.
138+
override def supportsNativePartialToSparkFinal(fn: Average): Boolean =
139+
supportsSparkPartialToNativeFinal(fn)
140+
136141
override def supportsSparkPartialToNativeFinal(fn: Average): Boolean =
137142
// Non-decimal AVG has a (sum: double, count: long) buffer matching Spark. Decimal AVG is
138143
// deferred (overflow nulls count differently) and stays unsafe for mixed execution.

‎spark/src/test/scala/org/apache/spark/sql/benchmark/CometAggregateExpressionBenchmark.scala‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,13 @@
1919

2020
package org.apache.spark.sql.benchmark
2121

22+
import org.apache.spark.sql.comet.CometHashAggregateExec
23+
import org.apache.spark.sql.comet.execution.shuffle.CometShuffleExchangeExec
24+
import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
25+
import org.apache.spark.sql.internal.SQLConf
26+
27+
import org.apache.comet.CometConf
28+
2229
case class AggExprConfig(
2330
name: String,
2431
query: String,
@@ -33,6 +40,87 @@ case class AggExprConfig(
3340
*/
3441
object CometAggregateExpressionBenchmark extends CometBenchmarkBase {
3542

43+
/**
44+
* Run with `--wide-decimal-shuffle` on both revisions to measure the cost of routing wide
45+
* decimal hash keys through Spark's partition assignments. Fixture generation, result checks,
46+
* and plan reporting are not timed. The benchmark session uses local[1]. Add `--reverse` to
47+
* reverse the order of the native and auto cases, or `--validate-only` to check the fixture,
48+
* results and plans without collecting timings.
49+
*/
50+
private def wideDecimalShuffleBenchmark(reverse: Boolean, validateOnly: Boolean): Unit = {
51+
val rows = 1024 * 1024
52+
val groups = 10000
53+
val partitions = 4
54+
val filePartitionBytes = 16 * 1024 * 1024
55+
withSQLConf(
56+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
57+
SQLConf.SHUFFLE_PARTITIONS.key -> partitions.toString,
58+
// Keep each file in one split, and prevent Spark from combining files into one task.
59+
SQLConf.FILES_MAX_PARTITION_BYTES.key -> filePartitionBytes.toString,
60+
SQLConf.FILES_OPEN_COST_IN_BYTES.key -> filePartitionBytes.toString) {
61+
withTempPath { dir =>
62+
withTempTable("parquetV1Table") {
63+
prepareTable(
64+
dir,
65+
spark
66+
.range(0L, rows.toLong, 1L, partitions)
67+
.selectExpr(
68+
s"CAST(id % $groups AS DECIMAL(38, 2)) AS k",
69+
"CAST(id % 97 AS DECIMAL(20, 2)) AS v"))
70+
val query = "SELECT k, AVG(v) FROM parquetV1Table GROUP BY k"
71+
val expected = withSQLConf(CometConf.COMET_ENABLED.key -> "false") {
72+
val input = spark.table("parquetV1Table")
73+
assert(input.inputFiles.length == partitions)
74+
val inputPartitions = input.rdd.getNumPartitions
75+
assert(
76+
inputPartitions == partitions,
77+
s"Expected $partitions inputs, got $inputPartitions")
78+
val df = spark.sql(query)
79+
val result = df.collect()
80+
assert(result.length == groups)
81+
println(
82+
"Wide-decimal Spark baseline plan:\n" + df.queryExecution.executedPlan.treeString)
83+
result.toSet
84+
}
85+
val modes = Seq("native", "auto")
86+
for (mode <- (if (reverse) modes.reverse else modes)) {
87+
val configs = Map(
88+
CometConf.COMET_SHUFFLE_ENABLED.key -> "true",
89+
CometConf.COMET_SHUFFLE_MODE.key -> mode)
90+
withSQLConf(
91+
(configs.toSeq ++ Seq(
92+
CometConf.COMET_ENABLED.key -> "true",
93+
CometConf.COMET_EXEC_ENABLED.key -> "true")): _*) {
94+
val df = spark.sql(query)
95+
val result = df.collect()
96+
assert(result.length == groups)
97+
assert(result.toSet == expected, s"Wide-decimal results differ from Spark: $mode")
98+
val plan = df.queryExecution.executedPlan
99+
val cometShuffles = plan.collect { case exchange: CometShuffleExchangeExec =>
100+
exchange.shuffleType.toString
101+
}
102+
val sparkShuffles = plan.collect { case _: ShuffleExchangeExec => 1 }.sum
103+
val nativeAggregates = plan.collect { case _: CometHashAggregateExec => 1 }.sum
104+
assert(cometShuffles.size + sparkShuffles == 1)
105+
// Report the actual routing, including the expected Spark aggregate fallback in
106+
// native-only mode. A faster result does not justify an incompatible hash key.
107+
println(s"Wide-decimal shuffle mode=$mode, rows=$rows, groups=$groups, " +
108+
s"inputPartitions=$partitions, shufflePartitions=$partitions, " +
109+
s"filePartitionBytes=$filePartitionBytes, fileOpenCostBytes=$filePartitionBytes, " +
110+
s"master=${spark.sparkContext.master}, sparkVersion=${spark.version}, " +
111+
s"resultMatchesSpark=true, cometShuffles=${cometShuffles.mkString(",")}, " +
112+
s"sparkShuffles=$sparkShuffles, nativeAggregates=$nativeAggregates")
113+
println(plan.treeString)
114+
}
115+
if (!validateOnly) {
116+
runExpressionBenchmark(s"wide_decimal_shuffle_$mode", rows, query, configs)
117+
}
118+
}
119+
}
120+
}
121+
}
122+
}
123+
36124
private val basicAggregates = List(
37125
AggExprConfig("count", "SELECT COUNT(*) FROM parquetV1Table GROUP BY grp"),
38126
AggExprConfig("count_col", "SELECT COUNT(c_int) FROM parquetV1Table GROUP BY grp"),
@@ -169,6 +257,12 @@ object CometAggregateExpressionBenchmark extends CometBenchmarkBase {
169257
"SELECT approx_percentile(c_double, 0.5) FROM parquetV1Table GROUP BY high_card_grp"))
170258

171259
override def runCometBenchmark(mainArgs: Array[String]): Unit = {
260+
if (mainArgs.contains("--wide-decimal-shuffle")) {
261+
wideDecimalShuffleBenchmark(
262+
mainArgs.contains("--reverse"),
263+
mainArgs.contains("--validate-only"))
264+
return
265+
}
172266
val values = 1024 * 1024
173267

174268
runBenchmarkWithTable("Aggregate function benchmarks", values) { v =>

0 commit comments

Comments
 (0)