1919
2020package org .apache .spark .sql .benchmark
2121
22+ import org .apache .spark .sql .comet .CometHashAggregateExec
23+ import org .apache .spark .sql .comet .execution .shuffle .CometShuffleExchangeExec
24+ import org .apache .spark .sql .execution .exchange .ShuffleExchangeExec
25+ import org .apache .spark .sql .internal .SQLConf
26+
27+ import org .apache .comet .CometConf
28+
2229case class AggExprConfig (
2330 name : String ,
2431 query : String ,
@@ -33,6 +40,87 @@ case class AggExprConfig(
3340 */
3441object CometAggregateExpressionBenchmark extends CometBenchmarkBase {
3542
43+ /**
44+ * Run with `--wide-decimal-shuffle` on both revisions to measure the cost of routing wide
45+ * decimal hash keys through Spark's partition assignments. Fixture generation, result checks,
46+ * and plan reporting are not timed. The benchmark session uses local[1]. Add `--reverse` to
47+ * reverse the order of the native and auto cases, or `--validate-only` to check the fixture,
48+ * results and plans without collecting timings.
49+ */
50+ private def wideDecimalShuffleBenchmark (reverse : Boolean , validateOnly : Boolean ): Unit = {
51+ val rows = 1024 * 1024
52+ val groups = 10000
53+ val partitions = 4
54+ val filePartitionBytes = 16 * 1024 * 1024
55+ withSQLConf(
56+ SQLConf .ADAPTIVE_EXECUTION_ENABLED .key -> " false" ,
57+ SQLConf .SHUFFLE_PARTITIONS .key -> partitions.toString,
58+ // Keep each file in one split, and prevent Spark from combining files into one task.
59+ SQLConf .FILES_MAX_PARTITION_BYTES .key -> filePartitionBytes.toString,
60+ SQLConf .FILES_OPEN_COST_IN_BYTES .key -> filePartitionBytes.toString) {
61+ withTempPath { dir =>
62+ withTempTable(" parquetV1Table" ) {
63+ prepareTable(
64+ dir,
65+ spark
66+ .range(0L , rows.toLong, 1L , partitions)
67+ .selectExpr(
68+ s " CAST(id % $groups AS DECIMAL(38, 2)) AS k " ,
69+ " CAST(id % 97 AS DECIMAL(20, 2)) AS v" ))
70+ val query = " SELECT k, AVG(v) FROM parquetV1Table GROUP BY k"
71+ val expected = withSQLConf(CometConf .COMET_ENABLED .key -> " false" ) {
72+ val input = spark.table(" parquetV1Table" )
73+ assert(input.inputFiles.length == partitions)
74+ val inputPartitions = input.rdd.getNumPartitions
75+ assert(
76+ inputPartitions == partitions,
77+ s " Expected $partitions inputs, got $inputPartitions" )
78+ val df = spark.sql(query)
79+ val result = df.collect()
80+ assert(result.length == groups)
81+ println(
82+ " Wide-decimal Spark baseline plan:\n " + df.queryExecution.executedPlan.treeString)
83+ result.toSet
84+ }
85+ val modes = Seq (" native" , " auto" )
86+ for (mode <- (if (reverse) modes.reverse else modes)) {
87+ val configs = Map (
88+ CometConf .COMET_SHUFFLE_ENABLED .key -> " true" ,
89+ CometConf .COMET_SHUFFLE_MODE .key -> mode)
90+ withSQLConf(
91+ (configs.toSeq ++ Seq (
92+ CometConf .COMET_ENABLED .key -> " true" ,
93+ CometConf .COMET_EXEC_ENABLED .key -> " true" )): _* ) {
94+ val df = spark.sql(query)
95+ val result = df.collect()
96+ assert(result.length == groups)
97+ assert(result.toSet == expected, s " Wide-decimal results differ from Spark: $mode" )
98+ val plan = df.queryExecution.executedPlan
99+ val cometShuffles = plan.collect { case exchange : CometShuffleExchangeExec =>
100+ exchange.shuffleType.toString
101+ }
102+ val sparkShuffles = plan.collect { case _ : ShuffleExchangeExec => 1 }.sum
103+ val nativeAggregates = plan.collect { case _ : CometHashAggregateExec => 1 }.sum
104+ assert(cometShuffles.size + sparkShuffles == 1 )
105+ // Report the actual routing, including the expected Spark aggregate fallback in
106+ // native-only mode. A faster result does not justify an incompatible hash key.
107+ println(s " Wide-decimal shuffle mode= $mode, rows= $rows, groups= $groups, " +
108+ s " inputPartitions= $partitions, shufflePartitions= $partitions, " +
109+ s " filePartitionBytes= $filePartitionBytes, fileOpenCostBytes= $filePartitionBytes, " +
110+ s " master= ${spark.sparkContext.master}, sparkVersion= ${spark.version}, " +
111+ s " resultMatchesSpark=true, cometShuffles= ${cometShuffles.mkString(" ," )}, " +
112+ s " sparkShuffles= $sparkShuffles, nativeAggregates= $nativeAggregates" )
113+ println(plan.treeString)
114+ }
115+ if (! validateOnly) {
116+ runExpressionBenchmark(s " wide_decimal_shuffle_ $mode" , rows, query, configs)
117+ }
118+ }
119+ }
120+ }
121+ }
122+ }
123+
36124 private val basicAggregates = List (
37125 AggExprConfig (" count" , " SELECT COUNT(*) FROM parquetV1Table GROUP BY grp" ),
38126 AggExprConfig (" count_col" , " SELECT COUNT(c_int) FROM parquetV1Table GROUP BY grp" ),
@@ -169,6 +257,12 @@ object CometAggregateExpressionBenchmark extends CometBenchmarkBase {
169257 " SELECT approx_percentile(c_double, 0.5) FROM parquetV1Table GROUP BY high_card_grp" ))
170258
171259 override def runCometBenchmark (mainArgs : Array [String ]): Unit = {
260+ if (mainArgs.contains(" --wide-decimal-shuffle" )) {
261+ wideDecimalShuffleBenchmark(
262+ mainArgs.contains(" --reverse" ),
263+ mainArgs.contains(" --validate-only" ))
264+ return
265+ }
172266 val values = 1024 * 1024
173267
174268 runBenchmarkWithTable(" Aggregate function benchmarks" , values) { v =>
0 commit comments