Skip to content

Commit 8514d68

Browse files
committed
test: fix the benchmark's hash key and drop a duplicate shuffle test
1 parent 3c5040e commit 8514d68

2 files changed

Lines changed: 61 additions & 30 deletions

File tree

‎spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -948,7 +948,8 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar
948948
// verdict must not gate the exchange (#5971). Turning the dispatcher off is only a cheap and
949949
// stable way to obtain an unserializable partition key; any expression without a serde reaches
950950
// the same gate, and unlike the strictFloatingPoint cases above nothing here depends on strict
951-
// mode.
951+
// mode. The hash counterpart is the partition assignment test below, which covers the same
952+
// routing and additionally compares where the rows land.
952953
test("range partitioning on an unserializable expression uses columnar shuffle") {
953954
withSQLConf(CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "false") {
954955
withParquetTable((0 until 20).map(i => (i, i.toString)), "range_udf_tbl") {
@@ -959,16 +960,6 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar
959960
}
960961
}
961962

962-
test("hash partitioning on an unserializable expression uses columnar shuffle") {
963-
withSQLConf(CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "false") {
964-
withParquetTable((0 until 20).map(i => (i, i.toString)), "hash_udf_tbl") {
965-
val bump = udf((x: Int) => x + 1)
966-
val df = sql("SELECT _1, _2 FROM hash_udf_tbl").repartition(4, bump(col("_1")))
967-
checkShuffleAnswer(df, 1)
968-
}
969-
}
970-
}
971-
972963
test("collation introduced above the scan still falls back to Spark's shuffle") {
973964
assume(isSpark40Plus, "string collation requires Spark 4.0+")
974965
withParquetTable((0 until 20).map(i => (i, (i % 4).toString)), "coll_above_tbl") {
@@ -992,6 +983,7 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar
992983
*/
993984
private def checkPartitionAssignmentMatchesSpark(df: => DataFrame, clue: String): Unit = {
994985
checkCometExchange(df, 1, false)
986+
checkSparkAnswer(df)
995987

996988
def pids: Array[(Int, Int)] =
997989
df.select(col("_1"), spark_partition_id().as("pid"))

‎spark/src/test/scala/org/apache/spark/sql/benchmark/CometUnserializableShuffleKeyBenchmark.scala‎

Lines changed: 58 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,14 @@
2020
package org.apache.spark.sql.benchmark
2121

2222
import org.apache.spark.benchmark.Benchmark
23-
import org.apache.spark.sql.comet.execution.shuffle.CometShuffleExchangeExec
23+
import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, RangePartitioning}
2424
import org.apache.spark.sql.execution.SparkPlan
2525
import org.apache.spark.sql.execution.adaptive.QueryStageExec
26-
import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
26+
import org.apache.spark.sql.execution.exchange.ShuffleExchangeLike
2727
import org.apache.spark.sql.internal.SQLConf
2828

2929
import org.apache.comet.CometConf
30+
import org.apache.comet.serde.QueryPlanSerde
3031

3132
/**
3233
* What a range-partitioned exchange costs when its sort key has no Comet serde, which is the
@@ -58,8 +59,16 @@ object CometUnserializableShuffleKeyBenchmark extends CometBenchmarkBase {
5859

5960
/** Range partitioning, as in the reported shape, and hash partitioning for comparison. */
6061
private val rangeQuery = "SELECT * FROM parquetV1Table ORDER BY bump(a), b"
61-
private val hashQuery =
62-
"SELECT max(b) FROM parquetV1Table GROUP BY bump(a) ORDER BY 1 LIMIT 10"
62+
// `GROUP BY bump(a)` would not do: Spark hoists the call into the partial aggregate and
63+
// partitions on the resulting attribute, which serializes fine, so that exchange never meets
64+
// the check this PR removes. `DISTRIBUTE BY` keeps the call in the partitioning itself.
65+
private val hashQuery = "SELECT * FROM parquetV1Table DISTRIBUTE BY bump(a)"
66+
67+
// The same exchange, but with an operator above it that can consume Arrow batches directly.
68+
// Every shape measured so far hands the exchange's output to Spark, which has to turn the
69+
// batches back into rows; this is the shape where the columnar path could pay for itself.
70+
private val hashCometConsumerQuery =
71+
"SELECT max(b) FROM (SELECT * FROM parquetV1Table DISTRIBUTE BY bump(a)) t"
6372

6473
override def runCometBenchmark(mainArgs: Array[String]): Unit = {
6574
spark.udf.register("bump", (x: Int) => x + 1)
@@ -76,6 +85,7 @@ object CometUnserializableShuffleKeyBenchmark extends CometBenchmarkBase {
7685
checkPlans()
7786
run("range partitioning on a key with no serde", rangeQuery)
7887
run("hash partitioning on a key with no serde", hashQuery)
88+
run("hash partitioning, Comet operator above the exchange", hashCometConsumerQuery)
7989
}
8090
}
8191
}
@@ -112,30 +122,59 @@ object CometUnserializableShuffleKeyBenchmark extends CometBenchmarkBase {
112122
CometConf.COMET_SHUFFLE_MODE.key -> "jvm",
113123
CometConf.COMET_SHUFFLE_CONVERT_FROM_SPARK_PLAN_ENABLED.key -> "true")
114124

125+
private val queries =
126+
Seq(rangeQuery, hashQuery, hashCometConsumerQuery)
127+
128+
/** Every shuffle exchange in `plan`, including the ones inside query stages. */
129+
private def exchangesOf(plan: SparkPlan): Seq[ShuffleExchangeLike] = {
130+
val stagePlans = collect(plan) { case s: QueryStageExec => s.plan }
131+
(plan +: stagePlans).flatMap(p => collect(p) { case e: ShuffleExchangeLike => e })
132+
}
133+
134+
/**
135+
* Whether the check this benchmark is about would have rejected `e`: the same `exprToProto`
136+
* call the columnar path used to make. An exchange that passes it is routed by rules this PR
137+
* does not touch, so timing it would say nothing about the change.
138+
*/
139+
private def meetsRemovedCheck(e: ShuffleExchangeLike): Boolean = {
140+
val inputs = e.child.output
141+
val keyExprs = e.outputPartitioning match {
142+
case HashPartitioning(expressions, _) => expressions
143+
case RangePartitioning(orderings, _) => orderings
144+
case _ => Nil
145+
}
146+
keyExprs.exists(expr => QueryPlanSerde.exprToProto(expr, inputs).isEmpty)
147+
}
148+
115149
/**
116-
* Fails unless each case really produces the exchange it is named for. Without this the
117-
* benchmark could silently time the same plan twice.
150+
* Fails unless each case really produces the exchange it is named for, and unless the
151+
* exchange's key is one the removed check rejected. The exchange's node name alone does not
152+
* establish the second: `GROUP BY f(x)` yields a `CometColumnarExchange` whose key is a plain
153+
* attribute, which the old check admitted, so timing it would measure Comet's columnar shuffle
154+
* against Spark's rather than anything this PR changes.
118155
*/
119156
private def checkPlans(): Unit = {
120-
for (query <- Seq(rangeQuery, hashQuery); c <- cases) {
157+
for (query <- queries; c <- cases) {
121158
withSQLConf(configs(c): _*) {
122159
val df = spark.sql(query)
123160
df.noop()
124161
val plan = stripAQEPlan(df.queryExecution.executedPlan)
125-
val stagePlans = collect(plan) { case s: QueryStageExec => s.plan }
126-
val exchanges = (plan +: stagePlans).flatMap { p: SparkPlan =>
127-
collect(p) {
128-
case e: CometShuffleExchangeExec => e.nodeName
129-
case e: ShuffleExchangeExec => e.nodeName
130-
}
131-
}
132-
val comet = exchanges.count(_.startsWith("CometColumnarExchange"))
133-
val spark0 = exchanges.count(_ == "Exchange")
134-
val ok = if (c.cometShuffle) comet >= 1 else comet == 0 && spark0 >= 1
135-
if (!ok) {
162+
val exchanges = exchangesOf(plan)
163+
val names = exchanges.map(_.nodeName)
164+
val comet = names.count(_.startsWith("CometColumnarExchange"))
165+
val sparkShuffles = names.count(_ == "Exchange")
166+
167+
if (if (c.cometShuffle) comet < 1 else comet > 0 || sparkShuffles < 1) {
136168
throw new IllegalStateException(
137169
s"${c.label}: expected ${if (c.cometShuffle) "a Comet columnar" else "a Spark"} " +
138-
s"exchange but found $exchanges in:\n$plan")
170+
s"exchange but found $names in:\n$plan")
171+
}
172+
173+
if (!exchanges.exists(meetsRemovedCheck)) {
174+
throw new IllegalStateException(
175+
s"${c.label}: no exchange here has a partition key the removed check rejected, so " +
176+
s"this case does not exercise the change. Partitionings: " +
177+
s"${exchanges.map(_.outputPartitioning)} in:\n$plan")
139178
}
140179
}
141180
}

0 commit comments

Comments
 (0)