2020package org .apache .spark .sql .benchmark
2121
2222import org .apache .spark .benchmark .Benchmark
23- import org .apache .spark .sql .comet . execution . shuffle . CometShuffleExchangeExec
23+ import org .apache .spark .sql .catalyst . plans . physical .{ HashPartitioning , RangePartitioning }
2424import org .apache .spark .sql .execution .SparkPlan
2525import org .apache .spark .sql .execution .adaptive .QueryStageExec
26- import org .apache .spark .sql .execution .exchange .ShuffleExchangeExec
26+ import org .apache .spark .sql .execution .exchange .ShuffleExchangeLike
2727import org .apache .spark .sql .internal .SQLConf
2828
2929import org .apache .comet .CometConf
30+ import org .apache .comet .serde .QueryPlanSerde
3031
3132/**
3233 * What a range-partitioned exchange costs when its sort key has no Comet serde, which is the
@@ -58,8 +59,16 @@ object CometUnserializableShuffleKeyBenchmark extends CometBenchmarkBase {
5859
5960 /** Range partitioning, as in the reported shape, and hash partitioning for comparison. */
6061 private val rangeQuery = " SELECT * FROM parquetV1Table ORDER BY bump(a), b"
61- private val hashQuery =
62- " SELECT max(b) FROM parquetV1Table GROUP BY bump(a) ORDER BY 1 LIMIT 10"
62+ // `GROUP BY bump(a)` would not do: Spark hoists the call into the partial aggregate and
63+ // partitions on the resulting attribute, which serializes fine, so that exchange never meets
64+ // the check this PR removes. `DISTRIBUTE BY` keeps the call in the partitioning itself.
65+ private val hashQuery = " SELECT * FROM parquetV1Table DISTRIBUTE BY bump(a)"
66+
67+ // The same exchange, but with an operator above it that can consume Arrow batches directly.
68+ // Every shape measured so far hands the exchange's output to Spark, which has to turn the
69+ // batches back into rows; this is the shape where the columnar path could pay for itself.
70+ private val hashCometConsumerQuery =
71+ " SELECT max(b) FROM (SELECT * FROM parquetV1Table DISTRIBUTE BY bump(a)) t"
6372
6473 override def runCometBenchmark (mainArgs : Array [String ]): Unit = {
6574 spark.udf.register(" bump" , (x : Int ) => x + 1 )
@@ -76,6 +85,7 @@ object CometUnserializableShuffleKeyBenchmark extends CometBenchmarkBase {
7685 checkPlans()
7786 run(" range partitioning on a key with no serde" , rangeQuery)
7887 run(" hash partitioning on a key with no serde" , hashQuery)
88+ run(" hash partitioning, Comet operator above the exchange" , hashCometConsumerQuery)
7989 }
8090 }
8191 }
@@ -112,30 +122,59 @@ object CometUnserializableShuffleKeyBenchmark extends CometBenchmarkBase {
112122 CometConf .COMET_SHUFFLE_MODE .key -> " jvm" ,
113123 CometConf .COMET_SHUFFLE_CONVERT_FROM_SPARK_PLAN_ENABLED .key -> " true" )
114124
125+ private val queries =
126+ Seq (rangeQuery, hashQuery, hashCometConsumerQuery)
127+
128+ /** Every shuffle exchange in `plan`, including the ones inside query stages. */
129+ private def exchangesOf (plan : SparkPlan ): Seq [ShuffleExchangeLike ] = {
130+ val stagePlans = collect(plan) { case s : QueryStageExec => s.plan }
131+ (plan +: stagePlans).flatMap(p => collect(p) { case e : ShuffleExchangeLike => e })
132+ }
133+
134+ /**
135+ * Whether the check this benchmark is about would have rejected `e`: the same `exprToProto`
136+ * call the columnar path used to make. An exchange that passes it is routed by rules this PR
137+ * does not touch, so timing it would say nothing about the change.
138+ */
139+ private def meetsRemovedCheck (e : ShuffleExchangeLike ): Boolean = {
140+ val inputs = e.child.output
141+ val keyExprs = e.outputPartitioning match {
142+ case HashPartitioning (expressions, _) => expressions
143+ case RangePartitioning (orderings, _) => orderings
144+ case _ => Nil
145+ }
146+ keyExprs.exists(expr => QueryPlanSerde .exprToProto(expr, inputs).isEmpty)
147+ }
148+
115149 /**
116- * Fails unless each case really produces the exchange it is named for. Without this the
117- * benchmark could silently time the same plan twice.
150+ * Fails unless each case really produces the exchange it is named for, and unless the
151+ * exchange's key is one the removed check rejected. The exchange's node name alone does not
152+ * establish the second: `GROUP BY f(x)` yields a `CometColumnarExchange` whose key is a plain
153+ * attribute, which the old check admitted, so timing it would measure Comet's columnar shuffle
154+ * against Spark's rather than anything this PR changes.
118155 */
119156 private def checkPlans (): Unit = {
120- for (query <- Seq (rangeQuery, hashQuery) ; c <- cases) {
157+ for (query <- queries ; c <- cases) {
121158 withSQLConf(configs(c): _* ) {
122159 val df = spark.sql(query)
123160 df.noop()
124161 val plan = stripAQEPlan(df.queryExecution.executedPlan)
125- val stagePlans = collect(plan) { case s : QueryStageExec => s.plan }
126- val exchanges = (plan +: stagePlans).flatMap { p : SparkPlan =>
127- collect(p) {
128- case e : CometShuffleExchangeExec => e.nodeName
129- case e : ShuffleExchangeExec => e.nodeName
130- }
131- }
132- val comet = exchanges.count(_.startsWith(" CometColumnarExchange" ))
133- val spark0 = exchanges.count(_ == " Exchange" )
134- val ok = if (c.cometShuffle) comet >= 1 else comet == 0 && spark0 >= 1
135- if (! ok) {
162+ val exchanges = exchangesOf(plan)
163+ val names = exchanges.map(_.nodeName)
164+ val comet = names.count(_.startsWith(" CometColumnarExchange" ))
165+ val sparkShuffles = names.count(_ == " Exchange" )
166+
167+ if (if (c.cometShuffle) comet < 1 else comet > 0 || sparkShuffles < 1 ) {
136168 throw new IllegalStateException (
137169 s " ${c.label}: expected ${if (c.cometShuffle) " a Comet columnar" else " a Spark" } " +
138- s " exchange but found $exchanges in: \n $plan" )
170+ s " exchange but found $names in: \n $plan" )
171+ }
172+
173+ if (! exchanges.exists(meetsRemovedCheck)) {
174+ throw new IllegalStateException (
175+ s " ${c.label}: no exchange here has a partition key the removed check rejected, so " +
176+ s " this case does not exercise the change. Partitionings: " +
177+ s " ${exchanges.map(_.outputPartitioning)} in: \n $plan" )
139178 }
140179 }
141180 }
0 commit comments