Skip to content

Commit fa70aff

Browse files
committed
test: check both routing properties on the same exchange
1 parent ce25516 commit fa70aff

1 file changed

Lines changed: 20 additions & 15 deletions

File tree

‎spark/src/test/scala/org/apache/spark/sql/benchmark/CometUnserializableShuffleKeyBenchmark.scala‎

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -147,11 +147,15 @@ object CometUnserializableShuffleKeyBenchmark extends CometBenchmarkBase {
147147
}
148148

149149
/**
150-
* Fails unless each case really produces the exchange it is named for, and unless the
151-
* exchange's key is one the removed check rejected. The exchange's node name alone does not
152-
* establish the second: `GROUP BY f(x)` yields a `CometColumnarExchange` whose key is a plain
153-
* attribute, which the old check admitted, so timing it would measure Comet's columnar shuffle
154-
* against Spark's rather than anything this PR changes.
150+
* Fails unless the exchange this benchmark is about -- the one whose partition key the removed
151+
* check rejected -- is routed the way the case is named for. Both properties have to hold of
152+
* the same exchange: the aggregate query has a second exchange for the final aggregate, so
153+
* asking separately whether any exchange is columnar and whether any exchange has a rejected
154+
* key would pass even if the two were different nodes and the measured one fell back.
155+
*
156+
* The key check matters on its own, too. `GROUP BY f(x)` yields a `CometColumnarExchange` whose
157+
* key is a plain attribute that the old check admitted, so timing it would compare Comet's
158+
* columnar shuffle with Spark's rather than anything this PR changes.
155159
*/
156160
private def checkPlans(): Unit = {
157161
for (query <- queries; c <- cases) {
@@ -160,22 +164,23 @@ object CometUnserializableShuffleKeyBenchmark extends CometBenchmarkBase {
160164
df.noop()
161165
val plan = stripAQEPlan(df.queryExecution.executedPlan)
162166
val exchanges = exchangesOf(plan)
163-
val names = exchanges.map(_.nodeName)
164-
val comet = names.count(_.startsWith("CometColumnarExchange"))
165-
val sparkShuffles = names.count(_ == "Exchange")
167+
val gated = exchanges.filter(meetsRemovedCheck)
166168

167-
if (if (c.cometShuffle) comet < 1 else comet > 0 || sparkShuffles < 1) {
168-
throw new IllegalStateException(
169-
s"${c.label}: expected ${if (c.cometShuffle) "a Comet columnar" else "a Spark"} " +
170-
s"exchange but found $names in:\n$plan")
171-
}
172-
173-
if (!exchanges.exists(meetsRemovedCheck)) {
169+
if (gated.isEmpty) {
174170
throw new IllegalStateException(
175171
s"${c.label}: no exchange here has a partition key the removed check rejected, so " +
176172
"this case does not exercise the change. Partitionings: " +
177173
s"${exchanges.map(_.outputPartitioning)} in:\n$plan")
178174
}
175+
176+
val expected = if (c.cometShuffle) "CometColumnarExchange" else "Exchange"
177+
val misrouted = gated.filterNot(_.nodeName == expected)
178+
if (misrouted.nonEmpty) {
179+
throw new IllegalStateException(
180+
s"${c.label}: the exchange whose key the removed check rejected is " +
181+
s"${misrouted.map(_.nodeName)}, not $expected, so this case does not measure " +
182+
s"what it claims. All exchanges: ${exchanges.map(_.nodeName)} in:\n$plan")
183+
}
179184
}
180185
}
181186
}

0 commit comments

Comments
 (0)