Skip to content

Commit 2241ccb

Browse files
committed
docs: expand revertRedundantColumnarShuffle docstring and add coverage tests
Document that correctness requires running in preColumnarTransitions and that SKIP_COMET_SHUFFLE_TAG guards against re-entry from both the AQE QueryStagePrepRule pass and the ColumnarRule pass. Add two CometExecRuleSuite cases asserting the revert does not fire when the revert config is disabled and when both aggregates convert to Comet native.
1 parent f407226 commit 2241ccb

2 files changed

Lines changed: 62 additions & 2 deletions

File tree

‎spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,8 +120,16 @@ case class CometExecRule(session: SparkSession)
120120
* main conversion already decided to keep both aggregates JVM - we never create the dangerous
121121
* mixed mode where a Comet partial feeds a JVM final (see issue #1389).
122122
*
123-
* Also tag the reverted shuffle so AQE stage-isolated re-planning does not convert it back to a
124-
* Comet shuffle when the outer aggregate context is no longer visible.
123+
* Correctness depends on running as part of `preColumnarTransitions`: if the revert ran after
124+
* Spark inserted `ColumnarToRowExec` between the aggregate and the columnar shuffle, the
125+
* pattern would no longer match (the shuffle would be separated from the aggregate by the
126+
* transition) and the unnecessary conversion could not be eliminated.
127+
*
128+
* The reverted shuffle is tagged with `SKIP_COMET_SHUFFLE_TAG` so both the AQE
129+
* `QueryStagePrepRule` pass and the `ColumnarRule` `preColumnarTransitions` pass leave it alone
130+
* on re-entry - AQE in particular re-runs the rule on each stage in isolation, where the outer
131+
* aggregate context is no longer visible and the shuffle would otherwise be re-wrapped as a
132+
* Comet columnar shuffle.
125133
*/
126134
private def revertRedundantColumnarShuffle(plan: SparkPlan): SparkPlan = {
127135
def isAggregate(p: SparkPlan): Boolean =

‎spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -262,4 +262,56 @@ class CometExecRuleSuite extends CometTestBase {
262262
}
263263
}
264264

265+
test("CometExecRule should not revert columnar shuffle when the revert config is disabled") {
266+
withTempView("test_data") {
267+
createTestDataFrame.createOrReplaceTempView("test_data")
268+
269+
val sparkPlan =
270+
createSparkPlan(spark, "SELECT COUNT(*), SUM(id) FROM test_data GROUP BY (id % 3)")
271+
272+
assert(countOperators(sparkPlan, classOf[ShuffleExchangeExec]) == 1)
273+
assert(countOperators(sparkPlan, classOf[HashAggregateExec]) == 2)
274+
275+
// Both aggregates fall back to JVM as in the prior test, but the revert optimization is
276+
// disabled, so the shuffle should still be wrapped in CometColumnarShuffle.
277+
withSQLConf(
278+
CometConf.COMET_EXEC_SHUFFLE_REVERT_REDUNDANT_COLUMNAR_ENABLED.key -> "false",
279+
CometConf.COMET_ENABLE_PARTIAL_HASH_AGGREGATE.key -> "false",
280+
CometConf.COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED.key -> "true") {
281+
val transformedPlan = applyCometExecRule(sparkPlan)
282+
283+
assert(countOperators(transformedPlan, classOf[HashAggregateExec]) == 2)
284+
assert(countOperators(transformedPlan, classOf[CometHashAggregateExec]) == 0)
285+
286+
assert(countOperators(transformedPlan, classOf[ShuffleExchangeExec]) == 0)
287+
assert(countOperators(transformedPlan, classOf[CometShuffleExchangeExec]) == 1)
288+
}
289+
}
290+
}
291+
292+
test("CometExecRule should not revert columnar shuffle when both aggregates go native") {
293+
withTempView("test_data") {
294+
createTestDataFrame.createOrReplaceTempView("test_data")
295+
296+
val sparkPlan =
297+
createSparkPlan(spark, "SELECT COUNT(*), SUM(id) FROM test_data GROUP BY (id % 3)")
298+
299+
assert(countOperators(sparkPlan, classOf[ShuffleExchangeExec]) == 1)
300+
assert(countOperators(sparkPlan, classOf[HashAggregateExec]) == 2)
301+
302+
// With default settings both aggregates convert to Comet native, so the shuffle between
303+
// them has a Comet consumer on both sides and must remain columnar - the revert must not
304+
// fire here.
305+
withSQLConf(CometConf.COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED.key -> "true") {
306+
val transformedPlan = applyCometExecRule(sparkPlan)
307+
308+
assert(countOperators(transformedPlan, classOf[HashAggregateExec]) == 0)
309+
assert(countOperators(transformedPlan, classOf[CometHashAggregateExec]) == 2)
310+
311+
assert(countOperators(transformedPlan, classOf[ShuffleExchangeExec]) == 0)
312+
assert(countOperators(transformedPlan, classOf[CometShuffleExchangeExec]) == 1)
313+
}
314+
}
315+
}
316+
265317
}

0 commit comments

Comments
 (0)