Skip to content

Commit 0107cbb

Browse files
committed
test: isolate and assert each join in the skewed rebalance test
Drop the aggregate above the join, which rejected the split on its own, and join the skewed side to a one-row-per-key side so the output stays small. Force the shuffled hash join explicitly and assert that the Comet join operator is in the plan. Each case fails when only that join's distribution delegation is removed.
1 parent ef4199b commit 0107cbb

1 file changed

Lines changed: 17 additions & 6 deletions

File tree

‎spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4010,8 +4010,13 @@ class CometExecSuite extends CometTestBase {
40104010
.parquet(dir.getCanonicalPath)
40114011

40124012
withParquetTable(dir.getCanonicalPath, "rebalance_t1") {
4013-
def checkNotSplit(query: String): Unit = {
4013+
def checkNotSplit(query: String, expected: Class[_]*): Unit = {
40144014
val (_, cometPlan) = checkSparkAnswer(query)
4015+
expected.foreach { cls =>
4016+
assert(
4017+
collect(cometPlan) { case op if cls.isInstance(op) => op }.nonEmpty,
4018+
cometPlan)
4019+
}
40154020
val skewedReads = collect(cometPlan) {
40164021
case r: AQEShuffleReadExec if r.hasSkewedPartition => r
40174022
}
@@ -4027,11 +4032,17 @@ class CometExecSuite extends CometTestBase {
40274032
s"SELECT k, sum(w) FROM (SELECT k, v + 1 AS w FROM $rebalanced WHERE v > 5) GROUP BY k")
40284033
checkNotSplit(window)
40294034
checkNotSplit(s"SELECT * FROM ($window) WHERE rn <= 2")
4030-
Seq("true", "false").foreach { preferSmj =>
4031-
withSQLConf(SQLConf.PREFER_SORTMERGEJOIN.key -> preferSmj) {
4032-
checkNotSplit(
4033-
s"SELECT a.k, count(*) FROM $rebalanced a JOIN $rebalanced b ON a.k = b.k " +
4034-
"GROUP BY a.k")
4035+
4036+
// The join is the only operator that needs the clustering, so it alone has to stop the
4037+
// split. The right side has one row per key, which keeps the join's output small.
4038+
withParquetTable((0 until 1000).map(i => (i.toLong, i.toLong)), "rebalance_t2") {
4039+
val rebalanced2 = "(SELECT /*+ REBALANCE(k) */ _1 AS k, _2 AS w FROM rebalance_t2)"
4040+
val join = s"SELECT a.k, a.v, b.w FROM $rebalanced a JOIN $rebalanced2 b ON a.k = b.k"
4041+
checkNotSplit(join, classOf[CometSortMergeJoinExec])
4042+
withSQLConf(
4043+
SQLConf.PREFER_SORTMERGEJOIN.key -> "false",
4044+
"spark.sql.join.forceApplyShuffledHashJoin" -> "true") {
4045+
checkNotSplit(join, classOf[CometHashJoinExec])
40354046
}
40364047
}
40374048
}

0 commit comments

Comments
 (0)