Skip to content

Commit 60730d8

Browse files
committed
test: cover a window over a join the hash join rewrite reordered
A window partitioned by the join key gets no sort of its own, because the sort-merge join's output ordering already satisfies it. Once RewriteJoin turns that join into a hash join, the window needs the ordering restored, or the native window fails its partition-ordering assertion. The test runs with AQE on and off and checks the join became a hash join.
1 parent 18bdefa commit 60730d8

1 file changed

Lines changed: 23 additions & 1 deletion

File tree

‎spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ import org.apache.spark.sql.catalyst.expressions.{And, AttributeReference, Dynam
3333
import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight}
3434
import org.apache.spark.sql.catalyst.plans.Inner
3535
import org.apache.spark.sql.catalyst.plans.logical.Join
36-
import org.apache.spark.sql.comet.{CometBroadcastExchangeExec, CometBroadcastHashJoinExec, CometBroadcastNestedLoopJoinExec, CometFilterExec, CometHashJoinExec, CometNativeScanExec, CometSortExec, CometSortMergeJoinExec, CometUnionExec}
36+
import org.apache.spark.sql.comet.{CometBroadcastExchangeExec, CometBroadcastHashJoinExec, CometBroadcastNestedLoopJoinExec, CometFilterExec, CometHashJoinExec, CometNativeScanExec, CometSortExec, CometSortMergeJoinExec, CometUnionExec, CometWindowExec}
3737
import org.apache.spark.sql.execution.{ColumnarToRowTransition, InputAdapter, LocalTableScanExec, SortExec, SparkPlan, WholeStageCodegenExec}
3838
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AQEShuffleReadExec, QueryStageExec}
3939
import org.apache.spark.sql.execution.datasources.SchemaColumnConvertNotSupportedException
@@ -1197,6 +1197,28 @@ class CometJoinSuite extends CometTestBase {
11971197
}
11981198
}
11991199

1200+
// A window partitioned by the join key gets no sort of its own, since the sort-merge join's
1201+
// output ordering satisfies it, so the window needs that ordering restored too.
1202+
for (adaptive <- Seq(false, true)) {
1203+
test(s"forceShuffledHashJoin restores ordering for a window over the join, AQE=$adaptive") {
1204+
withSQLConf(
1205+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> adaptive.toString,
1206+
SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
1207+
SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
1208+
SQLConf.SHUFFLE_PARTITIONS.key -> "2",
1209+
CometConf.COMET_FORCE_SHJ.key -> "true",
1210+
CometConf.COMET_FORCE_SHJ_MAX_BUILD_SIZE.key -> "-1") {
1211+
withChainedJoinTables(midRows = 3000, midKeys = 100) {
1212+
val (_, cometPlan) = checkSparkAnswer(
1213+
"SELECT big._1, big._2, small._2, count(*) OVER (PARTITION BY big._1) " +
1214+
"FROM big JOIN small ON big._1 = small._1")
1215+
assert(collect(cometPlan) { case j: CometHashJoinExec => j }.nonEmpty, cometPlan)
1216+
assert(collect(cometPlan) { case w: CometWindowExec => w }.nonEmpty, cometPlan)
1217+
}
1218+
}
1219+
}
1220+
}
1221+
12001222
test("BroadcastHashJoin with LeftAnti and NOT IN subquery (null-aware)") {
12011223
withSQLConf(
12021224
SQLConf.PREFER_SORTMERGEJOIN.key -> "false",

0 commit comments

Comments
 (0)