Skip to content

Commit bf2f258

Browse files
committed
feat: remove join type restrictions from RewriteJoin
GHJ supports all join types with either build side, so remove the canBuildShuffledHashJoinLeft/Right checks and the LeftAnti/LeftSemi BuildRight guard (apache#457, apache#2667).
1 parent 0155c75 commit bf2f258

1 file changed

Lines changed: 21 additions & 43 deletions

File tree

‎spark/src/main/scala/org/apache/comet/rules/RewriteJoin.scala‎

Lines changed: 21 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,7 @@
1919

2020
package org.apache.comet.rules
2121

22-
import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide, JoinSelectionHelper}
23-
import org.apache.spark.sql.catalyst.plans.{LeftAnti, LeftSemi}
22+
import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide}
2423
import org.apache.spark.sql.catalyst.plans.logical.Join
2524
import org.apache.spark.sql.execution.{SortExec, SparkPlan}
2625
import org.apache.spark.sql.execution.joins.{ShuffledHashJoinExec, SortMergeJoinExec}
@@ -33,21 +32,10 @@ import org.apache.comet.CometSparkSessionExtensions.withInfo
3332
*
3433
* This rule replaces [[SortMergeJoinExec]] with [[ShuffledHashJoinExec]].
3534
*/
36-
object RewriteJoin extends JoinSelectionHelper {
35+
object RewriteJoin {
3736

38-
private def getSmjBuildSide(join: SortMergeJoinExec): Option[BuildSide] = {
39-
val leftBuildable = canBuildShuffledHashJoinLeft(join.joinType)
40-
val rightBuildable = canBuildShuffledHashJoinRight(join.joinType)
41-
if (!leftBuildable && !rightBuildable) {
42-
return None
43-
}
44-
if (!leftBuildable) {
45-
return Some(BuildRight)
46-
}
47-
if (!rightBuildable) {
48-
return Some(BuildLeft)
49-
}
50-
val side = join.logicalLink
37+
private def getSmjBuildSide(join: SortMergeJoinExec): BuildSide = {
38+
join.logicalLink
5139
.flatMap {
5240
case join: Join => Some(getOptimalBuildSide(join))
5341
case _ => None
@@ -57,7 +45,6 @@ object RewriteJoin extends JoinSelectionHelper {
5745
// then we always choose left as build side.
5846
BuildLeft
5947
}
60-
Some(side)
6148
}
6249

6350
private def removeSort(plan: SparkPlan) = plan match {
@@ -89,32 +76,23 @@ object RewriteJoin extends JoinSelectionHelper {
8976

9077
def rewrite(plan: SparkPlan): SparkPlan = plan match {
9178
case smj: SortMergeJoinExec =>
92-
getSmjBuildSide(smj) match {
93-
case Some(BuildRight) if smj.joinType == LeftAnti || smj.joinType == LeftSemi =>
94-
// LeftAnti https://github.com/apache/datafusion-comet/issues/457
95-
// LeftSemi https://github.com/apache/datafusion-comet/issues/2667
96-
withInfo(
97-
smj,
98-
"Cannot rewrite SortMergeJoin to HashJoin: " +
99-
s"BuildRight with ${smj.joinType} is not supported")
100-
plan
101-
case Some(buildSide) if !buildSideSmallEnough(smj, buildSide) =>
102-
withInfo(
103-
smj,
104-
"Cannot rewrite SortMergeJoin to HashJoin: " +
105-
"build side exceeds spark.comet.exec.replaceSortMergeJoin.maxBuildSize")
106-
plan
107-
case Some(buildSide) =>
108-
ShuffledHashJoinExec(
109-
smj.leftKeys,
110-
smj.rightKeys,
111-
smj.joinType,
112-
buildSide,
113-
smj.condition,
114-
removeSort(smj.left),
115-
removeSort(smj.right),
116-
smj.isSkewJoin)
117-
case _ => plan
79+
val buildSide = getSmjBuildSide(smj)
80+
if (!buildSideSmallEnough(smj, buildSide)) {
81+
withInfo(
82+
smj,
83+
"Cannot rewrite SortMergeJoin to HashJoin: " +
84+
"build side exceeds spark.comet.exec.replaceSortMergeJoin.maxBuildSize")
85+
plan
86+
} else {
87+
ShuffledHashJoinExec(
88+
smj.leftKeys,
89+
smj.rightKeys,
90+
smj.joinType,
91+
buildSide,
92+
smj.condition,
93+
removeSort(smj.left),
94+
removeSort(smj.right),
95+
smj.isSkewJoin)
11896
}
11997
case _ => plan
12098
}

0 commit comments

Comments
 (0)