1919
2020package org .apache .comet .rules
2121
22- import org .apache .spark .sql .catalyst .optimizer .{BuildLeft , BuildRight , BuildSide , JoinSelectionHelper }
23- import org .apache .spark .sql .catalyst .plans .{LeftAnti , LeftSemi }
22+ import org .apache .spark .sql .catalyst .optimizer .{BuildLeft , BuildRight , BuildSide }
2423import org .apache .spark .sql .catalyst .plans .logical .Join
2524import org .apache .spark .sql .execution .{SortExec , SparkPlan }
2625import org .apache .spark .sql .execution .joins .{ShuffledHashJoinExec , SortMergeJoinExec }
@@ -33,21 +32,10 @@ import org.apache.comet.CometSparkSessionExtensions.withInfo
3332 *
3433 * This rule replaces [[SortMergeJoinExec ]] with [[ShuffledHashJoinExec ]].
3534 */
36- object RewriteJoin extends JoinSelectionHelper {
35+ object RewriteJoin {
3736
38- private def getSmjBuildSide (join : SortMergeJoinExec ): Option [BuildSide ] = {
39- val leftBuildable = canBuildShuffledHashJoinLeft(join.joinType)
40- val rightBuildable = canBuildShuffledHashJoinRight(join.joinType)
41- if (! leftBuildable && ! rightBuildable) {
42- return None
43- }
44- if (! leftBuildable) {
45- return Some (BuildRight )
46- }
47- if (! rightBuildable) {
48- return Some (BuildLeft )
49- }
50- val side = join.logicalLink
37+ private def getSmjBuildSide (join : SortMergeJoinExec ): BuildSide = {
38+ join.logicalLink
5139 .flatMap {
5240 case join : Join => Some (getOptimalBuildSide(join))
5341 case _ => None
@@ -57,7 +45,6 @@ object RewriteJoin extends JoinSelectionHelper {
5745 // then we always choose left as build side.
5846 BuildLeft
5947 }
60- Some (side)
6148 }
6249
6350 private def removeSort (plan : SparkPlan ) = plan match {
@@ -89,32 +76,23 @@ object RewriteJoin extends JoinSelectionHelper {
8976
9077 def rewrite (plan : SparkPlan ): SparkPlan = plan match {
9178 case smj : SortMergeJoinExec =>
92- getSmjBuildSide(smj) match {
93- case Some (BuildRight ) if smj.joinType == LeftAnti || smj.joinType == LeftSemi =>
94- // LeftAnti https://github.com/apache/datafusion-comet/issues/457
95- // LeftSemi https://github.com/apache/datafusion-comet/issues/2667
96- withInfo(
97- smj,
98- " Cannot rewrite SortMergeJoin to HashJoin: " +
99- s " BuildRight with ${smj.joinType} is not supported " )
100- plan
101- case Some (buildSide) if ! buildSideSmallEnough(smj, buildSide) =>
102- withInfo(
103- smj,
104- " Cannot rewrite SortMergeJoin to HashJoin: " +
105- " build side exceeds spark.comet.exec.replaceSortMergeJoin.maxBuildSize" )
106- plan
107- case Some (buildSide) =>
108- ShuffledHashJoinExec (
109- smj.leftKeys,
110- smj.rightKeys,
111- smj.joinType,
112- buildSide,
113- smj.condition,
114- removeSort(smj.left),
115- removeSort(smj.right),
116- smj.isSkewJoin)
117- case _ => plan
79+ val buildSide = getSmjBuildSide(smj)
80+ if (! buildSideSmallEnough(smj, buildSide)) {
81+ withInfo(
82+ smj,
83+ " Cannot rewrite SortMergeJoin to HashJoin: " +
84+ " build side exceeds spark.comet.exec.replaceSortMergeJoin.maxBuildSize" )
85+ plan
86+ } else {
87+ ShuffledHashJoinExec (
88+ smj.leftKeys,
89+ smj.rightKeys,
90+ smj.joinType,
91+ buildSide,
92+ smj.condition,
93+ removeSort(smj.left),
94+ removeSort(smj.right),
95+ smj.isSkewJoin)
11896 }
11997 case _ => plan
12098 }
0 commit comments