Skip to content

Commit be1b2e5

Browse files
viiryaclaude
andcommitted
feat: support regular BuildRight+LeftAnti hash join
DataFusion HashJoinExec supports LeftAnti in all build-side configurations, and the planner's existing swap_inputs logic handles BuildRight correctly. The original guard in operators.scala was added without a stated reason; the referenced issue apache#457 was specifically about null-aware anti-join (fixed via DataFusion 53), not regular BuildRight+LeftAnti. Changes: - Remove the BuildRight+LeftAnti rejection in CometHashJoin - Allow SortMergeJoin+BuildRight+LeftAnti to be rewritten to ShuffledHashJoin - Re-enable the previously-disabled SHJ BuildRight+LeftAnti test Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
1 parent fe3e019 commit be1b2e5

3 files changed

Lines changed: 5 additions & 16 deletions

File tree

‎spark/src/main/scala/org/apache/comet/rules/RewriteJoin.scala‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
package org.apache.comet.rules
2121

2222
import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide, JoinSelectionHelper}
23-
import org.apache.spark.sql.catalyst.plans.{LeftAnti, LeftSemi}
23+
import org.apache.spark.sql.catalyst.plans.LeftSemi
2424
import org.apache.spark.sql.catalyst.plans.logical.Join
2525
import org.apache.spark.sql.execution.{SortExec, SparkPlan}
2626
import org.apache.spark.sql.execution.joins.{ShuffledHashJoinExec, SortMergeJoinExec}
@@ -67,8 +67,7 @@ object RewriteJoin extends JoinSelectionHelper {
6767
def rewrite(plan: SparkPlan): SparkPlan = plan match {
6868
case smj: SortMergeJoinExec =>
6969
getSmjBuildSide(smj) match {
70-
case Some(BuildRight) if smj.joinType == LeftAnti || smj.joinType == LeftSemi =>
71-
// LeftAnti https://github.com/apache/datafusion-comet/issues/457
70+
case Some(BuildRight) if smj.joinType == LeftSemi =>
7271
// LeftSemi https://github.com/apache/datafusion-comet/issues/2667
7372
withInfo(
7473
smj,

‎spark/src/main/scala/org/apache/spark/sql/comet/operators.scala‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1710,22 +1710,12 @@ trait CometHashJoin {
17101710
return None
17111711
}
17121712

1713-
// Null-aware anti-join is only supported for BroadcastHashJoinExec.
1714-
// For ShuffledHashJoinExec, BuildRight+LeftAnti is a regular anti-join which
1715-
// does not have the NOT IN null semantics so it remains unsupported.
1713+
// Only BroadcastHashJoinExec can be null-aware (NOT IN subqueries).
17161714
val isNullAwareAntiJoin = join match {
17171715
case bhj: BroadcastHashJoinExec => bhj.isNullAwareAntiJoin
17181716
case _ => false
17191717
}
17201718

1721-
if (join.buildSide == BuildRight && join.joinType == LeftAnti && !isNullAwareAntiJoin) {
1722-
withInfo(
1723-
join,
1724-
"BuildRight with LeftAnti is not supported " +
1725-
"(use null-aware anti-join via NOT IN)")
1726-
return None
1727-
}
1728-
17291719
val condition = join.condition.map { cond =>
17301720
val condProto = exprToProto(cond, join.left.output ++ join.right.output)
17311721
if (condProto.isEmpty) {

‎spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -190,8 +190,8 @@ class CometJoinSuite extends CometTestBase {
190190
val df8 = left.join(right, left("_2") === right("_1"), "leftsemi")
191191
checkSparkAnswerAndOperator(df8)
192192

193-
// DataFusion HashJoin LeftAnti has bugs in handling nulls and is disabled for now.
194-
// left.join(right, left("_2") === right("_1"), "leftanti")
193+
val df9 = left.join(right, left("_2") === right("_1"), "leftanti")
194+
checkSparkAnswerAndOperator(df9)
195195
}
196196
}
197197
}

0 commit comments

Comments
 (0)