Skip to content

Commit 8e22f85

Browse files
viiryaclaude
andcommitted
feat: enable null-aware anti-join support via DataFusion 53
DataFusion 53.0.0 introduced null-aware anti-join support in HashJoinExec via a new boolean parameter to try_new(). This enables Comet to offload Spark's NOT IN subquery pattern (BroadcastHashJoinExec with isNullAwareAntiJoin=true) to the native layer. Changes: - Add null_aware_anti_join field (tag 6) to HashJoin proto message - Pass join.null_aware_anti_join to HashJoinExec::try_new() in planner.rs - Allow BuildRight+LeftAnti joins through when isNullAwareAntiJoin=true on BroadcastHashJoinExec; regular BuildRight+LeftAnti on ShuffledHashJoinExec remains blocked (issue apache#457) - Set NullAwareAntiJoin field in the proto builder from operators.scala Null-aware semantics: if the build side contains ANY null, all left rows are suppressed regardless of match status (SQL NOT IN with nullable column). Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
1 parent 964e578 commit 8e22f85

3 files changed

Lines changed: 16 additions & 8 deletions

File tree

‎native/core/src/execution/planner.rs‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1699,12 +1699,7 @@ impl PhysicalPlanner {
16991699
// null doesn't equal to null in Spark join key. If the join key is
17001700
// `EqualNullSafe`, Spark will rewrite it during planning.
17011701
NullEquality::NullEqualsNothing,
1702-
// null_aware is for null-aware anti joins (NOT IN subqueries).
1703-
// NullEquality controls whether NULL = NULL in join keys generally,
1704-
// while null_aware changes anti-join semantics so any NULL changes
1705-
// the entire result. Spark doesn't use this path (it rewrites
1706-
// EqualNullSafe at plan time), so false is correct.
1707-
false,
1702+
join.null_aware_anti_join,
17081703
)?);
17091704

17101705
// If the hash join is build right, we need to swap the left and right
@@ -4038,6 +4033,7 @@ mod tests {
40384033
join_type: 0,
40394034
condition: None,
40404035
build_side: 0,
4036+
null_aware_anti_join: false,
40414037
})),
40424038
};
40434039

‎native/proto/src/proto/operator.proto‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -341,6 +341,9 @@ message HashJoin {
341341
JoinType join_type = 3;
342342
optional spark.spark_expression.Expr condition = 4;
343343
BuildSide build_side = 5;
344+
// True for BroadcastHashJoinExec null-aware anti-joins (NOT IN subquery semantics).
345+
// When true, any null in the build side suppresses all left rows.
346+
bool null_aware_anti_join = 6;
344347
}
345348

346349
message SortMergeJoin {

‎spark/src/main/scala/org/apache/spark/sql/comet/operators.scala‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1710,9 +1710,17 @@ trait CometHashJoin {
17101710
return None
17111711
}
17121712

1713-
if (join.buildSide == BuildRight && join.joinType == LeftAnti) {
1713+
// Null-aware anti-join is only supported for BroadcastHashJoinExec.
1714+
// For ShuffledHashJoinExec, BuildRight+LeftAnti is a regular anti-join which
1715+
// does not have the NOT IN null semantics so it remains unsupported.
1716+
val isNullAwareAntiJoin = join match {
1717+
case bhj: BroadcastHashJoinExec => bhj.isNullAwareAntiJoin
1718+
case _ => false
1719+
}
1720+
1721+
if (join.buildSide == BuildRight && join.joinType == LeftAnti && !isNullAwareAntiJoin) {
17141722
// https://github.com/apache/datafusion-comet/issues/457
1715-
withInfo(join, "BuildRight with LeftAnti is not supported")
1723+
withInfo(join, "BuildRight with LeftAnti is not supported (use null-aware anti-join via NOT IN)")
17161724
return None
17171725
}
17181726

@@ -1754,6 +1762,7 @@ trait CometHashJoin {
17541762
.addAllRightJoinKeys(rightKeys.map(_.get).asJava)
17551763
.setBuildSide(if (join.buildSide == BuildLeft) OperatorOuterClass.BuildSide.BuildLeft
17561764
else OperatorOuterClass.BuildSide.BuildRight)
1765+
.setNullAwareAntiJoin(isNullAwareAntiJoin)
17571766
condition.foreach(joinBuilder.setCondition)
17581767
Some(builder.setHashJoin(joinBuilder).build())
17591768
} else {

0 commit comments

Comments
 (0)