Skip to content

Commit 509725f

Browse files
cool9850311claude
andcommitted
fix: reject null-aware LeftAnti hash join in Partitioned mode
A null-aware `LeftAnti` join emits build rows in the final stage, and the "a probe partition saw a NULL `NOT IN` key" fact is recorded on the build data (`JoinLeftData::probe_completion`). `PartitionMode::Partitioned` gives every output partition its own build side, so a NULL probe key - routed by hash to a single partition - only silenced that partition's build rows; the others still emitted their unmatched ones. With build `c1` = 1..8 and probe `c2` = {1, NULL} over four hash partitions the join returned `[3, 6, 8]` where `CollectLeft` correctly returned nothing. `NullAwareMode::try_new` already rejected `Partitioned` for the correlated `LeftAnti` and for `LeftMark`, which share the per-build-row null bitmap for the same reason. Widen that arm to cover the uncorrelated `LeftAnti` too, so the whole build-driven family is rejected consistently. The DataFusion planner never produced this plan - `physical_planner` does not repartition null-aware joins and `JoinSelection` pins them to `CollectLeft` - so the shape was only reachable from hand-built plans and from protobuf, which decodes the partition mode as written and goes through the same builder. `test_null_aware_left_anti_swaps_to_right_anti` loses its `Partitioned` case: that input plan can no longer be constructed, and the swap it checks is still covered by the `CollectLeft` and `Auto` cases. Closes apache#26496 Co-Authored-By: Claude Opus 5 <[email protected]>
1 parent e12901d commit 509725f

2 files changed

Lines changed: 83 additions & 7 deletions

File tree

‎datafusion/core/tests/physical_optimizer/join_selection.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -469,10 +469,13 @@ async fn test_join_with_swap_semi() {
469469
}
470470
}
471471

472+
/// `PartitionMode::Partitioned` is deliberately not a case here: a null-aware
473+
/// `LeftAnti` cannot be constructed in that mode at all, so there is no input
474+
/// plan to optimize. See
475+
/// `HashJoinExec::null_aware_mode`/`NullAwareMode::try_new`.
472476
#[rstest]
473477
#[case(PartitionMode::CollectLeft)]
474478
#[case(PartitionMode::Auto)]
475-
#[case(PartitionMode::Partitioned)]
476479
#[tokio::test]
477480
async fn test_null_aware_left_anti_swaps_to_right_anti(
478481
#[case] partition_mode: PartitionMode,

‎datafusion/physical-plan/src/joins/hash_join/exec.rs‎

Lines changed: 79 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -300,12 +300,14 @@ impl NullAwareMode {
300300
Self::RightAnti if num_keys != 1 => plan_err!(
301301
"null_aware {join_type} joins only support single column join key, got {num_keys} columns"
302302
),
303-
// Correlated joins share the per-build-row null bitmap across all
304-
// probe partitions.
305-
Self::LeftMark { .. }
306-
| Self::LeftAnti {
307-
correlated: true, ..
308-
} if partition_mode == PartitionMode::Partitioned => {
303+
// Build-driven joins decide in the final stage from state shared
304+
// across the probe partitions: the per-build-row null bitmap for
305+
// the correlated forms, and the probe-side NULL flag for the
306+
// uncorrelated ones. `Partitioned` gives each output partition its
307+
// own build side, so neither is global there.
308+
Self::LeftMark { .. } | Self::LeftAnti { .. }
309+
if partition_mode == PartitionMode::Partitioned =>
310+
{
309311
plan_err!(
310312
"null_aware joins require PartitionMode::CollectLeft, got PartitionMode::Partitioned"
311313
)
@@ -2743,9 +2745,37 @@ mod proto_tests {
27432745
let mut node = decodable_node();
27442746
node.join_type = protobuf::JoinType::Leftanti.into();
27452747
node.null_aware = true;
2748+
// `decodable_node` is `Partitioned`, which a null-aware join rejects
2749+
// (see `try_from_proto_rejects_a_null_aware_partitioned_join`).
2750+
node.partition_mode = protobuf::PartitionMode::CollectLeft.into();
27462751
assert!(as_join(&decode(node)).null_aware);
27472752
}
27482753

2754+
/// A null-aware join decides in the final stage from state shared across
2755+
/// the probe partitions, which `PartitionMode::Partitioned` does not give
2756+
/// it. Decoding goes through the same builder as `try_new`, so a node
2757+
/// written with that combination is rejected rather than silently
2758+
/// returning rows for a `NOT IN` whose subquery contains NULL.
2759+
#[test]
2760+
fn try_from_proto_rejects_a_null_aware_partitioned_join() {
2761+
let decoder = StubPlanDecoder::ok();
2762+
let ctx = ExecutionPlanDecodeCtx::new(&decoder);
2763+
let mut node = decodable_node();
2764+
node.join_type = protobuf::JoinType::Leftanti.into();
2765+
node.null_aware = true;
2766+
assert_eq!(
2767+
node.partition_mode,
2768+
protobuf::PartitionMode::Partitioned as i32
2769+
);
2770+
2771+
let err = HashJoinExec::try_from_proto(&join_node(node), &ctx).unwrap_err();
2772+
assert!(
2773+
err.to_string()
2774+
.contains("null_aware joins require PartitionMode::CollectLeft"),
2775+
"{err}"
2776+
);
2777+
}
2778+
27492779
/// A multi-column `NOT IN` keeps its value key count through the round
27502780
/// trip; a node written before the field existed decodes it as 0, which
27512781
/// is read as a scalar `NOT IN`.
@@ -10326,6 +10356,49 @@ mod tests {
1032610356
);
1032710357
}
1032810358

10359+
/// An uncorrelated null-aware `LeftAnti` must reject
10360+
/// `PartitionMode::Partitioned` just like the correlated forms do.
10361+
///
10362+
/// `LeftAnti` emits build rows in the final stage, and the "a probe
10363+
/// partition saw a NULL `NOT IN` key" flag lives on the build data
10364+
/// (`JoinLeftData::probe_completion`). In `Partitioned` mode every output
10365+
/// partition builds its own build side, so a NULL probe key, hashed to a
10366+
/// single partition, would only silence that partition's build rows while
10367+
/// the others still emitted their unmatched ones.
10368+
#[tokio::test]
10369+
async fn test_null_aware_validation_left_anti_partitioned() {
10370+
let left =
10371+
build_table_two_cols(("c1", &vec![Some(1)]), ("dummy", &vec![Some(10)]));
10372+
let right =
10373+
build_table_two_cols(("c2", &vec![Some(1)]), ("dummy", &vec![Some(100)]));
10374+
10375+
// A single join key and no filter, so this is the uncorrelated
10376+
// `LeftAnti` form, which the partition-mode check used to let through.
10377+
let on = vec![(
10378+
Arc::new(Column::new_with_schema("c1", &left.schema()).unwrap()) as _,
10379+
Arc::new(Column::new_with_schema("c2", &right.schema()).unwrap()) as _,
10380+
)];
10381+
10382+
let err = HashJoinExec::try_new(
10383+
left,
10384+
right,
10385+
on,
10386+
None,
10387+
&JoinType::LeftAnti,
10388+
None,
10389+
PartitionMode::Partitioned,
10390+
NullEquality::NullEqualsNothing,
10391+
true,
10392+
)
10393+
.expect_err("null_aware LeftAnti must reject PartitionMode::Partitioned");
10394+
10395+
assert!(
10396+
err.to_string()
10397+
.contains("null_aware joins require PartitionMode::CollectLeft"),
10398+
"{err}"
10399+
);
10400+
}
10401+
1032910402
/// Test that null_aware validation rejects PartitionMode::Partitioned (must be CollectLeft).
1033010403
#[tokio::test]
1033110404
async fn test_null_aware_validation_partitioned() {

0 commit comments

Comments
 (0)