Skip to content

Commit d3ccef0

Browse files
committed
fix: handle dictionary nulls in null-aware left mark joins
1 parent 2306a4b commit d3ccef0

3 files changed

Lines changed: 128 additions & 6 deletions

File tree

‎datafusion/physical-plan/src/joins/hash_join/exec.rs‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8147,6 +8147,59 @@ mod tests {
81478147
Ok(())
81488148
}
81498149

8150+
/// The dictionary counterpart of
8151+
/// [`test_null_aware_left_mark_probe_null_in_other_partition`], where the
8152+
/// probe NULL exists in a dictionary value instead of the key bitmap.
8153+
#[apply(hash_join_exec_configs)]
8154+
#[tokio::test]
8155+
async fn test_null_aware_left_mark_probe_logical_null_in_other_partition(
8156+
batch_size: usize,
8157+
) -> Result<()> {
8158+
let task_ctx = prepare_task_ctx(batch_size, false);
8159+
8160+
for order in [[0, 1], [1, 0]] {
8161+
let left = build_table_dict_key(
8162+
"c1",
8163+
vec![Some(1), Some(4)],
8164+
vec![0, 1],
8165+
"dummy",
8166+
vec![Some(10), Some(40)],
8167+
);
8168+
let right = build_two_partition_dict_probe_with_logical_null_in_partition_0();
8169+
8170+
let on = vec![(
8171+
Arc::new(Column::new_with_schema("c1", &left.schema())?) as _,
8172+
Arc::new(Column::new_with_schema("c2", &right.schema())?) as _,
8173+
)];
8174+
8175+
let join = HashJoinExec::try_new(
8176+
left,
8177+
right,
8178+
on,
8179+
None,
8180+
&JoinType::LeftMark,
8181+
None,
8182+
PartitionMode::CollectLeft,
8183+
NullEquality::NullEqualsNothing,
8184+
true,
8185+
)?;
8186+
8187+
let batches = collect_partitions_in_order(&join, &order, &task_ctx).await?;
8188+
8189+
allow_duplicates! {
8190+
assert_snapshot!(batches_to_sort_string(&batches), @r"
8191+
+----+-------+------+
8192+
| c1 | dummy | mark |
8193+
+----+-------+------+
8194+
| 1 | 10 | true |
8195+
| 4 | 40 | |
8196+
+----+-------+------+
8197+
");
8198+
}
8199+
}
8200+
Ok(())
8201+
}
8202+
81508203
/// Test null-aware anti join when build side (left) contains NULL keys
81518204
/// Expected: rows with NULL keys should not be output
81528205
#[apply(hash_join_exec_configs)]
@@ -9067,6 +9120,72 @@ mod tests {
90679120
Ok(())
90689121
}
90699122

9123+
/// A dictionary key that points to a NULL dictionary value is a NULL build
9124+
/// key even when the dictionary key bitmap itself has no NULLs.
9125+
#[apply(hash_join_exec_configs)]
9126+
#[tokio::test]
9127+
async fn test_null_aware_left_mark_build_logical_null(
9128+
batch_size: usize,
9129+
) -> Result<()> {
9130+
let task_ctx = prepare_task_ctx(batch_size, false);
9131+
9132+
let left = build_table_dict_key(
9133+
"c1",
9134+
vec![Some(1), None, Some(4)],
9135+
vec![0, 1, 2],
9136+
"dummy",
9137+
vec![Some(10), Some(0), Some(40)],
9138+
);
9139+
let left_key = left
9140+
.execute(0, Arc::clone(&task_ctx))?
9141+
.next()
9142+
.await
9143+
.unwrap()?;
9144+
assert_eq!(left_key.column(0).null_count(), 0);
9145+
assert_eq!(left_key.column(0).logical_null_count(), 1);
9146+
9147+
let right = build_table_dict_key(
9148+
"c2",
9149+
vec![Some(1), Some(2), Some(3)],
9150+
vec![0, 1, 2],
9151+
"dummy",
9152+
vec![Some(100), Some(200), Some(300)],
9153+
);
9154+
9155+
let on = vec![(
9156+
Arc::new(Column::new_with_schema("c1", &left.schema())?) as _,
9157+
Arc::new(Column::new_with_schema("c2", &right.schema())?) as _,
9158+
)];
9159+
9160+
let join = HashJoinExec::try_new(
9161+
left,
9162+
right,
9163+
on,
9164+
None,
9165+
&JoinType::LeftMark,
9166+
None,
9167+
PartitionMode::CollectLeft,
9168+
NullEquality::NullEqualsNothing,
9169+
true,
9170+
)?;
9171+
9172+
let stream = join.execute(0, task_ctx)?;
9173+
let batches = common::collect(stream).await?;
9174+
9175+
allow_duplicates! {
9176+
assert_snapshot!(batches_to_sort_string(&batches), @r"
9177+
+----+-------+-------+
9178+
| c1 | dummy | mark |
9179+
+----+-------+-------+
9180+
| | 0 | |
9181+
| 1 | 10 | true |
9182+
| 4 | 40 | false |
9183+
+----+-------+-------+
9184+
");
9185+
}
9186+
Ok(())
9187+
}
9188+
90709189
/// Scalar correlated null-aware left mark join where only the build side's
90719190
/// value key contains NULL.
90729191
///

‎datafusion/physical-plan/src/joins/hash_join/stream.rs‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1251,11 +1251,7 @@ fn null_aware_skip_probe_batch(
12511251
// `on[0]` is the `NOT IN` value key for both modes.
12521252
let probe_key_column = &state.values[0];
12531253
let is_anti = matches!(mode, NullAwareMode::LeftAnti { .. });
1254-
let probe_has_null = if is_anti {
1255-
probe_key_column.logical_null_count() > 0
1256-
} else {
1257-
probe_key_column.null_count() > 0
1258-
};
1254+
let probe_has_null = probe_key_column.logical_null_count() > 0;
12591255
// Only batches with rows count: `NULL NOT IN (empty)` is TRUE.
12601256
left_data.record_probe_batch(state.batch.num_rows() > 0, probe_has_null);
12611257
// Best-effort early exit; the final stage re-checks the flag

‎datafusion/physical-plan/src/joins/utils.rs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1391,12 +1391,19 @@ pub(crate) fn build_null_aware_left_mark_column(
13911391
probe_side_has_null: bool,
13921392
probe_side_non_empty: bool,
13931393
) -> ArrayRef {
1394+
let build_key_nulls = build_key_column.logical_nulls();
1395+
13941396
// Whether an unmatched build row's mark is NULL (UNKNOWN) instead of FALSE:
13951397
// correlated joins precomputed this per row in `null_indices_bitmap`; the
13961398
// uncorrelated rules are cases 1-4 in the doc above.
13971399
let unmatched_mark_is_null = |build_idx: usize| match null_indices_bitmap {
13981400
Some(bitmap) => bitmap.get_bit(build_idx),
1399-
None if build_key_column.is_null(build_idx) => probe_side_non_empty,
1401+
None if build_key_nulls
1402+
.as_ref()
1403+
.is_some_and(|nulls| nulls.is_null(build_idx)) =>
1404+
{
1405+
probe_side_non_empty
1406+
}
14001407
None => probe_side_has_null,
14011408
};
14021409

0 commit comments

Comments
 (0)