Skip to content

Commit 31f5ae6

Browse files
committed
perf: build the scope comparator once per null-aware scope lookup
for_each_scope_match passed a fresh comparator slot to each chunk even though the build and probe scope arrays stay the same for the whole loop, so multi-column scopes rebuilt the comparator per chunk. Keep one slot for the loop. Spell out on equal_rows_arr that the same-arrays rule is not checked and that an empty slot is always safe. Add a null-aware anti join test with a two-column correlation scope that splits each scope lookup into several chunks.
1 parent 92fe366 commit 31f5ae6

3 files changed

Lines changed: 152 additions & 2 deletions

File tree

‎datafusion/physical-plan/src/joins/hash_join/exec.rs‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10132,6 +10132,153 @@ mod tests {
1013210132
Ok(())
1013310133
}
1013410134

10135+
/// Correlated null-aware `LeftAnti` with a two-column correlation scope,
10136+
/// so scope lookups compare keys with the general multi-column comparator.
10137+
///
10138+
/// Each NULL-marking direction yields several scope matches for a single
10139+
/// probe batch, which small batch sizes split across many lookup chunks.
10140+
#[apply(hash_join_exec_configs)]
10141+
#[tokio::test]
10142+
async fn test_null_aware_left_anti_multi_column_scope(
10143+
batch_size: usize,
10144+
) -> Result<()> {
10145+
let task_ctx = prepare_task_ctx(batch_size, false);
10146+
10147+
fn table(
10148+
id: Vec<Option<i32>>,
10149+
g1: Vec<Option<i32>>,
10150+
g2: Vec<Option<i32>>,
10151+
) -> Arc<dyn ExecutionPlan> {
10152+
let schema = Arc::new(Schema::new(vec![
10153+
Field::new("id", DataType::Int32, true),
10154+
Field::new("g1", DataType::Int32, true),
10155+
Field::new("g2", DataType::Int32, true),
10156+
]));
10157+
let batch = RecordBatch::try_new(
10158+
Arc::clone(&schema),
10159+
vec![
10160+
Arc::new(Int32Array::from(id)),
10161+
Arc::new(Int32Array::from(g1)),
10162+
Arc::new(Int32Array::from(g2)),
10163+
],
10164+
)
10165+
.unwrap();
10166+
TestMemoryExec::try_new_exec(&[vec![batch]], schema, None).unwrap()
10167+
}
10168+
10169+
let left = table(
10170+
vec![
10171+
Some(1),
10172+
Some(2),
10173+
None,
10174+
None,
10175+
Some(6),
10176+
Some(8),
10177+
Some(9),
10178+
Some(10),
10179+
None,
10180+
],
10181+
vec![
10182+
Some(1),
10183+
Some(1),
10184+
Some(1),
10185+
Some(1),
10186+
Some(1),
10187+
Some(1),
10188+
Some(1),
10189+
Some(2),
10190+
Some(3),
10191+
],
10192+
vec![
10193+
Some(1),
10194+
Some(1),
10195+
Some(1),
10196+
Some(1),
10197+
Some(2),
10198+
Some(2),
10199+
Some(2),
10200+
Some(1),
10201+
Some(3),
10202+
],
10203+
);
10204+
let right = table(
10205+
vec![
10206+
Some(2),
10207+
Some(3),
10208+
Some(4),
10209+
Some(5),
10210+
None,
10211+
None,
10212+
Some(7),
10213+
Some(11),
10214+
],
10215+
vec![
10216+
Some(1),
10217+
Some(1),
10218+
Some(1),
10219+
Some(1),
10220+
Some(1),
10221+
Some(1),
10222+
Some(1),
10223+
Some(2),
10224+
],
10225+
vec![
10226+
Some(1),
10227+
Some(1),
10228+
Some(1),
10229+
Some(1),
10230+
Some(2),
10231+
Some(2),
10232+
Some(2),
10233+
Some(1),
10234+
],
10235+
);
10236+
10237+
let on = ["id", "g1", "g2"]
10238+
.into_iter()
10239+
.map(|name| {
10240+
Ok((
10241+
Arc::new(Column::new_with_schema(name, &left.schema())?) as _,
10242+
Arc::new(Column::new_with_schema(name, &right.schema())?) as _,
10243+
))
10244+
})
10245+
.collect::<Result<JoinOn>>()?;
10246+
10247+
let join = HashJoinExec::try_new(
10248+
left,
10249+
right,
10250+
on,
10251+
None,
10252+
&JoinType::LeftAnti,
10253+
None,
10254+
PartitionMode::CollectLeft,
10255+
NullEquality::NullEqualsNothing,
10256+
true,
10257+
)?;
10258+
10259+
let stream = join.execute(0, task_ctx)?;
10260+
let batches = common::collect(stream).await?;
10261+
10262+
// Scope (1, 1) holds 2, 3, 4 and 5: `(1, 1, 1)` is kept, `(2, 1, 1)`
10263+
// matches, and both NULL-valued build rows there are UNKNOWN. Scope
10264+
// (1, 2) holds two NULLs, so all of its build rows are UNKNOWN. Scope
10265+
// (2, 1) holds only 11, so `(10, 2, 1)` is kept, and `(NULL, 3, 3)`
10266+
// has an empty scope.
10267+
allow_duplicates! {
10268+
assert_snapshot!(batches_to_sort_string(&batches), @r"
10269+
+----+----+----+
10270+
| id | g1 | g2 |
10271+
+----+----+----+
10272+
| | 3 | 3 |
10273+
| 1 | 1 | 1 |
10274+
| 10 | 2 | 1 |
10275+
+----+----+----+
10276+
");
10277+
}
10278+
10279+
Ok(())
10280+
}
10281+
1013510282
#[test]
1013610283
fn test_lr_is_preserved() {
1013710284
assert_eq!(lr_is_preserved(JoinType::Inner), (true, true));

‎datafusion/physical-plan/src/joins/hash_join/stream.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1580,6 +1580,7 @@ fn for_each_scope_match(
15801580
mut f: impl FnMut(UInt64Array, UInt32Array) -> Result<()>,
15811581
) -> Result<()> {
15821582
let mut offset = (0, None);
1583+
let mut key_comparator = None;
15831584
loop {
15841585
let (build_indices, probe_indices, next_offset) = lookup_join_hashmap(
15851586
scope_map,
@@ -1592,7 +1593,7 @@ fn for_each_scope_match(
15921593
offset,
15931594
probe_indices_buffer,
15941595
build_indices_buffer,
1595-
&mut None,
1596+
&mut key_comparator,
15961597
)?;
15971598

15981599
if !build_indices.is_empty() {

‎datafusion/physical-plan/src/joins/utils.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2291,7 +2291,9 @@ pub(crate) fn matchable_join_keys(
22912291
/// `comparator` caches the general-path [`JoinKeyComparator`] between calls
22922292
/// that share the same `left_arrays` and `right_arrays`, so its setup cost is
22932293
/// paid once rather than per call. Pass an empty slot whenever either side's
2294-
/// arrays change.
2294+
/// arrays change. Nothing checks this: a slot kept across different arrays
2295+
/// silently returns wrong matches. An empty slot (`&mut None`) is always
2296+
/// correct and only rebuilds the comparator on each call.
22952297
pub(super) fn equal_rows_arr(
22962298
indices_left: &UInt64Array,
22972299
indices_right: &UInt32Array,

0 commit comments

Comments
 (0)