Skip to content

Commit 785332e

Browse files
viiryaIsaac
andcommitted
Address review: skip the per-batch UNKNOWN bitmap, trim overlapping tests
- A correlated null-aware RightAnti allocates the per-batch UNKNOWN bitmap only when the build side has a NULL-valued row or the probe batch has a NULL in a value key, and process_probe_batch matches on the join mode. - Rename `UnknownMarks::marked_row` to `unknown_row` and the build state flags to `with_null_candidate_state` / `with_build_unknown_bitmap`. - Move the all-NULL-valued subquery case to null_aware_multi_column.slt and drop the overlapping exec.rs test; fold the construction tests into one parametrized test and the swap tests into one rstest. - The fuzz walks the plan for HashJoinExec instead of matching the displayed plan, and plans each query once. - Move the memory-limit test to null_aware_memory_limit.slt. Co-authored-by: Isaac <[email protected]>
1 parent 2347a72 commit 785332e

8 files changed

Lines changed: 288 additions & 495 deletions

File tree

‎datafusion/core/tests/fuzz_cases/null_aware_join_fuzz.rs‎

Lines changed: 33 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,10 @@ use std::sync::Arc;
3131
use arrow::array::{Int32Array, RecordBatch};
3232
use arrow::datatypes::{DataType, Field, Schema};
3333
use arrow::util::pretty::pretty_format_batches;
34+
use datafusion::common::JoinType;
3435
use datafusion::datasource::MemTable;
35-
use datafusion::physical_plan::displayable;
36+
use datafusion::physical_plan::joins::HashJoinExec;
37+
use datafusion::physical_plan::{ExecutionPlan, collect};
3638
use datafusion::prelude::{SessionConfig, SessionContext};
3739
use rand::rngs::StdRng;
3840
use rand::{Rng, SeedableRng};
@@ -66,9 +68,9 @@ async fn null_aware_join_matches_reference() {
6668
ctx.register_table("o", Arc::clone(&outer) as _).unwrap();
6769
ctx.register_table("i", Arc::clone(&inner) as _).unwrap();
6870
for (query, reference) in cases() {
69-
plans.record(&ctx, &query).await;
70-
let actual = run(&ctx, &query).await;
71-
let expected = run(&ctx, &reference).await;
71+
let (actual, plan) = run(&ctx, &query).await;
72+
plans.record(plan.as_ref());
73+
let (expected, _) = run(&ctx, &reference).await;
7274
assert_eq!(
7375
actual, expected,
7476
"seed={seed} target_partitions={target_partitions} \
@@ -95,25 +97,25 @@ struct NullAwarePlans {
9597
}
9698

9799
impl NullAwarePlans {
98-
async fn record(&mut self, ctx: &SessionContext, sql: &str) {
99-
let plan = ctx
100-
.sql(sql)
101-
.await
102-
.unwrap()
103-
.create_physical_plan()
104-
.await
105-
.unwrap();
106-
let plan = displayable(plan.as_ref()).indent(true).to_string();
107-
for line in plan.lines().filter(|line| line.contains("null_aware")) {
108-
if line.contains("join_type=LeftAnti") {
109-
self.left_anti += 1;
110-
} else if line.contains("join_type=RightAnti") {
111-
self.right_anti += 1;
112-
if line.contains("), (") || line.contains("filter=") {
113-
self.correlated_right_anti += 1;
100+
/// Counts the null-aware anti joins in `plan` and its descendants.
101+
fn record(&mut self, plan: &dyn ExecutionPlan) {
102+
if let Some(join) = plan.downcast_ref::<HashJoinExec>()
103+
&& join.null_aware
104+
{
105+
match join.join_type() {
106+
JoinType::LeftAnti => self.left_anti += 1,
107+
JoinType::RightAnti => {
108+
self.right_anti += 1;
109+
if join.on().len() > 1 || join.filter().is_some() {
110+
self.correlated_right_anti += 1;
111+
}
114112
}
113+
_ => {}
115114
}
116115
}
116+
for child in plan.children() {
117+
self.record(child.as_ref());
118+
}
117119
}
118120
}
119121

@@ -215,9 +217,16 @@ fn random_table(rng: &mut StdRng, columns: &[&str], max_rows: i32) -> Arc<MemTab
215217
Arc::new(MemTable::try_new(schema, partitions).unwrap())
216218
}
217219

218-
/// The rows of `sql`, sorted.
219-
async fn run(ctx: &SessionContext, sql: &str) -> Vec<String> {
220-
let batches = ctx.sql(sql).await.unwrap().collect().await.unwrap();
220+
/// The rows of `sql`, sorted, and the physical plan that produced them.
221+
async fn run(ctx: &SessionContext, sql: &str) -> (Vec<String>, Arc<dyn ExecutionPlan>) {
222+
let plan = ctx
223+
.sql(sql)
224+
.await
225+
.unwrap()
226+
.create_physical_plan()
227+
.await
228+
.unwrap();
229+
let batches = collect(Arc::clone(&plan), ctx.task_ctx()).await.unwrap();
221230
let formatted = pretty_format_batches(&batches).unwrap().to_string();
222231
let mut rows: Vec<String> = formatted
223232
.lines()
@@ -226,5 +235,5 @@ async fn run(ctx: &SessionContext, sql: &str) -> Vec<String> {
226235
.map(String::from)
227236
.collect();
228237
rows.sort();
229-
rows
238+
(rows, plan)
230239
}

‎datafusion/core/tests/memory_limit/mod.rs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ use std::sync::{Arc, LazyLock};
2323
#[cfg(feature = "extended_tests")]
2424
mod memory_limit_validation;
2525
mod nlj_spill_unmatched;
26-
mod null_aware_not_in;
2726
mod repartition_mem_limit;
2827
mod union_nullable_spill;
2928
mod view_spill_compaction;

‎datafusion/core/tests/memory_limit/null_aware_not_in.rs‎

Lines changed: 0 additions & 103 deletions
This file was deleted.

‎datafusion/core/tests/physical_optimizer/join_selection.rs‎

Lines changed: 36 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -471,29 +471,41 @@ async fn test_join_with_swap_semi() {
471471
}
472472
}
473473

474+
/// A null-aware `LeftAnti` swaps to a null-aware `RightAnti` that builds on
475+
/// the smaller subquery side. Besides the scalar `NOT IN`, this covers a
476+
/// multi-key one (a multi-column `NOT IN`, correlation scope keys, or both),
477+
/// which keeps the number of `NOT IN` value keys and the key order, and a
478+
/// filtered one (a decorrelated correlated `NOT IN`), whose filter sides are
479+
/// swapped with the inputs.
474480
#[rstest]
475-
#[case(PartitionMode::CollectLeft)]
476-
#[case(PartitionMode::Auto)]
477-
#[case(PartitionMode::Partitioned)]
481+
#[case::collect_left(PartitionMode::CollectLeft, 1, 1, false)]
482+
#[case::auto(PartitionMode::Auto, 1, 1, false)]
483+
#[case::partitioned(PartitionMode::Partitioned, 1, 1, false)]
484+
#[case::filter(PartitionMode::CollectLeft, 1, 1, true)]
485+
#[case::scope_key(PartitionMode::CollectLeft, 2, 1, false)]
486+
#[case::multi_column(PartitionMode::CollectLeft, 2, 2, false)]
478487
#[tokio::test]
479488
async fn test_null_aware_left_anti_swaps_to_right_anti(
480489
#[case] partition_mode: PartitionMode,
490+
#[case] num_keys: usize,
491+
#[case] value_keys: usize,
492+
#[case] with_filter: bool,
481493
) -> Result<()> {
482494
let (big, small) = create_big_and_small();
483-
let join = HashJoinExec::try_new(
495+
let big_col = Arc::new(Column::new_with_schema("big_col", &big.schema())?) as _;
496+
let small_col = Arc::new(Column::new_with_schema("small_col", &small.schema())?) as _;
497+
let on = vec![(big_col, small_col); num_keys];
498+
let join = HashJoinExecBuilder::new(
484499
Arc::clone(&big),
485500
Arc::clone(&small),
486-
vec![(
487-
Arc::new(Column::new_with_schema("big_col", &big.schema())?),
488-
Arc::new(Column::new_with_schema("small_col", &small.schema())?),
489-
)],
490-
None,
491-
&JoinType::LeftAnti,
492-
None,
493-
partition_mode,
494-
NullEquality::NullEqualsNothing,
495-
true,
496-
)?;
501+
on,
502+
JoinType::LeftAnti,
503+
)
504+
.with_filter(if with_filter { nl_join_filter() } else { None })
505+
.with_partition_mode(partition_mode)
506+
.with_null_aware(true)
507+
.with_null_aware_value_keys(value_keys)
508+
.build()?;
497509
let original_schema = join.schema();
498510

499511
let optimized_join =
@@ -505,9 +517,18 @@ async fn test_null_aware_left_anti_swaps_to_right_anti(
505517
assert_eq!(*swapped_join.join_type(), JoinType::RightAnti);
506518
assert_eq!(*swapped_join.partition_mode(), PartitionMode::CollectLeft);
507519
assert!(swapped_join.null_aware);
520+
assert_eq!(swapped_join.null_aware_value_keys, value_keys);
521+
assert_eq!(swapped_join.on().len(), num_keys);
508522
assert_eq!(swapped_join.left().schema().field(0).name(), "small_col");
509523
assert_eq!(swapped_join.right().schema().field(0).name(), "big_col");
510524
assert_eq!(swapped_join.schema(), original_schema);
525+
let filter_sides = swapped_join
526+
.filter()
527+
.map(|filter| filter.column_indices().iter().map(|c| c.side).collect());
528+
assert_eq!(
529+
filter_sides,
530+
with_filter.then(|| vec![JoinSide::Right, JoinSide::Left])
531+
);
511532

512533
Ok(())
513534
}
@@ -619,97 +640,6 @@ async fn test_null_aware_left_anti_respects_disabled_join_reordering() -> Result
619640
Ok(())
620641
}
621642

622-
/// A filtered null-aware `LeftAnti` (produced by decorrelating a correlated
623-
/// `NOT IN`) swaps to a null-aware `RightAnti` that decides `NOT IN` per probe
624-
/// row; the filter's sides are swapped with the inputs.
625-
#[tokio::test]
626-
async fn test_null_aware_left_anti_with_filter_swaps() -> Result<()> {
627-
let (big, small) = create_big_and_small();
628-
let join = HashJoinExec::try_new(
629-
Arc::clone(&big),
630-
Arc::clone(&small),
631-
vec![(
632-
Arc::new(Column::new_with_schema("big_col", &big.schema())?),
633-
Arc::new(Column::new_with_schema("small_col", &small.schema())?),
634-
)],
635-
nl_join_filter(),
636-
&JoinType::LeftAnti,
637-
None,
638-
PartitionMode::CollectLeft,
639-
NullEquality::NullEqualsNothing,
640-
true,
641-
)?;
642-
643-
let optimized_join =
644-
JoinSelection::new().optimize(Arc::new(join), &ConfigOptions::new())?;
645-
let swapped_join = optimized_join
646-
.downcast_ref::<HashJoinExec>()
647-
.expect("anti join swap should not require a projection");
648-
649-
assert_eq!(*swapped_join.join_type(), JoinType::RightAnti);
650-
assert_eq!(*swapped_join.partition_mode(), PartitionMode::CollectLeft);
651-
assert!(swapped_join.null_aware);
652-
let filter = swapped_join.filter().expect("filter should be kept");
653-
assert_eq!(
654-
filter
655-
.column_indices()
656-
.iter()
657-
.map(|c| c.side)
658-
.collect::<Vec<_>>(),
659-
vec![JoinSide::Right, JoinSide::Left]
660-
);
661-
assert_eq!(swapped_join.left().schema().field(0).name(), "small_col");
662-
assert_eq!(swapped_join.right().schema().field(0).name(), "big_col");
663-
664-
Ok(())
665-
}
666-
667-
/// A multi-key null-aware `LeftAnti` (a multi-column `NOT IN`, correlation
668-
/// scope keys, or both) swaps to a `RightAnti` that keeps the number of
669-
/// `NOT IN` value keys and the key order.
670-
#[rstest]
671-
#[case(1)]
672-
#[case(2)]
673-
#[tokio::test]
674-
async fn test_null_aware_multi_key_left_anti_swaps_to_right_anti(
675-
#[case] value_keys: usize,
676-
) -> Result<()> {
677-
let (big, small) = create_big_and_small();
678-
let big_col = Arc::new(Column::new_with_schema("big_col", &big.schema())?) as _;
679-
let small_col = Arc::new(Column::new_with_schema("small_col", &small.schema())?) as _;
680-
let join = HashJoinExecBuilder::new(
681-
Arc::clone(&big),
682-
Arc::clone(&small),
683-
vec![
684-
(Arc::clone(&big_col), Arc::clone(&small_col)),
685-
(big_col, small_col),
686-
],
687-
JoinType::LeftAnti,
688-
)
689-
.with_partition_mode(PartitionMode::CollectLeft)
690-
.with_null_aware(true)
691-
.with_null_aware_value_keys(value_keys)
692-
.build()?;
693-
let original_schema = join.schema();
694-
695-
let optimized_join =
696-
JoinSelection::new().optimize(Arc::new(join), &ConfigOptions::new())?;
697-
let swapped_join = optimized_join
698-
.downcast_ref::<HashJoinExec>()
699-
.expect("anti join swap should not require a projection");
700-
701-
assert_eq!(*swapped_join.join_type(), JoinType::RightAnti);
702-
assert_eq!(*swapped_join.partition_mode(), PartitionMode::CollectLeft);
703-
assert!(swapped_join.null_aware);
704-
assert_eq!(swapped_join.null_aware_value_keys, value_keys);
705-
assert_eq!(swapped_join.on().len(), 2);
706-
assert_eq!(swapped_join.left().schema().field(0).name(), "small_col");
707-
assert_eq!(swapped_join.right().schema().field(0).name(), "big_col");
708-
assert_eq!(swapped_join.schema(), original_schema);
709-
710-
Ok(())
711-
}
712-
713643
#[tokio::test]
714644
async fn test_join_swap_uses_context_statistics_registry() -> Result<()> {
715645
let left: Arc<dyn ExecutionPlan> = Arc::new(StatisticsExec::new(

0 commit comments

Comments
 (0)