Skip to content

Commit a9d4896

Browse files
committed
refactor: rename Option<NotIn> to not_in, narrow NOT IN visibility, assert probe summary
1 parent 19c0fec commit a9d4896

4 files changed

Lines changed: 32 additions & 28 deletions

File tree

‎datafusion/core/tests/physical_optimizer/join_selection.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -481,7 +481,7 @@ async fn test_join_with_swap_semi() {
481481
/// `PartitionMode::Partitioned` is deliberately not a case here: a null-aware
482482
/// `LeftAnti` cannot be constructed in that mode at all, so there is no input
483483
/// plan to optimize. See
484-
/// `HashJoinExec::null_aware_mode`/`NotIn::try_new`.
484+
/// `HashJoinExec::not_in`/`NotIn::try_new`.
485485
#[rstest]
486486
#[case::collect_left(PartitionMode::CollectLeft, 1, 1, false)]
487487
#[case::auto(PartitionMode::Auto, 1, 1, false)]

‎datafusion/physical-plan/src/joins/hash_join/exec.rs‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,7 @@ struct JoinBuildData {
211211
/// Evaluated build-side key expressions, with float `-0.0` rewritten to
212212
/// `+0.0` so probe batches can compare against them without scanning the
213213
/// build side again. The dynamic filters (`bounds`, `membership`) and
214-
/// the `NOT IN` state were computed from the original keys in
214+
/// the `NOT IN` NULL masks were computed from the original keys in
215215
/// `collect_left_input`.
216216
values: Vec<ArrayRef>,
217217
/// Bounds computed from the build side; absent for an empty partition.
@@ -270,7 +270,7 @@ impl JoinLeftData {
270270

271271
/// Returns the build side key values, with float `-0.0` already rewritten
272272
/// to `+0.0` for key comparison. The dynamic filters and the `NOT IN`
273-
/// state were derived from the original keys instead.
273+
/// NULL masks were derived from the original keys instead.
274274
pub(super) fn values(&self) -> &[ArrayRef] {
275275
&self.build.values
276276
}
@@ -491,7 +491,7 @@ impl HashJoinExecBuilder {
491491
}
492492
}
493493
// Validate null_aware flag
494-
exec.null_aware_mode()?;
494+
exec.not_in()?;
495495

496496
if preserve_properties {
497497
return Ok(exec);
@@ -1199,7 +1199,7 @@ impl HashJoinExec {
11991199
/// The null-aware semantics of this join, if [`Self::null_aware`] is set.
12001200
///
12011201
/// Errors if `null_aware` is set on a join that does not support it.
1202-
pub(super) fn null_aware_mode(&self) -> Result<Option<NotIn>> {
1202+
pub(super) fn not_in(&self) -> Result<Option<NotIn>> {
12031203
self.null_aware
12041204
.then(|| {
12051205
NotIn::try_new(
@@ -1664,7 +1664,7 @@ impl ExecutionPlan for HashJoinExec {
16641664

16651665
let join_metrics = BuildProbeJoinMetrics::new(partition, &self.metrics);
16661666

1667-
let null_aware = self.null_aware_mode()?;
1667+
let not_in = self.not_in()?;
16681668

16691669
let array_map_created_count = MetricBuilder::new(&self.metrics)
16701670
.with_category(MetricCategory::Rows)
@@ -1691,7 +1691,7 @@ impl ExecutionPlan for HashJoinExec {
16911691
on_right,
16921692
repartition_random_state,
16931693
self.null_equality,
1694-
null_aware,
1694+
not_in,
16951695
))
16961696
})))
16971697
})
@@ -1725,7 +1725,7 @@ impl ExecutionPlan for HashJoinExec {
17251725
enable_dynamic_filter_pushdown,
17261726
Arc::clone(context.session_config().options()),
17271727
self.null_equality,
1728-
null_aware,
1728+
not_in,
17291729
array_map_created_count,
17301730
BuildMode::Ordinary,
17311731
))
@@ -1747,7 +1747,7 @@ impl ExecutionPlan for HashJoinExec {
17471747
enable_dynamic_filter_pushdown,
17481748
Arc::clone(context.session_config().options()),
17491749
self.null_equality,
1750-
null_aware,
1750+
not_in,
17511751
array_map_created_count,
17521752
BuildMode::Ordinary,
17531753
))
@@ -3072,7 +3072,7 @@ async fn collect_left_input(
30723072
should_compute_dynamic_filters: bool,
30733073
config: Arc<ConfigOptions>,
30743074
null_equality: NullEquality,
3075-
null_aware: Option<NotIn>,
3075+
not_in: Option<NotIn>,
30763076
array_map_created_count: Count,
30773077
mode: BuildMode,
30783078
) -> Result<JoinLeftData> {
@@ -3165,7 +3165,7 @@ async fn collect_left_input(
31653165
should_compute_dynamic_filters,
31663166
&config,
31673167
null_equality,
3168-
null_aware,
3168+
not_in,
31693169
&array_map_created_count,
31703170
prepared,
31713171
)
@@ -3196,7 +3196,7 @@ fn build_in_memory(
31963196
should_compute_dynamic_filters: bool,
31973197
config: &ConfigOptions,
31983198
null_equality: NullEquality,
3199-
null_aware: Option<NotIn>,
3199+
not_in: Option<NotIn>,
32003200
array_map_created_count: &Count,
32013201
prepared: bool,
32023202
) -> Result<JoinLeftData> {
@@ -3325,7 +3325,7 @@ fn build_in_memory(
33253325
BooleanBufferBuilder::new(0)
33263326
};
33273327

3328-
let not_in = null_aware
3328+
let not_in = not_in
33293329
.map(|not_in| {
33303330
NotInBuild::try_new(
33313331
not_in,
@@ -10163,7 +10163,7 @@ mod tests {
1016310163
output: NotInOutput::Rows,
1016410164
},
1016510165
};
10166-
assert_eq!(join.null_aware_mode()?, Some(expected));
10166+
assert_eq!(join.not_in()?, Some(expected));
1016710167
Ok(())
1016810168
}
1016910169

‎datafusion/physical-plan/src/joins/hash_join/not_in.rs‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -315,6 +315,10 @@ impl NotInBuild {
315315
left_side: UInt64Array,
316316
right_side: UInt32Array,
317317
) -> (UInt64Array, UInt32Array, Option<ArrayRef>) {
318+
debug_assert!(
319+
!summary.has_null || summary.non_empty,
320+
"a probe batch with a NULL key has rows"
321+
);
318322
match self {
319323
Self::ProbeRowsCorrelated { .. } | Self::ProbeRowsUncorrelated { .. } => {
320324
(left_side, right_side, None)
@@ -690,17 +694,17 @@ pub(super) struct NullValueBuildRows {
690694
/// build rows. Stored positions index into `scope_values`/`build_indices`,
691695
/// not the full build batch. `None` when the join has no correlation
692696
/// scope keys, so every probe row is in scope.
693-
pub(super) scope_map: Option<Box<dyn JoinHashMapType>>,
697+
scope_map: Option<Box<dyn JoinHashMapType>>,
694698
/// Correlation scope key values of the NULL-valued build rows.
695-
pub(super) scope_values: Vec<ArrayRef>,
699+
scope_values: Vec<ArrayRef>,
696700
/// Maps positions in `map`/`scope_values` back to row indices in the full
697701
/// build batch.
698-
pub(super) build_indices: UInt64Array,
702+
build_indices: UInt64Array,
699703
}
700704

701705
/// Returns the rows where at least one of `value_keys` is NULL, or `None` when
702706
/// there is no such row.
703-
pub(super) fn null_value_key_mask(value_keys: &[ArrayRef]) -> Option<BooleanArray> {
707+
fn null_value_key_mask(value_keys: &[ArrayRef]) -> Option<BooleanArray> {
704708
let logical_nulls: Vec<_> = value_keys.iter().map(|v| v.logical_nulls()).collect();
705709
let valid = NullBuffer::union_many(logical_nulls.iter().map(Option::as_ref))?;
706710
(valid.null_count() > 0).then(|| BooleanArray::new(!valid.inner(), None))
@@ -709,7 +713,7 @@ pub(super) fn null_value_key_mask(value_keys: &[ArrayRef]) -> Option<BooleanArra
709713
/// Where a correlated null-aware join records the rows whose `NOT IN` is
710714
/// UNKNOWN, see [`mark_null_candidates_for_probe_batch`].
711715
#[derive(Clone, Copy)]
712-
pub(super) enum UnknownMarks<'a> {
716+
enum UnknownMarks<'a> {
713717
/// `LeftAnti` and `LeftMark` mark build rows, in the null-indices bitmap
714718
/// shared by all probe partitions.
715719
Build(&'a SharedBitmapBuilder),

‎datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,7 @@ pub(crate) struct SharedBuildAccumulator {
274274
null_equality: NullEquality,
275275
/// Null-aware anti join (`NOT IN`). A probe-side NULL must reach the join so its
276276
/// three-valued logic can collapse the result, so the pushed filter keeps NULL rows.
277-
null_aware: Option<NotIn>,
277+
not_in: Option<NotIn>,
278278
}
279279

280280
/// Strategy for filter pushdown (decided at collection time)
@@ -386,7 +386,7 @@ impl SharedBuildAccumulator {
386386
on_right: Vec<PhysicalExprRef>,
387387
repartition_random_state: SeededRandomState,
388388
null_equality: NullEquality,
389-
null_aware: Option<NotIn>,
389+
not_in: Option<NotIn>,
390390
) -> Self {
391391
// Troubleshooting: If partition counts are incorrect, verify this logic matches
392392
// the actual execution pattern in collect_build_side()
@@ -443,7 +443,7 @@ impl SharedBuildAccumulator {
443443
probe_schema: right_child.schema(),
444444
probe_range_partitioning,
445445
null_equality,
446-
null_aware,
446+
not_in,
447447
}
448448
}
449449

@@ -823,13 +823,13 @@ impl SharedBuildAccumulator {
823823
// probe NULL makes `NOT IN` unknown for every build row. A null-equal join needs probe
824824
// NULLs only to match an actual build-side NULL, so a NULL-free build keeps the filter
825825
// at full selectivity.
826-
let needs_probe_nulls = self.null_aware.is_some()
826+
let needs_probe_nulls = self.not_in.is_some()
827827
|| (self.null_equality == NullEquality::NullEqualsNull
828828
&& build_keys_have_null);
829829
if !needs_probe_nulls {
830830
return Ok(filter_expr);
831831
}
832-
let keys = match self.null_aware {
832+
let keys = match self.not_in {
833833
Some(mode) => {
834834
let value_keys = mode.value_keys();
835835
assert_or_internal_err!(
@@ -899,7 +899,7 @@ pub(super) fn make_partitioned_accumulator_for_test(
899899
probe_schema,
900900
probe_range_partitioning: None,
901901
null_equality: NullEquality::NullEqualsNothing,
902-
null_aware: None,
902+
not_in: None,
903903
}
904904
}
905905

@@ -965,7 +965,7 @@ mod tests {
965965
probe_schema: test_probe_schema(),
966966
probe_range_partitioning: None,
967967
null_equality: NullEquality::NullEqualsNothing,
968-
null_aware: None,
968+
not_in: None,
969969
}
970970
}
971971

@@ -1453,7 +1453,7 @@ mod tests {
14531453
probe_schema,
14541454
probe_range_partitioning: None,
14551455
null_equality,
1456-
null_aware: null_aware.then_some(NotIn::BuildRows {
1456+
not_in: null_aware.then_some(NotIn::BuildRows {
14571457
scope: NotInScope::Uncorrelated,
14581458
output: NotInOutput::Rows,
14591459
}),
@@ -1668,7 +1668,7 @@ mod tests {
16681668
probe_schema: Arc::new(Schema::new(fields)),
16691669
probe_range_partitioning: None,
16701670
null_equality: NullEquality::NullEqualsNothing,
1671-
null_aware: Some(NotIn::BuildRows {
1671+
not_in: Some(NotIn::BuildRows {
16721672
scope: NotInScope::Correlated { value_keys },
16731673
output: NotInOutput::Mark,
16741674
}),

0 commit comments

Comments
 (0)