Skip to content

Commit 3919f28

Browse files
neilconwaylukekim
authored andcommitted
fix: Disable join dynamic filters for null-equal joins (apache#22965)
## Which issue does this PR close? - Closes #apache#22964 ## Rationale for this change We presently allow dynamic filter pushdown to be applied to null-equal hash joins. This might result in pushing a predicate down into the probe-side plan, where the predicate will not be evaluated with the null-equal semantics that are required. Longer-term, we might consider supporting this case with the correct semantics (e.g., generate a predicate with `OR IS NULL ...`), but for now disabling pushdown for null-equal joins seems much more practical. ## What changes are included in this PR? * Disable hash join dynamic filter pushdown for null-equal joins * Add SLT test with end-to-end repro * Add unit test ## Are these changes tested? Yes. ## Are there any user-facing changes? No. (cherry picked from commit baa497d) Conflicts resolved when applying to spiceai-54 (DataFusion 54.1): - datafusion/physical-plan/src/joins/hash_join/exec.rs: this branch already carries 5975a2d (apache#23173), which upstream merged later and which adds its own gate and tests at the same places. Placed this commit's gate and test ahead of those, the order they have upstream. - datafusion/sqllogictest/test_files/push_down_filter_parquet.slt: same; the null-equal block goes ahead of the apache#23173 block, as upstream has it.
1 parent 3e58817 commit 3919f28

2 files changed

Lines changed: 79 additions & 0 deletions

File tree

‎datafusion/physical-plan/src/joins/hash_join/exec.rs‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -905,6 +905,14 @@ impl<A: CollectLeftAccumulator + 'static> HashJoinExec<A> {
905905
return false;
906906
}
907907

908+
// Bounds and membership filters derived from the build side do not
909+
// account for null-equal matching: a probe-side NULL key evaluates
910+
// such predicates to NULL and would be pruned, even though it can
911+
// match a build-side NULL when nulls compare equal.
912+
if self.null_equality == NullEquality::NullEqualsNull {
913+
return false;
914+
}
915+
908916
// A null-aware anti join emits a build-side NULL only when the probe
909917
// is truly empty. The pushed filter can empty the probe by pruning
910918
// every row, which would surface that NULL wrongly. A NOT NULL build
@@ -6456,6 +6464,35 @@ mod tests {
64566464
Ok(())
64576465
}
64586466

6467+
#[test]
6468+
fn test_dynamic_filter_pushdown_rejects_null_equal_join() -> Result<()> {
6469+
let (_, _, on) = build_schema_and_on()?;
6470+
let left = build_table(("a1", &vec![1]), ("b1", &vec![1]), ("c1", &vec![1]));
6471+
let right = build_table(("a2", &vec![1]), ("b1", &vec![1]), ("c2", &vec![1]));
6472+
6473+
let mut session_config = SessionConfig::default();
6474+
session_config
6475+
.options_mut()
6476+
.optimizer
6477+
.enable_join_dynamic_filter_pushdown = true;
6478+
6479+
let join = HashJoinExec::try_new(
6480+
left,
6481+
right,
6482+
on,
6483+
None,
6484+
&JoinType::RightSemi,
6485+
None,
6486+
PartitionMode::CollectLeft,
6487+
NullEquality::NullEqualsNull,
6488+
false,
6489+
)?;
6490+
6491+
assert!(!join.allow_join_dynamic_filter_pushdown(session_config.options()));
6492+
6493+
Ok(())
6494+
}
6495+
64596496
#[test]
64606497
fn test_dynamic_filter_pushdown_rejects_null_aware_nullable_build_key() -> Result<()>
64616498
{

‎datafusion/sqllogictest/test_files/push_down_filter_parquet.slt‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1023,6 +1023,48 @@ statement ok
10231023
drop table int_probe;
10241024

10251025

1026+
########
1027+
# Dynamic filters must not be created for null-equal joins (IS NOT DISTINCT
1028+
# FROM, INTERSECT): min/max bounds and membership filters derived from the
1029+
# build side evaluate to NULL for probe-side NULL keys and would prune rows
1030+
# that can null-match a build-side NULL.
1031+
########
1032+
1033+
statement ok
1034+
COPY (SELECT * FROM (VALUES (11), (22), (NULL)) v(id)) TO 'test_files/scratch/push_down_filter_parquet/nej_probe.parquet' STORED AS PARQUET;
1035+
1036+
statement ok
1037+
COPY (SELECT * FROM (VALUES (11), (NULL)) v(id)) TO 'test_files/scratch/push_down_filter_parquet/nej_build.parquet' STORED AS PARQUET;
1038+
1039+
statement ok
1040+
CREATE EXTERNAL TABLE nej_probe STORED AS PARQUET LOCATION 'test_files/scratch/push_down_filter_parquet/nej_probe.parquet';
1041+
1042+
statement ok
1043+
CREATE EXTERNAL TABLE nej_build STORED AS PARQUET LOCATION 'test_files/scratch/push_down_filter_parquet/nej_build.parquet';
1044+
1045+
# The probe-side NULL key must survive to match the build-side NULL
1046+
query II rowsort
1047+
SELECT nej_build.id, nej_probe.id FROM nej_build JOIN nej_probe ON nej_build.id IS NOT DISTINCT FROM nej_probe.id
1048+
----
1049+
11 11
1050+
NULL NULL
1051+
1052+
# No DynamicFilter predicate may appear on the probe side of a null-equal join
1053+
query TT
1054+
EXPLAIN SELECT nej_build.id, nej_probe.id FROM nej_build JOIN nej_probe ON nej_build.id IS NOT DISTINCT FROM nej_probe.id
1055+
----
1056+
physical_plan
1057+
01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], NullsEqual: true
1058+
02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/nej_build.parquet]]}, projection=[id], file_type=parquet
1059+
03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/nej_probe.parquet]]}, projection=[id], file_type=parquet
1060+
1061+
statement ok
1062+
drop table nej_build;
1063+
1064+
statement ok
1065+
drop table nej_probe;
1066+
1067+
10261068
########
10271069
# Regression test for build-NULL + emptied-probe interaction in null-aware LeftAnti joins.
10281070
#

0 commit comments

Comments
 (0)