Skip to content

Commit 127731b

Browse files
authored
Include null_aware status in the relevant Join node display implementations (apache#22913)
## Which issue does this PR close? - Closes apache#22912. ## Rationale for this change This change makes testing null_aware behavior easier, and also makes the performance of various joins clearer - null_aware joins do extra work. This was originally part apache#21585, but it seems like there is a bunch of activity around null-aware joins, so I figured its worth splitting out. ## What changes are included in this PR? Add a `null_aware` indication to relevant Display implementations when appropriate. ## Are these changes tested? SLT tests ## Are there any user-facing changes? Display only --------- Signed-off-by: Adam Gutglick <[email protected]>
1 parent c14379b commit 127731b

7 files changed

Lines changed: 50 additions & 13 deletions

File tree

‎datafusion/expr/src/logical_plan/plan.rs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2073,6 +2073,7 @@ impl LogicalPlan {
20732073
filter,
20742074
join_constraint,
20752075
join_type,
2076+
null_aware,
20762077
..
20772078
}) => {
20782079
let join_expr: Vec<String> =
@@ -2081,6 +2082,8 @@ impl LogicalPlan {
20812082
.as_ref()
20822083
.map(|expr| format!(" Filter: {expr}"))
20832084
.unwrap_or_else(|| "".to_string());
2085+
let null_aware_expr =
2086+
if *null_aware { " null_aware" } else { "" };
20842087
let join_type = if filter.is_none()
20852088
&& keys.is_empty()
20862089
&& *join_type == JoinType::Inner
@@ -2100,15 +2103,17 @@ impl LogicalPlan {
21002103
filter_expr
21012104
)?;
21022105
}
2106+
write!(f, "{null_aware_expr}")?;
21032107
Ok(())
21042108
}
21052109
JoinConstraint::Using => {
21062110
write!(
21072111
f,
2108-
"{} Join: Using {}{}",
2112+
"{} Join: Using {}{}{}",
21092113
join_type,
21102114
join_expr.join(", "),
21112115
filter_expr,
2116+
null_aware_expr,
21122117
)
21132118
}
21142119
}

‎datafusion/physical-plan/src/joins/hash_join/exec.rs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1140,6 +1140,8 @@ impl DisplayAs for HashJoinExec {
11401140
let display_fetch = self
11411141
.fetch
11421142
.map_or_else(String::new, |f| format!(", fetch={f}"));
1143+
let display_null_aware =
1144+
if self.null_aware { ", null_aware" } else { "" };
11431145
let on = self
11441146
.on
11451147
.iter()
@@ -1148,14 +1150,15 @@ impl DisplayAs for HashJoinExec {
11481150
.join(", ");
11491151
write!(
11501152
f,
1151-
"HashJoinExec: mode={:?}, join_type={:?}, on=[{}]{}{}{}{}",
1153+
"HashJoinExec: mode={:?}, join_type={:?}, on=[{}]{}{}{}{}{}",
11521154
self.mode,
11531155
self.join_type,
11541156
on,
11551157
display_filter,
11561158
display_projections,
11571159
display_null_equality,
11581160
display_fetch,
1161+
display_null_aware,
11591162
)
11601163
}
11611164
DisplayFormatType::TreeRender => {
@@ -1178,6 +1181,10 @@ impl DisplayAs for HashJoinExec {
11781181
writeln!(f, "NullsEqual: true")?;
11791182
}
11801183

1184+
if self.null_aware {
1185+
writeln!(f, "null_aware")?;
1186+
}
1187+
11811188
if let Some(filter) = self.filter.as_ref() {
11821189
writeln!(f, "filter={filter}")?;
11831190
}

‎datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -374,14 +374,14 @@ FROM left_parquet l
374374
WHERE l.id NOT IN (SELECT r.id FROM right_parquet r);
375375
----
376376
logical_plan
377-
01)LeftAnti Join: l.id = __correlated_sq_1.id
377+
01)LeftAnti Join: l.id = __correlated_sq_1.id null_aware
378378
02)--SubqueryAlias: l
379379
03)----TableScan: left_parquet projection=[id, data]
380380
04)--SubqueryAlias: __correlated_sq_1
381381
05)----SubqueryAlias: r
382382
06)------TableScan: right_parquet projection=[id]
383383
physical_plan
384-
01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)]
384+
01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], null_aware
385385
02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet
386386
03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ]
387387

@@ -469,15 +469,15 @@ ORDER BY l.id LIMIT 2;
469469
----
470470
logical_plan
471471
01)Sort: l.id ASC NULLS LAST, fetch=2
472-
02)--LeftAnti Join: l.id = __correlated_sq_1.id
472+
02)--LeftAnti Join: l.id = __correlated_sq_1.id null_aware
473473
03)----SubqueryAlias: l
474474
04)------TableScan: left_parquet projection=[id, data]
475475
05)----SubqueryAlias: __correlated_sq_1
476476
06)------SubqueryAlias: r
477477
07)--------TableScan: right_parquet projection=[id]
478478
physical_plan
479479
01)SortExec: TopK(fetch=2), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
480-
02)--HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)]
480+
02)--HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], null_aware
481481
03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ]
482482
04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ]
483483

‎datafusion/sqllogictest/test_files/explain_tree.slt‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1100,6 +1100,31 @@ physical_plan
11001100
24)-----------------------------│ format: csv │
11011101
25)-----------------------------└───────────────────────────┘
11021102

1103+
# Query with null-aware anti join (NOT IN subquery).
1104+
query TT
1105+
explain select int_col from table1 where int_col not in (select int_col from table2);
1106+
----
1107+
physical_plan
1108+
01)┌───────────────────────────┐
1109+
02)│ HashJoinExec │
1110+
03)│ -------------------- │
1111+
04)│ join_type: LeftAnti │
1112+
05)│ │
1113+
06)│ null_aware ├──────────────┐
1114+
07)│ │ │
1115+
08)│ on: │ │
1116+
09)│ (int_col = int_col) │ │
1117+
10)└─────────────┬─────────────┘ │
1118+
11)┌─────────────┴─────────────┐┌─────────────┴─────────────┐
1119+
12)│ DataSourceExec ││ DataSourceExec │
1120+
13)│ -------------------- ││ -------------------- │
1121+
14)│ files: 1 ││ files: 1 │
1122+
15)│ format: csv ││ format: parquet │
1123+
16)│ ││ │
1124+
17)│ ││ predicate: │
1125+
18)│ ││ DynamicFilter [ empty ] │
1126+
19)└───────────────────────────┘└───────────────────────────┘
1127+
11031128
# Query with nested loop join.
11041129
query TT
11051130
explain select int_col from table1 where exists (select count(*) from table2);

‎datafusion/sqllogictest/test_files/joins.slt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1989,7 +1989,7 @@ where join_t1.t1_id + 12 not in
19891989
(select join_t2.t2_id + 1 from join_t2 where join_t1.t1_int > 0)
19901990
----
19911991
logical_plan
1992-
01)LeftAnti Join: CAST(join_t1.t1_id AS Int64) + Int64(12) = __correlated_sq_1.join_t2.t2_id + Int64(1) Filter: join_t1.t1_int > UInt32(0)
1992+
01)LeftAnti Join: CAST(join_t1.t1_id AS Int64) + Int64(12) = __correlated_sq_1.join_t2.t2_id + Int64(1) Filter: join_t1.t1_int > UInt32(0) null_aware
19931993
02)--TableScan: join_t1 projection=[t1_id, t1_name, t1_int]
19941994
03)--SubqueryAlias: __correlated_sq_1
19951995
04)----Projection: CAST(join_t2.t2_id AS Int64) + Int64(1)

‎datafusion/sqllogictest/test_files/null_aware_anti_join.slt‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,12 +53,12 @@ query TT
5353
EXPLAIN SELECT * FROM outer_table WHERE id NOT IN (SELECT id FROM inner_table_no_null);
5454
----
5555
logical_plan
56-
01)LeftAnti Join: outer_table.id = __correlated_sq_1.id
56+
01)LeftAnti Join: outer_table.id = __correlated_sq_1.id null_aware
5757
02)--TableScan: outer_table projection=[id, value]
5858
03)--SubqueryAlias: __correlated_sq_1
5959
04)----TableScan: inner_table_no_null projection=[id]
6060
physical_plan
61-
01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)]
61+
01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], null_aware
6262
02)--DataSourceExec: partitions=1, partition_sizes=[1]
6363
03)--DataSourceExec: partitions=1, partition_sizes=[1]
6464

@@ -193,12 +193,12 @@ query TT
193193
EXPLAIN SELECT * FROM outer_table WHERE id NOT IN (SELECT id FROM inner_table_with_null);
194194
----
195195
logical_plan
196-
01)LeftAnti Join: outer_table.id = __correlated_sq_1.id
196+
01)LeftAnti Join: outer_table.id = __correlated_sq_1.id null_aware
197197
02)--TableScan: outer_table projection=[id, value]
198198
03)--SubqueryAlias: __correlated_sq_1
199199
04)----TableScan: inner_table_with_null projection=[id]
200200
physical_plan
201-
01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)]
201+
01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], null_aware
202202
02)--DataSourceExec: partitions=1, partition_sizes=[1]
203203
03)--DataSourceExec: partitions=1, partition_sizes=[1]
204204

‎datafusion/sqllogictest/test_files/tpch/plans/q16.slt.part‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ logical_plan
5454
02)--Projection: part.p_brand, part.p_type, part.p_size, count(alias1) AS supplier_cnt
5555
03)----Aggregate: groupBy=[[part.p_brand, part.p_type, part.p_size]], aggr=[[count(alias1)]]
5656
04)------Aggregate: groupBy=[[part.p_brand, part.p_type, part.p_size, partsupp.ps_suppkey AS alias1]], aggr=[[]]
57-
05)--------LeftAnti Join: partsupp.ps_suppkey = __correlated_sq_1.s_suppkey
57+
05)--------LeftAnti Join: partsupp.ps_suppkey = __correlated_sq_1.s_suppkey null_aware
5858
06)----------Projection: partsupp.ps_suppkey, part.p_brand, part.p_type, part.p_size
5959
07)------------Inner Join: partsupp.ps_partkey = part.p_partkey
6060
08)--------------TableScan: partsupp projection=[ps_partkey, ps_suppkey]
@@ -74,7 +74,7 @@ physical_plan
7474
07)------------AggregateExec: mode=FinalPartitioned, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, alias1@3 as alias1], aggr=[]
7575
08)--------------RepartitionExec: partitioning=Hash([p_brand@0, p_type@1, p_size@2, alias1@3], 4), input_partitions=4
7676
09)----------------AggregateExec: mode=Partial, gby=[p_brand@1 as p_brand, p_type@2 as p_type, p_size@3 as p_size, ps_suppkey@0 as alias1], aggr=[]
77-
10)------------------HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(ps_suppkey@0, s_suppkey@0)]
77+
10)------------------HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(ps_suppkey@0, s_suppkey@0)], null_aware
7878
11)--------------------CoalescePartitionsExec
7979
12)----------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ps_partkey@0, p_partkey@0)], projection=[ps_suppkey@1, p_brand@3, p_type@4, p_size@5]
8080
13)------------------------RepartitionExec: partitioning=Hash([ps_partkey@0], 4), input_partitions=4

0 commit comments

Comments
 (0)