Skip to content

Commit 4900ba2

Browse files
committed
Covered a NOT NULL probe key in SLT.
A declared NOT NULL probe key skips the disjunct even when the build holds a NULL: no probe row can be NULL, so there is nothing to keep.
1 parent 95d7aee commit 4900ba2

1 file changed

Lines changed: 41 additions & 0 deletions

File tree

‎datafusion/sqllogictest/test_files/push_down_filter_parquet.slt‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1150,6 +1150,47 @@ statement ok
11501150
drop table nnb_probe;
11511151

11521152

1153+
# A probe key declared NOT NULL skips the disjunct even when the build holds a NULL:
1154+
# no probe row can be NULL, so there is nothing to keep.
1155+
statement ok
1156+
COPY (SELECT * FROM (VALUES (11), (NULL)) v(id)) TO 'test_files/scratch/push_down_filter_parquet/nnp_build.parquet' STORED AS PARQUET;
1157+
1158+
statement ok
1159+
COPY (SELECT * FROM (VALUES (11), (33)) v(id)) TO 'test_files/scratch/push_down_filter_parquet/nnp_probe.parquet' STORED AS PARQUET;
1160+
1161+
statement ok
1162+
CREATE EXTERNAL TABLE nnp_build STORED AS PARQUET LOCATION 'test_files/scratch/push_down_filter_parquet/nnp_build.parquet';
1163+
1164+
statement ok
1165+
CREATE EXTERNAL TABLE nnp_probe (id BIGINT NOT NULL) STORED AS PARQUET LOCATION 'test_files/scratch/push_down_filter_parquet/nnp_probe.parquet';
1166+
1167+
# The build NULL matches nothing here: the probe cannot produce a NULL.
1168+
query II rowsort
1169+
SELECT nnp_build.id, nnp_probe.id FROM nnp_build JOIN nnp_probe ON nnp_build.id IS NOT DISTINCT FROM nnp_probe.id
1170+
----
1171+
11 11
1172+
1173+
statement ok
1174+
set datafusion.explain.analyze_categories = 'rows';
1175+
1176+
query TT
1177+
EXPLAIN ANALYZE SELECT nnp_build.id, nnp_probe.id FROM nnp_build JOIN nnp_probe ON nnp_build.id IS NOT DISTINCT FROM nnp_probe.id
1178+
----
1179+
Plan with Metrics
1180+
01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], NullsEqual: true, metrics=[output_rows=1, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=1, input_rows=1, avg_fanout=100% (1/1), probe_hit_rate=100% (1/1)]
1181+
02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/nnp_build.parquet]]}, projection=[id], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=12.92% (65/503)]
1182+
03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/nnp_probe.parquet]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ id@0 >= 11 AND id@0 <= 11 AND id@0 IN (SET) ([11, NULL]) ], dynamic_rg_pruning=eligible, pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 >= 11 AND id_null_count@1 != row_count@2 AND id_min@3 <= 11 AND (id_null_count@1 != row_count@2 AND id_min@3 <= 11 AND 11 <= id_max@0 OR id_null_count@1 != row_count@2 AND id_min@3 <= NULL AND NULL <= id_max@0), required_guarantees=[id in (11, NULL)], metrics=[output_rows=1, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=1 total → 1 matched, page_index_rows_pruned=2 total → 2 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=1, pushdown_rows_pruned=1, predicate_cache_inner_records=2, predicate_cache_records=1, scan_efficiency_ratio=13.71% (68/496)]
1183+
1184+
statement ok
1185+
reset datafusion.explain.analyze_categories;
1186+
1187+
statement ok
1188+
drop table nnp_build;
1189+
1190+
statement ok
1191+
drop table nnp_probe;
1192+
1193+
11531194
# Partitioned mode: the per-partition CASE filter gets the same IS NULL widening, so a
11541195
# probe NULL routed to a pruning branch still reaches the join.
11551196
statement ok

0 commit comments

Comments
 (0)