Skip to content

Commit d56edfd

Browse files
adriangbclaude
andcommitted
test: cover InList pushdown dedup end to end
Adds a sqllogictest where the build side has 6 rows but only 2 distinct keys, asserting the pushed-down dynamic filter is `IN (SET) ([11, 22])` and the derived `required_guarantees` is `[id in (11, 22)]`. Before the previous commit both listed all 6 build rows. Co-Authored-By: Claude Opus 5 <[email protected]>
1 parent 3296003 commit d56edfd

1 file changed

Lines changed: 54 additions & 0 deletions

File tree

‎datafusion/sqllogictest/test_files/push_down_filter_parquet.slt‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1336,3 +1336,57 @@ RESET datafusion.execution.parquet.pushdown_filters;
13361336

13371337
statement ok
13381338
drop table t;
1339+
1340+
1341+
########
1342+
# The InList pushdown is gated on the build side's *distinct* key count, but the
1343+
# values it ships come from the raw build-side key arrays (one entry per build
1344+
# row). Duplicated build keys must not inflate the pushed-down `IN (SET)` list or
1345+
# the `required_guarantees` derived from it.
1346+
########
1347+
1348+
statement ok
1349+
COPY (SELECT * FROM (VALUES (11), (11), (11), (22), (22), (22)) v(id)) TO 'test_files/scratch/push_down_filter_parquet/dedup_build.parquet' STORED AS PARQUET;
1350+
1351+
statement ok
1352+
COPY (SELECT * FROM (VALUES (11), (22), (33), (44), (55), (66), (77), (88), (99), (110), (121), (132)) v(id)) TO 'test_files/scratch/push_down_filter_parquet/dedup_probe.parquet' STORED AS PARQUET;
1353+
1354+
statement ok
1355+
CREATE EXTERNAL TABLE dedup_build STORED AS PARQUET LOCATION 'test_files/scratch/push_down_filter_parquet/dedup_build.parquet';
1356+
1357+
statement ok
1358+
CREATE EXTERNAL TABLE dedup_probe STORED AS PARQUET LOCATION 'test_files/scratch/push_down_filter_parquet/dedup_probe.parquet';
1359+
1360+
# Data-correctness: deduplicating the pushed-down set must not drop join output
1361+
# rows, which still repeat once per matching build row.
1362+
query II rowsort
1363+
SELECT dedup_build.id, dedup_probe.id FROM dedup_build JOIN dedup_probe ON dedup_build.id = dedup_probe.id
1364+
----
1365+
11 11
1366+
11 11
1367+
11 11
1368+
22 22
1369+
22 22
1370+
22 22
1371+
1372+
statement ok
1373+
set datafusion.explain.analyze_categories = 'rows';
1374+
1375+
# The six build rows carry only two distinct keys, so the pushed-down filter is
1376+
# `IN (SET) ([11, 22])`, not `([11, 11, 11, 22, 22, 22])`.
1377+
query TT
1378+
EXPLAIN ANALYZE SELECT dedup_build.id, dedup_probe.id FROM dedup_build JOIN dedup_probe ON dedup_build.id = dedup_probe.id
1379+
----
1380+
Plan with Metrics
1381+
01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], metrics=[output_rows=6, output_batches=1, array_map_created_count=1, build_input_batches=1, build_input_rows=6, input_batches=1, input_rows=12, avg_fanout=300% (6/2), probe_hit_rate=16.67% (2/12)]
1382+
02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/dedup_build.parquet]]}, projection=[id], file_type=parquet, metrics=[output_rows=6, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=13.71% (68/496)]
1383+
03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/dedup_probe.parquet]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ id@0 >= 11 AND id@0 <= 22 AND id@0 IN (SET) ([11, 22]) ], dynamic_rg_pruning=eligible, pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 >= 11 AND id_null_count@1 != row_count@2 AND id_min@3 <= 22 AND (id_null_count@1 != row_count@2 AND id_min@3 <= 11 AND 11 <= id_max@0 OR id_null_count@1 != row_count@2 AND id_min@3 <= 22 AND 22 <= id_max@0), required_guarantees=[id in (11, 22)], metrics=[output_rows=12, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=1 total → 1 matched, page_index_rows_pruned=12 total → 12 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=18.34% (97/529)]
1384+
1385+
statement ok
1386+
reset datafusion.explain.analyze_categories;
1387+
1388+
statement ok
1389+
drop table dedup_build;
1390+
1391+
statement ok
1392+
drop table dedup_probe;

0 commit comments

Comments
 (0)