@@ -1336,3 +1336,57 @@ RESET datafusion.execution.parquet.pushdown_filters;
13361336
13371337statement ok
13381338drop table t;
1339+
1340+
1341+ ########
1342+ # The InList pushdown is gated on the build side's *distinct* key count, but the
1343+ # values it ships come from the raw build-side key arrays (one entry per build
1344+ # row). Duplicated build keys must not inflate the pushed-down `IN (SET)` list or
1345+ # the `required_guarantees` derived from it.
1346+ ########
1347+
1348+ statement ok
1349+ COPY (SELECT * FROM (VALUES (11), (11), (11), (22), (22), (22)) v(id)) TO 'test_files/scratch/push_down_filter_parquet/dedup_build.parquet' STORED AS PARQUET;
1350+
1351+ statement ok
1352+ COPY (SELECT * FROM (VALUES (11), (22), (33), (44), (55), (66), (77), (88), (99), (110), (121), (132)) v(id)) TO 'test_files/scratch/push_down_filter_parquet/dedup_probe.parquet' STORED AS PARQUET;
1353+
1354+ statement ok
1355+ CREATE EXTERNAL TABLE dedup_build STORED AS PARQUET LOCATION 'test_files/scratch/push_down_filter_parquet/dedup_build.parquet';
1356+
1357+ statement ok
1358+ CREATE EXTERNAL TABLE dedup_probe STORED AS PARQUET LOCATION 'test_files/scratch/push_down_filter_parquet/dedup_probe.parquet';
1359+
1360+ # Data-correctness: deduplicating the pushed-down set must not drop join output
1361+ # rows, which still repeat once per matching build row.
1362+ query II rowsort
1363+ SELECT dedup_build.id, dedup_probe.id FROM dedup_build JOIN dedup_probe ON dedup_build.id = dedup_probe.id
1364+ ----
1365+ 11 11
1366+ 11 11
1367+ 11 11
1368+ 22 22
1369+ 22 22
1370+ 22 22
1371+
1372+ statement ok
1373+ set datafusion.explain.analyze_categories = 'rows';
1374+
1375+ # The six build rows carry only two distinct keys, so the pushed-down filter is
1376+ # `IN (SET) ([11, 22])`, not `([11, 11, 11, 22, 22, 22])`.
1377+ query TT
1378+ EXPLAIN ANALYZE SELECT dedup_build.id, dedup_probe.id FROM dedup_build JOIN dedup_probe ON dedup_build.id = dedup_probe.id
1379+ ----
1380+ Plan with Metrics
1381+ 01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], metrics=[output_rows=6, output_batches=1, array_map_created_count=1, build_input_batches=1, build_input_rows=6, input_batches=1, input_rows=12, avg_fanout=300% (6/2), probe_hit_rate=16.67% (2/12)]
1382+ 02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/dedup_build.parquet]]}, projection=[id], file_type=parquet, metrics=[output_rows=6, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=13.71% (68/496)]
1383+ 03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/dedup_probe.parquet]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ id@0 >= 11 AND id@0 <= 22 AND id@0 IN (SET) ([11, 22]) ], dynamic_rg_pruning=eligible, pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 >= 11 AND id_null_count@1 != row_count@2 AND id_min@3 <= 22 AND (id_null_count@1 != row_count@2 AND id_min@3 <= 11 AND 11 <= id_max@0 OR id_null_count@1 != row_count@2 AND id_min@3 <= 22 AND 22 <= id_max@0), required_guarantees=[id in (11, 22)], metrics=[output_rows=12, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=1 total → 1 matched, page_index_rows_pruned=12 total → 12 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=18.34% (97/529)]
1384+
1385+ statement ok
1386+ reset datafusion.explain.analyze_categories;
1387+
1388+ statement ok
1389+ drop table dedup_build;
1390+
1391+ statement ok
1392+ drop table dedup_probe;
0 commit comments