Skip to content

Commit 1c1d157

Browse files
committed
test: add apache#24352 regression test for TopK dynamic filter + pushdown re-read
Adds the reporter's fixture as an slt case: the filter column (search_phrase) differs from the sort column (event_time), and one row group has an empty post-predicate selection invisible to statistics. Before the rg_plan/ decoder-frontier sync, a runtime prune re-read an already-delivered row group, duplicating rows and dropping the top-k tail. The query now returns the correct p0, p4096..p4104.
1 parent 35883b5 commit 1c1d157

1 file changed

Lines changed: 68 additions & 0 deletions

File tree

‎datafusion/sqllogictest/test_files/dynamic_row_group_pruning.slt‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,3 +152,71 @@ RESET datafusion.execution.parquet.pushdown_filters;
152152

153153
statement ok
154154
RESET datafusion.explain.analyze_level;
155+
156+
# Regression test for #24352: TopK dynamic filter + `pushdown_filters` must not
157+
# re-read an already-delivered row group. The filter column (`search_phrase`)
158+
# differs from the sort column (`event_time`), and one row group has an empty
159+
# post-predicate selection that row-group statistics cannot see — its only small
160+
# `event_time` (50) sits on the row where `search_phrase = ''`. arrow-rs finishes
161+
# that RG without handing back a reader; before `rg_plan` was synced to the
162+
# decoder frontier via `peek_next_row_group`, `rg_plan` trailed the decoder by
163+
# one, so a later runtime prune rebuilt the decoder from a stale plan and
164+
# re-read an already-delivered RG — duplicating rows and dropping the true
165+
# top-k tail.
166+
statement ok
167+
set datafusion.execution.parquet.pushdown_filters = true;
168+
169+
statement ok
170+
set datafusion.execution.target_partitions = 1;
171+
172+
statement ok
173+
CREATE TABLE q26_src AS
174+
SELECT
175+
CAST(CASE
176+
WHEN i < 2048 THEN i * 1000
177+
WHEN i < 4096 THEN (CASE WHEN i = 2048 THEN 50 ELSE 20000 + i END)
178+
WHEN i < 6144 THEN 100 + (i - 4096)
179+
ELSE 5000 + (i - 6144)
180+
END AS BIGINT) AS event_time,
181+
CASE WHEN i = 2048 THEN '' ELSE 'p' || CAST(i AS VARCHAR) END AS search_phrase
182+
FROM generate_series(0, 8191) AS t(i);
183+
184+
statement ok
185+
COPY (SELECT * FROM q26_src)
186+
TO 'test_files/scratch/dynamic_row_group_pruning/q26.parquet'
187+
STORED AS PARQUET
188+
OPTIONS ('format.max_row_group_size' '2048');
189+
190+
statement ok
191+
drop table q26_src;
192+
193+
statement ok
194+
CREATE EXTERNAL TABLE q26 (event_time BIGINT NOT NULL, search_phrase VARCHAR NOT NULL)
195+
STORED AS PARQUET
196+
LOCATION 'test_files/scratch/dynamic_row_group_pruning/q26.parquet';
197+
198+
# Each search_phrase is unique, so any repeated value would be the same source
199+
# row emitted twice. The result must be the 10 smallest-`event_time` non-empty
200+
# phrases with no duplicates (matches DuckDB and pushdown-off DataFusion).
201+
query T
202+
SELECT search_phrase FROM q26 WHERE search_phrase <> '' ORDER BY event_time LIMIT 10;
203+
----
204+
p0
205+
p4096
206+
p4097
207+
p4098
208+
p4099
209+
p4100
210+
p4101
211+
p4102
212+
p4103
213+
p4104
214+
215+
statement ok
216+
drop table q26;
217+
218+
statement ok
219+
RESET datafusion.execution.parquet.pushdown_filters;
220+
221+
statement ok
222+
RESET datafusion.execution.target_partitions;

0 commit comments

Comments
 (0)