Skip to content

Commit 7dd27a6

Browse files
committed
fix(parquet): don't runtime-prune row groups while a page-index RowSelection is live (apache#24355)
The runtime row-group pruner rebuilds the decoder via into_builder().with_row_groups(new_indices), which drops row groups without slicing the carried page-index RowSelection to match. A dropped RG's selectors are then applied to the next surviving RG, silently returning wrong results. Disable the runtime pruner when a row selection is present, mirroring reorder_by_statistics which already declines to reorder in that case. The proper fix (slice the selection alongside the row groups) belongs in arrow-rs and is tracked in apache#24358. Adds an slt regression test: it fails on main (returns 50..54 instead of 0..4) and passes with this change.
1 parent 00eba79 commit 7dd27a6

2 files changed

Lines changed: 108 additions & 20 deletions

File tree

‎datafusion/datasource-parquet/src/opener/mod.rs‎

Lines changed: 33 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1435,7 +1435,7 @@ impl RowGroupsPrunedParquetOpen {
14351435
prepared.virtual_state.as_deref(),
14361436
)?;
14371437

1438-
let (decoder, rg_plan) = {
1438+
let (decoder, rg_plan, has_row_selection) = {
14391439
let pushdown_predicate = prepared
14401440
.pushdown_filters
14411441
.then_some(prepared.predicate.as_ref())
@@ -1464,6 +1464,14 @@ impl RowGroupsPrunedParquetOpen {
14641464
};
14651465

14661466
let prepared_access_plan = prepare_access_plan(access_plan)?;
1467+
// #24355: a page-index row selection is carried by the decoder as one
1468+
// flat selection over the concatenation of the remaining row groups.
1469+
// The runtime pruner's `into_builder().with_row_groups(...)` rebuild
1470+
// drops row groups without slicing that selection to match, so record
1471+
// whether a selection is present and disable runtime pruning below
1472+
// when it is (mirroring `reorder_by_statistics`, which also bails when
1473+
// a row selection is present).
1474+
let has_row_selection = prepared_access_plan.row_selection.is_some();
14671475
let rg_plan: VecDeque<RgPlanEntry> = prepared_access_plan
14681476
.row_group_indexes
14691477
.iter()
@@ -1482,7 +1490,7 @@ impl RowGroupsPrunedParquetOpen {
14821490
}
14831491
}
14841492

1485-
(builder.build()?, rg_plan)
1493+
(builder.build()?, rg_plan, has_row_selection)
14861494
};
14871495

14881496
let predicate_cache_inner_records =
@@ -1504,24 +1512,29 @@ impl RowGroupsPrunedParquetOpen {
15041512
// via the `DynamicFilterTracker` watch channel (#22460), so detecting
15051513
// a threshold change is a single atomic load — not a tree walk per
15061514
// RG check.
1507-
let row_group_pruner = match (&prepared.predicate, rg_plan.len() > 1) {
1508-
(Some(predicate), true)
1509-
if matches!(
1510-
DynamicFilterTracking::classify(predicate),
1511-
DynamicFilterTracking::Watching(_)
1512-
) =>
1513-
{
1514-
Some(RowGroupPruner::new(
1515-
Arc::clone(predicate),
1516-
Arc::clone(&prepared.physical_file_schema),
1517-
Arc::clone(reader_metadata.metadata()),
1518-
prepared.predicate_creation_errors.clone(),
1519-
prepared.file_metrics.predicate_evaluation_errors.clone(),
1520-
prepared.max_in_list_size,
1521-
))
1522-
}
1523-
_ => None,
1524-
};
1515+
// Also disabled when a page-index row selection is live (#24355): the
1516+
// pruner rebuilds the decoder via `with_row_groups(...)`, which drops row
1517+
// groups without slicing the carried selection to match, so pruning under
1518+
// a live selection returns wrong results. Decline to prune in that case.
1519+
let row_group_pruner =
1520+
match (&prepared.predicate, rg_plan.len() > 1, has_row_selection) {
1521+
(Some(predicate), true, false)
1522+
if matches!(
1523+
DynamicFilterTracking::classify(predicate),
1524+
DynamicFilterTracking::Watching(_)
1525+
) =>
1526+
{
1527+
Some(RowGroupPruner::new(
1528+
Arc::clone(predicate),
1529+
Arc::clone(&prepared.physical_file_schema),
1530+
Arc::clone(reader_metadata.metadata()),
1531+
prepared.predicate_creation_errors.clone(),
1532+
prepared.file_metrics.predicate_evaluation_errors.clone(),
1533+
prepared.max_in_list_size,
1534+
))
1535+
}
1536+
_ => None,
1537+
};
15251538
let row_groups_pruned_dynamic = prepared
15261539
.file_metrics
15271540
.row_groups_pruned_dynamic_filter

‎datafusion/sqllogictest/test_files/dynamic_row_group_pruning.slt‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,3 +194,78 @@ RESET datafusion.optimizer.enable_dynamic_filter_pushdown;
194194

195195
statement ok
196196
RESET datafusion.optimizer.enable_topk_dynamic_filter_pushdown;
197+
198+
# Regression test for #24355: the runtime row-group pruner rebuilds the decoder
199+
# via `into_builder().with_row_groups(...)`, which drops row groups without
200+
# slicing the carried page-index `RowSelection` to match — a dropped RG's
201+
# selectors are then applied to the next surviving RG. Layout (RG size 100):
202+
# RG 0: b=1000..1099, a=100..199 (a>=50 keeps all)
203+
# RG 1: b=2000..2099, a=0..99 (a>=50 keeps rows 50..99 — page-index prunes
204+
# the first 5 pages, leaving `skip 50, select 50`)
205+
# RG 2: b=3000..3099, a=100..199 (keeps all)
206+
# RG 3: b=0..99, a=100..199 (keeps all)
207+
# `ORDER BY b ASC LIMIT 5` tightens the TopK threshold; the runtime pruner drops
208+
# RG 1 and RG 2, rebuilds with row_groups=[3], but the unsliced `skip 50,
209+
# select 50` is applied to RG 3 — dropping b=0..49, which is the correct answer.
210+
# Without the fix (#24355) this returns 50..54; with it the pruner is disabled
211+
# while a row selection is live, so the answer is correct.
212+
# `data_page_row_count_limit`/`write_batch_size` force multiple pages per RG so
213+
# page-index pruning can produce an intra-RG selection.
214+
statement ok
215+
set datafusion.execution.target_partitions = 1;
216+
217+
statement ok
218+
set datafusion.execution.parquet.pushdown_filters = true;
219+
220+
statement ok
221+
CREATE TABLE rgsel_src AS
222+
SELECT
223+
CAST(CASE WHEN i / 100 = 1 THEN i % 100 ELSE 100 + (i % 100) END AS BIGINT) AS a,
224+
CAST(CASE
225+
WHEN i < 100 THEN 1000 + i
226+
WHEN i < 200 THEN 2000 + (i - 100)
227+
WHEN i < 300 THEN 3000 + (i - 200)
228+
ELSE (i - 300)
229+
END AS BIGINT) AS b
230+
FROM generate_series(0, 399) AS t(i);
231+
232+
statement ok
233+
COPY (SELECT * FROM rgsel_src)
234+
TO 'test_files/scratch/dynamic_row_group_pruning/rgsel.parquet'
235+
STORED AS PARQUET
236+
OPTIONS (
237+
'format.max_row_group_size' '100',
238+
'format.data_page_row_count_limit' '10',
239+
'format.write_batch_size' '10'
240+
);
241+
242+
statement ok
243+
drop table rgsel_src;
244+
245+
statement ok
246+
CREATE EXTERNAL TABLE rgsel (a BIGINT NOT NULL, b BIGINT NOT NULL)
247+
STORED AS PARQUET
248+
LOCATION 'test_files/scratch/dynamic_row_group_pruning/rgsel.parquet';
249+
250+
# The correct top-5 by `b` among rows with `a >= 50` is b = 0..4 (they live in
251+
# RG 3, all of whose rows satisfy `a >= 50`). The bug returns 50..54.
252+
query I
253+
SELECT b FROM rgsel WHERE a >= 50 ORDER BY b ASC LIMIT 5;
254+
----
255+
0
256+
1
257+
2
258+
3
259+
4
260+
261+
statement ok
262+
drop table rgsel;
263+
264+
# The SLT runner sets `target_partitions` to 4 instead of using the default, so
265+
# restore it explicitly rather than RESET (which would revert to the system
266+
# default = num_cpus and leak modified config out of this file).
267+
statement ok
268+
set datafusion.execution.target_partitions = 4;
269+
270+
statement ok
271+
RESET datafusion.execution.parquet.pushdown_filters;

0 commit comments

Comments
 (0)