Skip to content

Commit 5617063

Browse files
committed
test: add rust integration test for the apache#24352 rg_plan desync
Adds topk_pushdown_does_not_reread_delivered_row_group in dynamic_row_group_pruning.rs (same style as the existing tests): builds the reporter's 4-row-group fixture via with_custom_data (one RG with an empty post-predicate selection invisible to statistics), and asserts the top-k has no duplicate (p4096 appears exactly once) and the true tail is present.
1 parent 4faf38e commit 5617063

1 file changed

Lines changed: 100 additions & 1 deletion

File tree

‎datafusion/core/tests/parquet/dynamic_row_group_pruning.rs‎

Lines changed: 100 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
3333
use std::sync::Arc;
3434

35-
use arrow::array::{ArrayRef, Int64Array, RecordBatch};
35+
use arrow::array::{ArrayRef, Int64Array, RecordBatch, StringArray};
3636
use arrow_schema::{DataType, Field, Schema};
3737

3838
use crate::parquet::Unit::RowGroup;
@@ -585,3 +585,102 @@ async fn dynamic_rg_pruning_fires_for_multi_column_sort_leading_tied() {
585585
output.description(),
586586
);
587587
}
588+
589+
/// Build the #24352 fixture: four 2048-row row groups where the filter column
590+
/// (`search_phrase`) differs from the sort column (`event_time`), and one row
591+
/// group (the second) has an empty post-predicate selection invisible to
592+
/// statistics — its only small `event_time` (50) sits on the row whose
593+
/// `search_phrase` is `''`.
594+
///
595+
/// RG 0: event_time = i*1000 (i in 0..2048)
596+
/// RG 1: i=2048 -> (50, ''), else (20000+i, 'p'||i) (i in 2048..4096)
597+
/// RG 2: event_time = 100 + (i-4096) (i in 4096..6144)
598+
/// RG 3: event_time = 5000 + (i-6144) (i in 6144..8192)
599+
fn build_q26_batches(schema: &Arc<Schema>) -> Vec<RecordBatch> {
600+
(0..4i64)
601+
.map(|rg| {
602+
let mut event_time = Vec::with_capacity(2048);
603+
let mut search_phrase: Vec<String> = Vec::with_capacity(2048);
604+
for j in 0..2048i64 {
605+
let i = rg * 2048 + j;
606+
let (et, sp) = if i < 2048 {
607+
(i * 1000, format!("p{i}"))
608+
} else if i < 4096 {
609+
if i == 2048 {
610+
(50, String::new())
611+
} else {
612+
(20000 + i, format!("p{i}"))
613+
}
614+
} else if i < 6144 {
615+
(100 + (i - 4096), format!("p{i}"))
616+
} else {
617+
(5000 + (i - 6144), format!("p{i}"))
618+
};
619+
event_time.push(et);
620+
search_phrase.push(sp);
621+
}
622+
RecordBatch::try_new(
623+
Arc::clone(schema),
624+
vec![
625+
Arc::new(Int64Array::from(event_time)) as ArrayRef,
626+
Arc::new(StringArray::from(search_phrase)) as ArrayRef,
627+
],
628+
)
629+
.unwrap()
630+
})
631+
.collect()
632+
}
633+
634+
/// Regression for #24352: with `pushdown_filters` + TopK dynamic filter, a row
635+
/// group whose post-predicate selection is empty is silently finished by
636+
/// arrow-rs without handing back a reader. Before `rg_plan` was synced to the
637+
/// decoder frontier (`peek_next_row_group`), it trailed the decoder by one, so
638+
/// a later runtime prune rebuilt the decoder from a stale plan and re-read an
639+
/// already-delivered row group — the duplicate rows displaced the true top-k.
640+
#[tokio::test]
641+
async fn topk_pushdown_does_not_reread_delivered_row_group() {
642+
let schema = Arc::new(Schema::new(vec![
643+
Field::new("event_time", DataType::Int64, false),
644+
Field::new("search_phrase", DataType::Utf8, false),
645+
]));
646+
let batches = build_q26_batches(&schema);
647+
648+
// `RowGroup(2048)` writes one row group per 2048-row batch (4 RGs) and
649+
// enables `pushdown_filters`, required for the dynamic filter to reach the
650+
// parquet scan.
651+
let mut ctx = ContextWithParquet::with_custom_data(
652+
Scenario::Int,
653+
RowGroup(2048),
654+
Arc::clone(&schema),
655+
batches,
656+
)
657+
.await;
658+
659+
let output = ctx
660+
.query(
661+
"SELECT search_phrase FROM t \
662+
WHERE search_phrase <> '' ORDER BY event_time LIMIT 10",
663+
)
664+
.await;
665+
666+
// `search_phrase` is unique per row, so any repeated value is the same
667+
// source row emitted twice. The correct answer is the 10 smallest-
668+
// `event_time` non-empty phrases, matching DuckDB / pushdown-off.
669+
assert_eq!(output.result_rows, 10, "{}", output.description());
670+
let formatted = output.pretty_results();
671+
for p in [
672+
"p0", "p4096", "p4097", "p4098", "p4099", "p4100", "p4101", "p4102", "p4103",
673+
"p4104",
674+
] {
675+
assert!(
676+
formatted.contains(&format!("| {p} ")),
677+
"missing {p} from top-k; got:\n{formatted}",
678+
);
679+
}
680+
// The bug emitted p4096 twice (and dropped p4101..=p4104); assert no dup.
681+
assert_eq!(
682+
formatted.matches("| p4096 ").count(),
683+
1,
684+
"p4096 emitted more than once — rg_plan/decoder desync; got:\n{formatted}",
685+
);
686+
}

0 commit comments

Comments
 (0)