|
32 | 32 |
|
33 | 33 | use std::sync::Arc; |
34 | 34 |
|
35 | | -use arrow::array::{ArrayRef, Int64Array, RecordBatch}; |
| 35 | +use arrow::array::{ArrayRef, Int64Array, RecordBatch, StringArray}; |
36 | 36 | use arrow_schema::{DataType, Field, Schema}; |
37 | 37 |
|
38 | 38 | use crate::parquet::Unit::RowGroup; |
@@ -585,3 +585,102 @@ async fn dynamic_rg_pruning_fires_for_multi_column_sort_leading_tied() { |
585 | 585 | output.description(), |
586 | 586 | ); |
587 | 587 | } |
| 588 | + |
| 589 | +/// Build the #24352 fixture: four 2048-row row groups where the filter column |
| 590 | +/// (`search_phrase`) differs from the sort column (`event_time`), and one row |
| 591 | +/// group (the second) has an empty post-predicate selection invisible to |
| 592 | +/// statistics — its only small `event_time` (50) sits on the row whose |
| 593 | +/// `search_phrase` is `''`. |
| 594 | +/// |
| 595 | +/// RG 0: event_time = i*1000 (i in 0..2048) |
| 596 | +/// RG 1: i=2048 -> (50, ''), else (20000+i, 'p'||i) (i in 2048..4096) |
| 597 | +/// RG 2: event_time = 100 + (i-4096) (i in 4096..6144) |
| 598 | +/// RG 3: event_time = 5000 + (i-6144) (i in 6144..8192) |
| 599 | +fn build_q26_batches(schema: &Arc<Schema>) -> Vec<RecordBatch> { |
| 600 | + (0..4i64) |
| 601 | + .map(|rg| { |
| 602 | + let mut event_time = Vec::with_capacity(2048); |
| 603 | + let mut search_phrase: Vec<String> = Vec::with_capacity(2048); |
| 604 | + for j in 0..2048i64 { |
| 605 | + let i = rg * 2048 + j; |
| 606 | + let (et, sp) = if i < 2048 { |
| 607 | + (i * 1000, format!("p{i}")) |
| 608 | + } else if i < 4096 { |
| 609 | + if i == 2048 { |
| 610 | + (50, String::new()) |
| 611 | + } else { |
| 612 | + (20000 + i, format!("p{i}")) |
| 613 | + } |
| 614 | + } else if i < 6144 { |
| 615 | + (100 + (i - 4096), format!("p{i}")) |
| 616 | + } else { |
| 617 | + (5000 + (i - 6144), format!("p{i}")) |
| 618 | + }; |
| 619 | + event_time.push(et); |
| 620 | + search_phrase.push(sp); |
| 621 | + } |
| 622 | + RecordBatch::try_new( |
| 623 | + Arc::clone(schema), |
| 624 | + vec![ |
| 625 | + Arc::new(Int64Array::from(event_time)) as ArrayRef, |
| 626 | + Arc::new(StringArray::from(search_phrase)) as ArrayRef, |
| 627 | + ], |
| 628 | + ) |
| 629 | + .unwrap() |
| 630 | + }) |
| 631 | + .collect() |
| 632 | +} |
| 633 | + |
| 634 | +/// Regression for #24352: with `pushdown_filters` + TopK dynamic filter, a row |
| 635 | +/// group whose post-predicate selection is empty is silently finished by |
| 636 | +/// arrow-rs without handing back a reader. Before `rg_plan` was synced to the |
| 637 | +/// decoder frontier (`peek_next_row_group`), it trailed the decoder by one, so |
| 638 | +/// a later runtime prune rebuilt the decoder from a stale plan and re-read an |
| 639 | +/// already-delivered row group — the duplicate rows displaced the true top-k. |
| 640 | +#[tokio::test] |
| 641 | +async fn topk_pushdown_does_not_reread_delivered_row_group() { |
| 642 | + let schema = Arc::new(Schema::new(vec![ |
| 643 | + Field::new("event_time", DataType::Int64, false), |
| 644 | + Field::new("search_phrase", DataType::Utf8, false), |
| 645 | + ])); |
| 646 | + let batches = build_q26_batches(&schema); |
| 647 | + |
| 648 | + // `RowGroup(2048)` writes one row group per 2048-row batch (4 RGs) and |
| 649 | + // enables `pushdown_filters`, required for the dynamic filter to reach the |
| 650 | + // parquet scan. |
| 651 | + let mut ctx = ContextWithParquet::with_custom_data( |
| 652 | + Scenario::Int, |
| 653 | + RowGroup(2048), |
| 654 | + Arc::clone(&schema), |
| 655 | + batches, |
| 656 | + ) |
| 657 | + .await; |
| 658 | + |
| 659 | + let output = ctx |
| 660 | + .query( |
| 661 | + "SELECT search_phrase FROM t \ |
| 662 | + WHERE search_phrase <> '' ORDER BY event_time LIMIT 10", |
| 663 | + ) |
| 664 | + .await; |
| 665 | + |
| 666 | + // `search_phrase` is unique per row, so any repeated value is the same |
| 667 | + // source row emitted twice. The correct answer is the 10 smallest- |
| 668 | + // `event_time` non-empty phrases, matching DuckDB / pushdown-off. |
| 669 | + assert_eq!(output.result_rows, 10, "{}", output.description()); |
| 670 | + let formatted = output.pretty_results(); |
| 671 | + for p in [ |
| 672 | + "p0", "p4096", "p4097", "p4098", "p4099", "p4100", "p4101", "p4102", "p4103", |
| 673 | + "p4104", |
| 674 | + ] { |
| 675 | + assert!( |
| 676 | + formatted.contains(&format!("| {p} ")), |
| 677 | + "missing {p} from top-k; got:\n{formatted}", |
| 678 | + ); |
| 679 | + } |
| 680 | + // The bug emitted p4096 twice (and dropped p4101..=p4104); assert no dup. |
| 681 | + assert_eq!( |
| 682 | + formatted.matches("| p4096 ").count(), |
| 683 | + 1, |
| 684 | + "p4096 emitted more than once — rg_plan/decoder desync; got:\n{formatted}", |
| 685 | + ); |
| 686 | +} |
0 commit comments