Skip to content

Commit c0ec512

Browse files
Braedon-Wooding-Displayralambclaude
authored
perf: don't build a parquet stream for a file range with no row group left (apache#26476)
I've got a few fixes to upstream so just starting with small ones. ## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123. --> - Closes apache#26475 ## Rationale for this change If row-group statistics or Bloom filters prune every row group in a file range, the opener still builds the decoder read plans, the decoder and the row filter, then hands back a stream morsel that yields nothing. `PruneFile` already ends early when file statistics prune the range; this does the same once row-group pruning has. ## What changes are included in this PR? `PruneWithStatistics` and `PruneWithBloomFilters` return `ParquetOpenState::Done` when no row group survives. ## What is the testing strategy for this PR? Testing to ensure that we do actually skip them! ## Are there any user-facing changes? Nope! --------- Co-authored-by: Andrew Lamb <[email protected]> Co-authored-by: Claude Fable 5.1 <[email protected]>
1 parent d090d29 commit c0ec512

2 files changed

Lines changed: 96 additions & 3 deletions

File tree

‎datafusion/datasource-parquet/src/opener/mod.rs‎

Lines changed: 71 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -660,6 +660,9 @@ impl ParquetOpenState {
660660
}
661661
ParquetOpenState::PruneWithStatistics(prepared) => {
662662
let mut prepared_row_groups = (*prepared).prune_row_groups()?;
663+
if prepared_row_groups.row_groups.is_fully_pruned() {
664+
return Ok(ParquetOpenState::Done);
665+
}
663666
if prepared_row_groups.should_load_page_index()? {
664667
Ok(ParquetOpenState::LoadPageIndex(
665668
prepared_row_groups.load_page_index().boxed(),
@@ -690,9 +693,13 @@ impl ParquetOpenState {
690693
ParquetOpenState::LoadBloomFilters(future) => {
691694
Ok(ParquetOpenState::LoadBloomFilters(future))
692695
}
693-
ParquetOpenState::PruneWithBloomFilters(loaded) => Ok(
694-
ParquetOpenState::BuildStream(Box::new(loaded.prune_bloom_filters())),
695-
),
696+
ParquetOpenState::PruneWithBloomFilters(loaded) => {
697+
let pruned = loaded.prune_bloom_filters();
698+
if pruned.row_groups.is_fully_pruned() {
699+
return Ok(ParquetOpenState::Done);
700+
}
701+
Ok(ParquetOpenState::BuildStream(Box::new(pruned)))
702+
}
696703
ParquetOpenState::BuildStream(prepared) => {
697704
Ok(ParquetOpenState::Ready(prepared.build_stream()?))
698705
}
@@ -2690,6 +2697,26 @@ mod test {
26902697
}
26912698
}
26922699

2700+
/// Drives a [`ParquetMorselizer`] to completion and returns how many
2701+
/// morsels it produced.
2702+
async fn count_morsels(
2703+
morselizer: &dyn Morselizer,
2704+
file: PartitionedFile,
2705+
) -> Result<usize> {
2706+
let mut planners = VecDeque::from([morselizer.plan_file(file)?]);
2707+
let mut morsels = 0;
2708+
while let Some(planner) = planners.pop_front() {
2709+
if let Some(mut plan) = planner.plan()? {
2710+
morsels += plan.take_morsels().len();
2711+
planners.extend(plan.take_ready_planners());
2712+
if let Some(pending_planner) = plan.take_pending_planner() {
2713+
planners.push_front(pending_planner.await?);
2714+
}
2715+
}
2716+
}
2717+
Ok(morsels)
2718+
}
2719+
26932720
fn constant_int_stats() -> (Statistics, SchemaRef) {
26942721
let schema = Arc::new(Schema::new(vec![
26952722
Field::new("a", DataType::Int32, false),
@@ -3234,6 +3261,47 @@ mod test {
32343261
assert_eq!(num_rows, 0);
32353262
}
32363263

3264+
#[tokio::test]
3265+
async fn test_no_morsel_when_every_row_group_is_pruned() {
3266+
let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
3267+
let batches = vec![
3268+
record_batch!(("a", Int32, vec![0, 2, 4])).unwrap(),
3269+
record_batch!(("a", Int32, vec![0, 2, 4])).unwrap(),
3270+
];
3271+
let schema = batches[0].schema();
3272+
let props = WriterProperties::builder()
3273+
.set_max_row_group_row_count(Some(3))
3274+
.set_bloom_filter_enabled(true)
3275+
.set_statistics_enabled(EnabledStatistics::Chunk)
3276+
.build();
3277+
let data_size = write_parquet_batches(
3278+
Arc::clone(&store),
3279+
"test.parquet",
3280+
batches,
3281+
Some(props),
3282+
)
3283+
.await;
3284+
let file = PartitionedFile::new("test.parquet".to_string(), data_size as u64);
3285+
3286+
let morsels = |expr: Expr| {
3287+
let morselizer = ParquetMorselizerBuilder::new()
3288+
.with_store(Arc::clone(&store))
3289+
.with_schema(Arc::clone(&schema))
3290+
.with_predicate(logical2physical(&expr, &schema))
3291+
.with_row_group_stats_pruning(true)
3292+
.with_enable_bloom_filter(true)
3293+
.build();
3294+
let file = file.clone();
3295+
async move { count_morsels(&morselizer, file).await.unwrap() }
3296+
};
3297+
3298+
// Statistics prune both row groups.
3299+
assert_eq!(morsels(col("a").gt(lit(10))).await, 0);
3300+
// Statistics keep both row groups; their Bloom filters prune both.
3301+
assert_eq!(morsels(col("a").eq(lit(1))).await, 0);
3302+
assert_eq!(morsels(col("a").eq(lit(2))).await, 1);
3303+
}
3304+
32373305
#[tokio::test]
32383306
async fn test_prune_on_partition_statistics_with_dynamic_expression() {
32393307
let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;

‎datafusion/datasource-parquet/src/row_group_filter.rs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,12 @@ impl RowGroupAccessPlanFilter {
9999
self.access_plan.is_empty()
100100
}
101101

102+
/// Return true if no row groups remain to be scanned, either because the
103+
/// file has none or because pruning has skipped every one of them.
104+
pub fn is_fully_pruned(&self) -> bool {
105+
self.access_plan.row_group_index_iter().next().is_none()
106+
}
107+
102108
/// Return the number of row groups that are currently expected to be scanned
103109
pub fn remaining_row_group_count(&self) -> usize {
104110
self.access_plan.row_group_index_iter().count()
@@ -706,6 +712,25 @@ mod tests {
706712
assert_eq!(filter.remaining_row_group_count(), 2);
707713
}
708714

715+
#[test]
716+
fn is_fully_pruned_only_when_every_row_group_is_skipped() {
717+
let mut filter = RowGroupAccessPlanFilter::new(ParquetAccessPlan::new_all(2));
718+
assert!(!filter.is_fully_pruned());
719+
720+
filter.access_plan.skip(0);
721+
assert!(!filter.is_fully_pruned());
722+
723+
filter.access_plan.skip(1);
724+
assert!(filter.is_fully_pruned());
725+
// Skipped row groups are still part of the plan
726+
assert!(!filter.is_empty());
727+
728+
assert!(
729+
RowGroupAccessPlanFilter::new(ParquetAccessPlan::new_all(0))
730+
.is_fully_pruned()
731+
);
732+
}
733+
709734
/// A row group with one column chunk of `size` bytes at the given offsets
710735
fn row_group_at(
711736
dictionary_page_offset: Option<i64>,

0 commit comments

Comments
 (0)