Skip to content

Commit bbb552f

Browse files
Renan GEHANalamb
andauthored
[Parquet] Skip bloom filter predicate evaluation when no bloom filters are loaded (#24101)
## Which issue does this PR close? - Closes #24107 ## Rationale for this change Bloom filter pruning codepath evaluates pruning predicates even when no bloom filters are defined for the row group. For wide / expensive predicates, this can add visible latency. This PR suggests shunting the evaluation of the predicate altogether when a row group provably doesn't contain bloom filters. ## What changes are included in this PR? Short-circuits the pruning predicate evaluation in bloom filter pruning codepath when there are no bloom filters. ## Are these changes tested? Yes, but the assertions are weak and cannot differentiate between a skipped pruning predicate (no bloom filter case) vs a pruning predicate that was evaluated and matches the row group. I suppose we could build a test with an expensive predicate, and rely on the timer, but I find time-based tests to be generally weak/flaky. Let me know if you prefer this solution / if you see a better approach. ## Are there any user-facing changes? None --------- Co-authored-by: Andrew Lamb <[email protected]>
1 parent 766ca58 commit bbb552f

2 files changed

Lines changed: 49 additions & 0 deletions

File tree

‎datafusion/datasource-parquet/src/bloom_filter.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,11 @@ impl BloomFilterStatistics {
6161
}
6262
}
6363

64+
/// Returns true if no per-column Bloom filters have been loaded.
65+
pub fn is_empty(&self) -> bool {
66+
self.column_sbbf.is_empty()
67+
}
68+
6469
/// Add a Bloom filter for the specified column, along with the column's
6570
/// Parquet physical [`Type`] and type length from the column descriptor.
6671
pub fn insert(

‎datafusion/datasource-parquet/src/row_group_filter.rs‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -434,6 +434,15 @@ impl RowGroupAccessPlanFilter {
434434
continue;
435435
}
436436

437+
// A row group without any loaded bloom filter cannot be pruned by this pass: with no
438+
// bloom statistics to consult, evaluation can only conclude "may match". Skip the
439+
// evaluation in that case: it runs once per row group and can be expensive for wide
440+
// predicates, a cost files written without bloom filters would pay for nothing.
441+
if stats.is_empty() {
442+
metrics.row_groups_pruned_bloom_filter.add_matched(1);
443+
continue;
444+
}
445+
437446
// Can this group be pruned?
438447
let prune_group = match predicate.prune(stats) {
439448
Ok(values) => !values[0],
@@ -544,6 +553,7 @@ mod tests {
544553
use parquet::arrow::ArrowSchemaConverter;
545554
use parquet::arrow::arrow_reader::{RowSelection, RowSelector};
546555
use parquet::basic::LogicalType;
556+
use parquet::bloom_filter::Sbbf;
547557
use parquet::data_type::{ByteArray, FixedLenByteArray};
548558
use parquet::file::metadata::ColumnChunkMetaData;
549559
use parquet::{
@@ -1382,6 +1392,40 @@ mod tests {
13821392
assert_pruned(row_groups, ExpectedPruning::Some(vec![1, 2]));
13831393
}
13841394

1395+
#[test]
1396+
fn bloom_filter_pruning_evaluates_row_groups_with_bloom_filters() {
1397+
// c1 = 15
1398+
let schema =
1399+
Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)]));
1400+
let expr = logical2physical(&col("c1").eq(lit(15)), &schema);
1401+
let pruning_predicate = PruningPredicateBuilder::new()
1402+
.with_file_schema(schema)
1403+
.build(expr)
1404+
.unwrap();
1405+
1406+
// Row group 0 has no bloom filter; row group 1 has a bloom filter
1407+
// whose values do not include 15, so it can be pruned.
1408+
let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap();
1409+
sbbf.insert(&1_i32);
1410+
sbbf.insert(&2_i32);
1411+
let mut with_bloom_filter = BloomFilterStatistics::new();
1412+
with_bloom_filter.insert("c1", sbbf, PhysicalType::INT32, 4);
1413+
1414+
let metrics = parquet_file_metrics();
1415+
let mut row_groups = RowGroupAccessPlanFilter::new(ParquetAccessPlan::new_all(2));
1416+
row_groups.prune_by_bloom_filters(
1417+
&pruning_predicate,
1418+
&metrics,
1419+
&[BloomFilterStatistics::new(), with_bloom_filter],
1420+
);
1421+
1422+
// The bloom-filter-less row group is kept without evaluation while
1423+
// the row group with a bloom filter is still evaluated and pruned.
1424+
assert_pruned(row_groups, ExpectedPruning::Some(vec![0]));
1425+
assert_eq!(metrics.row_groups_pruned_bloom_filter.pruned(), 1);
1426+
assert_eq!(metrics.row_groups_pruned_bloom_filter.matched(), 1);
1427+
}
1428+
13851429
fn get_row_group_meta_data(
13861430
schema_descr: &SchemaDescPtr,
13871431
column_statistics: Vec<ParquetStatistics>,

0 commit comments

Comments
 (0)