Skip to content

Commit 0f6cdeb

Browse files
committed
make PruningPredicateConfig a field in opener
1 parent e03847d commit 0f6cdeb

2 files changed

Lines changed: 8 additions & 10 deletions

File tree

‎datafusion/datasource-parquet/src/opener.rs‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -106,8 +106,8 @@ pub(super) struct ParquetOpener {
106106
pub enable_bloom_filter: bool,
107107
/// Should row group pruning be applied
108108
pub enable_row_group_stats_pruning: bool,
109-
/// Maximum number of elements (inclusive) in InList exprs to be eligible for pruning
110-
pub pruning_max_inlist_limit: usize,
109+
/// Internals configuration of predicate pruning
110+
pub(crate) pruning_predicate_config: PruningPredicateConfig,
111111
/// Coerce INT96 timestamps to specific TimeUnit
112112
pub coerce_int96: Option<TimeUnit>,
113113
/// Optional parquet FileDecryptionProperties
@@ -284,9 +284,7 @@ impl FileOpener for ParquetOpener {
284284

285285
let reverse_row_groups = self.reverse_row_groups;
286286
let preserve_order = self.preserve_order;
287-
let pruning_predicate_config = PruningPredicateConfig {
288-
max_in_list: self.pruning_max_inlist_limit,
289-
};
287+
let pruning_predicate_config = self.pruning_predicate_config.clone();
290288

291289
Ok(Box::pin(async move {
292290
#[cfg(feature = "parquet_encryption")]
@@ -1047,6 +1045,7 @@ mod test {
10471045
DefaultPhysicalExprAdapterFactory, replace_columns_with_literals,
10481046
};
10491047
use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;
1048+
use datafusion_pruning::PruningPredicateConfig;
10501049
use futures::{Stream, StreamExt};
10511050
use object_store::{ObjectStore, memory::InMemory, path::Path};
10521051
use parquet::arrow::ArrowWriter;
@@ -1200,7 +1199,7 @@ mod test {
12001199
enable_page_index: self.enable_page_index,
12011200
enable_bloom_filter: self.enable_bloom_filter,
12021201
enable_row_group_stats_pruning: self.enable_row_group_stats_pruning,
1203-
pruning_max_inlist_limit: 20,
1202+
pruning_predicate_config: PruningPredicateConfig::default(),
12041203
coerce_int96: self.coerce_int96,
12051204
#[cfg(feature = "parquet_encryption")]
12061205
file_decryption_properties: None,

‎datafusion/datasource-parquet/src/source.rs‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -561,10 +561,9 @@ impl FileSource for ParquetSource {
561561
enable_page_index: self.enable_page_index(),
562562
enable_bloom_filter: self.bloom_filter_on_read(),
563563
enable_row_group_stats_pruning: self.table_parquet_options.global.pruning,
564-
pruning_max_inlist_limit: self
565-
.table_parquet_options
566-
.global
567-
.pruning_max_inlist_limit,
564+
pruning_predicate_config: PruningPredicateConfig {
565+
max_in_list: self.table_parquet_options.global.pruning_max_inlist_limit,
566+
},
568567
coerce_int96,
569568
#[cfg(feature = "parquet_encryption")]
570569
file_decryption_properties,

0 commit comments

Comments
 (0)