Skip to content

Commit 56c95dc

Browse files
committed
Updates
1 parent d6d6e42 commit 56c95dc

2 files changed

Lines changed: 25 additions & 8 deletions

File tree

‎datafusion/datasource-parquet/src/opener.rs‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -940,19 +940,26 @@ impl MetadataLoadedParquetOpen {
940940

941941
impl MetadataLoadedParquetOpen {
942942
/// Returns true if the footer metadata contains page index locations for
943-
/// every column chunk.
943+
/// every file column referenced by the page pruning predicate.
944944
///
945945
/// This is different from `metadata().offset_index()` /
946946
/// `metadata().column_index()`: the `*_offset()` accessors tell us where
947947
/// the page index payload lives in the file, while `offset_index()` and
948948
/// `column_index()` tell us whether the index has actually been fetched and
949949
/// decoded.
950-
fn has_page_index_location(&self) -> bool {
950+
fn has_page_index_location(
951+
&self,
952+
page_pruning_predicate: &PagePruningAccessPlanFilter,
953+
) -> bool {
954+
let required_columns = page_pruning_predicate.required_column_names();
951955
self.reader_metadata
952956
.metadata()
953957
.row_groups()
954958
.iter()
955959
.flat_map(|row_group| row_group.columns())
960+
.filter(|column| {
961+
required_columns.contains(column.column_path().string().as_str())
962+
})
956963
.all(|column| {
957964
column.column_index_offset().is_some()
958965
&& column.offset_index_offset().is_some()
@@ -992,15 +999,16 @@ impl FiltersPreparedParquetOpen {
992999
return Ok(self);
9931000
}
9941001

995-
// If not all column chunks have page indexes, skip page pruning.
996-
if !self.loaded.has_page_index_location() {
997-
return Ok(self);
998-
}
999-
10001002
// Build the predicate used for page pruning.
10011003
self.page_pruning_predicate = self.loaded.prepared.build_page_pruning_predicate();
10021004
// If there is no usable page pruning predicate, there is nothing to load.
1003-
if self.page_pruning_predicate.is_none() {
1005+
let Some(page_pruning_predicate) = self.page_pruning_predicate.as_ref() else {
1006+
return Ok(self);
1007+
};
1008+
1009+
// If the columns in the pruning predicate do not have page index
1010+
// locations, don't bother to load them as it won't be successful anyway
1011+
if !self.loaded.has_page_index_location(page_pruning_predicate) {
10041012
return Ok(self);
10051013
}
10061014

‎datafusion/datasource-parquet/src/page_filter.rs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,15 @@ impl PagePruningAccessPlanFilter {
297297
pub fn filter_number(&self) -> usize {
298298
self.predicates.len()
299299
}
300+
301+
/// Returns the file columns referenced by the page pruning predicates.
302+
pub fn required_column_names(&self) -> HashSet<&str> {
303+
self.predicates
304+
.iter()
305+
.filter_map(|predicate| predicate.required_columns().single_column())
306+
.map(|column| column.name())
307+
.collect()
308+
}
300309
}
301310

302311
fn update_selection(

0 commit comments

Comments
 (0)