Skip to content

Commit 1fd29c9

Browse files
Skip loading Parquet page index when row-group statistics already prove it cannot prune (#22857)
## Which issue does this PR close? - Closes #22795 ## Rationale for this change The Parquet opener was loading the page index (ColumnIndex + OffsetIndex) before row-group statistics pruning. When all surviving row groups are fully matched by row-group statistics (for example, `IS NOT NULL` on a non-null column), page index I/O cannot prune further and is wasted. ## What changes are included in this PR? - Reorder the opener state machine: `PrepareFilters → PruneWithStatistics → LoadPageIndex? → LoadBloomFilters` - Skip `load_page_index` when there is no page-pruning predicate, no surviving row groups, or every surviving row group is fully matched - Add unit and integration tests for the gate and the fully-matched `IS NOT NULL` case ## Are these changes tested? - `cargo test -p datafusion-datasource-parquet should_load` - `cargo test -p datafusion-datasource-parquet page_index_skip` - `cargo test -p datafusion-datasource-parquet opener::test::test_page_pruning` - `cargo test -p datafusion --test parquet_integration` - `cargo clippy -p datafusion-datasource-parquet --all-targets -- -D warnings` ## Are there any user-facing changes? No user-facing API changes. This reduces unnecessary Parquet page index I/O during scan planning when row-group statistics already prove no further pruning is possible. Made with [Cursor](https://cursor.com) --------- Co-authored-by: Cursor <[email protected]>
1 parent b0afa3a commit 1fd29c9

6 files changed

Lines changed: 439 additions & 72 deletions

File tree

‎datafusion-cli/src/main.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -641,9 +641,9 @@ mod tests {
641641
+-----------------------------------+-----------------+---------------------+------+------------------+
642642
| filename | file_size_bytes | metadata_size_bytes | hits | extra |
643643
+-----------------------------------+-----------------+---------------------+------+------------------+
644-
| alltypes_plain.parquet | 1851 | 8794 | 2 | page_index=false |
644+
| alltypes_plain.parquet | 1851 | 8794 | 1 | page_index=false |
645645
| alltypes_tiny_pages.parquet | 454233 | 268970 | 2 | page_index=true |
646-
| lz4_raw_compressed_larger.parquet | 380836 | 1331 | 2 | page_index=false |
646+
| lz4_raw_compressed_larger.parquet | 380836 | 1331 | 1 | page_index=false |
647647
+-----------------------------------+-----------------+---------------------+------+------------------+
648648
");
649649

@@ -672,9 +672,9 @@ mod tests {
672672
+-----------------------------------+-----------------+---------------------+------+------------------+
673673
| filename | file_size_bytes | metadata_size_bytes | hits | extra |
674674
+-----------------------------------+-----------------+---------------------+------+------------------+
675-
| alltypes_plain.parquet | 1851 | 8794 | 5 | page_index=false |
675+
| alltypes_plain.parquet | 1851 | 8794 | 4 | page_index=false |
676676
| alltypes_tiny_pages.parquet | 454233 | 268970 | 2 | page_index=true |
677-
| lz4_raw_compressed_larger.parquet | 380836 | 1331 | 3 | page_index=false |
677+
| lz4_raw_compressed_larger.parquet | 380836 | 1331 | 2 | page_index=false |
678678
+-----------------------------------+-----------------+---------------------+------+------------------+
679679
");
680680

‎datafusion/datasource-parquet/src/metadata.rs‎

Lines changed: 106 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
//! [`DFParquetMetadata`] for fetching Parquet file metadata, statistics
1919
//! and schema information.
2020
21+
use crate::file_format::ObjectStoreFetch;
2122
use crate::{Int96Coercer, apply_file_schema_type_coercions};
2223
use arrow::array::{Array, ArrayRef, BooleanArray};
2324
use arrow::compute::kernels::cmp::eq;
@@ -42,8 +43,8 @@ use parquet::DecodeResult;
4243
use parquet::arrow::arrow_reader::statistics::StatisticsConverter;
4344
use parquet::arrow::{parquet_column, parquet_to_arrow_schema};
4445
use parquet::file::metadata::{
45-
PageIndexPolicy, ParquetMetaData, ParquetMetaDataPushDecoder, RowGroupMetaData,
46-
SortingColumn,
46+
PageIndexPolicy, ParquetMetaData, ParquetMetaDataPushDecoder, ParquetMetaDataReader,
47+
RowGroupMetaData, SortingColumn,
4748
};
4849
use parquet::file::statistics::Statistics as ParquetStatistics;
4950
use parquet::schema::types::SchemaDescriptor;
@@ -69,6 +70,7 @@ pub struct DFParquetMetadata<'a> {
6970
metadata_size_hint: Option<usize>,
7071
decryption_properties: Option<Arc<FileDecryptionProperties>>,
7172
file_metadata_cache: Option<Arc<FileMetadataCache>>,
73+
page_index_policy: Option<PageIndexPolicy>,
7274
/// timeunit to coerce INT96 timestamps to
7375
pub coerce_int96: Option<TimeUnit>,
7476
/// Optional timezone applied to INT96-coerced timestamps.
@@ -83,6 +85,7 @@ impl<'a> DFParquetMetadata<'a> {
8385
metadata_size_hint: None,
8486
decryption_properties: None,
8587
file_metadata_cache: None,
88+
page_index_policy: None,
8689
coerce_int96: None,
8790
coerce_int96_tz: None,
8891
}
@@ -112,6 +115,15 @@ impl<'a> DFParquetMetadata<'a> {
112115
self
113116
}
114117

118+
/// Sets the policy for loading parquet page index structures (column and offset indexes).
119+
pub fn with_page_index_policy(
120+
mut self,
121+
page_index_policy: Option<PageIndexPolicy>,
122+
) -> Self {
123+
self.page_index_policy = page_index_policy;
124+
self
125+
}
126+
115127
/// Set timeunit to coerce INT96 timestamps to
116128
pub fn with_coerce_int96(mut self, time_unit: Option<TimeUnit>) -> Self {
117129
self.coerce_int96 = time_unit;
@@ -126,9 +138,27 @@ impl<'a> DFParquetMetadata<'a> {
126138

127139
/// Fetch parquet metadata from the remote object store
128140
pub async fn fetch_metadata(&self) -> Result<Arc<ParquetMetaData>> {
129-
// implementation to fetch parquet metadata
141+
// fetch_metadata
142+
// │
143+
// ├─ cache_metadata = encryption check
144+
// ├─ page_index_policy = caller override OR default
145+
// │
146+
// ├─ CACHE HIT?
147+
// │ │
148+
// │ ├─ has index OR policy=Skip? → return cache
149+
// │ │
150+
// │ └─ else (footer only, wants index)
151+
// │ → load_page_index (index bytes only)
152+
// │ → cache_metadata() if allowed
153+
// │ → return
154+
// │
155+
// └─ CACHE MISS
156+
// → fetch_metadata_from_store(policy)
157+
// → cache_metadata() if allowed
158+
// → return
130159
let cache_metadata =
131160
!cfg!(feature = "parquet_encryption") || self.decryption_properties.is_none();
161+
let page_index_policy = self.effective_page_index_policy(cache_metadata);
132162

133163
if cache_metadata
134164
&& let Some(file_metadata_cache) = self.file_metadata_cache.as_ref()
@@ -139,9 +169,61 @@ impl<'a> DFParquetMetadata<'a> {
139169
.as_any()
140170
.downcast_ref::<CachedParquetMetaData>()
141171
{
142-
return Ok(Arc::clone(cached_parquet.parquet_metadata()));
172+
let cached_metadata = Arc::clone(cached_parquet.parquet_metadata());
173+
// Reuse the cache when it already has page index, or when the caller
174+
// asked to skip page index I/O (footer-only metadata is sufficient).
175+
if Self::metadata_has_page_index(cached_metadata.as_ref())
176+
|| page_index_policy == PageIndexPolicy::Skip
177+
{
178+
return Ok(cached_metadata);
179+
}
180+
let metadata =
181+
Self::load_page_index(self.store, self.object_meta, cached_metadata)
182+
.await?;
183+
if cache_metadata {
184+
self.cache_metadata(Arc::clone(&metadata)).await?;
185+
}
186+
return Ok(metadata);
187+
}
188+
189+
let metadata = self.fetch_metadata_from_store(page_index_policy).await?;
190+
if cache_metadata {
191+
self.cache_metadata(Arc::clone(&metadata)).await?;
143192
}
193+
Ok(metadata)
194+
}
195+
196+
fn effective_page_index_policy(&self, cache_metadata: bool) -> PageIndexPolicy {
197+
self.page_index_policy.unwrap_or_else(|| {
198+
if cache_metadata && self.file_metadata_cache.is_some() {
199+
PageIndexPolicy::Optional
200+
} else {
201+
PageIndexPolicy::Skip
202+
}
203+
})
204+
}
144205

206+
fn metadata_has_page_index(metadata: &ParquetMetaData) -> bool {
207+
metadata.column_index().is_some() && metadata.offset_index().is_some()
208+
}
209+
210+
async fn cache_metadata(&self, metadata: Arc<ParquetMetaData>) -> Result<()> {
211+
if let Some(file_metadata_cache) = &self.file_metadata_cache {
212+
file_metadata_cache.put(
213+
&self.object_meta.location,
214+
CachedFileMetadataEntry::new(
215+
self.object_meta.clone(),
216+
Arc::new(CachedParquetMetaData::new(metadata)),
217+
),
218+
);
219+
}
220+
Ok(())
221+
}
222+
223+
async fn fetch_metadata_from_store(
224+
&self,
225+
page_index_policy: PageIndexPolicy,
226+
) -> Result<Arc<ParquetMetaData>> {
145227
let file_size = self.object_meta.size;
146228
let mut decoder = ParquetMetaDataPushDecoder::try_new(file_size)
147229
.map_err(DataFusionError::from)?;
@@ -152,14 +234,8 @@ impl<'a> DFParquetMetadata<'a> {
152234
.with_file_decryption_properties(Some(Arc::clone(decryption_properties)));
153235
}
154236

155-
if cache_metadata && self.file_metadata_cache.is_some() {
156-
// Need to retrieve the entire metadata for the caching to be effective.
157-
decoder = decoder.with_page_index_policy(PageIndexPolicy::Optional);
158-
} else {
159-
decoder = decoder.with_page_index_policy(PageIndexPolicy::Skip);
160-
}
237+
decoder = decoder.with_page_index_policy(page_index_policy);
161238

162-
// If we have a size hint, prefetch that many bytes from the end of the file
163239
if let Some(hint) = self.metadata_size_hint {
164240
let prefetch_start = file_size.saturating_sub(hint as u64);
165241
let prefetch_range = prefetch_start..file_size;
@@ -198,19 +274,27 @@ impl<'a> DFParquetMetadata<'a> {
198274
}
199275
};
200276

201-
let metadata = Arc::new(metadata);
277+
Ok(Arc::new(metadata))
278+
}
202279

203-
if cache_metadata && let Some(file_metadata_cache) = &self.file_metadata_cache {
204-
file_metadata_cache.put(
205-
&self.object_meta.location,
206-
CachedFileMetadataEntry::new(
207-
self.object_meta.clone(),
208-
Arc::new(CachedParquetMetaData::new(Arc::clone(&metadata))),
209-
),
210-
);
280+
async fn load_page_index(
281+
store: &dyn ObjectStore,
282+
object_meta: &ObjectMeta,
283+
metadata: Arc<ParquetMetaData>,
284+
) -> Result<Arc<ParquetMetaData>> {
285+
if metadata.column_index().is_some() && metadata.offset_index().is_some() {
286+
return Ok(metadata);
211287
}
212-
213-
Ok(metadata)
288+
let metadata =
289+
Arc::try_unwrap(metadata).unwrap_or_else(|shared| (*shared).clone());
290+
let mut reader = ParquetMetaDataReader::new_with_metadata(metadata)
291+
.with_page_index_policy(PageIndexPolicy::Optional);
292+
let fetch = ObjectStoreFetch::new(store, object_meta);
293+
reader
294+
.load_page_index(fetch)
295+
.await
296+
.map_err(DataFusionError::from)?;
297+
Ok(Arc::new(reader.finish().map_err(DataFusionError::from)?))
214298
}
215299

216300
/// Read and parse the schema of the Parquet file

‎datafusion/datasource-parquet/src/metrics.rs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,4 +243,23 @@ impl ParquetFileMetrics {
243243
.counter("page_index_pages_skipped_by_fully_matched", partition);
244244
count.add(n);
245245
}
246+
247+
/// Record that page index I/O was skipped because row-group statistics
248+
/// already proved page index could not prune further.
249+
pub(crate) fn add_page_index_load_skipped(
250+
metrics: &ExecutionPlanMetricsSet,
251+
partition: usize,
252+
filename: &str,
253+
n: usize,
254+
) {
255+
if n == 0 {
256+
return;
257+
}
258+
259+
let count = MetricBuilder::new(metrics)
260+
.with_new_label("filename", filename.to_string())
261+
.with_type(MetricType::Summary)
262+
.counter("page_index_load_skipped", partition);
263+
count.add(n);
264+
}
246265
}

0 commit comments

Comments
 (0)