Skip to content

Commit b77cac0

Browse files
Skip Parquet page index I/O when row groups are fully matched (apache#22795)
Skip loading page index when row-group statistics already prove it cannot prune further, including on the cached metadata reader path. Honor PageIndexPolicy::Skip in DFParquetMetadata, add page_index_load_skipped metric, and add integration tests for default and cached reader factories. Co-authored-by: Cursor <[email protected]>
1 parent b8998c7 commit b77cac0

4 files changed

Lines changed: 423 additions & 67 deletions

File tree

‎datafusion/datasource-parquet/src/metadata.rs‎

Lines changed: 109 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
//! [`DFParquetMetadata`] for fetching Parquet file metadata, statistics
1919
//! and schema information.
2020
21+
use crate::file_format::ObjectStoreFetch;
2122
use crate::{Int96Coercer, apply_file_schema_type_coercions};
2223
use arrow::array::{Array, ArrayRef, BooleanArray};
2324
use arrow::compute::kernels::cmp::eq;
@@ -42,8 +43,8 @@ use parquet::DecodeResult;
4243
use parquet::arrow::arrow_reader::statistics::StatisticsConverter;
4344
use parquet::arrow::{parquet_column, parquet_to_arrow_schema};
4445
use parquet::file::metadata::{
45-
PageIndexPolicy, ParquetMetaData, ParquetMetaDataPushDecoder, RowGroupMetaData,
46-
SortingColumn,
46+
PageIndexPolicy, ParquetMetaData, ParquetMetaDataPushDecoder, ParquetMetaDataReader,
47+
RowGroupMetaData, SortingColumn,
4748
};
4849
use parquet::file::statistics::Statistics as ParquetStatistics;
4950
use parquet::schema::types::SchemaDescriptor;
@@ -70,6 +71,10 @@ pub struct DFParquetMetadata<'a> {
7071
metadata_size_hint: Option<usize>,
7172
decryption_properties: Option<Arc<FileDecryptionProperties>>,
7273
file_metadata_cache: Option<Arc<dyn FileMetadataCache>>,
74+
/// When set, controls whether page index structures are loaded. When unset,
75+
/// [`Self::fetch_metadata`] uses [`PageIndexPolicy::Optional`] if a metadata
76+
/// cache is configured and [`PageIndexPolicy::Skip`] otherwise.
77+
page_index_policy: Option<PageIndexPolicy>,
7378
/// timeunit to coerce INT96 timestamps to
7479
pub coerce_int96: Option<TimeUnit>,
7580
/// Optional timezone applied to INT96-coerced timestamps.
@@ -84,6 +89,7 @@ impl<'a> DFParquetMetadata<'a> {
8489
metadata_size_hint: None,
8590
decryption_properties: None,
8691
file_metadata_cache: None,
92+
page_index_policy: None,
8793
coerce_int96: None,
8894
coerce_int96_tz: None,
8995
}
@@ -113,6 +119,15 @@ impl<'a> DFParquetMetadata<'a> {
113119
self
114120
}
115121

122+
/// Sets the policy for loading parquet page index structures (column and offset indexes).
123+
pub fn with_page_index_policy(
124+
mut self,
125+
page_index_policy: Option<PageIndexPolicy>,
126+
) -> Self {
127+
self.page_index_policy = page_index_policy;
128+
self
129+
}
130+
116131
/// Set timeunit to coerce INT96 timestamps to
117132
pub fn with_coerce_int96(mut self, time_unit: Option<TimeUnit>) -> Self {
118133
self.coerce_int96 = time_unit;
@@ -127,9 +142,27 @@ impl<'a> DFParquetMetadata<'a> {
127142

128143
/// Fetch parquet metadata from the remote object store
129144
pub async fn fetch_metadata(&self) -> Result<Arc<ParquetMetaData>> {
130-
// implementation to fetch parquet metadata
145+
// fetch_metadata
146+
// │
147+
// ├─ cache_metadata = encryption check
148+
// ├─ page_index_policy = caller override OR default
149+
// │
150+
// ├─ CACHE HIT?
151+
// │ │
152+
// │ ├─ has index OR policy=Skip? → return cache
153+
// │ │
154+
// │ └─ else (footer only, wants index)
155+
// │ → load_page_index (index bytes only)
156+
// │ → cache_metadata() if allowed
157+
// │ → return
158+
// │
159+
// └─ CACHE MISS
160+
// → fetch_metadata_from_store(policy)
161+
// → cache_metadata() if allowed
162+
// → return
131163
let cache_metadata =
132164
!cfg!(feature = "parquet_encryption") || self.decryption_properties.is_none();
165+
let page_index_policy = self.effective_page_index_policy(cache_metadata);
133166

134167
if cache_metadata
135168
&& let Some(file_metadata_cache) = self.file_metadata_cache.as_ref()
@@ -140,9 +173,61 @@ impl<'a> DFParquetMetadata<'a> {
140173
.as_any()
141174
.downcast_ref::<CachedParquetMetaData>()
142175
{
143-
return Ok(Arc::clone(cached_parquet.parquet_metadata()));
176+
let cached_metadata = Arc::clone(cached_parquet.parquet_metadata());
177+
// Reuse the cache when it already has page index, or when the caller
178+
// asked to skip page index I/O (footer-only metadata is sufficient).
179+
if Self::metadata_has_page_index(cached_metadata.as_ref())
180+
|| page_index_policy == PageIndexPolicy::Skip
181+
{
182+
return Ok(cached_metadata);
183+
}
184+
let metadata =
185+
Self::load_page_index(self.store, self.object_meta, cached_metadata)
186+
.await?;
187+
if cache_metadata {
188+
self.cache_metadata(Arc::clone(&metadata)).await?;
189+
}
190+
return Ok(metadata);
191+
}
192+
193+
let metadata = self.fetch_metadata_from_store(page_index_policy).await?;
194+
if cache_metadata {
195+
self.cache_metadata(Arc::clone(&metadata)).await?;
144196
}
197+
Ok(metadata)
198+
}
199+
200+
fn effective_page_index_policy(&self, cache_metadata: bool) -> PageIndexPolicy {
201+
self.page_index_policy.unwrap_or_else(|| {
202+
if cache_metadata && self.file_metadata_cache.is_some() {
203+
PageIndexPolicy::Optional
204+
} else {
205+
PageIndexPolicy::Skip
206+
}
207+
})
208+
}
145209

210+
fn metadata_has_page_index(metadata: &ParquetMetaData) -> bool {
211+
metadata.column_index().is_some() && metadata.offset_index().is_some()
212+
}
213+
214+
async fn cache_metadata(&self, metadata: Arc<ParquetMetaData>) -> Result<()> {
215+
if let Some(file_metadata_cache) = &self.file_metadata_cache {
216+
file_metadata_cache.put(
217+
&self.object_meta.location,
218+
CachedFileMetadataEntry::new(
219+
self.object_meta.clone(),
220+
Arc::new(CachedParquetMetaData::new(metadata)),
221+
),
222+
);
223+
}
224+
Ok(())
225+
}
226+
227+
async fn fetch_metadata_from_store(
228+
&self,
229+
page_index_policy: PageIndexPolicy,
230+
) -> Result<Arc<ParquetMetaData>> {
146231
let file_size = self.object_meta.size;
147232
let mut decoder = ParquetMetaDataPushDecoder::try_new(file_size)
148233
.map_err(DataFusionError::from)?;
@@ -153,14 +238,8 @@ impl<'a> DFParquetMetadata<'a> {
153238
.with_file_decryption_properties(Some(Arc::clone(decryption_properties)));
154239
}
155240

156-
if cache_metadata && self.file_metadata_cache.is_some() {
157-
// Need to retrieve the entire metadata for the caching to be effective.
158-
decoder = decoder.with_page_index_policy(PageIndexPolicy::Optional);
159-
} else {
160-
decoder = decoder.with_page_index_policy(PageIndexPolicy::Skip);
161-
}
241+
decoder = decoder.with_page_index_policy(page_index_policy);
162242

163-
// If we have a size hint, prefetch that many bytes from the end of the file
164243
if let Some(hint) = self.metadata_size_hint {
165244
let prefetch_start = file_size.saturating_sub(hint as u64);
166245
let prefetch_range = prefetch_start..file_size;
@@ -199,19 +278,27 @@ impl<'a> DFParquetMetadata<'a> {
199278
}
200279
};
201280

202-
let metadata = Arc::new(metadata);
281+
Ok(Arc::new(metadata))
282+
}
203283

204-
if cache_metadata && let Some(file_metadata_cache) = &self.file_metadata_cache {
205-
file_metadata_cache.put(
206-
&self.object_meta.location,
207-
CachedFileMetadataEntry::new(
208-
self.object_meta.clone(),
209-
Arc::new(CachedParquetMetaData::new(Arc::clone(&metadata))),
210-
),
211-
);
284+
async fn load_page_index(
285+
store: &dyn ObjectStore,
286+
object_meta: &ObjectMeta,
287+
metadata: Arc<ParquetMetaData>,
288+
) -> Result<Arc<ParquetMetaData>> {
289+
if metadata.column_index().is_some() && metadata.offset_index().is_some() {
290+
return Ok(metadata);
212291
}
213-
214-
Ok(metadata)
292+
let metadata =
293+
Arc::try_unwrap(metadata).unwrap_or_else(|shared| (*shared).clone());
294+
let mut reader = ParquetMetaDataReader::new_with_metadata(metadata)
295+
.with_page_index_policy(PageIndexPolicy::Optional);
296+
let fetch = ObjectStoreFetch::new(store, object_meta);
297+
reader
298+
.load_page_index(fetch)
299+
.await
300+
.map_err(DataFusionError::from)?;
301+
Ok(Arc::new(reader.finish().map_err(DataFusionError::from)?))
215302
}
216303

217304
/// Read and parse the schema of the Parquet file

‎datafusion/datasource-parquet/src/metrics.rs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,9 @@ pub struct ParquetFileMetrics {
7171
pub page_index_pages_pruned: PruningMetrics,
7272
/// Total time spent evaluating parquet page index filters
7373
pub page_index_eval_time: Time,
74+
/// Number of times page index I/O was skipped because row-group statistics
75+
/// already proved page index could not prune further
76+
pub page_index_load_skipped: Count,
7477
/// Total time spent reading and parsing metadata from the footer
7578
pub metadata_load_time: Time,
7679
/// Scan Efficiency Ratio, calculated as bytes_scanned / total_file_size
@@ -185,6 +188,11 @@ impl ParquetFileMetrics {
185188
.clone()
186189
.subset_time("page_index_eval_time", partition);
187190

191+
let page_index_load_skipped = builder
192+
.clone()
193+
.with_type(MetricType::Summary)
194+
.counter("page_index_load_skipped", partition);
195+
188196
let page_index_rows_pruned = builder
189197
.clone()
190198
.pruning_metrics("page_index_rows_pruned", partition);
@@ -213,6 +221,7 @@ impl ParquetFileMetrics {
213221
statistics_eval_time,
214222
bloom_filter_eval_time,
215223
page_index_eval_time,
224+
page_index_load_skipped,
216225
metadata_load_time,
217226
scan_efficiency_ratio,
218227
predicate_cache_inner_records,

0 commit comments

Comments
 (0)