1818//! [`DFParquetMetadata`] for fetching Parquet file metadata, statistics
1919//! and schema information.
2020
21+ use crate :: file_format:: ObjectStoreFetch ;
2122use crate :: { Int96Coercer , apply_file_schema_type_coercions} ;
2223use arrow:: array:: { Array , ArrayRef , BooleanArray } ;
2324use arrow:: compute:: kernels:: cmp:: eq;
@@ -42,8 +43,8 @@ use parquet::DecodeResult;
4243use parquet:: arrow:: arrow_reader:: statistics:: StatisticsConverter ;
4344use parquet:: arrow:: { parquet_column, parquet_to_arrow_schema} ;
4445use parquet:: file:: metadata:: {
45- PageIndexPolicy , ParquetMetaData , ParquetMetaDataPushDecoder , RowGroupMetaData ,
46- SortingColumn ,
46+ PageIndexPolicy , ParquetMetaData , ParquetMetaDataPushDecoder , ParquetMetaDataReader ,
47+ RowGroupMetaData , SortingColumn ,
4748} ;
4849use parquet:: file:: statistics:: Statistics as ParquetStatistics ;
4950use parquet:: schema:: types:: SchemaDescriptor ;
@@ -70,6 +71,10 @@ pub struct DFParquetMetadata<'a> {
7071 metadata_size_hint : Option < usize > ,
7172 decryption_properties : Option < Arc < FileDecryptionProperties > > ,
7273 file_metadata_cache : Option < Arc < dyn FileMetadataCache > > ,
74+ /// When set, controls whether page index structures are loaded. When unset,
75+ /// [`Self::fetch_metadata`] uses [`PageIndexPolicy::Optional`] if a metadata
76+ /// cache is configured and [`PageIndexPolicy::Skip`] otherwise.
77+ page_index_policy : Option < PageIndexPolicy > ,
7378 /// timeunit to coerce INT96 timestamps to
7479 pub coerce_int96 : Option < TimeUnit > ,
7580 /// Optional timezone applied to INT96-coerced timestamps.
@@ -84,6 +89,7 @@ impl<'a> DFParquetMetadata<'a> {
8489 metadata_size_hint : None ,
8590 decryption_properties : None ,
8691 file_metadata_cache : None ,
92+ page_index_policy : None ,
8793 coerce_int96 : None ,
8894 coerce_int96_tz : None ,
8995 }
@@ -113,6 +119,15 @@ impl<'a> DFParquetMetadata<'a> {
113119 self
114120 }
115121
122+ /// Sets the policy for loading parquet page index structures (column and offset indexes).
123+ pub fn with_page_index_policy (
124+ mut self ,
125+ page_index_policy : Option < PageIndexPolicy > ,
126+ ) -> Self {
127+ self . page_index_policy = page_index_policy;
128+ self
129+ }
130+
116131 /// Set timeunit to coerce INT96 timestamps to
117132 pub fn with_coerce_int96 ( mut self , time_unit : Option < TimeUnit > ) -> Self {
118133 self . coerce_int96 = time_unit;
@@ -127,9 +142,27 @@ impl<'a> DFParquetMetadata<'a> {
127142
128143 /// Fetch parquet metadata from the remote object store
129144 pub async fn fetch_metadata ( & self ) -> Result < Arc < ParquetMetaData > > {
130- // implementation to fetch parquet metadata
145+ // fetch_metadata
146+ // │
147+ // ├─ cache_metadata = encryption check
148+ // ├─ page_index_policy = caller override OR default
149+ // │
150+ // ├─ CACHE HIT?
151+ // │ │
152+ // │ ├─ has index OR policy=Skip? → return cache
153+ // │ │
154+ // │ └─ else (footer only, wants index)
155+ // │ → load_page_index (index bytes only)
156+ // │ → cache_metadata() if allowed
157+ // │ → return
158+ // │
159+ // └─ CACHE MISS
160+ // → fetch_metadata_from_store(policy)
161+ // → cache_metadata() if allowed
162+ // → return
131163 let cache_metadata =
132164 !cfg ! ( feature = "parquet_encryption" ) || self . decryption_properties . is_none ( ) ;
165+ let page_index_policy = self . effective_page_index_policy ( cache_metadata) ;
133166
134167 if cache_metadata
135168 && let Some ( file_metadata_cache) = self . file_metadata_cache . as_ref ( )
@@ -140,9 +173,61 @@ impl<'a> DFParquetMetadata<'a> {
140173 . as_any ( )
141174 . downcast_ref :: < CachedParquetMetaData > ( )
142175 {
143- return Ok ( Arc :: clone ( cached_parquet. parquet_metadata ( ) ) ) ;
176+ let cached_metadata = Arc :: clone ( cached_parquet. parquet_metadata ( ) ) ;
177+ // Reuse the cache when it already has page index, or when the caller
178+ // asked to skip page index I/O (footer-only metadata is sufficient).
179+ if Self :: metadata_has_page_index ( cached_metadata. as_ref ( ) )
180+ || page_index_policy == PageIndexPolicy :: Skip
181+ {
182+ return Ok ( cached_metadata) ;
183+ }
184+ let metadata =
185+ Self :: load_page_index ( self . store , self . object_meta , cached_metadata)
186+ . await ?;
187+ if cache_metadata {
188+ self . cache_metadata ( Arc :: clone ( & metadata) ) . await ?;
189+ }
190+ return Ok ( metadata) ;
191+ }
192+
193+ let metadata = self . fetch_metadata_from_store ( page_index_policy) . await ?;
194+ if cache_metadata {
195+ self . cache_metadata ( Arc :: clone ( & metadata) ) . await ?;
144196 }
197+ Ok ( metadata)
198+ }
199+
200+ fn effective_page_index_policy ( & self , cache_metadata : bool ) -> PageIndexPolicy {
201+ self . page_index_policy . unwrap_or_else ( || {
202+ if cache_metadata && self . file_metadata_cache . is_some ( ) {
203+ PageIndexPolicy :: Optional
204+ } else {
205+ PageIndexPolicy :: Skip
206+ }
207+ } )
208+ }
145209
210+ fn metadata_has_page_index ( metadata : & ParquetMetaData ) -> bool {
211+ metadata. column_index ( ) . is_some ( ) && metadata. offset_index ( ) . is_some ( )
212+ }
213+
214+ async fn cache_metadata ( & self , metadata : Arc < ParquetMetaData > ) -> Result < ( ) > {
215+ if let Some ( file_metadata_cache) = & self . file_metadata_cache {
216+ file_metadata_cache. put (
217+ & self . object_meta . location ,
218+ CachedFileMetadataEntry :: new (
219+ self . object_meta . clone ( ) ,
220+ Arc :: new ( CachedParquetMetaData :: new ( metadata) ) ,
221+ ) ,
222+ ) ;
223+ }
224+ Ok ( ( ) )
225+ }
226+
227+ async fn fetch_metadata_from_store (
228+ & self ,
229+ page_index_policy : PageIndexPolicy ,
230+ ) -> Result < Arc < ParquetMetaData > > {
146231 let file_size = self . object_meta . size ;
147232 let mut decoder = ParquetMetaDataPushDecoder :: try_new ( file_size)
148233 . map_err ( DataFusionError :: from) ?;
@@ -153,14 +238,8 @@ impl<'a> DFParquetMetadata<'a> {
153238 . with_file_decryption_properties ( Some ( Arc :: clone ( decryption_properties) ) ) ;
154239 }
155240
156- if cache_metadata && self . file_metadata_cache . is_some ( ) {
157- // Need to retrieve the entire metadata for the caching to be effective.
158- decoder = decoder. with_page_index_policy ( PageIndexPolicy :: Optional ) ;
159- } else {
160- decoder = decoder. with_page_index_policy ( PageIndexPolicy :: Skip ) ;
161- }
241+ decoder = decoder. with_page_index_policy ( page_index_policy) ;
162242
163- // If we have a size hint, prefetch that many bytes from the end of the file
164243 if let Some ( hint) = self . metadata_size_hint {
165244 let prefetch_start = file_size. saturating_sub ( hint as u64 ) ;
166245 let prefetch_range = prefetch_start..file_size;
@@ -199,19 +278,27 @@ impl<'a> DFParquetMetadata<'a> {
199278 }
200279 } ;
201280
202- let metadata = Arc :: new ( metadata) ;
281+ Ok ( Arc :: new ( metadata) )
282+ }
203283
204- if cache_metadata && let Some ( file_metadata_cache) = & self . file_metadata_cache {
205- file_metadata_cache. put (
206- & self . object_meta . location ,
207- CachedFileMetadataEntry :: new (
208- self . object_meta . clone ( ) ,
209- Arc :: new ( CachedParquetMetaData :: new ( Arc :: clone ( & metadata) ) ) ,
210- ) ,
211- ) ;
284+ async fn load_page_index (
285+ store : & dyn ObjectStore ,
286+ object_meta : & ObjectMeta ,
287+ metadata : Arc < ParquetMetaData > ,
288+ ) -> Result < Arc < ParquetMetaData > > {
289+ if metadata. column_index ( ) . is_some ( ) && metadata. offset_index ( ) . is_some ( ) {
290+ return Ok ( metadata) ;
212291 }
213-
214- Ok ( metadata)
292+ let metadata =
293+ Arc :: try_unwrap ( metadata) . unwrap_or_else ( |shared| ( * shared) . clone ( ) ) ;
294+ let mut reader = ParquetMetaDataReader :: new_with_metadata ( metadata)
295+ . with_page_index_policy ( PageIndexPolicy :: Optional ) ;
296+ let fetch = ObjectStoreFetch :: new ( store, object_meta) ;
297+ reader
298+ . load_page_index ( fetch)
299+ . await
300+ . map_err ( DataFusionError :: from) ?;
301+ Ok ( Arc :: new ( reader. finish ( ) . map_err ( DataFusionError :: from) ?) )
215302 }
216303
217304 /// Read and parse the schema of the Parquet file
0 commit comments