4545//!
4646//! Filed upstream as apache/datafusion#23978. Once the opener merges its deferred page-index
4747//! load back into `FileMetadataCache` instead of bypassing it, the eager policy can go, but the
48- //! factory cannot: `get_metadata` is the one per-file hook that sees the raw footer, and two
49- //! other things hang off it.
48+ //! factory cannot: `get_metadata` is the per-file hook that sees the raw footer.
5049//!
5150//! The first is Spark's missing field id check. `ParquetReadSupport` refuses to open a file
5251//! whose Parquet schema carries no field id when the requested schema carries one, unless
5958//!
6059//! The second is the Variant footer rewrite, `with_spark_arrow_schema`, which replaces the
6160//! Arrow schema hint in the footer for scans that project Variant.
62-
63- use arrow:: datatypes:: { DataType , FieldRef , Schema } ;
61+ //!
62+ //! Conversion checks also use the raw footer to preserve legacy LIST clipping failures. With
63+ //! row filters enabled, a deferred conversion failure is raised on the first data-page request.
64+ //! Footer, Bloom-filter and page-index reads still proceed, so format pruning can discard a file
65+ //! without decoding it, but row selection cannot hide the failure.
66+
67+ use crate :: parquet:: parquet_support:: SparkParquetOptions ;
68+ use crate :: parquet:: schema_adapter:: {
69+ check_file_conversions, RejectOnNonEmpty , REPEATED_PRIMITIVE_KEY ,
70+ } ;
71+ use arrow:: datatypes:: { DataType , FieldRef , Schema , SchemaRef } ;
6472use async_trait:: async_trait;
6573use bytes:: Bytes ;
6674use datafusion:: common:: Result as DFResult ;
@@ -93,10 +101,11 @@ use parquet::file::metadata::{
93101 FooterTail , PageIndexPolicy , ParquetMetaData , ParquetMetaDataReader ,
94102} ;
95103use parquet:: schema:: types:: { ColumnDescPtr , SchemaDescriptor , Type as ParquetType } ;
104+ use std:: collections:: HashMap ;
96105use std:: fmt:: { Debug , Display , Formatter } ;
97106use std:: ops:: Range ;
98107use std:: sync:: atomic:: { AtomicBool , AtomicUsize , Ordering } ;
99- use std:: sync:: { Arc , OnceLock } ;
108+ use std:: sync:: { Arc , Mutex , OnceLock } ;
100109
101110#[ derive( Debug , Clone , Copy , PartialEq , Eq ) ]
102111pub ( crate ) enum ScanIoSource {
@@ -181,6 +190,8 @@ pub struct EagerPageIndexReaderFactory {
181190 // Refuse a file whose Parquet schema carries no field id, as Spark's `ParquetReadSupport`
182191 // does when the requested schema carries one and `ignoreMissing` is not set.
183192 require_field_ids : bool ,
193+ conversion_check : Option < ( SchemaRef , SparkParquetOptions , bool ) > ,
194+ deferred_rejections : Arc < Mutex < HashMap < Path , ( ObjectMeta , RejectOnNonEmpty ) > > > ,
184195}
185196
186197impl EagerPageIndexReaderFactory {
@@ -210,9 +221,23 @@ impl EagerPageIndexReaderFactory {
210221 scan_io_metrics,
211222 spark_variant_schema : false ,
212223 require_field_ids : false ,
224+ conversion_check : None ,
225+ deferred_rejections : Arc :: new ( Mutex :: new ( HashMap :: new ( ) ) ) ,
213226 }
214227 }
215228
229+ /// Retain raw LIST encoding for open-time validation. When row filters are enabled,
230+ /// enforce deferred failures at the first data-page read, after format pruning.
231+ pub ( crate ) fn with_conversion_check (
232+ mut self ,
233+ required : SchemaRef ,
234+ options : SparkParquetOptions ,
235+ row_filters : bool ,
236+ ) -> Self {
237+ self . conversion_check = Some ( ( required, options, row_filters) ) ;
238+ self
239+ }
240+
216241 pub fn with_spark_variant_schema ( mut self , enabled : bool ) -> Self {
217242 self . spark_variant_schema = enabled;
218243 self
@@ -254,6 +279,15 @@ impl ParquetFileReaderFactory for EagerPageIndexReaderFactory {
254279 metrics,
255280 ) ;
256281
282+ // DataFusion replaces the metadata reader after Bloom-filter pruning. The replacement
283+ // receives already-loaded metadata, so retain deferred failures in this scan's factory.
284+ let deferred_rejection = self
285+ . deferred_rejections
286+ . lock ( )
287+ . unwrap ( )
288+ . get ( & partitioned_file. object_meta . location )
289+ . filter ( |( meta, _) | meta == & partitioned_file. object_meta )
290+ . map ( |( _, rejection) | rejection. clone ( ) ) ;
257291 Ok ( Box :: new ( EagerPageIndexReader {
258292 file_metrics,
259293 scan_io_metrics : Arc :: clone ( & self . scan_io_metrics ) ,
@@ -263,6 +297,9 @@ impl ParquetFileReaderFactory for EagerPageIndexReaderFactory {
263297 metadata_size_hint,
264298 spark_variant_schema : self . spark_variant_schema ,
265299 require_field_ids : self . require_field_ids ,
300+ conversion_check : self . conversion_check . clone ( ) ,
301+ deferred_rejections : Arc :: clone ( & self . deferred_rejections ) ,
302+ deferred_rejection,
266303 } ) )
267304 }
268305}
@@ -279,6 +316,9 @@ struct EagerPageIndexReader {
279316 metadata_size_hint : Option < usize > ,
280317 spark_variant_schema : bool ,
281318 require_field_ids : bool ,
319+ conversion_check : Option < ( SchemaRef , SparkParquetOptions , bool ) > ,
320+ deferred_rejections : Arc < Mutex < HashMap < Path , ( ObjectMeta , RejectOnNonEmpty ) > > > ,
321+ deferred_rejection : Option < RejectOnNonEmpty > ,
282322}
283323
284324// Arrow infers ENUM as Binary, losing the distinction from raw binary that Spark needs.
@@ -411,6 +451,53 @@ fn with_spark_arrow_schema(metadata: Arc<ParquetMetaData>) -> ParquetResult<Arc<
411451 ) )
412452}
413453
454+ /// Arrow normalizes both standard and legacy LISTs to the same type. Carry the repeated
455+ /// primitive distinction in an ephemeral schema used by the conversion check, never in the
456+ /// shared footer cache or in the schema given to the decoder.
457+ fn mark_repeated_primitives ( schema : & Schema , parquet : & SchemaDescriptor ) -> Schema {
458+ fn mark ( field : & FieldRef , columns : & [ ColumnDescPtr ] , index : & mut usize ) -> FieldRef {
459+ let data_type = match field. data_type ( ) {
460+ DataType :: Struct ( fields) => DataType :: Struct (
461+ fields
462+ . iter ( )
463+ . map ( |f| mark ( f, columns, index) )
464+ . collect :: < Vec < _ > > ( )
465+ . into ( ) ,
466+ ) ,
467+ DataType :: List ( item) => DataType :: List ( mark ( item, columns, index) ) ,
468+ DataType :: LargeList ( item) => DataType :: LargeList ( mark ( item, columns, index) ) ,
469+ DataType :: FixedSizeList ( item, size) => {
470+ DataType :: FixedSizeList ( mark ( item, columns, index) , * size)
471+ }
472+ DataType :: Map ( entries, sorted) => DataType :: Map ( mark ( entries, columns, index) , * sorted) ,
473+ _ => {
474+ let repeated = columns. get ( * index) . is_some_and ( |column| {
475+ column. self_type ( ) . get_basic_info ( ) . repetition ( )
476+ == parquet:: basic:: Repetition :: REPEATED
477+ } ) ;
478+ * index += 1 ;
479+ let mut field = field. as_ref ( ) . clone ( ) ;
480+ if repeated {
481+ let mut metadata = field. metadata ( ) . clone ( ) ;
482+ metadata. insert ( REPEATED_PRIMITIVE_KEY . to_string ( ) , "true" . to_string ( ) ) ;
483+ field = field. with_metadata ( metadata) ;
484+ }
485+ return Arc :: new ( field) ;
486+ }
487+ } ;
488+ Arc :: new ( field. as_ref ( ) . clone ( ) . with_data_type ( data_type) )
489+ }
490+ let mut index = 0 ;
491+ Schema :: new_with_metadata (
492+ schema
493+ . fields ( )
494+ . iter ( )
495+ . map ( |f| mark ( f, parquet. columns ( ) , & mut index) )
496+ . collect :: < Vec < _ > > ( ) ,
497+ schema. metadata ( ) . clone ( ) ,
498+ )
499+ }
500+
414501impl AsyncFileReader for EagerPageIndexReader {
415502 /// Reads a metadata range, counting its requested size before I/O and its returned
416503 /// bytes only on success. The returned future borrows this reader; store errors retain
@@ -449,6 +536,12 @@ impl AsyncFileReader for EagerPageIndexReader {
449536 where
450537 Self : Send ,
451538 {
539+ if ranges. iter ( ) . any ( |range| !range. is_empty ( ) ) {
540+ if let Some ( rejection) = & self . deferred_rejection {
541+ let error = ParquetError :: External ( Box :: new ( rejection. error ( ) ) ) ;
542+ return async move { Err ( error) } . boxed ( ) ;
543+ }
544+ }
452545 let requested = ranges_bytes ( & ranges) ;
453546 self . file_metrics . bytes_scanned . add ( requested) ;
454547 let scan_io_metrics = Arc :: clone ( & self . scan_io_metrics ) ;
@@ -550,11 +643,43 @@ impl AsyncFileReader for EagerPageIndexReader {
550643 } ,
551644 ) ) ) ;
552645 }
553- if spark_variant_schema {
554- with_spark_arrow_schema ( metadata)
646+ let metadata = if spark_variant_schema {
647+ with_spark_arrow_schema ( metadata) ?
555648 } else {
556- Ok ( metadata)
649+ metadata
650+ } ;
651+ if let Some ( ( required, conversion_options, row_filters) ) =
652+ self . conversion_check . as_ref ( )
653+ {
654+ let file = metadata. file_metadata ( ) ;
655+ let needs_check = * row_filters
656+ || file. schema_descr ( ) . columns ( ) . iter ( ) . any ( |column| {
657+ column. self_type ( ) . get_basic_info ( ) . repetition ( )
658+ == parquet:: basic:: Repetition :: REPEATED
659+ } ) ;
660+ if !needs_check {
661+ return Ok ( metadata) ;
662+ }
663+ let schema =
664+ parquet_to_arrow_schema ( file. schema_descr ( ) , file. key_value_metadata ( ) ) ?;
665+ let schema = mark_repeated_primitives ( & schema, file. schema_descr ( ) ) ;
666+ let rejection = check_file_conversions (
667+ Arc :: clone ( required) ,
668+ Arc :: new ( schema) ,
669+ conversion_options. clone ( ) ,
670+ )
671+ . map_err ( |e| ParquetError :: External ( Box :: new ( e) ) ) ?;
672+ if * row_filters {
673+ if let Some ( rejection) = & rejection {
674+ self . deferred_rejections . lock ( ) . unwrap ( ) . insert (
675+ object_meta. location . clone ( ) ,
676+ ( object_meta. clone ( ) , rejection. clone ( ) ) ,
677+ ) ;
678+ }
679+ self . deferred_rejection = rejection;
680+ }
557681 }
682+ Ok ( metadata)
558683 }
559684 . boxed ( )
560685 }
0 commit comments