4545//!
4646//! Filed upstream as apache/datafusion#23978. Once the opener merges its deferred page-index
4747//! load back into `FileMetadataCache` instead of bypassing it, the eager policy can go, but the
48- //! factory cannot: `get_metadata` is the one per-file hook that sees the raw footer, and two
49- //! other things hang off it.
48+ //! factory cannot: `get_metadata` is the per-file hook that sees the raw footer.
5049//!
5150//! The first is Spark's missing field id check. `ParquetReadSupport` refuses to open a file
5251//! whose Parquet schema carries no field id when the requested schema carries one, unless
5958//!
6059//! The second is the Variant footer rewrite, `with_spark_arrow_schema`, which replaces the
6160//! Arrow schema hint in the footer for scans that project Variant.
62-
63- use arrow:: datatypes:: { DataType , FieldRef , Schema } ;
61+ //!
62+ //! Conversion checks also use the raw footer to preserve legacy LIST clipping failures. With
63+ //! row filters enabled, a deferred conversion failure is raised on the first data-page request.
64+ //! Footer, Bloom-filter and page-index reads still proceed, so format pruning can discard a file
65+ //! without decoding it, but row selection cannot hide the failure.
66+
67+ use crate :: parquet:: parquet_support:: SparkParquetOptions ;
68+ use crate :: parquet:: schema_adapter:: {
69+ check_file_conversions, RejectOnNonEmpty , REPEATED_PRIMITIVE_KEY ,
70+ } ;
71+ use arrow:: datatypes:: { DataType , FieldRef , Schema , SchemaRef } ;
6472use async_trait:: async_trait;
6573use bytes:: Bytes ;
66- use datafusion:: common:: Result as DFResult ;
74+ use datafusion:: common:: { DataFusionError , Result as DFResult } ;
6775use datafusion:: datasource:: physical_plan:: parquet:: metadata:: DFParquetMetadata ;
6876use datafusion:: datasource:: physical_plan:: parquet:: {
6977 ParquetFileMetrics , ParquetFileReaderFactory ,
@@ -93,10 +101,11 @@ use parquet::file::metadata::{
93101 FooterTail , PageIndexPolicy , ParquetMetaData , ParquetMetaDataReader ,
94102} ;
95103use parquet:: schema:: types:: { ColumnDescPtr , SchemaDescriptor , Type as ParquetType } ;
104+ use std:: collections:: HashMap ;
96105use std:: fmt:: { Debug , Display , Formatter } ;
97106use std:: ops:: Range ;
98107use std:: sync:: atomic:: { AtomicBool , AtomicUsize , Ordering } ;
99- use std:: sync:: { Arc , OnceLock } ;
108+ use std:: sync:: { Arc , Mutex , OnceLock } ;
100109
101110#[ derive( Debug , Clone , Copy , PartialEq , Eq ) ]
102111pub ( crate ) enum ScanIoSource {
@@ -181,6 +190,8 @@ pub struct EagerPageIndexReaderFactory {
181190 // Refuse a file whose Parquet schema carries no field id, as Spark's `ParquetReadSupport`
182191 // does when the requested schema carries one and `ignoreMissing` is not set.
183192 require_field_ids : bool ,
193+ conversion_check : Option < ( SchemaRef , SparkParquetOptions , bool ) > ,
194+ deferred_rejections : Arc < Mutex < HashMap < Path , ( ObjectMeta , RejectOnNonEmpty ) > > > ,
184195}
185196
186197impl EagerPageIndexReaderFactory {
@@ -210,9 +221,23 @@ impl EagerPageIndexReaderFactory {
210221 scan_io_metrics,
211222 spark_variant_schema : false ,
212223 require_field_ids : false ,
224+ conversion_check : None ,
225+ deferred_rejections : Arc :: new ( Mutex :: new ( HashMap :: new ( ) ) ) ,
213226 }
214227 }
215228
229+ /// Retain raw LIST encoding for open-time validation. When row filters are enabled,
230+ /// enforce deferred failures at the first data-page read, after format pruning.
231+ pub ( crate ) fn with_conversion_check (
232+ mut self ,
233+ required : SchemaRef ,
234+ options : SparkParquetOptions ,
235+ row_filters : bool ,
236+ ) -> Self {
237+ self . conversion_check = Some ( ( required, options, row_filters) ) ;
238+ self
239+ }
240+
216241 pub fn with_spark_variant_schema ( mut self , enabled : bool ) -> Self {
217242 self . spark_variant_schema = enabled;
218243 self
@@ -254,6 +279,15 @@ impl ParquetFileReaderFactory for EagerPageIndexReaderFactory {
254279 metrics,
255280 ) ;
256281
282+ // DataFusion replaces the metadata reader after Bloom-filter pruning. The replacement
283+ // receives already-loaded metadata, so retain deferred failures in this scan's factory.
284+ let deferred_rejection = self
285+ . deferred_rejections
286+ . lock ( )
287+ . unwrap ( )
288+ . get ( & partitioned_file. object_meta . location )
289+ . filter ( |( meta, _) | meta == & partitioned_file. object_meta )
290+ . map ( |( _, rejection) | rejection. clone ( ) ) ;
257291 Ok ( Box :: new ( EagerPageIndexReader {
258292 file_metrics,
259293 scan_io_metrics : Arc :: clone ( & self . scan_io_metrics ) ,
@@ -263,6 +297,9 @@ impl ParquetFileReaderFactory for EagerPageIndexReaderFactory {
263297 metadata_size_hint,
264298 spark_variant_schema : self . spark_variant_schema ,
265299 require_field_ids : self . require_field_ids ,
300+ conversion_check : self . conversion_check . clone ( ) ,
301+ deferred_rejections : Arc :: clone ( & self . deferred_rejections ) ,
302+ deferred_rejection,
266303 } ) )
267304 }
268305}
@@ -279,6 +316,9 @@ struct EagerPageIndexReader {
279316 metadata_size_hint : Option < usize > ,
280317 spark_variant_schema : bool ,
281318 require_field_ids : bool ,
319+ conversion_check : Option < ( SchemaRef , SparkParquetOptions , bool ) > ,
320+ deferred_rejections : Arc < Mutex < HashMap < Path , ( ObjectMeta , RejectOnNonEmpty ) > > > ,
321+ deferred_rejection : Option < RejectOnNonEmpty > ,
282322}
283323
284324// Arrow infers ENUM as Binary, losing the distinction from raw binary that Spark needs.
@@ -411,6 +451,53 @@ fn with_spark_arrow_schema(metadata: Arc<ParquetMetaData>) -> ParquetResult<Arc<
411451 ) )
412452}
413453
454+ /// Arrow normalizes both standard and legacy LISTs to the same type. Carry the repeated
455+ /// primitive distinction in an ephemeral schema used by the conversion check, never in the
456+ /// shared footer cache or in the schema given to the decoder.
457+ fn mark_repeated_primitives ( schema : & Schema , parquet : & SchemaDescriptor ) -> Schema {
458+ fn mark ( field : & FieldRef , columns : & [ ColumnDescPtr ] , index : & mut usize ) -> FieldRef {
459+ let data_type = match field. data_type ( ) {
460+ DataType :: Struct ( fields) => DataType :: Struct (
461+ fields
462+ . iter ( )
463+ . map ( |f| mark ( f, columns, index) )
464+ . collect :: < Vec < _ > > ( )
465+ . into ( ) ,
466+ ) ,
467+ DataType :: List ( item) => DataType :: List ( mark ( item, columns, index) ) ,
468+ DataType :: LargeList ( item) => DataType :: LargeList ( mark ( item, columns, index) ) ,
469+ DataType :: FixedSizeList ( item, size) => {
470+ DataType :: FixedSizeList ( mark ( item, columns, index) , * size)
471+ }
472+ DataType :: Map ( entries, sorted) => DataType :: Map ( mark ( entries, columns, index) , * sorted) ,
473+ _ => {
474+ let repeated = columns. get ( * index) . is_some_and ( |column| {
475+ column. self_type ( ) . get_basic_info ( ) . repetition ( )
476+ == parquet:: basic:: Repetition :: REPEATED
477+ } ) ;
478+ * index += 1 ;
479+ let mut field = field. as_ref ( ) . clone ( ) ;
480+ if repeated {
481+ let mut metadata = field. metadata ( ) . clone ( ) ;
482+ metadata. insert ( REPEATED_PRIMITIVE_KEY . to_string ( ) , "true" . to_string ( ) ) ;
483+ field = field. with_metadata ( metadata) ;
484+ }
485+ return Arc :: new ( field) ;
486+ }
487+ } ;
488+ Arc :: new ( field. as_ref ( ) . clone ( ) . with_data_type ( data_type) )
489+ }
490+ let mut index = 0 ;
491+ Schema :: new_with_metadata (
492+ schema
493+ . fields ( )
494+ . iter ( )
495+ . map ( |f| mark ( f, parquet. columns ( ) , & mut index) )
496+ . collect :: < Vec < _ > > ( ) ,
497+ schema. metadata ( ) . clone ( ) ,
498+ )
499+ }
500+
414501impl AsyncFileReader for EagerPageIndexReader {
415502 /// Reads a metadata range, counting its requested size before I/O and its returned
416503 /// bytes only on success. The returned future borrows this reader; store errors retain
@@ -449,6 +536,12 @@ impl AsyncFileReader for EagerPageIndexReader {
449536 where
450537 Self : Send ,
451538 {
539+ if ranges. iter ( ) . any ( |range| !range. is_empty ( ) ) {
540+ if let Some ( rejection) = & self . deferred_rejection {
541+ let error = ParquetError :: External ( Box :: new ( rejection. error ( ) ) ) ;
542+ return async move { Err ( error) } . boxed ( ) ;
543+ }
544+ }
452545 let requested = ranges_bytes ( & ranges) ;
453546 self . file_metrics . bytes_scanned . add ( requested) ;
454547 let scan_io_metrics = Arc :: clone ( & self . scan_io_metrics ) ;
@@ -550,11 +643,48 @@ impl AsyncFileReader for EagerPageIndexReader {
550643 } ,
551644 ) ) ) ;
552645 }
553- if spark_variant_schema {
554- with_spark_arrow_schema ( metadata)
646+ let metadata = if spark_variant_schema {
647+ with_spark_arrow_schema ( metadata) ?
555648 } else {
556- Ok ( metadata)
649+ metadata
650+ } ;
651+ if let Some ( ( required, conversion_options, row_filters) ) =
652+ self . conversion_check . as_ref ( )
653+ {
654+ let file = metadata. file_metadata ( ) ;
655+ let needs_check = * row_filters
656+ || file. schema_descr ( ) . columns ( ) . iter ( ) . any ( |column| {
657+ column. self_type ( ) . get_basic_info ( ) . repetition ( )
658+ == parquet:: basic:: Repetition :: REPEATED
659+ } ) ;
660+ if !needs_check {
661+ return Ok ( metadata) ;
662+ }
663+ let schema =
664+ parquet_to_arrow_schema ( file. schema_descr ( ) , file. key_value_metadata ( ) ) ?;
665+ let schema = mark_repeated_primitives ( & schema, file. schema_descr ( ) ) ;
666+ let rejection = check_file_conversions (
667+ Arc :: clone ( required) ,
668+ Arc :: new ( schema) ,
669+ conversion_options. clone ( ) ,
670+ )
671+ . map_err ( |e| match e {
672+ // Keep structured Spark errors directly inside the Parquet wrapper so
673+ // the JNI error converter preserves Spark's exception type and class.
674+ DataFusionError :: External ( source) => ParquetError :: External ( source) ,
675+ other => ParquetError :: External ( Box :: new ( other) ) ,
676+ } ) ?;
677+ if * row_filters {
678+ if let Some ( rejection) = & rejection {
679+ self . deferred_rejections . lock ( ) . unwrap ( ) . insert (
680+ object_meta. location . clone ( ) ,
681+ ( object_meta. clone ( ) , rejection. clone ( ) ) ,
682+ ) ;
683+ }
684+ self . deferred_rejection = rejection;
685+ }
557686 }
687+ Ok ( metadata)
558688 }
559689 . boxed ( )
560690 }
@@ -1183,6 +1313,64 @@ mod tests {
11831313 assert ! ( metadata_for( false , without_ids) . await . is_ok( ) ) ;
11841314 }
11851315
1316+ #[ tokio:: test]
1317+ async fn conversion_check_preserves_structured_spark_errors ( ) {
1318+ let store = Arc :: new ( InMemory :: new ( ) ) ;
1319+ let schema = Arc :: new ( Schema :: new ( vec ! [
1320+ arrow:: datatypes:: Field :: new( "B" , DataType :: Int32 , false ) ,
1321+ arrow:: datatypes:: Field :: new( "b" , DataType :: Int32 , false ) ,
1322+ ] ) ) ;
1323+ let values: arrow:: array:: ArrayRef = Arc :: new ( Int32Array :: from ( vec ! [ 1 , 2 ] ) ) ;
1324+ let batch =
1325+ RecordBatch :: try_new ( Arc :: clone ( & schema) , vec ! [ Arc :: clone( & values) , values] ) . unwrap ( ) ;
1326+ let mut bytes = Vec :: new ( ) ;
1327+ let mut writer = ArrowWriter :: try_new ( & mut bytes, schema, None ) . unwrap ( ) ;
1328+ writer. write ( & batch) . unwrap ( ) ;
1329+ writer. close ( ) . unwrap ( ) ;
1330+ let size = bytes. len ( ) as u64 ;
1331+ let location = "duplicates.parquet" ;
1332+ store
1333+ . put ( & Path :: from ( location) , Bytes :: from ( bytes) . into ( ) )
1334+ . await
1335+ . unwrap ( ) ;
1336+ let runtime = datafusion:: execution:: runtime_env:: RuntimeEnv :: default ( ) ;
1337+ let metrics = ExecutionPlanMetricsSet :: new ( ) ;
1338+ let required = Arc :: new ( Schema :: new ( vec ! [ arrow:: datatypes:: Field :: new(
1339+ "B" ,
1340+ DataType :: Int32 ,
1341+ false ,
1342+ ) ] ) ) ;
1343+ let mut options =
1344+ SparkParquetOptions :: new ( datafusion_comet_spark_expr:: EvalMode :: Legacy , "UTC" , false ) ;
1345+ options. case_sensitive = false ;
1346+ let factory = EagerPageIndexReaderFactory :: new (
1347+ store,
1348+ runtime. cache_manager . get_file_metadata_cache ( ) ,
1349+ ScanIoSource :: Local ,
1350+ & metrics,
1351+ )
1352+ . with_conversion_check ( required, options, true ) ;
1353+ let mut reader = factory
1354+ . create_reader (
1355+ 0 ,
1356+ PartitionedFile :: new ( location. to_string ( ) , size) ,
1357+ None ,
1358+ & metrics,
1359+ )
1360+ . unwrap ( ) ;
1361+ let error = reader. get_metadata ( None ) . await . unwrap_err ( ) ;
1362+ let ParquetError :: External ( source) = error else {
1363+ panic ! ( "expected structured Spark error, got {error}" ) ;
1364+ } ;
1365+ assert ! (
1366+ matches!(
1367+ source. downcast_ref:: <SparkError >( ) ,
1368+ Some ( SparkError :: DuplicateFieldCaseInsensitive { .. } )
1369+ ) ,
1370+ "unexpected error: {source}"
1371+ ) ;
1372+ }
1373+
11861374 #[ test]
11871375 fn variant_policy_preserves_footer_metadata_and_indexes ( ) {
11881376 let schema = Arc :: new ( Schema :: new_with_metadata (
0 commit comments