4545//!
4646//! Filed upstream as apache/datafusion#23978. Revert this once the opener merges its deferred
4747//! page-index load back into `FileMetadataCache` instead of bypassing it.
48+ //!
49+ //! The reader also carries Spark's missing field id check, because the footer is first at hand
50+ //! here. Spark's `ParquetReadSupport` refuses to open a file whose raw schema carries no field
51+ //! id when the requested schema carries one, unless `ignoreMissing` is set, and it walks the
52+ //! raw `MessageType` to decide. The Arrow schema the schema adapter sees later cannot stand in
53+ //! for that walk: the INT96 coercion rebuilds container fields without their metadata, and an
54+ //! id on a `list` or `key_value` group, or on the message root, never reaches an Arrow field.
4855
4956use arrow:: datatypes:: { DataType , FieldRef , Schema } ;
5057use async_trait:: async_trait;
@@ -58,6 +65,7 @@ use datafusion::execution::cache::cache_manager::FileMetadataCache;
5865use datafusion:: physical_plan:: metrics:: {
5966 Count , ExecutionPlanMetricsSet , MetricBuilder , MetricCategory , MetricType ,
6067} ;
68+ use datafusion_comet_common:: SparkError ;
6169use datafusion_datasource:: PartitionedFile ;
6270use futures:: future:: BoxFuture ;
6371use futures:: { FutureExt , StreamExt , TryStreamExt } ;
@@ -77,7 +85,7 @@ use parquet::file::metadata::{FileMetaData, KeyValue, ParquetMetaDataBuilder};
7785use parquet:: file:: metadata:: {
7886 FooterTail , PageIndexPolicy , ParquetMetaData , ParquetMetaDataReader ,
7987} ;
80- use parquet:: schema:: types:: { ColumnDescPtr , SchemaDescriptor } ;
88+ use parquet:: schema:: types:: { ColumnDescPtr , SchemaDescriptor , Type as ParquetType } ;
8189use std:: fmt:: { Debug , Display , Formatter } ;
8290use std:: ops:: Range ;
8391use std:: sync:: atomic:: { AtomicBool , AtomicUsize , Ordering } ;
@@ -163,6 +171,11 @@ pub struct EagerPageIndexReaderFactory {
163171 // Enable the footer workaround only for scans that project Variant.
164172 // https://github.com/apache/datafusion-comet/issues/5477
165173 spark_variant_schema : bool ,
174+ // Whether the schema Spark asked the scan for carries a field id at any depth, computed
175+ // once by the planner. Together with `ignore_missing_field_id` it decides whether a file
176+ // whose Parquet schema carries no id is refused, as Spark's `ParquetReadSupport` does.
177+ requested_schema_has_field_ids : bool ,
178+ ignore_missing_field_id : bool ,
166179}
167180
168181impl EagerPageIndexReaderFactory {
@@ -191,13 +204,40 @@ impl EagerPageIndexReaderFactory {
191204 metadata_cache,
192205 scan_io_metrics,
193206 spark_variant_schema : false ,
207+ requested_schema_has_field_ids : false ,
208+ ignore_missing_field_id : false ,
194209 }
195210 }
196211
197212 pub fn with_spark_variant_schema ( mut self , enabled : bool ) -> Self {
198213 self . spark_variant_schema = enabled;
199214 self
200215 }
216+
217+ /// Arm Spark's missing field id check. A file whose Parquet schema carries no field id
218+ /// is refused on open when `requested_schema_has_field_ids` is set and
219+ /// `ignore_missing_field_id` is not. Both default to off, so a factory that never calls
220+ /// this reads every file.
221+ pub fn with_missing_field_id_check (
222+ mut self ,
223+ requested_schema_has_field_ids : bool ,
224+ ignore_missing_field_id : bool ,
225+ ) -> Self {
226+ self . requested_schema_has_field_ids = requested_schema_has_field_ids;
227+ self . ignore_missing_field_id = ignore_missing_field_id;
228+ self
229+ }
230+ }
231+
232+ /// True when `node` or any node under it carries a field id, the way Spark's
233+ /// `containsFieldIds` answers it over the raw Parquet schema, message root included.
234+ fn parquet_schema_has_field_ids ( node : & ParquetType ) -> bool {
235+ node. get_basic_info ( ) . has_id ( )
236+ || ( node. is_group ( )
237+ && node
238+ . get_fields ( )
239+ . iter ( )
240+ . any ( |field| parquet_schema_has_field_ids ( field) ) )
201241}
202242
203243impl ParquetFileReaderFactory for EagerPageIndexReaderFactory {
@@ -225,6 +265,8 @@ impl ParquetFileReaderFactory for EagerPageIndexReaderFactory {
225265 metadata_cache : Arc :: clone ( & self . metadata_cache ) ,
226266 metadata_size_hint,
227267 spark_variant_schema : self . spark_variant_schema ,
268+ requested_schema_has_field_ids : self . requested_schema_has_field_ids ,
269+ ignore_missing_field_id : self . ignore_missing_field_id ,
228270 } ) )
229271 }
230272}
@@ -240,6 +282,8 @@ struct EagerPageIndexReader {
240282 metadata_cache : Arc < FileMetadataCache > ,
241283 metadata_size_hint : Option < usize > ,
242284 spark_variant_schema : bool ,
285+ requested_schema_has_field_ids : bool ,
286+ ignore_missing_field_id : bool ,
243287}
244288
245289// Arrow infers ENUM as Binary, losing the distinction from raw binary that Spark needs.
@@ -439,6 +483,8 @@ impl AsyncFileReader for EagerPageIndexReader {
439483 let metadata_size_hint = self . metadata_size_hint ;
440484 let scan_io_metrics = Arc :: clone ( & self . scan_io_metrics ) ;
441485 let spark_variant_schema = self . spark_variant_schema ;
486+ let require_file_field_ids =
487+ self . requested_schema_has_field_ids && !self . ignore_missing_field_id ;
442488 async move {
443489 let file_decryption_properties = options
444490 . and_then ( |o| o. file_decryption_properties ( ) )
@@ -498,6 +544,20 @@ impl AsyncFileReader for EagerPageIndexReader {
498544 }
499545
500546 let metadata = metadata?;
547+ // Spark's `ParquetReadSupport` refuses to open a file that carries no field ids when
548+ // the requested schema carries some, unless `ignoreMissing` is set, and it walks the
549+ // raw `MessageType` to decide. The same walk runs here over the footer's schema. The
550+ // error keeps its Spark type through `ParquetError::External`, which the JNI layer
551+ // unwraps, so the JVM sees the same exception Spark raises.
552+ if require_file_field_ids
553+ && !parquet_schema_has_field_ids (
554+ metadata. file_metadata ( ) . schema_descr ( ) . root_schema ( ) ,
555+ )
556+ {
557+ return Err ( ParquetError :: External ( Box :: new (
558+ SparkError :: ParquetMissingFieldIds ,
559+ ) ) ) ;
560+ }
501561 if spark_variant_schema {
502562 with_spark_arrow_schema ( metadata)
503563 } else {
@@ -992,6 +1052,69 @@ mod tests {
9921052 ) ;
9931053 }
9941054
1055+ /// The raw schema walk answers like Spark's `containsFieldIds`: an id on the message root,
1056+ /// on a leaf, or on a repeated `list` group counts, and a schema without any does not.
1057+ #[ test]
1058+ fn parquet_schema_has_field_ids_sees_ids_on_any_node ( ) {
1059+ use parquet:: basic:: Type as PhysicalType ;
1060+ use parquet:: schema:: types:: TypePtr ;
1061+
1062+ let leaf = |name : & str , id : Option < i32 > | -> TypePtr {
1063+ Arc :: new (
1064+ ParquetType :: primitive_type_builder ( name, PhysicalType :: INT32 )
1065+ . with_id ( id)
1066+ . build ( )
1067+ . unwrap ( ) ,
1068+ )
1069+ } ;
1070+ let group = |name : & str , id : Option < i32 > , fields : Vec < TypePtr > | -> TypePtr {
1071+ Arc :: new (
1072+ ParquetType :: group_type_builder ( name)
1073+ . with_id ( id)
1074+ . with_fields ( fields)
1075+ . build ( )
1076+ . unwrap ( ) ,
1077+ )
1078+ } ;
1079+
1080+ assert ! ( !parquet_schema_has_field_ids( & group(
1081+ "schema" ,
1082+ None ,
1083+ vec![ ]
1084+ ) ) ) ;
1085+ assert ! ( !parquet_schema_has_field_ids( & group(
1086+ "schema" ,
1087+ None ,
1088+ vec![ leaf( "a" , None ) ]
1089+ ) ) ) ;
1090+ assert ! ( parquet_schema_has_field_ids( & group(
1091+ "schema" ,
1092+ Some ( 1 ) ,
1093+ vec![ leaf( "a" , None ) ]
1094+ ) ) ) ;
1095+ assert ! ( parquet_schema_has_field_ids( & group(
1096+ "schema" ,
1097+ None ,
1098+ vec![ leaf( "a" , Some ( 1 ) ) ]
1099+ ) ) ) ;
1100+ let list_group_only = group (
1101+ "schema" ,
1102+ None ,
1103+ vec ! [ group(
1104+ "l" ,
1105+ None ,
1106+ vec![ group( "list" , Some ( 5 ) , vec![ leaf( "element" , None ) ] ) ] ,
1107+ ) ] ,
1108+ ) ;
1109+ assert ! ( parquet_schema_has_field_ids( & list_group_only) ) ;
1110+ let nested_without_ids = group (
1111+ "schema" ,
1112+ None ,
1113+ vec ! [ group( "s" , None , vec![ leaf( "a" , None ) ] ) ] ,
1114+ ) ;
1115+ assert ! ( !parquet_schema_has_field_ids( & nested_without_ids) ) ;
1116+ }
1117+
9951118 #[ test]
9961119 fn variant_policy_preserves_footer_metadata_and_indexes ( ) {
9971120 let schema = Arc :: new ( Schema :: new_with_metadata (
0 commit comments