4747//! page-index load back into `FileMetadataCache` instead of bypassing it. Preserve the
4848//! duplicate-field validation when replacing this factory.
4949
50- use arrow:: datatypes:: { DataType , FieldRef , Schema } ;
50+ use crate :: parquet:: name_fold:: { fold_name, fold_schema_names} ;
51+ use arrow:: datatypes:: { DataType , FieldRef , Schema , SchemaRef } ;
5152use async_trait:: async_trait;
5253use bytes:: Bytes ;
5354use datafusion:: common:: Result as DFResult ;
@@ -165,6 +166,8 @@ pub struct EagerPageIndexReaderFactory {
165166 // Enable the footer workaround only for scans that project Variant.
166167 // https://github.com/apache/datafusion-comet/issues/5477
167168 spark_variant_schema : bool ,
169+ projected_fields : Option < Arc < HashSet < String > > > ,
170+ case_sensitive : bool ,
168171}
169172
170173impl EagerPageIndexReaderFactory {
@@ -193,9 +196,29 @@ impl EagerPageIndexReaderFactory {
193196 metadata_cache,
194197 scan_io_metrics,
195198 spark_variant_schema : false ,
199+ projected_fields : None ,
200+ case_sensitive : true ,
196201 }
197202 }
198203
204+ pub ( crate ) fn with_required_schema (
205+ mut self ,
206+ schema : & SchemaRef ,
207+ case_sensitive : bool ,
208+ use_field_id : bool ,
209+ ) -> Self {
210+ // Field-ID projections can rename columns, so names cannot safely restrict the walk.
211+ self . projected_fields = ( !use_field_id) . then ( || {
212+ Arc :: new (
213+ fold_schema_names ( schema, case_sensitive)
214+ . into_iter ( )
215+ . collect ( ) ,
216+ )
217+ } ) ;
218+ self . case_sensitive = case_sensitive;
219+ self
220+ }
221+
199222 pub fn with_spark_variant_schema ( mut self , enabled : bool ) -> Self {
200223 self . spark_variant_schema = enabled;
201224 self
@@ -227,6 +250,8 @@ impl ParquetFileReaderFactory for EagerPageIndexReaderFactory {
227250 metadata_cache : Arc :: clone ( & self . metadata_cache ) ,
228251 metadata_size_hint,
229252 spark_variant_schema : self . spark_variant_schema ,
253+ projected_fields : self . projected_fields . clone ( ) ,
254+ case_sensitive : self . case_sensitive ,
230255 } ) )
231256 }
232257}
@@ -242,6 +267,8 @@ struct EagerPageIndexReader {
242267 metadata_cache : Arc < FileMetadataCache > ,
243268 metadata_size_hint : Option < usize > ,
244269 spark_variant_schema : bool ,
270+ projected_fields : Option < Arc < HashSet < String > > > ,
271+ case_sensitive : bool ,
245272}
246273
247274// Arrow infers ENUM as Binary, losing the distinction from raw binary that Spark needs.
@@ -376,20 +403,29 @@ fn with_spark_arrow_schema(metadata: Arc<ParquetMetaData>) -> ParquetResult<Arc<
376403
377404// Duplicate sibling names can make the decoder combine distinct leaves into one column,
378405// multiplying rows before schema adaptation can reject or resolve the duplicate (#5783).
379- // Reject the entire file, including unprojected fields, until the decoder can safely
380- // resolve duplicate siblings. Names in separate groups do not collide.
381- fn validate_field_names ( schema : & Type ) -> parquet:: errors:: Result < ( ) > {
406+ // Only selected top-level subtrees can reach the decoder. Recurse fully within each selected
407+ // subtree because nested projection does not safely separate duplicate leaves (#5884).
408+ fn validate_field_names (
409+ schema : & Type ,
410+ projected_fields : Option < & HashSet < String > > ,
411+ case_sensitive : bool ,
412+ ) -> parquet:: errors:: Result < ( ) > {
382413 if let Type :: GroupType { fields, .. } = schema {
383414 let mut names = HashSet :: with_capacity ( fields. len ( ) ) ;
384415 for field in fields {
416+ if projected_fields. is_some_and ( |projected| {
417+ !projected. contains ( & fold_name ( field. name ( ) , case_sensitive) )
418+ } ) {
419+ continue ;
420+ }
385421 if !names. insert ( field. name ( ) ) {
386422 return Err ( ParquetError :: General ( format ! (
387423 "Comet native scan does not support duplicate Parquet field name '{}' in group '{}'" ,
388424 field. name( ) ,
389425 schema. name( )
390426 ) ) ) ;
391427 }
392- validate_field_names ( field) ?;
428+ validate_field_names ( field, None , case_sensitive ) ?;
393429 }
394430 }
395431 Ok ( ( ) )
@@ -462,6 +498,8 @@ impl AsyncFileReader for EagerPageIndexReader {
462498 let metadata_size_hint = self . metadata_size_hint ;
463499 let scan_io_metrics = Arc :: clone ( & self . scan_io_metrics ) ;
464500 let spark_variant_schema = self . spark_variant_schema ;
501+ let projected_fields = self . projected_fields . clone ( ) ;
502+ let case_sensitive = self . case_sensitive ;
465503 async move {
466504 let file_decryption_properties = options
467505 . and_then ( |o| o. file_decryption_properties ( ) )
@@ -522,7 +560,11 @@ impl AsyncFileReader for EagerPageIndexReader {
522560
523561 let metadata = metadata?;
524562 // Validate cache hits too, before Arrow constructs a decoder for any projection.
525- validate_field_names ( metadata. file_metadata ( ) . schema_descr ( ) . root_schema ( ) ) ?;
563+ validate_field_names (
564+ metadata. file_metadata ( ) . schema_descr ( ) . root_schema ( ) ,
565+ projected_fields. as_deref ( ) ,
566+ case_sensitive,
567+ ) ?;
526568 if spark_variant_schema {
527569 with_spark_arrow_schema ( metadata)
528570 } else {
@@ -866,6 +908,83 @@ mod tests {
866908 } ,
867909 } ;
868910
911+ #[ test]
912+ fn projected_fields_skip_unselected_roots ( ) {
913+ let schema = parquet:: schema:: parser:: parse_message_type (
914+ "message root { optional int64 a; optional int64 a; optional int64 b; }" ,
915+ )
916+ . unwrap ( ) ;
917+ let selected = HashSet :: from ( [ "b" . to_string ( ) ] ) ;
918+ validate_field_names ( & schema, Some ( & selected) , true ) . unwrap ( ) ;
919+ validate_field_names ( & schema, Some ( & HashSet :: new ( ) ) , true ) . unwrap ( ) ;
920+ let selected = HashSet :: from ( [ "a" . to_string ( ) ] ) ;
921+ assert ! ( validate_field_names( & schema, Some ( & selected) , true ) . is_err( ) ) ;
922+ }
923+
924+ #[ test]
925+ fn projected_fields_match_case_insensitively_and_recurse_fully ( ) {
926+ let schema = parquet:: schema:: parser:: parse_message_type (
927+ "message root { optional group Selected {
928+ optional int64 valid; optional int64 dup; optional int64 dup;
929+ } optional int64 unrelated; }" ,
930+ )
931+ . unwrap ( ) ;
932+ let selected = HashSet :: from ( [ "selected" . to_string ( ) ] ) ;
933+ assert ! ( validate_field_names( & schema, Some ( & selected) , false ) . is_err( ) ) ;
934+ let selected = HashSet :: from ( [ "unrelated" . to_string ( ) ] ) ;
935+ validate_field_names ( & schema, Some ( & selected) , false ) . unwrap ( ) ;
936+ }
937+
938+ #[ test]
939+ fn duplicate_names_in_list_element_are_rejected ( ) {
940+ let schema = parquet:: schema:: parser:: parse_message_type (
941+ "message root { optional group a (LIST) { repeated group list {
942+ optional group element { optional int64 dup; optional int64 dup; }
943+ } } }" ,
944+ )
945+ . unwrap ( ) ;
946+ assert ! ( validate_field_names( & schema, None , true )
947+ . unwrap_err( )
948+ . to_string( )
949+ . contains( "group 'element'" ) ) ;
950+ }
951+
952+ #[ test]
953+ fn duplicate_names_in_map_key_value_are_rejected ( ) {
954+ let schema = parquet:: schema:: parser:: parse_message_type (
955+ "message root { optional group a (MAP) { repeated group key_value {
956+ required binary key (UTF8); optional int64 value; optional int64 value;
957+ } } }" ,
958+ )
959+ . unwrap ( ) ;
960+ assert ! ( validate_field_names( & schema, None , true )
961+ . unwrap_err( )
962+ . to_string( )
963+ . contains( "group 'key_value'" ) ) ;
964+ }
965+
966+ #[ test]
967+ fn repeated_names_in_separate_groups_are_valid ( ) {
968+ let schema = parquet:: schema:: parser:: parse_message_type (
969+ "message root { optional group a { optional int64 same; }
970+ optional group b { optional int64 same; } }" ,
971+ )
972+ . unwrap ( ) ;
973+ validate_field_names ( & schema, None , true ) . unwrap ( ) ;
974+ }
975+
976+ #[ test]
977+ fn duplicate_root_names_are_rejected ( ) {
978+ let schema = parquet:: schema:: parser:: parse_message_type (
979+ "message root { optional int64 a; optional int64 a; optional int64 b; }" ,
980+ )
981+ . unwrap ( ) ;
982+ assert ! ( validate_field_names( & schema, None , true )
983+ . unwrap_err( )
984+ . to_string( )
985+ . contains( "group 'root'" ) ) ;
986+ }
987+
869988 #[ derive( Debug ) ]
870989 struct RecordingRangeStore {
871990 inner : InMemory ,
0 commit comments