@@ -24,7 +24,7 @@ use arrow::{
2424 record_batch:: RecordBatch ,
2525} ;
2626
27- use crate :: parquet:: parquet_support:: { spark_parquet_convert, SparkParquetOptions } ;
27+ use crate :: parquet:: parquet_support:: { field_id , spark_parquet_convert, SparkParquetOptions } ;
2828use datafusion:: common:: format:: DEFAULT_CAST_OPTIONS ;
2929use datafusion:: common:: Result as DataFusionResult ;
3030use datafusion:: common:: ScalarValue ;
@@ -73,6 +73,21 @@ fn types_differ_only_in_field_names(physical: &DataType, logical: &DataType) ->
7373 }
7474}
7575
76+ /// Returns true if any struct field in the type tree carries Parquet field-id
77+ /// metadata, mirroring the per-struct `should_match_by_id` check in
78+ /// `parquet_convert_struct_to_struct`.
79+ fn contains_field_id_metadata ( data_type : & DataType ) -> bool {
80+ match data_type {
81+ DataType :: Struct ( fields) => fields
82+ . iter ( )
83+ . any ( |f| field_id ( f) . is_some ( ) || contains_field_id_metadata ( f. data_type ( ) ) ) ,
84+ DataType :: List ( f) | DataType :: LargeList ( f) | DataType :: Map ( f, _) => {
85+ contains_field_id_metadata ( f. data_type ( ) )
86+ }
87+ _ => false ,
88+ }
89+ }
90+
7691/// Recursively relabel an array so its DataType matches `target_type`.
7792/// This only changes metadata (field names, nullability flags in nested fields);
7893/// it does NOT change the underlying buffer data.
@@ -271,6 +286,15 @@ impl PhysicalExpr for CometCastColumnExpr {
271286 let input_physical_field = self . input_physical_field . data_type ( ) ;
272287 let target_field = self . target_field . data_type ( ) ;
273288
289+ // An id-based read must resolve struct fields through spark_parquet_convert,
290+ // which validates the requested ids (e.g. a requested id duplicated in the
291+ // file schema errors); metadata-only relabeling would skip that validation.
292+ let id_based_read = self
293+ . parquet_options
294+ . as_ref ( )
295+ . is_some_and ( |opts| opts. use_field_id )
296+ && contains_field_id_metadata ( target_field) ;
297+
274298 // Handle specific type conversions with custom casts
275299 match ( input_physical_field, target_field) {
276300 // Timestamp(Microsecond) -> Timestamp(Millisecond)
@@ -291,8 +315,11 @@ impl PhysicalExpr for CometCastColumnExpr {
291315 // Nested types that differ only in field names (e.g., List element named
292316 // "item" vs "element", or Map entries named "key_value" vs "entries").
293317 // Re-label the array so the DataType metadata matches the logical schema.
318+ // Skipped for id-based reads so field-id validation still runs below.
294319 ( physical, logical)
295- if physical != logical && types_differ_only_in_field_names ( physical, logical) =>
320+ if !id_based_read
321+ && physical != logical
322+ && types_differ_only_in_field_names ( physical, logical) =>
296323 {
297324 match value {
298325 ColumnarValue :: Array ( array) => {
@@ -352,6 +379,140 @@ mod tests {
352379 use arrow:: array:: { Array , Int32Array , StringArray } ;
353380 use arrow:: datatypes:: { Field , Fields } ;
354381 use datafusion:: physical_expr:: expressions:: Column ;
382+ use datafusion_comet_spark_expr:: EvalMode ;
383+ use parquet:: arrow:: PARQUET_FIELD_ID_META_KEY ;
384+ use std:: collections:: HashMap ;
385+
386+ fn int_field_with_id ( name : & str , id : i32 ) -> Field {
387+ Field :: new ( name, DataType :: Int32 , true ) . with_metadata ( HashMap :: from ( [ (
388+ PARQUET_FIELD_ID_META_KEY . to_string ( ) ,
389+ id. to_string ( ) ,
390+ ) ] ) )
391+ }
392+
393+ /// A file struct with two children sharing field id 1 must raise Spark's
394+ /// duplicate-id error when the requested schema looks id 1 up, even though
395+ /// every child name matches and the relabel shortcut would otherwise apply.
396+ #[ test]
397+ fn test_field_id_read_rejects_duplicate_ids_despite_matching_names ( ) {
398+ let physical_fields = Fields :: from ( vec ! [
399+ int_field_with_id( "x" , 1 ) ,
400+ int_field_with_id( "y" , 1 ) ,
401+ int_field_with_id( "z" , 2 ) ,
402+ ] ) ;
403+ let logical_fields = Fields :: from ( vec ! [
404+ int_field_with_id( "x" , 1 ) ,
405+ int_field_with_id( "y" , 3 ) ,
406+ int_field_with_id( "z" , 2 ) ,
407+ ] ) ;
408+
409+ let input_field = Arc :: new ( Field :: new (
410+ "s" ,
411+ DataType :: Struct ( physical_fields. clone ( ) ) ,
412+ true ,
413+ ) ) ;
414+ let target_field = Arc :: new ( Field :: new ( "s" , DataType :: Struct ( logical_fields) , true ) ) ;
415+
416+ let columns: Vec < ArrayRef > = vec ! [
417+ Arc :: new( Int32Array :: from( vec![ 42 ] ) ) ,
418+ Arc :: new( Int32Array :: from( vec![ 43 ] ) ) ,
419+ Arc :: new( Int32Array :: from( vec![ 44 ] ) ) ,
420+ ] ;
421+ let struct_arr = StructArray :: new ( physical_fields, columns, None ) ;
422+ let schema = Schema :: new ( vec ! [ Arc :: clone( & input_field) ] ) ;
423+ let batch = RecordBatch :: try_new ( Arc :: new ( schema) , vec ! [ Arc :: new( struct_arr) ] ) . unwrap ( ) ;
424+
425+ let mut opts = SparkParquetOptions :: new ( EvalMode :: Legacy , "UTC" , false ) ;
426+ opts. use_field_id = true ;
427+
428+ let col_expr: Arc < dyn PhysicalExpr > = Arc :: new ( Column :: new ( "s" , 0 ) ) ;
429+ let cast_expr = CometCastColumnExpr :: new ( col_expr, input_field, target_field, None )
430+ . with_parquet_options ( opts) ;
431+
432+ let err = cast_expr
433+ . evaluate ( & batch)
434+ . expect_err ( "requested field id 1 matches two file fields and must error" ) ;
435+ let msg = err. to_string ( ) ;
436+ assert ! (
437+ msg. contains( "_LEGACY_ERROR_TEMP_2094" ) && msg. contains( "id=1" ) ,
438+ "expected duplicate field id error, got: {msg}"
439+ ) ;
440+ }
441+
442+ /// Companion guard: without any field ids the relabel shortcut must keep
443+ /// handling name-only differences, whether or not id read mode is enabled.
444+ #[ test]
445+ fn test_relabel_shortcut_kept_for_name_only_differences_without_ids ( ) {
446+ // Physical: s { col: List(Field("item", Int32)) }
447+ // Logical: s { col: List(Field("element", Int32)) }
448+ let physical_list_field = Arc :: new ( Field :: new ( "item" , DataType :: Int32 , true ) ) ;
449+ let logical_list_field = Arc :: new ( Field :: new ( "element" , DataType :: Int32 , true ) ) ;
450+ let physical_fields = Fields :: from ( vec ! [ Field :: new(
451+ "col" ,
452+ DataType :: List ( Arc :: clone( & physical_list_field) ) ,
453+ true ,
454+ ) ] ) ;
455+ let logical_fields = Fields :: from ( vec ! [ Field :: new(
456+ "col" ,
457+ DataType :: List ( logical_list_field) ,
458+ true ,
459+ ) ] ) ;
460+
461+ let input_field = Arc :: new ( Field :: new (
462+ "s" ,
463+ DataType :: Struct ( physical_fields. clone ( ) ) ,
464+ true ,
465+ ) ) ;
466+ let target_field = Arc :: new ( Field :: new (
467+ "s" ,
468+ DataType :: Struct ( logical_fields. clone ( ) ) ,
469+ true ,
470+ ) ) ;
471+
472+ let values = Int32Array :: from ( vec ! [ 1 , 2 , 3 ] ) ;
473+ let list = ListArray :: new (
474+ physical_list_field,
475+ arrow:: buffer:: OffsetBuffer :: new ( vec ! [ 0 , 2 , 3 ] . into ( ) ) ,
476+ Arc :: new ( values) ,
477+ None ,
478+ ) ;
479+ let struct_arr = StructArray :: new ( physical_fields, vec ! [ Arc :: new( list) as ArrayRef ] , None ) ;
480+ let schema = Schema :: new ( vec ! [ Arc :: clone( & input_field) ] ) ;
481+ let batch = RecordBatch :: try_new ( Arc :: new ( schema) , vec ! [ Arc :: new( struct_arr) ] ) . unwrap ( ) ;
482+
483+ let col_expr: Arc < dyn PhysicalExpr > = Arc :: new ( Column :: new ( "s" , 0 ) ) ;
484+
485+ // Without parquet options the fallback arm would return the value
486+ // unchanged, so a relabeled result proves the shortcut itself fired.
487+ let plain_expr = CometCastColumnExpr :: new (
488+ Arc :: clone ( & col_expr) ,
489+ Arc :: clone ( & input_field) ,
490+ Arc :: clone ( & target_field) ,
491+ None ,
492+ ) ;
493+
494+ // Enabling id read mode without any id metadata must not disable the
495+ // shortcut either.
496+ let mut opts = SparkParquetOptions :: new ( EvalMode :: Legacy , "UTC" , false ) ;
497+ opts. use_field_id = true ;
498+ let id_mode_expr = CometCastColumnExpr :: new ( col_expr, input_field, target_field, None )
499+ . with_parquet_options ( opts) ;
500+
501+ for cast_expr in [ plain_expr, id_mode_expr] {
502+ let result = cast_expr. evaluate ( & batch) . unwrap ( ) ;
503+ let ColumnarValue :: Array ( arr) = result else {
504+ panic ! ( "expected array result" ) ;
505+ } ;
506+ assert_eq ! ( arr. data_type( ) , & DataType :: Struct ( logical_fields. clone( ) ) ) ;
507+ let result_struct = arr. as_any ( ) . downcast_ref :: < StructArray > ( ) . unwrap ( ) ;
508+ let result_list = result_struct
509+ . column ( 0 )
510+ . as_any ( )
511+ . downcast_ref :: < ListArray > ( )
512+ . unwrap ( ) ;
513+ assert_eq ! ( result_list. len( ) , 2 ) ;
514+ }
515+ }
355516
356517 #[ test]
357518 fn test_cast_timestamp_micros_to_millis_array ( ) {
0 commit comments