@@ -24,7 +24,9 @@ use arrow::{
2424 record_batch:: RecordBatch ,
2525} ;
2626
27- use crate :: parquet:: parquet_support:: { field_id, spark_parquet_convert, SparkParquetOptions } ;
27+ use crate :: parquet:: parquet_support:: {
28+ spark_parquet_convert_with_mapping, FieldMapping , SparkParquetOptions ,
29+ } ;
2830use datafusion:: common:: format:: DEFAULT_CAST_OPTIONS ;
2931use datafusion:: common:: Result as DataFusionResult ;
3032use datafusion:: common:: ScalarValue ;
@@ -73,21 +75,6 @@ fn types_differ_only_in_field_names(physical: &DataType, logical: &DataType) ->
7375 }
7476}
7577
76- /// Returns true if any struct field in the type tree carries Parquet field-id
77- /// metadata, mirroring the per-struct `should_match_by_id` check in
78- /// `parquet_convert_struct_to_struct`.
79- fn contains_field_id_metadata ( data_type : & DataType ) -> bool {
80- match data_type {
81- DataType :: Struct ( fields) => fields
82- . iter ( )
83- . any ( |f| field_id ( f) . is_some ( ) || contains_field_id_metadata ( f. data_type ( ) ) ) ,
84- DataType :: List ( f) | DataType :: LargeList ( f) | DataType :: Map ( f, _) => {
85- contains_field_id_metadata ( f. data_type ( ) )
86- }
87- _ => false ,
88- }
89- }
90-
9178/// Recursively relabel an array so its DataType matches `target_type`.
9279/// This only changes metadata (field names, nullability flags in nested fields);
9380/// it does NOT change the underlying buffer data.
@@ -202,8 +189,11 @@ pub struct CometCastColumnExpr {
202189 /// Options forwarded to [`cast_column`].
203190 cast_options : CastOptions < ' static > ,
204191 /// Spark parquet options for complex nested type conversions.
205- /// When present, enables `spark_parquet_convert` as a fallback.
192+ /// When present, enables the nested conversion as a fallback.
206193 parquet_options : Option < SparkParquetOptions > ,
194+ /// Which file field supplies each requested nested field, resolved once per file and
195+ /// reused for every batch. Set together with `parquet_options`.
196+ field_mapping : Option < Arc < FieldMapping > > ,
207197}
208198
209199// Manually derive `PartialEq`/`Hash` as `Arc<dyn PhysicalExpr>` does not
@@ -215,6 +205,7 @@ impl PartialEq for CometCastColumnExpr {
215205 && self . target_field . eq ( & other. target_field )
216206 && self . cast_options . eq ( & other. cast_options )
217207 && self . parquet_options . eq ( & other. parquet_options )
208+ && self . field_mapping . eq ( & other. field_mapping )
218209 }
219210}
220211
@@ -225,6 +216,7 @@ impl Hash for CometCastColumnExpr {
225216 self . target_field . hash ( state) ;
226217 self . cast_options . hash ( state) ;
227218 self . parquet_options . hash ( state) ;
219+ self . field_mapping . hash ( state) ;
228220 }
229221}
230222
@@ -242,12 +234,19 @@ impl CometCastColumnExpr {
242234 target_field,
243235 cast_options : cast_options. unwrap_or ( DEFAULT_CAST_OPTIONS ) ,
244236 parquet_options : None ,
237+ field_mapping : None ,
245238 }
246239 }
247240
248- /// Set Spark parquet options to enable complex nested type conversions.
249- pub fn with_parquet_options ( mut self , options : SparkParquetOptions ) -> Self {
241+ /// Enable nested type conversions with Spark parquet options and the field mapping
242+ /// resolved for this expression's physical and target types.
243+ pub fn with_parquet_options (
244+ mut self ,
245+ options : SparkParquetOptions ,
246+ field_mapping : Arc < FieldMapping > ,
247+ ) -> Self {
250248 self . parquet_options = Some ( options) ;
249+ self . field_mapping = Some ( field_mapping) ;
251250 self
252251 }
253252}
@@ -286,14 +285,13 @@ impl PhysicalExpr for CometCastColumnExpr {
286285 let input_physical_field = self . input_physical_field . data_type ( ) ;
287286 let target_field = self . target_field . data_type ( ) ;
288287
289- // An id-based read must resolve struct fields through spark_parquet_convert,
290- // which validates the requested ids (e.g. a requested id duplicated in the
291- // file schema errors); metadata-only relabeling would skip that validation .
292- let id_based_read = self
293- . parquet_options
288+ // Relabeling only swaps metadata, so it is right when every requested field reads
289+ // the file field at its own position. A mapping that reorders fields (ids resolved
290+ // to other positions) has to go through the nested conversion below .
291+ let positional = self
292+ . field_mapping
294293 . as_ref ( )
295- . is_some_and ( |opts| opts. use_field_id )
296- && contains_field_id_metadata ( target_field) ;
294+ . is_none_or ( |mapping| mapping. is_positional ( ) ) ;
297295
298296 // Handle specific type conversions with custom casts
299297 match ( input_physical_field, target_field) {
@@ -315,9 +313,8 @@ impl PhysicalExpr for CometCastColumnExpr {
315313 // Nested types that differ only in field names (e.g., List element named
316314 // "item" vs "element", or Map entries named "key_value" vs "entries").
317315 // Re-label the array so the DataType metadata matches the logical schema.
318- // Skipped for id-based reads so field-id validation still runs below.
319316 ( physical, logical)
320- if !id_based_read
317+ if positional
321318 && physical != logical
322319 && types_differ_only_in_field_names ( physical, logical) =>
323320 {
@@ -329,16 +326,17 @@ impl PhysicalExpr for CometCastColumnExpr {
329326 other => Ok ( other) ,
330327 }
331328 }
332- // Fallback: use spark_parquet_convert for complex nested type conversions
333- // (e.g., List<Struct{a,b,c}> → List<Struct{a,c}>, Map field selection, etc.)
334- _ => {
335- if let Some ( parquet_options) = & self . parquet_options {
336- let converted = spark_parquet_convert ( value, target_field, parquet_options) ?;
337- Ok ( converted)
338- } else {
339- Ok ( value)
340- }
341- }
329+ // Fallback: nested conversion through the resolved mapping
330+ // (e.g., List<Struct{a,b,c}> -> List<Struct{a,c}>, Map field selection, etc.)
331+ _ => match ( & self . parquet_options , & self . field_mapping ) {
332+ ( Some ( parquet_options) , Some ( mapping) ) => spark_parquet_convert_with_mapping (
333+ value,
334+ target_field,
335+ mapping,
336+ parquet_options,
337+ ) ,
338+ _ => Ok ( value) ,
339+ } ,
342340 }
343341 }
344342
@@ -362,8 +360,8 @@ impl PhysicalExpr for CometCastColumnExpr {
362360 Arc :: clone ( & self . target_field ) ,
363361 Some ( self . cast_options . clone ( ) ) ,
364362 ) ;
365- if let Some ( opts) = & self . parquet_options {
366- new_expr = new_expr. with_parquet_options ( opts. clone ( ) ) ;
363+ if let ( Some ( opts) , Some ( mapping ) ) = ( & self . parquet_options , & self . field_mapping ) {
364+ new_expr = new_expr. with_parquet_options ( opts. clone ( ) , Arc :: clone ( mapping ) ) ;
367365 }
368366 Ok ( Arc :: new ( new_expr) )
369367 }
@@ -376,6 +374,7 @@ impl PhysicalExpr for CometCastColumnExpr {
376374#[ cfg( test) ]
377375mod tests {
378376 use super :: * ;
377+ use crate :: parquet:: parquet_support:: resolve_field_mapping;
379378 use arrow:: array:: { Array , Int32Array , StringArray } ;
380379 use arrow:: datatypes:: { Field , Fields } ;
381380 use datafusion:: physical_expr:: expressions:: Column ;
@@ -390,53 +389,65 @@ mod tests {
390389 ) ] ) )
391390 }
392391
393- /// A file struct with two children sharing field id 1 must raise Spark's
394- /// duplicate-id error when the requested schema looks id 1 up, even though
395- /// every child name matches and the relabel shortcut would otherwise apply.
392+ /// File struct `x` (id 1) = 42, `y` (id 2) = 43; requested struct names them the same
393+ /// but swaps the ids. Names and types match, so only the positional gate keeps the
394+ /// relabel shortcut from firing: the mapping reads by id and the result must be
395+ /// `x` = 43, `y` = 42.
396396 #[ test]
397- fn test_field_id_read_rejects_duplicate_ids_despite_matching_names ( ) {
398- let physical_fields = Fields :: from ( vec ! [
399- int_field_with_id( "x" , 1 ) ,
400- int_field_with_id( "y" , 1 ) ,
401- int_field_with_id( "z" , 2 ) ,
402- ] ) ;
403- let logical_fields = Fields :: from ( vec ! [
404- int_field_with_id( "x" , 1 ) ,
405- int_field_with_id( "y" , 3 ) ,
406- int_field_with_id( "z" , 2 ) ,
407- ] ) ;
397+ fn test_swapped_field_ids_bypass_relabel_shortcut ( ) {
398+ let physical_fields =
399+ Fields :: from ( vec ! [ int_field_with_id( "x" , 1 ) , int_field_with_id( "y" , 2 ) ] ) ;
400+ let logical_fields =
401+ Fields :: from ( vec ! [ int_field_with_id( "x" , 2 ) , int_field_with_id( "y" , 1 ) ] ) ;
408402
409403 let input_field = Arc :: new ( Field :: new (
410404 "s" ,
411405 DataType :: Struct ( physical_fields. clone ( ) ) ,
412406 true ,
413407 ) ) ;
414- let target_field = Arc :: new ( Field :: new ( "s" , DataType :: Struct ( logical_fields) , true ) ) ;
408+ let target_field = Arc :: new ( Field :: new (
409+ "s" ,
410+ DataType :: Struct ( logical_fields. clone ( ) ) ,
411+ true ,
412+ ) ) ;
415413
416414 let columns: Vec < ArrayRef > = vec ! [
417415 Arc :: new( Int32Array :: from( vec![ 42 ] ) ) ,
418416 Arc :: new( Int32Array :: from( vec![ 43 ] ) ) ,
419- Arc :: new( Int32Array :: from( vec![ 44 ] ) ) ,
420417 ] ;
421418 let struct_arr = StructArray :: new ( physical_fields, columns, None ) ;
422419 let schema = Schema :: new ( vec ! [ Arc :: clone( & input_field) ] ) ;
423420 let batch = RecordBatch :: try_new ( Arc :: new ( schema) , vec ! [ Arc :: new( struct_arr) ] ) . unwrap ( ) ;
424421
425422 let mut opts = SparkParquetOptions :: new ( EvalMode :: Legacy , "UTC" , false ) ;
426423 opts. use_field_id = true ;
424+ let mapping = Arc :: new (
425+ resolve_field_mapping ( input_field. data_type ( ) , target_field. data_type ( ) , & opts)
426+ . unwrap ( ) ,
427+ ) ;
428+ assert ! ( !mapping. is_positional( ) ) ;
427429
428430 let col_expr: Arc < dyn PhysicalExpr > = Arc :: new ( Column :: new ( "s" , 0 ) ) ;
429431 let cast_expr = CometCastColumnExpr :: new ( col_expr, input_field, target_field, None )
430- . with_parquet_options ( opts) ;
431-
432- let err = cast_expr
433- . evaluate ( & batch)
434- . expect_err ( "requested field id 1 matches two file fields and must error" ) ;
435- let msg = err. to_string ( ) ;
436- assert ! (
437- msg. contains( "_LEGACY_ERROR_TEMP_2094" ) && msg. contains( "id=1" ) ,
438- "expected duplicate field id error, got: {msg}"
439- ) ;
432+ . with_parquet_options ( opts, mapping) ;
433+
434+ let ColumnarValue :: Array ( arr) = cast_expr. evaluate ( & batch) . unwrap ( ) else {
435+ panic ! ( "expected array result" ) ;
436+ } ;
437+ assert_eq ! ( arr. data_type( ) , & DataType :: Struct ( logical_fields) ) ;
438+ let result = arr. as_any ( ) . downcast_ref :: < StructArray > ( ) . unwrap ( ) ;
439+ let x = result
440+ . column ( 0 )
441+ . as_any ( )
442+ . downcast_ref :: < Int32Array > ( )
443+ . unwrap ( ) ;
444+ let y = result
445+ . column ( 1 )
446+ . as_any ( )
447+ . downcast_ref :: < Int32Array > ( )
448+ . unwrap ( ) ;
449+ assert_eq ! ( x. value( 0 ) , 43 ) ;
450+ assert_eq ! ( y. value( 0 ) , 42 ) ;
440451 }
441452
442453 /// Companion guard: without any field ids the relabel shortcut must keep
@@ -492,11 +503,16 @@ mod tests {
492503 ) ;
493504
494505 // Enabling id read mode without any id metadata must not disable the
495- // shortcut either.
506+ // shortcut either: the resolved mapping is positional .
496507 let mut opts = SparkParquetOptions :: new ( EvalMode :: Legacy , "UTC" , false ) ;
497508 opts. use_field_id = true ;
509+ let mapping = Arc :: new (
510+ resolve_field_mapping ( input_field. data_type ( ) , target_field. data_type ( ) , & opts)
511+ . unwrap ( ) ,
512+ ) ;
513+ assert ! ( mapping. is_positional( ) ) ;
498514 let id_mode_expr = CometCastColumnExpr :: new ( col_expr, input_field, target_field, None )
499- . with_parquet_options ( opts) ;
515+ . with_parquet_options ( opts, mapping ) ;
500516
501517 for cast_expr in [ plain_expr, id_mode_expr] {
502518 let result = cast_expr. evaluate ( & batch) . unwrap ( ) ;
0 commit comments