@@ -166,6 +166,24 @@ fn remap_physical_schema(
166166 } else {
167167 std:: collections:: HashSet :: new ( )
168168 } ;
169+ // Names of logical fields that resolve by ID (their ID is present in the file). Spark
170+ // never name-matches these (`matchIdField` runs first), so any other physical field
171+ // carrying such a name must be hidden from the adapter's name lookup, otherwise a
172+ // first-name match can select the wrong physical column ahead of the ID-matched one
173+ // (e.g. physical `v BINARY` ID 2 shadowing `other VARIANT` ID 1 renamed to `v`).
174+ let id_matched_logical_names: Vec < String > = if should_match_by_id {
175+ logical_schema
176+ . fields ( )
177+ . iter ( )
178+ . filter_map ( |lf| {
179+ parse_field_id ( lf)
180+ . filter ( |id| id_to_phys_names. contains_key ( id) )
181+ . map ( |_| lf. name ( ) . clone ( ) )
182+ } )
183+ . collect ( )
184+ } else {
185+ Vec :: new ( )
186+ } ;
169187 let mut fake_counter: usize = 0 ;
170188
171189 let mut name_map: HashMap < String , String > = HashMap :: new ( ) ;
@@ -208,6 +226,27 @@ fn remap_physical_schema(
208226 ) ;
209227 }
210228
229+ // A physical field that was not ID-matched must not keep a name that an
230+ // ID-matched logical field resolves to; hide it like Spark's fake-name scheme
231+ // so the ID-matched column stays the only name match. Exact names collide in
232+ // case-sensitive mode; ASCII-case-insensitive names collide otherwise.
233+ if should_match_by_id
234+ && id_matched_logical_names. iter ( ) . any ( |name| {
235+ if case_sensitive {
236+ name == field. name ( )
237+ } else {
238+ name. eq_ignore_ascii_case ( field. name ( ) )
239+ }
240+ } )
241+ {
242+ fake_counter += 1 ;
243+ let fake_name = format ! ( "__comet_id_shadowed_field_{}" , fake_counter) ;
244+ return Arc :: new (
245+ Field :: new ( fake_name, field. data_type ( ) . clone ( ) , field. is_nullable ( ) )
246+ . with_metadata ( field. metadata ( ) . clone ( ) ) ,
247+ ) ;
248+ }
249+
211250 // Name match. Spark's `matchIdField` does not fall through to a name match for
212251 // ID-bearing logical fields, so skip those when the schema is ID-bearing.
213252 if !case_sensitive {
@@ -350,29 +389,50 @@ fn check_column_duplicate(col_name: &str, physical_schema: &SchemaRef) -> Option
350389 }
351390}
352391
353- /// Detect a physical Parquet column that Spark's case-insensitive resolver would match to
354- /// `col_name` but the ASCII-only matcher cannot. Spark groups physical names by Unicode
355- /// lowercasing (`toLowerCase(Locale.ROOT)` in `ParquetReadSupport`), so a physical name such
356- /// as `K` (U+212A KELVIN SIGN) resolves to a requested ASCII `k` there while staying
357- /// unmatched here, and the scan-planning gate cannot see physical file names. Returns the
358- /// folded physical name when no ASCII-insensitive match exists but a non-ASCII physical name
359- /// Unicode-lowercases to the same string as `col_name`.
360- fn check_column_unicode_fold_mismatch (
392+ /// Outcome of Spark-parity Unicode fold analysis for one requested column name.
393+ enum UnicodeFoldIssue {
394+ /// More than one physical name folds to the requested name; Spark rejects the read as
395+ /// ambiguous, so the native scan must raise the same duplicate-field error. Carries the
396+ /// bracketed matched-name list in Spark's message format.
397+ Ambiguous ( String ) ,
398+ /// Exactly one physical name folds to the requested name, but the ASCII-only matcher
399+ /// cannot see it, so the column would be silently null-filled. Carries that name.
400+ FoldOnlyMatch ( String ) ,
401+ }
402+
403+ /// Compare Spark's case-insensitive resolution of `col_name` against the ASCII-only matcher.
404+ /// Spark groups physical names by Unicode lowercasing (`toLowerCase(Locale.ROOT)` in
405+ /// `ParquetReadSupport`), so a physical name such as `K` (U+212A KELVIN SIGN) resolves to a
406+ /// requested ASCII `k` there while staying unmatched here, and two physical names that both
407+ /// fold to the requested name are ambiguous there even when only one matches here. The
408+ /// scan-planning gate cannot see physical file names, so this runs per opened file.
409+ fn check_column_unicode_fold (
361410 col_name : & str ,
362411 physical_schema : & SchemaRef ,
363- ) -> Option < String > {
364- if physical_schema
412+ ) -> Option < UnicodeFoldIssue > {
413+ let folded = col_name. to_lowercase ( ) ;
414+ let matches: Vec < & str > = physical_schema
365415 . fields ( )
366416 . iter ( )
367- . any ( |pf| pf. name ( ) . eq_ignore_ascii_case ( col_name) )
368- {
369- return None ;
417+ . filter ( |pf| pf. name ( ) . to_lowercase ( ) == folded)
418+ . map ( |pf| pf. name ( ) . as_str ( ) )
419+ . collect ( ) ;
420+ match matches. as_slice ( ) {
421+ [ ] => None ,
422+ [ single] => {
423+ // An ASCII-case-insensitive match always folds equal, so a single fold match
424+ // that the ASCII matcher also sees is the ordinary working case.
425+ if single. eq_ignore_ascii_case ( col_name) {
426+ None
427+ } else {
428+ Some ( UnicodeFoldIssue :: FoldOnlyMatch ( single. to_string ( ) ) )
429+ }
430+ }
431+ _ => Some ( UnicodeFoldIssue :: Ambiguous ( format ! (
432+ "[{}]" ,
433+ matches. join( ", " )
434+ ) ) ) ,
370435 }
371- physical_schema
372- . fields ( )
373- . iter ( )
374- . find ( |pf| !pf. name ( ) . is_ascii ( ) && pf. name ( ) . to_lowercase ( ) == col_name. to_lowercase ( ) )
375- . map ( |pf| pf. name ( ) . clone ( ) )
376436}
377437
378438impl PhysicalExprAdapterFactory for SparkPhysicalExprAdapterFactory {
@@ -519,19 +579,31 @@ impl PhysicalExprAdapter for SparkPhysicalExprAdapter {
519579 ) ) ) ;
520580 }
521581 if column_err. is_none ( ) && variant_scan {
522- if let Some ( phys) =
523- check_column_unicode_fold_mismatch ( col. name ( ) , orig_physical)
524- {
525- column_err = Some ( DataFusionError :: Execution ( format ! (
526- "Case-insensitive scan of a Variant-bearing schema requires \
527- Unicode column matching: requested column '{name}' resolves \
528- to physical Parquet column '{phys}' only under Unicode case \
529- folding, which Comet's native scan does not support yet \
530- (https://github.com/apache/datafusion-comet/issues/5495). \
531- Retry with spark.comet.enabled=false, or align the column \
532- name case.",
533- name = col. name( ) ,
534- ) ) ) ;
582+ match check_column_unicode_fold ( col. name ( ) , orig_physical) {
583+ Some ( UnicodeFoldIssue :: Ambiguous ( matched) ) => {
584+ // Spark rejects the fold-ambiguous read, so raise its
585+ // duplicate-field error rather than serving one match.
586+ column_err = Some ( DataFusionError :: External ( Box :: new (
587+ SparkError :: DuplicateFieldCaseInsensitive {
588+ required_field_name : col. name ( ) . to_string ( ) ,
589+ matched_fields : matched,
590+ } ,
591+ ) ) ) ;
592+ }
593+ Some ( UnicodeFoldIssue :: FoldOnlyMatch ( phys) ) => {
594+ column_err = Some ( DataFusionError :: Execution ( format ! (
595+ "Case-insensitive scan of a Variant-bearing schema \
596+ requires Unicode column matching: requested column \
597+ '{name}' resolves to physical Parquet column '{phys}' \
598+ only under Unicode case folding, which Comet's native \
599+ scan does not support yet \
600+ (https://github.com/apache/datafusion-comet/issues/5495). \
601+ Retry with spark.comet.enabled=false, or align the \
602+ column name case.",
603+ name = col. name( ) ,
604+ ) ) ) ;
605+ }
606+ None => { }
535607 }
536608 }
537609 }
@@ -2067,8 +2139,8 @@ mod test {
20672139 }
20682140
20692141 #[ test]
2070- fn test_check_column_unicode_fold_mismatch ( ) {
2071- use super :: check_column_unicode_fold_mismatch ;
2142+ fn test_check_column_unicode_fold ( ) {
2143+ use super :: { check_column_unicode_fold , UnicodeFoldIssue } ;
20722144
20732145 // Physical `K` (U+212A KELVIN SIGN) folds to `k` in Spark's Unicode resolver but not
20742146 // in the ASCII matcher: the guard must report it.
@@ -2077,14 +2149,25 @@ mod test {
20772149 DataType :: Int32 ,
20782150 true ,
20792151 ) ] ) ) ;
2080- assert_eq ! (
2081- check_column_unicode_fold_mismatch ( "k" , & kelvin) ,
2082- Some ( "\u{212A} " . to_string ( ) )
2083- ) ;
2152+ assert ! ( matches !(
2153+ check_column_unicode_fold ( "k" , & kelvin) ,
2154+ Some ( UnicodeFoldIssue :: FoldOnlyMatch ( name ) ) if name == "\u{212A} "
2155+ ) ) ;
20842156
20852157 // ASCII case differences are handled by the existing matcher.
20862158 let ascii: SchemaRef = Arc :: new ( Schema :: new ( vec ! [ Field :: new( "K" , DataType :: Int32 , true ) ] ) ) ;
2087- assert_eq ! ( check_column_unicode_fold_mismatch( "k" , & ascii) , None ) ;
2159+ assert ! ( check_column_unicode_fold( "k" , & ascii) . is_none( ) ) ;
2160+
2161+ // ASCII `K` and Kelvin `K` both fold to `k`: Spark rejects the read as ambiguous,
2162+ // so the guard must report the duplicate instead of letting the ASCII match win.
2163+ let ambiguous: SchemaRef = Arc :: new ( Schema :: new ( vec ! [
2164+ Field :: new( "K" , DataType :: Int32 , true ) ,
2165+ Field :: new( "\u{212A} " , DataType :: Int32 , true ) ,
2166+ ] ) ) ;
2167+ assert ! ( matches!(
2168+ check_column_unicode_fold( "k" , & ambiguous) ,
2169+ Some ( UnicodeFoldIssue :: Ambiguous ( matched) ) if matched == "[K, \u{212A} ]"
2170+ ) ) ;
20882171
20892172 // `ſ` (U+017F) does not lowercase to `s`, so Spark also treats the requested column
20902173 // as missing; both engines agree on the missing-column path.
@@ -2093,15 +2176,15 @@ mod test {
20932176 DataType :: Int32 ,
20942177 true ,
20952178 ) ] ) ) ;
2096- assert_eq ! ( check_column_unicode_fold_mismatch ( "s" , & long_s) , None ) ;
2179+ assert ! ( check_column_unicode_fold ( "s" , & long_s) . is_none ( ) ) ;
20972180
20982181 // A genuinely missing column reports nothing.
20992182 let other: SchemaRef = Arc :: new ( Schema :: new ( vec ! [ Field :: new(
21002183 "other" ,
21012184 DataType :: Int32 ,
21022185 true ,
21032186 ) ] ) ) ;
2104- assert_eq ! ( check_column_unicode_fold_mismatch ( "k" , & other) , None ) ;
2187+ assert ! ( check_column_unicode_fold ( "k" , & other) . is_none ( ) ) ;
21052188 }
21062189
21072190 fn variant_field ( name : & str , id : Option < i32 > ) -> Field {
@@ -2235,4 +2318,37 @@ mod test {
22352318 let err = rewrite_column ( logical, physical, false , "v" ) . unwrap_err ( ) ;
22362319 assert ! ( err. to_string( ) . contains( "Found duplicate field" ) , "{err}" ) ;
22372320 }
2321+
2322+ #[ test]
2323+ fn test_unicode_fold_ambiguity_raises_duplicate_error ( ) {
2324+ // ASCII `K` and Kelvin `K` both fold to the requested `k` in Spark's resolver, which
2325+ // rejects the read as ambiguous; serving the single ASCII match would diverge.
2326+ let logical: SchemaRef = Arc :: new ( Schema :: new ( vec ! [ variant_field( "k" , None ) ] ) ) ;
2327+ let physical: SchemaRef = Arc :: new ( Schema :: new ( vec ! [
2328+ variant_field( "K" , None ) ,
2329+ variant_field( "\u{212A} " , None ) ,
2330+ ] ) ) ;
2331+ let err = rewrite_column ( logical, physical, false , "k" ) . unwrap_err ( ) ;
2332+ assert ! ( err. to_string( ) . contains( "Found duplicate field" ) , "{err}" ) ;
2333+ }
2334+
2335+ #[ test]
2336+ fn test_id_matched_name_shadowed_by_wrong_id_physical_field ( ) {
2337+ // Physical `v BINARY` (ID 2) precedes `other VARIANT` (ID 1). Spark resolves the
2338+ // requested `v` (ID 1) by field ID to `other`; the remap renames `other` to `v` and
2339+ // must hide the wrong-ID physical `v` so the first-name lookup cannot select the
2340+ // Binary column ahead of the ID-matched one.
2341+ let logical: SchemaRef = Arc :: new ( Schema :: new ( vec ! [ variant_field( "v" , Some ( 1 ) ) ] ) ) ;
2342+ let physical_v = {
2343+ let field = Field :: new ( "v" , DataType :: Binary , true ) ;
2344+ let mut metadata = std:: collections:: HashMap :: new ( ) ;
2345+ metadata. insert ( "PARQUET:field_id" . to_string ( ) , "2" . to_string ( ) ) ;
2346+ field. with_metadata ( metadata)
2347+ } ;
2348+ let physical: SchemaRef = Arc :: new ( Schema :: new ( vec ! [
2349+ physical_v,
2350+ variant_field( "other" , Some ( 1 ) ) ,
2351+ ] ) ) ;
2352+ rewrite_column ( logical, physical, true , "v" ) . unwrap ( ) ;
2353+ }
22382354}
0 commit comments