@@ -682,6 +682,10 @@ impl ArrowReader {
682682 Self :: include_leaf_field_id ( & map_type. key_field , field_ids) ;
683683 Self :: include_leaf_field_id ( & map_type. value_field , field_ids) ;
684684 }
685+ // Variant is a leaf type for Parquet projection purposes (like a primitive).
686+ Type :: Variant ( _) => {
687+ field_ids. push ( field. id ) ;
688+ }
685689 }
686690 }
687691
@@ -754,8 +758,46 @@ impl ArrowReader {
754758 arrow_schema : & ArrowSchemaRef ,
755759 type_promotion_is_valid : fn ( Option < & PrimitiveType > , Option < & PrimitiveType > ) -> bool ,
756760 ) -> Result < ProjectionMask > {
757- let mut column_map = HashMap :: new ( ) ;
761+ // Maps field_id → leaf column indices. Vec because variant contributes two
762+ // leaves (metadata + value) under a single field ID.
763+ let mut column_map: HashMap < i32 , Vec < usize > > = HashMap :: new ( ) ;
758764 let fields = arrow_schema. fields ( ) ;
765+ // HashSet for O(1) membership checks instead of O(n) slice scans.
766+ let leaf_field_id_set: HashSet < i32 > = leaf_field_ids. iter ( ) . copied ( ) . collect ( ) ;
767+
768+ // Variant fields are an Iceberg leaf type but a Parquet GROUP. Their
769+ // sub-fields (metadata, value) carry no embedded field IDs — only the
770+ // parent group has the field ID. filter_leaves therefore never finds
771+ // them via the standard field-ID scan below.
772+ //
773+ // Java's PruneColumns.variant() simply returns the group as-is with no
774+ // type-compatibility check (isStruct() also short-circuits on isVariantType()).
775+ // We replicate that here: pre-scan top-level Arrow struct fields whose
776+ // field ID resolves to Type::Variant and record all their sub-fields so
777+ // the second filter_leaves can include them directly.
778+ let mut variant_sub_fields: HashMap < FieldRef , i32 > = HashMap :: new ( ) ;
779+ for top_field in fields. iter ( ) {
780+ let Some ( field_id) = top_field
781+ . metadata ( )
782+ . get ( PARQUET_FIELD_ID_META_KEY )
783+ . and_then ( |s| i32:: from_str ( s) . ok ( ) )
784+ else {
785+ continue ;
786+ } ;
787+ if !leaf_field_id_set. contains ( & field_id) {
788+ continue ;
789+ }
790+ let Some ( iceberg_field) = iceberg_schema_of_task. field_by_id ( field_id) else {
791+ continue ;
792+ } ;
793+ if let Type :: Variant ( _) = iceberg_field. field_type . as_ref ( )
794+ && let DataType :: Struct ( sub_fields) = top_field. data_type ( )
795+ {
796+ for sub_field in sub_fields {
797+ variant_sub_fields. insert ( sub_field. clone ( ) , field_id) ;
798+ }
799+ }
800+ }
759801
760802 // Pre-project only the fields that have been selected, possibly avoiding converting
761803 // some Arrow types that are not yet supported.
@@ -767,14 +809,22 @@ impl ArrowReader {
767809 . and_then ( |field_id| i32:: from_str ( field_id) . ok ( ) )
768810 . is_some_and ( |field_id| {
769811 projected_fields. insert ( ( * f) . clone ( ) , field_id) ;
770- leaf_field_ids . contains ( & field_id)
812+ leaf_field_id_set . contains ( & field_id)
771813 } )
772814 } ) ,
773815 arrow_schema. metadata ( ) . clone ( ) ,
774816 ) ;
775817 let iceberg_schema = arrow_schema_to_schema ( & projected_arrow_schema) ?;
776818
777819 fields. filter_leaves ( |idx, field| {
820+ // Variant sub-fields: parent group carries the field ID, not the leaf.
821+ // Skip type-promotion check — Type::Variant is not a primitive type
822+ // (matches Java's PruneColumns.variant() which returns the group unchanged).
823+ if let Some ( & variant_field_id) = variant_sub_fields. get ( field) {
824+ column_map. entry ( variant_field_id) . or_default ( ) . push ( idx) ;
825+ return true ;
826+ }
827+
778828 let Some ( field_id) = projected_fields. get ( field) . cloned ( ) else {
779829 return false ;
780830 } ;
@@ -796,16 +846,16 @@ impl ArrowReader {
796846 return false ;
797847 }
798848
799- column_map. insert ( field_id, idx) ;
849+ column_map. entry ( field_id) . or_default ( ) . push ( idx) ;
800850 true
801851 } ) ;
802852
803853 // Schema evolution: New columns may not exist in old Parquet files.
804854 // We only project existing columns; RecordBatchTransformer adds default/NULL values.
805855 let mut indices = vec ! [ ] ;
806856 for field_id in leaf_field_ids {
807- if let Some ( col_idx ) = column_map. get ( field_id) {
808- indices. push ( * col_idx ) ;
857+ if let Some ( col_indices ) = column_map. get ( field_id) {
858+ indices. extend_from_slice ( col_indices ) ;
809859 }
810860 }
811861
@@ -1835,7 +1885,8 @@ mod tests {
18351885 use crate :: io:: FileIO ;
18361886 use crate :: scan:: { FileScanTask , FileScanTaskDeleteFile , FileScanTaskStream } ;
18371887 use crate :: spec:: {
1838- DataContentType , DataFileFormat , Datum , NestedField , PrimitiveType , Schema , SchemaRef , Type ,
1888+ DataContentType , DataFileFormat , Datum , NestedField , PrimitiveType , Schema , SchemaRef ,
1889+ Type , VariantType ,
18391890 } ;
18401891
18411892 fn table_schema_simple ( ) -> SchemaRef {
@@ -2003,6 +2054,100 @@ message schema {
20032054 assert_eq ! ( mask, ProjectionMask :: leaves( & parquet_schema, vec![ 0 ] ) ) ;
20042055 }
20052056
2057+ /// Variant fields are an Iceberg leaf type but a Parquet GROUP whose sub-fields carry
2058+ /// no embedded field IDs. The projection mask must include both metadata and value
2059+ /// leaves when the variant field ID is requested, and must not drop the variant when
2060+ /// it is projected alongside ordinary primitive columns.
2061+ #[ test]
2062+ fn test_arrow_projection_mask_variant ( ) {
2063+ // Iceberg schema: c1 (String, id=1) + v (Variant, id=2)
2064+ let schema = Arc :: new (
2065+ Schema :: builder ( )
2066+ . with_schema_id ( 1 )
2067+ . with_fields ( vec ! [
2068+ NestedField :: required( 1 , "c1" , Type :: Primitive ( PrimitiveType :: String ) ) . into( ) ,
2069+ NestedField :: required( 2 , "v" , Type :: Variant ( VariantType ) ) . into( ) ,
2070+ ] )
2071+ . build ( )
2072+ . unwrap ( ) ,
2073+ ) ;
2074+
2075+ // Arrow schema: c1 with field ID 1; v as Struct(metadata: Binary, value: Binary)
2076+ // with field ID 2 on the struct but NO field IDs on the sub-fields — that is the
2077+ // Parquet variant wire format.
2078+ let arrow_schema = Arc :: new ( ArrowSchema :: new ( vec ! [
2079+ Field :: new( "c1" , DataType :: Utf8 , false ) . with_metadata( HashMap :: from( [ (
2080+ PARQUET_FIELD_ID_META_KEY . to_string( ) ,
2081+ "1" . to_string( ) ,
2082+ ) ] ) ) ,
2083+ Field :: new(
2084+ "v" ,
2085+ DataType :: Struct ( arrow_schema:: Fields :: from( vec![
2086+ Arc :: new( Field :: new( "metadata" , DataType :: Binary , false ) ) ,
2087+ Arc :: new( Field :: new( "value" , DataType :: Binary , false ) ) ,
2088+ ] ) ) ,
2089+ false ,
2090+ )
2091+ . with_metadata( HashMap :: from( [ (
2092+ PARQUET_FIELD_ID_META_KEY . to_string( ) ,
2093+ "2" . to_string( ) ,
2094+ ) ] ) ) ,
2095+ ] ) ) ;
2096+
2097+ // Parquet message: c1 is leaf 0; variant sub-fields metadata=leaf 1, value=leaf 2.
2098+ // No field IDs on sub-leaves — matching the real Iceberg/Spark-written variant format.
2099+ let message_type = "
2100+ message schema {
2101+ required binary c1 (STRING) = 1;
2102+ required group v = 2 {
2103+ required binary metadata;
2104+ required binary value;
2105+ }
2106+ }
2107+ " ;
2108+ let parquet_type = parse_message_type ( message_type) . expect ( "should parse schema" ) ;
2109+ let parquet_schema = SchemaDescriptor :: new ( Arc :: new ( parquet_type) ) ;
2110+
2111+ // Both fields: all three leaves must be included.
2112+ let mask = ArrowReader :: get_arrow_projection_mask (
2113+ & [ 1 , 2 ] ,
2114+ & schema,
2115+ & parquet_schema,
2116+ & arrow_schema,
2117+ false ,
2118+ )
2119+ . expect ( "projection mask for c1 + v" ) ;
2120+ assert_eq ! ( mask, ProjectionMask :: leaves( & parquet_schema, vec![ 0 , 1 , 2 ] ) ) ;
2121+
2122+ // Variant only: leaves 1 (metadata) and 2 (value) must be included.
2123+ let mask_variant_only = ArrowReader :: get_arrow_projection_mask (
2124+ & [ 2 ] ,
2125+ & schema,
2126+ & parquet_schema,
2127+ & arrow_schema,
2128+ false ,
2129+ )
2130+ . expect ( "projection mask for v only" ) ;
2131+ assert_eq ! (
2132+ mask_variant_only,
2133+ ProjectionMask :: leaves( & parquet_schema, vec![ 1 , 2 ] ) ,
2134+ ) ;
2135+
2136+ // Primitive only: leaf 0 (c1) must be included; variant NOT included.
2137+ let mask_primitive_only = ArrowReader :: get_arrow_projection_mask (
2138+ & [ 1 ] ,
2139+ & schema,
2140+ & parquet_schema,
2141+ & arrow_schema,
2142+ false ,
2143+ )
2144+ . expect ( "projection mask for c1 only" ) ;
2145+ assert_eq ! (
2146+ mask_primitive_only,
2147+ ProjectionMask :: leaves( & parquet_schema, vec![ 0 ] ) ,
2148+ ) ;
2149+ }
2150+
20062151 #[ tokio:: test]
20072152 async fn test_kleene_logic_or_behaviour ( ) {
20082153 // a IS NULL OR a = 'foo'
0 commit comments