@@ -1132,17 +1132,12 @@ impl PhysicalPlanner {
11321132 ) )
11331133 }
11341134 OpStruct :: IcebergScan ( scan) => {
1135- // Determine if we're in split mode or legacy mode
1135+ // Determine if we're in split mode (common + partition present) or legacy mode
11361136 let ( required_schema, catalog_properties, metadata_location, tasks) =
1137- if scan. split_mode . unwrap_or ( false ) {
1138- // Split mode: extract from embedded common + single partition
1139- let common = scan. common . as_ref ( ) . ok_or_else ( || {
1140- GeneralError ( "split_mode=true but common data missing" . into ( ) )
1141- } ) ?;
1142- let partition = scan. partition . as_ref ( ) . ok_or_else ( || {
1143- GeneralError ( "split_mode=true but partition data missing" . into ( ) )
1144- } ) ?;
1145-
1137+ if let ( Some ( common) , Some ( partition) ) =
1138+ ( scan. common . as_ref ( ) , scan. partition . as_ref ( ) )
1139+ {
1140+ // Split mode: read from embedded common + single partition
11461141 let schema =
11471142 convert_spark_types_to_arrow_schema ( common. required_schema . as_slice ( ) ) ;
11481143 let catalog_props: HashMap < String , String > = common
@@ -1151,12 +1146,11 @@ impl PhysicalPlanner {
11511146 . map ( |( k, v) | ( k. clone ( ) , v. clone ( ) ) )
11521147 . collect ( ) ;
11531148 let metadata_loc = common. metadata_location . clone ( ) ;
1154- let tasks =
1155- parse_file_scan_tasks_from_common ( common, & partition. file_scan_tasks ) ?;
1149+ let tasks = parse_file_scan_tasks ( common, & partition. file_scan_tasks ) ?;
11561150
11571151 ( schema, catalog_props, metadata_loc, tasks)
11581152 } else {
1159- // Legacy mode: all data in single message
1153+ // Legacy mode: all data in top-level IcebergScan fields
11601154 let schema =
11611155 convert_spark_types_to_arrow_schema ( scan. required_schema . as_slice ( ) ) ;
11621156 let catalog_props: HashMap < String , String > = scan
@@ -1171,7 +1165,7 @@ impl PhysicalPlanner {
11711165 "IcebergScan must have at least one file partition."
11721166 ) ;
11731167
1174- let tasks = parse_file_scan_tasks (
1168+ let tasks = parse_file_scan_tasks_legacy (
11751169 scan,
11761170 & scan. file_partitions [ self . partition as usize ] . file_scan_tasks ,
11771171 ) ?;
@@ -2767,18 +2761,19 @@ fn partition_data_to_struct(
27672761
27682762/// Converts protobuf FileScanTasks from Scala into iceberg-rust FileScanTask objects.
27692763///
2764+ /// Parses Iceberg FileScanTasks from protobuf messages into iceberg-rust's FileScanTask format.
2765+ ///
27702766/// Each task contains a residual predicate that is used for row-group level filtering
27712767/// during Parquet scanning.
27722768///
2773- /// This function uses deduplication pools from the IcebergScan to avoid redundant parsing
2769+ /// This function uses deduplication pools from IcebergScanCommon to avoid redundant parsing
27742770/// of schemas, partition specs, partition types, name mappings, and other repeated data.
27752771fn parse_file_scan_tasks (
2776- proto_scan : & spark_operator:: IcebergScan ,
2772+ proto_common : & spark_operator:: IcebergScanCommon ,
27772773 proto_tasks : & [ spark_operator:: IcebergFileScanTask ] ,
27782774) -> Result < Vec < iceberg:: scan:: FileScanTask > , ExecutionError > {
2779- // Build caches upfront: for 10K tasks with 1 schema, this parses the schema
2780- // once instead of 10K times, eliminating redundant JSON deserialization
2781- let schema_cache: Vec < Arc < iceberg:: spec:: Schema > > = proto_scan
2775+ // Build caches upfront from common data
2776+ let schema_cache: Vec < Arc < iceberg:: spec:: Schema > > = proto_common
27822777 . schema_pool
27832778 . iter ( )
27842779 . map ( |json| {
@@ -2791,7 +2786,7 @@ fn parse_file_scan_tasks(
27912786 } )
27922787 . collect :: < Result < Vec < _ > , _ > > ( ) ?;
27932788
2794- let partition_spec_cache: Vec < Option < Arc < iceberg:: spec:: PartitionSpec > > > = proto_scan
2789+ let partition_spec_cache: Vec < Option < Arc < iceberg:: spec:: PartitionSpec > > > = proto_common
27952790 . partition_spec_pool
27962791 . iter ( )
27972792 . map ( |json| {
@@ -2801,7 +2796,7 @@ fn parse_file_scan_tasks(
28012796 } )
28022797 . collect ( ) ;
28032798
2804- let name_mapping_cache: Vec < Option < Arc < iceberg:: spec:: NameMapping > > > = proto_scan
2799+ let name_mapping_cache: Vec < Option < Arc < iceberg:: spec:: NameMapping > > > = proto_common
28052800 . name_mapping_pool
28062801 . iter ( )
28072802 . map ( |json| {
@@ -2811,7 +2806,7 @@ fn parse_file_scan_tasks(
28112806 } )
28122807 . collect ( ) ;
28132808
2814- let delete_files_cache: Vec < Vec < iceberg:: scan:: FileScanTaskDeleteFile > > = proto_scan
2809+ let delete_files_cache: Vec < Vec < iceberg:: scan:: FileScanTaskDeleteFile > > = proto_common
28152810 . delete_files_pool
28162811 . iter ( )
28172812 . map ( |list| {
@@ -2823,7 +2818,7 @@ fn parse_file_scan_tasks(
28232818 "EQUALITY_DELETES" => iceberg:: spec:: DataContentType :: EqualityDeletes ,
28242819 other => {
28252820 return Err ( GeneralError ( format ! (
2826- "Invalid delete content type '{}'. This indicates a bug in Scala serialization. " ,
2821+ "Invalid delete content type '{}'" ,
28272822 other
28282823 ) ) )
28292824 }
@@ -2844,7 +2839,6 @@ fn parse_file_scan_tasks(
28442839 } )
28452840 . collect :: < Result < Vec < _ > , _ > > ( ) ?;
28462841
2847- // Partition data pool is in protobuf messages
28482842 let results: Result < Vec < _ > , _ > = proto_tasks
28492843 . iter ( )
28502844 . map ( |proto_task| {
@@ -2878,7 +2872,7 @@ fn parse_file_scan_tasks(
28782872 } ;
28792873
28802874 let bound_predicate = if let Some ( idx) = proto_task. residual_idx {
2881- proto_scan
2875+ proto_common
28822876 . residual_pool
28832877 . get ( idx as usize )
28842878 . and_then ( convert_spark_expr_to_predicate)
@@ -2898,24 +2892,22 @@ fn parse_file_scan_tasks(
28982892 } ;
28992893
29002894 let partition = if let Some ( partition_data_idx) = proto_task. partition_data_idx {
2901- // Get partition data from protobuf pool
2902- let partition_data_proto = proto_scan
2895+ let partition_data_proto = proto_common
29032896 . partition_data_pool
29042897 . get ( partition_data_idx as usize )
29052898 . ok_or_else ( || {
29062899 ExecutionError :: GeneralError ( format ! (
29072900 "Invalid partition_data_idx: {} (pool size: {})" ,
29082901 partition_data_idx,
2909- proto_scan . partition_data_pool. len( )
2902+ proto_common . partition_data_pool. len( )
29102903 ) )
29112904 } ) ?;
29122905
2913- // Convert protobuf PartitionData to iceberg Struct
29142906 match partition_data_to_struct ( partition_data_proto) {
29152907 Ok ( s) => Some ( s) ,
29162908 Err ( e) => {
29172909 return Err ( ExecutionError :: GeneralError ( format ! (
2918- "Failed to deserialize partition data from protobuf : {}" ,
2910+ "Failed to deserialize partition data: {}" ,
29192911 e
29202912 ) ) )
29212913 }
@@ -2934,14 +2926,14 @@ fn parse_file_scan_tasks(
29342926 . and_then ( |idx| name_mapping_cache. get ( idx as usize ) )
29352927 . and_then ( |opt| opt. clone ( ) ) ;
29362928
2937- let project_field_ids = proto_scan
2929+ let project_field_ids = proto_common
29382930 . project_field_ids_pool
29392931 . get ( proto_task. project_field_ids_idx as usize )
29402932 . ok_or_else ( || {
29412933 ExecutionError :: GeneralError ( format ! (
29422934 "Invalid project_field_ids_idx: {} (pool size: {})" ,
29432935 proto_task. project_field_ids_idx,
2944- proto_scan . project_field_ids_pool. len( )
2936+ proto_common . project_field_ids_pool. len( )
29452937 ) )
29462938 } ) ?
29472939 . field_ids
@@ -2968,14 +2960,13 @@ fn parse_file_scan_tasks(
29682960 results
29692961}
29702962
2971- /// Parse FileScanTasks from IcebergScanCommon (split mode).
2972- /// Similar to parse_file_scan_tasks but reads pools from IcebergScanCommon.
2973- fn parse_file_scan_tasks_from_common (
2974- proto_common : & spark_operator:: IcebergScanCommon ,
2963+ /// Parse FileScanTasks from IcebergScan (legacy mode - pools in top-level fields).
2964+ fn parse_file_scan_tasks_legacy (
2965+ proto_scan : & spark_operator:: IcebergScan ,
29752966 proto_tasks : & [ spark_operator:: IcebergFileScanTask ] ,
29762967) -> Result < Vec < iceberg:: scan:: FileScanTask > , ExecutionError > {
2977- // Build caches upfront from common data
2978- let schema_cache: Vec < Arc < iceberg:: spec:: Schema > > = proto_common
2968+ // Build caches upfront from top-level IcebergScan fields
2969+ let schema_cache: Vec < Arc < iceberg:: spec:: Schema > > = proto_scan
29792970 . schema_pool
29802971 . iter ( )
29812972 . map ( |json| {
@@ -2988,7 +2979,7 @@ fn parse_file_scan_tasks_from_common(
29882979 } )
29892980 . collect :: < Result < Vec < _ > , _ > > ( ) ?;
29902981
2991- let partition_spec_cache: Vec < Option < Arc < iceberg:: spec:: PartitionSpec > > > = proto_common
2982+ let partition_spec_cache: Vec < Option < Arc < iceberg:: spec:: PartitionSpec > > > = proto_scan
29922983 . partition_spec_pool
29932984 . iter ( )
29942985 . map ( |json| {
@@ -2998,7 +2989,7 @@ fn parse_file_scan_tasks_from_common(
29982989 } )
29992990 . collect ( ) ;
30002991
3001- let name_mapping_cache: Vec < Option < Arc < iceberg:: spec:: NameMapping > > > = proto_common
2992+ let name_mapping_cache: Vec < Option < Arc < iceberg:: spec:: NameMapping > > > = proto_scan
30022993 . name_mapping_pool
30032994 . iter ( )
30042995 . map ( |json| {
@@ -3008,7 +2999,7 @@ fn parse_file_scan_tasks_from_common(
30082999 } )
30093000 . collect ( ) ;
30103001
3011- let delete_files_cache: Vec < Vec < iceberg:: scan:: FileScanTaskDeleteFile > > = proto_common
3002+ let delete_files_cache: Vec < Vec < iceberg:: scan:: FileScanTaskDeleteFile > > = proto_scan
30123003 . delete_files_pool
30133004 . iter ( )
30143005 . map ( |list| {
@@ -3074,7 +3065,7 @@ fn parse_file_scan_tasks_from_common(
30743065 } ;
30753066
30763067 let bound_predicate = if let Some ( idx) = proto_task. residual_idx {
3077- proto_common
3068+ proto_scan
30783069 . residual_pool
30793070 . get ( idx as usize )
30803071 . and_then ( convert_spark_expr_to_predicate)
@@ -3094,14 +3085,14 @@ fn parse_file_scan_tasks_from_common(
30943085 } ;
30953086
30963087 let partition = if let Some ( partition_data_idx) = proto_task. partition_data_idx {
3097- let partition_data_proto = proto_common
3088+ let partition_data_proto = proto_scan
30983089 . partition_data_pool
30993090 . get ( partition_data_idx as usize )
31003091 . ok_or_else ( || {
31013092 ExecutionError :: GeneralError ( format ! (
31023093 "Invalid partition_data_idx: {} (pool size: {})" ,
31033094 partition_data_idx,
3104- proto_common . partition_data_pool. len( )
3095+ proto_scan . partition_data_pool. len( )
31053096 ) )
31063097 } ) ?;
31073098
@@ -3128,14 +3119,14 @@ fn parse_file_scan_tasks_from_common(
31283119 . and_then ( |idx| name_mapping_cache. get ( idx as usize ) )
31293120 . and_then ( |opt| opt. clone ( ) ) ;
31303121
3131- let project_field_ids = proto_common
3122+ let project_field_ids = proto_scan
31323123 . project_field_ids_pool
31333124 . get ( proto_task. project_field_ids_idx as usize )
31343125 . ok_or_else ( || {
31353126 ExecutionError :: GeneralError ( format ! (
31363127 "Invalid project_field_ids_idx: {} (pool size: {})" ,
31373128 proto_task. project_field_ids_idx,
3138- proto_common . project_field_ids_pool. len( )
3129+ proto_scan . project_field_ids_pool. len( )
31393130 ) )
31403131 } ) ?
31413132 . field_ids
0 commit comments