Skip to content

Commit d78d9dd

Browse files
andygroveclaude
andcommitted
Simplify split mode detection for Iceberg scan serialization
Remove the explicit split_mode boolean flag. Instead, detect split mode by checking if both common and partition fields are present. This simplifies the proto and code while maintaining compatibility with native shuffle which uses the serialized plan directly. - Remove split_mode field from IcebergScan proto - Keep both legacy fields (1-12) and split mode fields (20-21) - Detect split mode in Rust by checking common/partition presence - Simplify Scala code to always use CometIcebergSplitRDD Co-Authored-By: Claude Opus 4.5 <[email protected]>
1 parent 0af529d commit d78d9dd

5 files changed

Lines changed: 74 additions & 119 deletions

File tree

‎native/core/src/execution/planner.rs‎

Lines changed: 37 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1132,17 +1132,12 @@ impl PhysicalPlanner {
11321132
))
11331133
}
11341134
OpStruct::IcebergScan(scan) => {
1135-
// Determine if we're in split mode or legacy mode
1135+
// Determine if we're in split mode (common + partition present) or legacy mode
11361136
let (required_schema, catalog_properties, metadata_location, tasks) =
1137-
if scan.split_mode.unwrap_or(false) {
1138-
// Split mode: extract from embedded common + single partition
1139-
let common = scan.common.as_ref().ok_or_else(|| {
1140-
GeneralError("split_mode=true but common data missing".into())
1141-
})?;
1142-
let partition = scan.partition.as_ref().ok_or_else(|| {
1143-
GeneralError("split_mode=true but partition data missing".into())
1144-
})?;
1145-
1137+
if let (Some(common), Some(partition)) =
1138+
(scan.common.as_ref(), scan.partition.as_ref())
1139+
{
1140+
// Split mode: read from embedded common + single partition
11461141
let schema =
11471142
convert_spark_types_to_arrow_schema(common.required_schema.as_slice());
11481143
let catalog_props: HashMap<String, String> = common
@@ -1151,12 +1146,11 @@ impl PhysicalPlanner {
11511146
.map(|(k, v)| (k.clone(), v.clone()))
11521147
.collect();
11531148
let metadata_loc = common.metadata_location.clone();
1154-
let tasks =
1155-
parse_file_scan_tasks_from_common(common, &partition.file_scan_tasks)?;
1149+
let tasks = parse_file_scan_tasks(common, &partition.file_scan_tasks)?;
11561150

11571151
(schema, catalog_props, metadata_loc, tasks)
11581152
} else {
1159-
// Legacy mode: all data in single message
1153+
// Legacy mode: all data in top-level IcebergScan fields
11601154
let schema =
11611155
convert_spark_types_to_arrow_schema(scan.required_schema.as_slice());
11621156
let catalog_props: HashMap<String, String> = scan
@@ -1171,7 +1165,7 @@ impl PhysicalPlanner {
11711165
"IcebergScan must have at least one file partition."
11721166
);
11731167

1174-
let tasks = parse_file_scan_tasks(
1168+
let tasks = parse_file_scan_tasks_legacy(
11751169
scan,
11761170
&scan.file_partitions[self.partition as usize].file_scan_tasks,
11771171
)?;
@@ -2767,18 +2761,19 @@ fn partition_data_to_struct(
27672761

27682762
/// Converts protobuf FileScanTasks from Scala into iceberg-rust FileScanTask objects.
27692763
///
2764+
/// Parses Iceberg FileScanTasks from protobuf messages into iceberg-rust's FileScanTask format.
2765+
///
27702766
/// Each task contains a residual predicate that is used for row-group level filtering
27712767
/// during Parquet scanning.
27722768
///
2773-
/// This function uses deduplication pools from the IcebergScan to avoid redundant parsing
2769+
/// This function uses deduplication pools from IcebergScanCommon to avoid redundant parsing
27742770
/// of schemas, partition specs, partition types, name mappings, and other repeated data.
27752771
fn parse_file_scan_tasks(
2776-
proto_scan: &spark_operator::IcebergScan,
2772+
proto_common: &spark_operator::IcebergScanCommon,
27772773
proto_tasks: &[spark_operator::IcebergFileScanTask],
27782774
) -> Result<Vec<iceberg::scan::FileScanTask>, ExecutionError> {
2779-
// Build caches upfront: for 10K tasks with 1 schema, this parses the schema
2780-
// once instead of 10K times, eliminating redundant JSON deserialization
2781-
let schema_cache: Vec<Arc<iceberg::spec::Schema>> = proto_scan
2775+
// Build caches upfront from common data
2776+
let schema_cache: Vec<Arc<iceberg::spec::Schema>> = proto_common
27822777
.schema_pool
27832778
.iter()
27842779
.map(|json| {
@@ -2791,7 +2786,7 @@ fn parse_file_scan_tasks(
27912786
})
27922787
.collect::<Result<Vec<_>, _>>()?;
27932788

2794-
let partition_spec_cache: Vec<Option<Arc<iceberg::spec::PartitionSpec>>> = proto_scan
2789+
let partition_spec_cache: Vec<Option<Arc<iceberg::spec::PartitionSpec>>> = proto_common
27952790
.partition_spec_pool
27962791
.iter()
27972792
.map(|json| {
@@ -2801,7 +2796,7 @@ fn parse_file_scan_tasks(
28012796
})
28022797
.collect();
28032798

2804-
let name_mapping_cache: Vec<Option<Arc<iceberg::spec::NameMapping>>> = proto_scan
2799+
let name_mapping_cache: Vec<Option<Arc<iceberg::spec::NameMapping>>> = proto_common
28052800
.name_mapping_pool
28062801
.iter()
28072802
.map(|json| {
@@ -2811,7 +2806,7 @@ fn parse_file_scan_tasks(
28112806
})
28122807
.collect();
28132808

2814-
let delete_files_cache: Vec<Vec<iceberg::scan::FileScanTaskDeleteFile>> = proto_scan
2809+
let delete_files_cache: Vec<Vec<iceberg::scan::FileScanTaskDeleteFile>> = proto_common
28152810
.delete_files_pool
28162811
.iter()
28172812
.map(|list| {
@@ -2823,7 +2818,7 @@ fn parse_file_scan_tasks(
28232818
"EQUALITY_DELETES" => iceberg::spec::DataContentType::EqualityDeletes,
28242819
other => {
28252820
return Err(GeneralError(format!(
2826-
"Invalid delete content type '{}'. This indicates a bug in Scala serialization.",
2821+
"Invalid delete content type '{}'",
28272822
other
28282823
)))
28292824
}
@@ -2844,7 +2839,6 @@ fn parse_file_scan_tasks(
28442839
})
28452840
.collect::<Result<Vec<_>, _>>()?;
28462841

2847-
// Partition data pool is in protobuf messages
28482842
let results: Result<Vec<_>, _> = proto_tasks
28492843
.iter()
28502844
.map(|proto_task| {
@@ -2878,7 +2872,7 @@ fn parse_file_scan_tasks(
28782872
};
28792873

28802874
let bound_predicate = if let Some(idx) = proto_task.residual_idx {
2881-
proto_scan
2875+
proto_common
28822876
.residual_pool
28832877
.get(idx as usize)
28842878
.and_then(convert_spark_expr_to_predicate)
@@ -2898,24 +2892,22 @@ fn parse_file_scan_tasks(
28982892
};
28992893

29002894
let partition = if let Some(partition_data_idx) = proto_task.partition_data_idx {
2901-
// Get partition data from protobuf pool
2902-
let partition_data_proto = proto_scan
2895+
let partition_data_proto = proto_common
29032896
.partition_data_pool
29042897
.get(partition_data_idx as usize)
29052898
.ok_or_else(|| {
29062899
ExecutionError::GeneralError(format!(
29072900
"Invalid partition_data_idx: {} (pool size: {})",
29082901
partition_data_idx,
2909-
proto_scan.partition_data_pool.len()
2902+
proto_common.partition_data_pool.len()
29102903
))
29112904
})?;
29122905

2913-
// Convert protobuf PartitionData to iceberg Struct
29142906
match partition_data_to_struct(partition_data_proto) {
29152907
Ok(s) => Some(s),
29162908
Err(e) => {
29172909
return Err(ExecutionError::GeneralError(format!(
2918-
"Failed to deserialize partition data from protobuf: {}",
2910+
"Failed to deserialize partition data: {}",
29192911
e
29202912
)))
29212913
}
@@ -2934,14 +2926,14 @@ fn parse_file_scan_tasks(
29342926
.and_then(|idx| name_mapping_cache.get(idx as usize))
29352927
.and_then(|opt| opt.clone());
29362928

2937-
let project_field_ids = proto_scan
2929+
let project_field_ids = proto_common
29382930
.project_field_ids_pool
29392931
.get(proto_task.project_field_ids_idx as usize)
29402932
.ok_or_else(|| {
29412933
ExecutionError::GeneralError(format!(
29422934
"Invalid project_field_ids_idx: {} (pool size: {})",
29432935
proto_task.project_field_ids_idx,
2944-
proto_scan.project_field_ids_pool.len()
2936+
proto_common.project_field_ids_pool.len()
29452937
))
29462938
})?
29472939
.field_ids
@@ -2968,14 +2960,13 @@ fn parse_file_scan_tasks(
29682960
results
29692961
}
29702962

2971-
/// Parse FileScanTasks from IcebergScanCommon (split mode).
2972-
/// Similar to parse_file_scan_tasks but reads pools from IcebergScanCommon.
2973-
fn parse_file_scan_tasks_from_common(
2974-
proto_common: &spark_operator::IcebergScanCommon,
2963+
/// Parse FileScanTasks from IcebergScan (legacy mode - pools in top-level fields).
2964+
fn parse_file_scan_tasks_legacy(
2965+
proto_scan: &spark_operator::IcebergScan,
29752966
proto_tasks: &[spark_operator::IcebergFileScanTask],
29762967
) -> Result<Vec<iceberg::scan::FileScanTask>, ExecutionError> {
2977-
// Build caches upfront from common data
2978-
let schema_cache: Vec<Arc<iceberg::spec::Schema>> = proto_common
2968+
// Build caches upfront from top-level IcebergScan fields
2969+
let schema_cache: Vec<Arc<iceberg::spec::Schema>> = proto_scan
29792970
.schema_pool
29802971
.iter()
29812972
.map(|json| {
@@ -2988,7 +2979,7 @@ fn parse_file_scan_tasks_from_common(
29882979
})
29892980
.collect::<Result<Vec<_>, _>>()?;
29902981

2991-
let partition_spec_cache: Vec<Option<Arc<iceberg::spec::PartitionSpec>>> = proto_common
2982+
let partition_spec_cache: Vec<Option<Arc<iceberg::spec::PartitionSpec>>> = proto_scan
29922983
.partition_spec_pool
29932984
.iter()
29942985
.map(|json| {
@@ -2998,7 +2989,7 @@ fn parse_file_scan_tasks_from_common(
29982989
})
29992990
.collect();
30002991

3001-
let name_mapping_cache: Vec<Option<Arc<iceberg::spec::NameMapping>>> = proto_common
2992+
let name_mapping_cache: Vec<Option<Arc<iceberg::spec::NameMapping>>> = proto_scan
30022993
.name_mapping_pool
30032994
.iter()
30042995
.map(|json| {
@@ -3008,7 +2999,7 @@ fn parse_file_scan_tasks_from_common(
30082999
})
30093000
.collect();
30103001

3011-
let delete_files_cache: Vec<Vec<iceberg::scan::FileScanTaskDeleteFile>> = proto_common
3002+
let delete_files_cache: Vec<Vec<iceberg::scan::FileScanTaskDeleteFile>> = proto_scan
30123003
.delete_files_pool
30133004
.iter()
30143005
.map(|list| {
@@ -3074,7 +3065,7 @@ fn parse_file_scan_tasks_from_common(
30743065
};
30753066

30763067
let bound_predicate = if let Some(idx) = proto_task.residual_idx {
3077-
proto_common
3068+
proto_scan
30783069
.residual_pool
30793070
.get(idx as usize)
30803071
.and_then(convert_spark_expr_to_predicate)
@@ -3094,14 +3085,14 @@ fn parse_file_scan_tasks_from_common(
30943085
};
30953086

30963087
let partition = if let Some(partition_data_idx) = proto_task.partition_data_idx {
3097-
let partition_data_proto = proto_common
3088+
let partition_data_proto = proto_scan
30983089
.partition_data_pool
30993090
.get(partition_data_idx as usize)
31003091
.ok_or_else(|| {
31013092
ExecutionError::GeneralError(format!(
31023093
"Invalid partition_data_idx: {} (pool size: {})",
31033094
partition_data_idx,
3104-
proto_common.partition_data_pool.len()
3095+
proto_scan.partition_data_pool.len()
31053096
))
31063097
})?;
31073098

@@ -3128,14 +3119,14 @@ fn parse_file_scan_tasks_from_common(
31283119
.and_then(|idx| name_mapping_cache.get(idx as usize))
31293120
.and_then(|opt| opt.clone());
31303121

3131-
let project_field_ids = proto_common
3122+
let project_field_ids = proto_scan
31323123
.project_field_ids_pool
31333124
.get(proto_task.project_field_ids_idx as usize)
31343125
.ok_or_else(|| {
31353126
ExecutionError::GeneralError(format!(
31363127
"Invalid project_field_ids_idx: {} (pool size: {})",
31373128
proto_task.project_field_ids_idx,
3138-
proto_common.project_field_ids_pool.len()
3129+
proto_scan.project_field_ids_pool.len()
31393130
))
31403131
})?
31413132
.field_ids

‎native/proto/src/proto/operator.proto‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -201,15 +201,10 @@ message IcebergScan {
201201
repeated DeleteFileList delete_files_pool = 11;
202202
repeated spark.spark_expression.Expr residual_pool = 12;
203203

204-
// Split mode: when true, common data and single partition are embedded
205-
// instead of all partitions in file_partitions
206-
optional bool split_mode = 20;
207-
208-
// Embedded common data for split mode (pools, metadata, catalog props)
209-
optional IcebergScanCommon common = 21;
210-
211-
// Single partition's file tasks for split mode
212-
optional IcebergFilePartition partition = 22;
204+
// Split mode: common data and single partition embedded for per-task serialization
205+
// When set, pools are read from common instead of the above fields
206+
optional IcebergScanCommon common = 20;
207+
optional IcebergFilePartition partition = 21;
213208
}
214209

215210
// Helper message for deduplicating field ID lists

‎spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala‎

Lines changed: 19 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -687,6 +687,7 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit
687687
scan: CometBatchScanExec,
688688
builder: Operator.Builder,
689689
childOp: Operator*): Option[OperatorOuterClass.Operator] = {
690+
// Build IcebergScan for serialized plan (native shuffle uses this)
690691
val icebergScanBuilder = OperatorOuterClass.IcebergScan.newBuilder()
691692

692693
// Deduplication structures - map unique values to pool indices
@@ -1028,18 +1029,21 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit
10281029
s"$partitionDataPoolBytes bytes (protobuf)")
10291030
}
10301031

1031-
// Build split serialization data for efficient per-partition transfer
1032+
// Build the IcebergScan (used for serialized plan in native shuffle)
10321033
val builtScan = icebergScanBuilder.build()
1034+
1035+
// Build split serialization data for efficient per-partition transfer via CometIcebergSplitRDD
10331036
buildAndStoreSplitData(builtScan)
10341037

10351038
builder.clearChildren()
10361039
Some(builder.setIcebergScan(icebergScanBuilder).build())
10371040
}
10381041

1039-
/** Builds split data (IcebergScanCommon + per-partition bytes) and stores in thread-local. */
1042+
/** Builds IcebergScanCommon from IcebergScan and stores split data in thread-local. */
10401043
private def buildAndStoreSplitData(builtScan: OperatorOuterClass.IcebergScan): Unit = {
10411044
val commonBuilder = OperatorOuterClass.IcebergScanCommon.newBuilder()
10421045

1046+
// Copy pools and metadata from IcebergScan to IcebergScanCommon
10431047
commonBuilder.setMetadataLocation(builtScan.getMetadataLocation)
10441048
commonBuilder.putAllCatalogProperties(builtScan.getCatalogPropertiesMap)
10451049
builtScan.getRequiredSchemaList.forEach(f => commonBuilder.addRequiredSchema(f))
@@ -1072,21 +1076,20 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit
10721076
val metadataLocation = nativeOp.getIcebergScan.getMetadataLocation
10731077

10741078
// Retrieve split data from thread-local (set during convert())
1075-
val splitData = splitDataThreadLocal.get()
1079+
val splitData = splitDataThreadLocal.get().getOrElse {
1080+
throw new IllegalStateException(
1081+
"Programming error: Split data not found in thread-local. " +
1082+
"buildAndStoreSplitData should have been called during convert().")
1083+
}
10761084
splitDataThreadLocal.remove()
10771085

1078-
splitData match {
1079-
case Some(data) =>
1080-
CometIcebergNativeScanExec(
1081-
nativeOp,
1082-
op.wrapped,
1083-
op.session,
1084-
metadataLocation,
1085-
metadata,
1086-
data.commonBytes,
1087-
data.perPartitionBytes)
1088-
case None =>
1089-
CometIcebergNativeScanExec(nativeOp, op.wrapped, op.session, metadataLocation, metadata)
1090-
}
1086+
CometIcebergNativeScanExec(
1087+
nativeOp,
1088+
op.wrapped,
1089+
op.session,
1090+
metadataLocation,
1091+
metadata,
1092+
splitData.commonBytes,
1093+
splitData.perPartitionBytes)
10911094
}
10921095
}

0 commit comments

Comments
 (0)