Skip to content

Commit c168832

Browse files
andygroveclaude
andcommitted
Cache additional partition data reflection methods
Add caching for partition data extraction methods that were still being looked up per-field: - PartitionSpec.partitionType() - StructType.fields() - NestedField.type(), fieldId(), name(), isOptional() - StructLike.get(int, Class<?>) These methods are called for every partition field in every task, so caching them provides significant speedup. Co-Authored-By: Claude Opus 4.5 <[email protected]>
1 parent ab020dc commit c168832

2 files changed

Lines changed: 53 additions & 24 deletions

File tree

‎spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala‎

Lines changed: 42 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ object IcebergReflection extends Logging {
4343
schemaClass: Class[_],
4444
partitionSpecParserClass: Class[_],
4545
partitionSpecClass: Class[_],
46+
structLikeClass: Class[_],
47+
nestedFieldClass: Class[_],
4648
// ContentScanTask methods
4749
fileMethod: java.lang.reflect.Method,
4850
startMethod: java.lang.reflect.Method,
@@ -62,7 +64,17 @@ object IcebergReflection extends Logging {
6264
// Schema methods
6365
schemaToJsonMethod: java.lang.reflect.Method,
6466
// PartitionSpec methods
65-
partitionSpecToJsonMethod: java.lang.reflect.Method)
67+
partitionSpecToJsonMethod: java.lang.reflect.Method,
68+
partitionTypeMethod: java.lang.reflect.Method,
69+
// StructType methods
70+
structTypeFieldsMethod: java.lang.reflect.Method,
71+
// NestedField methods
72+
nestedFieldTypeMethod: java.lang.reflect.Method,
73+
nestedFieldIdMethod: java.lang.reflect.Method,
74+
nestedFieldNameMethod: java.lang.reflect.Method,
75+
nestedFieldIsOptionalMethod: java.lang.reflect.Method,
76+
// StructLike methods
77+
structLikeGetMethod: java.lang.reflect.Method)
6678

6779
/**
6880
* Creates a ReflectionCache by loading all classes and methods once.
@@ -77,6 +89,9 @@ object IcebergReflection extends Logging {
7789
val schemaClass = Class.forName(ClassNames.SCHEMA)
7890
val partitionSpecParserClass = Class.forName(ClassNames.PARTITION_SPEC_PARSER)
7991
val partitionSpecClass = Class.forName(ClassNames.PARTITION_SPEC)
92+
val structTypeClass = Class.forName(ClassNames.STRUCT_TYPE)
93+
val nestedFieldClass = Class.forName(ClassNames.NESTED_FIELD)
94+
val structLikeClass = Class.forName(ClassNames.STRUCT_LIKE)
8095
// scalastyle:on classforname
8196

8297
// ContentScanTask methods
@@ -108,9 +123,22 @@ object IcebergReflection extends Logging {
108123
val schemaToJsonMethod = schemaParserClass.getMethod("toJson", schemaClass)
109124
schemaToJsonMethod.setAccessible(true)
110125

111-
// PartitionSpec serialization
126+
// PartitionSpec methods
112127
val partitionSpecToJsonMethod =
113128
partitionSpecParserClass.getMethod("toJson", partitionSpecClass)
129+
val partitionTypeMethod = partitionSpecClass.getMethod("partitionType")
130+
131+
// StructType methods
132+
val structTypeFieldsMethod = structTypeClass.getMethod("fields")
133+
134+
// NestedField methods
135+
val nestedFieldTypeMethod = nestedFieldClass.getMethod("type")
136+
val nestedFieldIdMethod = nestedFieldClass.getMethod("fieldId")
137+
val nestedFieldNameMethod = nestedFieldClass.getMethod("name")
138+
val nestedFieldIsOptionalMethod = nestedFieldClass.getMethod("isOptional")
139+
140+
// StructLike methods
141+
val structLikeGetMethod = structLikeClass.getMethod("get", classOf[Int], classOf[Class[_]])
114142

115143
ReflectionCache(
116144
contentScanTaskClass = contentScanTaskClass,
@@ -121,6 +149,8 @@ object IcebergReflection extends Logging {
121149
schemaClass = schemaClass,
122150
partitionSpecParserClass = partitionSpecParserClass,
123151
partitionSpecClass = partitionSpecClass,
152+
structLikeClass = structLikeClass,
153+
nestedFieldClass = nestedFieldClass,
124154
fileMethod = fileMethod,
125155
startMethod = startMethod,
126156
lengthMethod = lengthMethod,
@@ -134,7 +164,14 @@ object IcebergReflection extends Logging {
134164
deleteSpecIdMethod = deleteSpecIdMethod,
135165
deleteEqualityIdsMethod = deleteEqualityIdsMethod,
136166
schemaToJsonMethod = schemaToJsonMethod,
137-
partitionSpecToJsonMethod = partitionSpecToJsonMethod)
167+
partitionSpecToJsonMethod = partitionSpecToJsonMethod,
168+
partitionTypeMethod = partitionTypeMethod,
169+
structTypeFieldsMethod = structTypeFieldsMethod,
170+
nestedFieldTypeMethod = nestedFieldTypeMethod,
171+
nestedFieldIdMethod = nestedFieldIdMethod,
172+
nestedFieldNameMethod = nestedFieldNameMethod,
173+
nestedFieldIsOptionalMethod = nestedFieldIsOptionalMethod,
174+
structLikeGetMethod = structLikeGetMethod)
138175
}
139176

140177
/**
@@ -154,6 +191,8 @@ object IcebergReflection extends Logging {
154191
val PARTITION_SPEC = "org.apache.iceberg.PartitionSpec"
155192
val PARTITION_FIELD = "org.apache.iceberg.PartitionField"
156193
val UNBOUND_PREDICATE = "org.apache.iceberg.expressions.UnboundPredicate"
194+
val STRUCT_TYPE = "org.apache.iceberg.types.Types$StructType"
195+
val NESTED_FIELD = "org.apache.iceberg.types.Types$NestedField"
157196
}
158197

159198
/**

‎spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala‎

Lines changed: 11 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -331,20 +331,17 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit
331331
val partitionData = cache.partitionMethod.invoke(task)
332332

333333
if (partitionData != null) {
334-
// Get the partition type/schema from the spec
335-
val partitionTypeMethod = spec.getClass.getMethod("partitionType")
336-
val partitionType = partitionTypeMethod.invoke(spec)
334+
// Get the partition type/schema from the spec (using cached method)
335+
val partitionType = cache.partitionTypeMethod.invoke(spec)
337336

338-
// Check if partition type has any fields before serializing
339-
val fieldsMethod = partitionType.getClass.getMethod("fields")
340-
val fields = fieldsMethod
337+
// Check if partition type has any fields before serializing (using cached method)
338+
val fields = cache.structTypeFieldsMethod
341339
.invoke(partitionType)
342340
.asInstanceOf[java.util.List[_]]
343341

344342
// Helper to get field type string (shared by both type and data serialization)
345343
def getFieldType(field: Any): String = {
346-
val typeMethod = field.getClass.getMethod("type")
347-
typeMethod.invoke(field).toString
344+
cache.nestedFieldTypeMethod.invoke(field).toString
348345
}
349346

350347
// Only serialize partition type if there are actual partition fields
@@ -373,15 +370,10 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit
373370
if (fieldTypeStr == IcebergReflection.TypeNames.UNKNOWN) {
374371
None
375372
} else {
376-
val fieldIdMethod = field.getClass.getMethod("fieldId")
377-
val fieldId = fieldIdMethod.invoke(field).asInstanceOf[Int]
378-
379-
val nameMethod = field.getClass.getMethod("name")
380-
val fieldName = nameMethod.invoke(field).asInstanceOf[String]
381-
382-
val isOptionalMethod = field.getClass.getMethod("isOptional")
373+
val fieldId = cache.nestedFieldIdMethod.invoke(field).asInstanceOf[Int]
374+
val fieldName = cache.nestedFieldNameMethod.invoke(field).asInstanceOf[String]
383375
val isOptional =
384-
isOptionalMethod.invoke(field).asInstanceOf[Boolean]
376+
cache.nestedFieldIsOptionalMethod.invoke(field).asInstanceOf[Boolean]
385377
val required = !isOptional
386378

387379
Some(
@@ -433,12 +425,10 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit
433425
None
434426
} else {
435427
// Use the partition type's field ID (same as in partition_type_json)
436-
val fieldIdMethod = field.getClass.getMethod("fieldId")
437-
val fieldId = fieldIdMethod.invoke(field).asInstanceOf[Int]
428+
val fieldId = cache.nestedFieldIdMethod.invoke(field).asInstanceOf[Int]
438429

439-
val getMethod =
440-
partitionData.getClass.getMethod("get", classOf[Int], classOf[Class[_]])
441-
val value = getMethod.invoke(partitionData, Integer.valueOf(idx), classOf[Object])
430+
val value = cache.structLikeGetMethod
431+
.invoke(partitionData, Integer.valueOf(idx), classOf[Object])
442432

443433
Some(partitionValueToProto(fieldId, fieldTypeStr, value))
444434
}

0 commit comments

Comments
 (0)