Skip to content

Commit 27b388f

Browse files
unikdahalandygrove
andauthored
fix: make Iceberg delete-file reflection failures fatal (apache#5515)
* fix: harden Iceberg delete-file reflection handling * test: cover Iceberg delete-file serde failures --------- Co-authored-by: Andy Grove <[email protected]>
1 parent f9dd86d commit 27b388f

2 files changed

Lines changed: 263 additions & 101 deletions

File tree

‎spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala‎

Lines changed: 102 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -282,6 +282,19 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit
282282
deleteFile: Any): Option[AnyRef] =
283283
IcebergReflection.findMethod(clazz, methodName).flatMap(m => Option(m.invoke(deleteFile)))
284284

285+
/**
286+
* Reads equality-delete field IDs for the native serde path. The accessor is required on every
287+
* supported Iceberg DeleteFile API, so a missing method or invocation failure must propagate. A
288+
* null return is distinct: Iceberg uses it for files without equality keys.
289+
*/
290+
private def requiredEqualityFieldIds(
291+
deleteFileClass: Class[_],
292+
deleteFile: Any): java.util.List[Integer] = {
293+
val method = IcebergReflection.getMethod(deleteFileClass, "equalityFieldIds")
294+
val ids = method.invoke(deleteFile).asInstanceOf[java.util.List[Integer]]
295+
if (ids == null) new java.util.ArrayList[Integer]() else ids
296+
}
297+
285298
/**
286299
* Extracts delete files from an Iceberg FileScanTask as a list (for deduplication).
287300
*
@@ -301,97 +314,15 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit
301314

302315
val deletes = IcebergReflection.getDeleteFilesFromTask(task, fileScanTaskClass)
303316

304-
deletes.asScala.map { deleteFile =>
305-
// The path is the one essential field. A delete file we cannot locate cannot be applied,
306-
// and silently skipping it would leak deleted rows, so treat a missing path as fatal.
307-
val deletePath = IcebergReflection
308-
.extractFileLocation(contentFileClass, deleteFile)
309-
.getOrElse(
310-
throw new RuntimeException(
311-
"Neither location() nor path() is declared on this Iceberg version's " +
312-
"ContentFile -- cannot extract delete file path from FileScanTask"))
313-
314-
val deleteBuilder = OperatorOuterClass.IcebergDeleteFile.newBuilder()
315-
deleteBuilder.setFilePathIdx(internPath(deletePath))
316-
317-
val contentType =
318-
try {
319-
val contentMethod = IcebergReflection.getMethod(deleteFileClass, "content")
320-
val content = contentMethod.invoke(deleteFile)
321-
content.toString match {
322-
case IcebergReflection.ContentTypes.POSITION_DELETES =>
323-
IcebergReflection.ContentTypes.POSITION_DELETES
324-
case IcebergReflection.ContentTypes.EQUALITY_DELETES =>
325-
IcebergReflection.ContentTypes.EQUALITY_DELETES
326-
case other => other
327-
}
328-
} catch {
329-
case _: Exception =>
330-
IcebergReflection.ContentTypes.POSITION_DELETES
331-
}
332-
deleteBuilder.setContentType(contentType)
333-
334-
// "PARQUET", or "PUFFIN" for a V3 deletion vector. iceberg-rust selects its
335-
// deletion-vector reader on this, and a wrong value reads a Puffin blob as Parquet, so an
336-
// undeterminable format is fatal: by serde time there is no fallback left.
337-
val fileFormat = IcebergReflection
338-
.getFileFormat(contentFileClass, deleteFile)
339-
.getOrElse(
340-
throw new RuntimeException(
341-
"ContentFile.format() is not declared on this Iceberg version -- cannot tell a " +
342-
"deletion vector from a Parquet delete file"))
343-
deleteBuilder.setFileFormat(fileFormat)
344-
345-
val specId =
346-
try {
347-
val specIdMethod = IcebergReflection.getMethod(deleteFileClass, "specId")
348-
specIdMethod.invoke(deleteFile).asInstanceOf[Int]
349-
} catch {
350-
case _: Exception => 0
351-
}
352-
deleteBuilder.setPartitionSpecId(specId)
353-
354-
try {
355-
val equalityIdsMethod =
356-
IcebergReflection.getMethod(deleteFileClass, "equalityFieldIds")
357-
val equalityIds = equalityIdsMethod
358-
.invoke(deleteFile)
359-
.asInstanceOf[java.util.List[Integer]]
360-
deleteBuilder.addAllEqualityIds(equalityIds)
361-
} catch {
362-
case _: Exception =>
363-
}
364-
365-
// Gated on the format, not on the accessors returning a value: Iceberg also sets
366-
// referencedDataFile on file-scoped Parquet position deletes, where iceberg-rust ignores
367-
// it. Forwarding it there would serialize a data-file path per delete file that nothing
368-
// reads, and would invite keying deletion-vector detection on the field instead of on
369-
// fileFormat, which is the only discriminator.
370-
if (fileFormat.equalsIgnoreCase(IcebergReflection.FileFormats.PUFFIN)) {
371-
deletionVectorField(deleteFileClass, "referencedDataFile", deleteFile)
372-
.foreach(p => deleteBuilder.setReferencedDataFile(p.asInstanceOf[String]))
373-
deletionVectorField(deleteFileClass, "contentOffset", deleteFile)
374-
.foreach(o => deleteBuilder.setContentOffset(o.asInstanceOf[java.lang.Long]))
375-
deletionVectorField(deleteFileClass, "contentSizeInBytes", deleteFile)
376-
.foreach(s => deleteBuilder.setContentSizeInBytes(s.asInstanceOf[java.lang.Long]))
377-
}
378-
379-
// recordCount is declared on ContentFile, so it is present on every supported Iceberg
380-
// version and a lookup failure is a real defect rather than an old-version absence.
381-
// iceberg-rust rejects a deletion vector without it, since it checks the count against
382-
// the cardinality it decodes from the blob.
383-
deleteBuilder.setRecordCount(
384-
IcebergReflection
385-
.getMethod(contentFileClass, "recordCount")
386-
.invoke(deleteFile)
387-
.asInstanceOf[java.lang.Long])
388-
389-
// Encrypted delete files carry a plaintext StandardKeyMetadata blob; forward it verbatim.
390-
// Unencrypted delete files leave the field unset.
391-
keyMetadataBytes(keyMetadataMethod, deleteFile).foreach(deleteBuilder.setKeyMetadata)
392-
393-
deleteBuilder.build()
394-
}.toSeq
317+
deletes.asScala
318+
.map(
319+
serializeDeleteFile(
320+
_,
321+
contentFileClass,
322+
deleteFileClass,
323+
keyMetadataMethod,
324+
internPath))
325+
.toSeq
395326
} catch {
396327
case e: Exception =>
397328
val msg =
@@ -402,6 +333,85 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit
402333
}
403334
}
404335

336+
/**
337+
* Serializes a single Iceberg DeleteFile to protobuf.
338+
*
339+
* Required delete metadata must never fall back to guessed values after native execution has
340+
* been selected. Missing accessors and invocation failures therefore propagate.
341+
*/
342+
private[operator] def serializeDeleteFile(
343+
deleteFile: Any,
344+
contentFileClass: Class[_],
345+
deleteFileClass: Class[_],
346+
keyMetadataMethod: java.lang.reflect.Method,
347+
internPath: String => Int): OperatorOuterClass.IcebergDeleteFile = {
348+
val deletePath = IcebergReflection
349+
.extractFileLocation(contentFileClass, deleteFile)
350+
.getOrElse(
351+
throw new RuntimeException(
352+
"Neither location() nor path() is declared on this Iceberg version's " +
353+
"ContentFile -- cannot extract delete file path from FileScanTask"))
354+
355+
val deleteBuilder = OperatorOuterClass.IcebergDeleteFile.newBuilder()
356+
deleteBuilder.setFilePathIdx(internPath(deletePath))
357+
358+
val contentMethod = IcebergReflection.getMethod(deleteFileClass, "content")
359+
val contentType = contentMethod.invoke(deleteFile).toString
360+
deleteBuilder.setContentType(contentType)
361+
362+
// "PARQUET", or "PUFFIN" for a V3 deletion vector. iceberg-rust selects its
363+
// deletion-vector reader on this, and a wrong value reads a Puffin blob as Parquet, so an
364+
// undeterminable format is fatal: by serde time there is no fallback left.
365+
val fileFormat = IcebergReflection
366+
.getFileFormat(contentFileClass, deleteFile)
367+
.getOrElse(
368+
throw new RuntimeException(
369+
"ContentFile.format() is not declared on this Iceberg version -- cannot tell a " +
370+
"deletion vector from a Parquet delete file"))
371+
deleteBuilder.setFileFormat(fileFormat)
372+
373+
val specIdMethod = IcebergReflection.getMethod(deleteFileClass, "specId")
374+
deleteBuilder.setPartitionSpecId(specIdMethod.invoke(deleteFile).asInstanceOf[Int])
375+
376+
val equalityFieldIds = requiredEqualityFieldIds(deleteFileClass, deleteFile)
377+
val isEqualityDelete = contentType == IcebergReflection.ContentTypes.EQUALITY_DELETES
378+
if (isEqualityDelete && equalityFieldIds.isEmpty) {
379+
throw new IllegalStateException(
380+
s"Iceberg equality delete file '$deletePath' has no equality field IDs")
381+
}
382+
deleteBuilder.addAllEqualityIds(equalityFieldIds)
383+
384+
// Gated on the format, not on the accessors returning a value: Iceberg also sets
385+
// referencedDataFile on file-scoped Parquet position deletes, where iceberg-rust ignores
386+
// it. Forwarding it there would serialize a data-file path per delete file that nothing
387+
// reads, and would invite keying deletion-vector detection on the field instead of on
388+
// fileFormat, which is the only discriminator.
389+
if (fileFormat.equalsIgnoreCase(IcebergReflection.FileFormats.PUFFIN)) {
390+
deletionVectorField(deleteFileClass, "referencedDataFile", deleteFile)
391+
.foreach(p => deleteBuilder.setReferencedDataFile(p.asInstanceOf[String]))
392+
deletionVectorField(deleteFileClass, "contentOffset", deleteFile)
393+
.foreach(o => deleteBuilder.setContentOffset(o.asInstanceOf[java.lang.Long]))
394+
deletionVectorField(deleteFileClass, "contentSizeInBytes", deleteFile)
395+
.foreach(s => deleteBuilder.setContentSizeInBytes(s.asInstanceOf[java.lang.Long]))
396+
}
397+
398+
// recordCount is declared on ContentFile, so it is present on every supported Iceberg
399+
// version and a lookup failure is a real defect rather than an old-version absence.
400+
// iceberg-rust rejects a deletion vector without it, since it checks the count against
401+
// the cardinality it decodes from the blob.
402+
deleteBuilder.setRecordCount(
403+
IcebergReflection
404+
.getMethod(contentFileClass, "recordCount")
405+
.invoke(deleteFile)
406+
.asInstanceOf[java.lang.Long])
407+
408+
// Encrypted delete files carry a plaintext StandardKeyMetadata blob; forward it verbatim.
409+
// Unencrypted delete files leave the field unset.
410+
keyMetadataBytes(keyMetadataMethod, deleteFile).foreach(deleteBuilder.setKeyMetadata)
411+
412+
deleteBuilder.build()
413+
}
414+
405415
/**
406416
* Serializes partition spec and data from an Iceberg FileScanTask.
407417
*
@@ -1134,10 +1144,7 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit
11341144
// dropped ones from the table's schema history (mirrors Iceberg-Java's
11351145
// DeleteFilter.fileProjection).
11361146
val equalityFieldIds = deletes.asScala.flatMap { df =>
1137-
IcebergReflection
1138-
.getEqualityFieldIds(deleteFileClass, df)
1139-
.asScala
1140-
.map(_.asInstanceOf[java.lang.Integer].intValue())
1147+
requiredEqualityFieldIds(deleteFileClass, df).asScala.map(_.intValue())
11411148
}.toSeq
11421149
if (equalityFieldIds.nonEmpty) {
11431150
IcebergReflection

0 commit comments

Comments
 (0)