@@ -282,6 +282,19 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit
282282 deleteFile : Any ): Option [AnyRef ] =
283283 IcebergReflection .findMethod(clazz, methodName).flatMap(m => Option (m.invoke(deleteFile)))
284284
285+ /**
286+ * Reads equality-delete field IDs for the native serde path. The accessor is required on every
287+ * supported Iceberg DeleteFile API, so a missing method or invocation failure must propagate. A
288+ * null return is distinct: Iceberg uses it for files without equality keys.
289+ */
290+ private def requiredEqualityFieldIds (
291+ deleteFileClass : Class [_],
292+ deleteFile : Any ): java.util.List [Integer ] = {
293+ val method = IcebergReflection .getMethod(deleteFileClass, " equalityFieldIds" )
294+ val ids = method.invoke(deleteFile).asInstanceOf [java.util.List [Integer ]]
295+ if (ids == null ) new java.util.ArrayList [Integer ]() else ids
296+ }
297+
285298 /**
286299 * Extracts delete files from an Iceberg FileScanTask as a list (for deduplication).
287300 *
@@ -301,97 +314,15 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit
301314
302315 val deletes = IcebergReflection .getDeleteFilesFromTask(task, fileScanTaskClass)
303316
304- deletes.asScala.map { deleteFile =>
305- // The path is the one essential field. A delete file we cannot locate cannot be applied,
306- // and silently skipping it would leak deleted rows, so treat a missing path as fatal.
307- val deletePath = IcebergReflection
308- .extractFileLocation(contentFileClass, deleteFile)
309- .getOrElse(
310- throw new RuntimeException (
311- " Neither location() nor path() is declared on this Iceberg version's " +
312- " ContentFile -- cannot extract delete file path from FileScanTask" ))
313-
314- val deleteBuilder = OperatorOuterClass .IcebergDeleteFile .newBuilder()
315- deleteBuilder.setFilePathIdx(internPath(deletePath))
316-
317- val contentType =
318- try {
319- val contentMethod = IcebergReflection .getMethod(deleteFileClass, " content" )
320- val content = contentMethod.invoke(deleteFile)
321- content.toString match {
322- case IcebergReflection .ContentTypes .POSITION_DELETES =>
323- IcebergReflection .ContentTypes .POSITION_DELETES
324- case IcebergReflection .ContentTypes .EQUALITY_DELETES =>
325- IcebergReflection .ContentTypes .EQUALITY_DELETES
326- case other => other
327- }
328- } catch {
329- case _ : Exception =>
330- IcebergReflection .ContentTypes .POSITION_DELETES
331- }
332- deleteBuilder.setContentType(contentType)
333-
334- // "PARQUET", or "PUFFIN" for a V3 deletion vector. iceberg-rust selects its
335- // deletion-vector reader on this, and a wrong value reads a Puffin blob as Parquet, so an
336- // undeterminable format is fatal: by serde time there is no fallback left.
337- val fileFormat = IcebergReflection
338- .getFileFormat(contentFileClass, deleteFile)
339- .getOrElse(
340- throw new RuntimeException (
341- " ContentFile.format() is not declared on this Iceberg version -- cannot tell a " +
342- " deletion vector from a Parquet delete file" ))
343- deleteBuilder.setFileFormat(fileFormat)
344-
345- val specId =
346- try {
347- val specIdMethod = IcebergReflection .getMethod(deleteFileClass, " specId" )
348- specIdMethod.invoke(deleteFile).asInstanceOf [Int ]
349- } catch {
350- case _ : Exception => 0
351- }
352- deleteBuilder.setPartitionSpecId(specId)
353-
354- try {
355- val equalityIdsMethod =
356- IcebergReflection .getMethod(deleteFileClass, " equalityFieldIds" )
357- val equalityIds = equalityIdsMethod
358- .invoke(deleteFile)
359- .asInstanceOf [java.util.List [Integer ]]
360- deleteBuilder.addAllEqualityIds(equalityIds)
361- } catch {
362- case _ : Exception =>
363- }
364-
365- // Gated on the format, not on the accessors returning a value: Iceberg also sets
366- // referencedDataFile on file-scoped Parquet position deletes, where iceberg-rust ignores
367- // it. Forwarding it there would serialize a data-file path per delete file that nothing
368- // reads, and would invite keying deletion-vector detection on the field instead of on
369- // fileFormat, which is the only discriminator.
370- if (fileFormat.equalsIgnoreCase(IcebergReflection .FileFormats .PUFFIN )) {
371- deletionVectorField(deleteFileClass, " referencedDataFile" , deleteFile)
372- .foreach(p => deleteBuilder.setReferencedDataFile(p.asInstanceOf [String ]))
373- deletionVectorField(deleteFileClass, " contentOffset" , deleteFile)
374- .foreach(o => deleteBuilder.setContentOffset(o.asInstanceOf [java.lang.Long ]))
375- deletionVectorField(deleteFileClass, " contentSizeInBytes" , deleteFile)
376- .foreach(s => deleteBuilder.setContentSizeInBytes(s.asInstanceOf [java.lang.Long ]))
377- }
378-
379- // recordCount is declared on ContentFile, so it is present on every supported Iceberg
380- // version and a lookup failure is a real defect rather than an old-version absence.
381- // iceberg-rust rejects a deletion vector without it, since it checks the count against
382- // the cardinality it decodes from the blob.
383- deleteBuilder.setRecordCount(
384- IcebergReflection
385- .getMethod(contentFileClass, " recordCount" )
386- .invoke(deleteFile)
387- .asInstanceOf [java.lang.Long ])
388-
389- // Encrypted delete files carry a plaintext StandardKeyMetadata blob; forward it verbatim.
390- // Unencrypted delete files leave the field unset.
391- keyMetadataBytes(keyMetadataMethod, deleteFile).foreach(deleteBuilder.setKeyMetadata)
392-
393- deleteBuilder.build()
394- }.toSeq
317+ deletes.asScala
318+ .map(
319+ serializeDeleteFile(
320+ _,
321+ contentFileClass,
322+ deleteFileClass,
323+ keyMetadataMethod,
324+ internPath))
325+ .toSeq
395326 } catch {
396327 case e : Exception =>
397328 val msg =
@@ -402,6 +333,85 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit
402333 }
403334 }
404335
336+ /**
337+ * Serializes a single Iceberg DeleteFile to protobuf.
338+ *
339+ * Required delete metadata must never fall back to guessed values after native execution has
340+ * been selected. Missing accessors and invocation failures therefore propagate.
341+ */
342+ private [operator] def serializeDeleteFile (
343+ deleteFile : Any ,
344+ contentFileClass : Class [_],
345+ deleteFileClass : Class [_],
346+ keyMetadataMethod : java.lang.reflect.Method ,
347+ internPath : String => Int ): OperatorOuterClass .IcebergDeleteFile = {
348+ val deletePath = IcebergReflection
349+ .extractFileLocation(contentFileClass, deleteFile)
350+ .getOrElse(
351+ throw new RuntimeException (
352+ " Neither location() nor path() is declared on this Iceberg version's " +
353+ " ContentFile -- cannot extract delete file path from FileScanTask" ))
354+
355+ val deleteBuilder = OperatorOuterClass .IcebergDeleteFile .newBuilder()
356+ deleteBuilder.setFilePathIdx(internPath(deletePath))
357+
358+ val contentMethod = IcebergReflection .getMethod(deleteFileClass, " content" )
359+ val contentType = contentMethod.invoke(deleteFile).toString
360+ deleteBuilder.setContentType(contentType)
361+
362+ // "PARQUET", or "PUFFIN" for a V3 deletion vector. iceberg-rust selects its
363+ // deletion-vector reader on this, and a wrong value reads a Puffin blob as Parquet, so an
364+ // undeterminable format is fatal: by serde time there is no fallback left.
365+ val fileFormat = IcebergReflection
366+ .getFileFormat(contentFileClass, deleteFile)
367+ .getOrElse(
368+ throw new RuntimeException (
369+ " ContentFile.format() is not declared on this Iceberg version -- cannot tell a " +
370+ " deletion vector from a Parquet delete file" ))
371+ deleteBuilder.setFileFormat(fileFormat)
372+
373+ val specIdMethod = IcebergReflection .getMethod(deleteFileClass, " specId" )
374+ deleteBuilder.setPartitionSpecId(specIdMethod.invoke(deleteFile).asInstanceOf [Int ])
375+
376+ val equalityFieldIds = requiredEqualityFieldIds(deleteFileClass, deleteFile)
377+ val isEqualityDelete = contentType == IcebergReflection .ContentTypes .EQUALITY_DELETES
378+ if (isEqualityDelete && equalityFieldIds.isEmpty) {
379+ throw new IllegalStateException (
380+ s " Iceberg equality delete file ' $deletePath' has no equality field IDs " )
381+ }
382+ deleteBuilder.addAllEqualityIds(equalityFieldIds)
383+
384+ // Gated on the format, not on the accessors returning a value: Iceberg also sets
385+ // referencedDataFile on file-scoped Parquet position deletes, where iceberg-rust ignores
386+ // it. Forwarding it there would serialize a data-file path per delete file that nothing
387+ // reads, and would invite keying deletion-vector detection on the field instead of on
388+ // fileFormat, which is the only discriminator.
389+ if (fileFormat.equalsIgnoreCase(IcebergReflection .FileFormats .PUFFIN )) {
390+ deletionVectorField(deleteFileClass, " referencedDataFile" , deleteFile)
391+ .foreach(p => deleteBuilder.setReferencedDataFile(p.asInstanceOf [String ]))
392+ deletionVectorField(deleteFileClass, " contentOffset" , deleteFile)
393+ .foreach(o => deleteBuilder.setContentOffset(o.asInstanceOf [java.lang.Long ]))
394+ deletionVectorField(deleteFileClass, " contentSizeInBytes" , deleteFile)
395+ .foreach(s => deleteBuilder.setContentSizeInBytes(s.asInstanceOf [java.lang.Long ]))
396+ }
397+
398+ // recordCount is declared on ContentFile, so it is present on every supported Iceberg
399+ // version and a lookup failure is a real defect rather than an old-version absence.
400+ // iceberg-rust rejects a deletion vector without it, since it checks the count against
401+ // the cardinality it decodes from the blob.
402+ deleteBuilder.setRecordCount(
403+ IcebergReflection
404+ .getMethod(contentFileClass, " recordCount" )
405+ .invoke(deleteFile)
406+ .asInstanceOf [java.lang.Long ])
407+
408+ // Encrypted delete files carry a plaintext StandardKeyMetadata blob; forward it verbatim.
409+ // Unencrypted delete files leave the field unset.
410+ keyMetadataBytes(keyMetadataMethod, deleteFile).foreach(deleteBuilder.setKeyMetadata)
411+
412+ deleteBuilder.build()
413+ }
414+
405415 /**
406416 * Serializes partition spec and data from an Iceberg FileScanTask.
407417 *
@@ -1134,10 +1144,7 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit
11341144 // dropped ones from the table's schema history (mirrors Iceberg-Java's
11351145 // DeleteFilter.fileProjection).
11361146 val equalityFieldIds = deletes.asScala.flatMap { df =>
1137- IcebergReflection
1138- .getEqualityFieldIds(deleteFileClass, df)
1139- .asScala
1140- .map(_.asInstanceOf [java.lang.Integer ].intValue())
1147+ requiredEqualityFieldIds(deleteFileClass, df).asScala.map(_.intValue())
11411148 }.toSeq
11421149 if (equalityFieldIds.nonEmpty) {
11431150 IcebergReflection
0 commit comments