@@ -709,71 +709,100 @@ object IcebergReflection extends Logging {
709709
710710 /**
711711 * Returns the `Types.NestedField` for `fieldId` at any depth of `schema`, or None when `schema`
712- * has no such field. A reflection failure propagates rather than reading as a missing field,
713- * because the callers run at serialization time, when the scan can no longer fall back.
712+ * has no such field. A reflection failure propagates rather than reading as a missing field.
714713 */
715714 def findFieldObject (schema : Any , fieldId : Int ): Option [Any ] =
716715 Option (
717716 getMethod(schema.getClass, " findField" , classOf [Int ])
718717 .invoke(schema, fieldId.asInstanceOf [AnyRef ]))
719718
720719 /**
721- * Returns a schema equal to `baseSchema` but guaranteed to contain `requiredFieldIds`. The ids
722- * not already present are appended as they are in one schema: the table's current schema, or
723- * the newest schema in its history (`table.schemas()`) that has them all under names
724- * `baseSchema` does not use.
720+ * The task schemas of a native Iceberg scan, built once for all of its tasks during planning:
721+ * the pruned scan schema, and the full schema read when pruning is off or cannot apply. Both
722+ * hold the partition sources and equality-delete keys (`requiredFieldIds`) the tasks need
723+ * beyond the projection: iceberg-rust resolves a task's partition spec against its schema, and
724+ * reads the keys to apply equality deletes. The pruned one is None when a required column is
725+ * nested inside a struct the query pruned away, as appending it at the top level would put it
726+ * outside its struct. The full one is None, so that the scan falls back, when no name fits.
725727 *
726- * Two callers need this: an equality delete may be keyed on a column since dropped from the
727- * current schema (mirroring Iceberg-Java's `DeleteFilter.fileProjection`), and a partition
728- * spec's source columns must be present for iceberg-rust to resolve `partition_type` when the
729- * query projects them out. Called at serialization time, so it throws on failure (a required id
730- * that cannot be resolved, or any reflection error) rather than silently degrading.
731- * CometScanRule is responsible for falling back before we get here.
732- */
733- def schemaWithRequiredFields (baseSchema : Any , table : Any , requiredFieldIds : Seq [Int ]): Any = {
728+ * Every task reads with the same schema because iceberg-rust builds an equality-delete
729+ * predicate once per delete file, with the column names of the task that loads it first, and
730+ * binds it by name in every other task of the reader. The names come from the full schema, the
731+ * newest schema, current first, that names every column of the scan schema the same way. That
732+ * is the current schema unless a VERSION AS OF snapshot named a column differently. A key that
733+ * schema lacks, dropped before it, takes the newest name that no other column has.
734+ */
735+ def nativeTaskSchemas (
736+ table : Any ,
737+ tableSchema : Any ,
738+ scanSchema : Any ,
739+ requiredFieldIds : Set [Int ]): (Option [Any ], Option [Any ]) = {
740+ val prunedScanSchema = withoutMetadataColumns(scanSchema)
741+ val scanColumns = columnNamesById(prunedScanSchema).toSet
742+ val schemas = tableSchema +: getAllSchemas(table).reverse
743+ val full = schemas.iterator
744+ .map(schema => schema -> columnNamesById(schema))
745+ .find { case (_, columns) => scanColumns.subsetOf(columns.toSet) }
746+ full match {
747+ case None => (None , None )
748+ case Some ((fullSchema, fullColumns)) =>
749+ val fullNames = fullColumns.values.toSet
750+ val ids = requiredFieldIds.toSeq.sorted
751+ val inFull = ids.flatMap(id => findFieldObject(fullSchema, id).map(id -> _)).toMap
752+ // Keys dropped before `fullSchema`, each under a name that no other column has.
753+ val dropped =
754+ ids.filterNot(inFull.contains).foldLeft(Option (Map .empty[Int , Any ])) { (resolved, id) =>
755+ resolved.flatMap { fields =>
756+ val taken = fullNames ++ fields.values.map(fieldName)
757+ schemas.iterator
758+ .flatMap(findFieldObject(_, id))
759+ .find(field => ! taken.contains(fieldName(field)))
760+ .map(field => fields + (id -> field))
761+ }
762+ }
763+ dropped match {
764+ case None => (None , None )
765+ case Some (droppedFields) =>
766+ val missing = ids.filter(findFieldObject(prunedScanSchema, _).isEmpty)
767+ val prunedTaskSchema =
768+ if (missing.forall(id => fullColumns.contains(id) || ! inFull.contains(id))) {
769+ val fields = missing.map(id => inFull.getOrElse(id, droppedFields(id)))
770+ Some (withColumns(prunedScanSchema, fields))
771+ } else {
772+ None
773+ }
774+ val fullFields = droppedFields.toSeq.sortBy(_._1).map(_._2)
775+ (prunedTaskSchema, Some (withColumns(fullSchema, fullFields)))
776+ }
777+ }
778+ }
779+
780+ /** The top-level columns of `schema`, as field id to name. */
781+ private def columnNamesById (schema : Any ): Map [Int , String ] = {
734782 import scala .jdk .CollectionConverters ._
735- // `findFieldObject` searches recursively, so a source column already present as a nested field
736- // (e.g. `s.region`) is not re-appended at the top level, which would create a duplicate field
737- // id. Empty `requiredFieldIds` (the common non-partitioned, no-delete task) does no lookups.
738- val missingIds =
739- requiredFieldIds.distinct.filter(id => findFieldObject(baseSchema, id).isEmpty)
740- if (missingIds.isEmpty) {
741- baseSchema
783+ getMethod(schema.getClass, " columns" )
784+ .invoke(schema)
785+ .asInstanceOf [java.util.List [_]]
786+ .asScala
787+ .map(column => fieldIdOf(column) -> fieldName(column))
788+ .toMap
789+ }
790+
791+ /** `schema` with `fields` appended at its top level, or `schema` itself when there are none. */
792+ private def withColumns (schema : Any , fields : Seq [Any ]): Any =
793+ if (fields.isEmpty) {
794+ schema
742795 } else {
743- logDebug(
744- s " Native Iceberg scan schema is missing field id(s) ${missingIds.mkString(" ," )}; " +
745- " resolving them from table schema history" )
746- val existing =
747- getMethod(baseSchema.getClass, " columns" )
748- .invoke(baseSchema)
749- .asInstanceOf [java.util.List [_]]
750- val existingNames = existing.asScala.map(fieldName).toSet
751- // Taking every field from one schema keeps their names apart, which a name picked per field
752- // cannot promise once columns have been renamed into each other's names. A VERSION AS OF
753- // scan schema carries the snapshot's names, so a schema also must not name a field like a
754- // column `baseSchema` has. The current schema comes first, to keep current names, and then
755- // the others newest first, so that a promoted column keeps its widest type.
756- val schemas = getMethod(table.getClass, " schema" ).invoke(table) +:
757- getAllSchemas(table).reverse
758- val resolvedFields = schemas.iterator
759- .map(schema => missingIds.flatMap(findFieldObject(schema, _)))
760- .find { fields =>
761- val names = fields.map(fieldName)
762- fields.length == missingIds.length && names.distinct.length == names.length &&
763- ! names.exists(existingNames.contains)
764- }
765- .getOrElse(throw new IllegalStateException (
766- s " Cannot resolve field ids ${missingIds.mkString(" ," )} from one schema in the table " +
767- " schema history under names the task schema does not use" ))
768- val newColumns = new java.util.ArrayList [Any ](existing.size + resolvedFields.length)
769- newColumns.addAll(existing)
770- resolvedFields.foreach(newColumns.add)
771- baseSchema.getClass
796+ val columns =
797+ getMethod(schema.getClass, " columns" ).invoke(schema).asInstanceOf [java.util.List [_]]
798+ val newColumns = new java.util.ArrayList [Any ](columns.size + fields.length)
799+ newColumns.addAll(columns)
800+ fields.foreach(newColumns.add)
801+ schema.getClass
772802 .getConstructor(classOf [java.util.List [_]])
773803 .newInstance(newColumns)
774804 .asInstanceOf [AnyRef ]
775805 }
776- }
777806
778807 /** The spec reserves field ids above `Integer.MAX_VALUE - 200` for metadata columns. */
779808 val MaxDataFieldId : Int = Int .MaxValue - 200
@@ -798,44 +827,16 @@ object IcebergReflection extends Logging {
798827 }
799828 }
800829
801- /** Whether `fieldId` is a field of `schema` nested below its top level. */
802- def isNestedField (schema : Any , fieldId : Int ): Boolean = {
803- import scala .jdk .CollectionConverters ._
804- findFieldObject(schema, fieldId).isDefined &&
805- ! getMethod(schema.getClass, " columns" )
806- .invoke(schema)
807- .asInstanceOf [java.util.List [_]]
808- .asScala
809- .exists(fieldIdOf(_) == fieldId)
810- }
811-
812830 /**
813- * The source column field ids referenced by a task's partition spec.
831+ * The `sourceId`s of a partition spec's fields, in spec order. Empty for an unpartitioned spec.
814832 *
815833 * iceberg-rust validates a `FileScanTask` by resolving its partition spec against the task
816- * schema (`partition_type(schema)`), so a task carrying a partition spec needs those source
817- * columns present in the schema even when the query projects them out (for example selecting
818- * only `_spec_id` or `_partition`). These ids are unioned into the native task schema via
819- * [[schemaWithRequiredFields ]]. project_field_ids still drives the read, so the columns are not
820- * materialized into the scan output. All ids resolve from the table's schema history, including
821- * a source column later dropped by partition evolution.
822- *
823- * Returns an empty sequence when the task has no partition spec.
824- */
825- def partitionSourceFieldIds (task : Any , fileScanTaskClass : Class [_]): Seq [Int ] = {
826- val spec =
827- try {
828- getMethod(fileScanTaskClass, " spec" ).invoke(task)
829- } catch {
830- case _ : Exception => null
831- }
832- if (spec == null ) Seq .empty else partitionFieldSourceIds(spec)
833- }
834-
835- /**
836- * The `sourceId`s of a partition spec's fields, in spec order. Empty for an unpartitioned spec.
834+ * schema (`partition_type(schema)`), so the source columns must be in the task schema even when
835+ * the query projects them out (for example selecting only `_spec_id` or `_partition`). See
836+ * [[nativeTaskSchemas ]]. project_field_ids still drives the read, so the columns are not
837+ * materialized into the scan output.
837838 */
838- private def partitionFieldSourceIds (spec : Any ): Seq [Int ] = {
839+ def partitionFieldSourceIds (spec : Any ): Seq [Int ] = {
839840 import scala .jdk .CollectionConverters ._
840841 // Exclude fields whose source column was dropped (unknown result type): their source is absent
841842 // from the current schema and the serialized spec/values omit them too (see
@@ -2390,10 +2391,14 @@ object IcebergReflection extends Logging {
23902391 * List of FileScanTask objects from Iceberg planning
23912392 * @param scanSchema
23922393 * The expectedSchema from the SparkScan (for schema evolution / VERSION AS OF)
2393- * @param prunedScanSchema
2394- * scanSchema without its metadata columns, which iceberg-rust resolves by reserved field id
2395- * rather than from the task schema. Holds only the nested fields Spark's schema pruning keeps,
2396- * so a task reading with it skips the others.
2394+ * @param prunedTaskSchema
2395+ * The schema every task reads with when nested schema pruning is on: scanSchema without its
2396+ * metadata columns, which iceberg-rust resolves by reserved field id, plus the partition
2397+ * sources and equality-delete keys the tasks need. None when one of those is nested inside a
2398+ * struct the query pruned away. See [[IcebergReflection.nativeTaskSchemas ]].
2399+ * @param fullTaskSchema
2400+ * The schema every task reads with otherwise: the scan's full table schema plus the
2401+ * equality-delete keys it lacks. None when no name fits every task, which falls back to Spark.
23972402 * @param globalFieldIdMapping
23982403 * Mapping from column names to Iceberg field IDs (built from scanSchema)
23992404 * @param catalogProperties
@@ -2408,7 +2413,8 @@ case class CometIcebergNativeScanMetadata(
24082413 nameMapping : Option [String ],
24092414 @ transient tasks : java.util.List [_],
24102415 scanSchema : Any ,
2411- prunedScanSchema : Any ,
2416+ prunedTaskSchema : Option [Any ],
2417+ fullTaskSchema : Option [Any ],
24122418 tableSchema : Any ,
24132419 globalFieldIdMapping : Map [String , Int ],
24142420 catalogProperties : Map [String , String ],
@@ -2433,14 +2439,18 @@ object CometIcebergNativeScanMetadata extends Logging {
24332439 * The scan's FileScanTasks (already extracted). Passed in rather than re-read via
24342440 * [[IcebergReflection.getTasks ]], which for a staged scan rebuilds a flattened list of every
24352441 * task on each call.
2442+ * @param requiredFieldIds
2443+ * The partition sources and equality-delete keys of those tasks, which every task schema
2444+ * holds
24362445 * @return
24372446 * Some(metadata) if all reflection succeeds, None to trigger fallback
24382447 */
24392448 def extract (
24402449 scan : Any ,
24412450 metadataLocation : String ,
24422451 catalogProperties : Map [String , String ],
2443- tasks : java.util.List [_]): Option [CometIcebergNativeScanMetadata ] = {
2452+ tasks : java.util.List [_],
2453+ requiredFieldIds : Set [Int ]): Option [CometIcebergNativeScanMetadata ] = {
24442454 import org .apache .comet .iceberg .IcebergReflection ._
24452455
24462456 for {
@@ -2459,6 +2469,8 @@ object CometIcebergNativeScanMetadata extends Logging {
24592469 }
24602470
24612471 val globalFieldIdMapping = buildFieldIdMapping(scanSchema)
2472+ val (prunedTaskSchema, fullTaskSchema) =
2473+ nativeTaskSchemas(table, tableSchema, scanSchema, requiredFieldIds)
24622474
24632475 // File format is always PARQUET,
24642476 // validated in CometScanRule.validateIcebergFileScanTasks()
@@ -2469,7 +2481,8 @@ object CometIcebergNativeScanMetadata extends Logging {
24692481 nameMapping = nameMapping,
24702482 tasks = tasks,
24712483 scanSchema = scanSchema,
2472- prunedScanSchema = withoutMetadataColumns(scanSchema),
2484+ prunedTaskSchema = prunedTaskSchema,
2485+ fullTaskSchema = fullTaskSchema,
24732486 tableSchema = tableSchema,
24742487 globalFieldIdMapping = globalFieldIdMapping,
24752488 catalogProperties = catalogProperties,
0 commit comments