@@ -727,16 +727,27 @@ case class CometScanRule(session: SparkSession)
727727 }
728728 }
729729
730+ // Projected roots are matched by field ID so historical snapshots still identify renamed
731+ // columns. Both schema checks below use them.
732+ val projectedDataColumns = scanExec.output.filterNot(_.isMetadataCol)
733+ val resolver = session.sessionState.conf.resolver
734+ val projectedFieldIds = projectedDataColumns.map { attr =>
735+ metadata.globalFieldIdMapping.collectFirst {
736+ case (fieldName, fieldId) if resolver(fieldName, attr.name) => fieldId
737+ }
738+ }
739+ val resolvedProjectedFieldIds = projectedFieldIds.flatten.toSet
740+ val hasUnresolvedProjectedFieldIds = projectedFieldIds.exists(_.isEmpty)
741+
730742 // The whole Iceberg table schema is serialized to native, but iceberg-rust can represent
731- // Variant in that schema as long as no projected field contains one. Match projected
732- // roots by field ID so historical snapshots still identify renamed columns, check them
733- // strictly, and allow Variant only under entirely unprojected roots. Other unsupported
734- // types still fail closed everywhere. An empty data projection is also strict because
735- // iceberg-rust currently interprets an empty field-id list as a request for every column.
743+ // Variant in that schema as long as no projected field contains one. Check projected
744+ // roots strictly, and allow Variant only under entirely unprojected roots. Other
745+ // unsupported types still fail closed everywhere. An empty data projection is also strict
746+ // because iceberg-rust currently interprets an empty field-id list as a request for every
747+ // column.
736748 val schemaTypesSupported =
737749 try {
738750 val fullSchema = IcebergReflection .toSparkSchema(metadata.tableSchema)
739- val projectedDataColumns = scanExec.output.filterNot(_.isMetadataCol)
740751 // DataTypeSupport recursively dispatches back to this override for struct fields,
741752 // array elements, and map entries, so Variant is allowed at any nesting depth only
742753 // when its entire top-level Iceberg field is unprojected.
@@ -747,15 +758,7 @@ case class CometScanRule(session: SparkSession)
747758 reasons : ListBuffer [String ]): Boolean =
748759 isVariantType(dt) || super .isTypeSupported(dt, name, reasons)
749760 }
750- val resolver = session.sessionState.conf.resolver
751761 val tableFieldIds = IcebergReflection .buildFieldIdMapping(metadata.tableSchema)
752- val projectedFieldIds = projectedDataColumns.map { attr =>
753- metadata.globalFieldIdMapping.collectFirst {
754- case (fieldName, fieldId) if resolver(fieldName, attr.name) => fieldId
755- }
756- }
757- val resolvedProjectedFieldIds = projectedFieldIds.flatten.toSet
758- val hasUnresolvedProjectedFieldIds = projectedFieldIds.exists(_.isEmpty)
759762
760763 fullSchema.fields.forall { field =>
761764 val isProjected = projectedDataColumns.isEmpty ||
@@ -771,6 +774,44 @@ case class CometScanRule(session: SparkSession)
771774 false
772775 }
773776
777+ // Neither iceberg-rust nor the batch adaptation after it matches a data file's nested
778+ // fields to the table's by field id (apache/iceberg-rust#2617). A file written before a
779+ // nested field was added (to a struct, or to a struct inside a list or map) fails the
780+ // native scan with "Incorrect number of arrays for StructArray fields", and a nested field
781+ // renamed since the file was written can read back as NULL. The native read asks for a
782+ // projected column's full nested type even when Spark prunes it, taken from the current
783+ // table schema, or from the scan schema when VERSION AS OF reads a dropped column (see
784+ // CometIcebergNativeScan). A FileScanTask does not record which schema wrote its file, so
785+ // fall back when any schema in the table's history lacks a nested field of a projected
786+ // column or names it differently.
787+ val nestedFieldsSupported =
788+ try {
789+ val schemas = Seq (metadata.tableSchema, metadata.scanSchema)
790+ val fieldIds =
791+ if (hasUnresolvedProjectedFieldIds) {
792+ schemas.flatMap(IcebergReflection .buildFieldIdMapping(_).values).toSet
793+ } else {
794+ resolvedProjectedFieldIds
795+ }
796+ val changed = schemas
797+ .flatMap { schema =>
798+ IcebergReflection .nestedFieldsAddedOrRenamed(metadata.table, schema, fieldIds)
799+ }
800+ .distinct
801+ .sorted
802+ if (changed.nonEmpty) {
803+ fallbackReasons += " Nested fields added or renamed by Iceberg schema evolution are " +
804+ " not yet supported by Comet's native reader, which cannot match them to data " +
805+ s " files written before the change: ${changed.mkString(" , " )}"
806+ }
807+ changed.isEmpty
808+ } catch {
809+ case e : Exception =>
810+ fallbackReasons += " Iceberg reflection failure: could not compare nested fields " +
811+ s " with the table's schema history: ${e.getMessage}"
812+ false
813+ }
814+
774815 // The native Parquet reader decrypts 128-bit and 256-bit AES-GCM data keys (16- or
775816 // 32-byte). 192-bit is unsupported because the underlying crypto has no AES-192-GCM. Fall
776817 // back for anything else. None means the table is unencrypted. Reflection failure also
@@ -1006,7 +1047,8 @@ case class CometScanRule(session: SparkSession)
10061047 defaultValuesSupported && schemaTypesSupported && encryptionKeyLengthSupported &&
10071048 taskValidation.allParquet && allSupportedFilesystems && allLocationsOpenable &&
10081049 metadataSchemeSupported && partitionTypesSupported && unifiedPartitionTypeSupported &&
1009- transformFunctionsSupported && deleteFileTypesSupported && dppSubqueriesSupported) {
1050+ transformFunctionsSupported && deleteFileTypesSupported && dppSubqueriesSupported &&
1051+ nestedFieldsSupported) {
10101052 CometBatchScanExec (
10111053 scanExec.clone().asInstanceOf [BatchScanExec ],
10121054 runtimeFilters = scanExec.runtimeFilters,
0 commit comments