@@ -711,16 +711,27 @@ case class CometScanRule(session: SparkSession)
711711 }
712712 }
713713
714+ // Projected roots are matched by field ID so historical snapshots still identify renamed
715+ // columns. Both schema checks below use them.
716+ val projectedDataColumns = scanExec.output.filterNot(_.isMetadataCol)
717+ val resolver = session.sessionState.conf.resolver
718+ val projectedFieldIds = projectedDataColumns.map { attr =>
719+ metadata.globalFieldIdMapping.collectFirst {
720+ case (fieldName, fieldId) if resolver(fieldName, attr.name) => fieldId
721+ }
722+ }
723+ val resolvedProjectedFieldIds = projectedFieldIds.flatten.toSet
724+ val hasUnresolvedProjectedFieldIds = projectedFieldIds.exists(_.isEmpty)
725+
714726 // The whole Iceberg table schema is serialized to native, but iceberg-rust can represent
715- // Variant in that schema as long as no projected field contains one. Match projected
716- // roots by field ID so historical snapshots still identify renamed columns, check them
717- // strictly, and allow Variant only under entirely unprojected roots. Other unsupported
718- // types still fail closed everywhere. An empty data projection is also strict because
719- // iceberg-rust currently interprets an empty field-id list as a request for every column.
727+ // Variant in that schema as long as no projected field contains one. Check projected
728+ // roots strictly, and allow Variant only under entirely unprojected roots. Other
729+ // unsupported types still fail closed everywhere. An empty data projection is also strict
730+ // because iceberg-rust currently interprets an empty field-id list as a request for every
731+ // column.
720732 val schemaTypesSupported =
721733 try {
722734 val fullSchema = IcebergReflection .toSparkSchema(metadata.tableSchema)
723- val projectedDataColumns = scanExec.output.filterNot(_.isMetadataCol)
724735 // DataTypeSupport recursively dispatches back to this override for struct fields,
725736 // array elements, and map entries, so Variant is allowed at any nesting depth only
726737 // when its entire top-level Iceberg field is unprojected.
@@ -731,15 +742,7 @@ case class CometScanRule(session: SparkSession)
731742 reasons : ListBuffer [String ]): Boolean =
732743 isVariantType(dt) || super .isTypeSupported(dt, name, reasons)
733744 }
734- val resolver = session.sessionState.conf.resolver
735745 val tableFieldIds = IcebergReflection .buildFieldIdMapping(metadata.tableSchema)
736- val projectedFieldIds = projectedDataColumns.map { attr =>
737- metadata.globalFieldIdMapping.collectFirst {
738- case (fieldName, fieldId) if resolver(fieldName, attr.name) => fieldId
739- }
740- }
741- val resolvedProjectedFieldIds = projectedFieldIds.flatten.toSet
742- val hasUnresolvedProjectedFieldIds = projectedFieldIds.exists(_.isEmpty)
743746
744747 fullSchema.fields.forall { field =>
745748 val isProjected = projectedDataColumns.isEmpty ||
@@ -755,6 +758,44 @@ case class CometScanRule(session: SparkSession)
755758 false
756759 }
757760
761+ // Neither iceberg-rust nor the batch adaptation after it matches a data file's nested
762+ // fields to the table's by field id (apache/iceberg-rust#2617). A file written before a
763+ // nested field was added (to a struct, or to a struct inside a list or map) fails the
764+ // native scan with "Incorrect number of arrays for StructArray fields", and a nested field
765+ // renamed since the file was written can read back as NULL. The native read asks for a
766+ // projected column's full nested type even when Spark prunes it, taken from the current
767+ // table schema, or from the scan schema when VERSION AS OF reads a dropped column (see
768+ // CometIcebergNativeScan). A FileScanTask does not record which schema wrote its file, so
769+ // fall back when any schema in the table's history lacks a nested field of a projected
770+ // column or names it differently.
771+ val nestedFieldsSupported =
772+ try {
773+ val schemas = Seq (metadata.tableSchema, metadata.scanSchema)
774+ val fieldIds =
775+ if (hasUnresolvedProjectedFieldIds) {
776+ schemas.flatMap(IcebergReflection .buildFieldIdMapping(_).values).toSet
777+ } else {
778+ resolvedProjectedFieldIds
779+ }
780+ val changed = schemas
781+ .flatMap { schema =>
782+ IcebergReflection .nestedFieldsAddedOrRenamed(metadata.table, schema, fieldIds)
783+ }
784+ .distinct
785+ .sorted
786+ if (changed.nonEmpty) {
787+ fallbackReasons += " Nested fields added or renamed by Iceberg schema evolution are " +
788+ " not yet supported by Comet's native reader, which cannot match them to data " +
789+ s " files written before the change: ${changed.mkString(" , " )}"
790+ }
791+ changed.isEmpty
792+ } catch {
793+ case e : Exception =>
794+ fallbackReasons += " Iceberg reflection failure: could not compare nested fields " +
795+ s " with the table's schema history: ${e.getMessage}"
796+ false
797+ }
798+
758799 // The native Parquet reader decrypts 128-bit and 256-bit AES-GCM data keys (16- or
759800 // 32-byte). 192-bit is unsupported because the underlying crypto has no AES-192-GCM. Fall
760801 // back for anything else. None means the table is unencrypted. Reflection failure also
@@ -990,7 +1031,8 @@ case class CometScanRule(session: SparkSession)
9901031 defaultValuesSupported && schemaTypesSupported && encryptionKeyLengthSupported &&
9911032 taskValidation.allParquet && allSupportedFilesystems && allLocationsOpenable &&
9921033 metadataSchemeSupported && partitionTypesSupported && unifiedPartitionTypeSupported &&
993- transformFunctionsSupported && deleteFileTypesSupported && dppSubqueriesSupported) {
1034+ transformFunctionsSupported && deleteFileTypesSupported && dppSubqueriesSupported &&
1035+ nestedFieldsSupported) {
9941036 CometBatchScanExec (
9951037 scanExec.clone().asInstanceOf [BatchScanExec ],
9961038 runtimeFilters = scanExec.runtimeFilters,
0 commit comments