Skip to content

Commit 486be2c

Browse files
authored
fix: fall back from the native Iceberg scan when a nested field was added or renamed (#6543)
iceberg-rust does not match a data file's nested fields to the table's by field id (apache/iceberg-rust#2617). A file written before a nested field was added fails the native scan with "Incorrect number of arrays for StructArray fields", and a nested field renamed since the file was written can read back as NULL. CometScanRule now declines the native scan when a projected column has a nested field that some schema in the table's history lacks or names differently. Closes #6504.
1 parent ca1c143 commit 486be2c

4 files changed

Lines changed: 400 additions & 15 deletions

File tree

‎docs/source/user-guide/latest/iceberg.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,10 @@ The following scenarios will fall back to the JVM Iceberg reader:
192192
- Scans with residual filters using `truncate`, `bucket`, `year`, `month`, `day`, or `hour`
193193
transform functions (partition pruning still works, but row-level filtering of these
194194
transforms falls back)
195+
- Scans that read a struct, array, or map column with a nested field that schema evolution added
196+
or renamed. The native reader cannot yet match such a field to data files written before the
197+
change. The check uses the table's schema history, so the fallback stays after those files are
198+
rewritten
195199

196200
Writes are not covered by this list. By default Iceberg writes use Spark's own writer; see
197201
[Iceberg Writes](iceberg-writes.md) for the experimental native writer and when it applies.

‎spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1149,6 +1149,74 @@ object IcebergReflection extends Logging {
11491149
here ++ nested
11501150
}
11511151

1152+
/**
1153+
* Returns the nested fields of the `fieldIds` columns of `schema` that some schema in the
1154+
* table's history (`table.schemas()`) lacks or names differently. Each is a dotted path with
1155+
* the change, for example `items.element.z (renamed from a)`. A data file written under that
1156+
* older schema lacks the field, or has it under the old name. Fields are matched by id at every
1157+
* level (struct fields, list elements, map keys and values), so a reorder or a type promotion
1158+
* is not a change, and an older schema that lacks the column itself is skipped. Throws on
1159+
* reflection failure so the caller can fall back.
1160+
*/
1161+
def nestedFieldsAddedOrRenamed(table: Any, schema: Any, fieldIds: Set[Int]): Seq[String] = {
1162+
import scala.jdk.CollectionConverters._
1163+
val findField = getMethod(schema.getClass, "findField", classOf[Int])
1164+
def fieldById(s: Any, id: Int): Option[Any] =
1165+
Option(findField.invoke(s, id.asInstanceOf[AnyRef]))
1166+
lazy val history = getMethod(table.getClass, "schemas")
1167+
.invoke(table)
1168+
.asInstanceOf[java.util.Map[_, _]]
1169+
.values()
1170+
.asScala
1171+
.toSeq
1172+
fieldIds.toSeq.flatMap { id =>
1173+
fieldById(schema, id).filter(childFields(_).nonEmpty).toSeq.flatMap { field =>
1174+
history.flatMap(fieldById(_, id)).flatMap { older =>
1175+
childFieldsChangedFrom(field, older, fieldName(field))
1176+
}
1177+
}
1178+
}.distinct
1179+
}
1180+
1181+
/** The nested fields under `field`, at any depth, that `older` lacks or names differently. */
1182+
private def childFieldsChangedFrom(field: Any, older: Any, path: String): Seq[String] = {
1183+
val olderChildren = childFields(older).map(child => fieldIdOf(child) -> child).toMap
1184+
childFields(field).flatMap { child =>
1185+
val name = fieldName(child)
1186+
val childPath = s"$path.$name"
1187+
olderChildren.get(fieldIdOf(child)) match {
1188+
case Some(olderChild) =>
1189+
val olderName = fieldName(olderChild)
1190+
val renamed =
1191+
if (olderName != name) Seq(s"$childPath (renamed from $olderName)") else Nil
1192+
renamed ++ childFieldsChangedFrom(child, olderChild, childPath)
1193+
case None => Seq(s"$childPath (added)")
1194+
}
1195+
}
1196+
}
1197+
1198+
/** A struct's fields, a list's element, or a map's key and value. Empty for other types. */
1199+
private def childFields(field: Any): Seq[Any] = {
1200+
import scala.jdk.CollectionConverters._
1201+
val fieldType = getMethod(field.getClass, "type").invoke(field)
1202+
if (getMethod(fieldType.getClass, "isNestedType").invoke(fieldType).asInstanceOf[Boolean]) {
1203+
val nestedType = getMethod(fieldType.getClass, "asNestedType").invoke(fieldType)
1204+
getMethod(nestedType.getClass, "fields")
1205+
.invoke(nestedType)
1206+
.asInstanceOf[java.util.List[_]]
1207+
.asScala
1208+
.toSeq
1209+
} else {
1210+
Nil
1211+
}
1212+
}
1213+
1214+
private def fieldName(field: Any): String =
1215+
getMethod(field.getClass, "name").invoke(field).asInstanceOf[String]
1216+
1217+
private def fieldIdOf(field: Any): Int =
1218+
getMethod(field.getClass, "fieldId").invoke(field).asInstanceOf[Int]
1219+
11521220
/**
11531221
* Converts an Iceberg `Schema` to the Spark `StructType` it reads as, via
11541222
* `SparkSchemaUtil.convert`. Comet serializes the whole table/scan schema to native (not just

‎spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala‎

Lines changed: 57 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -727,16 +727,27 @@ case class CometScanRule(session: SparkSession)
727727
}
728728
}
729729

730+
// Projected roots are matched by field ID so historical snapshots still identify renamed
731+
// columns. Both schema checks below use them.
732+
val projectedDataColumns = scanExec.output.filterNot(_.isMetadataCol)
733+
val resolver = session.sessionState.conf.resolver
734+
val projectedFieldIds = projectedDataColumns.map { attr =>
735+
metadata.globalFieldIdMapping.collectFirst {
736+
case (fieldName, fieldId) if resolver(fieldName, attr.name) => fieldId
737+
}
738+
}
739+
val resolvedProjectedFieldIds = projectedFieldIds.flatten.toSet
740+
val hasUnresolvedProjectedFieldIds = projectedFieldIds.exists(_.isEmpty)
741+
730742
// The whole Iceberg table schema is serialized to native, but iceberg-rust can represent
731-
// Variant in that schema as long as no projected field contains one. Match projected
732-
// roots by field ID so historical snapshots still identify renamed columns, check them
733-
// strictly, and allow Variant only under entirely unprojected roots. Other unsupported
734-
// types still fail closed everywhere. An empty data projection is also strict because
735-
// iceberg-rust currently interprets an empty field-id list as a request for every column.
743+
// Variant in that schema as long as no projected field contains one. Check projected
744+
// roots strictly, and allow Variant only under entirely unprojected roots. Other
745+
// unsupported types still fail closed everywhere. An empty data projection is also strict
746+
// because iceberg-rust currently interprets an empty field-id list as a request for every
747+
// column.
736748
val schemaTypesSupported =
737749
try {
738750
val fullSchema = IcebergReflection.toSparkSchema(metadata.tableSchema)
739-
val projectedDataColumns = scanExec.output.filterNot(_.isMetadataCol)
740751
// DataTypeSupport recursively dispatches back to this override for struct fields,
741752
// array elements, and map entries, so Variant is allowed at any nesting depth only
742753
// when its entire top-level Iceberg field is unprojected.
@@ -747,15 +758,7 @@ case class CometScanRule(session: SparkSession)
747758
reasons: ListBuffer[String]): Boolean =
748759
isVariantType(dt) || super.isTypeSupported(dt, name, reasons)
749760
}
750-
val resolver = session.sessionState.conf.resolver
751761
val tableFieldIds = IcebergReflection.buildFieldIdMapping(metadata.tableSchema)
752-
val projectedFieldIds = projectedDataColumns.map { attr =>
753-
metadata.globalFieldIdMapping.collectFirst {
754-
case (fieldName, fieldId) if resolver(fieldName, attr.name) => fieldId
755-
}
756-
}
757-
val resolvedProjectedFieldIds = projectedFieldIds.flatten.toSet
758-
val hasUnresolvedProjectedFieldIds = projectedFieldIds.exists(_.isEmpty)
759762

760763
fullSchema.fields.forall { field =>
761764
val isProjected = projectedDataColumns.isEmpty ||
@@ -771,6 +774,44 @@ case class CometScanRule(session: SparkSession)
771774
false
772775
}
773776

777+
// Neither iceberg-rust nor the batch adaptation after it matches a data file's nested
778+
// fields to the table's by field id (apache/iceberg-rust#2617). A file written before a
779+
// nested field was added (to a struct, or to a struct inside a list or map) fails the
780+
// native scan with "Incorrect number of arrays for StructArray fields", and a nested field
781+
// renamed since the file was written can read back as NULL. The native read asks for a
782+
// projected column's full nested type even when Spark prunes it, taken from the current
783+
// table schema, or from the scan schema when VERSION AS OF reads a dropped column (see
784+
// CometIcebergNativeScan). A FileScanTask does not record which schema wrote its file, so
785+
// fall back when any schema in the table's history lacks a nested field of a projected
786+
// column or names it differently.
787+
val nestedFieldsSupported =
788+
try {
789+
val schemas = Seq(metadata.tableSchema, metadata.scanSchema)
790+
val fieldIds =
791+
if (hasUnresolvedProjectedFieldIds) {
792+
schemas.flatMap(IcebergReflection.buildFieldIdMapping(_).values).toSet
793+
} else {
794+
resolvedProjectedFieldIds
795+
}
796+
val changed = schemas
797+
.flatMap { schema =>
798+
IcebergReflection.nestedFieldsAddedOrRenamed(metadata.table, schema, fieldIds)
799+
}
800+
.distinct
801+
.sorted
802+
if (changed.nonEmpty) {
803+
fallbackReasons += "Nested fields added or renamed by Iceberg schema evolution are " +
804+
"not yet supported by Comet's native reader, which cannot match them to data " +
805+
s"files written before the change: ${changed.mkString(", ")}"
806+
}
807+
changed.isEmpty
808+
} catch {
809+
case e: Exception =>
810+
fallbackReasons += "Iceberg reflection failure: could not compare nested fields " +
811+
s"with the table's schema history: ${e.getMessage}"
812+
false
813+
}
814+
774815
// The native Parquet reader decrypts 128-bit and 256-bit AES-GCM data keys (16- or
775816
// 32-byte). 192-bit is unsupported because the underlying crypto has no AES-192-GCM. Fall
776817
// back for anything else. None means the table is unencrypted. Reflection failure also
@@ -1006,7 +1047,8 @@ case class CometScanRule(session: SparkSession)
10061047
defaultValuesSupported && schemaTypesSupported && encryptionKeyLengthSupported &&
10071048
taskValidation.allParquet && allSupportedFilesystems && allLocationsOpenable &&
10081049
metadataSchemeSupported && partitionTypesSupported && unifiedPartitionTypeSupported &&
1009-
transformFunctionsSupported && deleteFileTypesSupported && dppSubqueriesSupported) {
1050+
transformFunctionsSupported && deleteFileTypesSupported && dppSubqueriesSupported &&
1051+
nestedFieldsSupported) {
10101052
CometBatchScanExec(
10111053
scanExec.clone().asInstanceOf[BatchScanExec],
10121054
runtimeFilters = scanExec.runtimeFilters,

0 commit comments

Comments
 (0)