Skip to content

Commit 86eaa60

Browse files
committed
Match field types when picking the full task schema
The full task schema was the newest schema whose top-level columns match the scan schema by id and name. For a VERSION AS OF read across a type promotion, that could be a newer schema with the promoted type, which the batch adaptation after iceberg-rust cannot narrow back, so the native scan failed. Pick the newest schema in which every field the scan reads, at any depth, has the same name and type instead. The time travel test reads a promoted int, decimal, and nested field, on partitioned and unpartitioned tables, with pruning on and off.
1 parent 5395e50 commit 86eaa60

2 files changed

Lines changed: 69 additions & 9 deletions

File tree

‎spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala‎

Lines changed: 35 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -728,25 +728,31 @@ object IcebergReflection extends Logging {
728728
*
729729
* Every task reads with the same schema because iceberg-rust builds an equality-delete
730730
* predicate once per delete file, with the column names of the task that loads it first, and
731-
* binds it by name in every other task of the reader. The names come from the full schema, the
732-
* newest schema, current first, that names every column of the scan schema the same way. That
733-
* is the current schema unless a VERSION AS OF snapshot named a column differently. A key that
734-
* schema lacks, dropped before it, takes the newest name that no other column has.
731+
* binds it by name in every other task of the reader. The full schema is the newest schema,
732+
* current first, that has every field the scan reads, at any depth, with the same name and
733+
* type, as the batch adaptation after iceberg-rust cannot narrow a promoted type back. That is
734+
* the current schema unless a VERSION AS OF snapshot named or typed a field differently. A key
735+
* that schema lacks, dropped before it, takes the newest name that no other column has.
735736
*/
736737
def nativeTaskSchemas(
737738
table: Any,
738739
tableSchema: Any,
739740
scanSchema: Any,
740741
requiredFieldIds: Set[Int]): (Option[Any], Option[Any]) = {
741742
val prunedScanSchema = withoutMetadataColumns(scanSchema)
742-
val scanColumns = columnNamesById(prunedScanSchema).toSet
743+
val scanFields = fieldsAtAnyDepth(prunedScanSchema)
743744
val schemas = tableSchema +: getAllSchemas(table).reverse
744-
val full = schemas.iterator
745-
.map(schema => schema -> columnNamesById(schema))
746-
.find { case (_, columns) => scanColumns.subsetOf(columns.toSet) }
745+
val full = schemas.find { schema =>
746+
scanFields.forall { case (id, name, leafType) =>
747+
findFieldObject(schema, id).exists { field =>
748+
fieldName(field) == name && leafType.forall(_ == fieldTypeOf(field))
749+
}
750+
}
751+
}
747752
full match {
748753
case None => (None, None)
749-
case Some((fullSchema, fullColumns)) =>
754+
case Some(fullSchema) =>
755+
val fullColumns = columnNamesById(fullSchema)
750756
val fullNames = fullColumns.values.toSet
751757
val ids = requiredFieldIds.toSeq.sorted
752758
val inFull = ids.flatMap(id => findFieldObject(fullSchema, id).map(id -> _)).toMap
@@ -789,6 +795,24 @@ object IcebergReflection extends Logging {
789795
.toMap
790796
}
791797

798+
/**
799+
* The fields of `schema` at any depth, as id, name, and the type of a field with no children.
800+
*/
801+
private def fieldsAtAnyDepth(schema: Any): Seq[(Int, String, Option[Any])] = {
802+
import scala.jdk.CollectionConverters._
803+
def walk(field: Any): Seq[(Int, String, Option[Any])] = {
804+
val children = childFields(field)
805+
val leafType = if (children.isEmpty) Some(fieldTypeOf(field)) else None
806+
(fieldIdOf(field), fieldName(field), leafType) +: children.flatMap(walk)
807+
}
808+
getMethod(schema.getClass, "columns")
809+
.invoke(schema)
810+
.asInstanceOf[java.util.List[_]]
811+
.asScala
812+
.toSeq
813+
.flatMap(walk)
814+
}
815+
792816
/** `schema` with `fields` appended at its top level, or `schema` itself when there are none. */
793817
private def withColumns(schema: Any, fields: Seq[Any]): Any =
794818
if (fields.isEmpty) {
@@ -1384,6 +1408,8 @@ object IcebergReflection extends Logging {
13841408
private def fieldIdOf(field: Any): Int =
13851409
getMethod(field.getClass, "fieldId").invoke(field).asInstanceOf[Int]
13861410

1411+
private def fieldTypeOf(field: Any): Any = getMethod(field.getClass, "type").invoke(field)
1412+
13871413
/**
13881414
* Converts an Iceberg `Schema` to the Spark `StructType` it reads as, via
13891415
* `SparkSchemaUtil.convert`. Comet serializes the whole table/scan schema to native (not just

‎spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2529,6 +2529,40 @@ class CometIcebergNativeSuite
25292529
checkPrunedNativeScan(
25302530
s"SELECT id, c, s.a FROM $twoSources VERSION AS OF $twoSourcesSnapshot ORDER BY id")
25312531

2532+
// VERSION AS OF reads across type promotions. The batch adaptation after iceberg-rust
2533+
// cannot narrow a promoted type back, so the full schema must have the snapshot's types,
2534+
// not just be the newest schema that still has the dropped `c`. Each query reads one
2535+
// promoted column, so another promotion cannot steer the choice. The pruned schema cannot
2536+
// hold the nested partition source `s.k`, so the partitioned table reads the full schema
2537+
// with pruning on too.
2538+
Seq("promoted" -> "", "promoted_partitioned" -> "PARTITIONED BY (s.k)").foreach {
2539+
case (name, partitioning) =>
2540+
val promoted = s"test_cat.db.nested_pruning_$name"
2541+
spark.sql(s"""
2542+
CREATE TABLE $promoted (
2543+
id INT, x INT, d DECIMAL(9, 2), c INT, s STRUCT<k: STRING, a: INT>)
2544+
USING iceberg $partitioning
2545+
""")
2546+
spark.sql(s"""
2547+
INSERT INTO $promoted VALUES
2548+
(1, 10, 1.25, 100, named_struct('k', 'east', 'a', 1)),
2549+
(2, NULL, NULL, NULL, named_struct('k', 'west', 'a', CAST(NULL AS INT)))
2550+
""")
2551+
val snapshot = latestSnapshotId(promoted)
2552+
Seq(
2553+
"ALTER COLUMN x TYPE BIGINT",
2554+
"ALTER COLUMN d TYPE DECIMAL(18, 2)",
2555+
"ALTER COLUMN s.a TYPE BIGINT",
2556+
"DROP COLUMN c").foreach(change => spark.sql(s"ALTER TABLE $promoted $change"))
2557+
Seq("x", "d", "s.a").foreach { column =>
2558+
val query =
2559+
s"SELECT id, $column, c FROM $promoted VERSION AS OF $snapshot ORDER BY id"
2560+
checkIcebergNativeScan(query)
2561+
checkFullSchemaNativeScan(query)
2562+
}
2563+
spark.sql(s"DROP TABLE $promoted")
2564+
}
2565+
25322566
Seq(mor, partitioned, twoSources).foreach(t => spark.sql(s"DROP TABLE $t"))
25332567
}
25342568
}

0 commit comments

Comments
 (0)