Skip to content

Commit d88698c

Browse files
committed
Resolve appended task-schema columns from one schema
Take every missing partition source and equality-delete key from one schema, the current one or the newest in the history that has them all under names the task schema does not use. Picking a name per field clashed when two partition sources were renamed into each other's names. Tasks that need no extra columns skip the memo lookups, and tasks without deletes skip the equality-key work.
1 parent 335e7e0 commit d88698c

3 files changed

Lines changed: 69 additions & 37 deletions

File tree

‎spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala‎

Lines changed: 25 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -718,10 +718,10 @@ object IcebergReflection extends Logging {
718718
.invoke(schema, fieldId.asInstanceOf[AnyRef]))
719719

720720
/**
721-
* Returns a schema equal to `baseSchema` but guaranteed to contain `requiredFieldIds`. Any id
722-
* not already present is appended, resolved from the table's current schema, or from its schema
723-
* history (`table.schemas()`) when the column has since been dropped or its current name is
724-
* taken.
721+
* Returns a schema equal to `baseSchema` but guaranteed to contain `requiredFieldIds`. The ids
722+
* not already present are appended as they are in one schema: the table's current schema, or
723+
* the newest schema in its history (`table.schemas()`) that has them all under names
724+
* `baseSchema` does not use.
725725
*
726726
* Two callers need this: an equality delete may be keyed on a column since dropped from the
727727
* current schema (mirroring Iceberg-Java's `DeleteFilter.fileProjection`), and a partition
@@ -747,23 +747,27 @@ object IcebergReflection extends Logging {
747747
getMethod(baseSchema.getClass, "columns")
748748
.invoke(baseSchema)
749749
.asInstanceOf[java.util.List[_]]
750-
val newColumns = new java.util.ArrayList[Any](existing)
751-
// A field is appended under a name it has had that the task schema does not use yet. The
752-
// current schema comes first, to keep a live column's current name and type, and then
753-
// table.schemas(), oldest first. A VERSION AS OF scan schema carries the snapshot's names,
754-
// so a current name can clash with a column the snapshot still has.
755-
val schemas = getMethod(table.getClass, "schema").invoke(table) +: getAllSchemas(table)
756-
val names = scala.collection.mutable.Set(existing.asScala.map(fieldName).toSeq: _*)
757-
missingIds.foreach { id =>
758-
val (field, name) = schemas.iterator
759-
.flatMap(findFieldObject(_, id))
760-
.map(f => (f, fieldName(f)))
761-
.find { case (_, n) => !names.contains(n) }
762-
.getOrElse(throw new IllegalStateException(
763-
s"Cannot resolve field id $id in table schema history under an unused name"))
764-
names += name
765-
newColumns.add(field)
766-
}
750+
val existingNames = existing.asScala.map(fieldName).toSet
751+
// Taking every field from one schema keeps their names apart, which a name picked per field
752+
// cannot promise once columns have been renamed into each other's names. A VERSION AS OF
753+
// scan schema carries the snapshot's names, so a schema also must not name a field like a
754+
// column `baseSchema` has. The current schema comes first, to keep current names, and then
755+
// the others newest first, so that a promoted column keeps its widest type.
756+
val schemas = getMethod(table.getClass, "schema").invoke(table) +:
757+
getAllSchemas(table).reverse
758+
val resolvedFields = schemas.iterator
759+
.map(schema => missingIds.flatMap(findFieldObject(schema, _)))
760+
.find { fields =>
761+
val names = fields.map(fieldName)
762+
fields.length == missingIds.length && names.distinct.length == names.length &&
763+
!names.exists(existingNames.contains)
764+
}
765+
.getOrElse(throw new IllegalStateException(
766+
s"Cannot resolve field ids ${missingIds.mkString(",")} from one schema in the table " +
767+
"schema history under names the task schema does not use"))
768+
val newColumns = new java.util.ArrayList[Any](existing.size + resolvedFields.length)
769+
newColumns.addAll(existing)
770+
resolvedFields.foreach(newColumns.add)
767771
baseSchema.getClass
768772
.getConstructor(classOf[java.util.List[_]])
769773
.newInstance(newColumns)

‎spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala‎

Lines changed: 26 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1035,22 +1035,28 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit
10351035
// that needs one reads with the full schema instead.
10361036
val canUsePrunedSchemaCache = mutable.HashMap[Seq[Int], Boolean]()
10371037
def canUsePrunedSchema(requiredFieldIds: Seq[Int]): Boolean =
1038-
nestedSchemaPruningEnabled && canUsePrunedSchemaCache.getOrElseUpdate(
1039-
requiredFieldIds,
1040-
requiredFieldIds.forall { id =>
1041-
IcebergReflection.findFieldObject(metadata.prunedScanSchema, id).isDefined ||
1042-
!IcebergReflection.isNestedField(metadata.tableSchema, id)
1043-
})
1038+
nestedSchemaPruningEnabled && (requiredFieldIds.isEmpty ||
1039+
canUsePrunedSchemaCache.getOrElseUpdate(
1040+
requiredFieldIds,
1041+
requiredFieldIds.forall { id =>
1042+
IcebergReflection.findFieldObject(metadata.prunedScanSchema, id).isDefined ||
1043+
!IcebergReflection.isNestedField(metadata.tableSchema, id)
1044+
}))
10441045
// schemaWithRequiredFields builds a new Schema whenever it appends a column. Memoize it by
10451046
// base schema (Iceberg's Schema compares by identity) and ids, so tasks that need the same
1046-
// columns share one schema object, and with it one schema pool entry.
1047+
// columns share one schema object, and with it one schema pool entry. Most tasks need no extra
1048+
// columns, so they skip both lookups.
10471049
val requiredFieldsCache = mutable.HashMap[(AnyRef, Seq[Int]), AnyRef]()
10481050
def withRequiredFields(baseSchema: AnyRef, requiredFieldIds: Seq[Int]): AnyRef =
1049-
requiredFieldsCache.getOrElseUpdate(
1050-
(baseSchema, requiredFieldIds),
1051-
IcebergReflection
1052-
.schemaWithRequiredFields(baseSchema, metadata.table, requiredFieldIds)
1053-
.asInstanceOf[AnyRef])
1051+
if (requiredFieldIds.isEmpty) {
1052+
baseSchema
1053+
} else {
1054+
requiredFieldsCache.getOrElseUpdate(
1055+
(baseSchema, requiredFieldIds),
1056+
IcebergReflection
1057+
.schemaWithRequiredFields(baseSchema, metadata.table, requiredFieldIds)
1058+
.asInstanceOf[AnyRef])
1059+
}
10541060

10551061
val perPartitionBuilders = mutable.ArrayBuffer[OperatorOuterClass.IcebergScan]()
10561062

@@ -1165,9 +1171,14 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit
11651171
// so union the equality-delete field ids into the task schema, resolving any
11661172
// dropped ones from the table's schema history (mirrors Iceberg-Java's
11671173
// DeleteFilter.fileProjection).
1168-
val equalityFieldIds = deletes.asScala.flatMap { df =>
1169-
requiredEqualityFieldIds(deleteFileClass, df).asScala.map(_.intValue())
1170-
}.toSeq
1174+
val equalityFieldIds =
1175+
if (hasDeletes) {
1176+
deletes.asScala.flatMap { df =>
1177+
requiredEqualityFieldIds(deleteFileClass, df).asScala.map(_.intValue())
1178+
}.toSeq
1179+
} else {
1180+
Nil
1181+
}
11711182

11721183
// iceberg-rust validates a FileScanTask by resolving its partition spec against
11731184
// the task schema, so a task carrying a partition spec needs that spec's source

‎spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2458,7 +2458,24 @@ class CometIcebergNativeSuite
24582458
s"SELECT id, c, s.a FROM $partitioned VERSION AS OF $partitionedSnapshot ORDER BY id",
24592459
oneSchema = true)
24602460

2461-
Seq(mor, partitioned).foreach(t => spark.sql(s"DROP TABLE $t"))
2461+
// Two partition sources renamed into each other's names: `q` takes the dropped `c`'s name
2462+
// and `p` takes `q`'s. Names picked one field at a time clash here, so both are appended
2463+
// as one older schema names them.
2464+
val twoSources = "test_cat.db.nested_pruning_two_sources"
2465+
spark.sql(s"""
2466+
CREATE TABLE $twoSources (id INT, p INT, q INT, c INT, s STRUCT<a: INT, pad: STRING>)
2467+
USING iceberg PARTITIONED BY (bucket(4, p), bucket(4, q))
2468+
""")
2469+
spark.sql(s"INSERT INTO $twoSources SELECT id, id % 3, id % 5, id * 10, s FROM ($rows)")
2470+
val twoSourcesSnapshot = latestSnapshotId(twoSources)
2471+
Seq("DROP COLUMN c", "RENAME COLUMN q TO c", "RENAME COLUMN p TO q").foreach { change =>
2472+
spark.sql(s"ALTER TABLE $twoSources $change")
2473+
}
2474+
checkPrunedNativeScan(
2475+
s"SELECT id, c, s.a FROM $twoSources VERSION AS OF $twoSourcesSnapshot ORDER BY id",
2476+
oneSchema = true)
2477+
2478+
Seq(mor, partitioned, twoSources).foreach(t => spark.sql(s"DROP TABLE $t"))
24622479
}
24632480
}
24642481
}

0 commit comments

Comments
 (0)