Skip to content

Commit 335e7e0

Browse files
committed
feedback
1 parent 4125e42 commit 335e7e0

2 files changed

Lines changed: 51 additions & 23 deletions

File tree

‎spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala‎

Lines changed: 20 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -719,8 +719,9 @@ object IcebergReflection extends Logging {
719719

720720
/**
721721
* Returns a schema equal to `baseSchema` but guaranteed to contain `requiredFieldIds`. Any id
722-
* not already present is resolved from the table's current schema, or from its schema history
723-
* (`table.schemas()`) for a column since dropped, and appended.
722+
* not already present is appended, resolved from the table's current schema, or from its schema
723+
* history (`table.schemas()`) when the column has since been dropped or its current name is
724+
* taken.
724725
*
725726
* Two callers need this: an equality delete may be keyed on a column since dropped from the
726727
* current schema (mirroring Iceberg-Java's `DeleteFilter.fileProjection`), and a partition
@@ -730,6 +731,7 @@ object IcebergReflection extends Logging {
730731
* CometScanRule is responsible for falling back before we get here.
731732
*/
732733
def schemaWithRequiredFields(baseSchema: Any, table: Any, requiredFieldIds: Seq[Int]): Any = {
734+
import scala.jdk.CollectionConverters._
733735
// `findFieldObject` searches recursively, so a source column already present as a nested field
734736
// (e.g. `s.region`) is not re-appended at the top level, which would create a duplicate field
735737
// id. Empty `requiredFieldIds` (the common non-partitioned, no-delete task) does no lookups.
@@ -741,23 +743,27 @@ object IcebergReflection extends Logging {
741743
logDebug(
742744
s"Native Iceberg scan schema is missing field id(s) ${missingIds.mkString(",")}; " +
743745
"resolving them from table schema history")
744-
// table.schemas() lists the oldest schema first, so a column the table still has is looked
745-
// up in the current schema before it, to keep its current name and type. An older name
746-
// could clash with a column that took it over.
747-
val schemas = getMethod(table.getClass, "schema").invoke(table) +: getAllSchemas(table)
748-
val resolvedFields = missingIds.map { id =>
749-
schemas.iterator
750-
.map(findFieldObject(_, id))
751-
.collectFirst { case Some(field) => field }
752-
.getOrElse(throw new IllegalStateException(
753-
s"Cannot resolve field id $id in table schema history"))
754-
}
755746
val existing =
756747
getMethod(baseSchema.getClass, "columns")
757748
.invoke(baseSchema)
758749
.asInstanceOf[java.util.List[_]]
759750
val newColumns = new java.util.ArrayList[Any](existing)
760-
resolvedFields.foreach(newColumns.add)
751+
// A field is appended under a name it has had that the task schema does not use yet. The
752+
// current schema comes first, to keep a live column's current name and type, and then
753+
// table.schemas(), oldest first. A VERSION AS OF scan schema carries the snapshot's names,
754+
// so a current name can clash with a column the snapshot still has.
755+
val schemas = getMethod(table.getClass, "schema").invoke(table) +: getAllSchemas(table)
756+
val names = scala.collection.mutable.Set(existing.asScala.map(fieldName).toSeq: _*)
757+
missingIds.foreach { id =>
758+
val (field, name) = schemas.iterator
759+
.flatMap(findFieldObject(_, id))
760+
.map(f => (f, fieldName(f)))
761+
.find { case (_, n) => !names.contains(n) }
762+
.getOrElse(throw new IllegalStateException(
763+
s"Cannot resolve field id $id in table schema history under an unused name"))
764+
names += name
765+
newColumns.add(field)
766+
}
761767
baseSchema.getClass
762768
.getConstructor(classOf[java.util.List[_]])
763769
.newInstance(newColumns)

‎spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala‎

Lines changed: 31 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2351,7 +2351,8 @@ class CometIcebergNativeSuite
23512351
// A pruned task schema still needs the columns iceberg-rust uses beyond the projection: the
23522352
// partition source and the equality-delete key when the query projects neither. Tasks with
23532353
// deletes read with the pruned schema too, and the tasks of every partition share the one schema
2354-
// that appending the partition source builds.
2354+
// that appending the partition source builds. VERSION AS OF reads of columns dropped, renamed,
2355+
// or swapped after the snapshot use the snapshot's names.
23552356
test("nested schema pruning with deletes, partitions, and time travel") {
23562357
assume(icebergAvailable, "Iceberg not available in classpath")
23572358

@@ -2404,37 +2405,58 @@ class CometIcebergNativeSuite
24042405

24052406
val mor = "test_cat.db.nested_pruning_mor"
24062407
spark.sql(s"""
2407-
CREATE TABLE $mor (id INT, c STRING, s STRUCT<a: INT, pad: STRING>)
2408+
CREATE TABLE $mor (
2409+
id INT, c STRING, r STRING, x INT, y INT, s STRUCT<a: INT, pad: STRING>)
24082410
USING iceberg $morProperties
24092411
""")
2410-
spark.sql(s"INSERT INTO $mor SELECT id, p, s FROM ($rows)")
2412+
spark.sql(s"""
2413+
INSERT INTO $mor
2414+
SELECT id, p, concat('r', CAST(id AS STRING)), id, -id, s FROM ($rows)
2415+
""")
24112416
val snapshotBeforeDeletes = latestSnapshotId(mor)
24122417
spark.sql(s"DELETE FROM $mor WHERE id % 10 = 0")
24132418
val snapshotWithDeletes = latestSnapshotId(mor)
24142419
commitEqualityDelete("test_cat", "db", "nested_pruning_mor", "id", 7, warehouseDir)
2415-
spark.sql(s"ALTER TABLE $mor DROP COLUMN c")
2420+
// Drop `c`, rename `r`, and swap the names of `x` and `y`.
2421+
Seq(
2422+
"DROP COLUMN c",
2423+
"RENAME COLUMN r TO r2",
2424+
"RENAME COLUMN x TO tmp",
2425+
"RENAME COLUMN y TO x",
2426+
"RENAME COLUMN tmp TO y").foreach(change => spark.sql(s"ALTER TABLE $mor $change"))
24162427
checkPrunedNativeScan(s"SELECT id, s.a FROM $mor ORDER BY id")
24172428
// The equality-delete key `id` is not projected. Iceberg gives the delete only to the data
24182429
// file whose `id` range holds 7, so only that file's task appends `id`.
24192430
checkPrunedNativeScan(s"SELECT s.a FROM $mor ORDER BY s.a")
24202431
checkPrunedNativeScan(
24212432
s"SELECT id, s.a FROM $mor VERSION AS OF $snapshotBeforeDeletes ORDER BY id")
2422-
// `c` was dropped after this snapshot, so the current table schema lacks it, and these
2423-
// tasks with deletes must read with the scan schema.
2433+
// The current table schema lacks `c` and names `r`, `x`, and `y` differently, so these
2434+
// tasks with deletes must read with the scan schema, which keeps the snapshot's names.
24242435
checkPrunedNativeScan(
2425-
s"SELECT id, c, s.a FROM $mor VERSION AS OF $snapshotWithDeletes ORDER BY id")
2436+
s"SELECT id, c, r, x, y, s.a FROM $mor VERSION AS OF $snapshotWithDeletes ORDER BY id")
24262437

24272438
val partitioned = "test_cat.db.nested_pruning_partitioned"
24282439
spark.sql(s"""
2429-
CREATE TABLE $partitioned (id INT, p STRING, s STRUCT<a: INT, pad: STRING>)
2440+
CREATE TABLE $partitioned (id INT, p STRING, c STRING, s STRUCT<a: INT, pad: STRING>)
24302441
USING iceberg PARTITIONED BY (p) $morProperties
24312442
""")
2432-
spark.sql(s"INSERT INTO $partitioned $rows")
2443+
spark.sql(s"""
2444+
INSERT INTO $partitioned
2445+
SELECT id, p, concat('c', CAST(id AS STRING)), s FROM ($rows)
2446+
""")
24332447
spark.sql(s"DELETE FROM $partitioned WHERE id % 10 = 0")
2448+
val partitionedSnapshot = latestSnapshotId(partitioned)
24342449
// The partition source `p` is not projected. Every task appends it, and they share the
24352450
// memoized result.
24362451
checkPrunedNativeScan(s"SELECT id, s.a FROM $partitioned ORDER BY id", oneSchema = true)
24372452
checkPrunedNativeScan(s"SELECT p, count(s.a) FROM $partitioned GROUP BY p ORDER BY p")
2453+
// Once `p` takes the dropped `c`'s name, the partition source's current name clashes with
2454+
// the snapshot's `c`, so it is appended under its older name.
2455+
spark.sql(s"ALTER TABLE $partitioned DROP COLUMN c")
2456+
spark.sql(s"ALTER TABLE $partitioned RENAME COLUMN p TO c")
2457+
checkPrunedNativeScan(
2458+
s"SELECT id, c, s.a FROM $partitioned VERSION AS OF $partitionedSnapshot ORDER BY id",
2459+
oneSchema = true)
24382460

24392461
Seq(mor, partitioned).foreach(t => spark.sql(s"DROP TABLE $t"))
24402462
}

0 commit comments

Comments
 (0)