@@ -2351,7 +2351,8 @@ class CometIcebergNativeSuite
23512351 // A pruned task schema still needs the columns iceberg-rust uses beyond the projection: the
23522352 // partition source and the equality-delete key when the query projects neither. Tasks with
23532353 // deletes read with the pruned schema too, and the tasks of every partition share the one schema
2354- // that appending the partition source builds.
2354+ // that appending the partition source builds. VERSION AS OF reads of columns dropped, renamed,
2355+ // or swapped after the snapshot use the snapshot's names.
23552356 test(" nested schema pruning with deletes, partitions, and time travel" ) {
23562357 assume(icebergAvailable, " Iceberg not available in classpath" )
23572358
@@ -2404,37 +2405,58 @@ class CometIcebergNativeSuite
24042405
24052406 val mor = " test_cat.db.nested_pruning_mor"
24062407 spark.sql(s """
2407- CREATE TABLE $mor (id INT, c STRING, s STRUCT<a: INT, pad: STRING>)
2408+ CREATE TABLE $mor (
2409+ id INT, c STRING, r STRING, x INT, y INT, s STRUCT<a: INT, pad: STRING>)
24082410 USING iceberg $morProperties
24092411 """ )
2410- spark.sql(s " INSERT INTO $mor SELECT id, p, s FROM ( $rows) " )
2412+ spark.sql(s """
2413+ INSERT INTO $mor
2414+ SELECT id, p, concat('r', CAST(id AS STRING)), id, -id, s FROM ( $rows)
2415+ """ )
24112416 val snapshotBeforeDeletes = latestSnapshotId(mor)
24122417 spark.sql(s " DELETE FROM $mor WHERE id % 10 = 0 " )
24132418 val snapshotWithDeletes = latestSnapshotId(mor)
24142419 commitEqualityDelete(" test_cat" , " db" , " nested_pruning_mor" , " id" , 7 , warehouseDir)
2415- spark.sql(s " ALTER TABLE $mor DROP COLUMN c " )
2420+ // Drop `c`, rename `r`, and swap the names of `x` and `y`.
2421+ Seq (
2422+ " DROP COLUMN c" ,
2423+ " RENAME COLUMN r TO r2" ,
2424+ " RENAME COLUMN x TO tmp" ,
2425+ " RENAME COLUMN y TO x" ,
2426+ " RENAME COLUMN tmp TO y" ).foreach(change => spark.sql(s " ALTER TABLE $mor $change" ))
24162427 checkPrunedNativeScan(s " SELECT id, s.a FROM $mor ORDER BY id " )
24172428 // The equality-delete key `id` is not projected. Iceberg gives the delete only to the data
24182429 // file whose `id` range holds 7, so only that file's task appends `id`.
24192430 checkPrunedNativeScan(s " SELECT s.a FROM $mor ORDER BY s.a " )
24202431 checkPrunedNativeScan(
24212432 s " SELECT id, s.a FROM $mor VERSION AS OF $snapshotBeforeDeletes ORDER BY id " )
2422- // `c` was dropped after this snapshot, so the current table schema lacks it, and these
2423- // tasks with deletes must read with the scan schema.
2433+ // The current table schema lacks `c` and names `r`, `x`, and `y` differently, so these
2434+ // tasks with deletes must read with the scan schema, which keeps the snapshot's names .
24242435 checkPrunedNativeScan(
2425- s " SELECT id, c, s.a FROM $mor VERSION AS OF $snapshotWithDeletes ORDER BY id " )
2436+ s " SELECT id, c, r, x, y, s.a FROM $mor VERSION AS OF $snapshotWithDeletes ORDER BY id " )
24262437
24272438 val partitioned = " test_cat.db.nested_pruning_partitioned"
24282439 spark.sql(s """
2429- CREATE TABLE $partitioned (id INT, p STRING, s STRUCT<a: INT, pad: STRING>)
2440+ CREATE TABLE $partitioned (id INT, p STRING, c STRING, s STRUCT<a: INT, pad: STRING>)
24302441 USING iceberg PARTITIONED BY (p) $morProperties
24312442 """ )
2432- spark.sql(s " INSERT INTO $partitioned $rows" )
2443+ spark.sql(s """
2444+ INSERT INTO $partitioned
2445+ SELECT id, p, concat('c', CAST(id AS STRING)), s FROM ( $rows)
2446+ """ )
24332447 spark.sql(s " DELETE FROM $partitioned WHERE id % 10 = 0 " )
2448+ val partitionedSnapshot = latestSnapshotId(partitioned)
24342449 // The partition source `p` is not projected. Every task appends it, and they share the
24352450 // memoized result.
24362451 checkPrunedNativeScan(s " SELECT id, s.a FROM $partitioned ORDER BY id " , oneSchema = true )
24372452 checkPrunedNativeScan(s " SELECT p, count(s.a) FROM $partitioned GROUP BY p ORDER BY p " )
2453+ // Once `p` takes the dropped `c`'s name, the partition source's current name clashes with
2454+ // the snapshot's `c`, so it is appended under its older name.
2455+ spark.sql(s " ALTER TABLE $partitioned DROP COLUMN c " )
2456+ spark.sql(s " ALTER TABLE $partitioned RENAME COLUMN p TO c " )
2457+ checkPrunedNativeScan(
2458+ s " SELECT id, c, s.a FROM $partitioned VERSION AS OF $partitionedSnapshot ORDER BY id " ,
2459+ oneSchema = true )
24382460
24392461 Seq (mor, partitioned).foreach(t => spark.sql(s " DROP TABLE $t" ))
24402462 }
0 commit comments