Skip to content

Commit 8566834

Browse files
committed
Check the full task schema for nested evolution
The full task schema of a VERSION AS OF read can be older than the table's schema and hold a nested field that an earlier data file lacks, which the native scan cannot read. Walk it in the nested-evolution check too, so such a scan falls back to Spark.
1 parent 6213d05 commit 8566834

2 files changed

Lines changed: 29 additions & 6 deletions

File tree

‎spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -781,13 +781,15 @@ case class CometScanRule(session: SparkSession)
781781
// native scan with "Incorrect number of arrays for StructArray fields", and a nested field
782782
// renamed since the file was written can read back as NULL. The native read usually asks
783783
// only for the nested fields Spark's schema pruning keeps, but a scan can still read with
784-
// the full schema (see IcebergReflection.nativeTaskSchemas), so this check walks both
785-
// schemas and falls back even when the changed field is pruned away. A FileScanTask does
786-
// not record which schema wrote its file, so fall back when any schema in the table's
784+
// its full task schema (see IcebergReflection.nativeTaskSchemas), which for VERSION AS OF
785+
// can be an older schema than the table's. So this check walks the table, scan, and full
786+
// task schemas, and falls back even when the changed field is pruned away. A FileScanTask
787+
// does not record which schema wrote its file, so fall back when any schema in the table's
787788
// history lacks a nested field of a projected column or names it differently.
788789
val nestedFieldsSupported =
789790
try {
790-
val schemas = Seq(metadata.tableSchema, metadata.scanSchema)
791+
val schemas =
792+
Seq(metadata.tableSchema, metadata.scanSchema) ++ metadata.fullTaskSchema
791793
val fieldIds =
792794
if (hasUnresolvedProjectedFieldIds) {
793795
schemas.flatMap(IcebergReflection.buildFieldIdMapping(_).values).toSet

‎spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1827,15 +1827,36 @@ class CometIcebergNativeSuite
18271827
checkNestedFieldEvolutionFallback(
18281828
s"SELECT l.id, r.id FROM $table l JOIN $table r ON l.s = r.s ORDER BY l.id",
18291829
"s.b (added)")
1830-
// The old snapshot's files lack s.b too, and the native read uses the current schema.
1830+
// The old snapshot's files lack s.b too, and the scan's full schema is the current one,
1831+
// which has it.
18311832
checkNestedFieldEvolutionFallback(
18321833
s"SELECT id, s FROM $table VERSION AS OF $snapshotBeforeAdd ORDER BY id",
18331834
"s.b (added)")
18341835

18351836
// A read that does not project the evolved column stays native.
18361837
checkIcebergNativeScan(s"SELECT id FROM $table ORDER BY id")
18371838

1838-
spark.sql(s"DROP TABLE $table")
1839+
// A VERSION AS OF read whose full schema is older than the table's: the pruned-away
1840+
// nested partition source `s.k` makes the scan read its full schema, and the dropped `c`
1841+
// makes that the snapshot's, which still has the s.b that the first file predates.
1842+
val historical = "test_cat.db.evo_struct_historical"
1843+
spark.sql(s"""
1844+
CREATE TABLE $historical (id INT, c INT, s STRUCT<k: STRING, a: INT>)
1845+
USING iceberg PARTITIONED BY (s.k)
1846+
""")
1847+
spark.sql(s"INSERT INTO $historical VALUES (1, 100, named_struct('k', 'east', 'a', 10))")
1848+
spark.sql(s"ALTER TABLE $historical ADD COLUMN s.b INT")
1849+
spark.sql(s"""
1850+
INSERT INTO $historical VALUES (2, 200, named_struct('k', 'east', 'a', 20, 'b', 99))
1851+
""")
1852+
val snapshotWithB = latestSnapshotId(historical)
1853+
spark.sql(s"ALTER TABLE $historical DROP COLUMN c")
1854+
spark.sql(s"ALTER TABLE $historical DROP COLUMN s.b")
1855+
checkNestedFieldEvolutionFallback(
1856+
s"SELECT id, c, s.a FROM $historical VERSION AS OF $snapshotWithB ORDER BY id",
1857+
"s.b (added)")
1858+
1859+
Seq(table, historical).foreach(t => spark.sql(s"DROP TABLE $t"))
18391860
}
18401861
}
18411862
}

0 commit comments

Comments
 (0)