Skip to content

Commit ab20cde

Browse files
committed
test: expect a native scan for case-only nested renames now that iceberg-rust maps nested fields by id
iceberg-rust b154603a relabels a file's nested fields by field id, so these reads no longer fall back. A new test covers a struct under A that shares no fields with the file beside a: it falls back only when A resolves to its own field id.
1 parent 90fa7d0 commit ab20cde

1 file changed

Lines changed: 35 additions & 4 deletions

File tree

‎spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala‎

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1650,7 +1650,7 @@ class CometIcebergNativeSuite
16501650
s"SELECT id, s FROM $t ORDER BY id",
16511651
s"SELECT id, s.X FROM $t ORDER BY id",
16521652
s"SELECT id FROM $t WHERE s.X = 20").foreach { query =>
1653-
checkNestedFieldEvolutionFallback(query, "s.X (renamed from x)")
1653+
checkIcebergNativeScan(query)
16541654
}
16551655
spark.sql(s"DROP TABLE $t")
16561656
}
@@ -1676,19 +1676,50 @@ class CometIcebergNativeSuite
16761676
s"INSERT INTO $t VALUES " +
16771677
"(named_struct('x', 5, 'z', 6), named_struct('x', 30, 'z', 300))")
16781678
}
1679-
// A case-insensitive lookup of `A` finds `a` first and would skip the check on `A.z`.
1679+
// iceberg-rust matches the nested fields of `A` by field id, so `A.z` reads natively.
1680+
withCaseCatalog(warehouseDir, caseSensitive = false) {
1681+
val query = s"SELECT * FROM $t"
1682+
if (isSpark41Plus) {
1683+
val e = intercept[org.apache.spark.sql.AnalysisException](spark.sql(query).collect())
1684+
assert(e.getMessage.contains("COLUMN_ALREADY_EXISTS"), e.getMessage)
1685+
} else {
1686+
checkIcebergNativeScan(query)
1687+
}
1688+
}
1689+
withCaseCatalog(warehouseDir, caseSensitive = true) {
1690+
checkIcebergNativeScan(s"SELECT a FROM $t")
1691+
checkIcebergNativeScan(s"SELECT A FROM $t")
1692+
spark.sql(s"DROP TABLE $t")
1693+
}
1694+
}
1695+
}
1696+
1697+
test("issue 5540: struct under A with no fields in common beside a") {
1698+
assume(icebergAvailable, "Iceberg not available in classpath")
1699+
withTempIcebergDir { warehouseDir =>
1700+
val t = "case_cat.db.case_pair_disjoint"
1701+
withCaseCatalog(warehouseDir, caseSensitive = true) {
1702+
// `a` has the shape `A` ends up with, since case-insensitive schema pruning requests `A`
1703+
// with the type of `a`. The file written first has none of the current fields of `A`.
1704+
spark.sql(s"CREATE TABLE $t (a STRUCT<z: INT>, A STRUCT<y: INT>) USING iceberg")
1705+
spark.sql(s"INSERT INTO $t VALUES (named_struct('z', 1), named_struct('y', 10))")
1706+
spark.sql(s"ALTER TABLE $t ADD COLUMN A.z INT")
1707+
spark.sql(s"ALTER TABLE $t DROP COLUMN A.y")
1708+
}
1709+
// The field id of `A`, not of `a`, has to reach the check for structs with no fields in
1710+
// common.
16801711
withCaseCatalog(warehouseDir, caseSensitive = false) {
16811712
val query = s"SELECT * FROM $t"
16821713
if (isSpark41Plus) {
16831714
val e = intercept[org.apache.spark.sql.AnalysisException](spark.sql(query).collect())
16841715
assert(e.getMessage.contains("COLUMN_ALREADY_EXISTS"), e.getMessage)
16851716
} else {
1686-
checkNestedFieldEvolutionFallback(query, "A.z (renamed from y)")
1717+
checkNestedFieldEvolutionFallback(query, "A")
16871718
}
16881719
}
16891720
withCaseCatalog(warehouseDir, caseSensitive = true) {
16901721
checkIcebergNativeScan(s"SELECT a FROM $t")
1691-
checkNestedFieldEvolutionFallback(s"SELECT A FROM $t", "A.z (renamed from y)")
1722+
checkNestedFieldEvolutionFallback(s"SELECT A FROM $t", "A")
16921723
spark.sql(s"DROP TABLE $t")
16931724
}
16941725
}

0 commit comments

Comments
 (0)