Skip to content

Commit 7497946

Browse files
committed
fix: look up projected Iceberg field ids by exact name in CometScanRule
With spark.sql.caseSensitive=false the rule matched projected columns to field ids through the session resolver, so in a table with columns a and A both resolved to a's id. The nested-field check then skipped A, and on Spark 3.5 and 4.0 a field renamed under A read back as NULL from files written before the rename. Look ids up by exact name, as the serde does.
1 parent c4d7b48 commit 7497946

2 files changed

Lines changed: 41 additions & 7 deletions

File tree

‎spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -772,12 +772,8 @@ case class CometScanRule(session: SparkSession)
772772
// Projected roots are matched by field ID so historical snapshots still identify renamed
773773
// columns. Both schema checks below use them.
774774
val projectedDataColumns = scanExec.output.filterNot(_.isMetadataCol)
775-
val resolver = session.sessionState.conf.resolver
776-
val projectedFieldIds = projectedDataColumns.map { attr =>
777-
metadata.globalFieldIdMapping.collectFirst {
778-
case (fieldName, fieldId) if resolver(fieldName, attr.name) => fieldId
779-
}
780-
}
775+
val projectedFieldIds =
776+
projectedDataColumns.map(attr => metadata.globalFieldIdMapping.get(attr.name))
781777
val resolvedProjectedFieldIds = projectedFieldIds.flatten.toSet
782778
val hasUnresolvedProjectedFieldIds = projectedFieldIds.exists(_.isEmpty)
783779

‎spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1518,7 +1518,8 @@ class CometIcebergNativeSuite
15181518
"PARTITIONED BY (Region) TBLPROPERTIES ('format-version' = '2')")
15191519
spark.sql(s"INSERT INTO $t VALUES (1, 'a', 10), (2, 'a', 20), (3, 'b', 30)")
15201520
}
1521-
// Metadata columns resolve by reserved field id, so the exact name check must skip them.
1521+
// Metadata columns resolve by reserved field id and keep Iceberg's reserved names, so the
1522+
// native scan's exact name check finds them too.
15221523
Seq(true, false).foreach { caseSensitive =>
15231524
withCaseCatalog(warehouseDir, caseSensitive) {
15241525
failures ++= failingNativeScanChecks(Seq(
@@ -1561,6 +1562,43 @@ class CometIcebergNativeSuite
15611562
}
15621563
}
15631564

1565+
test("issue 5540: nested field renamed under A beside a") {
1566+
assume(icebergAvailable, "Iceberg not available in classpath")
1567+
withTempIcebergDir { warehouseDir =>
1568+
val t = "case_cat.db.case_pair_nested"
1569+
withCaseCatalog(warehouseDir, caseSensitive = true) {
1570+
// After the rename `a` and `A` share the shape <x, z>, since case-insensitive schema
1571+
// pruning requests `A` with the type of `a`.
1572+
spark.sql(
1573+
s"CREATE TABLE $t (a STRUCT<x: INT, z: INT>, A STRUCT<x: INT, y: INT>) USING iceberg")
1574+
spark.sql(
1575+
s"INSERT INTO $t VALUES " +
1576+
"(named_struct('x', 1, 'z', 2), named_struct('x', 10, 'y', 100)), " +
1577+
"(named_struct('x', 3, 'z', 4), " +
1578+
"named_struct('x', CAST(NULL AS INT), 'y', CAST(NULL AS INT)))")
1579+
spark.sql(s"ALTER TABLE $t RENAME COLUMN A.y TO z")
1580+
spark.sql(
1581+
s"INSERT INTO $t VALUES " +
1582+
"(named_struct('x', 5, 'z', 6), named_struct('x', 30, 'z', 300))")
1583+
}
1584+
// A case-insensitive lookup of `A` finds `a` first and would skip the check on `A.z`.
1585+
withCaseCatalog(warehouseDir, caseSensitive = false) {
1586+
val query = s"SELECT * FROM $t"
1587+
if (isSpark41Plus) {
1588+
val e = intercept[org.apache.spark.sql.AnalysisException](spark.sql(query).collect())
1589+
assert(e.getMessage.contains("COLUMN_ALREADY_EXISTS"), e.getMessage)
1590+
} else {
1591+
checkNestedFieldEvolutionFallback(query, "A.z (renamed from y)")
1592+
}
1593+
}
1594+
withCaseCatalog(warehouseDir, caseSensitive = true) {
1595+
checkIcebergNativeScan(s"SELECT a FROM $t")
1596+
checkNestedFieldEvolutionFallback(s"SELECT A FROM $t", "A.z (renamed from y)")
1597+
spark.sql(s"DROP TABLE $t")
1598+
}
1599+
}
1600+
}
1601+
15641602
test("MOR table with multiple delete operations - mixed delete types") {
15651603
assume(icebergAvailable, "Iceberg not available in classpath")
15661604

0 commit comments

Comments
 (0)