Skip to content

Commit b8518b8

Browse files
committed
test: move Iceberg case-sensitive read checks into a SQL fixture
Read and filter checks for case-variant columns now live in sql-tests/iceberg/case_sensitive_columns.sql. The suite keeps the equality-delete coverage in one test for both key columns and shares a helper for importing Parquet files into a table.
1 parent 60b4745 commit b8518b8

2 files changed

Lines changed: 132 additions & 107 deletions

File tree

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
-- Licensed to the Apache Software Foundation (ASF) under one
2+
-- or more contributor license agreements. See the NOTICE file
3+
-- distributed with this work for additional information
4+
-- regarding copyright ownership. The ASF licenses this file
5+
-- to you under the Apache License, Version 2.0 (the
6+
-- "License"); you may not use this file except in compliance
7+
-- with the License. You may obtain a copy of the License at
8+
--
9+
-- http://www.apache.org/licenses/LICENSE-2.0
10+
--
11+
-- Unless required by applicable law or agreed to in writing,
12+
-- software distributed under the License is distributed on an
13+
-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
-- KIND, either express or implied. See the License for the
15+
-- specific language governing permissions and limitations
16+
-- under the License.
17+
18+
-- Columns `a` and `A` in one Iceberg table under spark.sql.caseSensitive=true. The native scan
19+
-- has to bind each projected column and each filter to its own field rather than match names
20+
-- case-insensitively (https://github.com/apache/datafusion-comet/issues/5540).
21+
-- CometIcebergNativeSuite covers equality deletes, name mapping, and metadata columns.
22+
23+
-- No Iceberg spark-runtime is published for Spark 4.2 yet, and the 4.0 runtime the build reuses
24+
-- is binary-incompatible with it. See https://github.com/apache/datafusion-comet/issues/4969.
25+
-- MaxSparkVersion: 4.1
26+
27+
-- Config: spark.sql.catalog.test_cat=org.apache.iceberg.spark.SparkCatalog
28+
-- Config: spark.sql.catalog.test_cat.type=hadoop
29+
-- Config: spark.sql.catalog.test_cat.warehouse=/tmp/comet-iceberg-sql-test
30+
-- Config: spark.comet.enabled=true
31+
-- Config: spark.comet.exec.enabled=true
32+
-- Config: spark.comet.scan.icebergNative.enabled=true
33+
-- Config: spark.sql.caseSensitive=true
34+
35+
statement
36+
DROP TABLE IF EXISTS test_cat.db.case_pair
37+
38+
statement
39+
CREATE TABLE test_cat.db.case_pair (a INT, A INT) USING iceberg
40+
41+
-- `a` spans [1, 3] and `A` spans [20, 40], so a filter bound to the other column prunes the
42+
-- whole row group or keeps the wrong row.
43+
statement
44+
INSERT INTO test_cat.db.case_pair VALUES (1, 20), (3, 40)
45+
46+
-- A plain query fails on any operator that is not Comet's, so each one also checks that the
47+
-- Iceberg scan ran natively.
48+
query
49+
SELECT a, A FROM test_cat.db.case_pair ORDER BY a
50+
51+
query
52+
SELECT A, a FROM test_cat.db.case_pair ORDER BY a
53+
54+
query
55+
SELECT A FROM test_cat.db.case_pair ORDER BY A
56+
57+
query
58+
SELECT a FROM test_cat.db.case_pair ORDER BY a
59+
60+
query
61+
SELECT * FROM test_cat.db.case_pair ORDER BY a
62+
63+
query
64+
SELECT a FROM test_cat.db.case_pair WHERE A = 40
65+
66+
query
67+
SELECT A FROM test_cat.db.case_pair WHERE a = 3
68+
69+
query
70+
SELECT A FROM test_cat.db.case_pair WHERE A = 40
71+
72+
query
73+
SELECT a FROM test_cat.db.case_pair WHERE a = 3
74+
75+
query
76+
SELECT a, A FROM test_cat.db.case_pair WHERE A = 40
77+
78+
query
79+
SELECT a, A FROM test_cat.db.case_pair WHERE a = 3
80+
81+
query
82+
SELECT a FROM test_cat.db.case_pair WHERE A > 30
83+
84+
query
85+
SELECT a FROM test_cat.db.case_pair WHERE A IN (20)
86+
87+
query
88+
SELECT A FROM test_cat.db.case_pair WHERE a < 2
89+
90+
statement
91+
DROP TABLE test_cat.db.case_pair

‎spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala‎

Lines changed: 41 additions & 107 deletions
Original file line numberDiff line numberDiff line change
@@ -1300,73 +1300,62 @@ class CometIcebergNativeSuite
13001300
CometConf.COMET_EXEC_ENABLED.key -> "true",
13011301
CometConf.COMET_ICEBERG_NATIVE_ENABLED.key -> "true")(f)
13021302

1303-
test("issue 5540: case-sensitive read of columns a and A") {
1304-
assume(icebergAvailable, "Iceberg not available in classpath")
1305-
withTempIcebergDir { warehouseDir =>
1306-
withCaseCatalog(warehouseDir, caseSensitive = true) {
1307-
val t = "case_cat.db.case_pair"
1308-
spark.sql(s"CREATE TABLE $t (a INT, A INT) USING iceberg")
1309-
spark.sql(s"INSERT INTO $t VALUES (1, 20), (3, 40)")
1310-
val failures = failingNativeScanChecks(
1311-
Seq(
1312-
s"SELECT a, A FROM $t ORDER BY a",
1313-
s"SELECT A, a FROM $t ORDER BY a",
1314-
s"SELECT A FROM $t ORDER BY A",
1315-
s"SELECT a FROM $t ORDER BY a",
1316-
s"SELECT * FROM $t ORDER BY a"))
1317-
spark.sql(s"DROP TABLE $t")
1318-
assert(failures.isEmpty, failures.mkString("\n\n=====\n\n"))
1319-
}
1320-
}
1321-
}
1322-
1323-
test("issue 5540: case-sensitive filters on columns a and A") {
1324-
assume(icebergAvailable, "Iceberg not available in classpath")
1325-
withTempIcebergDir { warehouseDir =>
1326-
withCaseCatalog(warehouseDir, caseSensitive = true) {
1327-
val t = "case_cat.db.case_pair_filter"
1328-
spark.sql(s"CREATE TABLE $t (a INT, A INT) USING iceberg")
1329-
spark.sql(s"INSERT INTO $t VALUES (1, 20), (3, 40)")
1330-
// a spans [1, 3] and A spans [20, 40]: a filter on one column bound to the other would
1331-
// prune the whole row group or keep the wrong row.
1332-
val failures = failingNativeScanChecks(
1333-
Seq(
1334-
s"SELECT a FROM $t WHERE A = 40",
1335-
s"SELECT A FROM $t WHERE a = 3",
1336-
s"SELECT A FROM $t WHERE A = 40",
1337-
s"SELECT a FROM $t WHERE a = 3",
1338-
s"SELECT a, A FROM $t WHERE A = 40",
1339-
s"SELECT a, A FROM $t WHERE a = 3",
1340-
s"SELECT a FROM $t WHERE A > 30",
1341-
s"SELECT a FROM $t WHERE A IN (20)",
1342-
s"SELECT A FROM $t WHERE a < 2"))
1343-
spark.sql(s"DROP TABLE $t")
1344-
assert(failures.isEmpty, failures.mkString("\n\n=====\n\n"))
1345-
}
1346-
}
1347-
}
1348-
1349-
test("issue 5540: case-sensitive equality deletes keyed on a and on A") {
1303+
/** Imports the files of session table `sourceTable` into `table` with SparkTableUtil. */
1304+
private def importParquetInto(
1305+
table: org.apache.iceberg.Table,
1306+
sourceTable: String,
1307+
stagingDir: String): Unit =
1308+
Class
1309+
.forName("org.apache.iceberg.spark.SparkTableUtil")
1310+
.getMethod(
1311+
"importSparkTable",
1312+
classOf[org.apache.spark.sql.SparkSession],
1313+
classOf[org.apache.spark.sql.catalyst.TableIdentifier],
1314+
classOf[org.apache.iceberg.Table],
1315+
classOf[String])
1316+
.invoke(
1317+
null,
1318+
spark,
1319+
new org.apache.spark.sql.catalyst.TableIdentifier(sourceTable),
1320+
table,
1321+
stagingDir)
1322+
1323+
test("issue 5540: equality deletes keyed on columns a and A") {
13501324
assume(icebergAvailable, "Iceberg not available in classpath")
13511325
withTempIcebergDir { warehouseDir =>
1326+
val t = "case_cat.db.case_pair_eq"
1327+
var failures = Seq.empty[String]
13521328
withCaseCatalog(warehouseDir, caseSensitive = true) {
1353-
val t = "case_cat.db.case_pair_eq"
13541329
spark.sql(
13551330
s"CREATE TABLE $t (a INT, A INT) USING iceberg TBLPROPERTIES ('format-version' = '2')")
13561331
spark.sql(s"INSERT INTO $t VALUES (1, 20), (3, 40), (5, 60)")
13571332
// One delete keyed on `a` and one keyed on `A`: binding either key by a folded name
13581333
// misreads one of them whichever field the fold resolves to. Only (5, 60) survives.
13591334
commitEqualityDelete("case_cat", "db", "case_pair_eq", "a", 1, warehouseDir)
13601335
commitEqualityDelete("case_cat", "db", "case_pair_eq", "A", 40, warehouseDir)
1361-
val failures = failingNativeScanChecks(
1336+
failures = failingNativeScanChecks(
13621337
Seq(
13631338
s"SELECT a, A FROM $t ORDER BY a",
13641339
s"SELECT A FROM $t ORDER BY A",
13651340
s"SELECT a FROM $t ORDER BY a",
13661341
s"SELECT count(*) FROM $t"))
1342+
}
1343+
// `a` is ambiguous when case-insensitive, so these queries name no column. Spark 4.1+
1344+
// rejects the duplicate columns itself before any scan runs.
1345+
withCaseCatalog(warehouseDir, caseSensitive = false) {
1346+
Seq(s"SELECT * FROM $t", s"SELECT count(*) FROM $t").foreach { query =>
1347+
if (isSpark41Plus) {
1348+
val e = intercept[org.apache.spark.sql.AnalysisException](spark.sql(query).collect())
1349+
assert(e.getMessage.contains("COLUMN_ALREADY_EXISTS"), e.getMessage)
1350+
} else {
1351+
checkIcebergNativeScan(query)
1352+
}
1353+
}
1354+
}
1355+
withCaseCatalog(warehouseDir, caseSensitive = true) {
13671356
spark.sql(s"DROP TABLE $t")
1368-
assert(failures.isEmpty, failures.mkString("\n\n=====\n\n"))
13691357
}
1358+
assert(failures.isEmpty, failures.mkString("\n\n=====\n\n"))
13701359
}
13711360
}
13721361

@@ -1391,35 +1380,6 @@ class CometIcebergNativeSuite
13911380
}
13921381
}
13931382

1394-
test("issue 5540: case-insensitive read of columns a and A with equality deletes") {
1395-
assume(icebergAvailable, "Iceberg not available in classpath")
1396-
withTempIcebergDir { warehouseDir =>
1397-
val t = "case_cat.db.case_pair_ci"
1398-
withCaseCatalog(warehouseDir, caseSensitive = true) {
1399-
spark.sql(
1400-
s"CREATE TABLE $t (a INT, A INT) USING iceberg TBLPROPERTIES ('format-version' = '2')")
1401-
spark.sql(s"INSERT INTO $t VALUES (1, 20), (3, 40), (5, 60)")
1402-
commitEqualityDelete("case_cat", "db", "case_pair_ci", "a", 1, warehouseDir)
1403-
commitEqualityDelete("case_cat", "db", "case_pair_ci", "A", 40, warehouseDir)
1404-
}
1405-
// `a` is ambiguous here, so the queries name no column: only (5, 60) survives. Spark 4.1+
1406-
// rejects the duplicate columns itself before any scan runs.
1407-
withCaseCatalog(warehouseDir, caseSensitive = false) {
1408-
Seq(s"SELECT * FROM $t", s"SELECT count(*) FROM $t").foreach { query =>
1409-
if (isSpark41Plus) {
1410-
val e = intercept[org.apache.spark.sql.AnalysisException](spark.sql(query).collect())
1411-
assert(e.getMessage.contains("COLUMN_ALREADY_EXISTS"), e.getMessage)
1412-
} else {
1413-
checkIcebergNativeScan(query)
1414-
}
1415-
}
1416-
}
1417-
withCaseCatalog(warehouseDir, caseSensitive = true) {
1418-
spark.sql(s"DROP TABLE $t")
1419-
}
1420-
}
1421-
}
1422-
14231383
test("issue 5540: equality delete survives a case-only column rename") {
14241384
assume(icebergAvailable, "Iceberg not available in classpath")
14251385
withTempIcebergDir { warehouseDir =>
@@ -1470,20 +1430,7 @@ class CometIcebergNativeSuite
14701430
.table()
14711431
spark.sql(s"CREATE TABLE parquet_case_mapped USING parquet LOCATION '$dataPath'")
14721432
try {
1473-
Class
1474-
.forName("org.apache.iceberg.spark.SparkTableUtil")
1475-
.getMethod(
1476-
"importSparkTable",
1477-
classOf[org.apache.spark.sql.SparkSession],
1478-
classOf[org.apache.spark.sql.catalyst.TableIdentifier],
1479-
classOf[org.apache.iceberg.Table],
1480-
classOf[String])
1481-
.invoke(
1482-
null,
1483-
spark,
1484-
new org.apache.spark.sql.catalyst.TableIdentifier("parquet_case_mapped"),
1485-
icebergTable,
1486-
s"${warehouseDir.getAbsolutePath}/staging")
1433+
importParquetInto(icebergTable, "parquet_case_mapped", s"$warehouseDir/staging")
14871434
Seq(true, false).foreach { caseSensitive =>
14881435
withCaseCatalog(warehouseDir, caseSensitive) {
14891436
checkIcebergNativeScan(s"SELECT id, val FROM $t ORDER BY id")
@@ -1530,20 +1477,7 @@ class CometIcebergNativeSuite
15301477
icebergTable.refresh()
15311478
spark.sql(s"CREATE TABLE parquet_$name USING parquet LOCATION '$dataPath'")
15321479
try {
1533-
Class
1534-
.forName("org.apache.iceberg.spark.SparkTableUtil")
1535-
.getMethod(
1536-
"importSparkTable",
1537-
classOf[org.apache.spark.sql.SparkSession],
1538-
classOf[org.apache.spark.sql.catalyst.TableIdentifier],
1539-
classOf[org.apache.iceberg.Table],
1540-
classOf[String])
1541-
.invoke(
1542-
null,
1543-
spark,
1544-
new org.apache.spark.sql.catalyst.TableIdentifier(s"parquet_$name"),
1545-
icebergTable,
1546-
s"${warehouseDir.getAbsolutePath}/staging")
1480+
importParquetInto(icebergTable, s"parquet_$name", s"$warehouseDir/staging")
15471481
// Spark itself reads the mapped children, so matching Spark means reading values.
15481482
withSQLConf(CometConf.COMET_ENABLED.key -> "false") {
15491483
assert(

0 commit comments

Comments
 (0)