@@ -1300,73 +1300,62 @@ class CometIcebergNativeSuite
13001300 CometConf .COMET_EXEC_ENABLED .key -> " true" ,
13011301 CometConf .COMET_ICEBERG_NATIVE_ENABLED .key -> " true" )(f)
13021302
1303- test(" issue 5540: case-sensitive read of columns a and A" ) {
1304- assume(icebergAvailable, " Iceberg not available in classpath" )
1305- withTempIcebergDir { warehouseDir =>
1306- withCaseCatalog(warehouseDir, caseSensitive = true ) {
1307- val t = " case_cat.db.case_pair"
1308- spark.sql(s " CREATE TABLE $t (a INT, A INT) USING iceberg " )
1309- spark.sql(s " INSERT INTO $t VALUES (1, 20), (3, 40) " )
1310- val failures = failingNativeScanChecks(
1311- Seq (
1312- s " SELECT a, A FROM $t ORDER BY a " ,
1313- s " SELECT A, a FROM $t ORDER BY a " ,
1314- s " SELECT A FROM $t ORDER BY A " ,
1315- s " SELECT a FROM $t ORDER BY a " ,
1316- s " SELECT * FROM $t ORDER BY a " ))
1317- spark.sql(s " DROP TABLE $t" )
1318- assert(failures.isEmpty, failures.mkString(" \n\n =====\n\n " ))
1319- }
1320- }
1321- }
1322-
1323- test(" issue 5540: case-sensitive filters on columns a and A" ) {
1324- assume(icebergAvailable, " Iceberg not available in classpath" )
1325- withTempIcebergDir { warehouseDir =>
1326- withCaseCatalog(warehouseDir, caseSensitive = true ) {
1327- val t = " case_cat.db.case_pair_filter"
1328- spark.sql(s " CREATE TABLE $t (a INT, A INT) USING iceberg " )
1329- spark.sql(s " INSERT INTO $t VALUES (1, 20), (3, 40) " )
1330- // a spans [1, 3] and A spans [20, 40]: a filter on one column bound to the other would
1331- // prune the whole row group or keep the wrong row.
1332- val failures = failingNativeScanChecks(
1333- Seq (
1334- s " SELECT a FROM $t WHERE A = 40 " ,
1335- s " SELECT A FROM $t WHERE a = 3 " ,
1336- s " SELECT A FROM $t WHERE A = 40 " ,
1337- s " SELECT a FROM $t WHERE a = 3 " ,
1338- s " SELECT a, A FROM $t WHERE A = 40 " ,
1339- s " SELECT a, A FROM $t WHERE a = 3 " ,
1340- s " SELECT a FROM $t WHERE A > 30 " ,
1341- s " SELECT a FROM $t WHERE A IN (20) " ,
1342- s " SELECT A FROM $t WHERE a < 2 " ))
1343- spark.sql(s " DROP TABLE $t" )
1344- assert(failures.isEmpty, failures.mkString(" \n\n =====\n\n " ))
1345- }
1346- }
1347- }
1348-
1349- test(" issue 5540: case-sensitive equality deletes keyed on a and on A" ) {
1303+ /** Imports the files of session table `sourceTable` into `table` with SparkTableUtil. */
1304+ private def importParquetInto (
1305+ table : org.apache.iceberg.Table ,
1306+ sourceTable : String ,
1307+ stagingDir : String ): Unit =
1308+ Class
1309+ .forName(" org.apache.iceberg.spark.SparkTableUtil" )
1310+ .getMethod(
1311+ " importSparkTable" ,
1312+ classOf [org.apache.spark.sql.SparkSession ],
1313+ classOf [org.apache.spark.sql.catalyst.TableIdentifier ],
1314+ classOf [org.apache.iceberg.Table ],
1315+ classOf [String ])
1316+ .invoke(
1317+ null ,
1318+ spark,
1319+ new org.apache.spark.sql.catalyst.TableIdentifier (sourceTable),
1320+ table,
1321+ stagingDir)
1322+
1323+ test(" issue 5540: equality deletes keyed on columns a and A" ) {
13501324 assume(icebergAvailable, " Iceberg not available in classpath" )
13511325 withTempIcebergDir { warehouseDir =>
1326+ val t = " case_cat.db.case_pair_eq"
1327+ var failures = Seq .empty[String ]
13521328 withCaseCatalog(warehouseDir, caseSensitive = true ) {
1353- val t = " case_cat.db.case_pair_eq"
13541329 spark.sql(
13551330 s " CREATE TABLE $t (a INT, A INT) USING iceberg TBLPROPERTIES ('format-version' = '2') " )
13561331 spark.sql(s " INSERT INTO $t VALUES (1, 20), (3, 40), (5, 60) " )
13571332 // One delete keyed on `a` and one keyed on `A`: binding either key by a folded name
13581333 // misreads one of them whichever field the fold resolves to. Only (5, 60) survives.
13591334 commitEqualityDelete(" case_cat" , " db" , " case_pair_eq" , " a" , 1 , warehouseDir)
13601335 commitEqualityDelete(" case_cat" , " db" , " case_pair_eq" , " A" , 40 , warehouseDir)
1361- val failures = failingNativeScanChecks(
1336+ failures = failingNativeScanChecks(
13621337 Seq (
13631338 s " SELECT a, A FROM $t ORDER BY a " ,
13641339 s " SELECT A FROM $t ORDER BY A " ,
13651340 s " SELECT a FROM $t ORDER BY a " ,
13661341 s " SELECT count(*) FROM $t" ))
1342+ }
1343+ // `a` is ambiguous when case-insensitive, so these queries name no column. Spark 4.1+
1344+ // rejects the duplicate columns itself before any scan runs.
1345+ withCaseCatalog(warehouseDir, caseSensitive = false ) {
1346+ Seq (s " SELECT * FROM $t" , s " SELECT count(*) FROM $t" ).foreach { query =>
1347+ if (isSpark41Plus) {
1348+ val e = intercept[org.apache.spark.sql.AnalysisException ](spark.sql(query).collect())
1349+ assert(e.getMessage.contains(" COLUMN_ALREADY_EXISTS" ), e.getMessage)
1350+ } else {
1351+ checkIcebergNativeScan(query)
1352+ }
1353+ }
1354+ }
1355+ withCaseCatalog(warehouseDir, caseSensitive = true ) {
13671356 spark.sql(s " DROP TABLE $t" )
1368- assert(failures.isEmpty, failures.mkString(" \n\n =====\n\n " ))
13691357 }
1358+ assert(failures.isEmpty, failures.mkString(" \n\n =====\n\n " ))
13701359 }
13711360 }
13721361
@@ -1391,35 +1380,6 @@ class CometIcebergNativeSuite
13911380 }
13921381 }
13931382
1394- test(" issue 5540: case-insensitive read of columns a and A with equality deletes" ) {
1395- assume(icebergAvailable, " Iceberg not available in classpath" )
1396- withTempIcebergDir { warehouseDir =>
1397- val t = " case_cat.db.case_pair_ci"
1398- withCaseCatalog(warehouseDir, caseSensitive = true ) {
1399- spark.sql(
1400- s " CREATE TABLE $t (a INT, A INT) USING iceberg TBLPROPERTIES ('format-version' = '2') " )
1401- spark.sql(s " INSERT INTO $t VALUES (1, 20), (3, 40), (5, 60) " )
1402- commitEqualityDelete(" case_cat" , " db" , " case_pair_ci" , " a" , 1 , warehouseDir)
1403- commitEqualityDelete(" case_cat" , " db" , " case_pair_ci" , " A" , 40 , warehouseDir)
1404- }
1405- // `a` is ambiguous here, so the queries name no column: only (5, 60) survives. Spark 4.1+
1406- // rejects the duplicate columns itself before any scan runs.
1407- withCaseCatalog(warehouseDir, caseSensitive = false ) {
1408- Seq (s " SELECT * FROM $t" , s " SELECT count(*) FROM $t" ).foreach { query =>
1409- if (isSpark41Plus) {
1410- val e = intercept[org.apache.spark.sql.AnalysisException ](spark.sql(query).collect())
1411- assert(e.getMessage.contains(" COLUMN_ALREADY_EXISTS" ), e.getMessage)
1412- } else {
1413- checkIcebergNativeScan(query)
1414- }
1415- }
1416- }
1417- withCaseCatalog(warehouseDir, caseSensitive = true ) {
1418- spark.sql(s " DROP TABLE $t" )
1419- }
1420- }
1421- }
1422-
14231383 test(" issue 5540: equality delete survives a case-only column rename" ) {
14241384 assume(icebergAvailable, " Iceberg not available in classpath" )
14251385 withTempIcebergDir { warehouseDir =>
@@ -1470,20 +1430,7 @@ class CometIcebergNativeSuite
14701430 .table()
14711431 spark.sql(s " CREATE TABLE parquet_case_mapped USING parquet LOCATION ' $dataPath' " )
14721432 try {
1473- Class
1474- .forName(" org.apache.iceberg.spark.SparkTableUtil" )
1475- .getMethod(
1476- " importSparkTable" ,
1477- classOf [org.apache.spark.sql.SparkSession ],
1478- classOf [org.apache.spark.sql.catalyst.TableIdentifier ],
1479- classOf [org.apache.iceberg.Table ],
1480- classOf [String ])
1481- .invoke(
1482- null ,
1483- spark,
1484- new org.apache.spark.sql.catalyst.TableIdentifier (" parquet_case_mapped" ),
1485- icebergTable,
1486- s " ${warehouseDir.getAbsolutePath}/staging " )
1433+ importParquetInto(icebergTable, " parquet_case_mapped" , s " $warehouseDir/staging " )
14871434 Seq (true , false ).foreach { caseSensitive =>
14881435 withCaseCatalog(warehouseDir, caseSensitive) {
14891436 checkIcebergNativeScan(s " SELECT id, val FROM $t ORDER BY id " )
@@ -1530,20 +1477,7 @@ class CometIcebergNativeSuite
15301477 icebergTable.refresh()
15311478 spark.sql(s " CREATE TABLE parquet_ $name USING parquet LOCATION ' $dataPath' " )
15321479 try {
1533- Class
1534- .forName(" org.apache.iceberg.spark.SparkTableUtil" )
1535- .getMethod(
1536- " importSparkTable" ,
1537- classOf [org.apache.spark.sql.SparkSession ],
1538- classOf [org.apache.spark.sql.catalyst.TableIdentifier ],
1539- classOf [org.apache.iceberg.Table ],
1540- classOf [String ])
1541- .invoke(
1542- null ,
1543- spark,
1544- new org.apache.spark.sql.catalyst.TableIdentifier (s " parquet_ $name" ),
1545- icebergTable,
1546- s " ${warehouseDir.getAbsolutePath}/staging " )
1480+ importParquetInto(icebergTable, s " parquet_ $name" , s " $warehouseDir/staging " )
15471481 // Spark itself reads the mapped children, so matching Spark means reading values.
15481482 withSQLConf(CometConf .COMET_ENABLED .key -> " false" ) {
15491483 assert(
0 commit comments