Skip to content

Commit b4befe7

Browse files
committed
test: skip only the format-dependent checks in the Spark SQL diffs
SPARK-36120, SPARK-33687, SPARK-22673 and SPARK-43376 now run with Comet's cache format and skip only the size or subquery-reuse assertions that depend on it. The union test in CoalesceShufflePartitionsSuite runs its coalescing check again on Spark 3.5 and later, where CometCoalesceShufflePartitions coalesces the shuffles on the union's join side. From Spark 3.5, CometTestSettings also makes Comet's cache serializer the default of the sql and hive test JVMs, next to the shuffle manager. Spark keeps the first cache serializer it loads for the rest of the JVM, so a suite that caches first through a session of its own left the suites after it in Spark's format.
1 parent 684c1e0 commit b4befe7

5 files changed

Lines changed: 325 additions & 306 deletions

File tree

‎dev/diffs/3.5.9.diff‎

Lines changed: 78 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ index 49eca0f7555..b9e5d69a6c6 100644
3838
</dependencyManagement>
3939

4040
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
41-
index daa3ebac992..2b6138c4280 100644
41+
index daa3ebac992..2c721d85961 100644
4242
--- a/project/SparkBuild.scala
4343
+++ b/project/SparkBuild.scala
4444
@@ -459,9 +459,11 @@ object SparkBuild extends PomBuild {
@@ -53,24 +53,34 @@ index daa3ebac992..2b6138c4280 100644
5353

5454
enable(SparkConnectCommon.settings)(connectCommon)
5555
enable(SparkConnect.settings)(connect)
56-
@@ -1198,6 +1200,25 @@ object SQL {
56+
@@ -1198,6 +1200,35 @@ object SQL {
5757
)
5858
}
5959

6060
+/**
6161
+ * Comet disables itself when its shuffle manager is not registered. When Comet is enabled, make
6262
+ * that shuffle manager the default for every test SparkConf in the modules that have Comet on
6363
+ * their classpath, so that suites which build their own SparkSession or SparkContext run Comet
64-
+ * too, not just the ones that go through SharedSparkSession or TestHive.
64+
+ * too, not just the ones that go through SharedSparkSession or TestHive. Comet's cache serializer
65+
+ * is made the default the same way. Spark keeps the first cache serializer it loads for the rest
66+
+ * of the JVM, so a suite that caches through a session of its own would otherwise leave the
67+
+ * suites after it in Spark's cache format.
6568
+ */
6669
+object CometTestSettings {
6770
+ private val cometEnabled =
6871
+ sys.env.get("ENABLE_COMET").forall(v => v == "1" || v.equalsIgnoreCase("true"))
6972
+
73+
+ private val defaults = Seq(
74+
+ "spark.shuffle.manager" ->
75+
+ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager",
76+
+ "spark.sql.cache.serializer" ->
77+
+ "org.apache.spark.sql.comet.execution.arrow.ArrowCachedBatchSerializer")
78+
+
7079
+ lazy val settings: Seq[Setting[_]] =
71-
+ if (cometEnabled && !sys.props.contains("spark.shuffle.manager")) {
72-
+ Seq((Test / javaOptions) +=
73-
+ "-Dspark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager")
80+
+ if (cometEnabled) {
81+
+ defaults.filterNot { case (key, _) => sys.props.contains(key) }.map {
82+
+ case (key, value) => (Test / javaOptions) += s"-D$key=$value"
83+
+ }
7484
+ } else {
7585
+ Seq.empty
7686
+ }
@@ -260,7 +270,7 @@ index 0efe0877e9b..423d3b3d76d 100644
260270
-- SELECT_HAVING
261271
-- https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/select_having.sql
262272
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
263-
index e5494726695..7a2a2d8b721 100644
273+
index e5494726695..9ceb28855db 100644
264274
--- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
265275
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
266276
@@ -35,10 +35,11 @@ import org.apache.spark.sql.catalyst.analysis.TempTableAlreadyExistsException
@@ -307,16 +317,18 @@ index e5494726695..7a2a2d8b721 100644
307317
}
308318

309319
test("A cached table preserves the partitioning and ordering of its cached SparkPlan") {
310-
@@ -1574,7 +1581,8 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
320+
@@ -1581,7 +1588,10 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
321+
checkAnswer(spark.table(tableName), Row(LocalDateTime.parse("2021-01-01T00:00:00")))
322+
spark.table(tableName).queryExecution.withCachedData.collect {
323+
case cached: InMemoryRelation =>
324+
- assert(cached.stats.sizeInBytes === 8)
325+
+ // Comet's cache format reports Arrow buffer sizes.
326+
+ if (!isCometEnabled) {
327+
+ assert(cached.stats.sizeInBytes === 8)
328+
+ }
329+
}
330+
sql(s"UNCACHE TABLE $tableName")
311331
}
312-
}
313-
314-
- test("SPARK-36120: Support cache/uncache table with TimestampNTZ type") {
315-
+ test("SPARK-36120: Support cache/uncache table with TimestampNTZ type",
316-
+ IgnoreComet("Comet's cache format reports Arrow buffer sizes")) {
317-
val tableName = "ntzCache"
318-
withTable(tableName) {
319-
sql(s"CACHE TABLE $tableName AS SELECT TIMESTAMP_NTZ'2021-01-01 00:00:00'")
320332
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
321333
index 6f3090d8908..52523429c87 100644
322334
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
@@ -1336,19 +1348,32 @@ index 8b4ac474f87..b097a14a972 100644
13361348
assert(df.queryExecution.executedPlan.isInstanceOf[AdaptiveSparkPlanExec])
13371349
df.queryExecution.executedPlan.asInstanceOf[AdaptiveSparkPlanExec].executedPlan
13381350
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
1339-
index e827396009d..066d1a6bf04 100644
1351+
index e827396009d..efbee3975e3 100644
13401352
--- a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
13411353
+++ b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
1342-
@@ -811,7 +811,8 @@ class StatisticsCollectionSuite extends StatisticsCollectionTestBase with Shared
1343-
}
1344-
}
1354+
@@ -829,14 +829,19 @@ class StatisticsCollectionSuite extends StatisticsCollectionTestBase with Shared
1355+
checkTableStats("t1", hasSizeInBytes = true, expectedRowCounts = None)
1356+
checkTableStats("t2", hasSizeInBytes = true, expectedRowCounts = None)
1357+
assert(getCatalogTable("v1").stats.isEmpty)
1358+
- checkOptimizedPlanStats(spark.table("v1"), 4, Some(1), Seq.empty)
1359+
+ // Comet's cache format reports Arrow buffer sizes for a materialized cached view.
1360+
+ if (!isCometEnabled) {
1361+
+ checkOptimizedPlanStats(spark.table("v1"), 4, Some(1), Seq.empty)
1362+
+ }
1363+
checkOptimizedPlanStats(spark.table("v2"), 1, None, Seq.empty)
13451364

1346-
- test("SPARK-33687: analyze all tables in a specific database") {
1347-
+ test("SPARK-33687: analyze all tables in a specific database",
1348-
+ IgnoreComet("Comet's cache format reports Arrow buffer sizes")) {
1349-
withTempDatabase { database =>
1350-
spark.catalog.setCurrentDatabase(database)
1351-
withTempDir { dir =>
1365+
sql("ANALYZE TABLES COMPUTE STATISTICS")
1366+
checkTableStats("t1", hasSizeInBytes = true, expectedRowCounts = Some(10))
1367+
checkTableStats("t2", hasSizeInBytes = true, expectedRowCounts = Some(20))
1368+
- checkOptimizedPlanStats(spark.table("v1"), 4, Some(1), Seq.empty)
1369+
- checkOptimizedPlanStats(spark.table("v2"), 4, Some(1), Seq.empty)
1370+
+ if (!isCometEnabled) {
1371+
+ checkOptimizedPlanStats(spark.table("v1"), 4, Some(1), Seq.empty)
1372+
+ checkOptimizedPlanStats(spark.table("v2"), 4, Some(1), Seq.empty)
1373+
+ }
1374+
}
1375+
}
1376+
}
13521377
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
13531378
index 04702201f82..4d38d8d6e51 100644
13541379
--- a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
@@ -1586,7 +1611,7 @@ index ae1c0a86a14..1d3b914fd64 100644
15861611
// Fail to read ancient datetime values.
15871612
withSQLConf(SQLConf.PARQUET_REBASE_MODE_IN_READ.key -> EXCEPTION.toString) {
15881613
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala
1589-
index e11191da6a9..005fd4fa657 100644
1614+
index e11191da6a9..0988faf4fcd 100644
15901615
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala
15911616
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala
15921617
@@ -51,6 +51,11 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite {
@@ -1646,23 +1671,6 @@ index e11191da6a9..005fd4fa657 100644
16461671
}
16471672
}
16481673
}
1649-
@@ -473,10 +484,12 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite {
1650-
// aggregate on the other side of the union.
1651-
val finalPlan = resultDf.queryExecution.executedPlan
1652-
.asInstanceOf[AdaptiveSparkPlanExec].executedPlan
1653-
- assert(
1654-
- finalPlan.collect {
1655-
- case r @ CoalescedShuffleRead() => r
1656-
- }.size == 2)
1657-
+ if (!isCometEnabled) {
1658-
+ assert(
1659-
+ finalPlan.collect {
1660-
+ case r @ CoalescedShuffleRead() => r
1661-
+ }.size == 2)
1662-
+ }
1663-
}
1664-
withSparkSession(test, 100, None)
1665-
}
16661674
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala
16671675
index 418ca3430bb..eb8267192f8 100644
16681676
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala
@@ -1929,7 +1937,7 @@ index 5a413c77754..207b66e1d7b 100644
19291937

19301938
import testImplicits._
19311939
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
1932-
index 2f8e401e743..74c046f5db4 100644
1940+
index 2f8e401e743..fc4ce35be4b 100644
19331941
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
19341942
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
19351943
@@ -27,9 +27,11 @@ import org.scalatest.time.SpanSugar._
@@ -2335,17 +2343,18 @@ index 2f8e401e743..74c046f5db4 100644
23352343
}.size == (if (firstAccess) 2 else 0))
23362344
assert(collect(initialExecutedPlan) {
23372345
case i: InMemoryTableScanLike => i
2338-
@@ -2923,7 +2966,8 @@ class AdaptiveQueryExecSuite
2339-
}
2340-
}
2346+
@@ -2932,13 +2975,18 @@ class AdaptiveQueryExecSuite
23412347

2342-
- test("SPARK-43376: Improve reuse subquery with table cache") {
2343-
+ test("SPARK-43376: Improve reuse subquery with table cache",
2344-
+ IgnoreComet("Comet's cache scan does not plan the subqueries in its pruning predicates")) {
2345-
withSQLConf(SQLConf.CAN_CHANGE_CACHED_PLAN_OUTPUT_PARTITIONING.key -> "true") {
2346-
withTable("t1", "t2") {
2347-
withCache("t1") {
2348-
@@ -2938,7 +2982,8 @@ class AdaptiveQueryExecSuite
2348+
val (_, adaptive) = runAdaptiveAndVerifyResult(
2349+
"SELECT * FROM t1 WHERE c1 < (SELECT c2 FROM t2)")
2350+
- assert(findReusedSubquery(adaptive).size == 1)
2351+
+ // Comet's cache scan does not plan the subqueries in its pruning predicates, so only the
2352+
+ // filter above it runs this one and there is nothing to reuse.
2353+
+ if (!isCometEnabled) {
2354+
+ assert(findReusedSubquery(adaptive).size == 1)
2355+
+ }
2356+
}
2357+
}
23492358
}
23502359
}
23512360

@@ -2355,7 +2364,7 @@ index 2f8e401e743..74c046f5db4 100644
23552364
val emptyDf = spark.range(1).where("false")
23562365
val aggDf1 = emptyDf.agg(sum("id").as("id")).withColumn("name", lit("df1"))
23572366
val aggDf2 = emptyDf.agg(sum("id").as("id")).withColumn("name", lit("df2"))
2358-
@@ -2980,7 +3025,9 @@ class AdaptiveQueryExecSuite
2367+
@@ -2980,7 +3028,9 @@ class AdaptiveQueryExecSuite
23592368

23602369
val plan = df.queryExecution.executedPlan.asInstanceOf[AdaptiveSparkPlanExec]
23612370
assert(plan.inputPlan.isInstanceOf[TakeOrderedAndProjectExec])
@@ -2367,16 +2376,10 @@ index 2f8e401e743..74c046f5db4 100644
23672376
assert(o1.semanticEquals(o2), "Different output column order after AQE optimization")
23682377
}
23692378
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala
2370-
index de04938f247..fa019925b6f 100644
2379+
index de04938f247..f3351a86946 100644
23712380
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala
23722381
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala
2373-
@@ -22,11 +22,12 @@ import java.sql.{Date, Timestamp}
2374-
import java.util.concurrent.atomic.AtomicInteger
2375-
2376-
import org.apache.spark.rdd.RDD
2377-
-import org.apache.spark.sql.{DataFrame, QueryTest, Row}
2378-
+import org.apache.spark.sql.{DataFrame, IgnoreComet, QueryTest, Row}
2379-
import org.apache.spark.sql.catalyst.InternalRow
2382+
@@ -27,6 +27,7 @@ import org.apache.spark.sql.catalyst.InternalRow
23802383
import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, AttributeSet, In}
23812384
import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning
23822385
import org.apache.spark.sql.columnar.CachedBatch
@@ -2405,16 +2408,18 @@ index de04938f247..fa019925b6f 100644
24052408
}
24062409

24072410
test("SPARK-25727 - otherCopyArgs in InMemoryRelation does not include outputOrdering") {
2408-
@@ -522,7 +529,8 @@ class InMemoryColumnarQuerySuite extends QueryTest
2409-
assert(json.contains("outputOrdering"))
2410-
}
2411+
@@ -543,7 +550,10 @@ class InMemoryColumnarQuerySuite extends QueryTest
24112412

2412-
- test("SPARK-22673: InMemoryRelation should utilize existing stats of the plan to be cached") {
2413-
+ test("SPARK-22673: InMemoryRelation should utilize existing stats of the plan to be cached",
2414-
+ IgnoreComet("Comet's cache format reports Arrow buffer sizes")) {
2415-
Seq("orc", "").foreach { useV1SourceReaderList =>
2416-
// This test case depends on the size of ORC in statistics.
2417-
withSQLConf(
2413+
// InMemoryRelation's stats is updated after materializing RDD
2414+
dfFromFile.collect()
2415+
- assert(inMemoryRelation.computeStats().sizeInBytes === 16)
2416+
+ // Comet's cache format reports Arrow buffer sizes.
2417+
+ if (!isCometEnabled) {
2418+
+ assert(inMemoryRelation.computeStats().sizeInBytes === 16)
2419+
+ }
2420+
2421+
// test of catalog table
2422+
val dfFromTable = spark.catalog.createTable("table1", workDirPath).cache()
24182423
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala
24192424
index 2c73622739a..5d0efeb263a 100644
24202425
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala

0 commit comments

Comments
 (0)