@@ -38,7 +38,7 @@ index 49eca0f7555..b9e5d69a6c6 100644
3838 </dependencyManagement>
3939
4040diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
41- index daa3ebac992..2b6138c4280 100644
41+ index daa3ebac992..2c721d85961 100644
4242--- a/project/SparkBuild.scala
4343+++ b/project/SparkBuild.scala
4444@@ -459,9 +459,11 @@ object SparkBuild extends PomBuild {
@@ -53,24 +53,34 @@ index daa3ebac992..2b6138c4280 100644
5353
5454 enable(SparkConnectCommon.settings)(connectCommon)
5555 enable(SparkConnect.settings)(connect)
56- @@ -1198,6 +1200,25 @@ object SQL {
56+ @@ -1198,6 +1200,35 @@ object SQL {
5757 )
5858 }
5959
6060+ /**
6161+ * Comet disables itself when its shuffle manager is not registered. When Comet is enabled, make
6262+ * that shuffle manager the default for every test SparkConf in the modules that have Comet on
6363+ * their classpath, so that suites which build their own SparkSession or SparkContext run Comet
64- + * too, not just the ones that go through SharedSparkSession or TestHive.
64+ + * too, not just the ones that go through SharedSparkSession or TestHive. Comet's cache serializer
65+ + * is made the default the same way. Spark keeps the first cache serializer it loads for the rest
66+ + * of the JVM, so a suite that caches through a session of its own would otherwise leave the
67+ + * suites after it in Spark's cache format.
6568+ */
6669+ object CometTestSettings {
6770+ private val cometEnabled =
6871+ sys.env.get("ENABLE_COMET").forall(v => v == "1" || v.equalsIgnoreCase("true"))
6972+
73+ + private val defaults = Seq(
74+ + "spark.shuffle.manager" ->
75+ + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager",
76+ + "spark.sql.cache.serializer" ->
77+ + "org.apache.spark.sql.comet.execution.arrow.ArrowCachedBatchSerializer")
78+ +
7079+ lazy val settings: Seq[Setting[_]] =
71- + if (cometEnabled && !sys.props.contains("spark.shuffle.manager")) {
72- + Seq((Test / javaOptions) +=
73- + "-Dspark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager")
80+ + if (cometEnabled) {
81+ + defaults.filterNot { case (key, _) => sys.props.contains(key) }.map {
82+ + case (key, value) => (Test / javaOptions) += s"-D$key=$value"
83+ + }
7484+ } else {
7585+ Seq.empty
7686+ }
@@ -260,7 +270,7 @@ index 0efe0877e9b..423d3b3d76d 100644
260270 -- SELECT_HAVING
261271 -- https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/select_having.sql
262272diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
263- index e5494726695..7a2a2d8b721 100644
273+ index e5494726695..9ceb28855db 100644
264274--- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
265275+++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
266276@@ -35,10 +35,11 @@ import org.apache.spark.sql.catalyst.analysis.TempTableAlreadyExistsException
@@ -307,16 +317,18 @@ index e5494726695..7a2a2d8b721 100644
307317 }
308318
309319 test("A cached table preserves the partitioning and ordering of its cached SparkPlan") {
310- @@ -1574,7 +1581,8 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
320+ @@ -1581,7 +1588,10 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
321+ checkAnswer(spark.table(tableName), Row(LocalDateTime.parse("2021-01-01T00:00:00")))
322+ spark.table(tableName).queryExecution.withCachedData.collect {
323+ case cached: InMemoryRelation =>
324+ - assert(cached.stats.sizeInBytes === 8)
325+ + // Comet's cache format reports Arrow buffer sizes.
326+ + if (!isCometEnabled) {
327+ + assert(cached.stats.sizeInBytes === 8)
328+ + }
329+ }
330+ sql(s"UNCACHE TABLE $tableName")
311331 }
312- }
313-
314- - test("SPARK-36120: Support cache/uncache table with TimestampNTZ type") {
315- + test("SPARK-36120: Support cache/uncache table with TimestampNTZ type",
316- + IgnoreComet("Comet's cache format reports Arrow buffer sizes")) {
317- val tableName = "ntzCache"
318- withTable(tableName) {
319- sql(s"CACHE TABLE $tableName AS SELECT TIMESTAMP_NTZ'2021-01-01 00:00:00'")
320332diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
321333index 6f3090d8908..52523429c87 100644
322334--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
@@ -1336,19 +1348,32 @@ index 8b4ac474f87..b097a14a972 100644
13361348 assert(df.queryExecution.executedPlan.isInstanceOf[AdaptiveSparkPlanExec])
13371349 df.queryExecution.executedPlan.asInstanceOf[AdaptiveSparkPlanExec].executedPlan
13381350diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
1339- index e827396009d..066d1a6bf04 100644
1351+ index e827396009d..efbee3975e3 100644
13401352--- a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
13411353+++ b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
1342- @@ -811,7 +811,8 @@ class StatisticsCollectionSuite extends StatisticsCollectionTestBase with Shared
1343- }
1344- }
1354+ @@ -829,14 +829,19 @@ class StatisticsCollectionSuite extends StatisticsCollectionTestBase with Shared
1355+ checkTableStats("t1", hasSizeInBytes = true, expectedRowCounts = None)
1356+ checkTableStats("t2", hasSizeInBytes = true, expectedRowCounts = None)
1357+ assert(getCatalogTable("v1").stats.isEmpty)
1358+ - checkOptimizedPlanStats(spark.table("v1"), 4, Some(1), Seq.empty)
1359+ + // Comet's cache format reports Arrow buffer sizes for a materialized cached view.
1360+ + if (!isCometEnabled) {
1361+ + checkOptimizedPlanStats(spark.table("v1"), 4, Some(1), Seq.empty)
1362+ + }
1363+ checkOptimizedPlanStats(spark.table("v2"), 1, None, Seq.empty)
13451364
1346- - test("SPARK-33687: analyze all tables in a specific database") {
1347- + test("SPARK-33687: analyze all tables in a specific database",
1348- + IgnoreComet("Comet's cache format reports Arrow buffer sizes")) {
1349- withTempDatabase { database =>
1350- spark.catalog.setCurrentDatabase(database)
1351- withTempDir { dir =>
1365+ sql("ANALYZE TABLES COMPUTE STATISTICS")
1366+ checkTableStats("t1", hasSizeInBytes = true, expectedRowCounts = Some(10))
1367+ checkTableStats("t2", hasSizeInBytes = true, expectedRowCounts = Some(20))
1368+ - checkOptimizedPlanStats(spark.table("v1"), 4, Some(1), Seq.empty)
1369+ - checkOptimizedPlanStats(spark.table("v2"), 4, Some(1), Seq.empty)
1370+ + if (!isCometEnabled) {
1371+ + checkOptimizedPlanStats(spark.table("v1"), 4, Some(1), Seq.empty)
1372+ + checkOptimizedPlanStats(spark.table("v2"), 4, Some(1), Seq.empty)
1373+ + }
1374+ }
1375+ }
1376+ }
13521377diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
13531378index 04702201f82..4d38d8d6e51 100644
13541379--- a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
@@ -1586,7 +1611,7 @@ index ae1c0a86a14..1d3b914fd64 100644
15861611 // Fail to read ancient datetime values.
15871612 withSQLConf(SQLConf.PARQUET_REBASE_MODE_IN_READ.key -> EXCEPTION.toString) {
15881613diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala
1589- index e11191da6a9..005fd4fa657 100644
1614+ index e11191da6a9..0988faf4fcd 100644
15901615--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala
15911616+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala
15921617@@ -51,6 +51,11 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite {
@@ -1646,23 +1671,6 @@ index e11191da6a9..005fd4fa657 100644
16461671 }
16471672 }
16481673 }
1649- @@ -473,10 +484,12 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite {
1650- // aggregate on the other side of the union.
1651- val finalPlan = resultDf.queryExecution.executedPlan
1652- .asInstanceOf[AdaptiveSparkPlanExec].executedPlan
1653- - assert(
1654- - finalPlan.collect {
1655- - case r @ CoalescedShuffleRead() => r
1656- - }.size == 2)
1657- + if (!isCometEnabled) {
1658- + assert(
1659- + finalPlan.collect {
1660- + case r @ CoalescedShuffleRead() => r
1661- + }.size == 2)
1662- + }
1663- }
1664- withSparkSession(test, 100, None)
1665- }
16661674diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala
16671675index 418ca3430bb..eb8267192f8 100644
16681676--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala
@@ -1929,7 +1937,7 @@ index 5a413c77754..207b66e1d7b 100644
19291937
19301938 import testImplicits._
19311939diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
1932- index 2f8e401e743..74c046f5db4 100644
1940+ index 2f8e401e743..fc4ce35be4b 100644
19331941--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
19341942+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
19351943@@ -27,9 +27,11 @@ import org.scalatest.time.SpanSugar._
@@ -2335,17 +2343,18 @@ index 2f8e401e743..74c046f5db4 100644
23352343 }.size == (if (firstAccess) 2 else 0))
23362344 assert(collect(initialExecutedPlan) {
23372345 case i: InMemoryTableScanLike => i
2338- @@ -2923,7 +2966,8 @@ class AdaptiveQueryExecSuite
2339- }
2340- }
2346+ @@ -2932,13 +2975,18 @@ class AdaptiveQueryExecSuite
23412347
2342- - test("SPARK-43376: Improve reuse subquery with table cache") {
2343- + test("SPARK-43376: Improve reuse subquery with table cache",
2344- + IgnoreComet("Comet's cache scan does not plan the subqueries in its pruning predicates")) {
2345- withSQLConf(SQLConf.CAN_CHANGE_CACHED_PLAN_OUTPUT_PARTITIONING.key -> "true") {
2346- withTable("t1", "t2") {
2347- withCache("t1") {
2348- @@ -2938,7 +2982,8 @@ class AdaptiveQueryExecSuite
2348+ val (_, adaptive) = runAdaptiveAndVerifyResult(
2349+ "SELECT * FROM t1 WHERE c1 < (SELECT c2 FROM t2)")
2350+ - assert(findReusedSubquery(adaptive).size == 1)
2351+ + // Comet's cache scan does not plan the subqueries in its pruning predicates, so only the
2352+ + // filter above it runs this one and there is nothing to reuse.
2353+ + if (!isCometEnabled) {
2354+ + assert(findReusedSubquery(adaptive).size == 1)
2355+ + }
2356+ }
2357+ }
23492358 }
23502359 }
23512360
@@ -2355,7 +2364,7 @@ index 2f8e401e743..74c046f5db4 100644
23552364 val emptyDf = spark.range(1).where("false")
23562365 val aggDf1 = emptyDf.agg(sum("id").as("id")).withColumn("name", lit("df1"))
23572366 val aggDf2 = emptyDf.agg(sum("id").as("id")).withColumn("name", lit("df2"))
2358- @@ -2980,7 +3025 ,9 @@ class AdaptiveQueryExecSuite
2367+ @@ -2980,7 +3028 ,9 @@ class AdaptiveQueryExecSuite
23592368
23602369 val plan = df.queryExecution.executedPlan.asInstanceOf[AdaptiveSparkPlanExec]
23612370 assert(plan.inputPlan.isInstanceOf[TakeOrderedAndProjectExec])
@@ -2367,16 +2376,10 @@ index 2f8e401e743..74c046f5db4 100644
23672376 assert(o1.semanticEquals(o2), "Different output column order after AQE optimization")
23682377 }
23692378diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala
2370- index de04938f247..fa019925b6f 100644
2379+ index de04938f247..f3351a86946 100644
23712380--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala
23722381+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala
2373- @@ -22,11 +22,12 @@ import java.sql.{Date, Timestamp}
2374- import java.util.concurrent.atomic.AtomicInteger
2375-
2376- import org.apache.spark.rdd.RDD
2377- - import org.apache.spark.sql.{DataFrame, QueryTest, Row}
2378- + import org.apache.spark.sql.{DataFrame, IgnoreComet, QueryTest, Row}
2379- import org.apache.spark.sql.catalyst.InternalRow
2382+ @@ -27,6 +27,7 @@ import org.apache.spark.sql.catalyst.InternalRow
23802383 import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, AttributeSet, In}
23812384 import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning
23822385 import org.apache.spark.sql.columnar.CachedBatch
@@ -2405,16 +2408,18 @@ index de04938f247..fa019925b6f 100644
24052408 }
24062409
24072410 test("SPARK-25727 - otherCopyArgs in InMemoryRelation does not include outputOrdering") {
2408- @@ -522,7 +529,8 @@ class InMemoryColumnarQuerySuite extends QueryTest
2409- assert(json.contains("outputOrdering"))
2410- }
2411+ @@ -543,7 +550,10 @@ class InMemoryColumnarQuerySuite extends QueryTest
24112412
2412- - test("SPARK-22673: InMemoryRelation should utilize existing stats of the plan to be cached") {
2413- + test("SPARK-22673: InMemoryRelation should utilize existing stats of the plan to be cached",
2414- + IgnoreComet("Comet's cache format reports Arrow buffer sizes")) {
2415- Seq("orc", "").foreach { useV1SourceReaderList =>
2416- // This test case depends on the size of ORC in statistics.
2417- withSQLConf(
2413+ // InMemoryRelation's stats is updated after materializing RDD
2414+ dfFromFile.collect()
2415+ - assert(inMemoryRelation.computeStats().sizeInBytes === 16)
2416+ + // Comet's cache format reports Arrow buffer sizes.
2417+ + if (!isCometEnabled) {
2418+ + assert(inMemoryRelation.computeStats().sizeInBytes === 16)
2419+ + }
2420+
2421+ // test of catalog table
2422+ val dfFromTable = spark.catalog.createTable("table1", workDirPath).cache()
24182423diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala
24192424index 2c73622739a..5d0efeb263a 100644
24202425--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala
0 commit comments