@@ -37,6 +37,48 @@ index d3544881af1..0c0618a5e80 100644
3737 </dependencies>
3838 </dependencyManagement>
3939
40+ diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
41+ index 1cbd9a61289..74aa1f684c9 100644
42+ --- a/project/SparkBuild.scala
43+ +++ b/project/SparkBuild.scala
44+ @@ -445,9 +445,11 @@ object SparkBuild extends PomBuild {
45+
46+ /* Spark SQL Core console settings */
47+ enable(SQL.settings)(sql)
48+ + enable(CometTestSettings.settings)(sql)
49+
50+ /* Hive console settings */
51+ enable(Hive.settings)(hive)
52+ + enable(CometTestSettings.settings)(hive)
53+
54+ enable(SparkConnectCommon.settings)(connectCommon)
55+ enable(SparkConnect.settings)(connect)
56+ @@ -1183,6 +1185,25 @@ object SQL {
57+ )
58+ }
59+
60+ + /**
61+ + * Comet disables itself when its shuffle manager is not registered. When Comet is enabled, make
62+ + * that shuffle manager the default for every test SparkConf in the modules that have Comet on
63+ + * their classpath, so that suites which build their own SparkSession or SparkContext run Comet
64+ + * too, not just the ones that go through SharedSparkSession or TestHive.
65+ + */
66+ + object CometTestSettings {
67+ + private val cometEnabled =
68+ + sys.env.get("ENABLE_COMET").forall(v => v == "1" || v.equalsIgnoreCase("true"))
69+ +
70+ + lazy val settings: Seq[Setting[_]] =
71+ + if (cometEnabled && !sys.props.contains("spark.shuffle.manager")) {
72+ + Seq((Test / javaOptions) +=
73+ + "-Dspark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager")
74+ + } else {
75+ + Seq.empty
76+ + }
77+ + }
78+ +
79+ object Hive {
80+
81+ lazy val settings = Seq(
4082diff --git a/sql/core/pom.xml b/sql/core/pom.xml
4183index b386d135da1..46449e3f3f1 100644
4284--- a/sql/core/pom.xml
@@ -1056,6 +1098,50 @@ index 2dabcf01be7..8fcec0d1ce4 100644
10561098 }
10571099 }
10581100 }
1101+ diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
1102+ index 48ad10992c5..a164e273b76 100644
1103+ --- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
1104+ +++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
1105+ @@ -165,7 +165,15 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper {
1106+ }
1107+ }
1108+
1109+ + // Comet: Comet's own columnar rules take over these plans, so the rules the tests below
1110+ + // inject never see the Spark operators they act on or check for.
1111+ + private def assumeCometDisabled(): Unit = {
1112+ + assume(!org.apache.spark.sql.SparkSession.isCometEnabled,
1113+ + "Skipped when Comet is enabled: Comet replaces the operators the injected rules act on")
1114+ + }
1115+ +
1116+ test("inject adaptive query prep rule") {
1117+ + assumeCometDisabled()
1118+ val extensions = create { extensions =>
1119+ // inject rule that will run during AQE query stage preparation and will add custom tags
1120+ // to the plan
1121+ @@ -259,6 +267,7 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper {
1122+ }
1123+
1124+ private def testInjectColumnar(enableAQE: Boolean): Unit = {
1125+ + assumeCometDisabled()
1126+ def collectPlanSteps(plan: SparkPlan): Seq[Int] = plan match {
1127+ case a: AdaptiveSparkPlanExec =>
1128+ assert(a.toString.startsWith("AdaptiveSparkPlan isFinalPlan=true"))
1129+ @@ -314,6 +323,7 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper {
1130+ }
1131+
1132+ test("reset column vectors") {
1133+ + assumeCometDisabled()
1134+ val session = SparkSession.builder()
1135+ .master("local[1]")
1136+ .config(COLUMN_BATCH_SIZE.key, 2)
1137+ @@ -482,6 +492,7 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper {
1138+ }
1139+
1140+ test("SPARK-38697: Extend SparkSessionExtensions to inject rules into AQE Optimizer") {
1141+ + assumeCometDisabled()
1142+ def executedPlan(df: Dataset[java.lang.Long]): SparkPlan = {
1143+ assert(df.queryExecution.executedPlan.isInstanceOf[AdaptiveSparkPlanExec])
1144+ df.queryExecution.executedPlan.asInstanceOf[AdaptiveSparkPlanExec].executedPlan
10591145diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
10601146index 18123a4d6ec..0fe185baa33 100644
10611147--- a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
@@ -1328,6 +1414,84 @@ index c0ec8a58bd5..4e8bc6ed3c5 100644
13281414
13291415 // Fail to read ancient datetime values.
13301416 withSQLConf(SQLConf.PARQUET_REBASE_MODE_IN_READ.key -> EXCEPTION.toString) {
1417+ diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala
1418+ index 24a98dd83f3..4db28a7a1ce 100644
1419+ --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala
1420+ +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala
1421+ @@ -51,6 +51,11 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite {
1422+ }
1423+ }
1424+
1425+ + // Comet: AQE coalesces post-shuffle partitions by map output size, and Comet's shuffle
1426+ + // writes Arrow IPC blocks whose sizes differ from Spark's, so the expected partition
1427+ + // counts below only hold for Spark's shuffle.
1428+ + private def isCometEnabled: Boolean = org.apache.spark.sql.SparkSession.isCometEnabled
1429+ +
1430+ val numInputPartitions: Int = 10
1431+
1432+ def withSparkSession(
1433+ @@ -163,9 +168,11 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite {
1434+ assert(shuffleReads.isEmpty)
1435+
1436+ case None =>
1437+ - assert(shuffleReads.length === 2)
1438+ - shuffleReads.foreach { read =>
1439+ - assert(read.outputPartitioning.numPartitions === 2)
1440+ + if (!isCometEnabled) {
1441+ + assert(shuffleReads.length === 2)
1442+ + shuffleReads.foreach { read =>
1443+ + assert(read.outputPartitioning.numPartitions === 2)
1444+ + }
1445+ }
1446+ }
1447+ }
1448+ @@ -214,9 +221,11 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite {
1449+ assert(shuffleReads.isEmpty)
1450+
1451+ case None =>
1452+ - assert(shuffleReads.length === 2)
1453+ - shuffleReads.foreach { read =>
1454+ - assert(read.outputPartitioning.numPartitions === 2)
1455+ + if (!isCometEnabled) {
1456+ + assert(shuffleReads.length === 2)
1457+ + shuffleReads.foreach { read =>
1458+ + assert(read.outputPartitioning.numPartitions === 2)
1459+ + }
1460+ }
1461+ }
1462+ }
1463+ @@ -265,9 +274,11 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite {
1464+ assert(shuffleReads.isEmpty)
1465+
1466+ case None =>
1467+ - assert(shuffleReads.length === 2)
1468+ - shuffleReads.foreach { read =>
1469+ - assert(read.outputPartitioning.numPartitions === 3)
1470+ + if (!isCometEnabled) {
1471+ + assert(shuffleReads.length === 2)
1472+ + shuffleReads.foreach { read =>
1473+ + assert(read.outputPartitioning.numPartitions === 3)
1474+ + }
1475+ }
1476+ }
1477+ }
1478+ @@ -412,10 +423,12 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite {
1479+ // aggregate on the other side of the union.
1480+ val finalPlan = resultDf.queryExecution.executedPlan
1481+ .asInstanceOf[AdaptiveSparkPlanExec].executedPlan
1482+ - assert(
1483+ - finalPlan.collect {
1484+ - case r @ CoalescedShuffleRead() => r
1485+ - }.size == 2)
1486+ + if (!isCometEnabled) {
1487+ + assert(
1488+ + finalPlan.collect {
1489+ + case r @ CoalescedShuffleRead() => r
1490+ + }.size == 2)
1491+ + }
1492+ }
1493+ withSparkSession(test, 100, None)
1494+ }
13311495diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala
13321496index 418ca3430bb..eb8267192f8 100644
13331497--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala
@@ -2433,6 +2597,29 @@ index 3a0bd35cb70..b28f06a757f 100644
24332597 withTempPath { workDir =>
24342598 val workDirPath = workDir.getAbsolutePath
24352599 val input = spark.range(5).toDF("id")
2600+ diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
2601+ index 6333808b420..81b2704300c 100644
2602+ --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
2603+ +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
2604+ @@ -21,7 +21,7 @@ import scala.reflect.ClassTag
2605+
2606+ import org.apache.spark.AccumulatorSuite
2607+ import org.apache.spark.internal.config.EXECUTOR_MEMORY
2608+ - import org.apache.spark.sql.{Dataset, QueryTest, Row, SparkSession}
2609+ + import org.apache.spark.sql.{Dataset, IgnoreComet, QueryTest, Row, SparkSession}
2610+ import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BitwiseAnd, BitwiseOr, Cast, Expression, Literal, ShiftLeft}
2611+ import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide}
2612+ import org.apache.spark.sql.catalyst.plans.Inner
2613+ @@ -486,7 +486,8 @@ abstract class BroadcastJoinSuiteBase extends QueryTest with SQLTestUtils
2614+ }
2615+ }
2616+
2617+ - test("broadcast join where streamed side's output partitioning is PartitioningCollection") {
2618+ + test("broadcast join where streamed side's output partitioning is PartitioningCollection",
2619+ + IgnoreComet("Comet replaces the join and shuffle operators this test inspects")) {
2620+ withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "500") {
2621+ val t1 = (0 until 100).map(i => (i % 5, i % 13)).toDF("i1", "j1")
2622+ val t2 = (0 until 100).map(i => (i % 5, i % 14)).toDF("i2", "j2")
24362623diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
24372624index 26e61c6b58d..cb09d7e116a 100644
24382625--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
0 commit comments