Skip to content

Commit 01d9a57

Browse files
committed
Merge remote-tracking branch 'upstream/main' into HEAD
# Conflicts: # docs/source/user-guide/latest/compatibility/floating-point.md # spark/src/test/scala/org/apache/comet/CometFloatSemanticsSuite.scala
2 parents f9014b1 + a2d7370 commit 01d9a57

54 files changed

Lines changed: 4194 additions & 369 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎dev/diffs/3.4.3.diff‎

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,48 @@ index d3544881af1..0c0618a5e80 100644
3737
</dependencies>
3838
</dependencyManagement>
3939

40+
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
41+
index 1cbd9a61289..74aa1f684c9 100644
42+
--- a/project/SparkBuild.scala
43+
+++ b/project/SparkBuild.scala
44+
@@ -445,9 +445,11 @@ object SparkBuild extends PomBuild {
45+
46+
/* Spark SQL Core console settings */
47+
enable(SQL.settings)(sql)
48+
+ enable(CometTestSettings.settings)(sql)
49+
50+
/* Hive console settings */
51+
enable(Hive.settings)(hive)
52+
+ enable(CometTestSettings.settings)(hive)
53+
54+
enable(SparkConnectCommon.settings)(connectCommon)
55+
enable(SparkConnect.settings)(connect)
56+
@@ -1183,6 +1185,25 @@ object SQL {
57+
)
58+
}
59+
60+
+/**
61+
+ * Comet disables itself when its shuffle manager is not registered. When Comet is enabled, make
62+
+ * that shuffle manager the default for every test SparkConf in the modules that have Comet on
63+
+ * their classpath, so that suites which build their own SparkSession or SparkContext run Comet
64+
+ * too, not just the ones that go through SharedSparkSession or TestHive.
65+
+ */
66+
+object CometTestSettings {
67+
+ private val cometEnabled =
68+
+ sys.env.get("ENABLE_COMET").forall(v => v == "1" || v.equalsIgnoreCase("true"))
69+
+
70+
+ lazy val settings: Seq[Setting[_]] =
71+
+ if (cometEnabled && !sys.props.contains("spark.shuffle.manager")) {
72+
+ Seq((Test / javaOptions) +=
73+
+ "-Dspark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager")
74+
+ } else {
75+
+ Seq.empty
76+
+ }
77+
+}
78+
+
79+
object Hive {
80+
81+
lazy val settings = Seq(
4082
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
4183
index b386d135da1..46449e3f3f1 100644
4284
--- a/sql/core/pom.xml
@@ -1056,6 +1098,50 @@ index 2dabcf01be7..8fcec0d1ce4 100644
10561098
}
10571099
}
10581100
}
1101+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
1102+
index 48ad10992c5..a164e273b76 100644
1103+
--- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
1104+
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
1105+
@@ -165,7 +165,15 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper {
1106+
}
1107+
}
1108+
1109+
+ // Comet: Comet's own columnar rules take over these plans, so the rules the tests below
1110+
+ // inject never see the Spark operators they act on or check for.
1111+
+ private def assumeCometDisabled(): Unit = {
1112+
+ assume(!org.apache.spark.sql.SparkSession.isCometEnabled,
1113+
+ "Skipped when Comet is enabled: Comet replaces the operators the injected rules act on")
1114+
+ }
1115+
+
1116+
test("inject adaptive query prep rule") {
1117+
+ assumeCometDisabled()
1118+
val extensions = create { extensions =>
1119+
// inject rule that will run during AQE query stage preparation and will add custom tags
1120+
// to the plan
1121+
@@ -259,6 +267,7 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper {
1122+
}
1123+
1124+
private def testInjectColumnar(enableAQE: Boolean): Unit = {
1125+
+ assumeCometDisabled()
1126+
def collectPlanSteps(plan: SparkPlan): Seq[Int] = plan match {
1127+
case a: AdaptiveSparkPlanExec =>
1128+
assert(a.toString.startsWith("AdaptiveSparkPlan isFinalPlan=true"))
1129+
@@ -314,6 +323,7 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper {
1130+
}
1131+
1132+
test("reset column vectors") {
1133+
+ assumeCometDisabled()
1134+
val session = SparkSession.builder()
1135+
.master("local[1]")
1136+
.config(COLUMN_BATCH_SIZE.key, 2)
1137+
@@ -482,6 +492,7 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper {
1138+
}
1139+
1140+
test("SPARK-38697: Extend SparkSessionExtensions to inject rules into AQE Optimizer") {
1141+
+ assumeCometDisabled()
1142+
def executedPlan(df: Dataset[java.lang.Long]): SparkPlan = {
1143+
assert(df.queryExecution.executedPlan.isInstanceOf[AdaptiveSparkPlanExec])
1144+
df.queryExecution.executedPlan.asInstanceOf[AdaptiveSparkPlanExec].executedPlan
10591145
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
10601146
index 18123a4d6ec..0fe185baa33 100644
10611147
--- a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
@@ -1328,6 +1414,84 @@ index c0ec8a58bd5..4e8bc6ed3c5 100644
13281414

13291415
// Fail to read ancient datetime values.
13301416
withSQLConf(SQLConf.PARQUET_REBASE_MODE_IN_READ.key -> EXCEPTION.toString) {
1417+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala
1418+
index 24a98dd83f3..4db28a7a1ce 100644
1419+
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala
1420+
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala
1421+
@@ -51,6 +51,11 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite {
1422+
}
1423+
}
1424+
1425+
+ // Comet: AQE coalesces post-shuffle partitions by map output size, and Comet's shuffle
1426+
+ // writes Arrow IPC blocks whose sizes differ from Spark's, so the expected partition
1427+
+ // counts below only hold for Spark's shuffle.
1428+
+ private def isCometEnabled: Boolean = org.apache.spark.sql.SparkSession.isCometEnabled
1429+
+
1430+
val numInputPartitions: Int = 10
1431+
1432+
def withSparkSession(
1433+
@@ -163,9 +168,11 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite {
1434+
assert(shuffleReads.isEmpty)
1435+
1436+
case None =>
1437+
- assert(shuffleReads.length === 2)
1438+
- shuffleReads.foreach { read =>
1439+
- assert(read.outputPartitioning.numPartitions === 2)
1440+
+ if (!isCometEnabled) {
1441+
+ assert(shuffleReads.length === 2)
1442+
+ shuffleReads.foreach { read =>
1443+
+ assert(read.outputPartitioning.numPartitions === 2)
1444+
+ }
1445+
}
1446+
}
1447+
}
1448+
@@ -214,9 +221,11 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite {
1449+
assert(shuffleReads.isEmpty)
1450+
1451+
case None =>
1452+
- assert(shuffleReads.length === 2)
1453+
- shuffleReads.foreach { read =>
1454+
- assert(read.outputPartitioning.numPartitions === 2)
1455+
+ if (!isCometEnabled) {
1456+
+ assert(shuffleReads.length === 2)
1457+
+ shuffleReads.foreach { read =>
1458+
+ assert(read.outputPartitioning.numPartitions === 2)
1459+
+ }
1460+
}
1461+
}
1462+
}
1463+
@@ -265,9 +274,11 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite {
1464+
assert(shuffleReads.isEmpty)
1465+
1466+
case None =>
1467+
- assert(shuffleReads.length === 2)
1468+
- shuffleReads.foreach { read =>
1469+
- assert(read.outputPartitioning.numPartitions === 3)
1470+
+ if (!isCometEnabled) {
1471+
+ assert(shuffleReads.length === 2)
1472+
+ shuffleReads.foreach { read =>
1473+
+ assert(read.outputPartitioning.numPartitions === 3)
1474+
+ }
1475+
}
1476+
}
1477+
}
1478+
@@ -412,10 +423,12 @@ class CoalesceShufflePartitionsSuite extends SparkFunSuite {
1479+
// aggregate on the other side of the union.
1480+
val finalPlan = resultDf.queryExecution.executedPlan
1481+
.asInstanceOf[AdaptiveSparkPlanExec].executedPlan
1482+
- assert(
1483+
- finalPlan.collect {
1484+
- case r @ CoalescedShuffleRead() => r
1485+
- }.size == 2)
1486+
+ if (!isCometEnabled) {
1487+
+ assert(
1488+
+ finalPlan.collect {
1489+
+ case r @ CoalescedShuffleRead() => r
1490+
+ }.size == 2)
1491+
+ }
1492+
}
1493+
withSparkSession(test, 100, None)
1494+
}
13311495
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala
13321496
index 418ca3430bb..eb8267192f8 100644
13331497
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala
@@ -2433,6 +2597,29 @@ index 3a0bd35cb70..b28f06a757f 100644
24332597
withTempPath { workDir =>
24342598
val workDirPath = workDir.getAbsolutePath
24352599
val input = spark.range(5).toDF("id")
2600+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
2601+
index 6333808b420..81b2704300c 100644
2602+
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
2603+
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
2604+
@@ -21,7 +21,7 @@ import scala.reflect.ClassTag
2605+
2606+
import org.apache.spark.AccumulatorSuite
2607+
import org.apache.spark.internal.config.EXECUTOR_MEMORY
2608+
-import org.apache.spark.sql.{Dataset, QueryTest, Row, SparkSession}
2609+
+import org.apache.spark.sql.{Dataset, IgnoreComet, QueryTest, Row, SparkSession}
2610+
import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BitwiseAnd, BitwiseOr, Cast, Expression, Literal, ShiftLeft}
2611+
import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide}
2612+
import org.apache.spark.sql.catalyst.plans.Inner
2613+
@@ -486,7 +486,8 @@ abstract class BroadcastJoinSuiteBase extends QueryTest with SQLTestUtils
2614+
}
2615+
}
2616+
2617+
- test("broadcast join where streamed side's output partitioning is PartitioningCollection") {
2618+
+ test("broadcast join where streamed side's output partitioning is PartitioningCollection",
2619+
+ IgnoreComet("Comet replaces the join and shuffle operators this test inspects")) {
2620+
withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "500") {
2621+
val t1 = (0 until 100).map(i => (i % 5, i % 13)).toDF("i1", "j1")
2622+
val t2 = (0 until 100).map(i => (i % 5, i % 14)).toDF("i2", "j2")
24362623
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
24372624
index 26e61c6b58d..cb09d7e116a 100644
24382625
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala

0 commit comments

Comments
 (0)