Skip to content

Commit 7047b8a

Browse files
authored
Merge branch 'main' into feat/cache-enabled-by-default
2 parents 83ca07a + 00a4b42 commit 7047b8a

162 files changed

Lines changed: 11330 additions & 1702 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/pr_build_linux.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -460,6 +460,7 @@ jobs:
460460
value: |
461461
org.apache.comet.parquet.CometParquetWriterSuite
462462
org.apache.comet.parquet.CometEmptyRelationParquetWriterSuite
463+
org.apache.spark.sql.comet.CometNativeWriteSuite
463464
org.apache.comet.parquet.ParquetReadV1Suite
464465
org.apache.comet.parquet.ParquetTimestampLtzAsNtzSuite
465466
org.apache.spark.sql.comet.ParquetEncryptionITCase
@@ -486,6 +487,7 @@ jobs:
486487
org.apache.comet.exec.CometShuffle4_0Suite
487488
org.apache.comet.exec.CometNativeColumnarToRowSuite
488489
org.apache.comet.exec.CometNativeShuffleSuite
490+
org.apache.comet.exec.CometShuffleInputConversionSuite
489491
org.apache.comet.shuffle.CelebornShufflePartitionPusherSuite
490492
org.apache.spark.sql.comet.execution.shuffle.CometCelebornShuffleManagerSuite
491493
org.apache.spark.sql.comet.execution.shuffle.CometCelebornNativeShuffleWriterSuite
@@ -503,6 +505,7 @@ jobs:
503505
org.apache.comet.exec.CometShuffleManagerSuite
504506
org.apache.comet.exec.DisableAQECometShuffleSuite
505507
org.apache.spark.shuffle.comet.CometUnboundedShuffleMemoryAllocatorSuite
508+
org.apache.spark.shuffle.comet.CometUnifiedShuffleMemoryAllocatorSuite
506509
org.apache.spark.shuffle.sort.SpillSorterSuite
507510
- name: "exec"
508511
value: |

‎.github/workflows/pr_build_macos.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,7 @@ jobs:
164164
value: |
165165
org.apache.comet.parquet.CometParquetWriterSuite
166166
org.apache.comet.parquet.CometEmptyRelationParquetWriterSuite
167+
org.apache.spark.sql.comet.CometNativeWriteSuite
167168
org.apache.comet.parquet.ParquetReadV1Suite
168169
org.apache.comet.parquet.ParquetTimestampLtzAsNtzSuite
169170
org.apache.spark.sql.comet.ParquetEncryptionITCase
@@ -190,6 +191,7 @@ jobs:
190191
org.apache.comet.exec.CometShuffle4_0Suite
191192
org.apache.comet.exec.CometNativeColumnarToRowSuite
192193
org.apache.comet.exec.CometNativeShuffleSuite
194+
org.apache.comet.exec.CometShuffleInputConversionSuite
193195
org.apache.comet.shuffle.CelebornShufflePartitionPusherSuite
194196
org.apache.spark.sql.comet.execution.shuffle.CometCelebornShuffleManagerSuite
195197
org.apache.spark.sql.comet.execution.shuffle.CometCelebornNativeShuffleWriterSuite
@@ -207,6 +209,7 @@ jobs:
207209
org.apache.comet.exec.CometShuffleManagerSuite
208210
org.apache.comet.exec.DisableAQECometShuffleSuite
209211
org.apache.spark.shuffle.comet.CometUnboundedShuffleMemoryAllocatorSuite
212+
org.apache.spark.shuffle.comet.CometUnifiedShuffleMemoryAllocatorSuite
210213
org.apache.spark.shuffle.sort.SpillSorterSuite
211214
- name: "exec"
212215
value: |

‎.github/workflows/spark_sql_writer_tests.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,13 @@ jobs:
115115
spark-short-version: ${{ inputs.spark-version }}
116116
skip-native-build: true
117117

118+
- name: Pre-compile Spark SQL test classes
119+
run: |
120+
cd apache-spark
121+
rm -rf /root/.m2/repository/org/apache/parquet
122+
NOLINT_ON_COMPILE=true build/sbt -Dsbt.log.noformat=true -mem 3072 \
123+
'sql/Test/compile'
124+
118125
- name: Run Parquet writer tests
119126
run: |
120127
cd apache-spark

‎benchmarks/pyspark/run_all_benchmarks.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ $SPARK_HOME/bin/spark-submit \
6767
# Run Comet JVM shuffle
6868
echo ""
6969
echo ">>> Running COMET JVM shuffle benchmark..."
70+
# maxBuildSize=-1 measures the hash join alone: no build-side size limit on the rewrite.
7071
$SPARK_HOME/bin/spark-submit \
7172
--master "$SPARK_MASTER" \
7273
--executor-memory "$EXECUTOR_MEMORY" \
@@ -86,6 +87,7 @@ $SPARK_HOME/bin/spark-submit \
8687
--conf spark.comet.shuffle.mode=jvm \
8788
--conf spark.comet.shuffle.mode=jvm \
8889
--conf spark.comet.exec.replaceSortMergeJoin=true \
90+
--conf spark.comet.exec.forceShuffledHashJoin.maxBuildSize=-1 \
8991
--conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \
9092
--conf spark.sql.extensions=org.apache.comet.CometSparkSessionExtensions \
9193
--conf spark.comet.cast.allowIncompatible=true \
@@ -96,6 +98,7 @@ $SPARK_HOME/bin/spark-submit \
9698
# Run Comet Native shuffle
9799
echo ""
98100
echo ">>> Running COMET NATIVE shuffle benchmark..."
101+
# maxBuildSize=-1 measures the hash join alone: no build-side size limit on the rewrite.
99102
$SPARK_HOME/bin/spark-submit \
100103
--master "$SPARK_MASTER" \
101104
--executor-memory "$EXECUTOR_MEMORY" \
@@ -114,6 +117,7 @@ $SPARK_HOME/bin/spark-submit \
114117
--conf spark.comet.explain.fallback.enabled=true \
115118
--conf spark.comet.shuffle.mode=native \
116119
--conf spark.comet.exec.replaceSortMergeJoin=true \
120+
--conf spark.comet.exec.forceShuffledHashJoin.maxBuildSize=-1 \
117121
--conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \
118122
--conf spark.sql.extensions=org.apache.comet.CometSparkSessionExtensions \
119123
--conf spark.comet.cast.allowIncompatible=true \

‎benchmarks/tpc/engines/comet-hashjoin.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,6 @@ driver_class_path = ["$COMET_JAR"]
3131
"spark.plugins" = "org.apache.spark.CometPlugin"
3232
"spark.shuffle.manager" = "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager"
3333
"spark.comet.exec.replaceSortMergeJoin" = "true"
34+
# Measure the hash join alone: no build-side size limit on the rewrite.
35+
"spark.comet.exec.forceShuffledHashJoin.maxBuildSize" = "-1"
3436
"spark.comet.expression.Cast.allowIncompatible" = "true"

‎benchmarks/tpc/engines/comet-iceberg-hashjoin.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ driver_class_path = ["$COMET_JAR", "$ICEBERG_JAR"]
3434
"spark.plugins" = "org.apache.spark.CometPlugin"
3535
"spark.shuffle.manager" = "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager"
3636
"spark.comet.exec.replaceSortMergeJoin" = "true"
37+
# Measure the hash join alone: no build-side size limit on the rewrite.
38+
"spark.comet.exec.forceShuffledHashJoin.maxBuildSize" = "-1"
3739
"spark.comet.expression.Cast.allowIncompatible" = "true"
3840
"spark.comet.enabled" = "true"
3941
"spark.comet.exec.enabled" = "true"

‎dev/diffs/4.2.0.diff‎

Lines changed: 4 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -550,7 +550,7 @@ index 085dbcd8046..8cee9832317 100644
550550
assert(subqueryInMemoryTableScan.size == 1)
551551
checkAnswer(cteInSubquery, Row(1) :: Nil)
552552
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
553-
index 5b8154d2900..eb3de47f387 100644
553+
index 5b8154d2900..d42374b127f 100644
554554
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
555555
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
556556
@@ -30,8 +30,9 @@ import org.apache.spark.sql.catalyst.util.AUTO_GENERATED_ALIAS
@@ -564,18 +564,7 @@ index 5b8154d2900..eb3de47f387 100644
564564
import org.apache.spark.sql.expressions.Window
565565
import org.apache.spark.sql.functions._
566566
import org.apache.spark.sql.internal.SQLConf
567-
@@ -667,7 +668,9 @@ class DataFrameAggregateSuite extends SharedSparkSession
568-
df.selectExpr("sort_array(collect_set(b) RESPECT NULLS)"), Seq(Row(Seq(null, 2))))
569-
}
570-
571-
- test("SPARK-57298: collect_set normalizes NaN and -0.0 for floating-point types") {
572-
+ test("SPARK-57298: collect_set normalizes NaN and -0.0 for floating-point types",
573-
+ // Comet's native collect_set does not yet normalize NaN / -0.0.
574-
+ IgnoreComet("https://github.com/apache/datafusion-comet/issues/4966")) {
575-
checkAnswer(
576-
sql("SELECT collect_set(v) FROM VALUES (double('NaN')), (double('NaN')) AS t(v)"),
577-
Row(Seq(Double.NaN)))
578-
@@ -932,7 +935,9 @@ class DataFrameAggregateSuite extends SharedSparkSession
567+
@@ -932,7 +933,9 @@ class DataFrameAggregateSuite extends SharedSparkSession
579568
case _ => false
580569
}.isDefined)
581570
} else {
@@ -586,7 +575,7 @@ index 5b8154d2900..eb3de47f387 100644
586575
}
587576

588577
// test case for ObjectHashAggregate and SortAggregate
589-
@@ -941,7 +946,8 @@ class DataFrameAggregateSuite extends SharedSparkSession
578+
@@ -941,7 +944,8 @@ class DataFrameAggregateSuite extends SharedSparkSession
590579
val objHashAggOrSortAggPlan =
591580
stripAQEPlan(objHashAggOrSortAggDF.queryExecution.executedPlan)
592581
if (useObjectHashAgg) {
@@ -596,7 +585,7 @@ index 5b8154d2900..eb3de47f387 100644
596585
} else {
597586
assert(objHashAggOrSortAggPlan.isInstanceOf[SortAggregateExec])
598587
}
599-
@@ -991,12 +997,12 @@ class DataFrameAggregateSuite extends SharedSparkSession
588+
@@ -991,12 +995,12 @@ class DataFrameAggregateSuite extends SharedSparkSession
600589
assert(sortAggPlans.isEmpty)
601590

602591
val objHashAggPlans = collect(aggPlan) {

0 commit comments

Comments
 (0)