Skip to content

Commit 24c0270

Browse files
committed
Merge apache/main into fix/6613-array-null-short-circuit
# Conflicts: # spark/src/main/scala/org/apache/comet/serde/arrays.scala
2 parents 9c9ff57 + bfacc43 commit 24c0270

114 files changed

Lines changed: 9353 additions & 1906 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/pr_build_linux.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -460,6 +460,7 @@ jobs:
460460
value: |
461461
org.apache.comet.parquet.CometParquetWriterSuite
462462
org.apache.comet.parquet.CometEmptyRelationParquetWriterSuite
463+
org.apache.spark.sql.comet.CometNativeWriteSuite
463464
org.apache.comet.parquet.ParquetReadV1Suite
464465
org.apache.comet.parquet.ParquetTimestampLtzAsNtzSuite
465466
org.apache.spark.sql.comet.ParquetEncryptionITCase
@@ -504,6 +505,7 @@ jobs:
504505
org.apache.comet.exec.CometShuffleManagerSuite
505506
org.apache.comet.exec.DisableAQECometShuffleSuite
506507
org.apache.spark.shuffle.comet.CometUnboundedShuffleMemoryAllocatorSuite
508+
org.apache.spark.shuffle.comet.CometUnifiedShuffleMemoryAllocatorSuite
507509
org.apache.spark.shuffle.sort.SpillSorterSuite
508510
- name: "exec"
509511
value: |

‎.github/workflows/pr_build_macos.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,7 @@ jobs:
164164
value: |
165165
org.apache.comet.parquet.CometParquetWriterSuite
166166
org.apache.comet.parquet.CometEmptyRelationParquetWriterSuite
167+
org.apache.spark.sql.comet.CometNativeWriteSuite
167168
org.apache.comet.parquet.ParquetReadV1Suite
168169
org.apache.comet.parquet.ParquetTimestampLtzAsNtzSuite
169170
org.apache.spark.sql.comet.ParquetEncryptionITCase
@@ -208,6 +209,7 @@ jobs:
208209
org.apache.comet.exec.CometShuffleManagerSuite
209210
org.apache.comet.exec.DisableAQECometShuffleSuite
210211
org.apache.spark.shuffle.comet.CometUnboundedShuffleMemoryAllocatorSuite
212+
org.apache.spark.shuffle.comet.CometUnifiedShuffleMemoryAllocatorSuite
211213
org.apache.spark.shuffle.sort.SpillSorterSuite
212214
- name: "exec"
213215
value: |

‎.github/workflows/spark_sql_writer_tests.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,13 @@ jobs:
115115
spark-short-version: ${{ inputs.spark-version }}
116116
skip-native-build: true
117117

118+
- name: Pre-compile Spark SQL test classes
119+
run: |
120+
cd apache-spark
121+
rm -rf /root/.m2/repository/org/apache/parquet
122+
NOLINT_ON_COMPILE=true build/sbt -Dsbt.log.noformat=true -mem 3072 \
123+
'sql/Test/compile'
124+
118125
- name: Run Parquet writer tests
119126
run: |
120127
cd apache-spark

‎benchmarks/pyspark/run_all_benchmarks.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ $SPARK_HOME/bin/spark-submit \
6767
# Run Comet JVM shuffle
6868
echo ""
6969
echo ">>> Running COMET JVM shuffle benchmark..."
70+
# maxBuildSize=-1 measures the hash join alone: no build-side size limit on the rewrite.
7071
$SPARK_HOME/bin/spark-submit \
7172
--master "$SPARK_MASTER" \
7273
--executor-memory "$EXECUTOR_MEMORY" \
@@ -86,6 +87,7 @@ $SPARK_HOME/bin/spark-submit \
8687
--conf spark.comet.shuffle.mode=jvm \
8788
--conf spark.comet.shuffle.mode=jvm \
8889
--conf spark.comet.exec.replaceSortMergeJoin=true \
90+
--conf spark.comet.exec.forceShuffledHashJoin.maxBuildSize=-1 \
8991
--conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \
9092
--conf spark.sql.extensions=org.apache.comet.CometSparkSessionExtensions \
9193
--conf spark.comet.cast.allowIncompatible=true \
@@ -96,6 +98,7 @@ $SPARK_HOME/bin/spark-submit \
9698
# Run Comet Native shuffle
9799
echo ""
98100
echo ">>> Running COMET NATIVE shuffle benchmark..."
101+
# maxBuildSize=-1 measures the hash join alone: no build-side size limit on the rewrite.
99102
$SPARK_HOME/bin/spark-submit \
100103
--master "$SPARK_MASTER" \
101104
--executor-memory "$EXECUTOR_MEMORY" \
@@ -114,6 +117,7 @@ $SPARK_HOME/bin/spark-submit \
114117
--conf spark.comet.explain.fallback.enabled=true \
115118
--conf spark.comet.shuffle.mode=native \
116119
--conf spark.comet.exec.replaceSortMergeJoin=true \
120+
--conf spark.comet.exec.forceShuffledHashJoin.maxBuildSize=-1 \
117121
--conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \
118122
--conf spark.sql.extensions=org.apache.comet.CometSparkSessionExtensions \
119123
--conf spark.comet.cast.allowIncompatible=true \

‎benchmarks/tpc/engines/comet-hashjoin.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,6 @@ driver_class_path = ["$COMET_JAR"]
3131
"spark.plugins" = "org.apache.spark.CometPlugin"
3232
"spark.shuffle.manager" = "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager"
3333
"spark.comet.exec.replaceSortMergeJoin" = "true"
34+
# Measure the hash join alone: no build-side size limit on the rewrite.
35+
"spark.comet.exec.forceShuffledHashJoin.maxBuildSize" = "-1"
3436
"spark.comet.expression.Cast.allowIncompatible" = "true"

‎benchmarks/tpc/engines/comet-iceberg-hashjoin.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ driver_class_path = ["$COMET_JAR", "$ICEBERG_JAR"]
3434
"spark.plugins" = "org.apache.spark.CometPlugin"
3535
"spark.shuffle.manager" = "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager"
3636
"spark.comet.exec.replaceSortMergeJoin" = "true"
37+
# Measure the hash join alone: no build-side size limit on the rewrite.
38+
"spark.comet.exec.forceShuffledHashJoin.maxBuildSize" = "-1"
3739
"spark.comet.expression.Cast.allowIncompatible" = "true"
3840
"spark.comet.enabled" = "true"
3941
"spark.comet.exec.enabled" = "true"

0 commit comments

Comments
 (0)