@@ -67,6 +67,7 @@ $SPARK_HOME/bin/spark-submit \
6767# Run Comet JVM shuffle
6868echo " "
6969echo " >>> Running COMET JVM shuffle benchmark..."
70+ # maxBuildSize=-1 measures the hash join alone: no build-side size limit on the rewrite.
7071$SPARK_HOME /bin/spark-submit \
7172 --master " $SPARK_MASTER " \
7273 --executor-memory " $EXECUTOR_MEMORY " \
@@ -86,6 +87,7 @@ $SPARK_HOME/bin/spark-submit \
8687 --conf spark.comet.shuffle.mode=jvm \
8788 --conf spark.comet.shuffle.mode=jvm \
8889 --conf spark.comet.exec.replaceSortMergeJoin=true \
90+ --conf spark.comet.exec.forceShuffledHashJoin.maxBuildSize=-1 \
8991 --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \
9092 --conf spark.sql.extensions=org.apache.comet.CometSparkSessionExtensions \
9193 --conf spark.comet.cast.allowIncompatible=true \
@@ -96,6 +98,7 @@ $SPARK_HOME/bin/spark-submit \
9698# Run Comet Native shuffle
9799echo " "
98100echo " >>> Running COMET NATIVE shuffle benchmark..."
101+ # maxBuildSize=-1 measures the hash join alone: no build-side size limit on the rewrite.
99102$SPARK_HOME /bin/spark-submit \
100103 --master " $SPARK_MASTER " \
101104 --executor-memory " $EXECUTOR_MEMORY " \
@@ -114,6 +117,7 @@ $SPARK_HOME/bin/spark-submit \
114117 --conf spark.comet.explain.fallback.enabled=true \
115118 --conf spark.comet.shuffle.mode=native \
116119 --conf spark.comet.exec.replaceSortMergeJoin=true \
120+ --conf spark.comet.exec.forceShuffledHashJoin.maxBuildSize=-1 \
117121 --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \
118122 --conf spark.sql.extensions=org.apache.comet.CometSparkSessionExtensions \
119123 --conf spark.comet.cast.allowIncompatible=true \
0 commit comments