Skip to content

Commit 21651aa

Browse files
committed
refactor: convert only the input of a shuffle that would use the JVM columnar shuffle
convertsInputForNativeShuffle now starts from shuffleSupported, whose checks it restated, and which also rules out Celeborn and calendar intervals. With spark.comet.shuffle.mode=native, a Spark child keeps Spark's shuffle, as the config doc says.
1 parent 73c727c commit 21651aa

3 files changed

Lines changed: 45 additions & 32 deletions

File tree

‎spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala‎

Lines changed: 17 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ import org.apache.spark.sql.execution.adaptive.ShuffleQueryStageExec
4343
import org.apache.spark.sql.execution.exchange.{ENSURE_REQUIREMENTS, ShuffleExchangeExec, ShuffleExchangeLike, ShuffleOrigin}
4444
import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics, SQLShuffleReadMetricsReporter, SQLShuffleWriteMetricsReporter}
4545
import org.apache.spark.sql.internal.SQLConf
46-
import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType, CalendarIntervalType, DataType, DateType, DecimalType, DoubleType, FloatType, IntegerType, LongType, MapType, ShortType, StringType, StructField, StructType, TimestampNTZType, TimestampType}
46+
import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType, DataType, DateType, DecimalType, DoubleType, FloatType, IntegerType, LongType, MapType, ShortType, StringType, StructField, StructType, TimestampNTZType, TimestampType}
4747
import org.apache.spark.sql.vectorized.ColumnarBatch
4848
import org.apache.spark.util.MutablePair
4949
import org.apache.spark.util.collection.unsafe.sort.{PrefixComparators, RecordComparator}
@@ -549,33 +549,23 @@ object CometShuffleExchangeExec
549549
}
550550

551551
/**
552-
* Whether a shuffle whose child is a Spark row-based plan can convert the child's rows to Arrow
553-
* with `CometSparkToColumnarExec` and use native shuffle, instead of the JVM columnar shuffle.
554-
* See [[CometConf.COMET_CONVERT_FROM_SHUFFLE_INPUT_ENABLED]]. The checks are those that
555-
* [[shuffleSupported]] would make for the converted plan, so `CometExecRule` only converts a
556-
* shuffle that will become native. Pure: does not tag the node.
552+
* Whether a shuffle that would use the JVM columnar shuffle, because its child is a Spark
553+
* row-based plan, can convert the child's rows to Arrow with `CometSparkToColumnarExec` and use
554+
* native shuffle instead. See [[CometConf.COMET_CONVERT_FROM_SHUFFLE_INPUT_ENABLED]]. The
555+
* native checks are those that [[shuffleSupported]] would make for the converted plan, so
556+
* `CometExecRule` only converts a shuffle that will become native. Like [[shuffleSupported]],
557+
* tags the node when no Comet shuffle can take it.
557558
*/
558-
def convertsInputForNativeShuffle(s: ShuffleExchangeExec): Boolean = {
559-
val conf = s.conf
560-
CometConf.COMET_CONVERT_FROM_SHUFFLE_INPUT_ENABLED.get(conf) &&
561-
// A decision an earlier pass recorded stands, as in shuffleSupported.
562-
!hasFallbackReason(s) &&
563-
isCometShuffleEnabledReason(s).isEmpty &&
564-
CometConf.COMET_SHUFFLE_CONVERT_FROM_SPARK_PLAN_ENABLED.get(conf) &&
565-
// Not exercised with Celeborn, which has no JVM columnar shuffle to replace.
566-
!isCometCelebornShuffleManagerEnabled(conf) &&
567-
!isCometPlan(s.child) &&
568-
!s.child.supportsColumnar &&
569-
!isShuffleOperator(s.child) &&
570-
!stageContainsDPPScan(s) &&
571-
CometSparkToColumnarExec.isSchemaSupported(s.child.schema, ListBuffer.empty) &&
572-
// Arrow holds the time part of an interval in nanoseconds, so the conversion overflows on a
573-
// calendar interval with more microseconds than that can hold. The JVM columnar shuffle
574-
// leaves calendar intervals to Spark's shuffle as well.
575-
!s.child.schema.existsRecursively(_.isInstanceOf[CalendarIntervalType]) &&
576-
nativeShuffleFailureReasons(s).isEmpty &&
577-
!hashesDifferentlyFromSpark(s)
578-
}
559+
def convertsInputForNativeShuffle(s: ShuffleExchangeExec): Boolean =
560+
CometConf.COMET_CONVERT_FROM_SHUFFLE_INPUT_ENABLED.get(s.conf) &&
561+
!isCometPlan(s.child) &&
562+
// This also rules out Celeborn, which has no JVM columnar shuffle, and calendar intervals.
563+
// Arrow holds the time part of an interval in nanoseconds, so the conversion would
564+
// overflow on one with more microseconds than that can hold.
565+
shuffleSupported(s).contains(CometColumnarShuffle) &&
566+
CometSparkToColumnarExec.isSchemaSupported(s.child.schema, ListBuffer.empty) &&
567+
nativeShuffleFailureReasons(s).isEmpty &&
568+
!hashesDifferentlyFromSpark(s)
579569

580570
/**
581571
* Whether the shuffle hashes into more than one partition a key that native shuffle, reading

‎spark/src/test/scala/org/apache/comet/exec/CometShuffleInputConversionSuite.scala‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -317,11 +317,19 @@ class CometShuffleInputConversionSuite extends CometTestBase {
317317
assert(convertedShuffles(plan).length == 1, plan)
318318
}
319319

320-
convertTest("the JVM shuffle mode keeps the JVM columnar shuffle") {
321-
withSQLConf(CometConf.COMET_SHUFFLE_MODE.key -> "jvm") {
322-
val (_, plan) = checkSparkAnswer(rowsDf().repartition(10, col("k")).groupBy("k").count())
323-
assert(conversions(plan).isEmpty, plan)
324-
assert(cometShuffles(plan).forall(_.shuffleType == CometColumnarShuffle), plan)
320+
convertTest("the conversion only takes over from the JVM columnar shuffle") {
321+
// In `jvm` mode the shuffle keeps the JVM columnar shuffle. In `native` mode, where Comet has
322+
// no shuffle for a Spark operator's rows, it keeps Spark's shuffle.
323+
Seq("jvm" -> Seq(CometColumnarShuffle), "native" -> Seq.empty).foreach {
324+
case (mode, shuffleTypes) =>
325+
withSQLConf(CometConf.COMET_SHUFFLE_MODE.key -> mode) {
326+
val (_, plan) =
327+
checkSparkAnswer(rowsDf().repartition(10, col("k")).groupBy("k").count())
328+
assert(conversions(plan).isEmpty, s"$mode:\n$plan")
329+
assert(
330+
cometShuffles(plan).map(_.shuffleType).distinct == shuffleTypes,
331+
s"$mode:\n$plan")
332+
}
325333
}
326334
}
327335

‎spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShufflePlanningSuite.scala‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,21 @@ class CometCelebornShufflePlanningSuite extends CometTestBase {
265265
}
266266
}
267267

268+
test("the shuffle input conversion leaves a Spark child to Spark's shuffle") {
269+
// The conversion only takes over from the JVM columnar shuffle, which Celeborn does not
270+
// have, and Celeborn's native shuffle needs a native child.
271+
withSQLConf(
272+
CometConf.COMET_SHUFFLE_MODE.key -> "native",
273+
CometConf.COMET_CONVERT_FROM_SHUFFLE_INPUT_ENABLED.key -> "true") {
274+
val leaf = sparkLeaf()
275+
val exchange = ShuffleExchangeExec(HashPartitioning(leaf.output, 2), leaf)
276+
assert(!CometShuffleExchangeExec.convertsInputForNativeShuffle(exchange))
277+
val planned = CometExecRule(spark)(exchange)
278+
assert(planned.isInstanceOf[ShuffleExchangeExec], planned)
279+
assert(collect(planned) { case c: CometSparkToColumnarExec => c }.isEmpty, planned)
280+
}
281+
}
282+
268283
test("fallback is sticky after AQE reshapes the child or native mode becomes available") {
269284
val child = nativeChild()
270285
val exchange = ShuffleExchangeExec(SinglePartition, child)

0 commit comments

Comments
 (0)