@@ -43,7 +43,7 @@ import org.apache.spark.sql.execution.adaptive.ShuffleQueryStageExec
4343import org .apache .spark .sql .execution .exchange .{ENSURE_REQUIREMENTS , ShuffleExchangeExec , ShuffleExchangeLike , ShuffleOrigin }
4444import org .apache .spark .sql .execution .metric .{SQLMetric , SQLMetrics , SQLShuffleReadMetricsReporter , SQLShuffleWriteMetricsReporter }
4545import org .apache .spark .sql .internal .SQLConf
46- import org .apache .spark .sql .types .{ArrayType , BinaryType , BooleanType , ByteType , CalendarIntervalType , DataType , DateType , DecimalType , DoubleType , FloatType , IntegerType , LongType , MapType , ShortType , StringType , StructField , StructType , TimestampNTZType , TimestampType }
46+ import org .apache .spark .sql .types .{ArrayType , BinaryType , BooleanType , ByteType , DataType , DateType , DecimalType , DoubleType , FloatType , IntegerType , LongType , MapType , ShortType , StringType , StructField , StructType , TimestampNTZType , TimestampType }
4747import org .apache .spark .sql .vectorized .ColumnarBatch
4848import org .apache .spark .util .MutablePair
4949import org .apache .spark .util .collection .unsafe .sort .{PrefixComparators , RecordComparator }
@@ -549,33 +549,23 @@ object CometShuffleExchangeExec
549549 }
550550
551551 /**
552- * Whether a shuffle whose child is a Spark row-based plan can convert the child's rows to Arrow
553- * with `CometSparkToColumnarExec` and use native shuffle, instead of the JVM columnar shuffle.
554- * See [[CometConf.COMET_CONVERT_FROM_SHUFFLE_INPUT_ENABLED ]]. The checks are those that
555- * [[shuffleSupported ]] would make for the converted plan, so `CometExecRule` only converts a
556- * shuffle that will become native. Pure: does not tag the node.
552+ * Whether a shuffle that would use the JVM columnar shuffle, because its child is a Spark
553+ * row-based plan, can convert the child's rows to Arrow with `CometSparkToColumnarExec` and use
554+ * native shuffle instead. See [[CometConf.COMET_CONVERT_FROM_SHUFFLE_INPUT_ENABLED ]]. The
555+ * native checks are those that [[shuffleSupported ]] would make for the converted plan, so
556+ * `CometExecRule` only converts a shuffle that will become native. Like [[shuffleSupported ]],
557+ * tags the node when no Comet shuffle can take it.
557558 */
558- def convertsInputForNativeShuffle (s : ShuffleExchangeExec ): Boolean = {
559- val conf = s.conf
560- CometConf .COMET_CONVERT_FROM_SHUFFLE_INPUT_ENABLED .get(conf) &&
561- // A decision an earlier pass recorded stands, as in shuffleSupported.
562- ! hasFallbackReason(s) &&
563- isCometShuffleEnabledReason(s).isEmpty &&
564- CometConf .COMET_SHUFFLE_CONVERT_FROM_SPARK_PLAN_ENABLED .get(conf) &&
565- // Not exercised with Celeborn, which has no JVM columnar shuffle to replace.
566- ! isCometCelebornShuffleManagerEnabled(conf) &&
567- ! isCometPlan(s.child) &&
568- ! s.child.supportsColumnar &&
569- ! isShuffleOperator(s.child) &&
570- ! stageContainsDPPScan(s) &&
571- CometSparkToColumnarExec .isSchemaSupported(s.child.schema, ListBuffer .empty) &&
572- // Arrow holds the time part of an interval in nanoseconds, so the conversion overflows on a
573- // calendar interval with more microseconds than that can hold. The JVM columnar shuffle
574- // leaves calendar intervals to Spark's shuffle as well.
575- ! s.child.schema.existsRecursively(_.isInstanceOf [CalendarIntervalType ]) &&
576- nativeShuffleFailureReasons(s).isEmpty &&
577- ! hashesDifferentlyFromSpark(s)
578- }
559+ def convertsInputForNativeShuffle (s : ShuffleExchangeExec ): Boolean =
560+ CometConf .COMET_CONVERT_FROM_SHUFFLE_INPUT_ENABLED .get(s.conf) &&
561+ ! isCometPlan(s.child) &&
562+ // This also rules out Celeborn, which has no JVM columnar shuffle, and calendar intervals.
563+ // Arrow holds the time part of an interval in nanoseconds, so the conversion would
564+ // overflow on one with more microseconds than that can hold.
565+ shuffleSupported(s).contains(CometColumnarShuffle ) &&
566+ CometSparkToColumnarExec .isSchemaSupported(s.child.schema, ListBuffer .empty) &&
567+ nativeShuffleFailureReasons(s).isEmpty &&
568+ ! hashesDifferentlyFromSpark(s)
579569
580570 /**
581571 * Whether the shuffle hashes into more than one partition a key that native shuffle, reading
0 commit comments