Skip to content

Commit 1ec0e66

Browse files
committed
fix: keep wide-decimal shuffles above a typed Dataset conversion on JVM shuffle
A typed Dataset conversion moves the shuffle above it from Comet's columnar shuffle, which partitions with Spark's hash, to native shuffle. Native shuffle hashes decimals wider than 18 digits differently from Spark (#5994), so a join on such keys with an input that stays on columnar shuffle, for example one with an array<int> column the conversion declines, put matching keys in different partitions and returned 9 rows instead of 100. Such a shuffle now stays on columnar shuffle unless it has one partition, the same rule #6005 proposes for every native shuffle. Also declare the benchmark's row count as a Long, which the strict Spark 3.5 compile requires.
1 parent 2b87063 commit 1ec0e66

4 files changed

Lines changed: 74 additions & 4 deletions

File tree

‎docs/source/contributor-guide/native_shuffle.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,12 @@ Native shuffle (`CometExchange`) is selected when all of the following condition
7171
Spark's `mapsort` normalization makes physical entry order irrelevant. A collated string at any
7272
depth still disqualifies the key. The config defaults to `false` pending measurement of the
7373
nested hashing paths, so by default a complex hash key falls back to JVM shuffle.
74+
- A hash key that is or contains a decimal wider than 18 digits stays on JVM shuffle when the
75+
shuffle's stage starts at a typed `Dataset` conversion
76+
(`spark.comet.convert.typedDataset.enabled`) and the shuffle has more than one partition.
77+
Native shuffle hashes such decimals differently from Spark
78+
([#5994](https://github.com/apache/datafusion-comet/issues/5994)). Without the conversion
79+
this shuffle would have used JVM shuffle, and a join partner may still use it.
7480

7581
## Architecture
7682

‎spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala‎

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,11 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, BoundReference, Exp
3434
import org.apache.spark.sql.catalyst.expressions.codegen.LazilyGeneratedOrdering
3535
import org.apache.spark.sql.catalyst.plans.logical.Statistics
3636
import org.apache.spark.sql.catalyst.plans.physical._
37-
import org.apache.spark.sql.comet.{CometFilterExec, CometMetricNode, CometNativeExec, CometNativeScanExec, CometPlan, CometProjectExec, CometSinkPlaceHolder, NativeExecContext}
37+
import org.apache.spark.sql.comet.{CometFilterExec, CometMetricNode, CometNativeExec, CometNativeScanExec, CometPlan, CometProjectExec, CometScanWrapper, CometSinkPlaceHolder, CometSparkToColumnarExec, NativeExecContext}
3838
import org.apache.spark.sql.comet.execution.arrow.CometArrowStream
3939
import org.apache.spark.sql.execution._
40-
import org.apache.spark.sql.execution.adaptive.ShuffleQueryStageExec
41-
import org.apache.spark.sql.execution.exchange.{ENSURE_REQUIREMENTS, ShuffleExchangeExec, ShuffleExchangeLike, ShuffleOrigin}
40+
import org.apache.spark.sql.execution.adaptive.{QueryStageExec, ShuffleQueryStageExec}
41+
import org.apache.spark.sql.execution.exchange.{ENSURE_REQUIREMENTS, Exchange, ShuffleExchangeExec, ShuffleExchangeLike, ShuffleOrigin}
4242
import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics, SQLShuffleReadMetricsReporter, SQLShuffleWriteMetricsReporter}
4343
import org.apache.spark.sql.internal.SQLConf
4444
import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType, DataType, DateType, DecimalType, DoubleType, FloatType, IntegerType, LongType, MapType, ShortType, StringType, StructField, StructType, TimestampNTZType, TimestampType}
@@ -524,6 +524,29 @@ object CometShuffleExchangeExec
524524
None
525525
}
526526

527+
/** Whether `dt` is, or contains, a decimal that native shuffle does not hash as Spark does. */
528+
private def hasWideDecimal(dt: DataType): Boolean = dt match {
529+
case d: DecimalType => d.precision > 18
530+
case StructType(fields) => fields.exists(f => hasWideDecimal(f.dataType))
531+
case ArrayType(elementType, _) => hasWideDecimal(elementType)
532+
case MapType(keyType, valueType, _) => hasWideDecimal(keyType) || hasWideDecimal(valueType)
533+
case _ => false
534+
}
535+
536+
/**
537+
* Whether the stage feeding a shuffle starts at a typed Dataset conversion (see
538+
* [[CometConf.COMET_CONVERT_FROM_TYPED_DATASET_ENABLED]]). `CometExecRule` decides the shuffle
539+
* before it removes its placeholders, so the conversion is still inside its `CometScanWrapper`.
540+
*/
541+
private def readsTypedDatasetConversion(plan: SparkPlan): Boolean = plan match {
542+
case _: Exchange | _: QueryStageExec => false
543+
case CometScanWrapper(_, conversion: CometSparkToColumnarExec) =>
544+
conversion.child.isInstanceOf[SerializeFromObjectExec]
545+
case conversion: CometSparkToColumnarExec =>
546+
conversion.child.isInstanceOf[SerializeFromObjectExec]
547+
case other => other.children.exists(readsTypedDatasetConversion)
548+
}
549+
527550
/**
528551
* Reasons the native shuffle path cannot handle this shuffle. Empty means native is supported.
529552
* Pure: does not tag the node.
@@ -622,6 +645,17 @@ object CometShuffleExchangeExec
622645
reasons += s"unsupported hash partitioning data type for native shuffle: $dt"
623646
}
624647
}
648+
// A typed Dataset conversion moves the shuffle above it from Comet's columnar shuffle,
649+
// which partitions with Spark's hash, to native shuffle. Native shuffle hashes a decimal
650+
// wider than 18 digits differently from Spark (#5994), so a join with an input that is
651+
// still on the columnar shuffle would put matching keys in different partitions. Leave
652+
// such a shuffle where it was. A single partition hashes nothing.
653+
if (partitioning.numPartitions > 1 &&
654+
expressions.exists(e => hasWideDecimal(e.dataType)) &&
655+
readsTypedDatasetConversion(s.child)) {
656+
reasons += "a shuffle above a typed Dataset conversion that hashes a decimal wider " +
657+
"than 18 digits stays on Comet's columnar shuffle, which hashes it as Spark does"
658+
}
625659
case SinglePartition =>
626660
// we already checked that the input types are supported
627661
case RangePartitioning(orderings, _) =>

‎spark/src/test/scala/org/apache/comet/exec/CometTypedDatasetSuite.scala‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import java.util.concurrent.atomic.AtomicLong
2424
import org.apache.spark.sql.{CometTestBase, DataFrame, Dataset}
2525
import org.apache.spark.sql.catalyst.expressions.aggregate.Partial
2626
import org.apache.spark.sql.comet.{CometBroadcastHashJoinExec, CometHashAggregateExec, CometSparkToColumnarExec}
27+
import org.apache.spark.sql.comet.execution.shuffle.{CometColumnarShuffle, CometShuffleExchangeExec}
2728
import org.apache.spark.sql.execution.{ColumnarToRowExec, ColumnarToRowTransition, InputAdapter, SerializeFromObjectExec, SparkPlan, WholeStageCodegenExec}
2829
import org.apache.spark.sql.functions.{broadcast, col, size, sum}
2930
import org.apache.spark.sql.internal.SQLConf
@@ -39,6 +40,10 @@ case class TypedDsNested(id: Int, inner: TypedDsRec, tags: Seq[String])
3940

4041
case class TypedDsInts(id: Int, xs: Seq[Int])
4142

43+
case class TypedDsDecimal(k: java.math.BigDecimal, v: Long)
44+
45+
case class TypedDsDecimalInts(k: java.math.BigDecimal, xs: Seq[Int])
46+
4247
/** Counts calls to a user function. Comet tests run in local mode, so tasks see this object. */
4348
object TypedDsCounter {
4449
val calls = new AtomicLong(0)
@@ -209,6 +214,31 @@ class CometTypedDatasetSuite extends CometTestBase {
209214
}
210215
}
211216

217+
convertTest("a join on wide decimal keys with an input that is not converted") {
218+
// The left input converts and the right one, with its array<int> column, does not. Native
219+
// shuffle hashes a decimal wider than 18 digits differently from Spark's partitioner (#5994),
220+
// so the shuffle above the conversion has to stay on Comet's columnar shuffle like the right
221+
// input's, or matching keys land in different partitions and the join loses rows.
222+
withSQLConf(
223+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
224+
SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
225+
SQLConf.SHUFFLE_PARTITIONS.key -> "10") {
226+
val left = spark
227+
.range(0, 100, 1, 2)
228+
.map(i => TypedDsDecimal(new java.math.BigDecimal(i.longValue), i.longValue))
229+
.alias("l")
230+
val right = spark
231+
.range(0, 100, 1, 2)
232+
.map(i => TypedDsDecimalInts(new java.math.BigDecimal(i.longValue), Seq(i.intValue)))
233+
.alias("r")
234+
val (_, plan) = checkSparkAnswer(
235+
left.join(right, col("l.k") === col("r.k")).select(col("l.v"), col("r.xs")))
236+
assert(conversions(plan).nonEmpty, plan)
237+
val shuffles = collectWithSubqueries(plan) { case s: CometShuffleExchangeExec => s }
238+
assert(shuffles.nonEmpty && shuffles.forall(_.shuffleType == CometColumnarShuffle), plan)
239+
}
240+
}
241+
212242
convertTest("the user function runs once per row") {
213243
withRecs(500) { ds =>
214244
TypedDsCounter.calls.set(0)

‎spark/src/test/scala/org/apache/spark/sql/benchmark/CometTypedDatasetBenchmark.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ case class TypedDatasetBenchWide(a: Long, b: String, c: Long, d: String)
5454
*/
5555
object CometTypedDatasetBenchmark extends CometBenchmarkBase {
5656

57-
private val numRows = 4 * 1024 * 1024
57+
private val numRows = 4L * 1024 * 1024
5858
private val loKeys = 100
5959
private val hiKeys = 1024 * 1024
6060

0 commit comments

Comments
 (0)