Skip to content

Commit 7e0e5d2

Browse files
authored
fix: Native shuffle fails with a 2GB task serialization OOM on jobs with many partitions (#5392)
* fix: Native shuffle fails with a 2GB task serialization OOM on jobs with very many partitions
1 parent 05a3083 commit 7e0e5d2

8 files changed

Lines changed: 181 additions & 16 deletions

File tree

‎.github/workflows/pr_build_linux.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -327,6 +327,7 @@ jobs:
327327
org.apache.comet.exec.CometShuffle4_0Suite
328328
org.apache.comet.exec.CometNativeColumnarToRowSuite
329329
org.apache.comet.exec.CometNativeShuffleSuite
330+
org.apache.spark.sql.comet.execution.shuffle.CometNativeShuffleInputRDDSuite
330331
org.apache.comet.exec.CometShuffleEncryptionSuite
331332
org.apache.comet.exec.CometShuffleManagerSuite
332333
org.apache.comet.exec.CometAsyncShuffleSuite

‎.github/workflows/pr_build_macos.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,7 @@ jobs:
143143
org.apache.comet.exec.CometShuffle4_0Suite
144144
org.apache.comet.exec.CometNativeColumnarToRowSuite
145145
org.apache.comet.exec.CometNativeShuffleSuite
146+
org.apache.spark.sql.comet.execution.shuffle.CometNativeShuffleInputRDDSuite
146147
org.apache.comet.exec.CometShuffleEncryptionSuite
147148
org.apache.comet.exec.CometShuffleManagerSuite
148149
org.apache.comet.exec.CometAsyncShuffleSuite

‎spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleInputRDD.scala‎

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,8 @@ private[shuffle] class CometNativeShuffleInputRDD(
3838
sc: SparkContext,
3939
var inputRDDs: Seq[RDD[_]],
4040
numPartitionsParam: Int,
41-
shuffleScanIndices: Set[Int])
41+
shuffleScanIndices: Set[Int],
42+
@transient perPartitionByKey: Map[String, Array[Array[Byte]]] = Map.empty)
4243
extends RDD[Product2[Int, ColumnarBatch]](
4344
sc,
4445
inputRDDs.map(rdd => new OneToOneDependency(rdd))) {
@@ -50,7 +51,13 @@ private[shuffle] class CometNativeShuffleInputRDD(
5051
// `leafRdd.partitions` on the executor, which would otherwise trigger getPartitions and
5152
// hit the @transient-null trap (e.g. CometExecRDD.perPartitionByKey).
5253
val inputParts = inputRDDs.map(_.partitions(i)).toArray
53-
new CometNativeShuffleInputPartition(i, inputParts)
54+
// Slice this partition's plan data off the @transient full map here on the driver. Carrying
55+
// only the per-partition slice on the Partition object (serialized per task) keeps the full
56+
// O(numPartitions) map out of the broadcast task binary, which otherwise blows the 2GB
57+
// ByteArrayOutputStream limit on jobs with tens of millions of partitions. Mirrors
58+
// CometExecRDD.getPartitions.
59+
val planDataByKey = perPartitionByKey.map { case (key, arr) => key -> arr(i) }
60+
new CometNativeShuffleInputPartition(i, inputParts, planDataByKey)
5461
}.toArray
5562

5663
override def compute(
@@ -63,7 +70,11 @@ private[shuffle] class CometNativeShuffleInputRDD(
6370
partition.inputPartitions,
6471
shuffleScanIndices,
6572
context)
66-
new CometNativeShuffleInputIterator(partition.index, inputObjects, shuffleBlockIters)
73+
new CometNativeShuffleInputIterator(
74+
partition.index,
75+
inputObjects,
76+
shuffleBlockIters,
77+
partition.planDataByKey)
6778
}
6879

6980
override def getPreferredLocations(split: Partition): Seq[String] = {
@@ -84,19 +95,23 @@ private[shuffle] class CometNativeShuffleInputRDD(
8495

8596
private[shuffle] class CometNativeShuffleInputPartition(
8697
override val index: Int,
87-
val inputPartitions: Array[Partition])
98+
val inputPartitions: Array[Partition],
99+
val planDataByKey: Map[String, Array[Byte]])
88100
extends Partition
89101

90102
/**
91103
* Iterator handed to [[CometNativeShuffleWriter.write]] via Spark's ShuffleMapTask. Reports no
92-
* elements; the writer downcasts and reads `partitionIndex`, `inputObjects`, and
93-
* `shuffleBlockIterators` directly to drive the unified native plan. `inputObjects` are the
94-
* already-resolved native input slots (see [[CometExecRDD.resolveInputObjects]]).
104+
* elements; the writer downcasts and reads `partitionIndex`, `inputObjects`,
105+
* `shuffleBlockIterators`, and `planDataByKey` directly to drive the unified native plan.
106+
* `inputObjects` are the already-resolved native input slots (see
107+
* [[CometExecRDD.resolveInputObjects]]). `planDataByKey` is this partition's slice of the scan
108+
* plan data (one entry per scan key); the writer injects it into the native plan.
95109
*/
96110
private[shuffle] class CometNativeShuffleInputIterator(
97111
val partitionIndex: Int,
98112
val inputObjects: Array[Object],
99-
val shuffleBlockIterators: Map[Int, CometShuffleBlockIterator])
113+
val shuffleBlockIterators: Map[Int, CometShuffleBlockIterator],
114+
val planDataByKey: Map[String, Array[Byte]])
100115
extends Iterator[Product2[Int, ColumnarBatch]] {
101116

102117
override def hasNext: Boolean = false

‎spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -97,10 +97,14 @@ class CometNativeShuffleWriter[K, V](
9797
val unifiedPlan = buildUnifiedPlan(tempDataFilename, tempIndexFilename)
9898
val ctx = spec.execContext
9999
val finalNativePlan = if (ctx.commonByKey.nonEmpty) {
100-
val partitionDataByKey = ctx.perPartitionByKey.map { case (k, arr) =>
101-
k -> arr(partitionIdx)
102-
}
103-
PlanDataInjector.injectPlanData(unifiedPlan, ctx.commonByKey, partitionDataByKey)
100+
// This partition's plan-data slice rides on the input iterator's Partition object (populated
101+
// in CometNativeShuffleInputRDD.getPartitions on the driver), not on the spec. The spec's
102+
// execContext.perPartitionByKey is emptied in prepareNativeShuffleDependency so the full
103+
// O(numPartitions) map stays out of the broadcast task binary.
104+
PlanDataInjector.injectPlanData(
105+
unifiedPlan,
106+
ctx.commonByKey,
107+
shuffleInputIter.planDataByKey)
104108
} else {
105109
unifiedPlan
106110
}

‎spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,8 @@ case class CometShuffleExchangeExec(
122122
sparkContext,
123123
ctx.inputs,
124124
ctx.numPartitions,
125-
ctx.shuffleScanIndices)
125+
ctx.shuffleScanIndices,
126+
ctx.perPartitionByKey)
126127
case None =>
127128
// Non-native child (e.g. CometSparkToColumnarExec): no subtree to inline. The dep gets
128129
// built via the convenience overload below; we just need a real RDD of batches.
@@ -786,8 +787,17 @@ object CometShuffleExchangeExec
786787
case e: Expression => e.collect { case s: ScalarSubquery => s }
787788
case _ => Nil
788789
}
789-
val augmentedSpec = spec.copy(execContext =
790-
spec.execContext.copy(subqueries = spec.execContext.subqueries ++ partitioningSubqueries))
790+
// Drop the per-partition plan-data map off the spec that lands on the (non-transient)
791+
// CometShuffleDependency.nativeShuffleSpec. Each partition's slice now rides on the thin RDD's
792+
// Partition objects (see CometNativeShuffleInputRDD.getPartitions), so the full
793+
// O(numPartitions) map is dead weight here and would blow the 2GB ByteArrayOutputStream limit
794+
// at stage submission on very-high-partition-count jobs. NativeExecContext.perPartitionByKey is
795+
// also @transient (the structural guard against any build path), but we empty it explicitly
796+
// here too so the map isn't retained on the driver via this dependency. commonByKey stays
797+
// (O(#scans), not O(#partitions), and the writer still needs it).
798+
val augmentedSpec = spec.copy(execContext = spec.execContext.copy(
799+
subqueries = spec.execContext.subqueries ++ partitioningSubqueries,
800+
perPartitionByKey = Map.empty))
791801

792802
// The code block below is mostly brought over from
793803
// ShuffleExchangeExec::prepareShuffleDependency

‎spark/src/main/scala/org/apache/spark/sql/comet/operators.scala‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -539,7 +539,12 @@ private[comet] case class NativeExecContext(
539539
broadcastedHadoopConfForEncryption: Option[Broadcast[SerializableConfiguration]],
540540
encryptedFilePaths: Seq[String],
541541
commonByKey: Map[String, Array[Byte]],
542-
perPartitionByKey: Map[String, Array[Array[Byte]]],
542+
// @transient: this holds one serialized scan-plan-data blob per partition, so at high partition
543+
// counts it is huge. It is only read on the driver (to slice per partition onto each task's
544+
// Partition object - see CometNativeShuffleInputRDD / CometExecRDD); the executor reads its own
545+
// slice, never this map. Keeping it off the wire stops it from bloating the broadcast task
546+
// binary when this context rides on the non-transient CometShuffleDependency.nativeShuffleSpec.
547+
@transient perPartitionByKey: Map[String, Array[Array[Byte]]],
543548
shuffleScanIndices: Set[Int],
544549
hasScanInput: Boolean) {
545550
// Catch shape divergence (e.g. broadcast scans with different partition counts after DPP

‎spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,34 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper
108108
}
109109
}
110110

111+
test("native shuffle over a multi-partition native scan re-threads per-partition plan data") {
112+
// End-to-end companion to CometNativeShuffleInputRDDSuite: that suite proves the per-partition
113+
// scan plan data no longer rides the broadcast task binary; this one proves each task still
114+
// gets its OWN slice at write time. Reading real Parquet files gives a CometNativeScanExec with
115+
// several map partitions, so perPartitionByKey holds one file-list slice per partition and a
116+
// correct result depends on task i seeing slice i (not partition 0's).
117+
withTempDir { dir =>
118+
val path = new Path(dir.toURI.toString, "multi.parquet")
119+
// Spread rows across 8 files so the native scan can yield multiple map partitions.
120+
spark
121+
.range(0, 10000, 1, numPartitions = 8)
122+
.selectExpr("id AS _1", "CAST(id AS STRING) AS _2")
123+
.write
124+
.parquet(path.toString)
125+
126+
// Force one scan partition per file split so the per-partition array has multiple distinct
127+
// entries; otherwise Spark coalesces the tiny files into a single partition.
128+
withSQLConf(
129+
"spark.sql.files.maxPartitionBytes" -> "1024",
130+
"spark.sql.files.openCostInBytes" -> "0") {
131+
readParquetFile(path.toString) { df =>
132+
val shuffled = df.repartition(17, col("_1"))
133+
checkShuffleAnswer(shuffled, 1, checkNativeOperators = true)
134+
}
135+
}
136+
}
137+
}
138+
111139
test("hash-based native shuffle") {
112140
withParquetTable((0 until 5).map(i => (i, (i + 1).toLong)), "tbl") {
113141
val df = sql("SELECT * FROM tbl").sortWithinPartitions($"_1".desc)
Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.spark.sql.comet.execution.shuffle
21+
22+
import org.apache.spark.HashPartitioner
23+
import org.apache.spark.serializer.JavaSerializer
24+
import org.apache.spark.sql.CometTestBase
25+
import org.apache.spark.sql.comet.{CometMetricNode, NativeExecContext}
26+
import org.apache.spark.sql.execution.metric.SQLMetrics
27+
import org.apache.spark.sql.vectorized.ColumnarBatch
28+
29+
import org.apache.comet.serde.OperatorOuterClass.Operator
30+
31+
/**
32+
* Ensure the serialized shuffle-map-stage task binary does not grow with the number of
33+
* partitions.
34+
*
35+
* `DAGScheduler.submitMissingTasks` broadcasts the serialized `(stage.rdd, stage.shuffleDep)`
36+
* pair, which must fit in ~2GB. On the native-shuffle path the scan plan data is one serialized
37+
* blob per map partition; the leak lived under
38+
* `CometShuffleDependency.nativeShuffleSpec.execContext`, not on the thin RDD. So this builds
39+
* both carriers and serializes the pair, which catches a regression in either the RDD or the
40+
* dependency guards without needing a real 2GB allocation.
41+
*
42+
* Lives in the `execution.shuffle` package so it can construct the `private[shuffle]`
43+
* [[CometNativeShuffleInputRDD]] and the `private[comet]` [[NativeExecContext]] directly.
44+
*/
45+
class CometNativeShuffleInputRDDSuite extends CometTestBase {
46+
47+
test("serialized (rdd, dep) task binary size is independent of partition count") {
48+
val sc = spark.sparkContext
49+
val ser = new JavaSerializer(sc.getConf).newInstance()
50+
51+
// One scan key with a 1KB blob per map partition -- the per-partition plan data. Build both
52+
// objects the DAGScheduler serializes for a shuffle-map stage: the thin RDD, and the
53+
// CometShuffleDependency whose NativeShuffleSpec holds a NativeExecContext with this map.
54+
def build(numPartitions: Int): (
55+
CometNativeShuffleInputRDD,
56+
CometShuffleDependency[Int, ColumnarBatch, ColumnarBatch]) = {
57+
val perPartitionByKey =
58+
Map("scan-0" -> Array.fill(numPartitions)(new Array[Byte](1024)))
59+
val rdd = new CometNativeShuffleInputRDD(
60+
sc,
61+
inputRDDs = Seq.empty,
62+
numPartitionsParam = numPartitions,
63+
shuffleScanIndices = Set.empty,
64+
perPartitionByKey = perPartitionByKey)
65+
val execContext = NativeExecContext(
66+
inputs = Seq.empty,
67+
numPartitions = numPartitions,
68+
subqueries = Seq.empty,
69+
broadcastedHadoopConfForEncryption = None,
70+
encryptedFilePaths = Seq.empty,
71+
commonByKey = Map.empty,
72+
perPartitionByKey = perPartitionByKey,
73+
shuffleScanIndices = Set.empty,
74+
hasScanInput = false)
75+
val spec =
76+
NativeShuffleSpec(Operator.getDefaultInstance, CometMetricNode(Map.empty), execContext)
77+
val dep = new CometShuffleDependency[Int, ColumnarBatch, ColumnarBatch](
78+
_rdd = rdd,
79+
partitioner = new HashPartitioner(numPartitions),
80+
decodeTime = SQLMetrics.createMetric(sc, "decode time"),
81+
nativeShuffleSpec = Some(spec))
82+
(rdd, dep)
83+
}
84+
85+
// Pre-fix this pair grew from ~13KB to ~10MB between 10 and 10000 partitions. With the map held
86+
// @transient on both the RDD and the NativeExecContext, the pair stays roughly constant.
87+
val (_, smallDep) = build(10)
88+
val (largeRdd, largeDep) = build(10000)
89+
val smallPair = ser.serialize((smallDep.rdd, smallDep)).limit()
90+
val largePair = ser.serialize((largeDep.rdd, largeDep)).limit()
91+
assert(
92+
math.abs(largePair - smallPair) < 100 * 1024,
93+
s"serialized (rdd, dep) grew with partition count (small=$smallPair, large=$largePair); " +
94+
"the per-partition plan-data map is leaking into the broadcast task binary")
95+
96+
// Each task's Partition object still carries its own slice so the writer can inject plan data.
97+
val part = largeRdd.partitions(7).asInstanceOf[CometNativeShuffleInputPartition]
98+
assert(part.planDataByKey.keySet == Set("scan-0"))
99+
assert(part.planDataByKey("scan-0").length == 1024)
100+
}
101+
}

0 commit comments

Comments
 (0)