Skip to content

Commit bcf4862

Browse files
committed
fix: apply the captured configs to the Celeborn frame size and capture DEBUG logs in tests
The native Celeborn writer read spark.comet.shuffle.rss.maxFrameBytes in getWriter, before the writer applies the captured configs. Carry the configs on NativeShuffleSpec, which getWriter can reach through the dependency, and apply them to that read as well as in the writer, in place of CometNativeShuffleInputRDD and its iterator. Spark's LogAppender keeps INFO and above by default, so set its threshold to DEBUG for the memory pool line the tests read.
1 parent 04cdf52 commit bcf4862

7 files changed

Lines changed: 38 additions & 46 deletions

File tree

‎spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleManager.scala‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -170,10 +170,11 @@ class CometCelebornShuffleManager private[shuffle] (
170170
.computeIfAbsent(handle.shuffleId, _ => new ConcurrentHashMap[Int, AnyRef]())
171171
.put(resolved.celebornShuffleId, resolved.client)
172172

173+
val spec = dependency.nativeShuffleSpec.getOrElse {
174+
throw new IllegalStateException("Native Comet shuffle has no execution plan")
175+
}
173176
new CometNativeShuffleWriter[K, V](
174-
dependency.nativeShuffleSpec.getOrElse {
175-
throw new IllegalStateException("Native Comet shuffle has no execution plan")
176-
},
177+
spec,
177178
dependency.outputPartitioning.getOrElse {
178179
throw new IllegalStateException("Native Comet shuffle has no output partitioning")
179180
},
@@ -189,7 +190,7 @@ class CometCelebornShuffleManager private[shuffle] (
189190
CelebornNativeShuffleDestination(
190191
resolved.pusher,
191192
CometCelebornShuffleManager.maxNativeFrameBytes(
192-
CometConf.COMET_SHUFFLE_RSS_MAX_FRAME_BYTES.get().toInt,
193+
spec.sqlConf.withConf(CometConf.COMET_SHUFFLE_RSS_MAX_FRAME_BYTES.get().toInt),
193194
resolved.pusher),
194195
dependency.partitioner.numPartitions,
195196
commitAuthorized = true,

‎spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleInputRDD.scala‎

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ package org.apache.spark.sql.comet.execution.shuffle
2121

2222
import org.apache.spark._
2323
import org.apache.spark.rdd.{DeterministicLevel, RDD}
24-
import org.apache.spark.sql.comet.{CapturedSQLConf, CometExecRDD, CometMetricNode}
24+
import org.apache.spark.sql.comet.{CometExecRDD, CometMetricNode}
2525
import org.apache.spark.sql.vectorized.ColumnarBatch
2626

2727
import org.apache.comet.CometShuffleBlockIterator
@@ -37,9 +37,6 @@ import org.apache.comet.CometShuffleBlockIterator
3737
* @param positionalRoundRobin
3838
* whether the writer fed by this RDD places rows by position rather than by content; see
3939
* [[CometShuffleExchangeExec.positionalRoundRobinSpec]] and `getOutputDeterministicLevel`.
40-
* @param sqlConf
41-
* the session's Comet configs, captured on the driver, which the writer applies; see
42-
* [[CapturedSQLConf]].
4340
*/
4441
private[shuffle] class CometNativeShuffleInputRDD(
4542
sc: SparkContext,
@@ -48,7 +45,6 @@ private[shuffle] class CometNativeShuffleInputRDD(
4845
shuffleScanIndices: Set[Int],
4946
spillMetricNode: CometMetricNode,
5047
@transient perPartitionByKey: Map[String, Array[Array[Byte]]] = Map.empty,
51-
sqlConf: CapturedSQLConf,
5248
positionalRoundRobin: Boolean = false)
5349
extends RDD[Product2[Int, ColumnarBatch]](
5450
sc,
@@ -67,7 +63,6 @@ private[shuffle] class CometNativeShuffleInputRDD(
6763
shuffleScanIndices,
6864
spillMetricNode,
6965
perPartitionByKey,
70-
sqlConf,
7166
positionalRoundRobin)
7267

7368
/**
@@ -127,8 +122,7 @@ private[shuffle] class CometNativeShuffleInputRDD(
127122
partition.index,
128123
inputObjects,
129124
shuffleBlockIters,
130-
partition.planDataByKey,
131-
sqlConf)
125+
partition.planDataByKey)
132126
}
133127

134128
override def getPreferredLocations(split: Partition): Seq[String] = {
@@ -165,8 +159,7 @@ private[shuffle] class CometNativeShuffleInputIterator(
165159
val partitionIndex: Int,
166160
val inputObjects: Array[Object],
167161
val shuffleBlockIterators: Map[Int, CometShuffleBlockIterator],
168-
val planDataByKey: Map[String, Array[Byte]],
169-
val sqlConf: CapturedSQLConf)
162+
val planDataByKey: Map[String, Array[Byte]])
170163
extends Iterator[Product2[Int, ColumnarBatch]] {
171164

172165
override def hasNext: Boolean = false

‎spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala‎

Lines changed: 13 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -95,8 +95,7 @@ class CometNativeShuffleWriter[K, V](
9595

9696
override def write(inputs: Iterator[Product2[K, V]]): Unit = {
9797
try {
98-
val input = nativeShuffleInput(inputs)
99-
input.sqlConf.withConf(writeInternal(input))
98+
spec.sqlConf.withConf(writeInternal(inputs))
10099
} catch {
101100
case failure: Throwable =>
102101
remoteDestination.foreach { destination =>
@@ -112,20 +111,7 @@ class CometNativeShuffleWriter[K, V](
112111
}
113112
}
114113

115-
// The dep's _rdd is always a CometNativeShuffleInputRDD on this path. Pattern-match instead
116-
// of asInstanceOf so a future RDD-layering change produces a clear error here rather than a
117-
// bare ClassCastException deeper in the stack.
118-
private def nativeShuffleInput(
119-
inputs: Iterator[Product2[K, V]]): CometNativeShuffleInputIterator = inputs match {
120-
case it: CometNativeShuffleInputIterator => it
121-
case other =>
122-
throw new IllegalStateException(
123-
"CometNativeShuffleWriter expects its input iterator to be a " +
124-
"CometNativeShuffleInputIterator (produced by CometNativeShuffleInputRDD), got " +
125-
s"${other.getClass.getName}")
126-
}
127-
128-
private def writeInternal(shuffleInputIter: CometNativeShuffleInputIterator): Unit = {
114+
private def writeInternal(inputs: Iterator[Product2[K, V]]): Unit = {
129115
val localOutput = if (remoteDestination.isEmpty) {
130116
val resolver =
131117
SparkEnv.get.shuffleManager.shuffleBlockResolver.asInstanceOf[IndexShuffleBlockResolver]
@@ -137,6 +123,17 @@ class CometNativeShuffleWriter[K, V](
137123
None
138124
}
139125

126+
// The dep's _rdd is always a CometNativeShuffleInputRDD on this path. Pattern-match instead
127+
// of asInstanceOf so a future RDD-layering change produces a clear error here rather than a
128+
// bare ClassCastException deeper in the stack.
129+
val shuffleInputIter = inputs match {
130+
case it: CometNativeShuffleInputIterator => it
131+
case other =>
132+
throw new IllegalStateException(
133+
"CometNativeShuffleWriter expects its input iterator to be a " +
134+
"CometNativeShuffleInputIterator (produced by CometNativeShuffleInputRDD), got " +
135+
s"${other.getClass.getName}")
136+
}
140137
val partitionIdx = shuffleInputIter.partitionIndex
141138
val inputObjects = shuffleInputIter.inputObjects
142139
val shuffleBlockIters = shuffleInputIter.shuffleBlockIterators

‎spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleDependency.scala‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ import org.apache.spark.shuffle.ShuffleWriteProcessor
2828
import org.apache.spark.sql.catalyst.InternalRow
2929
import org.apache.spark.sql.catalyst.expressions.Attribute
3030
import org.apache.spark.sql.catalyst.plans.physical.Partitioning
31-
import org.apache.spark.sql.comet.{CometMetricNode, NativeExecContext}
31+
import org.apache.spark.sql.comet.{CapturedSQLConf, CometMetricNode, NativeExecContext}
3232
import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
3333
import org.apache.spark.sql.execution.metric.SQLMetric
3434
import org.apache.spark.sql.types.StructType
@@ -44,6 +44,8 @@ case class NativeShuffleSpec(
4444
childNativeOp: OperatorOuterClass.Operator,
4545
childMetricNode: CometMetricNode,
4646
execContext: NativeExecContext,
47+
/** The session's Comet configs, captured on the driver; see [[CapturedSQLConf]]. */
48+
sqlConf: CapturedSQLConf,
4749
/**
4850
* Set when round-robin placement is positional rather than content-hashed. Both the decision
4951
* and the group size are resolved once on the driver: the decision because it depends on the

‎spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,6 @@ case class CometShuffleExchangeExec(
144144
ctx.shuffleScanIndices,
145145
CometMetricNode(metrics, Seq(nativeChildMetricNode)),
146146
ctx.perPartitionByKey,
147-
CapturedSQLConf.capture(),
148147
positionalRoundRobin.isDefined)
149148
case None =>
150149
child match {
@@ -236,6 +235,7 @@ case class CometShuffleExchangeExec(
236235
nativeChild.nativeOp,
237236
nativeChildMetricNode,
238237
ctx,
238+
CapturedSQLConf.capture(),
239239
positionalRoundRobin))
240240
case None =>
241241
child match {
@@ -947,8 +947,7 @@ object CometShuffleExchangeExec
947947
Seq(streamRDD),
948948
streamRDD.getNumPartitions,
949949
shuffleScanIndices = Set.empty,
950-
spillMetricNode = CometMetricNode(metrics, Seq(childMetricNode)),
951-
sqlConf = CapturedSQLConf.capture())
950+
spillMetricNode = CometMetricNode(metrics, Seq(childMetricNode)))
952951

953952
val ctx = NativeExecContext(
954953
inputs = Seq(streamRDD),
@@ -970,7 +969,7 @@ object CometShuffleExchangeExec
970969
outputPartitioning,
971970
serializer,
972971
metrics,
973-
NativeShuffleSpec(scanOp, childMetricNode, ctx))
972+
NativeShuffleSpec(scanOp, childMetricNode, ctx, CapturedSQLConf.capture()))
974973
}
975974

976975
/**

‎spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2005,6 +2005,7 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper
20052005
/** Runs `df` from its RDD and checks that every native plan got the session's memory pool. */
20062006
private def checkNativePlansUseSessionConfs(df: DataFrame): Unit = {
20072007
val appender = new LogAppender("native plan memory pools")
2008+
appender.setThreshold(Level.DEBUG)
20082009
withLogAppender(appender, Seq(classOf[CometExecIterator].getName), Some(Level.DEBUG)) {
20092010
df.queryExecution.toRdd.foreach(_ => ())
20102011
}

‎spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleInputRDDSuite.scala‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,7 @@ class CometNativeShuffleInputRDDSuite extends CometTestBase {
6161
Seq(parent),
6262
0,
6363
Set.empty,
64-
CometMetricNode(Map.empty),
65-
sqlConf = CapturedSQLConf.capture())
64+
CometMetricNode(Map.empty))
6665
assert(input.outputDeterministicLevel == level)
6766
assert(input.copyForLocalShuffle().outputDeterministicLevel == level)
6867
}
@@ -91,7 +90,6 @@ class CometNativeShuffleInputRDDSuite extends CometTestBase {
9190
0,
9291
Set.empty,
9392
CometMetricNode(Map.empty),
94-
sqlConf = CapturedSQLConf.capture(),
9593
positionalRoundRobin = true)
9694
assert(input.outputDeterministicLevel == expected, s"parent was $parentLevel")
9795
// The flag has to survive the copy, or a local-shuffle fallback silently drops the
@@ -121,8 +119,7 @@ class CometNativeShuffleInputRDDSuite extends CometTestBase {
121119
2,
122120
Set.empty,
123121
CometMetricNode(Map.empty),
124-
planData,
125-
CapturedSQLConf.capture())
122+
planData)
126123
val local = remote.copyForLocalShuffle()
127124

128125
assert(local.id != remote.id)
@@ -175,12 +172,11 @@ class CometNativeShuffleInputRDDSuite extends CometTestBase {
175172
Seq(nestedInput),
176173
1,
177174
Set.empty,
178-
CometMetricNode(writerMetrics, Seq(childMetrics)),
179-
sqlConf = CapturedSQLConf.capture())
175+
CometMetricNode(writerMetrics, Seq(childMetrics)))
180176

181177
inputRDD.iterator(inputRDD.partitions.head, taskContext)
182178
new CometNativeShuffleWriter[Int, Any](
183-
NativeShuffleSpec(null, childMetrics, null),
179+
NativeShuffleSpec(null, childMetrics, null, CapturedSQLConf.capture()),
184180
null,
185181
Nil,
186182
writerMetrics,
@@ -222,8 +218,7 @@ class CometNativeShuffleInputRDDSuite extends CometTestBase {
222218
numPartitionsParam = numPartitions,
223219
shuffleScanIndices = Set.empty,
224220
spillMetricNode = CometMetricNode(writerMetrics, Seq(childMetricNode)),
225-
perPartitionByKey = perPartitionByKey,
226-
sqlConf = CapturedSQLConf.capture())
221+
perPartitionByKey = perPartitionByKey)
227222
val execContext = NativeExecContext(
228223
inputs = Seq.empty,
229224
numPartitions = numPartitions,
@@ -237,7 +232,11 @@ class CometNativeShuffleInputRDDSuite extends CometTestBase {
237232
perPartitionFilePaths = Array.tabulate(numPartitions) { idx =>
238233
Seq(s"file:/tmp/part-$idx.parquet")
239234
})
240-
val spec = NativeShuffleSpec(Operator.getDefaultInstance, childMetricNode, execContext)
235+
val spec = NativeShuffleSpec(
236+
Operator.getDefaultInstance,
237+
childMetricNode,
238+
execContext,
239+
CapturedSQLConf.capture())
241240
val dep = new CometShuffleDependency[Int, ColumnarBatch, ColumnarBatch](
242241
_rdd = rdd,
243242
partitioner = new HashPartitioner(numPartitions),

0 commit comments

Comments
 (0)