Skip to content

Commit 88a1f48

Browse files
authored
fix: preserve current AQE logical-stage links on Comet operators (#5483)
* fix: preserve current AQE logical-stage links on Comet operators * docs: link Spark AQE physical-root reuse behavior * test: cover AQE DPP broadcast logical links * test: fix AQE callback syntax for Scala 2.12 * test: invalidate the actual positional shuffle dependency
1 parent a86c967 commit 88a1f48

4 files changed

Lines changed: 298 additions & 6 deletions

File tree

‎spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ import org.apache.spark.sql.comet.execution.shuffle.{CometColumnarShuffle, Comet
3434
import org.apache.spark.sql.comet.shims.ShimCometEmptyRelation
3535
import org.apache.spark.sql.comet.util.Utils
3636
import org.apache.spark.sql.execution._
37-
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AQEShuffleReadExec, BroadcastQueryStageExec, QueryStageExec, ShuffleQueryStageExec}
37+
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AQEShuffleReadExec, BroadcastQueryStageExec, LogicalQueryStage, QueryStageExec, ShuffleQueryStageExec}
3838
import org.apache.spark.sql.execution.aggregate.{BaseAggregateExec, HashAggregateExec, ObjectHashAggregateExec}
3939
import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
4040
import org.apache.spark.sql.execution.command.{DataWritingCommandExec, ExecutedCommandExec}
@@ -806,6 +806,23 @@ case class CometExecRule(session: SparkSession)
806806

807807
// Set up logical links
808808
newPlan = newPlan.transform {
809+
case op: CometExec
810+
if op
811+
.getTagValue(SparkPlan.LOGICAL_PLAN_TAG)
812+
.exists(_.isInstanceOf[LogicalQueryStage]) =>
813+
// AQE replanning reuses this physical root and links it to the current logical stage.
814+
// originalPlan can still point to a subtree hidden inside that logical leaf, which
815+
// AQE cannot replace in the current logical plan. Only preserve a direct stage link,
816+
// not a link inherited from an ancestor.
817+
// On the ordinary exchange path, the exchange itself is behind a QueryStageExec
818+
// leaf and is not visited by this transform.
819+
// Spark 4.1.3 returns the existing root in LogicalQueryStageStrategy and then calls
820+
// setLogicalLink from SparkStrategies.plan:
821+
// scalastyle:off line.size.limit
822+
// https://github.com/apache/spark/blob/v4.1.3/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStageStrategy.scala#L64-L65
823+
// https://github.com/apache/spark/blob/v4.1.3/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala#L78-L87
824+
// scalastyle:on line.size.limit
825+
op
809826
case op: CometExec =>
810827
if (op.originalPlan.logicalLink.isEmpty) {
811828
op.unsetTagValue(SparkPlan.LOGICAL_PLAN_TAG)

‎spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala‎

Lines changed: 50 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,12 @@ import org.apache.spark.sql._
3333
import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
3434
import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStatistics, CatalogTable}
3535
import org.apache.spark.sql.catalyst.expressions.{DynamicPruningExpression, Expression, ExpressionInfo, Hex, Literal}
36-
import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateMode, BloomFilterAggregate}
36+
import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateMode, BloomFilterAggregate, Final}
3737
import org.apache.spark.sql.comet._
3838
import org.apache.spark.sql.comet.execution.shuffle.{CometColumnarShuffle, CometNativeShuffle, CometShuffleExchangeExec}
3939
import org.apache.spark.sql.connector.catalog.InMemoryTableCatalog
4040
import org.apache.spark.sql.execution._
41-
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, BroadcastQueryStageExec}
41+
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, BroadcastQueryStageExec, LogicalQueryStage}
4242
import org.apache.spark.sql.execution.columnar.{CometInMemoryRelationHelper, InMemoryTableScanExec}
4343
import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
4444
import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, BroadcastExchangeLike, ReusedExchangeExec, ShuffleExchangeExec}
@@ -2132,6 +2132,54 @@ class CometExecSuite extends CometTestBase {
21322132
}
21332133
}
21342134

2135+
test("AQE broadcasts native aggregates after replanning") {
2136+
withSQLConf(
2137+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
2138+
SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
2139+
SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "10485760",
2140+
SQLConf.SHUFFLE_PARTITIONS.key -> "4",
2141+
CometConf.COMET_SHUFFLE_MODE.key -> "native",
2142+
CometConf.COMET_SPARK_TO_ARROW_SUPPORTED_OPERATOR_LIST.key -> "Range") {
2143+
val df = sql("""
2144+
|WITH s AS (
2145+
| SELECT id % 64 AS k, SUM(id) AS v FROM range(0, 4096, 1, 4) GROUP BY id % 64
2146+
|), r1 AS (
2147+
| SELECT id % 64 AS k, SUM(id + 1) AS v FROM range(0, 3072, 1, 4) GROUP BY id % 64
2148+
|), r2 AS (
2149+
| SELECT id % 64 AS k, SUM(id + 7) AS v FROM range(0, 2048, 1, 4) GROUP BY id % 64
2150+
|), g AS (
2151+
| SELECT SUM(id) AS v FROM range(0, 1024, 1, 4)
2152+
|)
2153+
|SELECT SUM(s.v + COALESCE(r1.v, 0) + COALESCE(r2.v, 0) + g.v)
2154+
|FROM s LEFT JOIN r1 ON s.k = r1.k LEFT JOIN r2 ON s.k = r2.k CROSS JOIN g
2155+
|""".stripMargin)
2156+
val adaptive = df.queryExecution.executedPlan.asInstanceOf[AdaptiveSparkPlanExec]
2157+
assert(collect(adaptive.executedPlan) { case b: CometBroadcastHashJoinExec => b }.isEmpty)
2158+
2159+
checkAnswer(df, Seq(Row(48738816L)))
2160+
2161+
val finalPlan = adaptive.executedPlan
2162+
assert(collect(finalPlan) { case b: CometBroadcastHashJoinExec => b }.size == 2)
2163+
val broadcasts = collect(finalPlan) { case b: CometBroadcastExchangeExec => b }
2164+
val aggregates = broadcasts.flatMap { broadcast =>
2165+
collect(broadcast.child) {
2166+
case a: CometHashAggregateExec
2167+
if a.modes.contains(Final) && a.groupingExpressions.nonEmpty =>
2168+
a
2169+
}
2170+
}
2171+
assert(aggregates.size == 2)
2172+
aggregates.foreach { aggregate =>
2173+
assert(aggregate.longMetric("output_rows").value == 64)
2174+
assert(aggregate.longMetric("elapsed_compute").value > 0)
2175+
assert(
2176+
aggregate
2177+
.getTagValue(SparkPlan.LOGICAL_PLAN_TAG)
2178+
.exists(_.isInstanceOf[LogicalQueryStage]))
2179+
}
2180+
}
2181+
}
2182+
21352183
test("CometShuffleExchangeExec logical link should be correct") {
21362184
withTempView("v") {
21372185
spark.sparkContext

‎spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala‎

Lines changed: 229 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,25 +19,32 @@
1919

2020
package org.apache.comet.rules
2121

22+
import java.util.concurrent.ConcurrentLinkedQueue
23+
24+
import scala.jdk.CollectionConverters._
2225
import scala.util.Random
2326

27+
import org.scalatest.PrivateMethodTester._
28+
2429
import org.apache.logging.log4j.Level
2530
import org.apache.spark.sql._
2631
import org.apache.spark.sql.catalyst.FunctionIdentifier
2732
import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression, ExpressionInfo, In, InSet, KnownFloatingPointNormalized, Literal, Not}
2833
import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, BloomFilterAggregate, Final, Min, Partial, PartialMerge}
2934
import org.apache.spark.sql.catalyst.optimizer.NormalizeNaNAndZero
35+
import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan}
36+
import org.apache.spark.sql.catalyst.rules.Rule
3037
import org.apache.spark.sql.comet._
3138
import org.apache.spark.sql.comet.execution.shuffle.CometShuffleExchangeExec
3239
import org.apache.spark.sql.execution._
33-
import org.apache.spark.sql.execution.adaptive.{QueryStageExec, ShuffleQueryStageExec}
40+
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, BroadcastQueryStageExec, LogicalQueryStage, QueryStageExec, ShuffleQueryStageExec, SimpleCost, SimpleCostEvaluator}
3441
import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec}
3542
import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
3643
import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ShuffleExchangeExec}
3744
import org.apache.spark.sql.internal.SQLConf
3845
import org.apache.spark.sql.types.{DataTypes, DoubleType, FloatType, StructField, StructType}
3946

40-
import org.apache.comet.{CometConf, CometCoverageStats, CometExplainInfo, ExtendedExplainInfo}
47+
import org.apache.comet.{CometConf, CometCoverageStats, CometExplainInfo, CometSparkSessionExtensions, ExtendedExplainInfo}
4148
import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus, isSpark42Plus, withFallbackReason}
4249
import org.apache.comet.serde.{CometAggregateExpressionSerde, Compatible, ExprOuterClass, QueryPlanSerde, Unsupported}
4350
import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator}
@@ -49,6 +56,40 @@ import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator}
4956
*/
5057
class CometExecRuleSuite extends CometTestBase {
5158

59+
// The observers are active only during the DPP lifecycle regression below. AQE can prepare
60+
// subqueries on different threads, so publish the callbacks and pair each thread's invocations.
61+
@volatile private var beforeCometPreparation: SparkPlan => Unit = (_: SparkPlan) => ()
62+
@volatile private var afterCometPreparation: SparkPlan => Unit = (_: SparkPlan) => ()
63+
64+
override protected def createSparkSession: SparkSessionType = {
65+
SparkSession.clearActiveSession()
66+
SparkSession.clearDefaultSession()
67+
SparkSession
68+
.builder()
69+
.config(sparkContext.getConf)
70+
.withExtensions { extensions =>
71+
extensions.injectQueryStagePrepRule { _ =>
72+
new Rule[SparkPlan] {
73+
override def apply(plan: SparkPlan): SparkPlan = {
74+
beforeCometPreparation(plan)
75+
plan
76+
}
77+
}
78+
}
79+
new CometSparkSessionExtensions().apply(extensions)
80+
extensions.injectQueryStagePrepRule { _ =>
81+
new Rule[SparkPlan] {
82+
override def apply(plan: SparkPlan): SparkPlan = {
83+
afterCometPreparation(plan)
84+
plan
85+
}
86+
}
87+
}
88+
}
89+
.getOrCreate()
90+
.asInstanceOf[SparkSessionType]
91+
}
92+
5293
/** Helper method to apply CometExecRule and return the transformed plan */
5394
private def applyCometExecRule(plan: SparkPlan): SparkPlan = {
5495
CometExecRule(spark).apply(stripAQEPlan(plan))
@@ -106,6 +147,192 @@ class CometExecRuleSuite extends CometTestBase {
106147
case a: ObjectHashAggregateExec if a.aggregateExpressions.forall(_.mode == Partial) => a
107148
}.get
108149

150+
/** A native final aggregate over a shuffle stage, as reused by AQE replanning. */
151+
private def createAdaptiveAggregate(): CometHashAggregateExec = {
152+
val plan = createSparkPlan(
153+
spark,
154+
"SELECT id % 3 AS k, SUM(id) AS total FROM range(0, 100, 1, 2) GROUP BY id % 3")
155+
val aggregate = applyCometExecRule(plan).asInstanceOf[CometHashAggregateExec]
156+
val shuffle = aggregate.child.asInstanceOf[CometShuffleExchangeExec]
157+
aggregate
158+
.withNewChildren(Seq(ShuffleQueryStageExec(0, shuffle, shuffle.canonicalized)))
159+
.asInstanceOf[CometHashAggregateExec]
160+
}
161+
162+
test("CometExecRule preserves the current direct AQE logical link") {
163+
withSQLConf(
164+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
165+
SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false",
166+
CometConf.COMET_SPARK_TO_ARROW_SUPPORTED_OPERATOR_LIST.key -> "Range") {
167+
val originalTags =
168+
Seq(Some(SparkPlan.LOGICAL_PLAN_TAG), Some(SparkPlan.LOGICAL_PLAN_INHERITED_TAG), None)
169+
originalTags.foreach { originalTag =>
170+
withClue(s"original logical tag: $originalTag") {
171+
val aggregate = createAdaptiveAggregate()
172+
val original = aggregate.originalPlan
173+
val originalLogicalPlan = original.logicalLink.get
174+
original.unsetTagValue(SparkPlan.LOGICAL_PLAN_TAG)
175+
original.unsetTagValue(SparkPlan.LOGICAL_PLAN_INHERITED_TAG)
176+
originalTag.foreach(original.setTagValue(_, originalLogicalPlan))
177+
178+
var current: SparkPlan = aggregate
179+
(1 to 2).foreach { _ =>
180+
val logicalStage = LogicalQueryStage(originalLogicalPlan, current)
181+
val replanned = spark.sessionState.planner.plan(logicalStage).next()
182+
assert(replanned eq current)
183+
assert(replanned.getTagValue(SparkPlan.LOGICAL_PLAN_TAG).exists(_ eq logicalStage))
184+
185+
current = applyCometExecRule(replanned)
186+
assert(current.getTagValue(SparkPlan.LOGICAL_PLAN_TAG).exists(_ eq logicalStage))
187+
}
188+
}
189+
}
190+
}
191+
}
192+
193+
test("CometExecRule repairs ordinary and inherited logical links from the original plan") {
194+
withSQLConf(
195+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
196+
SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false",
197+
CometConf.COMET_SPARK_TO_ARROW_SUPPORTED_OPERATOR_LIST.key -> "Range") {
198+
val originalTags =
199+
Seq(Some(SparkPlan.LOGICAL_PLAN_TAG), Some(SparkPlan.LOGICAL_PLAN_INHERITED_TAG), None)
200+
for (originalTag <- originalTags; hasDirectLink <- Seq(false, true)) {
201+
withClue(s"original logical tag: $originalTag, ordinary direct link: $hasDirectLink") {
202+
val aggregate = createAdaptiveAggregate()
203+
val original = aggregate.originalPlan
204+
val originalLogicalPlan = original.logicalLink.get
205+
original.unsetTagValue(SparkPlan.LOGICAL_PLAN_TAG)
206+
original.unsetTagValue(SparkPlan.LOGICAL_PLAN_INHERITED_TAG)
207+
originalTag.foreach(original.setTagValue(_, originalLogicalPlan))
208+
209+
aggregate.unsetTagValue(SparkPlan.LOGICAL_PLAN_TAG)
210+
aggregate.setTagValue(
211+
SparkPlan.LOGICAL_PLAN_INHERITED_TAG,
212+
LogicalQueryStage(originalLogicalPlan, aggregate))
213+
if (hasDirectLink) {
214+
aggregate.setTagValue(SparkPlan.LOGICAL_PLAN_TAG, LocalRelation(aggregate.output))
215+
}
216+
217+
val transformed = applyCometExecRule(aggregate)
218+
if (originalTag.isDefined) {
219+
assert(transformed.logicalLink.exists(_ eq originalLogicalPlan))
220+
} else {
221+
assert(transformed.getTagValue(SparkPlan.LOGICAL_PLAN_TAG).isEmpty)
222+
assert(transformed.getTagValue(SparkPlan.LOGICAL_PLAN_INHERITED_TAG).isEmpty)
223+
}
224+
}
225+
}
226+
}
227+
}
228+
229+
test("AQE DPP broadcast roots retain temporary logical links after an unchanged replan") {
230+
assume(isSpark35Plus, "Native AQE DPP requires Spark 3.5+")
231+
withSQLConf(
232+
SQLConf.USE_V1_SOURCE_LIST.key -> "parquet",
233+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
234+
SQLConf.ADAPTIVE_FORCE_OPTIMIZE_SKEWED_JOIN.key -> "false",
235+
SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
236+
SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true",
237+
SQLConf.SHUFFLE_PARTITIONS.key -> "2",
238+
CometConf.COMET_SHUFFLE_MODE.key -> "native") {
239+
withTempDir { dir =>
240+
withTempView("dpp_link_fact", "dpp_link_dim") {
241+
withSQLConf(CometConf.COMET_ENABLED.key -> "false") {
242+
spark
243+
.range(64)
244+
.selectExpr("CAST(id % 8 AS INT) AS k", "id AS v")
245+
.write
246+
.partitionBy("k")
247+
.parquet(s"$dir/fact")
248+
spark
249+
.range(32)
250+
.selectExpr(
251+
"CAST(id % 8 AS INT) AS k",
252+
"id AS v",
253+
"IF(id % 2 = 0, 'DE', 'US') AS country")
254+
.write
255+
.parquet(s"$dir/dim")
256+
}
257+
spark.read.parquet(s"$dir/fact").createOrReplaceTempView("dpp_link_fact")
258+
spark.read.parquet(s"$dir/dim").createOrReplaceTempView("dpp_link_dim")
259+
260+
assert(
261+
spark.sessionState.conf.getConf(SQLConf.ADAPTIVE_CUSTOM_COST_EVALUATOR_CLASS).isEmpty)
262+
type Replan = (SparkPlan, LogicalPlan)
263+
val pending = new ThreadLocal[List[Option[Replan]]] {
264+
override def initialValue(): List[Option[Replan]] = Nil
265+
}
266+
val observed = new ConcurrentLinkedQueue[(CometBroadcastExchangeExec, LogicalPlan)]()
267+
val tempTag = AdaptiveSparkPlanExec.TEMP_LOGICAL_PLAN_TAG
268+
val costEvaluator = SimpleCostEvaluator(forceOptimizeSkewedJoin = false)
269+
beforeCometPreparation = plan => {
270+
val replan = plan match {
271+
case broadcast: CometBroadcastExchangeExec =>
272+
broadcast.getTagValue(SparkPlan.LOGICAL_PLAN_TAG).collect {
273+
case stage: LogicalQueryStage =>
274+
assert(stage.physicalPlan eq broadcast)
275+
assert(broadcast.getTagValue(tempTag).exists(_ eq stage.logicalPlan))
276+
(broadcast.clone(), stage.logicalPlan)
277+
}
278+
case _ => None
279+
}
280+
pending.set(replan :: pending.get())
281+
}
282+
afterCometPreparation = plan => {
283+
val replan = pending.get().head
284+
val remaining = pending.get().tail
285+
if (remaining.isEmpty) pending.remove() else pending.set(remaining)
286+
replan.foreach { case (previous, logicalPlan) =>
287+
val broadcast = plan.asInstanceOf[CometBroadcastExchangeExec]
288+
assert(broadcast.logicalLink.exists(_ eq logicalPlan))
289+
assert(broadcast.getTagValue(tempTag).exists(_ eq logicalPlan))
290+
// Spark rejects an equal-cost candidate when its physical tree is unchanged.
291+
// Pin both inputs to that decision, including Comet's retained temporary link.
292+
assert(previous == broadcast)
293+
assert(costEvaluator.evaluateCost(previous) == SimpleCost(0))
294+
assert(costEvaluator.evaluateCost(broadcast) == SimpleCost(0))
295+
observed.add(
296+
(broadcast.clone().asInstanceOf[CometBroadcastExchangeExec], logicalPlan))
297+
}
298+
}
299+
try {
300+
val df = sql("""
301+
|SELECT /*+ BROADCAST(d) */ f.k, f.total, d.total
302+
|FROM (SELECT k, SUM(v) AS total FROM dpp_link_fact GROUP BY k) f
303+
|JOIN (SELECT k, SUM(v) AS total FROM dpp_link_dim
304+
| WHERE country = 'DE' GROUP BY k) d ON f.k = d.k
305+
|""".stripMargin)
306+
QueryTest.checkAnswer(
307+
df,
308+
(0 until 8 by 2).map(k => Row(k, 224L + 8L * k, 48L + 4L * k)),
309+
checkToRDD = false)
310+
val plan = df.queryExecution.executedPlan
311+
assert(collect(plan) { case b: CometBroadcastHashJoinExec => b }.nonEmpty)
312+
assert(collectWithSubqueries(plan) { case s: CometSubqueryBroadcastExec =>
313+
s
314+
}.nonEmpty)
315+
assert(!observed.isEmpty, "Expected a DPP broadcast root with a direct logical stage")
316+
observed.iterator().asScala.foreach { case (broadcast, logicalPlan) =>
317+
// Give the isolated snapshot conflicting links to pin Spark's TEMP-over-direct
318+
// precedence, which would otherwise be invisible after Comet repairs both.
319+
broadcast.setLogicalLink(LogicalQueryStage(logicalPlan, broadcast))
320+
val stage = BroadcastQueryStageExec(0, broadcast, broadcast.canonicalized)
321+
val setStageLink = PrivateMethod[Unit](Symbol("setLogicalLinkForNewQueryStage"))
322+
plan
323+
.asInstanceOf[AdaptiveSparkPlanExec]
324+
.invokePrivate(setStageLink(stage, broadcast))
325+
assert(stage.logicalLink.exists(_ eq logicalPlan))
326+
}
327+
} finally {
328+
beforeCometPreparation = _ => ()
329+
afterCometPreparation = _ => ()
330+
}
331+
}
332+
}
333+
}
334+
}
335+
109336
test("expression-level fallback reasons are rolled up onto the operator that falls back") {
110337
// Extended explain only walks plan nodes, so a reason recorded on a sub-expression is
111338
// invisible unless CometExecRule lifts it onto the enclosing operator. Disabling a single

‎spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativePositionalRoundRobinSuite.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@ class CometNativePositionalRoundRobinSuite extends CometTestBase with AdaptiveSp
197197
// Drop the map output so the next job re-runs the map stage under the new batch size.
198198
SparkEnv.get.mapOutputTracker
199199
.asInstanceOf[MapOutputTrackerMaster]
200-
.unregisterAllMapAndMergeOutput(exchange.shuffleId)
200+
.unregisterAllMapAndMergeOutput(exchange.shuffleDependency.shuffleId)
201201
assert(placement() == first)
202202
}
203203
}

0 commit comments

Comments
 (0)