Skip to content

Commit 4ac0892

Browse files
committed
test: share the reused exchange helpers in CometTestBase
assertExchangeReuseOver and assertNoReusedExchangeOverDppScan now call one reusedExchangesOver, and assertSameExchangeReuseAsSpark replaces the reuse count comparison written out in CometExecSuite and CometDeltaNativeScanSuite.
1 parent f9625fe commit 4ac0892

3 files changed

Lines changed: 34 additions & 25 deletions

File tree

‎contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaNativeScanSuite.scala‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@ import org.apache.spark.sql.catalyst.expressions.{AttributeReference, DynamicPru
3131
import org.apache.spark.sql.comet.{CometDeltaNativeScanExec, CometSubqueryBroadcastExec}
3232
import org.apache.spark.sql.execution.{FileSourceScanExec, InSubqueryExec, QueryExecution, ReusedSubqueryExec, ScalarSubquery, SparkPlan, SubqueryBroadcastExec, SubqueryExec}
3333
import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec
34-
import org.apache.spark.sql.execution.exchange.ReusedExchangeExec
3534
import org.apache.spark.sql.functions.{col, lit, to_json}
3635
import org.apache.spark.sql.internal.SQLConf
3736
import org.apache.spark.sql.types.{ByteType, LongType, StringType, StructField, StructType}
@@ -1970,13 +1969,10 @@ class CometDeltaNativeScanSuite extends CometDeltaTestBase {
19701969
collect(cometPlan) { case s: CometDeltaNativeScanExec => s }.nonEmpty,
19711970
s"expected a native Delta scan:\n${cometPlan.treeString}")
19721971

1973-
def reused(plan: SparkPlan): Int =
1974-
collect(plan) { case r: ReusedExchangeExec => r }.size
1975-
assert(reused(sparkPlan) == 1, s"Spark should reuse the shuffle:\n$sparkPlan")
19761972
assert(
1977-
reused(cometPlan) == reused(sparkPlan),
1978-
s"expected ${reused(sparkPlan)} reused exchange, got ${reused(cometPlan)}:\n" +
1979-
cometPlan.treeString)
1973+
reusedExchanges(sparkPlan).size == 1,
1974+
s"Spark should reuse the shuffle once:\n$sparkPlan")
1975+
assertSameExchangeReuseAsSpark(sparkPlan, cometPlan)
19801976
}
19811977
}
19821978
}

‎spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1883,13 +1883,7 @@ class CometExecSuite extends CometTestBase {
18831883
val df = sql(dppShuffleQuery("a", "a"))
18841884
val (sparkPlan, cometPlan) =
18851885
if (isSpark35Plus) checkSparkAnswerAndOperator(df) else checkSparkAnswer(df)
1886-
def reused(plan: SparkPlan): Int =
1887-
collect(plan) { case r: ReusedExchangeExec => r }.size
1888-
assert(reused(sparkPlan) > 0, s"Spark should reuse the shuffle:\n$sparkPlan")
1889-
assert(
1890-
reused(cometPlan) == reused(sparkPlan),
1891-
s"expected ${reused(sparkPlan)} reused exchanges, got ${reused(cometPlan)}:\n" +
1892-
cometPlan.treeString)
1886+
assertSameExchangeReuseAsSpark(sparkPlan, cometPlan)
18931887
if (isSpark35Plus) {
18941888
assertExchangeReuseOver(cometPlan, "Expected the shuffle over a DPP scan to be reused")(
18951889
dppScan)

‎spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala‎

Lines changed: 30 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -70,21 +70,40 @@ abstract class CometTestBase
7070
protected val shuffleManager: String =
7171
"org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager"
7272

73+
/** The reused exchanges in `plan`. */
74+
protected def reusedExchanges(plan: SparkPlan): Seq[ReusedExchangeExec] =
75+
collect(plan) { case r: ReusedExchangeExec => r }
76+
77+
/** The reused exchanges in `plan` whose child contains a node that `pf` matches. */
78+
protected def reusedExchangesOver[T](plan: SparkPlan)(
79+
pf: PartialFunction[SparkPlan, T]): Seq[ReusedExchangeExec] =
80+
reusedExchanges(plan).filter(exchange => collect(exchange.child)(pf).nonEmpty)
81+
82+
/** Asserts that some reused exchange in `plan` sits over a node that `pf` matches. */
7383
protected def assertExchangeReuseOver[T](plan: SparkPlan, clue: String)(
74-
pf: PartialFunction[SparkPlan, T]): Unit = {
75-
val reused = collect(plan) {
76-
case exchange: ReusedExchangeExec if collect(exchange.child)(pf).nonEmpty => exchange
77-
}
78-
assert(reused.nonEmpty, s"$clue:\n$plan")
79-
}
84+
pf: PartialFunction[SparkPlan, T]): Unit =
85+
assert(reusedExchangesOver(plan)(pf).nonEmpty, s"$clue:\n$plan")
8086

8187
/** Asserts that no reused exchange in `plan` sits over a scan that `dppScan` matches. */
8288
protected def assertNoReusedExchangeOverDppScan[T](plan: SparkPlan)(
83-
dppScan: PartialFunction[SparkPlan, T]): Unit = {
84-
val reused = collect(plan) {
85-
case exchange: ReusedExchangeExec if collect(exchange.child)(dppScan).nonEmpty => exchange
86-
}
87-
assert(reused.isEmpty, s"An exchange over a DPP scan was reused by another branch:\n$plan")
89+
dppScan: PartialFunction[SparkPlan, T]): Unit =
90+
assert(
91+
reusedExchangesOver(plan)(dppScan).isEmpty,
92+
s"An exchange over a DPP scan was reused by another branch:\n$plan")
93+
94+
/**
95+
* Asserts that Spark reuses at least one exchange in `sparkPlan` and that Comet reuses as many
96+
* in `cometPlan`.
97+
*/
98+
protected def assertSameExchangeReuseAsSpark(
99+
sparkPlan: SparkPlan,
100+
cometPlan: SparkPlan): Unit = {
101+
val sparkReused = reusedExchanges(sparkPlan).size
102+
val cometReused = reusedExchanges(cometPlan).size
103+
assert(sparkReused > 0, s"Spark should reuse an exchange:\n$sparkPlan")
104+
assert(
105+
cometReused == sparkReused,
106+
s"expected $sparkReused reused exchanges, got $cometReused:\n" + cometPlan.treeString)
88107
}
89108

90109
/**

0 commit comments

Comments
 (0)