Skip to content

Commit 5cf4654

Browse files
committed
fix: strip every DPP filter from the Delta scan's originalPlan when canonicalizing
CometDeltaNativeScanExec kept a stale DPP placeholder in its originalPlan copy once filterUnusedDynamicPruningExpressions stopped dropping placeholders, so after AQE turned the live filter into TrueLiteral the scan no longer matched its twin and lost SPARK-32509 exchange reuse. Drop every DPP filter from originalPlan, as CometNativeScanExec does; runtimeFilters still carry the scan's DPP identity. Adds a Delta version of the SPARK-32509 reuse test.
1 parent 5e3f69a commit 5cf4654

2 files changed

Lines changed: 55 additions & 2 deletions

File tree

‎contrib/delta-spark/src/main/scala/org/apache/spark/sql/comet/CometDeltaNativeScanExec.scala‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -225,9 +225,11 @@ case class CometDeltaNativeScanExec(
225225
}
226226

227227
override def doCanonicalize(): CometDeltaNativeScanExec = {
228+
// originalPlan's DPP filters are a stale copy that no DPP rule rewrites, so drop them all.
229+
// runtimeFilters below carry the scan's DPP identity.
228230
val canonOriginal = if (originalPlan != null) {
229231
val stripped = originalPlan.copy(partitionFilters =
230-
CometScanUtils.filterUnusedDynamicPruningExpressions(originalPlan.partitionFilters))
232+
originalPlan.partitionFilters.filterNot(_.isInstanceOf[DynamicPruningExpression]))
231233
stripped.doCanonicalize()
232234
} else {
233235
null

‎contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaNativeScanSuite.scala‎

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,11 @@ import scala.collection.mutable.ListBuffer
2727
import org.apache.spark.CometListenerBusUtils
2828
import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd}
2929
import org.apache.spark.sql.{DataFrame, Row}
30-
import org.apache.spark.sql.catalyst.expressions.{AttributeReference, DynamicPruningExpression, Expression, NamedExpression, StructsToJson}
30+
import org.apache.spark.sql.catalyst.expressions.{AttributeReference, DynamicPruningExpression, DynamicPruningSubquery, Expression, NamedExpression, StructsToJson}
3131
import org.apache.spark.sql.comet.{CometDeltaNativeScanExec, CometSubqueryBroadcastExec}
3232
import org.apache.spark.sql.execution.{FileSourceScanExec, InSubqueryExec, QueryExecution, ReusedSubqueryExec, ScalarSubquery, SparkPlan, SubqueryBroadcastExec, SubqueryExec}
3333
import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec
34+
import org.apache.spark.sql.execution.exchange.ReusedExchangeExec
3435
import org.apache.spark.sql.functions.{col, lit, to_json}
3536
import org.apache.spark.sql.internal.SQLConf
3637
import org.apache.spark.sql.types.{ByteType, LongType, StringType, StructField, StructType}
@@ -1931,6 +1932,56 @@ class CometDeltaNativeScanSuite extends CometDeltaTestBase {
19311932
}
19321933
}
19331934

1935+
// SPARK-32509: with no broadcast join the DPP filter on one side of the self-join degrades to
1936+
// TrueLiteral, so both fact scans must canonicalize alike and share one shuffle.
1937+
test("unused DPP filter does not prevent exchange reuse (SPARK-32509)") {
1938+
withTempPath { dir =>
1939+
val path = dir.getAbsolutePath
1940+
spark
1941+
.range(1, 26)
1942+
.selectExpr(
1943+
"cast(id as int) as store_id",
1944+
"cast(case when id = 15 then 70 else id % 5 * 10 end as int) as units_sold")
1945+
.write
1946+
.format("delta")
1947+
.partitionBy("store_id")
1948+
.save(path)
1949+
1950+
withTempView("dpp_reuse_fact") {
1951+
spark.read.format("delta").load(path).createOrReplaceTempView("dpp_reuse_fact")
1952+
withSQLConf(
1953+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
1954+
SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true",
1955+
SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
1956+
val df = sql("""WITH v AS (
1957+
| SELECT store_id FROM dpp_reuse_fact WHERE units_sold = 70
1958+
|)
1959+
|SELECT * FROM v v1 JOIN v v2 ON v1.store_id = v2.store_id""".stripMargin)
1960+
assert(
1961+
df.queryExecution.optimizedPlan.exists(_.expressions.exists(_.exists {
1962+
case _: DynamicPruningSubquery => true
1963+
case _ => false
1964+
})),
1965+
s"expected a DPP filter in the optimized plan:\n${df.queryExecution.optimizedPlan}")
1966+
1967+
val (sparkPlan, cometPlan) = checkSparkAnswer(df)
1968+
checkAnswer(df, Row(15, 15) :: Nil)
1969+
assert(
1970+
collect(cometPlan) { case s: CometDeltaNativeScanExec => s }.nonEmpty,
1971+
s"expected a native Delta scan:\n${cometPlan.treeString}")
1972+
1973+
def reused(plan: SparkPlan): Int =
1974+
collect(plan) { case r: ReusedExchangeExec => r }.size
1975+
assert(reused(sparkPlan) == 1, s"Spark should reuse the shuffle:\n$sparkPlan")
1976+
assert(
1977+
reused(cometPlan) == reused(sparkPlan),
1978+
s"expected ${reused(sparkPlan)} reused exchange, got ${reused(cometPlan)}:\n" +
1979+
cometPlan.treeString)
1980+
}
1981+
}
1982+
}
1983+
}
1984+
19341985
test("scalar subquery in a partition filter does not force partitioning during AQE checks") {
19351986
// Crash shape: a scalar subquery used directly as
19361987
// a partition filter, e.g. `p = (SELECT max(p) FROM dim ...)`, references only the

0 commit comments

Comments
 (0)