Skip to content

Commit 891008e

Browse files
authored
fix: report SinglePartition from a native Iceberg scan that replaces one (#7424)
* fix: report SinglePartition from a native Iceberg scan that replaces one On Spark 3.4, BatchScanExec reports SinglePartition for a scan of one input partition, and Spark plans no exchange above it. The native Iceberg scan reported UnknownPartitioning(0). Since the aggregates declare their required distribution (#7231), EnsureRequirements, which AQE runs again over a DPP subquery's converted plan, inserted an exchange inside a native block that was already serialized, and the native scan lost its tasks. Report SinglePartition when the replaced scan does. * test: run the SPARK-34637 Iceberg DPP test on Spark 3.4 again
1 parent c7b2cfd commit 891008e

2 files changed

Lines changed: 20 additions & 9 deletions

File tree

‎spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergNativeScanExec.scala‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ import org.apache.spark.rdd.RDD
2626
import org.apache.spark.sql.SparkSession
2727
import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, SortOrder}
2828
import org.apache.spark.sql.catalyst.plans.QueryPlan
29-
import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, UnknownPartitioning}
29+
import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, SinglePartition, UnknownPartitioning}
3030
import org.apache.spark.sql.execution.{SparkPlan, SQLExecution}
3131
import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
3232
import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics}
@@ -135,7 +135,18 @@ case class CometIcebergNativeScanExec(
135135
// CometPlanAdaptiveDynamicPruningFilters converts the DPP placeholders, so serializing the
136136
// tasks here would run a placeholder subquery. At execution, CometNativeExec sizes the native
137137
// RDD from perPartitionData instead.
138-
override lazy val outputPartitioning: Partitioning = UnknownPartitioning(0)
138+
//
139+
// Spark 3.4's BatchScanExec reports SinglePartition for one input partition, which it counts
140+
// before any DPP filtering, and Spark plans no exchange above it on that basis. The scan reports
141+
// the same. Otherwise EnsureRequirements, which AQE runs again over a DPP subquery's converted
142+
// plan, inserts an exchange inside a native block that is already serialized (#7422).
143+
// serializePartitions keeps that one partition when DPP prunes it, as Spark does.
144+
override lazy val outputPartitioning: Partitioning =
145+
if (originalPlan != null && originalPlan.outputPartitioning == SinglePartition) {
146+
SinglePartition
147+
} else {
148+
UnknownPartitioning(0)
149+
}
139150

140151
override lazy val outputOrdering: Seq[SortOrder] = Nil
141152

‎spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6743,8 +6743,6 @@ class CometIcebergNativeSuite
67436743
// Spark's V2 path.
67446744
test("AQE DPP - V2 broadcast query stage creation order (SPARK-34637)") {
67456745
assume(icebergAvailable, "Iceberg not available")
6746-
// https://github.com/apache/datafusion-comet/issues/7422
6747-
assume(isSpark35Plus, "Spark 3.4 fails the DPP build side, see #7422")
67486746
withTempIcebergDir { warehouseDir =>
67496747
withSQLConf(
67506748
"spark.sql.catalog.aqe_cat" -> "org.apache.iceberg.spark.SparkCatalog",
@@ -6780,12 +6778,14 @@ class CometIcebergNativeSuite
67806778

67816779
assertNoLeftoverCSAB(cometPlan)
67826780

6783-
val cometSubqueries = collectWithSubqueries(cometPlan) {
6784-
case s: CometSubqueryBroadcastExec => s
6781+
if (isSpark35Plus) {
6782+
val cometSubqueries = collectWithSubqueries(cometPlan) {
6783+
case s: CometSubqueryBroadcastExec => s
6784+
}
6785+
assert(
6786+
cometSubqueries.nonEmpty,
6787+
s"Expected CometSubqueryBroadcastExec for DPP, got 0:\n${cometPlan.treeString}")
67856788
}
6786-
assert(
6787-
cometSubqueries.nonEmpty,
6788-
s"Expected CometSubqueryBroadcastExec for DPP, got 0:\n${cometPlan.treeString}")
67896789

67906790
spark.sql("DROP TABLE aqe_cat.db.q34637_fact")
67916791
}

0 commit comments

Comments
 (0)