@@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.InternalRow
2828import org .apache .spark .sql .catalyst .expressions ._
2929import org .apache .spark .sql .catalyst .plans .QueryPlan
3030import org .apache .spark .sql .catalyst .plans .physical .{Partitioning , UnknownPartitioning }
31- import org .apache .spark .sql .execution .{FileSourceScanExec , InSubqueryExec , SparkPlan , SubqueryBroadcastExec }
31+ import org .apache .spark .sql .execution .{FileSourceScanExec , InSubqueryExec , SparkPlan }
3232import org .apache .spark .sql .execution .metric .{SQLMetric , SQLMetrics }
3333import org .apache .spark .sql .internal .SQLConf
3434import org .apache .spark .sql .types ._
@@ -83,48 +83,59 @@ case class CometDeltaNativeScanExec(
8383
8484 override val nodeName : String = s " CometDeltaNativeScan $tableRoot"
8585
86- override protected def doPrepare (): Unit = {
87- // `prepare()` (not execute) is safe for any subquery plan, including an
88- // unconverted adaptive-broadcast placeholder.
89- dppFilters.foreach {
90- case DynamicPruningExpression (e : InSubqueryExec ) =>
91- e.plan.prepare()
92- case _ =>
93- }
94- super .doPrepare()
86+ // DPP support. The AQE DPP subquery on a partitioned Delta scan arrives as an
87+ // unexecutable placeholder: CometExecRule wraps Spark's
88+ // SubqueryAdaptiveBroadcastExec into CometSubqueryAdaptiveBroadcastExec, and
89+ // CometPlanAdaptiveDynamicPruningFilters rewrites it to an executable
90+ // (Comet)SubqueryBroadcastExec with proper broadcast reuse. That rewrite would
91+ // normally produce a copy of this scan, but the copy is dropped when the
92+ // enclosing native block is rebuilt (TreeNode.makeCopy can't carry @transient
93+ // fields, #3510). So the rule installs the rewrite IN PLACE via
94+ // `withDynamicPruningFilters` (below), which updates this transient
95+ // side-channel and returns `this` -- landing the executable subqueries on the
96+ // SAME instance that executes. `dppFilters` (the case-class field) is left
97+ // untouched so node equality/canonicalization is unaffected; everything at
98+ // execution reads `effectiveDppFilters`.
99+ @ transient private var dppFiltersOverride : Seq [Expression ] = null
100+
101+ private def effectiveDppFilters : Seq [Expression ] =
102+ if (dppFiltersOverride != null ) dppFiltersOverride else dppFilters
103+
104+ override def dynamicPruningFilters : Seq [Expression ] = effectiveDppFilters
105+
106+ override def withDynamicPruningFilters (filters : Seq [Expression ]): SparkPlan = {
107+ dppFiltersOverride = filters
108+ this
95109 }
96110
97111 /** True when a DPP subquery is an adaptive-broadcast placeholder we can't
98- * execute ourselves : the unwrapped Spark `SubqueryAdaptiveBroadcastExec` or
99- * the Comet-wrapped `CometSubqueryAdaptiveBroadcastExec`. Both throw from
100- * `doExecute()`; they only become executable once
101- * `CometPlanAdaptiveDynamicPruningFilters` rewrites them to a
102- * (Comet)SubqueryBroadcastExec . */
112+ * execute: the unwrapped Spark `SubqueryAdaptiveBroadcastExec` or the
113+ * Comet-wrapped `CometSubqueryAdaptiveBroadcastExec`. Both throw from
114+ * `doExecute()`. Normally the rule rewrites them in place (see above) before
115+ * execution; this guard skips any that slip through (e.g. the rule didn't
116+ * run) so we read all partitions instead of crashing . */
103117 private def isUnexecutableDpp (plan : SparkPlan ): Boolean =
104118 plan.isInstanceOf [org.apache.spark.sql.execution.SubqueryAdaptiveBroadcastExec ] ||
105119 plan.isInstanceOf [CometSubqueryAdaptiveBroadcastExec ]
106120
107- // DPP support. The AQE DPP subquery on a partitioned Delta scan arrives as an
108- // unexecutable placeholder: CometExecRule wraps Spark's
109- // SubqueryAdaptiveBroadcastExec into CometSubqueryAdaptiveBroadcastExec, and
110- // CometPlanAdaptiveDynamicPruningFilters is supposed to rewrite it to an
111- // executable (Comet)SubqueryBroadcastExec -- but that rewritten scan copy is
112- // dropped when the enclosing native block is rebuilt (TreeNode.makeCopy can't
113- // carry @transient fields, #3510), so the live scan keeps the placeholder.
114- // We therefore (1) skip the placeholder in `ensureSubqueriesResolved` so it
115- // doesn't execute and crash, and (2) convert it on the fly during partition
116- // pruning in `applyDppFilters` (where the broadcast is already materialized)
117- // to recover the pruning values. Results are always correct; pruning applies
118- // when the scan recomputes partitions at execution.
121+ override protected def doPrepare (): Unit = {
122+ // `prepare()` (not execute) is safe for any subquery plan, including a
123+ // placeholder.
124+ effectiveDppFilters.foreach {
125+ case DynamicPruningExpression (e : InSubqueryExec ) =>
126+ e.plan.prepare()
127+ case _ =>
128+ }
129+ super .doPrepare()
130+ }
119131
120132 // Comet's native-scan subquery lifecycle (see CometLeafExec). The default
121- // `prepare(); waitForSubqueries()` would execute EVERY collected subquery,
122- // including an unexecutable adaptive-broadcast placeholder -> crash. Resolve
123- // only the DPP subqueries we can execute; skip placeholders so pruning falls
124- // back to read-all instead of throwing.
133+ // `prepare(); waitForSubqueries()` would execute EVERY collected subquery; if
134+ // an unrewritten placeholder slipped through it would crash. Resolve only the
135+ // executable DPP subqueries; skip placeholders (read-all fallback).
125136 override def ensureSubqueriesResolved (): Unit = {
126137 prepare()
127- dppFilters .foreach {
138+ effectiveDppFilters .foreach {
128139 case DynamicPruningExpression (inSub : InSubqueryExec )
129140 if ! isUnexecutableDpp(inSub.plan) && inSub.values().isEmpty =>
130141 inSub.updateResult()
@@ -190,18 +201,30 @@ case class CometDeltaNativeScanExec(
190201 * reading the full table every time AQE is in the loop.
191202 */
192203 private def buildPerPartitionBytes (): Array [Array [Byte ]] = {
193- val tasks =
194- if (dppFilters.nonEmpty && partitionSchema.nonEmpty) applyDppFilters(allTasks)
195- else allTasks
196- if (tasks.isEmpty) {
197- Array .empty[Array [Byte ]]
198- } else {
199- packTasks(tasks).map { group =>
200- val builder = OperatorOuterClass .DeltaScan .newBuilder()
201- group.foreach(builder.addTasks)
202- builder.build().toByteArray
203- }.toArray
204- }
204+ // Group ALL tasks once (`taskGroups`) so the partition COUNT is fixed
205+ // regardless of DPP -- Spark pins `numPartitions` at planning and the native
206+ // RDD's partition count must not change at execution. DPP pruning then
207+ // happens WITHIN each group: pruned-out tasks are removed, and a group whose
208+ // tasks are all pruned becomes an empty DeltaScan (0 rows) -- but the group
209+ // (= partition slot) remains, keeping the count stable. This lets DPP prune
210+ // even when the scan executes inside a parent native block (MERGE/join),
211+ // where the parent reads `perPartitionData` rather than running the scan's
212+ // own `doExecuteColumnar`.
213+ val groups = taskGroups
214+ if (groups.isEmpty) return Array .empty[Array [Byte ]]
215+ val survivorPaths : Option [Set [String ]] =
216+ if (dppFilters.nonEmpty && partitionSchema.nonEmpty) {
217+ Some (applyDppFilters(allTasks).map(_.getFilePath).toSet)
218+ } else None
219+ groups.map { group =>
220+ val kept = survivorPaths match {
221+ case Some (s) => group.filter(t => s.contains(t.getFilePath))
222+ case None => group
223+ }
224+ val builder = OperatorOuterClass .DeltaScan .newBuilder()
225+ kept.foreach(builder.addTasks)
226+ builder.build().toByteArray
227+ }.toArray
205228 }
206229
207230 // #75 design A: when input_file_name() is needed (signal threaded from CometScanRule
@@ -241,41 +264,34 @@ case class CometDeltaNativeScanExec(
241264 out.toSeq
242265 }
243266
244- // Planning-time snapshot used by metrics, sourceKey derivations, and `numPartitions`.
245- // Execution-time recomputation happens inside `doExecuteColumnar`.
246- @ transient private lazy val planningPerPartitionBytes : Array [Array [Byte ]] =
247- buildPerPartitionBytes()
267+ // Stable task grouping = the partition layout. Computed once from ALL tasks so
268+ // the partition count is fixed across planning and execution (DPP prunes
269+ // tasks WITHIN groups, never changing the group count). `numPartitions` reads
270+ // this directly so counting partitions never triggers DPP broadcast
271+ // resolution.
272+ @ transient private lazy val taskGroups : Seq [Seq [OperatorOuterClass .DeltaScanTask ]] =
273+ if (allTasks.isEmpty) Seq .empty else packTasks(allTasks)
248274
249275 private def applyDppFilters (
250276 tasks : Seq [OperatorOuterClass .DeltaScanTask ]): Seq [OperatorOuterClass .DeltaScanTask ] = {
251277 // Resolve each DPP subquery to its runtime pruning values, then prune tasks
252- // by evaluating the partition predicate below.
253- //
254- // A DPP subquery may still be an adaptive-broadcast PLACEHOLDER that can't
255- // execute directly: the Comet-wrapped `CometSubqueryAdaptiveBroadcastExec`
256- // (produced by CometExecRule) or the unwrapped `SubqueryAdaptiveBroadcast
257- // Exec`. `CometPlanAdaptiveDynamicPruningFilters` normally rewrites these to
258- // an executable (Comet)SubqueryBroadcastExec, but that rewrite can be
259- // orphaned for a scan buried inside a native block (the converted copy is
260- // dropped when the parent block is rebuilt -- #3510), leaving the live scan
261- // with the placeholder. By the time we build per-partition data the
262- // broadcast the placeholder wraps is already materialized, so we convert it
263- // on the fly to an executable `SubqueryBroadcastExec` (its `child` is that
264- // materialized broadcast) and resolve THAT to recover the pruning values --
265- // so we still scan only the required partitions. Any failure falls back to
266- // returning all tasks (correct, just unpruned).
278+ // by evaluating the partition predicate below. By execution time the rule
279+ // has installed executable (Comet)SubqueryBroadcastExec subqueries in place
280+ // (see `withDynamicPruningFilters`); we resolve them here. If an
281+ // unexecutable placeholder slipped through (rule didn't run), skip pruning
282+ // and read all tasks (correct, just unpruned) rather than crashing.
283+ if (effectiveDppFilters.exists {
284+ case DynamicPruningExpression (inSub : InSubqueryExec ) => isUnexecutableDpp(inSub.plan)
285+ case _ => false
286+ }) {
287+ return tasks
288+ }
267289 val resolvedFilters : Seq [Expression ] =
268290 try {
269- dppFilters .map {
291+ effectiveDppFilters .map {
270292 case DynamicPruningExpression (inSub : InSubqueryExec ) =>
271- val executable = inSub.plan match {
272- case csab : CometSubqueryAdaptiveBroadcastExec =>
273- inSub.copy(plan = SubqueryBroadcastExec (
274- csab.name, csab.indices, csab.buildKeys, csab.child))
275- case _ => inSub
276- }
277- if (executable.values().isEmpty) executable.updateResult()
278- executable
293+ if (inSub.values().isEmpty) inSub.updateResult()
294+ inSub
279295 case DynamicPruningExpression (e) => e
280296 case other => other
281297 }
@@ -310,7 +326,11 @@ case class CometDeltaNativeScanExec(
310326 }
311327
312328 def commonData : Array [Byte ] = commonBytes
313- def perPartitionData : Array [Array [Byte ]] = planningPerPartitionBytes
329+ // Recomputed (not memoised) so that when a parent native block reads this at
330+ // execution -- after AQE has materialised the DPP broadcast -- the returned
331+ // per-partition task lists reflect DPP pruning. The partition COUNT is fixed
332+ // by `taskGroups`; only the tasks within each group are pruned.
333+ def perPartitionData : Array [Array [Byte ]] = buildPerPartitionBytes()
314334
315335 // Surface per-partition file paths to the unified `CometExecRDD` path in
316336 // `operators.scala` so `InputFileBlockHolder` is populated when this scan
@@ -319,7 +339,7 @@ case class CometDeltaNativeScanExec(
319339 // `CometExecRDD.compute` sees empty filePaths -> `input_file_name()`
320340 // returns "" -> `DELTA_FILE_TO_OVERWRITE_NOT_FOUND`.
321341 override def perPartitionFilePaths : Array [Seq [String ]] = {
322- planningPerPartitionBytes .map { bytes =>
342+ perPartitionData .map { bytes =>
323343 OperatorOuterClass .DeltaScan .parseFrom(bytes)
324344 .getTasksList.asScala.map(_.getFilePath).toSeq
325345 }
@@ -337,7 +357,7 @@ case class CometDeltaNativeScanExec(
337357 */
338358 def sourceKey : String = CometDeltaNativeScanExec .computeSourceKey(nativeOp)
339359
340- def numPartitions : Int = perPartitionData .length
360+ def numPartitions : Int = taskGroups .length
341361
342362 override lazy val outputPartitioning : Partitioning =
343363 UnknownPartitioning (math.max(1 , numPartitions))
0 commit comments