2020package org .apache .spark .sql .comet
2121
2222import java .util .Locale
23+ import java .util .ServiceLoader
2324
2425import scala .collection .mutable
2526import scala .collection .mutable .ArrayBuffer
2627import scala .jdk .CollectionConverters ._
28+ import scala .util .control .NonFatal
2729
2830import org .apache .spark .{Partition , TaskContext }
2931import org .apache .spark .broadcast .Broadcast
@@ -90,39 +92,31 @@ private[comet] trait PlanDataInjector {
9092 */
9193private [comet] object PlanDataInjector extends Logging {
9294
93- // Registry of injectors for different operator types. The contrib/delta integration's
94- // DeltaPlanDataInjector is appended via one reflective class lookup -- present only when
95- // the contrib was bundled (i.e. -Pcontrib-delta on the Maven build). Default builds get
96- // the empty Option and an unmodified injectors list, so there's zero contrib surface at
97- // runtime on default builds.
98- private val injectors : Seq [PlanDataInjector ] = {
95+ // Registry of injectors for different operator types. The built-in injectors live in core.
96+ // Out-of-tree contribs (e.g. contrib-delta's `DeltaPlanDataInjector`) are discovered via the
97+ // standard JDK `ServiceLoader`: a contrib ships a
98+ // `META-INF/services/org.apache.spark.sql.comet.PlanDataInjector` resource naming its
99+ // implementation, so it joins the registry without core holding any compile-time reference or
100+ // contrib-specific code. Default builds carry no such service file, so discovery yields nothing
101+ // and the registry is exactly the built-ins -- zero contrib surface at runtime.
102+ private [comet] val injectors : Seq [PlanDataInjector ] = {
99103 val builtin : Seq [PlanDataInjector ] = Seq (IcebergPlanDataInjector , NativeScanPlanDataInjector )
100- val deltaOpt : Option [PlanDataInjector ] =
104+ val discovered : Seq [PlanDataInjector ] =
101105 try {
102- // Scala compiles `object Foo` into BOTH `Foo.class` (a static-forwarder
103- // class with no MODULE$ field) AND `Foo$.class` (the module class that
104- // does have MODULE$). The trailing `$` selects the module class.
105- // scalastyle:off classforname
106- val cls = Class .forName(" org.apache.spark.sql.comet.DeltaPlanDataInjector$" )
107- // scalastyle:on classforname
108- Some (cls.getField(" MODULE$" ).get(null ).asInstanceOf [PlanDataInjector ])
106+ ServiceLoader .load(classOf [PlanDataInjector ], getClass.getClassLoader).asScala.toSeq
109107 } catch {
110- // Default builds (no -Pcontrib-delta) won't have the class -> silent None.
111- // This is the only EXPECTED miss, so it's the only quiet one.
112- case _ : ClassNotFoundException => None
113- // The class IS on the classpath but couldn't be bound: missing MODULE$,
114- // access drift (NoSuchField/IllegalAccess), an initializer/linkage error,
115- // or a CCE on the PlanDataInjector cast. That's a misbuilt contrib jar, not
116- // a default build -- warn so it's diagnosable, then still decline so the
117- // rest of the planner stays alive.
118- case e : Throwable =>
108+ // A misbuilt contrib jar -- a malformed service file, or a listed provider that can't be
109+ // instantiated -- surfaces as ServiceConfigurationError while the iterator is forced.
110+ // Warn so it's diagnosable, then continue with the built-ins so the planner stays alive.
111+ // NonFatal covers ServiceConfigurationError (it is not a LinkageError).
112+ case NonFatal (e) =>
119113 logWarning(
120- " Found org.apache.spark.sql.comet.DeltaPlanDataInjector$ on classpath " +
121- " but failed to load it; skipping contrib-delta plan-data injection " ,
114+ " Failed to load contrib PlanDataInjector services; " +
115+ " continuing with built-in injectors only " ,
122116 e)
123- None
117+ Seq .empty
124118 }
125- builtin ++ deltaOpt
119+ builtin ++ discovered
126120 }
127121
128122 // O(1) lookup by op kind: most operators in any tree don't match any injector, so the per-op
@@ -182,6 +176,73 @@ private[comet] object PlanDataInjector extends Logging {
182176 codedOutput.checkNoSpaceLeft()
183177 bytes
184178 }
179+
180+ /**
181+ * Find all plan nodes with per-partition planning data in the plan tree. Returns two maps keyed
182+ * by a unique identifier: one for common data (shared across partitions) and one for
183+ * per-partition data.
184+ *
185+ * Recognises Iceberg scans (keyed by metadata_location) plus any leaf scan that surfaces its
186+ * data via the [[CometScanWithPlanData ]] trait (`CometNativeScanExec` and out-of-tree contrib
187+ * scans such as the Delta contrib's `CometDeltaNativeScanExec`).
188+ *
189+ * Stops at stage boundaries (shuffle exchanges, etc.) because partition indices are only valid
190+ * within the same stage.
191+ *
192+ * @return
193+ * (commonByKey, perPartitionByKey) - common data is shared, per-partition varies
194+ */
195+ private [comet] def findAllPlanData (
196+ plan : SparkPlan ): (Map [String , Array [Byte ]], Map [String , Array [Array [Byte ]]]) = {
197+ plan match {
198+ case iceberg : CometIcebergNativeScanExec =>
199+ // Trigger Spark's standard prepare -> waitForSubqueries lifecycle so DPP
200+ // InSubqueryExec values are resolved before commonData is read. Without this,
201+ // the parent CometNativeExec.executeQuery flow never invokes the scan's
202+ // executeQuery, leaving DPP unresolved and forcing a sync-on-this await inside
203+ // the serializedPartitionData lazy val initializer (a known deadlock surface).
204+ iceberg.ensureSubqueriesResolved()
205+ if (iceberg.commonData.nonEmpty && iceberg.perPartitionData.nonEmpty) {
206+ (
207+ Map (iceberg.metadataLocation -> iceberg.commonData),
208+ Map (iceberg.metadataLocation -> iceberg.perPartitionData))
209+ } else {
210+ (Map .empty, Map .empty)
211+ }
212+
213+ // Generic path for leaf scans that surface planning data via the
214+ // `CometScanWithPlanData` trait. Catches `CometNativeScanExec` and any contrib
215+ // leaf scan (e.g. the Delta contrib's `CometDeltaNativeScanExec`) without
216+ // requiring core to compile-time reference contrib classes. The trait's
217+ // `self: CometLeafExec` self-type guarantees this is also a `CometLeafExec`, so
218+ // the compound pattern always matches and we can drive the subquery lifecycle
219+ // directly -- there is no silent "not a leaf" skip.
220+ case s : CometLeafExec with CometScanWithPlanData =>
221+ s.ensureSubqueriesResolved()
222+ if (s.commonData.nonEmpty && s.perPartitionData.nonEmpty) {
223+ (Map (s.sourceKey -> s.commonData), Map (s.sourceKey -> s.perPartitionData))
224+ } else {
225+ (Map .empty, Map .empty)
226+ }
227+
228+ // Broadcast stages are boundaries - don't collect per-partition data from inside them.
229+ // After DPP filtering, broadcast scans may have different partition counts than the
230+ // probe side, causing ArrayIndexOutOfBoundsException in CometExecRDD.getPartitions.
231+ case _ : BroadcastQueryStageExec | _ : CometBroadcastExchangeExec =>
232+ (Map .empty, Map .empty)
233+
234+ // Stage boundaries - stop searching (partition indices won't align after these)
235+ case _ : ShuffleQueryStageExec | _ : AQEShuffleReadExec | _ : CometShuffleExchangeExec |
236+ _ : CometUnionExec | _ : CometTakeOrderedAndProjectExec | _ : CometCoalesceExec |
237+ _ : ReusedExchangeExec | _ : CometSparkToColumnarExec =>
238+ (Map .empty, Map .empty)
239+
240+ // Continue searching through other operators, combining results from all children
241+ case _ =>
242+ val results = plan.children.map(findAllPlanData)
243+ (results.flatMap(_._1).toMap, results.flatMap(_._2).toMap)
244+ }
245+ }
185246}
186247
187248/**
@@ -568,7 +629,7 @@ abstract class CometNativeExec extends CometExec {
568629 }
569630
570631 // Find planning data within this stage (stops at shuffle boundaries).
571- val (commonByKey, perPartitionByKey) = findAllPlanData(this )
632+ val (commonByKey, perPartitionByKey) = PlanDataInjector . findAllPlanData(this )
572633
573634 // Collect the input batches from the child operators. Non-shuffle inputs become
574635 // RDD[ArrowArrayStream] (one stream per partition, exported via the C Stream Interface
@@ -791,68 +852,6 @@ abstract class CometNativeExec extends CometExec {
791852 indices.toSet
792853 }
793854
794- /**
795- * Find all plan nodes with per-partition planning data in the plan tree. Returns two maps keyed
796- * by a unique identifier: one for common data (shared across partitions) and one for
797- * per-partition data.
798- *
799- * Currently supports Iceberg scans (keyed by metadata_location). Additional scan types can be
800- * added by extending this method.
801- *
802- * Stops at stage boundaries (shuffle exchanges, etc.) because partition indices are only valid
803- * within the same stage.
804- *
805- * @return
806- * (commonByKey, perPartitionByKey) - common data is shared, per-partition varies
807- */
808- private def findAllPlanData (
809- plan : SparkPlan ): (Map [String , Array [Byte ]], Map [String , Array [Array [Byte ]]]) = {
810- plan match {
811- case iceberg : CometIcebergNativeScanExec =>
812- // Trigger Spark's standard prepare -> waitForSubqueries lifecycle so DPP
813- // InSubqueryExec values are resolved before commonData is read. Without this,
814- // the parent CometNativeExec.executeQuery flow never invokes the scan's
815- // executeQuery, leaving DPP unresolved and forcing a sync-on-this await inside
816- // the serializedPartitionData lazy val initializer (a known deadlock surface).
817- iceberg.ensureSubqueriesResolved()
818- if (iceberg.commonData.nonEmpty && iceberg.perPartitionData.nonEmpty) {
819- (
820- Map (iceberg.metadataLocation -> iceberg.commonData),
821- Map (iceberg.metadataLocation -> iceberg.perPartitionData))
822- } else {
823- (Map .empty, Map .empty)
824- }
825-
826- // Generic path for leaf scans that surface planning data via the
827- // `CometScanWithPlanData` trait. Catches `CometNativeScanExec` and any contrib
828- // leaf scan (e.g. the Delta contrib's `CometDeltaNativeScanExec`) without
829- // requiring core to compile-time reference contrib classes.
830- case s : CometScanWithPlanData =>
831- s match {
832- case leaf : CometLeafExec => leaf.ensureSubqueriesResolved()
833- case _ => // no DPP lifecycle to drive
834- }
835- (Map (s.sourceKey -> s.commonData), Map (s.sourceKey -> s.perPartitionData))
836-
837- // Broadcast stages are boundaries - don't collect per-partition data from inside them.
838- // After DPP filtering, broadcast scans may have different partition counts than the
839- // probe side, causing ArrayIndexOutOfBoundsException in CometExecRDD.getPartitions.
840- case _ : BroadcastQueryStageExec | _ : CometBroadcastExchangeExec =>
841- (Map .empty, Map .empty)
842-
843- // Stage boundaries - stop searching (partition indices won't align after these)
844- case _ : ShuffleQueryStageExec | _ : AQEShuffleReadExec | _ : CometShuffleExchangeExec |
845- _ : CometUnionExec | _ : CometTakeOrderedAndProjectExec | _ : CometCoalesceExec |
846- _ : ReusedExchangeExec | _ : CometSparkToColumnarExec =>
847- (Map .empty, Map .empty)
848-
849- // Continue searching through other operators, combining results from all children
850- case _ =>
851- val results = plan.children.map(findAllPlanData)
852- (results.flatMap(_._1).toMap, results.flatMap(_._2).toMap)
853- }
854- }
855-
856855 /**
857856 * Converts this native Comet operator and its children into a native block which can be
858857 * executed as a whole (i.e., in a single JNI call) from the native side.
@@ -953,10 +952,15 @@ abstract class CometLeafExec extends CometNativeExec with LeafExecNode {
953952 * parent's native execution receives an empty input. (`CometIcebergNativeScanExec` does NOT use
954953 * this trait; it has a dedicated `findAllPlanData` case.)
955954 *
956- * Each implementation also resolves its own DPP subqueries via `ensureSubqueriesResolved`
957- * (overridden from [[CometLeafExec ]]) before `commonData`/`perPartitionData` are read.
955+ * Each implementation also resolves its own DPP subqueries via `ensureSubqueriesResolved` before
956+ * `commonData`/`perPartitionData` are read. That method lives on [[CometLeafExec ]], so the `self:
957+ * CometLeafExec` self-type below makes "is a leaf scan" a compile-time requirement: an
958+ * implementer cannot forget to extend [[CometLeafExec ]] (which would otherwise compile and then
959+ * silently skip subquery resolution -- the deadlock surface `ensureSubqueriesResolved` exists to
960+ * prevent). It also lets [[PlanDataInjector.findAllPlanData ]] drive the lifecycle without a
961+ * runtime "not a leaf" fallback.
958962 */
959- trait CometScanWithPlanData {
963+ trait CometScanWithPlanData { self : CometLeafExec =>
960964 def sourceKey : String
961965 def commonData : Array [Byte ]
962966 def perPartitionData : Array [Array [Byte ]]
@@ -974,6 +978,10 @@ trait CometScanWithPlanData {
974978 // that gets dropped when the enclosing native block is rebuilt. Only called
975979 // when `dynamicPruningFilters` is non-empty, so the default is never reached
976980 // for scans that leave it empty.
981+ //
982+ // TODO(#3510): once TreeNode.makeCopy preserves @transient fields, the
983+ // mutate-and-return-`this` workaround can collapse back to a normal copy, mirroring
984+ // the matching TODO in CometPlanAdaptiveDynamicPruningFilters.
977985 def withDynamicPruningFilters (filters : Seq [Expression ]): SparkPlan =
978986 throw new UnsupportedOperationException (
979987 s " ${getClass.getSimpleName} exposes dynamicPruningFilters but does not " +
0 commit comments