Skip to content

Commit 8d0bb01

Browse files
authored
fix: initialize dispatched kernels with the partition the native plan computes (#6700)
* fix: initialize dispatched kernels with the partition the native plan computes The JVM codegen dispatcher seeded kernels from `TaskContext.partitionId()`, which differs from the partition being computed under a union, a coalesce and a cartesian product. It also kept one kernel per task, so the parent partitions of a coalesce shared random state. The native planner now passes its partition index and plan id through the UDF bridge. Nondeterministic kernels are cached per plan and initialized with that index. Deterministic kernels never read it and stay shared across the plans of a task. * fix: drop a closed plan's dispatched kernels and check every node for determinism A coalesce kept the nondeterministic kernel of every parent partition until the task ended. `CometExecIterator.close` now calls `CometUdfBridge.releasePlan`, which hands the closed plan to a new `CometUDF.releasePlan` hook, and the dispatcher drops that plan's kernels there. Kernels shared by every plan stay until the task ends. The dispatcher shared a kernel whenever the root expression reported deterministic, but `Invoke` leaves its target object out of `deterministic`. So `make_valid_utf8` over `spark_partition_id()` still shared one kernel across the plans of a coalesce. It now checks every node.
1 parent 6cd5ea0 commit 8d0bb01

11 files changed

Lines changed: 197 additions & 34 deletions

File tree

‎native/core/src/execution/planner.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1045,6 +1045,8 @@ impl PhysicalPlanner {
10451045
udf.return_nullable,
10461046
self.task_context.clone(),
10471047
self.class_loader.clone(),
1048+
self.partition,
1049+
self.exec_context_id,
10481050
)))
10491051
}
10501052
ExprStruct::NativeScalarUdf(call) => {

‎native/core/src/parquet/parquet_exec.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -417,6 +417,8 @@ mod tests {
417417
true,
418418
None,
419419
None,
420+
0,
421+
0,
420422
));
421423
let literal: Arc<dyn PhysicalExpr> = Arc::new(Literal::new(ScalarValue::Int32(Some(1))));
422424
let nested: Arc<dyn PhysicalExpr> =

‎native/jni-bridge/src/comet_udf_bridge.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ impl<'a> CometUdfBridge<'a> {
4242
JNIString::new(Self::JVM_CLASS),
4343
jni::jni_str!("evaluate"),
4444
jni::jni_sig!(
45-
"(Ljava/lang/String;[J[JJJILorg/apache/spark/TaskContext;Ljava/lang/ClassLoader;)V"
45+
"(Ljava/lang/String;[J[JJJIIJLorg/apache/spark/TaskContext;Ljava/lang/ClassLoader;)V"
4646
),
4747
)?,
4848
method_evaluate_ret: ReturnType::Primitive(Primitive::Void),

‎native/spark-expr/src/jvm_udf/mod.rs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,16 +54,23 @@ pub struct JvmScalarUdfExpr {
5454
/// duration of the call. `None` when no driving Spark task is available (unit tests, direct
5555
/// native driver runs); the bridge then installs nothing.
5656
class_loader: Option<Arc<Global<JObject<'static>>>>,
57+
/// Index of the partition this native plan computes. See `CometUDF.evaluate`.
58+
partition: i32,
59+
/// Id of this native plan. See `CometUDF.evaluate`.
60+
exec_context_id: i64,
5761
}
5862

5963
impl JvmScalarUdfExpr {
64+
#[allow(clippy::too_many_arguments)]
6065
pub fn new(
6166
class_name: String,
6267
args: Vec<Arc<dyn PhysicalExpr>>,
6368
return_type: DataType,
6469
return_nullable: bool,
6570
task_context: Option<Arc<Global<JObject<'static>>>>,
6671
class_loader: Option<Arc<Global<JObject<'static>>>>,
72+
partition: i32,
73+
exec_context_id: i64,
6774
) -> Self {
6875
debug_assert!(
6976
!class_name.is_empty(),
@@ -76,6 +83,8 @@ impl JvmScalarUdfExpr {
7683
return_nullable,
7784
task_context,
7885
class_loader,
86+
partition,
87+
exec_context_id,
7988
}
8089
}
8190
}
@@ -234,6 +243,8 @@ impl PhysicalExpr for JvmScalarUdfExpr {
234243
JValue::Long(out_arr_ptr).as_jni(),
235244
JValue::Long(out_sch_ptr).as_jni(),
236245
JValue::Int(batch.num_rows() as i32).as_jni(),
246+
JValue::Int(self.partition).as_jni(),
247+
JValue::Long(self.exec_context_id).as_jni(),
237248
JValue::Object(task_context_ref).as_jni(),
238249
JValue::Object(class_loader_ref).as_jni(),
239250
],
@@ -276,6 +287,8 @@ impl PhysicalExpr for JvmScalarUdfExpr {
276287
self.return_nullable,
277288
self.task_context.clone(),
278289
self.class_loader.clone(),
290+
self.partition,
291+
self.exec_context_id,
279292
)))
280293
}
281294
}

‎spark/src/main/java/org/apache/comet/udf/CometUdfBridge.java‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@
3232
import org.apache.spark.comet.CometTaskContextShim;
3333
import org.apache.spark.util.TaskCompletionListener;
3434

35+
import com.google.common.annotations.VisibleForTesting;
36+
3537
import org.apache.comet.util.ClassLoaders;
3638

3739
/**
@@ -86,6 +88,8 @@ public class CometUdfBridge {
8688
* @param numRows row count of the current batch. Mirrors DataFusion's {@code
8789
* ScalarFunctionArgs.number_rows}; the only batch-size signal a zero-input UDF (e.g. a
8890
* zero-arg non-deterministic ScalaUDF) ever sees.
91+
* @param partitionIndex index of the partition the calling native plan computes
92+
* @param planId id of the calling native plan
8993
* @param taskContext propagated Spark {@link TaskContext} from the driving Spark task thread, or
9094
* {@code null} outside a Spark task. Treated as ground truth for the call: installed as the
9195
* thread-local on entry, with the prior value (if any) saved and restored in {@code finally}.
@@ -113,6 +117,8 @@ public static void evaluate(
113117
long outArrayPtr,
114118
long outSchemaPtr,
115119
int numRows,
120+
int partitionIndex,
121+
long planId,
116122
TaskContext taskContext,
117123
ClassLoader classLoader) {
118124
assert udfClassName != null && !udfClassName.isEmpty() : "udfClassName must be non-empty";
@@ -148,6 +154,8 @@ public static void evaluate(
148154
outArrayPtr,
149155
outSchemaPtr,
150156
numRows,
157+
partitionIndex,
158+
planId,
151159
taskContext);
152160
} finally {
153161
// Unconditional: a no-op when nothing was installed, and it also undoes any change the
@@ -170,6 +178,8 @@ private static void evaluateInternal(
170178
long outArrayPtr,
171179
long outSchemaPtr,
172180
int numRows,
181+
int partitionIndex,
182+
long planId,
173183
TaskContext taskContext) {
174184
long taskAttemptId = (taskContext != null) ? taskContext.taskAttemptId() : NO_TASK_ID;
175185

@@ -228,7 +238,7 @@ private static void evaluateInternal(
228238
inputs[i] = Data.importVector(importAllocator, inArr, inSch, null);
229239
}
230240

231-
result = udf.evaluate(inputs, numRows);
241+
result = udf.evaluate(inputs, numRows, partitionIndex, planId);
232242
// Recorded before the checks below, so the invariant holds however this exits.
233243
resultIsInput = isOneOf(result, inputs);
234244
if (!(result instanceof FieldVector)) {
@@ -290,6 +300,27 @@ private static void evaluateInternal(
290300
}
291301
}
292302

303+
/**
304+
* Called by {@code CometExecIterator} once native plan {@code planId} of task {@code
305+
* taskAttemptId} has closed. Passes the plan to every {@link CometUDF} instance of the task, see
306+
* {@code CometUDF.releasePlan}.
307+
*/
308+
public static void releasePlan(long taskAttemptId, long planId) {
309+
ConcurrentHashMap<String, CometUDF> perTask = INSTANCES.get(taskAttemptId);
310+
if (perTask != null) {
311+
for (CometUDF udf : perTask.values()) {
312+
udf.releasePlan(planId);
313+
}
314+
}
315+
}
316+
317+
/** The instance of {@code udfClassName} that task {@code taskAttemptId} holds, or null. */
318+
@VisibleForTesting
319+
public static CometUDF instanceFor(long taskAttemptId, String udfClassName) {
320+
ConcurrentHashMap<String, CometUDF> perTask = INSTANCES.get(taskAttemptId);
321+
return perTask == null ? null : perTask.get(udfClassName);
322+
}
323+
293324
/** Whether the UDF handed back one of the vectors it was given, rather than a new one. */
294325
private static boolean isOneOf(ValueVector result, ValueVector[] inputs) {
295326
for (ValueVector input : inputs) {

‎spark/src/main/scala/org/apache/comet/CometExecIterator.scala‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ import org.apache.comet.exceptions.CometQueryExecutionException
4545
import org.apache.comet.parquet.CometFileKeyUnwrapper
4646
import org.apache.comet.serde.Config.ConfigMap
4747
import org.apache.comet.shuffle.ShufflePartitionPusher
48+
import org.apache.comet.udf.CometUdfBridge
4849
import org.apache.comet.vector.NativeUtil
4950

5051
/**
@@ -356,6 +357,10 @@ class CometExecIterator(
356357
// global refs.
357358
attempt(nativeLib.releasePlan(plan))
358359

360+
// Drops what the task's JVM UDFs keep for this plan, such as its dispatched kernels. `id` is
361+
// the plan id the native plan passed them.
362+
attempt(CometUdfBridge.releasePlan(taskAttemptId, id))
363+
359364
// Run the diagnostics even when teardown failed: a failed teardown is exactly when the
360365
// non-zero memory usage warning below is most informative.
361366
attempt {

‎spark/src/main/scala/org/apache/comet/udf/CometUDF.scala‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,4 +45,23 @@ import org.apache.arrow.vector.ValueVector
4545
*/
4646
trait CometUDF {
4747
def evaluate(inputs: Array[ValueVector], numRows: Int): ValueVector
48+
49+
/**
50+
* The overload the bridge calls. `partitionIndex` is the index of the partition the calling
51+
* native plan computes, which differs from `TaskContext.partitionId()` under a union, a
52+
* coalesce or a cartesian product. `planId` tells apart the plans one task runs, such as the
53+
* parent partitions of a coalesce. The default ignores both.
54+
*/
55+
def evaluate(
56+
inputs: Array[ValueVector],
57+
numRows: Int,
58+
partitionIndex: Int,
59+
planId: Long): ValueVector =
60+
evaluate(inputs, numRows)
61+
62+
/**
63+
* Called once native plan `planId` has closed, so an implementation that keeps state per plan
64+
* can drop it before the task ends. The default does nothing.
65+
*/
66+
def releasePlan(planId: Long): Unit = ()
4867
}

‎spark/src/main/scala/org/apache/comet/udf/codegen/CometScalaUDFCodegen.scala‎

Lines changed: 59 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ import scala.util.control.NonFatal
2929
import org.apache.arrow.vector._
3030
import org.apache.arrow.vector.complex.{ListVector, MapVector, StructVector}
3131
import org.apache.arrow.vector.types.pojo.Field
32-
import org.apache.spark.{SparkEnv, TaskContext}
32+
import org.apache.spark.SparkEnv
3333
import org.apache.spark.internal.Logging
3434
import org.apache.spark.sql.catalyst.expressions.Expression
3535
import org.apache.spark.sql.comet.util.Utils
@@ -41,8 +41,8 @@ import org.apache.comet.udf.CometUDF
4141

4242
/**
4343
* Arrow-direct codegen dispatcher. For each `(bound expression, input Arrow schema)` pair,
44-
* compiles a specialized [[CometBatchKernel]] on first encounter, initializes it with the task's
45-
* partition index, and caches the live instance.
44+
* compiles a specialized [[CometBatchKernel]] on first encounter, initializes it with the index
45+
* of the partition the calling native plan computes, and caches the live instance.
4646
*
4747
* Arg 0 is a `VarBinaryVector` scalar carrying the closure-serialized bound `Expression` bytes;
4848
* args 1..N are the data columns the `BoundReference`s read in ordinal order.
@@ -56,16 +56,20 @@ import org.apache.comet.udf.CometUDF
5656
* +----------------------------+ +----------------------------+ +----------------------------+
5757
* | Key: generated Java | | Key: task + UDF class | | Key: bound expression + |
5858
* | source | | | | input column shapes |
59-
* | Value: compiled Java class | | Value: dispatcher object | | Value: ready-to-run kernel |
60-
* | Scope: JVM, all queries | | Scope: one Spark task | | with state primed |
61-
* | share it | | | | Scope: one Spark task |
62-
* | Owner: Spark | | Owner: Comet | | (lives inside 2) |
59+
* | Value: compiled Java class | | Value: dispatcher object | | (+ native plan if |
60+
* | Scope: JVM, all queries | | Scope: one Spark task | | nondeterministic) |
61+
* | share it | | | | Value: ready-to-run kernel |
62+
* | Owner: Spark | | Owner: Comet | | with state primed |
63+
* | | | | | Scope: one Spark task |
64+
* | | | | | (lives inside 2), |
65+
* | | | | | or one native plan |
66+
* | | | | | if nondeterministic |
6367
* | | | | | Owner: Comet |
6468
* +----------------------------+ +----------------------------+ +----------------------------+
6569
* }}}
6670
*
67-
* Stateful expressions (`Rand`, `MonotonicallyIncreasingID`) advance inside the per-task kernel
68-
* across batches.
71+
* Stateful expressions (`Rand`, `MonotonicallyIncreasingID`) advance inside the per-plan kernel
72+
* across batches. `CometExecIterator.close` drops a plan's kernels through `releasePlan`.
6973
*
7074
* `evaluate` runs under `this.synchronized` because DataFusion operators like `HashJoinExec`
7175
* pipeline build/probe via `OnceAsync` (`tokio::spawn`), so multiple Tokio worker threads can
@@ -81,13 +85,34 @@ class CometScalaUDFCodegen extends CometUDF with Logging {
8185
* Per-task cache keyed on serialized expression bytes plus per-column specs. The deserialized
8286
* `boundExpr` carries mutable state (`NamedLambdaVariable.value` for HOFs, `Rand`'s
8387
* `XORShiftRandom`) that must not be shared across concurrent tasks running the same query;
84-
* keeping the cache per-task gives each task its own copy. Guarded by `this.synchronized`.
88+
* keeping the cache per-task gives each task its own copy. A nondeterministic kernel is seeded
89+
* from the partition its plan computes, so its key also holds the plan: each plan a task runs,
90+
* such as each parent partition of a coalesce, gets its own, dropped by `releasePlan` when the
91+
* plan closes. Guarded by `this.synchronized`.
8592
*/
8693
private val kernelCache
8794
: mutable.Map[CometScalaUDFCodegen.CacheKey, CometScalaUDFCodegen.CacheEntry] =
8895
mutable.HashMap.empty
8996

90-
override def evaluate(inputs: Array[ValueVector], numRows: Int): ValueVector = {
97+
// Kernels shared by every plan stay until the task ends.
98+
override def releasePlan(planId: Long): Unit = this.synchronized {
99+
kernelCache.keys.filter(_.planId == planId).toList.foreach(kernelCache.remove)
100+
}
101+
102+
/** Plan ids of the cached kernels, `NoPlan` for a shared one. */
103+
private[comet] def cachedPlanIds: List[Long] = this.synchronized {
104+
kernelCache.keysIterator.map(_.planId).toList.sorted
105+
}
106+
107+
// Callers that bypass the bridge (unit tests, benchmarks) have no native plan.
108+
override def evaluate(inputs: Array[ValueVector], numRows: Int): ValueVector =
109+
evaluate(inputs, numRows, partitionIndex = 0, planId = CometScalaUDFCodegen.NoPlan)
110+
111+
override def evaluate(
112+
inputs: Array[ValueVector],
113+
numRows: Int,
114+
partitionIndex: Int,
115+
planId: Long): ValueVector = {
91116
require(
92117
inputs.length >= 1,
93118
"CometScalaUDFCodegen requires at least 1 input (serialized expression), " +
@@ -115,12 +140,12 @@ class CometScalaUDFCodegen extends CometUDF with Logging {
115140
val n = numRows
116141
val specsSeq = specs.toIndexedSeq
117142

118-
val key = CometScalaUDFCodegen.CacheKey(ByteBuffer.wrap(bytes), specsSeq)
143+
val key = CometScalaUDFCodegen.CacheKey(planId, ByteBuffer.wrap(bytes), specsSeq)
119144

120145
// Cache lookup and `process` run under one lock to serialize concurrent Tokio callers that
121146
// would otherwise race on the kernel's per-batch instance fields.
122147
this.synchronized {
123-
val entry = lookupOrCompile(key, bytes, specsSeq)
148+
val entry = lookupOrCompile(key, bytes, specsSeq, partitionIndex)
124149

125150
val out = CometBatchKernelCodegen.allocateOutput(
126151
entry.outputField,
@@ -144,9 +169,14 @@ class CometScalaUDFCodegen extends CometUDF with Logging {
144169
private def lookupOrCompile(
145170
key: CometScalaUDFCodegen.CacheKey,
146171
bytes: Array[Byte],
147-
specs: IndexedSeq[ArrowColumnSpec]): CometScalaUDFCodegen.CacheEntry = {
172+
specs: IndexedSeq[ArrowColumnSpec],
173+
partitionIndex: Int): CometScalaUDFCodegen.CacheEntry = {
148174
assert(Thread.holdsLock(this), "lookupOrCompile must run under this.synchronized")
149-
kernelCache.get(key) match {
175+
// A deterministic kernel never reads the partition index, so one instance under the planless
176+
// key serves every plan in the task. Only a kernel with a nondeterministic node is stored per
177+
// plan.
178+
val sharedKey = key.copy(planId = CometScalaUDFCodegen.NoPlan)
179+
kernelCache.get(sharedKey).orElse(kernelCache.get(key)) match {
150180
case Some(entry) =>
151181
CometScalaUDFCodegen.cacheHitCount.incrementAndGet()
152182
entry
@@ -168,14 +198,17 @@ class CometScalaUDFCodegen extends CometUDF with Logging {
168198
}
169199
val compiled = CometBatchKernelCodegen.compile(boundExpr, specs)
170200
val kernel = compiled.newInstance()
171-
kernel.init(CometScalaUDFCodegen.currentPartitionIndex())
201+
kernel.init(partitionIndex)
172202
val outputField = CometBatchKernelCodegen.toFfiArrowField(
173203
"codegen_result",
174204
boundExpr.dataType,
175205
boundExpr.nullable)
176206
val entry =
177207
CometScalaUDFCodegen.CacheEntry(compiled, kernel, boundExpr.dataType, outputField)
178-
kernelCache.put(key, entry)
208+
// Walks the tree, because `deterministic` is not transitive everywhere. `Invoke` skips its
209+
// `targetObject`, which is where Spark 4's `make_valid_utf8` puts its input.
210+
val perPlan = boundExpr.exists(!_.deterministic)
211+
kernelCache.put(if (perPlan) key else sharedKey, entry)
179212
CometScalaUDFCodegen.compileCount.incrementAndGet()
180213
CometScalaUDFCodegen.recordCompiledSignature(specs, boundExpr.dataType)
181214
entry
@@ -300,21 +333,22 @@ object CometScalaUDFCodegen {
300333
}
301334

302335
/**
303-
* Partition index for the kernel's `init`. Expressions whose `doGenCode` calls
304-
* `addPartitionInitializationStatement` (`Rand`, `Randn`, `Uuid`) reseed mutable state from
305-
* this. Falls back to 0 when the dispatcher is exercised outside a Spark task (unit tests).
336+
* Plan id for kernels shared by every plan in a task, and for callers that bypass the bridge.
306337
*/
307-
private def currentPartitionIndex(): Int =
308-
Option(TaskContext.get()).map(_.partitionId()).getOrElse(0)
338+
private[comet] val NoPlan = -1L
309339

310340
/**
311-
* Cache key: serialized expression bytes plus per-column compile-time invariants. `hashCode`
312-
* walks `bytesKey` per lookup, so for large ScalaUDF closures it scales with closure size.
341+
* Cache key: calling native plan (`NoPlan` for a deterministic expression), serialized
342+
* expression bytes plus per-column compile-time invariants. `hashCode` walks `bytesKey` per
343+
* lookup, so for large ScalaUDF closures it scales with closure size.
313344
*
314345
* TODO(perf-cache-key): if hot, options are a driver-precomputed hash piggybacked through the
315346
* proto, per-instance last-key memoization, or a two-tier cache keyed on the generated source.
316347
*/
317-
final case class CacheKey(bytesKey: ByteBuffer, specs: IndexedSeq[ArrowColumnSpec])
348+
final case class CacheKey(
349+
planId: Long,
350+
bytesKey: ByteBuffer,
351+
specs: IndexedSeq[ArrowColumnSpec])
318352

319353
/** Snapshot of dispatcher cache counters and current size. */
320354
final case class DispatcherStats(compileCount: Long, cacheHitCount: Long, cacheSize: Int) {

‎spark/src/test/scala/org/apache/comet/CometCodegenSourceSuite.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1535,8 +1535,8 @@ class CometCodegenSourceSuite extends AnyFunSuite {
15351535
IndexedSeq[ArrowColumnSpec](ArrowColumnSpec(varCharVectorClass, nullable = true))
15361536
val nonNullable =
15371537
IndexedSeq[ArrowColumnSpec](ArrowColumnSpec(varCharVectorClass, nullable = false))
1538-
val k1 = CometScalaUDFCodegen.CacheKey(bytes, nullable)
1539-
val k2 = CometScalaUDFCodegen.CacheKey(bytes, nonNullable)
1538+
val k1 = CometScalaUDFCodegen.CacheKey(0L, bytes, nullable)
1539+
val k2 = CometScalaUDFCodegen.CacheKey(0L, bytes, nonNullable)
15401540
assert(
15411541
k1 != k2,
15421542
"expected nullable=true and nullable=false specs to produce distinct cache keys")

0 commit comments

Comments
 (0)