Skip to content

Commit 003f608

Browse files
authored
fix: propagate the Spark task ClassLoader to JVM UDF calls (#5282)
1 parent 93e0d7f commit 003f608

11 files changed

Lines changed: 302 additions & 28 deletions

File tree

‎.github/workflows/pr_build_linux.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -402,6 +402,7 @@ jobs:
402402
org.apache.comet.CometCodegenHOFSuite
403403
org.apache.comet.CometFuzzMathSuite
404404
org.apache.comet.CometCodegenFuzzSuite
405+
org.apache.comet.CometScalaUDFClassLoaderSuite
405406
org.apache.comet.codegen.CometSpecializedGettersDispatchSuite
406407
org.apache.comet.CometStringDecodeSuite
407408
org.apache.comet.CometWidthBucketSuite

‎.github/workflows/pr_build_macos.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,7 @@ jobs:
218218
org.apache.comet.CometCodegenHOFSuite
219219
org.apache.comet.CometFuzzMathSuite
220220
org.apache.comet.CometCodegenFuzzSuite
221+
org.apache.comet.CometScalaUDFClassLoaderSuite
221222
org.apache.comet.codegen.CometSpecializedGettersDispatchSuite
222223
org.apache.comet.CometStringDecodeSuite
223224
org.apache.comet.CometWidthBucketSuite

‎docs/source/user-guide/latest/scala_java_udfs.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ When a UDF is rejected, the reason surfaces through Comet's standard fallback di
5454

5555
- Non-deterministic expressions referenced from the argument tree (`rand`, `uuid`, `monotonically_increasing_id`) produce per-partition sequences consistent with Spark.
5656
- `TaskContext.get()` inside the user function returns the driving Spark task's context.
57+
- The Spark task thread's context ClassLoader is propagated to the thread that runs the user function, so functions defined in jars supplied with `--jars` / `spark.jars` resolve the same way they do under Spark's own execution.
5758
- The user function must be closure-serializable; the same function that works with Spark's executor execution works here.
5859

5960
## Known limitations

‎native/core/src/execution/jni_api.rs‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -352,6 +352,11 @@ struct ExecutionContext {
352352
/// cheap to clone; the underlying `Global<JObject>` releases its JNI global ref on drop
353353
/// via `jni`'s `Drop` impl.
354354
pub task_context: Option<Arc<Global<JObject<'static>>>>,
355+
/// Context `ClassLoader` of the driving Spark task thread, captured at `createPlan` time and
356+
/// threaded into every JVM scalar UDF the planner builds; see `CometUdfBridge.evaluate` for why
357+
/// it has to travel with the plan. `None` when no driving Spark task is present (unit tests,
358+
/// direct native driver runs). Lifetime is as for `task_context` above.
359+
pub class_loader: Option<Arc<Global<JObject<'static>>>>,
355360
}
356361

357362
/// Accept serialized query plan and return the address of the native query plan.
@@ -379,6 +384,7 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_createPlan(
379384
task_cpus: jlong,
380385
key_unwrapper_obj: JObject,
381386
task_context_obj: JObject,
387+
class_loader_obj: JObject,
382388
) -> jlong {
383389
try_unwrap_or_throw(&e, |env| {
384390
// Deserialize Spark configs
@@ -500,15 +506,21 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_createPlan(
500506
String::new()
501507
};
502508

503-
// Capture the driving Spark task's TaskContext as a JNI global reference when
504-
// non-null. The `Arc<Global<JObject>>` releases its global ref on drop, so cleanup
505-
// is automatic when the ExecutionContext drops.
509+
// Capture the driving Spark task's TaskContext and context ClassLoader as JNI global
510+
// references when non-null. The `Arc<Global<JObject>>` releases its global ref on
511+
// drop, so cleanup is automatic when the ExecutionContext drops.
506512
let task_context = if !task_context_obj.is_null() {
507513
Some(Arc::new(jni_new_global_ref!(env, task_context_obj)?))
508514
} else {
509515
None
510516
};
511517

518+
let class_loader = if !class_loader_obj.is_null() {
519+
Some(Arc::new(jni_new_global_ref!(env, class_loader_obj)?))
520+
} else {
521+
None
522+
};
523+
512524
let exec_context = Box::new(ExecutionContext {
513525
id,
514526
task_attempt_id,
@@ -536,6 +548,7 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_createPlan(
536548
),
537549
tracing_event_name,
538550
task_context,
551+
class_loader,
539552
});
540553

541554
Ok(Box::into_raw(exec_context) as i64)
@@ -782,7 +795,8 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_executePlan(
782795
PhysicalPlanner::new(Arc::clone(&exec_context.session_ctx), partition)
783796
.with_exec_id(exec_context_id)
784797
.with_sql_text_pool(&exec_context.spark_plan)
785-
.with_task_context(exec_context.task_context.clone());
798+
.with_task_context(exec_context.task_context.clone())
799+
.with_class_loader(exec_context.class_loader.clone());
786800
let (scans, shuffle_scans, root_op) = planner.create_plan(
787801
&exec_context.spark_plan,
788802
&mut exec_context.input_sources.clone(),

‎native/core/src/execution/planner.rs‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,10 @@ pub struct PhysicalPlanner {
250250
/// Captured at `createPlan` time on `ExecutionContext`; see that struct for the
251251
/// propagation rationale. `None` when no driving Spark task is available.
252252
task_context: Option<Arc<Global<JObject<'static>>>>,
253+
/// Context `ClassLoader` of the driving Spark task thread, captured at `createPlan` time on
254+
/// `ExecutionContext`; see that struct for the propagation rationale. `None` when no driving
255+
/// Spark task is available.
256+
class_loader: Option<Arc<Global<JObject<'static>>>>,
253257
}
254258

255259
impl Default for PhysicalPlanner {
@@ -267,6 +271,7 @@ impl PhysicalPlanner {
267271
query_context_registry: datafusion_comet_spark_expr::create_query_context_map(),
268272
sql_text_pool: vec![],
269273
task_context: None,
274+
class_loader: None,
270275
}
271276
}
272277

@@ -356,6 +361,17 @@ impl PhysicalPlanner {
356361
self
357362
}
358363

364+
/// Attach the driving Spark task thread's context `ClassLoader` as a global reference. Mirrors
365+
/// `with_task_context`: called by the JNI `executePlan` entry with whatever was captured at
366+
/// `createPlan` time, and cloned into every `JvmScalarUdfExpr` the planner builds.
367+
pub fn with_class_loader(
368+
mut self,
369+
class_loader: Option<Arc<Global<JObject<'static>>>>,
370+
) -> Self {
371+
self.class_loader = class_loader;
372+
self
373+
}
374+
359375
/// Return session context of this planner.
360376
pub fn session_ctx(&self) -> &Arc<SessionContext> {
361377
&self.session_ctx
@@ -876,19 +892,22 @@ impl PhysicalPlanner {
876892
to_arrow_datatype(udf.return_type.as_ref().ok_or_else(|| {
877893
GeneralError("JvmScalarUdf missing return_type".to_string())
878894
})?);
879-
// Invariant: task_context is propagated for every JvmScalarUdfExpr built during
880-
// normal execution. The TEST_EXEC_CONTEXT_ID path is the only context in which
881-
// task_context may legitimately be None (unit tests, direct native driver runs).
895+
// Invariant: task_context and class_loader are propagated for every
896+
// JvmScalarUdfExpr built during normal execution. The TEST_EXEC_CONTEXT_ID path is
897+
// the only context in which they may legitimately be None (unit tests, direct
898+
// native driver runs).
882899
debug_assert!(
883-
self.task_context.is_some() || self.exec_context_id == TEST_EXEC_CONTEXT_ID,
884-
"task_context must be set for non-test execution"
900+
(self.task_context.is_some() && self.class_loader.is_some())
901+
|| self.exec_context_id == TEST_EXEC_CONTEXT_ID,
902+
"task_context and class_loader must be set for non-test execution"
885903
);
886904
Ok(Arc::new(JvmScalarUdfExpr::new(
887905
udf.class_name.clone(),
888906
args,
889907
return_type,
890908
udf.return_nullable,
891909
self.task_context.clone(),
910+
self.class_loader.clone(),
892911
)))
893912
}
894913
expr => Err(GeneralError(format!("Not implemented: {expr:?}"))),

‎native/jni-bridge/src/comet_udf_bridge.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,9 @@ impl<'a> CometUdfBridge<'a> {
4141
method_evaluate: env.get_static_method_id(
4242
JNIString::new(Self::JVM_CLASS),
4343
jni::jni_str!("evaluate"),
44-
jni::jni_sig!("(Ljava/lang/String;[J[JJJILorg/apache/spark/TaskContext;)V"),
44+
jni::jni_sig!(
45+
"(Ljava/lang/String;[J[JJJILorg/apache/spark/TaskContext;Ljava/lang/ClassLoader;)V"
46+
),
4547
)?,
4648
method_evaluate_ret: ReturnType::Primitive(Primitive::Void),
4749
class,

‎native/spark-expr/src/jvm_udf/mod.rs‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,11 @@ pub struct JvmScalarUdfExpr {
4848
/// Spark task is available; the bridge then leaves whatever `TaskContext.get()` already
4949
/// returns in place.
5050
task_context: Option<Arc<Global<JObject<'static>>>>,
51+
/// Context `ClassLoader` of the driving Spark task thread, captured at `createPlan` time and
52+
/// threaded here by the planner. See `CometUdfBridge.evaluate`, which installs it for the
53+
/// duration of the call. `None` when no driving Spark task is available (unit tests, direct
54+
/// native driver runs); the bridge then installs nothing.
55+
class_loader: Option<Arc<Global<JObject<'static>>>>,
5156
}
5257

5358
impl JvmScalarUdfExpr {
@@ -57,6 +62,7 @@ impl JvmScalarUdfExpr {
5762
return_type: DataType,
5863
return_nullable: bool,
5964
task_context: Option<Arc<Global<JObject<'static>>>>,
65+
class_loader: Option<Arc<Global<JObject<'static>>>>,
6066
) -> Self {
6167
debug_assert!(
6268
!class_name.is_empty(),
@@ -68,6 +74,7 @@ impl JvmScalarUdfExpr {
6874
return_type,
6975
return_nullable,
7076
task_context,
77+
class_loader,
7178
}
7279
}
7380
}
@@ -197,14 +204,17 @@ impl PhysicalExpr for JvmScalarUdfExpr {
197204
.set_region(env, 0, &in_sch_ptrs)
198205
.map_err(|e| CometError::JNI { source: e })?;
199206

200-
// Resolve the TaskContext reference once before building the arg array so the
201-
// borrow lives until `call_static_method_unchecked` returns. When no TaskContext
202-
// was propagated, pass a null object so the bridge's null-guard leaves the thread-
203-
// local alone.
204-
let null_task_context = JObject::null();
207+
// Resolve the TaskContext and ClassLoader references once before building the arg
208+
// array so the borrows live until `call_static_method_unchecked` returns. Absent
209+
// values are passed as a null object, which the bridge's null-guards skip.
210+
let null_obj = JObject::null();
205211
let task_context_ref: &JObject = match &self.task_context {
206212
Some(gref) => gref.as_obj(),
207-
None => &null_task_context,
213+
None => &null_obj,
214+
};
215+
let class_loader_ref: &JObject = match &self.class_loader {
216+
Some(gref) => gref.as_obj(),
217+
None => &null_obj,
208218
};
209219
let ret = unsafe {
210220
env.call_static_method_unchecked(
@@ -219,6 +229,7 @@ impl PhysicalExpr for JvmScalarUdfExpr {
219229
JValue::Long(out_sch_ptr).as_jni(),
220230
JValue::Int(batch.num_rows() as i32).as_jni(),
221231
JValue::Object(task_context_ref).as_jni(),
232+
JValue::Object(class_loader_ref).as_jni(),
222233
],
223234
)
224235
};
@@ -255,6 +266,7 @@ impl PhysicalExpr for JvmScalarUdfExpr {
255266
self.return_type.clone(),
256267
self.return_nullable,
257268
self.task_context.clone(),
269+
self.class_loader.clone(),
258270
)))
259271
}
260272
}

‎spark/src/main/java/org/apache/comet/udf/CometUdfBridge.java‎

Lines changed: 29 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@
3131
import org.apache.spark.comet.CometTaskContextShim;
3232
import org.apache.spark.util.TaskCompletionListener;
3333

34+
import org.apache.comet.util.ClassLoaders;
35+
3436
/**
3537
* JNI entry point for native execution to invoke a {@link CometUDF}. Matches the static-method
3638
* pattern used by CometScalarSubquery so the native side can dispatch via
@@ -91,6 +93,17 @@ public class CometUdfBridge {
9193
* left on a worker by a previous task. Its task attempt ID also keys the UDF-instance cache,
9294
* so a UDF holding per-task state in fields sees a consistent instance for every call within
9395
* the task regardless of which Tokio worker is polling.
96+
* @param classLoader context ClassLoader captured on the driving Spark task thread, or {@code
97+
* null} outside a Spark task. Installed as this thread's context ClassLoader for the duration
98+
* of the call, with the prior value restored in {@code finally}. Tokio workers attach through
99+
* JNI and an attached thread has no context ClassLoader, so without this every lookup falls
100+
* back to the ClassLoader that loaded Comet, which never holds user jars ({@code --jars} /
101+
* {@code spark.jars}). Both the {@code CometUDF} resolution below and the closure
102+
* deserialization in {@code CometScalaUDFCodegen} depend on it. Installed per call rather
103+
* than once when the worker attaches, because the Tokio runtime is process-global: one worker
104+
* interleaves work from task attempts of different jobs, and under Spark Connect from
105+
* sessions with different artifact ClassLoaders. The loader is a property of the plan, not of
106+
* the thread.
94107
*/
95108
public static void evaluate(
96109
String udfClassName,
@@ -99,7 +112,8 @@ public static void evaluate(
99112
long outArrayPtr,
100113
long outSchemaPtr,
101114
int numRows,
102-
TaskContext taskContext) {
115+
TaskContext taskContext,
116+
ClassLoader classLoader) {
103117
assert udfClassName != null && !udfClassName.isEmpty() : "udfClassName must be non-empty";
104118
assert inputArrayPtrs != null && inputSchemaPtrs != null
105119
: "input pointer arrays must be non-null";
@@ -111,13 +125,20 @@ public static void evaluate(
111125

112126
// Save-and-restore rather than only-install-if-null: the propagated `taskContext` is the
113127
// ground truth for this call. Any value already on the thread is either (a) the same object
114-
// on a Spark task thread, or (b) stale from a prior task on a reused Tokio worker.
128+
// on a Spark task thread, or (b) stale from a prior task on a reused Tokio worker. The same
129+
// reasoning applies to the propagated `classLoader`.
115130
TaskContext prior = TaskContext.get();
116131
if (taskContext != null) {
117132
CometTaskContextShim.set(taskContext);
118133
assert TaskContext.get() == taskContext
119134
: "TaskContext install did not take effect on this thread";
120135
}
136+
Thread currentThread = Thread.currentThread();
137+
ClassLoader priorLoader = currentThread.getContextClassLoader();
138+
if (classLoader != null) {
139+
currentThread.setContextClassLoader(classLoader);
140+
}
141+
121142
try {
122143
evaluateInternal(
123144
udfClassName,
@@ -128,6 +149,9 @@ public static void evaluate(
128149
numRows,
129150
taskContext);
130151
} finally {
152+
// Unconditional: a no-op when nothing was installed, and it also undoes any change the
153+
// user function made to the ClassLoader of a worker that outlives this call.
154+
currentThread.setContextClassLoader(priorLoader);
131155
if (taskContext != null) {
132156
if (prior != null) {
133157
CometTaskContextShim.set(prior);
@@ -175,14 +199,10 @@ private static void evaluateInternal(
175199
udfClassName,
176200
name -> {
177201
try {
178-
// Resolve via the executor's context classloader so user-supplied UDF jars
179-
// (added via spark.jars / --jars) are visible.
180-
ClassLoader cl = Thread.currentThread().getContextClassLoader();
181-
if (cl == null) {
182-
cl = CometUdfBridge.class.getClassLoader();
183-
}
202+
// Resolves through the context ClassLoader installed by `evaluate`, so a
203+
// user-supplied CometUDF shipped in a user jar is visible.
184204
return (CometUDF)
185-
Class.forName(name, true, cl).getDeclaredConstructor().newInstance();
205+
ClassLoaders.loadClass(name).getDeclaredConstructor().newInstance();
186206
} catch (ReflectiveOperationException e) {
187207
throw new RuntimeException("Failed to instantiate CometUDF: " + name, e);
188208
}

‎spark/src/main/scala/org/apache/comet/CometExecIterator.scala‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -125,8 +125,11 @@ class CometExecIterator(
125125
taskCPUs,
126126
keyUnwrapper,
127127
// Propagated to Tokio workers running JVM UDFs so they see this Spark task's
128-
// TaskContext. See CometUdfBridge.evaluate.
129-
TaskContext.get())
128+
// TaskContext and context ClassLoader. Read here because this class is only ever
129+
// constructed on a Spark task thread (see `taskAttemptId` above); a JNI-attached Tokio
130+
// worker has neither. See CometUdfBridge.evaluate.
131+
TaskContext.get(),
132+
Thread.currentThread().getContextClassLoader)
130133
}
131134

132135
private var nextBatch: Option[ColumnarBatch] = None

‎spark/src/main/scala/org/apache/comet/Native.scala‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,11 @@ class Native extends NativeBase {
4848
* @param taskMemoryManager
4949
* the task-level memory manager that is responsible for tracking memory usage across JVM and
5050
* native side.
51+
* @param taskContext
52+
* the driving Spark task's `TaskContext`, propagated to Tokio workers running JVM UDFs.
53+
* @param classLoader
54+
* the calling Spark task thread's context ClassLoader, propagated to Tokio workers running
55+
* JVM UDFs. Must be read on the task thread; see `CometUdfBridge.evaluate`.
5156
* @return
5257
* the address to native query plan.
5358
*/
@@ -70,7 +75,8 @@ class Native extends NativeBase {
7075
taskAttemptId: Long,
7176
taskCPUs: Long,
7277
keyUnwrapper: CometFileKeyUnwrapper,
73-
taskContext: TaskContext): Long
78+
taskContext: TaskContext,
79+
classLoader: ClassLoader): Long
7480
// scalastyle:on
7581

7682
/**

0 commit comments

Comments
 (0)