Skip to content

Commit ae98cef

Browse files
committed
Merge current main into AVG compatibility fixes
Preserve the upstream RSS callback and protocol tests alongside the wide-decimal native shuffle routing expectation. Combine the conflicting DecimalType and StructType imports without changing either implementation.
2 parents 6dca7f1 + 525c05b commit ae98cef

38 files changed

Lines changed: 3631 additions & 91 deletions

File tree

‎docs/source/contributor-guide/expression-audits/string_funcs.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,14 @@
245245
- Spark 3.5.8 (audited 2026-05-27): registry alias of `Upper`. Same support as `upper`.
246246
- Spark 4.0.1 (audited 2026-05-27): unchanged alias of `Upper`.
247247

248+
## unbase64
249+
250+
- Spark 3.4.3 (audited 2026-08-24): identical to 3.5.8.
251+
- Spark 3.5.8 (audited 2026-08-24): baseline. `doGenCode` emits `java.util.Base64.getMimeDecoder().decode(child.toString())`. The MIME decoder skips every byte outside the base64 alphabet, so CRLF-wrapped output from Spark's own `base64` round-trips cleanly; four terminal-shape errors surface as `IllegalArgumentException`. `failOnError = true` is set only when the node is constructed from `to_binary('base64')` / `try_to_binary`, which use a stricter RFC 4648 validator.
252+
- Spark 4.0.1 (audited 2026-08-24): `NullIntolerant` becomes `override def nullIntolerant: Boolean = true`; `inputTypes` widens to `StringTypeWithCollation(supportsTrimCollation = true)`. Behaviour is byte-level and collation-independent, so no divergence for `UTF8_BINARY` and no shim is needed.
253+
- Spark 4.1.1 (audited 2026-08-24): adds `contextIndependentFoldable`; no behavioural change on the decode path.
254+
- Comet native implementation (`spark_unbase64`, `native/spark-expr/src/string_funcs/unbase64.rs`) ports the JDK MIME decoder rules: 256-entry decode LUT, a reused per-batch scratch `Vec<u8>` copied into a preallocated `BinaryBuilder`, all four error messages reproduced verbatim. `CometUnBase64` handles `failOnError = false` natively; `failOnError = true` (reachable from `to_binary('base64')` / `try_to_binary`) requires strict RFC 4648 validation and is not yet implemented natively, so those cases stay on the JVM codegen dispatcher via `CodegenDispatchFallback`.
255+
248256
## upper
249257

250258
- Spark 3.4.3 (audited 2026-05-27): identical to 3.5.8.

‎native/core/src/execution/jni_api.rs‎

Lines changed: 47 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ use crate::{
2424
metrics::utils::update_comet_metric, planner::PhysicalPlanner, serde::to_arrow_datatype,
2525
shuffle::spark_unsafe::row::process_sorted_row_partition, sort::RdxSort,
2626
},
27-
jvm_bridge::JVMClasses,
27+
jvm_bridge::{JVMClasses, JavaShufflePartitionPusher, ShufflePartitionPusher},
2828
};
2929
use std::collections::HashSet;
3030

@@ -393,6 +393,9 @@ struct ExecutionContext {
393393
/// it has to travel with the plan. `None` when no driving Spark task is present (unit tests,
394394
/// direct native driver runs). Lifetime is as for `task_context` above.
395395
pub class_loader: Option<Arc<Global<JObject<'static>>>>,
396+
/// Task-owned remote shuffle callback, registered before native planning starts.
397+
/// The callback owns a JNI global reference and can safely run on Tokio workers.
398+
pub shuffle_partition_pusher: Option<Arc<dyn ShufflePartitionPusher>>,
396399
/// Removes this context's tracing memory-pool entry on every exit path.
397400
memory_pool_registration: Option<ThreadMemoryPoolRegistration>,
398401
}
@@ -583,6 +586,7 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_createPlan(
583586
tracing_event_name,
584587
task_context,
585588
class_loader,
589+
shuffle_partition_pusher: None,
586590
memory_pool_registration,
587591
});
588592

@@ -591,6 +595,44 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_createPlan(
591595
})
592596
}
593597

598+
/// Binds one task-owned shuffle callback before native execution is initialized.
599+
///
600+
/// Keeping callback registration separate preserves the existing `createPlan` JNI ABI for
601+
/// all local shuffle and non-shuffle callers.
602+
#[no_mangle]
603+
pub extern "system" fn Java_org_apache_comet_Native_setShufflePartitionPusher(
604+
e: EnvUnowned,
605+
_class: JClass,
606+
exec_context: jlong,
607+
callback: JObject,
608+
) {
609+
try_unwrap_or_throw(&e, |env| {
610+
if exec_context == 0 {
611+
return Err(CometError::NullPointer(
612+
"Remote shuffle callback requires a valid native execution plan".to_string(),
613+
));
614+
}
615+
616+
let exec_context = get_execution_context(exec_context);
617+
if exec_context.root_op.is_some() {
618+
return Err(CometError::Internal(
619+
"Remote shuffle callback cannot be registered after native execution starts"
620+
.to_string(),
621+
));
622+
}
623+
624+
if exec_context.shuffle_partition_pusher.is_some() {
625+
return Err(CometError::Internal(
626+
"Remote shuffle callback has already been registered for this task".to_string(),
627+
));
628+
}
629+
630+
let pusher = JavaShufflePartitionPusher::try_new(env, &callback)?;
631+
exec_context.shuffle_partition_pusher = Some(Arc::new(pusher));
632+
Ok(())
633+
})
634+
}
635+
594636
/// Configure DataFusion session context.
595637
fn prepare_datafusion_session_context(
596638
batch_size: usize,
@@ -831,7 +873,10 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_executePlan(
831873
.with_exec_id(exec_context_id)
832874
.with_sql_text_pool(&exec_context.spark_plan)
833875
.with_task_context(exec_context.task_context.clone())
834-
.with_class_loader(exec_context.class_loader.clone());
876+
.with_class_loader(exec_context.class_loader.clone())
877+
.with_shuffle_partition_pusher(
878+
exec_context.shuffle_partition_pusher.clone(),
879+
);
835880
let (scans, shuffle_scans, root_op) = planner.create_plan(
836881
&exec_context.spark_plan,
837882
&mut exec_context.input_sources.clone(),

0 commit comments

Comments
 (0)