Skip to content

Commit 98cd8c9

Browse files
authored
feat: add raw Celeborn native shuffle reader (7/n) (#5531)
1 parent 92167ef commit 98cd8c9

25 files changed

Lines changed: 4447 additions & 121 deletions

‎.github/workflows/pr_build_linux.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -330,6 +330,7 @@ jobs:
330330
org.apache.comet.shuffle.CelebornShufflePartitionPusherSuite
331331
org.apache.spark.sql.comet.execution.shuffle.CometCelebornShuffleManagerSuite
332332
org.apache.spark.sql.comet.execution.shuffle.CometCelebornNativeShuffleWriterSuite
333+
org.apache.spark.sql.comet.execution.shuffle.CometCelebornShuffleReaderSuite
333334
org.apache.spark.sql.comet.execution.shuffle.CometNativeShuffleInputRDDSuite
334335
org.apache.comet.exec.CometShuffleEncryptionSuite
335336
org.apache.comet.exec.CometShuffleManagerSuite

‎.github/workflows/pr_build_macos.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ jobs:
146146
org.apache.comet.shuffle.CelebornShufflePartitionPusherSuite
147147
org.apache.spark.sql.comet.execution.shuffle.CometCelebornShuffleManagerSuite
148148
org.apache.spark.sql.comet.execution.shuffle.CometCelebornNativeShuffleWriterSuite
149+
org.apache.spark.sql.comet.execution.shuffle.CometCelebornShuffleReaderSuite
149150
org.apache.spark.sql.comet.execution.shuffle.CometNativeShuffleInputRDDSuite
150151
org.apache.comet.exec.CometShuffleEncryptionSuite
151152
org.apache.comet.exec.CometShuffleManagerSuite

‎native/core/src/execution/jni_api.rs‎

Lines changed: 62 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ use datafusion::{
4141
physical_plan::{display::DisplayableExecutionPlan, SendableRecordBatchStream},
4242
prelude::{SessionConfig, SessionContext},
4343
};
44-
use datafusion_comet_proto::spark_operator::Operator;
44+
use datafusion_comet_proto::spark_operator::{Operator, ShuffleScan};
4545
use datafusion_comet_spark_expr::url_funcs::{CometParseUrl, CometTryParseUrl};
4646
use datafusion_spark::function::array::array_contains::SparkArrayContains;
4747
use datafusion_spark::function::array::repeat::SparkArrayRepeat;
@@ -89,6 +89,7 @@ use jni::{
8989
Env, EnvUnowned,
9090
};
9191
use parking_lot::Mutex;
92+
use prost::Message;
9293
use std::collections::HashMap;
9394
use std::path::PathBuf;
9495
use std::time::{Duration, Instant};
@@ -100,7 +101,9 @@ use crate::execution::memory_pools::{
100101
create_memory_pool, handle_task_shared_pool_release, parse_memory_pool_config, MemoryPoolConfig,
101102
};
102103
use crate::execution::operators::{ScanExec, ShuffleScanExec};
103-
use crate::execution::shuffle::{read_ipc_compressed, CompressionCodec};
104+
use crate::execution::shuffle::{
105+
read_ipc_compressed, read_ipc_compressed_validated, validate_remote_schema, CompressionCodec,
106+
};
104107
use crate::execution::spark_plan::SparkPlan;
105108

106109
use crate::execution::tracing::{
@@ -1249,15 +1252,67 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_decodeShuffleBlock(
12491252
) -> jlong {
12501253
try_unwrap_or_throw(&e, |env| {
12511254
with_trace("decodeShuffleBlock", tracing_enabled != JNI_FALSE, || {
1252-
let raw_pointer = env.get_direct_buffer_address(&byte_buffer)?;
1253-
let length = length as usize;
1254-
let slice: &[u8] = unsafe { std::slice::from_raw_parts(raw_pointer, length) };
1255-
let batch = read_ipc_compressed(slice)?;
1256-
prepare_output(env, array_addrs, schema_addrs, batch, false)
1255+
decode_shuffle_block(env, byte_buffer, length, array_addrs, schema_addrs, None)
12571256
})
12581257
})
12591258
}
12601259

1260+
#[no_mangle]
1261+
/// Decode a remote native shuffle block with Arrow array and logical type validation enabled.
1262+
/// # Safety
1263+
/// This function is inherently unsafe since it deals with raw pointers passed from JNI.
1264+
pub unsafe extern "system" fn Java_org_apache_comet_Native_decodeShuffleBlockWithValidation(
1265+
e: EnvUnowned,
1266+
_class: JClass,
1267+
byte_buffer: JByteBuffer,
1268+
length: jint,
1269+
array_addrs: JLongArray,
1270+
schema_addrs: JLongArray,
1271+
tracing_enabled: jboolean,
1272+
expected_schema: JByteArray,
1273+
) -> jlong {
1274+
try_unwrap_or_throw(&e, |env| {
1275+
with_trace("decodeShuffleBlock", tracing_enabled != JNI_FALSE, || {
1276+
let bytes = env.convert_byte_array(expected_schema)?;
1277+
let schema = ShuffleScan::decode(bytes.as_slice()).map_err(|error| {
1278+
CometError::Internal(format!("Invalid expected remote shuffle schema: {error}"))
1279+
})?;
1280+
let expected_types: Vec<_> = schema.fields.iter().map(to_arrow_datatype).collect();
1281+
decode_shuffle_block(
1282+
env,
1283+
byte_buffer,
1284+
length,
1285+
array_addrs,
1286+
schema_addrs,
1287+
Some(&expected_types),
1288+
)
1289+
})
1290+
})
1291+
}
1292+
1293+
fn decode_shuffle_block(
1294+
env: &mut Env,
1295+
byte_buffer: JByteBuffer,
1296+
length: jint,
1297+
array_addrs: JLongArray,
1298+
schema_addrs: JLongArray,
1299+
expected_types: Option<&[ArrowDataType]>,
1300+
) -> CometResult<jlong> {
1301+
let raw_pointer = env.get_direct_buffer_address(&byte_buffer)?;
1302+
let length = length as usize;
1303+
let slice: &[u8] = unsafe { std::slice::from_raw_parts(raw_pointer, length) };
1304+
let batch = if let Some(expected_types) = expected_types {
1305+
let batch = read_ipc_compressed_validated(slice)?;
1306+
// Reject incompatible remote schemas before exporting arrays to the JVM. Casting or
1307+
// importing first can silently change values or bypass remote fetch-failure reporting.
1308+
validate_remote_schema(&batch, expected_types)?;
1309+
batch
1310+
} else {
1311+
read_ipc_compressed(slice)?
1312+
};
1313+
prepare_output(env, array_addrs, schema_addrs, batch, false)
1314+
}
1315+
12611316
#[no_mangle]
12621317
/// # Safety
12631318
/// This function is inherently unsafe since it deals with raw pointers passed from JNI.

‎native/core/src/execution/operators/shuffle_scan.rs‎

Lines changed: 170 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,13 @@
1818
use crate::{
1919
errors::CometError,
2020
execution::{
21-
operators::ExecutionError, planner::TEST_EXEC_CONTEXT_ID, shuffle::ipc::read_ipc_compressed,
21+
operators::ExecutionError,
22+
planner::TEST_EXEC_CONTEXT_ID,
23+
shuffle::{read_ipc_compressed, read_ipc_compressed_validated, validate_remote_schema},
2224
},
2325
jvm_bridge::{jni_call, JVMClasses},
2426
};
25-
use arrow::array::ArrayRef;
27+
use arrow::array::{ArrayRef, RecordBatch};
2628
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
2729
use datafusion::common::Result as DataFusionResult;
2830
use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};
@@ -68,6 +70,8 @@ pub struct ShuffleScanExec {
6870
baseline_metrics: BaselineMetrics,
6971
/// Time spent decoding compressed shuffle blocks.
7072
decode_time: Time,
73+
/// Remote inputs require Arrow array and logical schema validation; queried once at construction.
74+
requires_validation: bool,
7175
}
7276

7377
impl ShuffleScanExec {
@@ -76,6 +80,16 @@ impl ShuffleScanExec {
7680
input_source: Option<Arc<Global<JObject<'static>>>>,
7781
data_types: Vec<DataType>,
7882
) -> Result<Self, CometError> {
83+
let requires_validation = if exec_context_id == TEST_EXEC_CONTEXT_ID {
84+
false
85+
} else if let Some(input) = &input_source {
86+
JVMClasses::with_env(|env| unsafe {
87+
jni_call!(env,
88+
comet_shuffle_block_iterator(input.as_obj()).requires_validation() -> bool)
89+
})?
90+
} else {
91+
false
92+
};
7993
let metrics_set = ExecutionPlanMetricsSet::default();
8094
let baseline_metrics = BaselineMetrics::new(&metrics_set, 0);
8195
let decode_time = MetricBuilder::new(&metrics_set).subset_time("decode_time", 0);
@@ -99,6 +113,7 @@ impl ShuffleScanExec {
99113
baseline_metrics,
100114
schema,
101115
decode_time,
116+
requires_validation,
102117
})
103118
}
104119

@@ -123,6 +138,7 @@ impl ShuffleScanExec {
123138
self.input_source.as_ref().unwrap().as_obj(),
124139
&self.data_types,
125140
&self.decode_time,
141+
self.requires_validation,
126142
)?;
127143
*current_batch = Some(next_batch);
128144
}
@@ -138,6 +154,7 @@ impl ShuffleScanExec {
138154
iter: &JObject,
139155
data_types: &[DataType],
140156
decode_time: &Time,
157+
requires_validation: bool,
141158
) -> Result<InputBatch, CometError> {
142159
if exec_context_id == TEST_EXEC_CONTEXT_ID {
143160
return Ok(InputBatch::EOF);
@@ -173,7 +190,21 @@ impl ShuffleScanExec {
173190

174191
// Decode the compressed IPC data
175192
let mut timer = decode_time.timer();
176-
let batch = read_ipc_compressed(slice)?;
193+
let batch = match decode_shuffle_batch(slice, data_types, requires_validation) {
194+
Ok(batch) => batch,
195+
Err(failure) => {
196+
// Remote inputs must invalidate the failed shuffle generation even when
197+
// fetching bytes succeeded and only IPC/codec decoding or logical schema
198+
// validation detects the corruption. JNI preserves FetchFailedException.
199+
// Local inputs leave this callback as a no-op and keep the native error.
200+
let message = env.new_string(failure.to_string())?;
201+
unsafe {
202+
jni_call!(env,
203+
comet_shuffle_block_iterator(iter).on_decode_failure(&message) -> ())?;
204+
}
205+
return Err(failure.into());
206+
}
207+
};
177208
timer.stop();
178209

179210
let num_rows = batch.num_rows();
@@ -188,19 +219,37 @@ impl ShuffleScanExec {
188219
.map(|col| unpack_dictionary(col))
189220
.collect();
190221

191-
debug_assert_eq!(
192-
columns.len(),
193-
data_types.len(),
194-
"Shuffle block column count mismatch: got {} but expected {}",
195-
columns.len(),
196-
data_types.len()
197-
);
198-
199222
Ok(InputBatch::new(columns, Some(num_rows)))
200223
})
201224
}
202225
}
203226

227+
fn decode_shuffle_batch(
228+
bytes: &[u8],
229+
expected_types: &[DataType],
230+
requires_validation: bool,
231+
) -> DataFusionResult<RecordBatch> {
232+
if requires_validation {
233+
let batch = read_ipc_compressed_validated(bytes)?;
234+
// Validate before unpack_dictionary or cast_and_stamp_schema can hide an incompatible
235+
// wire type by changing values. Keep failures inside get_next's recovery callback.
236+
validate_remote_schema(&batch, expected_types)?;
237+
Ok(batch)
238+
} else {
239+
check_column_count(read_ipc_compressed(bytes)?, expected_types.len())
240+
}
241+
}
242+
243+
fn check_column_count(batch: RecordBatch, expected: usize) -> DataFusionResult<RecordBatch> {
244+
if batch.num_columns() != expected {
245+
return Err(datafusion::common::DataFusionError::Execution(format!(
246+
"Shuffle block column count mismatch: got {} but expected {expected}",
247+
batch.num_columns()
248+
)));
249+
}
250+
Ok(batch)
251+
}
252+
204253
/// If `array` is dictionary-encoded, cast it to the value type. Otherwise return as-is.
205254
fn unpack_dictionary(array: &ArrayRef) -> ArrayRef {
206255
if let DataType::Dictionary(_, value_type) = array.data_type() {
@@ -350,7 +399,7 @@ impl RecordBatchStream for ShuffleScanStream {
350399
#[cfg(test)]
351400
mod tests {
352401
use crate::execution::shuffle::{CompressionCodec, ShuffleBlockWriter};
353-
use arrow::array::{Int32Array, StringArray};
402+
use arrow::array::{Int32Array, RecordBatchOptions, StringArray, UInt32Array};
354403
use arrow::datatypes::{DataType, Field, Schema};
355404
use arrow::ipc::writer::CompressionContext;
356405
use arrow::record_batch::RecordBatch;
@@ -360,6 +409,107 @@ mod tests {
360409

361410
use crate::execution::shuffle::ipc::read_ipc_compressed;
362411

412+
fn uncompressed_shuffle_payload(batch: &RecordBatch) -> Vec<u8> {
413+
let writer = ShuffleBlockWriter::try_new(&batch.schema(), CompressionCodec::None).unwrap();
414+
let mut output = Cursor::new(Vec::new());
415+
writer
416+
.write_batch(
417+
batch,
418+
&mut output,
419+
&mut CompressionContext::default(),
420+
&Time::new(),
421+
)
422+
.unwrap();
423+
// The iterator has already checked the 8-byte frame length and 8-byte field count.
424+
output.into_inner()[16..].to_vec()
425+
}
426+
427+
#[test]
428+
fn remote_shuffle_rejects_schema_signedness_flip_before_casting() {
429+
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
430+
let batch =
431+
RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![-1]))]).unwrap();
432+
let mut payload = uncompressed_shuffle_payload(&batch);
433+
434+
// Corrupt only the schema's signed flag, leaving the field count, row count and all
435+
// value buffers valid. The same -1 bytes now represent UINT_MAX, which a later cast to
436+
// the Spark IntegerType would silently turn into null.
437+
let schema_start = 4 + 8; // codec, then IPC continuation marker and metadata length
438+
let signed_flag = {
439+
let message = arrow::ipc::root_as_message(&payload[schema_start..]).unwrap();
440+
let integer = message
441+
.header_as_schema()
442+
.unwrap()
443+
.fields()
444+
.unwrap()
445+
.get(0)
446+
.type_as_int()
447+
.unwrap();
448+
assert!(integer.is_signed());
449+
let offset = integer._tab.vtable().get(arrow::ipc::Int::VT_IS_SIGNED);
450+
assert_ne!(offset, 0);
451+
schema_start + integer._tab.loc() + usize::from(offset)
452+
};
453+
payload[signed_flag] = 0;
454+
455+
let decoded = super::read_ipc_compressed_validated(&payload).unwrap();
456+
assert_eq!(decoded.num_columns(), 1);
457+
assert_eq!(decoded.column(0).data_type(), &DataType::UInt32);
458+
assert_eq!(
459+
decoded
460+
.column(0)
461+
.as_any()
462+
.downcast_ref::<UInt32Array>()
463+
.unwrap()
464+
.values(),
465+
&[u32::MAX]
466+
);
467+
let error = super::decode_shuffle_batch(&payload, &[DataType::Int32], true)
468+
.unwrap_err()
469+
.to_string();
470+
assert!(error.contains("type mismatch at column 0"), "{error}");
471+
assert!(error.contains("UInt32"), "{error}");
472+
assert!(error.contains("Int32"), "{error}");
473+
474+
// The new logical validation is confined to remote inputs.
475+
assert_eq!(
476+
super::decode_shuffle_batch(&payload, &[DataType::Int32], false)
477+
.unwrap()
478+
.column(0)
479+
.data_type(),
480+
&DataType::UInt32
481+
);
482+
}
483+
484+
#[test]
485+
fn remote_shuffle_preserves_rows_for_zero_column_batches() {
486+
let batch = RecordBatch::try_new_with_options(
487+
Arc::new(Schema::empty()),
488+
vec![],
489+
&RecordBatchOptions::new().with_row_count(Some(3)),
490+
)
491+
.unwrap();
492+
let payload = uncompressed_shuffle_payload(&batch);
493+
let decoded = super::decode_shuffle_batch(&payload, &[], true).unwrap();
494+
assert_eq!(decoded.num_columns(), 0);
495+
assert_eq!(decoded.num_rows(), 3);
496+
}
497+
498+
#[test]
499+
fn test_shuffle_column_count_mismatch_returns_error() {
500+
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
501+
let batch =
502+
RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1, 2, 3]))]).unwrap();
503+
504+
assert!(super::check_column_count(batch.clone(), 1).is_ok());
505+
for expected in [0, 2] {
506+
let error = super::check_column_count(batch.clone(), expected)
507+
.unwrap_err()
508+
.to_string();
509+
assert!(error.contains("column count mismatch"), "{error}");
510+
}
511+
}
512+
363513
#[test]
364514
#[cfg_attr(miri, ignore)] // Miri cannot call FFI functions (zstd)
365515
fn test_read_compressed_ipc_block() {
@@ -464,8 +614,9 @@ mod tests {
464614
let bytes = buf.into_inner();
465615
let body = &bytes[16..];
466616

467-
// Confirm that read_ipc_compressed returns dictionary-encoded arrays
468-
let decoded = read_ipc_compressed(body).unwrap();
617+
// Remote schema validation must preserve the writer's dictionary representation.
618+
let decoded =
619+
super::decode_shuffle_batch(body, &[DataType::Int32, DataType::Utf8], true).unwrap();
469620
assert!(
470621
matches!(decoded.column(1).data_type(), DataType::Dictionary(_, _)),
471622
"Expected dictionary-encoded column from IPC, got {:?}",
@@ -536,13 +687,17 @@ mod tests {
536687
// declared the `flag` child nullable.
537688
let block_column = list_of_struct(false);
538689
let declared = list_of_struct_type(true);
690+
let block = RecordBatch::try_from_iter([("payload", block_column)]).unwrap();
691+
let payload = uncompressed_shuffle_payload(&block);
692+
let decoded =
693+
super::decode_shuffle_batch(&payload, std::slice::from_ref(&declared), true).unwrap();
539694
let mut scan = ShuffleScanExec::new(
540695
super::super::super::planner::TEST_EXEC_CONTEXT_ID,
541696
None,
542697
vec![declared.clone()],
543698
)
544699
.unwrap();
545-
scan.set_input_batch(InputBatch::new(vec![block_column], Some(2)));
700+
scan.set_input_batch(InputBatch::new(decoded.columns().to_vec(), Some(2)));
546701

547702
let rt = tokio::runtime::Runtime::new().unwrap();
548703
rt.block_on(async {

0 commit comments

Comments
 (0)