1818use crate :: {
1919 errors:: CometError ,
2020 execution:: {
21- operators:: ExecutionError , planner:: TEST_EXEC_CONTEXT_ID , shuffle:: ipc:: read_ipc_compressed,
21+ operators:: ExecutionError ,
22+ planner:: TEST_EXEC_CONTEXT_ID ,
23+ shuffle:: { read_ipc_compressed, read_ipc_compressed_validated, validate_remote_schema} ,
2224 } ,
2325 jvm_bridge:: { jni_call, JVMClasses } ,
2426} ;
25- use arrow:: array:: ArrayRef ;
27+ use arrow:: array:: { ArrayRef , RecordBatch } ;
2628use arrow:: datatypes:: { DataType , Field , Schema , SchemaRef } ;
2729use datafusion:: common:: Result as DataFusionResult ;
2830use datafusion:: physical_plan:: execution_plan:: { Boundedness , EmissionType } ;
@@ -68,6 +70,8 @@ pub struct ShuffleScanExec {
6870 baseline_metrics : BaselineMetrics ,
6971 /// Time spent decoding compressed shuffle blocks.
7072 decode_time : Time ,
73+ /// Remote inputs require Arrow array and logical schema validation; queried once at construction.
74+ requires_validation : bool ,
7175}
7276
7377impl ShuffleScanExec {
@@ -76,6 +80,16 @@ impl ShuffleScanExec {
7680 input_source : Option < Arc < Global < JObject < ' static > > > > ,
7781 data_types : Vec < DataType > ,
7882 ) -> Result < Self , CometError > {
83+ let requires_validation = if exec_context_id == TEST_EXEC_CONTEXT_ID {
84+ false
85+ } else if let Some ( input) = & input_source {
86+ JVMClasses :: with_env ( |env| unsafe {
87+ jni_call ! ( env,
88+ comet_shuffle_block_iterator( input. as_obj( ) ) . requires_validation( ) -> bool )
89+ } ) ?
90+ } else {
91+ false
92+ } ;
7993 let metrics_set = ExecutionPlanMetricsSet :: default ( ) ;
8094 let baseline_metrics = BaselineMetrics :: new ( & metrics_set, 0 ) ;
8195 let decode_time = MetricBuilder :: new ( & metrics_set) . subset_time ( "decode_time" , 0 ) ;
@@ -99,6 +113,7 @@ impl ShuffleScanExec {
99113 baseline_metrics,
100114 schema,
101115 decode_time,
116+ requires_validation,
102117 } )
103118 }
104119
@@ -123,6 +138,7 @@ impl ShuffleScanExec {
123138 self . input_source . as_ref ( ) . unwrap ( ) . as_obj ( ) ,
124139 & self . data_types ,
125140 & self . decode_time ,
141+ self . requires_validation ,
126142 ) ?;
127143 * current_batch = Some ( next_batch) ;
128144 }
@@ -138,6 +154,7 @@ impl ShuffleScanExec {
138154 iter : & JObject ,
139155 data_types : & [ DataType ] ,
140156 decode_time : & Time ,
157+ requires_validation : bool ,
141158 ) -> Result < InputBatch , CometError > {
142159 if exec_context_id == TEST_EXEC_CONTEXT_ID {
143160 return Ok ( InputBatch :: EOF ) ;
@@ -173,7 +190,21 @@ impl ShuffleScanExec {
173190
174191 // Decode the compressed IPC data
175192 let mut timer = decode_time. timer ( ) ;
176- let batch = read_ipc_compressed ( slice) ?;
193+ let batch = match decode_shuffle_batch ( slice, data_types, requires_validation) {
194+ Ok ( batch) => batch,
195+ Err ( failure) => {
196+ // Remote inputs must invalidate the failed shuffle generation even when
197+ // fetching bytes succeeded and only IPC/codec decoding or logical schema
198+ // validation detects the corruption. JNI preserves FetchFailedException.
199+ // Local inputs leave this callback as a no-op and keep the native error.
200+ let message = env. new_string ( failure. to_string ( ) ) ?;
201+ unsafe {
202+ jni_call ! ( env,
203+ comet_shuffle_block_iterator( iter) . on_decode_failure( & message) -> ( ) ) ?;
204+ }
205+ return Err ( failure. into ( ) ) ;
206+ }
207+ } ;
177208 timer. stop ( ) ;
178209
179210 let num_rows = batch. num_rows ( ) ;
@@ -188,19 +219,37 @@ impl ShuffleScanExec {
188219 . map ( |col| unpack_dictionary ( col) )
189220 . collect ( ) ;
190221
191- debug_assert_eq ! (
192- columns. len( ) ,
193- data_types. len( ) ,
194- "Shuffle block column count mismatch: got {} but expected {}" ,
195- columns. len( ) ,
196- data_types. len( )
197- ) ;
198-
199222 Ok ( InputBatch :: new ( columns, Some ( num_rows) ) )
200223 } )
201224 }
202225}
203226
227+ fn decode_shuffle_batch (
228+ bytes : & [ u8 ] ,
229+ expected_types : & [ DataType ] ,
230+ requires_validation : bool ,
231+ ) -> DataFusionResult < RecordBatch > {
232+ if requires_validation {
233+ let batch = read_ipc_compressed_validated ( bytes) ?;
234+ // Validate before unpack_dictionary or cast_and_stamp_schema can hide an incompatible
235+ // wire type by changing values. Keep failures inside get_next's recovery callback.
236+ validate_remote_schema ( & batch, expected_types) ?;
237+ Ok ( batch)
238+ } else {
239+ check_column_count ( read_ipc_compressed ( bytes) ?, expected_types. len ( ) )
240+ }
241+ }
242+
243+ fn check_column_count ( batch : RecordBatch , expected : usize ) -> DataFusionResult < RecordBatch > {
244+ if batch. num_columns ( ) != expected {
245+ return Err ( datafusion:: common:: DataFusionError :: Execution ( format ! (
246+ "Shuffle block column count mismatch: got {} but expected {expected}" ,
247+ batch. num_columns( )
248+ ) ) ) ;
249+ }
250+ Ok ( batch)
251+ }
252+
204253/// If `array` is dictionary-encoded, cast it to the value type. Otherwise return as-is.
205254fn unpack_dictionary ( array : & ArrayRef ) -> ArrayRef {
206255 if let DataType :: Dictionary ( _, value_type) = array. data_type ( ) {
@@ -350,7 +399,7 @@ impl RecordBatchStream for ShuffleScanStream {
350399#[ cfg( test) ]
351400mod tests {
352401 use crate :: execution:: shuffle:: { CompressionCodec , ShuffleBlockWriter } ;
353- use arrow:: array:: { Int32Array , StringArray } ;
402+ use arrow:: array:: { Int32Array , RecordBatchOptions , StringArray , UInt32Array } ;
354403 use arrow:: datatypes:: { DataType , Field , Schema } ;
355404 use arrow:: ipc:: writer:: CompressionContext ;
356405 use arrow:: record_batch:: RecordBatch ;
@@ -360,6 +409,107 @@ mod tests {
360409
361410 use crate :: execution:: shuffle:: ipc:: read_ipc_compressed;
362411
412+ fn uncompressed_shuffle_payload ( batch : & RecordBatch ) -> Vec < u8 > {
413+ let writer = ShuffleBlockWriter :: try_new ( & batch. schema ( ) , CompressionCodec :: None ) . unwrap ( ) ;
414+ let mut output = Cursor :: new ( Vec :: new ( ) ) ;
415+ writer
416+ . write_batch (
417+ batch,
418+ & mut output,
419+ & mut CompressionContext :: default ( ) ,
420+ & Time :: new ( ) ,
421+ )
422+ . unwrap ( ) ;
423+ // The iterator has already checked the 8-byte frame length and 8-byte field count.
424+ output. into_inner ( ) [ 16 ..] . to_vec ( )
425+ }
426+
427+ #[ test]
428+ fn remote_shuffle_rejects_schema_signedness_flip_before_casting ( ) {
429+ let schema = Arc :: new ( Schema :: new ( vec ! [ Field :: new( "id" , DataType :: Int32 , false ) ] ) ) ;
430+ let batch =
431+ RecordBatch :: try_new ( schema, vec ! [ Arc :: new( Int32Array :: from( vec![ -1 ] ) ) ] ) . unwrap ( ) ;
432+ let mut payload = uncompressed_shuffle_payload ( & batch) ;
433+
434+ // Corrupt only the schema's signed flag, leaving the field count, row count and all
435+ // value buffers valid. The same -1 bytes now represent UINT_MAX, which a later cast to
436+ // the Spark IntegerType would silently turn into null.
437+ let schema_start = 4 + 8 ; // codec, then IPC continuation marker and metadata length
438+ let signed_flag = {
439+ let message = arrow:: ipc:: root_as_message ( & payload[ schema_start..] ) . unwrap ( ) ;
440+ let integer = message
441+ . header_as_schema ( )
442+ . unwrap ( )
443+ . fields ( )
444+ . unwrap ( )
445+ . get ( 0 )
446+ . type_as_int ( )
447+ . unwrap ( ) ;
448+ assert ! ( integer. is_signed( ) ) ;
449+ let offset = integer. _tab . vtable ( ) . get ( arrow:: ipc:: Int :: VT_IS_SIGNED ) ;
450+ assert_ne ! ( offset, 0 ) ;
451+ schema_start + integer. _tab . loc ( ) + usize:: from ( offset)
452+ } ;
453+ payload[ signed_flag] = 0 ;
454+
455+ let decoded = super :: read_ipc_compressed_validated ( & payload) . unwrap ( ) ;
456+ assert_eq ! ( decoded. num_columns( ) , 1 ) ;
457+ assert_eq ! ( decoded. column( 0 ) . data_type( ) , & DataType :: UInt32 ) ;
458+ assert_eq ! (
459+ decoded
460+ . column( 0 )
461+ . as_any( )
462+ . downcast_ref:: <UInt32Array >( )
463+ . unwrap( )
464+ . values( ) ,
465+ & [ u32 :: MAX ]
466+ ) ;
467+ let error = super :: decode_shuffle_batch ( & payload, & [ DataType :: Int32 ] , true )
468+ . unwrap_err ( )
469+ . to_string ( ) ;
470+ assert ! ( error. contains( "type mismatch at column 0" ) , "{error}" ) ;
471+ assert ! ( error. contains( "UInt32" ) , "{error}" ) ;
472+ assert ! ( error. contains( "Int32" ) , "{error}" ) ;
473+
474+ // The new logical validation is confined to remote inputs.
475+ assert_eq ! (
476+ super :: decode_shuffle_batch( & payload, & [ DataType :: Int32 ] , false )
477+ . unwrap( )
478+ . column( 0 )
479+ . data_type( ) ,
480+ & DataType :: UInt32
481+ ) ;
482+ }
483+
484+ #[ test]
485+ fn remote_shuffle_preserves_rows_for_zero_column_batches ( ) {
486+ let batch = RecordBatch :: try_new_with_options (
487+ Arc :: new ( Schema :: empty ( ) ) ,
488+ vec ! [ ] ,
489+ & RecordBatchOptions :: new ( ) . with_row_count ( Some ( 3 ) ) ,
490+ )
491+ . unwrap ( ) ;
492+ let payload = uncompressed_shuffle_payload ( & batch) ;
493+ let decoded = super :: decode_shuffle_batch ( & payload, & [ ] , true ) . unwrap ( ) ;
494+ assert_eq ! ( decoded. num_columns( ) , 0 ) ;
495+ assert_eq ! ( decoded. num_rows( ) , 3 ) ;
496+ }
497+
498+ #[ test]
499+ fn test_shuffle_column_count_mismatch_returns_error ( ) {
500+ let schema = Arc :: new ( Schema :: new ( vec ! [ Field :: new( "id" , DataType :: Int32 , false ) ] ) ) ;
501+ let batch =
502+ RecordBatch :: try_new ( schema, vec ! [ Arc :: new( Int32Array :: from( vec![ 1 , 2 , 3 ] ) ) ] ) . unwrap ( ) ;
503+
504+ assert ! ( super :: check_column_count( batch. clone( ) , 1 ) . is_ok( ) ) ;
505+ for expected in [ 0 , 2 ] {
506+ let error = super :: check_column_count ( batch. clone ( ) , expected)
507+ . unwrap_err ( )
508+ . to_string ( ) ;
509+ assert ! ( error. contains( "column count mismatch" ) , "{error}" ) ;
510+ }
511+ }
512+
363513 #[ test]
364514 #[ cfg_attr( miri, ignore) ] // Miri cannot call FFI functions (zstd)
365515 fn test_read_compressed_ipc_block ( ) {
@@ -464,8 +614,9 @@ mod tests {
464614 let bytes = buf. into_inner ( ) ;
465615 let body = & bytes[ 16 ..] ;
466616
467- // Confirm that read_ipc_compressed returns dictionary-encoded arrays
468- let decoded = read_ipc_compressed ( body) . unwrap ( ) ;
617+ // Remote schema validation must preserve the writer's dictionary representation.
618+ let decoded =
619+ super :: decode_shuffle_batch ( body, & [ DataType :: Int32 , DataType :: Utf8 ] , true ) . unwrap ( ) ;
469620 assert ! (
470621 matches!( decoded. column( 1 ) . data_type( ) , DataType :: Dictionary ( _, _) ) ,
471622 "Expected dictionary-encoded column from IPC, got {:?}" ,
@@ -536,13 +687,17 @@ mod tests {
536687 // declared the `flag` child nullable.
537688 let block_column = list_of_struct ( false ) ;
538689 let declared = list_of_struct_type ( true ) ;
690+ let block = RecordBatch :: try_from_iter ( [ ( "payload" , block_column) ] ) . unwrap ( ) ;
691+ let payload = uncompressed_shuffle_payload ( & block) ;
692+ let decoded =
693+ super :: decode_shuffle_batch ( & payload, std:: slice:: from_ref ( & declared) , true ) . unwrap ( ) ;
539694 let mut scan = ShuffleScanExec :: new (
540695 super :: super :: super :: planner:: TEST_EXEC_CONTEXT_ID ,
541696 None ,
542697 vec ! [ declared. clone( ) ] ,
543698 )
544699 . unwrap ( ) ;
545- scan. set_input_batch ( InputBatch :: new ( vec ! [ block_column ] , Some ( 2 ) ) ) ;
700+ scan. set_input_batch ( InputBatch :: new ( decoded . columns ( ) . to_vec ( ) , Some ( 2 ) ) ) ;
546701
547702 let rt = tokio:: runtime:: Runtime :: new ( ) . unwrap ( ) ;
548703 rt. block_on ( async {
0 commit comments