@@ -21,6 +21,9 @@ use jni::objects::{Global, JMethodID, JObject, JValue};
2121use jni:: signature:: { Primitive , ReturnType } ;
2222use jni:: Env ;
2323
24+ /// OpenJDK's conservative soft array limit; actual JVM limits can vary.
25+ const MAX_JVM_ARRAY_LENGTH : i32 = i32:: MAX - 8 ;
26+
2427/// Receives a complete encoded shuffle block for one output partition.
2528///
2629/// Implementations must remain safe when invoked from native execution
@@ -72,11 +75,13 @@ impl JavaShufflePartitionPusher {
7275 ) ) ) ;
7376 }
7477
75- i32:: try_from ( payload_length) . map_err ( |_| {
76- DataFusionError :: Execution ( format ! (
77- "Remote shuffle payload size {payload_length} exceeds the JVM array limit"
78- ) )
79- } )
78+ match i32:: try_from ( payload_length) {
79+ Ok ( length) if length <= MAX_JVM_ARRAY_LENGTH => Ok ( length) ,
80+ _ => Err ( DataFusionError :: Execution ( format ! (
81+ "Remote shuffle payload size {payload_length} exceeds the JVM array limit of \
82+ {MAX_JVM_ARRAY_LENGTH} bytes"
83+ ) ) ) ,
84+ }
8085 }
8186}
8287
@@ -117,17 +122,18 @@ impl ShufflePartitionPusher for JavaShufflePartitionPusher {
117122
118123#[ cfg( test) ]
119124mod tests {
120- use super :: JavaShufflePartitionPusher ;
125+ use super :: { JavaShufflePartitionPusher , MAX_JVM_ARRAY_LENGTH } ;
121126
122127 #[ test]
123- fn accepts_payload_lengths_representable_by_jvm_arrays ( ) {
128+ fn accepts_payload_lengths_up_to_jvm_soft_array_limit ( ) {
124129 assert_eq ! (
125130 JavaShufflePartitionPusher :: checked_payload_length( 0 , 0 ) . unwrap( ) ,
126131 0
127132 ) ;
128133 assert_eq ! (
129- JavaShufflePartitionPusher :: checked_payload_length( 7 , i32 :: MAX as usize ) . unwrap( ) ,
130- i32 :: MAX
134+ JavaShufflePartitionPusher :: checked_payload_length( 7 , MAX_JVM_ARRAY_LENGTH as usize , )
135+ . unwrap( ) ,
136+ MAX_JVM_ARRAY_LENGTH
131137 ) ;
132138 }
133139
@@ -138,10 +144,18 @@ mod tests {
138144 }
139145
140146 #[ test]
141- fn rejects_payload_lengths_larger_than_a_jvm_array ( ) {
142- let oversized_length = i32:: MAX as usize + 1 ;
143- let error =
144- JavaShufflePartitionPusher :: checked_payload_length ( 0 , oversized_length) . unwrap_err ( ) ;
145- assert ! ( error. to_string( ) . contains( "exceeds the JVM array limit" ) ) ;
147+ fn rejects_payload_lengths_larger_than_jvm_soft_array_limit ( ) {
148+ for oversized_length in [
149+ MAX_JVM_ARRAY_LENGTH as usize + 1 ,
150+ i32:: MAX as usize ,
151+ i32:: MAX as usize + 1 ,
152+ usize:: MAX ,
153+ ] {
154+ let error = JavaShufflePartitionPusher :: checked_payload_length ( 0 , oversized_length)
155+ . unwrap_err ( ) ;
156+ let message = error. to_string ( ) ;
157+ assert ! ( message. contains( "exceeds the JVM array limit" ) ) ;
158+ assert ! ( message. contains( & MAX_JVM_ARRAY_LENGTH . to_string( ) ) ) ;
159+ }
146160 }
147161}
0 commit comments