@@ -31,7 +31,7 @@ use arrow::datatypes::{
3131} ;
3232use datafusion_common:: cast:: { as_int64_array, as_large_list_array, as_list_array} ;
3333use datafusion_common:: types:: { NativeType , logical_int64} ;
34- use datafusion_common:: { DataFusionError , Result } ;
34+ use datafusion_common:: { Result , exec_datafusion_err } ;
3535use datafusion_expr:: {
3636 ColumnarValue , Documentation , ScalarFunctionArgs , ScalarUDFImpl , Signature ,
3737 Volatility ,
@@ -179,7 +179,18 @@ fn general_repeat<O: OffsetSizeTrait>(
179179 array : & ArrayRef ,
180180 count_array : & Int64Array ,
181181) -> Result < ArrayRef > {
182- let ( offsets, total_repeated_values) = build_repeat_offsets :: < O > ( count_array) ?;
182+ let total_repeated_values =
183+ ( 0 ..count_array. len ( ) ) . try_fold ( 0usize , |total, idx| {
184+ total
185+ . checked_add ( repeat_count ( count_array, idx) . unwrap_or_default ( ) )
186+ . ok_or_else ( || {
187+ exec_datafusion_err ! (
188+ "array_repeat: total repeated values overflowed usize"
189+ )
190+ } )
191+ } ) ?;
192+ ensure_repeated_values_fit :: < O > ( total_repeated_values) ?;
193+ let ( offsets, _) = build_repeat_offsets :: < O > ( count_array) ?;
183194
184195 let mut take_indices = Vec :: with_capacity ( total_repeated_values) ;
185196
@@ -234,13 +245,13 @@ fn general_list_repeat<O: OffsetSizeTrait>(
234245 - list_offsets[ i] . to_usize ( ) . unwrap ( ) ;
235246 inner_total =
236247 checked_repeat_len_add ( inner_total, checked_repeat_len_mul ( len, count) ?) ?;
237- ensure_array_repeat_output_len :: < O > ( inner_total) ?;
248+ ensure_repeated_values_fit :: < O > ( inner_total) ?;
238249 }
239250 }
240251
241252 // Build inner structures
242- ensure_vec_capacity :: < O > ( checked_repeat_len_add ( outer_total, 1 ) ? ) ?;
243- let mut inner_offsets = Vec :: with_capacity ( outer_total + 1 ) ;
253+ let inner_offsets_capacity = checked_offset_slots_capacity :: < O > ( outer_total) ?;
254+ let mut inner_offsets = Vec :: with_capacity ( inner_offsets_capacity ) ;
244255 let mut take_indices = Vec :: with_capacity ( inner_total) ;
245256 let mut inner_nulls = BooleanBufferBuilder :: new ( outer_total) ;
246257 let mut inner_running = 0usize ;
@@ -257,12 +268,8 @@ fn general_list_repeat<O: OffsetSizeTrait>(
257268
258269 for _ in 0 ..count {
259270 inner_running = checked_repeat_len_add ( inner_running, row_len) ?;
260- ensure_array_repeat_output_len :: < O > ( inner_running) ?;
261- let offset = O :: from_usize ( inner_running) . ok_or_else ( || {
262- DataFusionError :: Execution ( format ! (
263- "array_repeat: offset {inner_running} exceeds the maximum value for offset type"
264- ) )
265- } ) ?;
271+ ensure_repeated_values_fit :: < O > ( inner_running) ?;
272+ let offset = checked_repeat_offset :: < O > ( inner_running) ?;
266273 inner_offsets. push ( offset) ;
267274 inner_nulls. append ( list_is_valid) ;
268275 if list_is_valid {
@@ -298,7 +305,8 @@ fn general_list_repeat<O: OffsetSizeTrait>(
298305fn build_repeat_offsets < O : OffsetSizeTrait > (
299306 count_array : & Int64Array ,
300307) -> Result < ( Vec < O > , usize ) > {
301- let mut offsets = Vec :: with_capacity ( count_array. len ( ) + 1 ) ;
308+ let offsets_capacity = checked_offset_slots_capacity :: < O > ( count_array. len ( ) ) ?;
309+ let mut offsets = Vec :: with_capacity ( offsets_capacity) ;
302310 offsets. push ( O :: zero ( ) ) ;
303311 let mut running_offset = 0usize ;
304312
@@ -308,60 +316,52 @@ fn build_repeat_offsets<O: OffsetSizeTrait>(
308316 continue ;
309317 } ;
310318 running_offset = checked_repeat_len_add ( running_offset, count) ?;
311- ensure_array_repeat_output_len :: < O > ( running_offset) ?;
312- let offset = O :: from_usize ( running_offset) . ok_or_else ( || {
313- DataFusionError :: Execution ( format ! (
314- "array_repeat: offset {running_offset} exceeds the maximum value for offset type"
315- ) )
316- } ) ?;
319+ ensure_repeated_values_fit :: < O > ( running_offset) ?;
320+ let offset = checked_repeat_offset :: < O > ( running_offset) ?;
317321 offsets. push ( offset) ;
318322 }
319323
320324 Ok ( ( offsets, running_offset) )
321325}
322326
323327fn checked_repeat_len_add ( lhs : usize , rhs : usize ) -> Result < usize > {
324- lhs. checked_add ( rhs) . ok_or_else ( || {
325- DataFusionError :: Execution ( ARRAY_REPEAT_LENGTH_EXCEEDED . to_string ( ) )
326- } )
328+ lhs. checked_add ( rhs)
329+ . ok_or_else ( || exec_datafusion_err ! ( "{}" , ARRAY_REPEAT_LENGTH_EXCEEDED ) )
327330}
328331
329332fn checked_repeat_len_mul ( lhs : usize , rhs : usize ) -> Result < usize > {
330- lhs. checked_mul ( rhs) . ok_or_else ( || {
331- DataFusionError :: Execution ( ARRAY_REPEAT_LENGTH_EXCEEDED . to_string ( ) )
332- } )
333+ lhs. checked_mul ( rhs)
334+ . ok_or_else ( || exec_datafusion_err ! ( "{}" , ARRAY_REPEAT_LENGTH_EXCEEDED ) )
333335}
334336
335- fn ensure_array_repeat_output_len < O : OffsetSizeTrait > ( len : usize ) -> Result < ( ) > {
336- if len > max_array_repeat_output_len :: < O > ( ) {
337- return Err ( DataFusionError :: Execution (
338- ARRAY_REPEAT_LENGTH_EXCEEDED . to_string ( ) ,
339- ) ) ;
340- }
337+ fn ensure_repeated_values_fit < O : OffsetSizeTrait > ( len : usize ) -> Result < ( ) > {
338+ ensure_vec_capacity :: < u64 > ( len) ?;
339+ checked_repeat_offset :: < O > ( len) ?;
341340
342341 Ok ( ( ) )
343342}
344343
345344fn ensure_vec_capacity < T > ( len : usize ) -> Result < ( ) > {
346345 if len > max_vec_elements :: < T > ( ) {
347- return Err ( DataFusionError :: Execution (
348- ARRAY_REPEAT_LENGTH_EXCEEDED . to_string ( ) ,
349- ) ) ;
346+ return Err ( exec_datafusion_err ! ( "{}" , ARRAY_REPEAT_LENGTH_EXCEEDED ) ) ;
350347 }
351348
352349 Ok ( ( ) )
353350}
354351
355- fn max_array_repeat_output_len < O : OffsetSizeTrait > ( ) -> usize {
356- max_offset_elements :: < O > ( ) . min ( max_vec_elements :: < u64 > ( ) )
352+ fn checked_offset_slots_capacity < O > ( len : usize ) -> Result < usize > {
353+ let capacity = checked_repeat_len_add ( len, 1 ) ?;
354+ ensure_vec_capacity :: < O > ( capacity) ?;
355+
356+ Ok ( capacity)
357357}
358358
359- fn max_offset_elements < O : OffsetSizeTrait > ( ) -> usize {
360- if size_of :: < O > ( ) == size_of :: < i32 > ( ) {
361- i32 :: MAX as usize
362- } else {
363- i64 :: MAX as usize
364- }
359+ fn checked_repeat_offset < O : OffsetSizeTrait > ( offset : usize ) -> Result < O > {
360+ O :: from_usize ( offset ) . ok_or_else ( || {
361+ exec_datafusion_err ! (
362+ "array_repeat: offset {offset} exceeds the maximum value for offset type"
363+ )
364+ } )
365365}
366366
367367fn max_vec_elements < T > ( ) -> usize {
@@ -451,9 +451,25 @@ mod tests {
451451 let count: ArrayRef = Arc :: new ( Int64Array :: from ( vec ! [ i64 :: MAX ] ) ) ;
452452
453453 let err = array_repeat_inner ( & [ element, count] ) . unwrap_err ( ) ;
454- assert_eq ! (
455- err. to_string( ) ,
456- "Execution error: array_repeat: requested length exceeds maximum array size"
454+ assert ! (
455+ err. to_string( ) . starts_with(
456+ "Execution error: array_repeat: requested length exceeds maximum array size"
457+ ) ,
458+ "unexpected error: {err}"
459+ ) ;
460+ }
461+
462+ #[ test]
463+ fn scalar_count_exceeding_list_offset_limit_returns_error ( ) {
464+ let element: ArrayRef = Arc :: new ( Int64Array :: from ( vec ! [ 1 ] ) ) ;
465+ let count: ArrayRef = Arc :: new ( Int64Array :: from ( vec ! [ i32 :: MAX as i64 + 1 ] ) ) ;
466+
467+ let err = array_repeat_inner ( & [ element, count] ) . unwrap_err ( ) ;
468+ assert ! (
469+ err. to_string( ) . starts_with(
470+ "Execution error: array_repeat: offset 2147483648 exceeds the maximum value for offset type"
471+ ) ,
472+ "unexpected error: {err}"
457473 ) ;
458474 }
459475}
0 commit comments