Skip to content

Commit e34912f

Browse files
authored
fix array_repeat scalar path overflows total repeated-value count (apache#22274)
## Which issue does this PR close? - Closes apache#22218 ## Rationale for this change The scalar path of array_repeat can overflow while computing the total number of repeated values. Previously, the implementation used unchecked accumulation for these totals, which could lead to overflow behavior that was not explicitly handled. This change makes the capacity and offset calculations overflow-safe and returns a clear execution error when the total output size exceeds usize. ## What changes are included in this PR? - Adds overflow checks for the total repeated value count in the scalar array_repeat path. - Adds overflow checks for outer and inner total size calculations in the list repeat path. - Preserves the existing behavior where non-positive counts are treated as zero. - Adds sqllogictest coverage for the overflow error case. ## Are these changes tested? - Yes. A new sqllogictest covers the scalar overflow failure case for array_repeat. - Existing related tests continue to pass. ## Are there any user-facing changes? - Yes. For extremely large inputs, array_repeat now returns a clear execution error instead of relying on implicit overflow behavior. - Normal inputs are unchanged.
1 parent 7a0d026 commit e34912f

2 files changed

Lines changed: 70 additions & 44 deletions

File tree

‎datafusion/functions-nested/src/repeat.rs‎

Lines changed: 60 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ use arrow::datatypes::{
3131
};
3232
use datafusion_common::cast::{as_int64_array, as_large_list_array, as_list_array};
3333
use datafusion_common::types::{NativeType, logical_int64};
34-
use datafusion_common::{DataFusionError, Result};
34+
use datafusion_common::{Result, exec_datafusion_err};
3535
use datafusion_expr::{
3636
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
3737
Volatility,
@@ -179,7 +179,18 @@ fn general_repeat<O: OffsetSizeTrait>(
179179
array: &ArrayRef,
180180
count_array: &Int64Array,
181181
) -> Result<ArrayRef> {
182-
let (offsets, total_repeated_values) = build_repeat_offsets::<O>(count_array)?;
182+
let total_repeated_values =
183+
(0..count_array.len()).try_fold(0usize, |total, idx| {
184+
total
185+
.checked_add(repeat_count(count_array, idx).unwrap_or_default())
186+
.ok_or_else(|| {
187+
exec_datafusion_err!(
188+
"array_repeat: total repeated values overflowed usize"
189+
)
190+
})
191+
})?;
192+
ensure_repeated_values_fit::<O>(total_repeated_values)?;
193+
let (offsets, _) = build_repeat_offsets::<O>(count_array)?;
183194

184195
let mut take_indices = Vec::with_capacity(total_repeated_values);
185196

@@ -234,13 +245,13 @@ fn general_list_repeat<O: OffsetSizeTrait>(
234245
- list_offsets[i].to_usize().unwrap();
235246
inner_total =
236247
checked_repeat_len_add(inner_total, checked_repeat_len_mul(len, count)?)?;
237-
ensure_array_repeat_output_len::<O>(inner_total)?;
248+
ensure_repeated_values_fit::<O>(inner_total)?;
238249
}
239250
}
240251

241252
// Build inner structures
242-
ensure_vec_capacity::<O>(checked_repeat_len_add(outer_total, 1)?)?;
243-
let mut inner_offsets = Vec::with_capacity(outer_total + 1);
253+
let inner_offsets_capacity = checked_offset_slots_capacity::<O>(outer_total)?;
254+
let mut inner_offsets = Vec::with_capacity(inner_offsets_capacity);
244255
let mut take_indices = Vec::with_capacity(inner_total);
245256
let mut inner_nulls = BooleanBufferBuilder::new(outer_total);
246257
let mut inner_running = 0usize;
@@ -257,12 +268,8 @@ fn general_list_repeat<O: OffsetSizeTrait>(
257268

258269
for _ in 0..count {
259270
inner_running = checked_repeat_len_add(inner_running, row_len)?;
260-
ensure_array_repeat_output_len::<O>(inner_running)?;
261-
let offset = O::from_usize(inner_running).ok_or_else(|| {
262-
DataFusionError::Execution(format!(
263-
"array_repeat: offset {inner_running} exceeds the maximum value for offset type"
264-
))
265-
})?;
271+
ensure_repeated_values_fit::<O>(inner_running)?;
272+
let offset = checked_repeat_offset::<O>(inner_running)?;
266273
inner_offsets.push(offset);
267274
inner_nulls.append(list_is_valid);
268275
if list_is_valid {
@@ -298,7 +305,8 @@ fn general_list_repeat<O: OffsetSizeTrait>(
298305
fn build_repeat_offsets<O: OffsetSizeTrait>(
299306
count_array: &Int64Array,
300307
) -> Result<(Vec<O>, usize)> {
301-
let mut offsets = Vec::with_capacity(count_array.len() + 1);
308+
let offsets_capacity = checked_offset_slots_capacity::<O>(count_array.len())?;
309+
let mut offsets = Vec::with_capacity(offsets_capacity);
302310
offsets.push(O::zero());
303311
let mut running_offset = 0usize;
304312

@@ -308,60 +316,52 @@ fn build_repeat_offsets<O: OffsetSizeTrait>(
308316
continue;
309317
};
310318
running_offset = checked_repeat_len_add(running_offset, count)?;
311-
ensure_array_repeat_output_len::<O>(running_offset)?;
312-
let offset = O::from_usize(running_offset).ok_or_else(|| {
313-
DataFusionError::Execution(format!(
314-
"array_repeat: offset {running_offset} exceeds the maximum value for offset type"
315-
))
316-
})?;
319+
ensure_repeated_values_fit::<O>(running_offset)?;
320+
let offset = checked_repeat_offset::<O>(running_offset)?;
317321
offsets.push(offset);
318322
}
319323

320324
Ok((offsets, running_offset))
321325
}
322326

323327
fn checked_repeat_len_add(lhs: usize, rhs: usize) -> Result<usize> {
324-
lhs.checked_add(rhs).ok_or_else(|| {
325-
DataFusionError::Execution(ARRAY_REPEAT_LENGTH_EXCEEDED.to_string())
326-
})
328+
lhs.checked_add(rhs)
329+
.ok_or_else(|| exec_datafusion_err!("{}", ARRAY_REPEAT_LENGTH_EXCEEDED))
327330
}
328331

329332
fn checked_repeat_len_mul(lhs: usize, rhs: usize) -> Result<usize> {
330-
lhs.checked_mul(rhs).ok_or_else(|| {
331-
DataFusionError::Execution(ARRAY_REPEAT_LENGTH_EXCEEDED.to_string())
332-
})
333+
lhs.checked_mul(rhs)
334+
.ok_or_else(|| exec_datafusion_err!("{}", ARRAY_REPEAT_LENGTH_EXCEEDED))
333335
}
334336

335-
fn ensure_array_repeat_output_len<O: OffsetSizeTrait>(len: usize) -> Result<()> {
336-
if len > max_array_repeat_output_len::<O>() {
337-
return Err(DataFusionError::Execution(
338-
ARRAY_REPEAT_LENGTH_EXCEEDED.to_string(),
339-
));
340-
}
337+
fn ensure_repeated_values_fit<O: OffsetSizeTrait>(len: usize) -> Result<()> {
338+
ensure_vec_capacity::<u64>(len)?;
339+
checked_repeat_offset::<O>(len)?;
341340

342341
Ok(())
343342
}
344343

345344
fn ensure_vec_capacity<T>(len: usize) -> Result<()> {
346345
if len > max_vec_elements::<T>() {
347-
return Err(DataFusionError::Execution(
348-
ARRAY_REPEAT_LENGTH_EXCEEDED.to_string(),
349-
));
346+
return Err(exec_datafusion_err!("{}", ARRAY_REPEAT_LENGTH_EXCEEDED));
350347
}
351348

352349
Ok(())
353350
}
354351

355-
fn max_array_repeat_output_len<O: OffsetSizeTrait>() -> usize {
356-
max_offset_elements::<O>().min(max_vec_elements::<u64>())
352+
fn checked_offset_slots_capacity<O>(len: usize) -> Result<usize> {
353+
let capacity = checked_repeat_len_add(len, 1)?;
354+
ensure_vec_capacity::<O>(capacity)?;
355+
356+
Ok(capacity)
357357
}
358358

359-
fn max_offset_elements<O: OffsetSizeTrait>() -> usize {
360-
if size_of::<O>() == size_of::<i32>() {
361-
i32::MAX as usize
362-
} else {
363-
i64::MAX as usize
364-
}
359+
fn checked_repeat_offset<O: OffsetSizeTrait>(offset: usize) -> Result<O> {
360+
O::from_usize(offset).ok_or_else(|| {
361+
exec_datafusion_err!(
362+
"array_repeat: offset {offset} exceeds the maximum value for offset type"
363+
)
364+
})
365365
}
366366

367367
fn max_vec_elements<T>() -> usize {
@@ -451,9 +451,25 @@ mod tests {
451451
let count: ArrayRef = Arc::new(Int64Array::from(vec![i64::MAX]));
452452

453453
let err = array_repeat_inner(&[element, count]).unwrap_err();
454-
assert_eq!(
455-
err.to_string(),
456-
"Execution error: array_repeat: requested length exceeds maximum array size"
454+
assert!(
455+
err.to_string().starts_with(
456+
"Execution error: array_repeat: requested length exceeds maximum array size"
457+
),
458+
"unexpected error: {err}"
459+
);
460+
}
461+
462+
#[test]
463+
fn scalar_count_exceeding_list_offset_limit_returns_error() {
464+
let element: ArrayRef = Arc::new(Int64Array::from(vec![1]));
465+
let count: ArrayRef = Arc::new(Int64Array::from(vec![i32::MAX as i64 + 1]));
466+
467+
let err = array_repeat_inner(&[element, count]).unwrap_err();
468+
assert!(
469+
err.to_string().starts_with(
470+
"Execution error: array_repeat: offset 2147483648 exceeds the maximum value for offset type"
471+
),
472+
"unexpected error: {err}"
457473
);
458474
}
459475
}

‎datafusion/sqllogictest/test_files/array/array_repeat.slt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,16 @@ Select
7979
----
8080
[] [] [] []
8181

82+
# array_repeat returns an execution error on scalar output-size overflow
83+
query error DataFusion error: Execution error: array_repeat: total repeated values overflowed usize
84+
SELECT array_repeat(1, c)
85+
FROM (
86+
VALUES
87+
(9223372036854775807),
88+
(9223372036854775807),
89+
(9223372036854775807)
90+
) AS t(c);
91+
8292
# array_repeat with columns #1
8393

8494
statement ok

0 commit comments

Comments
 (0)