Skip to content

Commit 32a1fe5

Browse files
Sean Kenneth Dohertyalamb
andauthored
fix: guard repeat array length overflow (apache#22293)
## Which issue does this PR close? - Closes apache#22217. ## Rationale for this change The array execution path for `repeat(string, count)` calculated `string.len() * count` before checking the configured string-size limit. For very large counts, that multiplication can overflow and panic instead of returning the same string-size overflow error used by the scalar path. ## What changes are included in this PR? - Adds checked count conversion and repeated-length calculation helpers. - Uses checked multiplication and checked total-capacity accumulation in the array path. - Adds Rust and sqllogictest coverage for the one-row columnar reproducer from the issue. ## Are these changes tested? - `cargo fmt --all` - `TMPDIR=/home/sean/Projects/datafusion-repeat-overflow/target/tmp cargo test -p datafusion-functions string::repeat::tests::test_repeat_string_array_overflow -- --nocapture` - `TMPDIR=/home/sean/Projects/datafusion-repeat-overflow/target/tmp cargo test --profile=ci --test sqllogictests -- string/string_literal.slt` - `TMPDIR=/home/sean/Projects/datafusion-repeat-overflow/target/tmp cargo clippy --all-targets --all-features -- -D warnings` - `git diff --check` ## Are there any user-facing changes? Invalid oversized `repeat` results in the columnar path now return a normal DataFusion string-size overflow error instead of panicking. --------- Co-authored-by: Andrew Lamb <[email protected]>
1 parent 34834dc commit 32a1fe5

2 files changed

Lines changed: 57 additions & 14 deletions

File tree

‎datafusion/functions/src/string/repeat.rs‎

Lines changed: 53 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,9 @@ use arrow::datatypes::DataType::{LargeUtf8, Utf8, Utf8View};
2626
use datafusion_common::cast::as_int64_array;
2727
use datafusion_common::types::{NativeType, logical_int64, logical_string};
2828
use datafusion_common::utils::take_function_args;
29-
use datafusion_common::{DataFusionError, Result, ScalarValue, exec_err, internal_err};
29+
use datafusion_common::{
30+
DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err, internal_err,
31+
};
3032
use datafusion_expr::{ColumnarValue, Documentation, Volatility};
3133
use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl, Signature};
3234
use datafusion_expr_common::signature::{Coercion, TypeSignatureClass};
@@ -166,15 +168,40 @@ fn compute_repeat(s: &str, count: i64, max_size: usize) -> Result<String> {
166168
if count <= 0 {
167169
return Ok(String::new());
168170
}
169-
let result_len = s.len().saturating_mul(count as usize);
171+
let result_len = repeat_len(s.len(), count, max_size)?;
172+
debug_assert!(result_len <= max_size);
173+
let count = repeat_count(count, max_size)?;
174+
Ok(s.repeat(count))
175+
}
176+
177+
fn repeat_len(string_len: usize, count: i64, max_size: usize) -> Result<usize> {
178+
let count = repeat_count(count, max_size)?;
179+
let result_len = string_len.checked_mul(count).ok_or_else(|| {
180+
exec_datafusion_err!(
181+
"string size overflow on repeat, max size is {}, but got {}",
182+
max_size,
183+
usize::MAX
184+
)
185+
})?;
170186
if result_len > max_size {
171187
return exec_err!(
172188
"string size overflow on repeat, max size is {}, but got {}",
173189
max_size,
174190
result_len
175191
);
176192
}
177-
Ok(s.repeat(count as usize))
193+
Ok(result_len)
194+
}
195+
196+
fn repeat_count(count: i64, max_size: usize) -> Result<usize> {
197+
match usize::try_from(count) {
198+
Ok(count) => Ok(count),
199+
Err(_) => exec_err!(
200+
"string size overflow on repeat, max size is {}, but got {}",
201+
max_size,
202+
usize::MAX
203+
),
204+
}
178205
}
179206

180207
/// Repeats string the specified number of times.
@@ -227,22 +254,22 @@ fn calculate_capacities<'a, S>(
227254
where
228255
S: StringArrayType<'a>,
229256
{
230-
let mut total_capacity = 0;
231-
let mut max_item_capacity = 0;
257+
let mut total_capacity = 0usize;
258+
let mut max_item_capacity = 0usize;
232259

233260
string_array.iter().zip(number_array.iter()).try_for_each(
234261
|(string, number)| -> Result<(), DataFusionError> {
235262
match (string, number) {
236263
(Some(string), Some(number)) if number >= 0 => {
237-
let item_capacity = string.len() * number as usize;
238-
if item_capacity > max_str_len {
239-
return exec_err!(
240-
"string size overflow on repeat, max size is {}, but got {}",
241-
max_str_len,
242-
number as usize * string.len()
243-
);
244-
}
245-
total_capacity += item_capacity;
264+
let item_capacity = repeat_len(string.len(), number, max_str_len)?;
265+
total_capacity =
266+
total_capacity.checked_add(item_capacity).ok_or_else(|| {
267+
exec_datafusion_err!(
268+
"string size overflow on repeat, max size is {}, but got {}",
269+
max_str_len,
270+
usize::MAX
271+
)
272+
})?;
246273
max_item_capacity = max_item_capacity.max(item_capacity);
247274
}
248275
_ => (),
@@ -487,6 +514,18 @@ mod tests {
487514
assert_sliced_offset_output::<StringArray>(result);
488515
}
489516

517+
#[test]
518+
fn test_repeat_string_array_overflow() {
519+
let strings: ArrayRef = Arc::new(StringArray::from(vec![Some("abc")]));
520+
let counts: ArrayRef = Arc::new(Int64Array::from(vec![Some(i64::MAX)]));
521+
522+
let err = super::repeat(&strings, &counts).unwrap_err().to_string();
523+
assert!(
524+
err.contains("string size overflow on repeat"),
525+
"unexpected error: {err}"
526+
);
527+
}
528+
490529
#[test]
491530
fn test_repeat_sliced_large_string_with_null_offset() {
492531
let (strings, counts) =

‎datafusion/sqllogictest/test_files/string/string_literal.slt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -391,6 +391,10 @@ SELECT repeat(arrow_cast('foo', 'Dictionary(Int32, Utf8)'), 3)
391391
----
392392
foofoofoo
393393

394+
query error DataFusion error: Execution error: string size overflow on repeat, max size is 2147483647, but got \d+
395+
SELECT repeat(x, 9223372036854775807)
396+
FROM (VALUES ('abc')) AS t(x);
397+
394398
query T
395399
SELECT arrow_typeof(repeat('foo', 3))
396400
----

0 commit comments

Comments
 (0)