Skip to content

Commit 7738f74

Browse files
authored
fix: return error instead of capacity overflow panic in generate_series (apache#22323)
## Which issue does this PR close? - Closes apache#22188. ## Rationale for this change Captured in issue description ## What changes are included in this PR? Bounds the element count at `isize:MAX / size_of::<i64>()` in `generate_range_values` and returns a DataFusion execution error when the limit is exceeded, so the function returns a normal DataFusion error instead of panicking inside `Vec::reserve`. Only the scalar UDF path path in `datafusion/functions-nested/src/range.rs` is affected, the table-valued generate_series already streams and is unchanged. This PR just takes care of the panic and does not attempt a larger refactor towards streaming or introducing a max-elements config value. Happy to follow up on either. ## Are these changes tested? Added sqllogictest to tests in `datafusion/sqllogictest/test_files/array/array_range.slt` to cover the issues presented in the issue. ## Are there any user-facing changes? Queries that previously crashed the process with capacity overflow now returns `Execution error: Range too large to materialize: would produce {count} elements ({MAX_RANGE_ELEMENTS})` Behavior for valid ranges is unchanged.
1 parent 821260f commit 7738f74

2 files changed

Lines changed: 40 additions & 11 deletions

File tree

‎datafusion/functions-nested/src/range.rs‎

Lines changed: 26 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -329,7 +329,7 @@ impl Range {
329329
step,
330330
self.include_upper_bound,
331331
&mut values,
332-
);
332+
)?;
333333
offsets.push(values.len() as i32);
334334
valid.append_non_null();
335335
}
@@ -538,6 +538,22 @@ fn retrieve_range_args(
538538
Some((start, stop, step))
539539
}
540540

541+
/// Reserve space for `count` more elements, returning an error when the
542+
/// allocation would overflow `Vec`'s capacity limit or the allocator
543+
/// rejects it, rather than panicking on user-supplied SQL.
544+
fn reserve_range_capacity(values: &mut Vec<i64>, count: u64) -> Result<()> {
545+
let count_usize = usize::try_from(count).map_err(|_| {
546+
exec_datafusion_err!(
547+
"Range too large to materialize: would produce {count} elements"
548+
)
549+
})?;
550+
values.try_reserve(count_usize).map_err(|e| {
551+
exec_datafusion_err!(
552+
"Range too large to materialize: failed to allocate {count} elements: {e}"
553+
)
554+
})
555+
}
556+
541557
/// Generate integer range values directly into the provided buffer.
542558
#[inline]
543559
fn generate_range_values(
@@ -546,9 +562,9 @@ fn generate_range_values(
546562
step: i64,
547563
include_upper: bool,
548564
values: &mut Vec<i64>,
549-
) {
565+
) -> Result<()> {
550566
if !include_upper && start == stop {
551-
return;
567+
return Ok(());
552568
}
553569

554570
if step > 0 {
@@ -558,11 +574,10 @@ fn generate_range_values(
558574
stop.saturating_sub(1)
559575
};
560576
if start > limit {
561-
return;
577+
return Ok(());
562578
}
563-
let count =
564-
(start.abs_diff(limit) / step.unsigned_abs()).saturating_add(1) as usize;
565-
values.reserve(count);
579+
let count = (start.abs_diff(limit) / step.unsigned_abs()).saturating_add(1);
580+
reserve_range_capacity(values, count)?;
566581
let mut current = start;
567582
while current <= limit {
568583
values.push(current);
@@ -578,11 +593,10 @@ fn generate_range_values(
578593
stop.saturating_add(1)
579594
};
580595
if start < limit {
581-
return;
596+
return Ok(());
582597
}
583-
let count =
584-
(start.abs_diff(limit) / step.unsigned_abs()).saturating_add(1) as usize;
585-
values.reserve(count);
598+
let count = (start.abs_diff(limit) / step.unsigned_abs()).saturating_add(1);
599+
reserve_range_capacity(values, count)?;
586600
let mut current = start;
587601
while current >= limit {
588602
values.push(current);
@@ -592,6 +606,7 @@ fn generate_range_values(
592606
}
593607
}
594608
}
609+
Ok(())
595610
}
596611

597612
fn parse_tz(tz: &Option<&str>) -> Result<Tz> {

‎datafusion/sqllogictest/test_files/array/array_range.slt‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,13 @@ select range(5),
3838
query error DataFusion error: Execution error: step can't be 0 for function range\(start \[, stop, step\]\)
3939
select range(1, 1, 0);
4040

41+
# Range too large to materialize should error instead of panicking
42+
query error DataFusion error: Execution error: Range too large to materialize
43+
select range(0, 9223372036854775807);
44+
45+
query error DataFusion error: Execution error: Range too large to materialize
46+
select range(9223372036854775807);
47+
4148
# Test range with big steps
4249
query ????
4350
select
@@ -352,6 +359,13 @@ select generate_series(1, 1, 0);
352359
query error DataFusion error: Execution error: Interval argument to generate_series must not be 0
353360
select generate_series(TIMESTAMP '2000-01-02', TIMESTAMP '2000-01-01', INTERVAL '0' MINUTE);
354361

362+
# Range too large to materialize should error instead of panicking
363+
query error DataFusion error: Execution error: Range too large to materialize
364+
select generate_series(0, 9223372036854775807);
365+
366+
query error DataFusion error: Execution error: Range too large to materialize
367+
select generate_series(-9223372036854775808, 9223372036854775807);
368+
355369
# Test generate_series with big steps
356370
query ????
357371
select

0 commit comments

Comments
 (0)