Skip to content

Commit 9aebcea

Browse files
authored
fix date_bin overflows scaling extreme Timestamp(Second) source (apache#22315)
## Which issue does this PR close? - Closes apache#22211. ## Rationale for this change `date_bin` could panic during planning or constant evaluation when scaling a non-nanosecond source timestamp to nanoseconds overflowed. This change makes that path return a regular error instead of panicking. ## What changes are included in this PR? - Added checked overflow handling for source timestamp scaling in `date_bin`. - Return an error for out-of-range source timestamp conversion instead of panicking. - Preserved existing `NULL` behavior for unrelated out-of-range `date_bin` cases. - Added Rust unit test and sqllogictest coverage for the overflow case. ## Are these changes tested? Yes. Verified with: - `cargo test -p datafusion-functions test_date_bin --lib` - `cargo test -p datafusion-sqllogictest --test sqllogictests date_bin_errors` ## Are there any user-facing changes? Yes. Queries that previously could panic now return a normal error: `Execution error: DATE_BIN source timestamp ... cannot be represented in nanoseconds`
1 parent 5b22857 commit 9aebcea

2 files changed

Lines changed: 111 additions & 55 deletions

File tree

‎datafusion/functions/src/datetime/date_bin.rs‎

Lines changed: 91 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,12 @@ use arrow::datatypes::{
3131
DataType, Time32MillisecondType, Time32SecondType, Time64MicrosecondType,
3232
Time64NanosecondType, TimeUnit,
3333
};
34+
use arrow::error::ArrowError;
3435
use arrow::temporal_conversions::NANOSECONDS_IN_DAY;
3536
use datafusion_common::cast::as_primitive_array;
36-
use datafusion_common::{Result, ScalarValue, exec_err, not_impl_err, plan_err};
37+
use datafusion_common::{
38+
DataFusionError, Result, ScalarValue, exec_err, not_impl_err, plan_err,
39+
};
3740
use datafusion_expr::TypeSignature::Exact;
3841
use datafusion_expr::sort_properties::{ExprProperties, SortProperties};
3942
use datafusion_expr::{
@@ -322,7 +325,7 @@ impl Interval {
322325
// return time in nanoseconds that the source timestamp falls into based on the stride and origin
323326
fn date_bin_nanos_interval(stride_nanos: i64, source: i64, origin: i64) -> Result<i64> {
324327
let time_diff = source.checked_sub(origin).ok_or_else(|| {
325-
arrow::error::ArrowError::InvalidArgumentError(format!(
328+
ArrowError::InvalidArgumentError(format!(
326329
"date_bin source timestamp {source} - origin {origin} overflows i64"
327330
))
328331
})?;
@@ -331,7 +334,7 @@ fn date_bin_nanos_interval(stride_nanos: i64, source: i64, origin: i64) -> Resul
331334
let time_delta = compute_distance(time_diff, stride_nanos)?;
332335

333336
origin.checked_add(time_delta).ok_or_else(|| {
334-
arrow::error::ArrowError::InvalidArgumentError(format!(
337+
ArrowError::InvalidArgumentError(format!(
335338
"date_bin origin {origin} + delta {time_delta} overflows i64"
336339
))
337340
.into()
@@ -341,20 +344,20 @@ fn date_bin_nanos_interval(stride_nanos: i64, source: i64, origin: i64) -> Resul
341344
// distance from origin to bin
342345
fn compute_distance(time_diff: i64, stride: i64) -> Result<i64> {
343346
let remainder = time_diff.checked_rem(stride).ok_or_else(|| {
344-
arrow::error::ArrowError::InvalidArgumentError(format!(
347+
ArrowError::InvalidArgumentError(format!(
345348
"date_bin compute_distance time_diff {time_diff} % stride {stride} overflows i64"
346349
))
347350
})?;
348351
let time_delta = time_diff.checked_sub(remainder).ok_or_else(|| {
349-
arrow::error::ArrowError::InvalidArgumentError(format!(
352+
ArrowError::InvalidArgumentError(format!(
350353
"date_bin compute_distance time_diff {time_diff} - remainder {remainder} overflows i64"
351354
))
352355
})?;
353356

354357
if time_diff < 0 && stride > 1 && time_delta != time_diff {
355358
// The origin is later than the source timestamp, round down to the previous bin
356359
time_delta.checked_sub(stride).ok_or_else(|| {
357-
arrow::error::ArrowError::InvalidArgumentError(format!(
360+
ArrowError::InvalidArgumentError(format!(
358361
"date_bin compute_distance time_delta {time_delta} - stride {stride} overflows i64"
359362
))
360363
.into()
@@ -594,53 +597,91 @@ fn date_bin_impl(
594597
return exec_err!("DATE_BIN stride must be non-zero");
595598
}
596599

597-
fn stride_map_fn<T: ArrowTimestampType>(
598-
origin: i64,
599-
stride: i64,
600-
stride_fn: BinFunction,
601-
) -> impl Fn(i64) -> Result<i64> {
602-
let scale = match T::UNIT {
600+
fn timestamp_scale<T: ArrowTimestampType>() -> i64 {
601+
match T::UNIT {
603602
Nanosecond => 1,
604603
Microsecond => NANOS_PER_MICRO,
605604
Millisecond => NANOS_PER_MILLI,
606605
Second => NANOSECONDS,
607-
};
608-
move |x: i64| match stride_fn(stride, x * scale, origin) {
609-
Ok(result) => Ok(result / scale),
610-
Err(e) => Err(e),
611606
}
612607
}
613608

609+
fn timestamp_scale_overflow_error(x: i64) -> DataFusionError {
610+
DataFusionError::Execution(format!(
611+
"DATE_BIN source timestamp {x} cannot be represented in nanoseconds"
612+
))
613+
}
614+
614615
Ok(match array {
615616
ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(v, tz_opt)) => {
616-
let apply_stride_fn =
617-
stride_map_fn::<TimestampNanosecondType>(origin, stride, stride_fn);
617+
let scale = timestamp_scale::<TimestampNanosecondType>();
618618
ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(
619-
v.and_then(|val| apply_stride_fn(val).ok()),
619+
match *v {
620+
Some(val) => {
621+
let scaled = val
622+
.checked_mul(scale)
623+
.ok_or_else(|| timestamp_scale_overflow_error(val))?;
624+
match stride_fn(stride, scaled, origin) {
625+
Ok(result) => Some(result / scale),
626+
Err(_) => None,
627+
}
628+
}
629+
None => None,
630+
},
620631
tz_opt.clone(),
621632
))
622633
}
623634
ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(v, tz_opt)) => {
624-
let apply_stride_fn =
625-
stride_map_fn::<TimestampMicrosecondType>(origin, stride, stride_fn);
635+
let scale = timestamp_scale::<TimestampMicrosecondType>();
626636
ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(
627-
v.and_then(|val| apply_stride_fn(val).ok()),
637+
match *v {
638+
Some(val) => {
639+
let scaled = val
640+
.checked_mul(scale)
641+
.ok_or_else(|| timestamp_scale_overflow_error(val))?;
642+
match stride_fn(stride, scaled, origin) {
643+
Ok(result) => Some(result / scale),
644+
Err(_) => None,
645+
}
646+
}
647+
None => None,
648+
},
628649
tz_opt.clone(),
629650
))
630651
}
631652
ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(v, tz_opt)) => {
632-
let apply_stride_fn =
633-
stride_map_fn::<TimestampMillisecondType>(origin, stride, stride_fn);
653+
let scale = timestamp_scale::<TimestampMillisecondType>();
634654
ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(
635-
v.and_then(|val| apply_stride_fn(val).ok()),
655+
match *v {
656+
Some(val) => {
657+
let scaled = val
658+
.checked_mul(scale)
659+
.ok_or_else(|| timestamp_scale_overflow_error(val))?;
660+
match stride_fn(stride, scaled, origin) {
661+
Ok(result) => Some(result / scale),
662+
Err(_) => None,
663+
}
664+
}
665+
None => None,
666+
},
636667
tz_opt.clone(),
637668
))
638669
}
639670
ColumnarValue::Scalar(ScalarValue::TimestampSecond(v, tz_opt)) => {
640-
let apply_stride_fn =
641-
stride_map_fn::<TimestampSecondType>(origin, stride, stride_fn);
671+
let scale = timestamp_scale::<TimestampSecondType>();
642672
ColumnarValue::Scalar(ScalarValue::TimestampSecond(
643-
v.and_then(|val| apply_stride_fn(val).ok()),
673+
match *v {
674+
Some(val) => {
675+
let scaled = val
676+
.checked_mul(scale)
677+
.ok_or_else(|| timestamp_scale_overflow_error(val))?;
678+
match stride_fn(stride, scaled, origin) {
679+
Ok(result) => Some(result / scale),
680+
Err(_) => None,
681+
}
682+
}
683+
None => None,
684+
},
644685
tz_opt.clone(),
645686
))
646687
}
@@ -710,20 +751,24 @@ fn date_bin_impl(
710751
T: ArrowTimestampType,
711752
{
712753
let array = as_primitive_array::<T>(array)?;
713-
let scale = match T::UNIT {
714-
Nanosecond => 1,
715-
Microsecond => NANOS_PER_MICRO,
716-
Millisecond => NANOS_PER_MILLI,
717-
Second => NANOSECONDS,
718-
};
719-
720-
let result: PrimitiveArray<T> = array.try_unary(|val| {
721-
stride_fn(stride, val * scale, origin)
722-
.map(|binned| binned / scale)
723-
.map_err(|e| {
724-
arrow::error::ArrowError::ComputeError(e.to_string())
725-
})
726-
})?;
754+
let scale = timestamp_scale::<T>();
755+
756+
let values = array
757+
.iter()
758+
.map(|val| match val {
759+
Some(val) => {
760+
let scaled = val
761+
.checked_mul(scale)
762+
.ok_or_else(|| timestamp_scale_overflow_error(val))?;
763+
Ok(stride_fn(stride, scaled, origin)
764+
.ok()
765+
.map(|binned| binned / scale))
766+
}
767+
None => Ok(None),
768+
})
769+
.collect::<Result<Vec<_>>>()?;
770+
771+
let result = PrimitiveArray::<T>::from_iter(values);
727772

728773
let array = result.with_timezone_opt(tz_opt.clone());
729774
Ok(ColumnarValue::Array(Arc::new(array)))
@@ -764,9 +809,7 @@ fn date_bin_impl(
764809
let nanos = binned_nanos % (NANOSECONDS_IN_DAY);
765810
(nanos / NANOS_PER_MILLI) as i32
766811
})
767-
.map_err(|e| {
768-
arrow::error::ArrowError::ComputeError(e.to_string())
769-
})
812+
.map_err(|e| ArrowError::ComputeError(e.to_string()))
770813
})?;
771814
ColumnarValue::Array(Arc::new(result))
772815
}
@@ -784,9 +827,7 @@ fn date_bin_impl(
784827
let nanos = binned_nanos % (NANOSECONDS_IN_DAY);
785828
(nanos / NANOS_PER_SEC) as i32
786829
})
787-
.map_err(|e| {
788-
arrow::error::ArrowError::ComputeError(e.to_string())
789-
})
830+
.map_err(|e| ArrowError::ComputeError(e.to_string()))
790831
})?;
791832
ColumnarValue::Array(Arc::new(result))
792833
}
@@ -804,9 +845,7 @@ fn date_bin_impl(
804845
let nanos = binned_nanos % (NANOSECONDS_IN_DAY);
805846
nanos / NANOS_PER_MICRO
806847
})
807-
.map_err(|e| {
808-
arrow::error::ArrowError::ComputeError(e.to_string())
809-
})
848+
.map_err(|e| ArrowError::ComputeError(e.to_string()))
810849
})?;
811850
ColumnarValue::Array(Arc::new(result))
812851
}
@@ -821,9 +860,7 @@ fn date_bin_impl(
821860
array.try_unary(|x| {
822861
stride_fn(stride, x, origin)
823862
.map(|binned_nanos| binned_nanos % (NANOSECONDS_IN_DAY))
824-
.map_err(|e| {
825-
arrow::error::ArrowError::ComputeError(e.to_string())
826-
})
863+
.map_err(|e| ArrowError::ComputeError(e.to_string()))
827864
})?;
828865
ColumnarValue::Array(Arc::new(result))
829866
}

‎datafusion/sqllogictest/test_files/date_bin_errors.slt‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,4 +77,23 @@ select date_bin(
7777
arrow_cast(-9223372036854775808, 'Timestamp(Nanosecond, None)')
7878
);
7979
----
80-
NULL
80+
NULL
81+
82+
# Source timestamp scaling to nanoseconds overflows: should return an error, not panic
83+
query error DataFusion error: Execution error: DATE_BIN source timestamp 9223372036854775807 cannot be represented in nanoseconds
84+
select date_bin(
85+
interval '1 nanosecond',
86+
arrow_cast(9223372036854775807, 'Timestamp(Second, None)'),
87+
timestamp '1970-01-01 00:00:00'
88+
);
89+
90+
# Source timestamp scaling to nanoseconds overflows in array path: should return an error, not panic
91+
query error DataFusion error: Execution error: DATE_BIN source timestamp 9223372036854775807 cannot be represented in nanoseconds
92+
select date_bin(
93+
interval '1 nanosecond',
94+
ts,
95+
timestamp '1970-01-01 00:00:00'
96+
)
97+
from (
98+
values (arrow_cast(9223372036854775807, 'Timestamp(Second, None)'))
99+
) as t(ts);

0 commit comments

Comments
 (0)