Skip to content

Commit b239d04

Browse files
authored
fix: prevent date_trunc array underflow (apache#25131)
## Which issue does this PR close? - Closes apache#25129. ## Rationale for this change `date_trunc` can panic in debug builds or return an incorrect timestamp in release builds when its array fast path truncates a timestamp near `i64::MIN`. ## What changes are included in this PR? Use checked subtraction in the fine-granularity array path and return an execution error when the truncated timestamp cannot be represented. ## What is the testing strategy for this PR? Added `test_date_trunc_fine_granularity_underflow`, which passes `i64::MIN` through the nanosecond timestamp array path and verifies that `date_trunc` returns an execution error. The original SQL reproduction was also verified with the locally compiled DataFusion CLI. ## Are there any user-facing changes? Invalid extreme timestamp truncations now return an execution error instead of panicking or returning a wrapped timestamp. --- Note: This PR was developed with AI assistance. All changes have been reviewed, and I take full responsibility for this contribution.
1 parent 90dc436 commit b239d04

1 file changed

Lines changed: 127 additions & 10 deletions

File tree

‎datafusion/functions/src/datetime/date_trunc.rs‎

Lines changed: 127 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -778,13 +778,30 @@ fn general_date_trunc_array_fine_granularity<T: ArrowTimestampType>(
778778

779779
if let Some(unit) = unit {
780780
let unit = unit.get();
781-
let array = PrimitiveArray::<T>::from_iter_values_with_nulls(
782-
array
783-
.values()
784-
.iter()
785-
.map(|v| *v - i64::rem_euclid(*v, unit)),
786-
array.nulls().cloned(),
787-
)
781+
// Truncation can only underflow within one `unit` of `i64::MIN`.
782+
// Track that possibility while computing the common case so the loop
783+
// remains infallible and can be vectorized.
784+
let underflow_bound = i64::MIN + unit;
785+
let mut maybe_underflow = false;
786+
let values: Vec<i64> = array
787+
.values()
788+
.iter()
789+
.map(|value| {
790+
maybe_underflow |= *value < underflow_bound;
791+
value.wrapping_sub(value.rem_euclid(unit))
792+
})
793+
.collect();
794+
let array: PrimitiveArray<T> = if maybe_underflow {
795+
array.try_unary(|value| {
796+
value.checked_sub(value.rem_euclid(unit)).ok_or_else(|| {
797+
exec_datafusion_err!(
798+
"Timestamp {value} out of range after truncating to {granularity}"
799+
)
800+
})
801+
})?
802+
} else {
803+
PrimitiveArray::new(values.into(), array.nulls().cloned())
804+
}
788805
.with_timezone_opt(tz_opt);
789806
Ok(Arc::new(array))
790807
} else {
@@ -856,12 +873,17 @@ mod tests {
856873
use std::sync::Arc;
857874

858875
use crate::datetime::date_trunc::{
859-
DateTruncFunc, date_trunc_coarse, parse_granularity,
876+
DateTruncFunc, NANOS_PER_MICROSECOND, date_trunc_coarse,
877+
general_date_trunc_array_fine_granularity, parse_granularity,
860878
};
861879

862880
use arrow::array::cast::as_primitive_array;
863-
use arrow::array::types::TimestampNanosecondType;
864-
use arrow::array::{Array, TimestampNanosecondArray};
881+
use arrow::array::types::{ArrowTimestampType, TimestampNanosecondType};
882+
use arrow::array::{
883+
Array, PrimitiveArray, TimestampMicrosecondArray, TimestampMillisecondArray,
884+
TimestampNanosecondArray, TimestampSecondArray,
885+
};
886+
use arrow::buffer::NullBuffer;
865887
use arrow::compute::DatePart;
866888
use arrow::compute::kernels::cast_utils::string_to_timestamp_nanos;
867889
use arrow::datatypes::{DataType, Field, TimeUnit};
@@ -1339,6 +1361,101 @@ mod tests {
13391361
}
13401362
}
13411363

1364+
fn assert_fine_granularity_underflow<T: ArrowTimestampType>(
1365+
array: PrimitiveArray<T>,
1366+
granularity: DatePart,
1367+
) {
1368+
let error =
1369+
general_date_trunc_array_fine_granularity(T::UNIT, &array, granularity, None)
1370+
.unwrap_err();
1371+
assert!(
1372+
error
1373+
.strip_backtrace()
1374+
.contains("Timestamp -9223372036854775808 out of range"),
1375+
"unexpected error: {error}"
1376+
);
1377+
}
1378+
1379+
#[test]
1380+
fn test_date_trunc_fine_granularity_underflow_for_each_timestamp_unit() {
1381+
assert_fine_granularity_underflow(
1382+
TimestampSecondArray::from(vec![i64::MIN]),
1383+
DatePart::Minute,
1384+
);
1385+
assert_fine_granularity_underflow(
1386+
TimestampMillisecondArray::from(vec![i64::MIN]),
1387+
DatePart::Second,
1388+
);
1389+
assert_fine_granularity_underflow(
1390+
TimestampMicrosecondArray::from(vec![i64::MIN]),
1391+
DatePart::Millisecond,
1392+
);
1393+
assert_fine_granularity_underflow(
1394+
TimestampNanosecondArray::from(vec![i64::MIN]),
1395+
DatePart::Microsecond,
1396+
);
1397+
}
1398+
1399+
#[test]
1400+
fn test_date_trunc_fine_granularity_minimum_safe_input() {
1401+
// The boundary below i64::MIN is unrepresentable. Inputs become safe
1402+
// at the next microsecond boundary.
1403+
let distance_to_next_boundary =
1404+
NANOS_PER_MICROSECOND - i64::MIN.rem_euclid(NANOS_PER_MICROSECOND);
1405+
let minimum_safe_input = i64::MIN + distance_to_next_boundary;
1406+
let unsafe_input = TimestampNanosecondArray::from(vec![minimum_safe_input - 1]);
1407+
assert!(
1408+
general_date_trunc_array_fine_granularity(
1409+
TimeUnit::Nanosecond,
1410+
&unsafe_input,
1411+
DatePart::Microsecond,
1412+
None,
1413+
)
1414+
.is_err()
1415+
);
1416+
1417+
let input = TimestampNanosecondArray::from(vec![
1418+
Some(minimum_safe_input),
1419+
Some(minimum_safe_input + 1),
1420+
None,
1421+
]);
1422+
let result = general_date_trunc_array_fine_granularity(
1423+
TimeUnit::Nanosecond,
1424+
&input,
1425+
DatePart::Microsecond,
1426+
None,
1427+
)
1428+
.unwrap();
1429+
let result = as_primitive_array::<TimestampNanosecondType>(&result);
1430+
let expected = TimestampNanosecondArray::from(vec![
1431+
Some(minimum_safe_input),
1432+
Some(minimum_safe_input),
1433+
None,
1434+
]);
1435+
assert_eq!(result, &expected);
1436+
}
1437+
1438+
#[test]
1439+
fn test_date_trunc_fine_granularity_ignores_null_slot_values() {
1440+
// Physical values behind null slots are arbitrary. An unsafe value in
1441+
// a null slot may trigger the checked fallback, but must not cause an
1442+
// error or alter the null bitmap.
1443+
let input = TimestampNanosecondArray::new(
1444+
vec![i64::MIN, 1].into(),
1445+
Some(NullBuffer::from(vec![false, true])),
1446+
);
1447+
let result = general_date_trunc_array_fine_granularity(
1448+
TimeUnit::Nanosecond,
1449+
&input,
1450+
DatePart::Microsecond,
1451+
None,
1452+
)
1453+
.unwrap();
1454+
let result = as_primitive_array::<TimestampNanosecondType>(&result);
1455+
let expected = TimestampNanosecondArray::from(vec![None, Some(0)]);
1456+
assert_eq!(result, &expected);
1457+
}
1458+
13421459
#[test]
13431460
fn test_date_trunc_fine_granularity_timezones() {
13441461
let cases = [

0 commit comments

Comments
 (0)