Skip to content

Commit e391b4d

Browse files
mbutrovichmbutrovich
committed
fix: keep fractional seconds in EXTRACT(EPOCH) for Duration
1 parent 353e4ce commit e391b4d

2 files changed

Lines changed: 50 additions & 44 deletions

File tree

‎datafusion/functions/src/datetime/date_part.rs‎

Lines changed: 13 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,9 @@ use crate::datetime::common::parse_tz;
3939
use datafusion_common::{
4040
Result, ScalarValue,
4141
cast::{
42-
as_date32_array, as_date64_array, as_int32_array, as_interval_dt_array,
42+
as_date32_array, as_date64_array, as_duration_microsecond_array,
43+
as_duration_millisecond_array, as_duration_nanosecond_array,
44+
as_duration_second_array, as_int32_array, as_interval_dt_array,
4345
as_interval_mdn_array, as_interval_ym_array, as_primitive_array,
4446
as_time32_millisecond_array, as_time32_second_array, as_time64_microsecond_array,
4547
as_time64_nanosecond_array, as_timestamp_microsecond_array,
@@ -504,48 +506,6 @@ fn seconds_as_i32(array: &dyn Array, unit: TimeUnit) -> Result<ArrayRef> {
504506
}
505507
}
506508

507-
/// Invoke [`date_part`] on an `array` (e.g. Timestamp) and convert the
508-
/// result to a total number of seconds, milliseconds, microseconds or
509-
/// nanoseconds
510-
///
511-
/// Given epoch return f64, this is a duplicated function to optimize for f64 type
512-
fn seconds(array: &dyn Array, unit: TimeUnit) -> Result<ArrayRef> {
513-
let sf = match unit {
514-
Second => 1_f64,
515-
Millisecond => 1_000_f64,
516-
Microsecond => 1_000_000_f64,
517-
Nanosecond => 1_000_000_000_f64,
518-
};
519-
let secs = date_part(array, DatePart::Second)?;
520-
// This assumes array is primitive and not a dictionary
521-
let secs = as_int32_array(secs.as_ref())?;
522-
let subsecs = date_part(array, DatePart::Nanosecond)?;
523-
let subsecs = as_int32_array(subsecs.as_ref())?;
524-
525-
// Special case where there are no nulls.
526-
if subsecs.null_count() == 0 {
527-
let r: Float64Array = binary(secs, subsecs, |secs, subsecs| {
528-
(secs as f64 + ((subsecs % 1_000_000_000) as f64 / 1_000_000_000_f64)) * sf
529-
})?;
530-
Ok(Arc::new(r))
531-
} else {
532-
// Nulls in secs are preserved, nulls in subsecs are treated as zero to account for the case
533-
// where the number of nanoseconds overflows.
534-
let r: Float64Array = secs
535-
.iter()
536-
.zip(subsecs)
537-
.map(|(secs, subsecs)| {
538-
secs.map(|secs| {
539-
let subsecs = subsecs.unwrap_or(0);
540-
(secs as f64 + ((subsecs % 1_000_000_000) as f64 / 1_000_000_000_f64))
541-
* sf
542-
})
543-
})
544-
.collect();
545-
Ok(Arc::new(r))
546-
}
547-
}
548-
549509
fn epoch(array: &dyn Array) -> Result<ArrayRef> {
550510
const SECONDS_IN_A_DAY: f64 = 86400_f64;
551511
// Note: Month-to-second conversion uses 30 days as an approximation.
@@ -589,7 +549,16 @@ fn epoch(array: &dyn Array) -> Result<ArrayRef> {
589549
+ x.nanoseconds as f64 / 1_000_000_000_f64
590550
})
591551
}
592-
Duration(_) => return seconds(array, Second),
552+
Duration(Second) => as_duration_second_array(array)?.unary(|x| x as f64),
553+
Duration(Millisecond) => {
554+
as_duration_millisecond_array(array)?.unary(|x| x as f64 / 1_000_f64)
555+
}
556+
Duration(Microsecond) => {
557+
as_duration_microsecond_array(array)?.unary(|x| x as f64 / 1_000_000_f64)
558+
}
559+
Duration(Nanosecond) => {
560+
as_duration_nanosecond_array(array)?.unary(|x| x as f64 / 1_000_000_000_f64)
561+
}
593562
d => return exec_err!("Cannot convert {d:?} to epoch"),
594563
};
595564
Ok(Arc::new(f))

‎datafusion/sqllogictest/test_files/datetime/date_part.slt‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1178,6 +1178,43 @@ SELECT extract(epoch from arrow_cast(2, 'Duration(Second)'))
11781178
----
11791179
2
11801180

1181+
# epoch keeps the fraction of a duration whose total sub-second units exceed i32::MAX
1182+
query RR
1183+
SELECT extract(epoch from arrow_cast(2147483647, 'Duration(Nanosecond)')),
1184+
extract(epoch from arrow_cast(2147483648, 'Duration(Nanosecond)'))
1185+
----
1186+
2.147483647 2.147483648
1187+
1188+
query R
1189+
SELECT extract(epoch from timestamp '2025-01-01 00:00:12.923350' - timestamp '2025-01-01 00:00:00')
1190+
----
1191+
12.92335
1192+
1193+
query R
1194+
SELECT extract(epoch from arrow_cast(2147483648, 'Duration(Microsecond)'))
1195+
----
1196+
2147.483648
1197+
1198+
query R
1199+
SELECT extract(epoch from arrow_cast(2147483648, 'Duration(Millisecond)'))
1200+
----
1201+
2147483.648
1202+
1203+
query R
1204+
SELECT extract(epoch from arrow_cast(2147483648, 'Duration(Second)'))
1205+
----
1206+
2147483648
1207+
1208+
query R
1209+
SELECT extract(epoch from arrow_cast(-2500000000, 'Duration(Nanosecond)'))
1210+
----
1211+
-2.5
1212+
1213+
query R
1214+
SELECT extract(epoch from arrow_cast(NULL, 'Duration(Nanosecond)'))
1215+
----
1216+
NULL
1217+
11811218
query I
11821219
SELECT extract(millisecond from arrow_cast(2, 'Duration(Second)'))
11831220
----

0 commit comments

Comments
 (0)