Skip to content

Commit 8bfc0ea

Browse files
Jefffreywaynexia
andauthored
Parse nanoseconds for intervals (#4186)
* Parse nanoseconds for intervals * Formatting Co-authored-by: Ruihang Xia <[email protected]> * Empty commit Co-authored-by: Ruihang Xia <[email protected]>
1 parent d91ce06 commit 8bfc0ea

2 files changed

Lines changed: 81 additions & 68 deletions

File tree

‎datafusion/common/src/parsers.rs‎

Lines changed: 78 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@
1919
use crate::{DataFusionError, Result, ScalarValue};
2020
use std::str::FromStr;
2121

22-
const SECONDS_PER_HOUR: f32 = 3_600_f32;
23-
const MILLIS_PER_SECOND: f32 = 1_000_f32;
22+
const SECONDS_PER_HOUR: f64 = 3_600_f64;
23+
const NANOS_PER_SECOND: f64 = 1_000_000_000_f64;
2424

2525
/// Parses a string with an interval like `'0.5 MONTH'` to an
2626
/// appropriately typed [`ScalarValue`]
@@ -29,73 +29,74 @@ pub fn parse_interval(leading_field: &str, value: &str) -> Result<ScalarValue> {
2929
// INTERVAL '0.5 MONTH' = 15 days, INTERVAL '1.5 MONTH' = 1 month 15 days
3030
// INTERVAL '0.5 DAY' = 12 hours, INTERVAL '1.5 DAY' = 1 day 12 hours
3131
let align_interval_parts =
32-
|month_part: f32, mut day_part: f32, mut milles_part: f32| -> (i32, i32, f32) {
32+
|month_part: f64, mut day_part: f64, mut nanos_part: f64| -> (i64, i64, f64) {
3333
// Convert fractional month to days, It's not supported by Arrow types, but anyway
34-
day_part += (month_part - (month_part as i32) as f32) * 30_f32;
34+
day_part += (month_part - (month_part as i64) as f64) * 30_f64;
3535

3636
// Convert fractional days to hours
37-
milles_part += (day_part - ((day_part as i32) as f32))
38-
* 24_f32
37+
nanos_part += (day_part - ((day_part as i64) as f64))
38+
* 24_f64
3939
* SECONDS_PER_HOUR
40-
* MILLIS_PER_SECOND;
40+
* NANOS_PER_SECOND;
4141

42-
(month_part as i32, day_part as i32, milles_part)
42+
(month_part as i64, day_part as i64, nanos_part)
4343
};
4444

45-
let calculate_from_part =
46-
|interval_period_str: &str, interval_type: &str| -> Result<(i32, i32, f32)> {
47-
// @todo It's better to use Decimal in order to protect rounding errors
48-
// Wait https://github.com/apache/arrow/pull/9232
49-
let interval_period = match f32::from_str(interval_period_str) {
50-
Ok(n) => n,
51-
Err(_) => {
52-
return Err(DataFusionError::NotImplemented(format!(
53-
"Unsupported Interval Expression with value {:?}",
54-
value
55-
)));
56-
}
57-
};
58-
59-
if interval_period > (i32::MAX as f32) {
45+
let calculate_from_part = |interval_period_str: &str,
46+
interval_type: &str|
47+
-> Result<(i64, i64, f64)> {
48+
// @todo It's better to use Decimal in order to protect rounding errors
49+
// Wait https://github.com/apache/arrow/pull/9232
50+
let interval_period = match f64::from_str(interval_period_str) {
51+
Ok(n) => n,
52+
Err(_) => {
6053
return Err(DataFusionError::NotImplemented(format!(
61-
"Interval field value out of range: {:?}",
54+
"Unsupported Interval Expression with value {:?}",
6255
value
6356
)));
6457
}
58+
};
6559

66-
match interval_type.to_lowercase().as_str() {
67-
"century" | "centuries" => {
68-
Ok(align_interval_parts(interval_period * 1200_f32, 0.0, 0.0))
69-
}
70-
"decade" | "decades" => {
71-
Ok(align_interval_parts(interval_period * 120_f32, 0.0, 0.0))
72-
}
73-
"year" | "years" => {
74-
Ok(align_interval_parts(interval_period * 12_f32, 0.0, 0.0))
75-
}
76-
"month" | "months" => Ok(align_interval_parts(interval_period, 0.0, 0.0)),
77-
"week" | "weeks" => {
78-
Ok(align_interval_parts(0.0, interval_period * 7_f32, 0.0))
79-
}
80-
"day" | "days" => Ok(align_interval_parts(0.0, interval_period, 0.0)),
81-
"hour" | "hours" => {
82-
Ok((0, 0, interval_period * SECONDS_PER_HOUR * MILLIS_PER_SECOND))
83-
}
84-
"minute" | "minutes" => {
85-
Ok((0, 0, interval_period * 60_f32 * MILLIS_PER_SECOND))
86-
}
87-
"second" | "seconds" => Ok((0, 0, interval_period * MILLIS_PER_SECOND)),
88-
"millisecond" | "milliseconds" => Ok((0, 0, interval_period)),
89-
_ => Err(DataFusionError::NotImplemented(format!(
90-
"Invalid input syntax for type interval: {:?}",
91-
value
92-
))),
60+
if interval_period > (i64::MAX as f64) {
61+
return Err(DataFusionError::NotImplemented(format!(
62+
"Interval field value out of range: {:?}",
63+
value
64+
)));
65+
}
66+
67+
match interval_type.to_lowercase().as_str() {
68+
"century" | "centuries" => {
69+
Ok(align_interval_parts(interval_period * 1200_f64, 0.0, 0.0))
9370
}
94-
};
71+
"decade" | "decades" => {
72+
Ok(align_interval_parts(interval_period * 120_f64, 0.0, 0.0))
73+
}
74+
"year" | "years" => {
75+
Ok(align_interval_parts(interval_period * 12_f64, 0.0, 0.0))
76+
}
77+
"month" | "months" => Ok(align_interval_parts(interval_period, 0.0, 0.0)),
78+
"week" | "weeks" => {
79+
Ok(align_interval_parts(0.0, interval_period * 7_f64, 0.0))
80+
}
81+
"day" | "days" => Ok(align_interval_parts(0.0, interval_period, 0.0)),
82+
"hour" | "hours" => {
83+
Ok((0, 0, interval_period * SECONDS_PER_HOUR * NANOS_PER_SECOND))
84+
}
85+
"minute" | "minutes" => {
86+
Ok((0, 0, interval_period * 60_f64 * NANOS_PER_SECOND))
87+
}
88+
"second" | "seconds" => Ok((0, 0, interval_period * NANOS_PER_SECOND)),
89+
"millisecond" | "milliseconds" => Ok((0, 0, interval_period * 1_000_000f64)),
90+
_ => Err(DataFusionError::NotImplemented(format!(
91+
"Invalid input syntax for type interval: {:?}",
92+
value
93+
))),
94+
}
95+
};
9596

9697
let mut result_month: i64 = 0;
9798
let mut result_days: i64 = 0;
98-
let mut result_millis: i64 = 0;
99+
let mut result_nanos: i128 = 0;
99100

100101
let mut parts = value.split_whitespace();
101102

@@ -107,7 +108,7 @@ pub fn parse_interval(leading_field: &str, value: &str) -> Result<ScalarValue> {
107108

108109
let unit = parts.next().unwrap_or(leading_field);
109110

110-
let (diff_month, diff_days, diff_millis) =
111+
let (diff_month, diff_days, diff_nanos) =
111112
calculate_from_part(interval_period_str.unwrap(), unit)?;
112113

113114
result_month += diff_month as i64;
@@ -128,9 +129,9 @@ pub fn parse_interval(leading_field: &str, value: &str) -> Result<ScalarValue> {
128129
)));
129130
}
130131

131-
result_millis += diff_millis as i64;
132+
result_nanos += diff_nanos as i128;
132133

133-
if result_millis > (i32::MAX as i64) {
134+
if result_nanos > (i64::MAX as i128) {
134135
return Err(DataFusionError::NotImplemented(format!(
135136
"Interval field value out of range: {:?}",
136137
value
@@ -142,11 +143,13 @@ pub fn parse_interval(leading_field: &str, value: &str) -> Result<ScalarValue> {
142143
// 1 day is not 24 hours because timezones, 1 year != 365/364! 30 days != 1 month
143144
// The true way to store and calculate intervals is to store it as it defined
144145
// It's why we there are 3 different interval types in Arrow
145-
if result_month != 0 && (result_days != 0 || result_millis != 0) {
146-
let result: i128 = ((result_month as i128) << 96)
147-
| ((result_days as i128) << 64)
148-
// IntervalMonthDayNano uses nanos, but IntervalDayTime uses milles
149-
| ((result_millis * 1_000_000_i64) as i128);
146+
147+
// If have a unit smaller than milliseconds then must use IntervalMonthDayNano
148+
if (result_nanos % 1_000_000 != 0)
149+
|| (result_month != 0 && (result_days != 0 || result_nanos != 0))
150+
{
151+
let result: i128 =
152+
((result_month as i128) << 96) | ((result_days as i128) << 64) | result_nanos;
150153

151154
return Ok(ScalarValue::IntervalMonthDayNano(Some(result)));
152155
}
@@ -156,7 +159,8 @@ pub fn parse_interval(leading_field: &str, value: &str) -> Result<ScalarValue> {
156159
return Ok(ScalarValue::IntervalYearMonth(Some(result_month as i32)));
157160
}
158161

159-
let result: i64 = (result_days << 32) | result_millis;
162+
// IntervalMonthDayNano uses nanos, but IntervalDayTime uses millis
163+
let result: i64 = (result_days << 32) | (result_nanos as i64 / 1_000_000);
160164
Ok(ScalarValue::IntervalDayTime(Some(result)))
161165
}
162166

@@ -165,6 +169,8 @@ mod test {
165169
use super::*;
166170
use crate::assert_contains;
167171

172+
const MILLIS_PER_SECOND: f64 = 1_000_f64;
173+
168174
#[test]
169175
fn test_parse_ym() {
170176
assert_eq!(
@@ -212,13 +218,20 @@ mod test {
212218
}
213219

214220
#[test]
215-
// https://github.com/apache/arrow-rs/pull/2235
216-
#[ignore]
217221
fn test_mdn() {
218-
// these should be the same, I think
219222
assert_eq!(
220223
parse_interval("months", "1 year 25 millisecond").unwrap(),
221224
ScalarValue::new_interval_mdn(12, 0, 25 * 1_000_000)
222225
);
226+
227+
assert_eq!(
228+
parse_interval("months", "1 year 1 day 0.000000001 seconds").unwrap(),
229+
ScalarValue::new_interval_mdn(12, 1, 1)
230+
);
231+
232+
assert_eq!(
233+
parse_interval("months", "1 year 1 day 0.1 milliseconds").unwrap(),
234+
ScalarValue::new_interval_mdn(12, 1, 1_00 * 1_000)
235+
);
223236
}
224237
}

‎datafusion/core/tests/sql/expr.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -917,19 +917,19 @@ async fn test_interval_expressions() -> Result<()> {
917917
);
918918
test_expression!(
919919
"interval '0.499 day'",
920-
"0 years 0 mons 0 days 11 hours 58 mins 33.596 secs"
920+
"0 years 0 mons 0 days 11 hours 58 mins 33.600 secs"
921921
);
922922
test_expression!(
923923
"interval '0.4999 day'",
924-
"0 years 0 mons 0 days 11 hours 59 mins 51.364 secs"
924+
"0 years 0 mons 0 days 11 hours 59 mins 51.360 secs"
925925
);
926926
test_expression!(
927927
"interval '0.49999 day'",
928928
"0 years 0 mons 0 days 11 hours 59 mins 59.136 secs"
929929
);
930930
test_expression!(
931931
"interval '0.49999999999 day'",
932-
"0 years 0 mons 0 days 12 hours 0 mins 0.00 secs"
932+
"0 years 0 mons 0 days 11 hours 59 mins 59.999999136 secs"
933933
);
934934
test_expression!(
935935
"interval '5 day'",

0 commit comments

Comments
 (0)