Skip to content

Commit a46b97e

Browse files
Support Time32 and Time64 for Type Coercion
1 parent 807a0c1 commit a46b97e

7 files changed

Lines changed: 358 additions & 43 deletions

File tree

‎datafusion/common/src/scalar.rs‎

Lines changed: 129 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -86,8 +86,14 @@ pub enum ScalarValue {
8686
Date32(Option<i32>),
8787
/// Date stored as a signed 64bit int milliseconds since UNIX epoch 1970-01-01
8888
Date64(Option<i64>),
89+
/// Time stored as a signed 32bit int as seconds since midnight
90+
Time32Second(Option<i32>),
91+
/// Time stored as a signed 32bit int as milliseconds since midnight
92+
Time32Millisecond(Option<i32>),
93+
/// Time stored as a signed 64bit int as microseconds since midnight
94+
Time64Microsecond(Option<i64>),
8995
/// Time stored as a signed 64bit int as nanoseconds since midnight
90-
Time64(Option<i64>),
96+
Time64Nanosecond(Option<i64>),
9197
/// Timestamp Second
9298
TimestampSecond(Option<i64>, Option<String>),
9399
/// Timestamp Milliseconds
@@ -168,8 +174,14 @@ impl PartialEq for ScalarValue {
168174
(Date32(_), _) => false,
169175
(Date64(v1), Date64(v2)) => v1.eq(v2),
170176
(Date64(_), _) => false,
171-
(Time64(v1), Time64(v2)) => v1.eq(v2),
172-
(Time64(_), _) => false,
177+
(Time32Second(v1), Time32Second(v2)) => v1.eq(v2),
178+
(Time32Second(_), _) => false,
179+
(Time32Millisecond(v1), Time32Millisecond(v2)) => v1.eq(v2),
180+
(Time32Millisecond(_), _) => false,
181+
(Time64Microsecond(v1), Time64Microsecond(v2)) => v1.eq(v2),
182+
(Time64Microsecond(_), _) => false,
183+
(Time64Nanosecond(v1), Time64Nanosecond(v2)) => v1.eq(v2),
184+
(Time64Nanosecond(_), _) => false,
173185
(TimestampSecond(v1, _), TimestampSecond(v2, _)) => v1.eq(v2),
174186
(TimestampSecond(_, _), _) => false,
175187
(TimestampMillisecond(v1, _), TimestampMillisecond(v2, _)) => v1.eq(v2),
@@ -262,8 +274,14 @@ impl PartialOrd for ScalarValue {
262274
(Date32(_), _) => None,
263275
(Date64(v1), Date64(v2)) => v1.partial_cmp(v2),
264276
(Date64(_), _) => None,
265-
(Time64(v1), Time64(v2)) => v1.partial_cmp(v2),
266-
(Time64(_), _) => None,
277+
(Time32Second(v1), Time32Second(v2)) => v1.partial_cmp(v2),
278+
(Time32Second(_), _) => None,
279+
(Time32Millisecond(v1), Time32Millisecond(v2)) => v1.partial_cmp(v2),
280+
(Time32Millisecond(_), _) => None,
281+
(Time64Microsecond(v1), Time64Microsecond(v2)) => v1.partial_cmp(v2),
282+
(Time64Microsecond(_), _) => None,
283+
(Time64Nanosecond(v1), Time64Nanosecond(v2)) => v1.partial_cmp(v2),
284+
(Time64Nanosecond(_), _) => None,
267285
(TimestampSecond(v1, _), TimestampSecond(v2, _)) => v1.partial_cmp(v2),
268286
(TimestampSecond(_, _), _) => None,
269287
(TimestampMillisecond(v1, _), TimestampMillisecond(v2, _)) => {
@@ -544,7 +562,10 @@ impl std::hash::Hash for ScalarValue {
544562
}
545563
Date32(v) => v.hash(state),
546564
Date64(v) => v.hash(state),
547-
Time64(v) => v.hash(state),
565+
Time32Second(v) => v.hash(state),
566+
Time32Millisecond(v) => v.hash(state),
567+
Time64Microsecond(v) => v.hash(state),
568+
Time64Nanosecond(v) => v.hash(state),
548569
TimestampSecond(v, _) => v.hash(state),
549570
TimestampMillisecond(v, _) => v.hash(state),
550571
TimestampMicrosecond(v, _) => v.hash(state),
@@ -909,7 +930,10 @@ impl ScalarValue {
909930
))),
910931
ScalarValue::Date32(_) => DataType::Date32,
911932
ScalarValue::Date64(_) => DataType::Date64,
912-
ScalarValue::Time64(_) => DataType::Time64(TimeUnit::Nanosecond),
933+
ScalarValue::Time32Second(_) => DataType::Time32(TimeUnit::Second),
934+
ScalarValue::Time32Millisecond(_) => DataType::Time32(TimeUnit::Millisecond),
935+
ScalarValue::Time64Microsecond(_) => DataType::Time64(TimeUnit::Microsecond),
936+
ScalarValue::Time64Nanosecond(_) => DataType::Time64(TimeUnit::Nanosecond),
913937
ScalarValue::IntervalYearMonth(_) => {
914938
DataType::Interval(IntervalUnit::YearMonth)
915939
}
@@ -992,7 +1016,10 @@ impl ScalarValue {
9921016
ScalarValue::List(v, _) => v.is_none(),
9931017
ScalarValue::Date32(v) => v.is_none(),
9941018
ScalarValue::Date64(v) => v.is_none(),
995-
ScalarValue::Time64(v) => v.is_none(),
1019+
ScalarValue::Time32Second(v) => v.is_none(),
1020+
ScalarValue::Time32Millisecond(v) => v.is_none(),
1021+
ScalarValue::Time64Microsecond(v) => v.is_none(),
1022+
ScalarValue::Time64Nanosecond(v) => v.is_none(),
9961023
ScalarValue::TimestampSecond(v, _) => v.is_none(),
9971024
ScalarValue::TimestampMillisecond(v, _) => v.is_none(),
9981025
ScalarValue::TimestampMicrosecond(v, _) => v.is_none(),
@@ -1215,8 +1242,17 @@ impl ScalarValue {
12151242
DataType::LargeBinary => build_array_string!(LargeBinaryArray, LargeBinary),
12161243
DataType::Date32 => build_array_primitive!(Date32Array, Date32),
12171244
DataType::Date64 => build_array_primitive!(Date64Array, Date64),
1245+
DataType::Time32(TimeUnit::Second) => {
1246+
build_array_primitive!(Time32SecondArray, Time32Second)
1247+
}
1248+
DataType::Time32(TimeUnit::Millisecond) => {
1249+
build_array_primitive!(Time32MillisecondArray, Time32Millisecond)
1250+
}
1251+
DataType::Time64(TimeUnit::Microsecond) => {
1252+
build_array_primitive!(Time64MicrosecondArray, Time64Microsecond)
1253+
}
12181254
DataType::Time64(TimeUnit::Nanosecond) => {
1219-
build_array_primitive!(Time64NanosecondArray, Time64)
1255+
build_array_primitive!(Time64NanosecondArray, Time64Nanosecond)
12201256
}
12211257
DataType::Timestamp(TimeUnit::Second, _) => {
12221258
build_array_primitive_tz!(TimestampSecondArray, TimestampSecond)
@@ -1360,8 +1396,10 @@ impl ScalarValue {
13601396
// explicitly enumerate unsupported types so newly added
13611397
// types must be aknowledged
13621398
DataType::Float16
1363-
| DataType::Time32(_)
1364-
| DataType::Time64(_)
1399+
| DataType::Time32(TimeUnit::Microsecond)
1400+
| DataType::Time32(TimeUnit::Nanosecond)
1401+
| DataType::Time64(TimeUnit::Second)
1402+
| DataType::Time64(TimeUnit::Millisecond)
13651403
| DataType::Duration(_)
13661404
| DataType::FixedSizeBinary(_)
13671405
| DataType::FixedSizeList(_, _)
@@ -1613,7 +1651,34 @@ impl ScalarValue {
16131651
ScalarValue::Date64(e) => {
16141652
build_array_from_option!(Date64, Date64Array, e, size)
16151653
}
1616-
ScalarValue::Time64(e) => {
1654+
ScalarValue::Time32Second(e) => {
1655+
build_array_from_option!(
1656+
Time32,
1657+
TimeUnit::Second,
1658+
Time32SecondArray,
1659+
e,
1660+
size
1661+
)
1662+
}
1663+
ScalarValue::Time32Millisecond(e) => {
1664+
build_array_from_option!(
1665+
Time32,
1666+
TimeUnit::Millisecond,
1667+
Time32MillisecondArray,
1668+
e,
1669+
size
1670+
)
1671+
}
1672+
ScalarValue::Time64Microsecond(e) => {
1673+
build_array_from_option!(
1674+
Time64,
1675+
TimeUnit::Microsecond,
1676+
Time64MicrosecondArray,
1677+
e,
1678+
size
1679+
)
1680+
}
1681+
ScalarValue::Time64Nanosecond(e) => {
16171682
build_array_from_option!(
16181683
Time64,
16191684
TimeUnit::Nanosecond,
@@ -1756,8 +1821,17 @@ impl ScalarValue {
17561821
DataType::Date64 => {
17571822
typed_cast!(array, index, Date64Array, Date64)
17581823
}
1824+
DataType::Time32(TimeUnit::Second) => {
1825+
typed_cast!(array, index, Time32SecondArray, Time32Second)
1826+
}
1827+
DataType::Time32(TimeUnit::Millisecond) => {
1828+
typed_cast!(array, index, Time32MillisecondArray, Time32Millisecond)
1829+
}
1830+
DataType::Time64(TimeUnit::Microsecond) => {
1831+
typed_cast!(array, index, Time64MicrosecondArray, Time64Microsecond)
1832+
}
17591833
DataType::Time64(TimeUnit::Nanosecond) => {
1760-
typed_cast!(array, index, Time64NanosecondArray, Time64)
1834+
typed_cast!(array, index, Time64NanosecondArray, Time64Nanosecond)
17611835
}
17621836
DataType::Timestamp(TimeUnit::Second, tz_opt) => {
17631837
typed_cast_tz!(
@@ -1951,7 +2025,16 @@ impl ScalarValue {
19512025
ScalarValue::Date64(val) => {
19522026
eq_array_primitive!(array, index, Date64Array, val)
19532027
}
1954-
ScalarValue::Time64(val) => {
2028+
ScalarValue::Time32Second(val) => {
2029+
eq_array_primitive!(array, index, Time32SecondArray, val)
2030+
}
2031+
ScalarValue::Time32Millisecond(val) => {
2032+
eq_array_primitive!(array, index, Time32MillisecondArray, val)
2033+
}
2034+
ScalarValue::Time64Microsecond(val) => {
2035+
eq_array_primitive!(array, index, Time64MicrosecondArray, val)
2036+
}
2037+
ScalarValue::Time64Nanosecond(val) => {
19552038
eq_array_primitive!(array, index, Time64NanosecondArray, val)
19562039
}
19572040
ScalarValue::TimestampSecond(val, _) => {
@@ -2090,7 +2173,9 @@ impl TryFrom<ScalarValue> for i32 {
20902173
fn try_from(value: ScalarValue) -> Result<Self> {
20912174
match value {
20922175
ScalarValue::Int32(Some(inner_value))
2093-
| ScalarValue::Date32(Some(inner_value)) => Ok(inner_value),
2176+
| ScalarValue::Date32(Some(inner_value))
2177+
| ScalarValue::Time32Second(Some(inner_value))
2178+
| ScalarValue::Time32Millisecond(Some(inner_value)) => Ok(inner_value),
20942179
_ => Err(DataFusionError::Internal(format!(
20952180
"Cannot convert {:?} to {}",
20962181
value,
@@ -2108,7 +2193,8 @@ impl TryFrom<ScalarValue> for i64 {
21082193
match value {
21092194
ScalarValue::Int64(Some(inner_value))
21102195
| ScalarValue::Date64(Some(inner_value))
2111-
| ScalarValue::Time64(Some(inner_value))
2196+
| ScalarValue::Time64Microsecond(Some(inner_value))
2197+
| ScalarValue::Time64Nanosecond(Some(inner_value))
21122198
| ScalarValue::TimestampNanosecond(Some(inner_value), _)
21132199
| ScalarValue::TimestampMicrosecond(Some(inner_value), _)
21142200
| ScalarValue::TimestampMillisecond(Some(inner_value), _)
@@ -2170,7 +2256,14 @@ impl TryFrom<&DataType> for ScalarValue {
21702256
DataType::LargeUtf8 => ScalarValue::LargeUtf8(None),
21712257
DataType::Date32 => ScalarValue::Date32(None),
21722258
DataType::Date64 => ScalarValue::Date64(None),
2173-
DataType::Time64(TimeUnit::Nanosecond) => ScalarValue::Time64(None),
2259+
DataType::Time32(TimeUnit::Second) => ScalarValue::Time32Second(None),
2260+
DataType::Time32(TimeUnit::Millisecond) => {
2261+
ScalarValue::Time32Millisecond(None)
2262+
}
2263+
DataType::Time64(TimeUnit::Microsecond) => {
2264+
ScalarValue::Time64Microsecond(None)
2265+
}
2266+
DataType::Time64(TimeUnit::Nanosecond) => ScalarValue::Time64Nanosecond(None),
21742267
DataType::Timestamp(TimeUnit::Second, tz_opt) => {
21752268
ScalarValue::TimestampSecond(None, tz_opt.clone())
21762269
}
@@ -2309,7 +2402,10 @@ impl fmt::Display for ScalarValue {
23092402
},
23102403
ScalarValue::Date32(e) => format_option!(f, e)?,
23112404
ScalarValue::Date64(e) => format_option!(f, e)?,
2312-
ScalarValue::Time64(e) => format_option!(f, e)?,
2405+
ScalarValue::Time32Second(e) => format_option!(f, e)?,
2406+
ScalarValue::Time32Millisecond(e) => format_option!(f, e)?,
2407+
ScalarValue::Time64Microsecond(e) => format_option!(f, e)?,
2408+
ScalarValue::Time64Nanosecond(e) => format_option!(f, e)?,
23132409
ScalarValue::IntervalDayTime(e) => format_option!(f, e)?,
23142410
ScalarValue::IntervalYearMonth(e) => format_option!(f, e)?,
23152411
ScalarValue::IntervalMonthDayNano(e) => format_option!(f, e)?,
@@ -2370,7 +2466,16 @@ impl fmt::Debug for ScalarValue {
23702466
ScalarValue::List(_, _) => write!(f, "List([{}])", self),
23712467
ScalarValue::Date32(_) => write!(f, "Date32(\"{}\")", self),
23722468
ScalarValue::Date64(_) => write!(f, "Date64(\"{}\")", self),
2373-
ScalarValue::Time64(_) => write!(f, "Time64(\"{}\")", self),
2469+
ScalarValue::Time32Second(_) => write!(f, "Time32Second(\"{}\")", self),
2470+
ScalarValue::Time32Millisecond(_) => {
2471+
write!(f, "Time32Millisecond(\"{}\")", self)
2472+
}
2473+
ScalarValue::Time64Microsecond(_) => {
2474+
write!(f, "Time64Microsecond(\"{}\")", self)
2475+
}
2476+
ScalarValue::Time64Nanosecond(_) => {
2477+
write!(f, "Time64Nanosecond(\"{}\")", self)
2478+
}
23742479
ScalarValue::IntervalDayTime(_) => {
23752480
write!(f, "IntervalDayTime(\"{}\")", self)
23762481
}
@@ -2999,7 +3104,10 @@ mod tests {
29993104
make_binary_test_case!(str_vals, LargeBinaryArray, LargeBinary),
30003105
make_test_case!(i32_vals, Date32Array, Date32),
30013106
make_test_case!(i64_vals, Date64Array, Date64),
3002-
make_test_case!(i64_vals, Time64NanosecondArray, Time64),
3107+
make_test_case!(i32_vals, Time32SecondArray, Time32Second),
3108+
make_test_case!(i32_vals, Time32MillisecondArray, Time32Millisecond),
3109+
make_test_case!(i64_vals, Time64MicrosecondArray, Time64Microsecond),
3110+
make_test_case!(i64_vals, Time64NanosecondArray, Time64Nanosecond),
30033111
make_test_case!(i64_vals, TimestampSecondArray, TimestampSecond, None),
30043112
make_test_case!(
30053113
i64_vals,
@@ -3810,4 +3918,4 @@ mod tests {
38103918
]
38113919
);
38123920
}
3813-
}
3921+
}

‎datafusion/expr/src/type_coercion/binary.rs‎

Lines changed: 54 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,9 @@
2020
use crate::type_coercion::is_numeric;
2121
use crate::Operator;
2222
use arrow::compute::can_cast_types;
23-
use arrow::datatypes::{DataType, DECIMAL128_MAX_PRECISION, DECIMAL128_MAX_SCALE};
23+
use arrow::datatypes::{
24+
DataType, TimeUnit, DECIMAL128_MAX_PRECISION, DECIMAL128_MAX_SCALE,
25+
};
2426
use datafusion_common::DataFusionError;
2527
use datafusion_common::Result;
2628

@@ -515,14 +517,29 @@ fn like_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataType> {
515517
/// casted to for the purpose of a date computation
516518
fn temporal_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataType> {
517519
use arrow::datatypes::DataType::*;
518-
use arrow::datatypes::TimeUnit;
519520
match (lhs_type, rhs_type) {
520521
(Date64, Date32) => Some(Date64),
521522
(Date32, Date64) => Some(Date64),
522523
(Utf8, Date32) => Some(Date32),
523524
(Date32, Utf8) => Some(Date32),
524525
(Utf8, Date64) => Some(Date64),
525526
(Date64, Utf8) => Some(Date64),
527+
(Utf8, Time32(unit)) => match check_time_unit(Time32(unit.clone())) {
528+
None => None,
529+
Some(unit) => Some(Time32(unit)),
530+
},
531+
(Time32(unit), Utf8) => match check_time_unit(Time32(unit.clone())) {
532+
None => None,
533+
Some(unit) => Some(Time32(unit)),
534+
},
535+
(Utf8, Time64(unit)) => match check_time_unit(Time64(unit.clone())) {
536+
None => None,
537+
Some(unit) => Some(Time64(unit)),
538+
},
539+
(Time64(unit), Utf8) => match check_time_unit(Time64(unit.clone())) {
540+
None => None,
541+
Some(unit) => Some(Time64(unit)),
542+
},
526543
(Timestamp(lhs_unit, lhs_tz), Timestamp(rhs_unit, rhs_tz)) => {
527544
let tz = match (lhs_tz, rhs_tz) {
528545
// can't cast across timezones
@@ -623,6 +640,17 @@ fn null_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataType> {
623640
}
624641
}
625642

643+
fn check_time_unit(datatype: DataType) -> Option<TimeUnit> {
644+
use arrow::datatypes::DataType::*;
645+
match datatype {
646+
Time32(TimeUnit::Second) => Some(TimeUnit::Second),
647+
Time32(TimeUnit::Millisecond) => Some(TimeUnit::Millisecond),
648+
Time64(TimeUnit::Microsecond) => Some(TimeUnit::Microsecond),
649+
Time64(TimeUnit::Nanosecond) => Some(TimeUnit::Nanosecond),
650+
_ => None,
651+
}
652+
}
653+
626654
#[cfg(test)]
627655
mod tests {
628656
use super::*;
@@ -824,6 +852,30 @@ mod tests {
824852
Operator::Lt,
825853
DataType::Date64
826854
);
855+
test_coercion_binary_rule!(
856+
DataType::Utf8,
857+
DataType::Time32(TimeUnit::Second),
858+
Operator::Eq,
859+
DataType::Time32(TimeUnit::Second)
860+
);
861+
test_coercion_binary_rule!(
862+
DataType::Utf8,
863+
DataType::Time32(TimeUnit::Millisecond),
864+
Operator::Eq,
865+
DataType::Time32(TimeUnit::Millisecond)
866+
);
867+
test_coercion_binary_rule!(
868+
DataType::Utf8,
869+
DataType::Time64(TimeUnit::Microsecond),
870+
Operator::Eq,
871+
DataType::Time64(TimeUnit::Microsecond)
872+
);
873+
test_coercion_binary_rule!(
874+
DataType::Utf8,
875+
DataType::Time64(TimeUnit::Nanosecond),
876+
Operator::Eq,
877+
DataType::Time64(TimeUnit::Nanosecond)
878+
);
827879
test_coercion_binary_rule!(
828880
DataType::Utf8,
829881
DataType::Utf8,

0 commit comments

Comments
 (0)