Skip to content

Commit 3a74ca5

Browse files
committed
add interval extension dtype
Signed-off-by: Matt Katz <[email protected]>
1 parent 7261620 commit 3a74ca5

20 files changed

Lines changed: 791 additions & 31 deletions

File tree

‎encodings/datetime-parts/src/compress.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,9 @@ pub struct TemporalParts {
3232
/// Splitting the components by granularity creates more small values, which enables better
3333
/// cascading compression.
3434
pub fn split_temporal(array: TemporalArray, ctx: &mut ExecutionCtx) -> VortexResult<TemporalParts> {
35-
let time_unit = array.temporal_metadata().time_unit();
35+
let Some(time_unit) = array.temporal_metadata().time_unit() else {
36+
vortex_bail!("Cannot handle data without a time unit");
37+
};
3638
if matches!(time_unit, TimeUnit::Days) {
3739
vortex_bail!("Cannot handle day-level data");
3840
}

‎vortex-array/src/arrays/datetime/mod.rs‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,16 +20,17 @@ use crate::dtype::extension::ExtDTypeRef;
2020
use crate::extension::datetime::AnyTemporal;
2121
use crate::extension::datetime::Date;
2222
use crate::extension::datetime::Duration;
23+
use crate::extension::datetime::Interval;
2324
use crate::extension::datetime::TemporalMetadata;
2425
use crate::extension::datetime::Time;
2526
use crate::extension::datetime::TimeUnit;
2627
use crate::extension::datetime::Timestamp;
2728

28-
/// An array wrapper for primitive values that have an associated temporal meaning.
29+
/// An array wrapper for values that have an associated temporal meaning.
2930
///
3031
/// This is a wrapper around ExtensionArrays containing numeric types, each of which corresponds to
31-
/// either a timestamp or julian date (both referenced to UNIX epoch), a time since midnight, OR a
32-
/// duration.
32+
/// either a timestamp or julian date (both referenced to UNIX epoch), a time since midnight, a
33+
/// duration, OR an interval, which is stored as a struct of months, days, and nanoseconds.
3334
///
3435
/// ## Arrow compatibility
3536
///
@@ -40,9 +41,11 @@ use crate::extension::datetime::Timestamp;
4041
/// * `Date32`
4142
/// * `Date64`
4243
/// * `Duration`
44+
/// * `Interval`
4345
///
44-
/// Anything that can be constructed and held in a `TemporalData` can also be zero-copy converted
45-
/// back to the relevant Arrow datatype.
46+
/// Anything that can be constructed and held in a `TemporalData` can also be converted back to the
47+
/// relevant Arrow datatype. This is zero-copy for everything except intervals, whose fields are
48+
/// interleaved into Arrow's `IntervalMonthDayNano` values.
4649
#[derive(Clone, Debug)]
4750
pub struct TemporalData {
4851
/// The underlying Vortex extension array holding all the numeric values.
@@ -141,6 +144,20 @@ impl TemporalData {
141144
),
142145
}
143146
}
147+
148+
/// Create a new `TemporalData` holding calendar intervals.
149+
///
150+
/// This is equivalent to the data described by the `Interval(MonthDayNano)` type from Arrow.
151+
///
152+
/// # Panics
153+
///
154+
/// If `array` is not a struct of non-nullable `months: i32`, `days: i32`, and
155+
/// `nanoseconds: i64` fields, the function will panic.
156+
pub fn new_interval(array: ArrayRef) -> Self {
157+
Self {
158+
ext: ExtensionArray::new(Interval::new(array.dtype().nullability()).erased(), array),
159+
}
160+
}
144161
}
145162

146163
impl TemporalData {

‎vortex-array/src/arrays/datetime/test.rs‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,22 +6,28 @@ use vortex_buffer::buffer;
66
use vortex_error::VortexExpect;
77
use vortex_error::VortexResult;
88

9+
use crate::ArrayRef;
910
use crate::EqMode;
1011
use crate::IntoArray;
1112
use crate::VortexSessionExecute;
1213
use crate::array_session;
14+
use crate::arrays::BoolArray;
1315
use crate::arrays::PrimitiveArray;
16+
use crate::arrays::StructArray;
1417
use crate::arrays::datetime::TemporalData;
1518
use crate::assert_arrays_eq;
19+
use crate::builtins::ArrayBuiltins;
1620
use crate::expr::gt;
1721
use crate::expr::lit;
1822
use crate::expr::root;
23+
use crate::extension::datetime::Interval;
1924
use crate::extension::datetime::TemporalMetadata;
2025
use crate::extension::datetime::TimeUnit;
2126
use crate::extension::datetime::Timestamp;
2227
use crate::extension::datetime::TimestampOptions;
2328
use crate::hash::ArrayEq;
2429
use crate::scalar::Scalar;
30+
use crate::scalar_fn::fns::operators::Operator;
2531
use crate::validity::Validity;
2632

2733
macro_rules! test_temporal_roundtrip {
@@ -35,7 +41,7 @@ macro_rules! test_temporal_roundtrip {
3541
PrimitiveArray::from_iter([100 as $prim]),
3642
&mut ctx
3743
);
38-
assert_eq!(temporal.temporal_metadata().time_unit(), $unit);
44+
assert_eq!(temporal.temporal_metadata().time_unit(), Some($unit));
3945
}};
4046
}
4147

@@ -170,6 +176,40 @@ test_fail_case!(
170176
TimeUnit::Days
171177
);
172178

179+
fn intervals(values: &[(i32, i32, i64)]) -> VortexResult<ArrayRef> {
180+
let storage = StructArray::try_new(
181+
Interval::FIELD_NAMES.into(),
182+
[
183+
PrimitiveArray::from_iter(values.iter().map(|v| v.0)).into_array(),
184+
PrimitiveArray::from_iter(values.iter().map(|v| v.1)).into_array(),
185+
PrimitiveArray::from_iter(values.iter().map(|v| v.2)).into_array(),
186+
],
187+
values.len(),
188+
Validity::NonNullable,
189+
)?;
190+
Ok(TemporalData::new_interval(storage.into_array()).into_array())
191+
}
192+
193+
#[test]
194+
fn test_interval_compares_field_by_field() -> VortexResult<()> {
195+
let mut ctx = array_session().create_execution_ctx();
196+
// 1 month against 100 days, 1 month against 30 days, and 1 day against 24 hours.
197+
let lhs = intervals(&[(1, 0, 0), (1, 0, 0), (0, 1, 0)])?;
198+
let rhs = intervals(&[(0, 100, 0), (0, 30, 0), (0, 0, 86_400_000_000_000)])?;
199+
200+
let gt = lhs.binary(rhs.clone(), Operator::Gt)?;
201+
assert_arrays_eq!(gt, BoolArray::from_iter([true, true, true]), &mut ctx);
202+
let eq = lhs.binary(rhs, Operator::Eq)?;
203+
assert_arrays_eq!(eq, BoolArray::from_iter([false, false, false]), &mut ctx);
204+
Ok(())
205+
}
206+
207+
#[test]
208+
#[should_panic]
209+
fn test_interval_fails_primitive() {
210+
TemporalData::new_interval(buffer![100i64].into_array());
211+
}
212+
173213
// We test Timestamp explicitly to avoid the macro getting too complex.
174214
#[test]
175215
fn test_timestamp() {

‎vortex-array/src/dtype/session.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ use crate::dtype::extension::ExtDTypePluginRef;
1616
use crate::dtype::extension::ExtVTable;
1717
use crate::extension::datetime::Date;
1818
use crate::extension::datetime::Duration;
19+
use crate::extension::datetime::Interval;
1920
use crate::extension::datetime::Time;
2021
use crate::extension::datetime::Timestamp;
2122

@@ -37,6 +38,7 @@ impl Default for DTypeSession {
3738
// Register built-in temporal extension dtypes
3839
this.register(Date);
3940
this.register(Duration);
41+
this.register(Interval);
4042
this.register(Time);
4143
this.register(Timestamp);
4244

‎vortex-array/src/extension/datetime/duration.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ mod tests {
134134
temporal,
135135
TemporalMetadata::Duration(&TimeUnit::Milliseconds)
136136
);
137-
assert_eq!(temporal.time_unit(), TimeUnit::Milliseconds);
137+
assert_eq!(temporal.time_unit(), Some(TimeUnit::Milliseconds));
138138
assert!(temporal.to_jiff(0).is_err());
139139
}
140140

0 commit comments

Comments
 (0)