Skip to content

Commit 61ea323

Browse files
committed
don't validate Timestamp with jiff
Signed-off-by: Mikhail Kot <[email protected]>
1 parent d443eab commit 61ea323

5 files changed

Lines changed: 175 additions & 46 deletions

File tree

‎vortex-array/src/aggregate_fn/fns/min_max/extension.rs‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ use crate::ExecutionCtx;
1010
use crate::aggregate_fn::NumericalAggregateOpts;
1111
use crate::arrays::ExtensionArray;
1212
use crate::arrays::extension::ExtensionArrayExt;
13+
use crate::dtype::DType;
1314
use crate::dtype::Nullability;
1415
use crate::scalar::Scalar;
1516

@@ -19,15 +20,23 @@ pub(super) fn accumulate_extension(
1920
ctx: &mut ExecutionCtx,
2021
) -> VortexResult<()> {
2122
let non_nullable_ext_dtype = array.ext_dtype().with_nullability(Nullability::NonNullable);
22-
let local = min_max(
23+
let Some(MinMaxResult { min, max }) = min_max(
2324
array.storage_array(),
2425
ctx,
2526
NumericalAggregateOpts::default(),
2627
)?
27-
.map(|MinMaxResult { min, max }| MinMaxResult {
28-
min: Scalar::extension_ref(non_nullable_ext_dtype.clone(), min),
29-
max: Scalar::extension_ref(non_nullable_ext_dtype, max),
30-
});
28+
else {
29+
return Ok(());
30+
};
31+
32+
let ext_dtype = DType::Extension(non_nullable_ext_dtype);
33+
let local = match (
34+
Scalar::try_new(ext_dtype.clone(), min.into_value()),
35+
Scalar::try_new(ext_dtype, max.into_value()),
36+
) {
37+
(Ok(min), Ok(max)) => Some(MinMaxResult { min, max }),
38+
_ => None,
39+
};
3140
partial.merge(local);
3241
Ok(())
3342
}

‎vortex-array/src/aggregate_fn/fns/min_max/mod.rs‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -465,6 +465,7 @@ mod tests {
465465
use crate::arrays::ChunkedArray;
466466
use crate::arrays::ConstantArray;
467467
use crate::arrays::DecimalArray;
468+
use crate::arrays::ExtensionArray;
468469
use crate::arrays::FixedSizeListArray;
469470
use crate::arrays::ListArray;
470471
use crate::arrays::NullArray;
@@ -476,7 +477,10 @@ mod tests {
476477
use crate::dtype::PType;
477478
use crate::expr::stats::Precision;
478479
use crate::expr::stats::Stat;
480+
use crate::extension::datetime::TimeUnit;
481+
use crate::extension::datetime::Timestamp;
479482
use crate::scalar::DecimalValue;
483+
use crate::scalar::PValue;
480484
use crate::scalar::Scalar;
481485
use crate::scalar::ScalarValue;
482486
use crate::validity::Validity;
@@ -1092,4 +1096,30 @@ mod tests {
10921096
);
10931097
Ok(())
10941098
}
1099+
1100+
#[test]
1101+
fn test_timestamp_min_max_beyond_jiff_range() -> VortexResult<()> {
1102+
let mut ctx = SESSION.create_execution_ctx();
1103+
let ext_dtype = Timestamp::new(TimeUnit::Microseconds, Nullability::NonNullable).erased();
1104+
// 9999-12-30T22:00:00Z is jiff's maximum, 9999-12-31T00:00:00Z is past it.
1105+
let in_range = 253_402_207_200_000_000i64;
1106+
let out_of_range = 253_402_214_400_000_000i64;
1107+
let storage = buffer![out_of_range, in_range].into_array();
1108+
let array = ExtensionArray::try_new(ext_dtype.clone(), storage)?.into_array();
1109+
1110+
let result = min_max(&array, &mut ctx, NumericalAggregateOpts::default())?
1111+
.vortex_expect("should have result");
1112+
let dtype = DType::Extension(ext_dtype);
1113+
let expected_min = Scalar::try_new(
1114+
dtype.clone(),
1115+
Some(ScalarValue::Primitive(PValue::I64(in_range))),
1116+
)?;
1117+
let expected_max = Scalar::try_new(
1118+
dtype,
1119+
Some(ScalarValue::Primitive(PValue::I64(out_of_range))),
1120+
)?;
1121+
assert_eq!(result.min, expected_min);
1122+
assert_eq!(result.max, expected_max);
1123+
Ok(())
1124+
}
10951125
}

‎vortex-array/src/arrays/extension/vtable/operations.rs‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ use crate::array::ArrayView;
88
use crate::array::OperationsVTable;
99
use crate::arrays::Extension;
1010
use crate::arrays::extension::ExtensionArrayExt;
11+
use crate::dtype::DType;
1112
use crate::scalar::Scalar;
1213

1314
impl OperationsVTable<Extension> for Extension {
@@ -16,9 +17,10 @@ impl OperationsVTable<Extension> for Extension {
1617
index: usize,
1718
ctx: &mut ExecutionCtx,
1819
) -> VortexResult<Scalar> {
19-
Ok(Scalar::extension_ref(
20-
array.ext_dtype().clone(),
21-
array.storage_array().execute_scalar(index, ctx)?,
22-
))
20+
let storage_scalar = array.storage_array().execute_scalar(index, ctx)?;
21+
Scalar::try_new(
22+
DType::Extension(array.ext_dtype().clone()),
23+
storage_scalar.into_value(),
24+
)
2325
}
2426
}

‎vortex-array/src/extension/datetime/timestamp.rs‎

Lines changed: 67 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -91,20 +91,44 @@ pub enum TimestampValue<'a> {
9191

9292
impl fmt::Display for TimestampValue<'_> {
9393
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
94-
let (span, tz) = match self {
95-
TimestampValue::Seconds(v, tz) => (Span::new().seconds(*v), *tz),
96-
TimestampValue::Milliseconds(v, tz) => (Span::new().milliseconds(*v), *tz),
97-
TimestampValue::Microseconds(v, tz) => (Span::new().microseconds(*v), *tz),
98-
TimestampValue::Nanoseconds(v, tz) => (Span::new().nanoseconds(*v), *tz),
94+
let (span, raw, unit, tz) = match self {
95+
TimestampValue::Seconds(v, tz) => {
96+
(Span::new().try_seconds(*v), *v, TimeUnit::Seconds, *tz)
97+
}
98+
TimestampValue::Milliseconds(v, tz) => (
99+
Span::new().try_milliseconds(*v),
100+
*v,
101+
TimeUnit::Milliseconds,
102+
*tz,
103+
),
104+
TimestampValue::Microseconds(v, tz) => (
105+
Span::new().try_microseconds(*v),
106+
*v,
107+
TimeUnit::Microseconds,
108+
*tz,
109+
),
110+
TimestampValue::Nanoseconds(v, tz) => (
111+
Span::new().try_nanoseconds(*v),
112+
*v,
113+
TimeUnit::Nanoseconds,
114+
*tz,
115+
),
116+
};
117+
118+
let Some(ts) = span
119+
.ok()
120+
.and_then(|span| jiff::Timestamp::UNIX_EPOCH.checked_add(span).ok())
121+
else {
122+
// storage values may exceed jiff's Timestamp range
123+
return write!(f, "{raw}{unit} since Unix epoch");
99124
};
100-
let ts = jiff::Timestamp::UNIX_EPOCH + span;
101125

102126
match tz {
103127
None => write!(f, "{ts}"),
104-
Some(tz) => {
105-
let adjusted_ts = ts.in_tz(tz.as_ref()).vortex_expect("unknown timezone");
106-
write!(f, "{adjusted_ts}",)
107-
}
128+
Some(tz) => match ts.in_tz(tz.as_ref()) {
129+
Ok(adjusted_ts) => write!(f, "{adjusted_ts}"),
130+
Err(_) => write!(f, "{ts}"),
131+
},
108132
}
109133
}
110134
}
@@ -226,37 +250,18 @@ impl ExtVTable for Timestamp {
226250
let ts_value = storage_value.as_primitive().cast::<i64>()?;
227251
let tz = metadata.tz.as_ref();
228252

229-
let (span, value) = match metadata.unit {
230-
TimeUnit::Nanoseconds => (
231-
Span::new().nanoseconds(ts_value),
232-
TimestampValue::Nanoseconds(ts_value, tz),
233-
),
234-
TimeUnit::Microseconds => (
235-
Span::new().microseconds(ts_value),
236-
TimestampValue::Microseconds(ts_value, tz),
237-
),
238-
TimeUnit::Milliseconds => (
239-
Span::new().milliseconds(ts_value),
240-
TimestampValue::Milliseconds(ts_value, tz),
241-
),
242-
TimeUnit::Seconds => (
243-
Span::new().seconds(ts_value),
244-
TimestampValue::Seconds(ts_value, tz),
245-
),
246-
TimeUnit::Days => vortex_bail!("Timestamp does not support Days time unit"),
247-
};
248-
249-
// Validate the storage value is within the valid range for Timestamp.
250-
let ts = jiff::Timestamp::UNIX_EPOCH
251-
.checked_add(span)
252-
.map_err(|e| vortex_err!("Invalid timestamp scalar: {}", e))?;
253-
254253
if let Some(tz) = tz {
255-
ts.in_tz(tz.as_ref())
254+
jiff::tz::TimeZone::get(tz.as_ref())
256255
.map_err(|e| vortex_err!("Invalid timezone for timestamp scalar: {}", e))?;
257256
}
258257

259-
Ok(value)
258+
Ok(match metadata.unit {
259+
TimeUnit::Nanoseconds => TimestampValue::Nanoseconds(ts_value, tz),
260+
TimeUnit::Microseconds => TimestampValue::Microseconds(ts_value, tz),
261+
TimeUnit::Milliseconds => TimestampValue::Milliseconds(ts_value, tz),
262+
TimeUnit::Seconds => TimestampValue::Seconds(ts_value, tz),
263+
TimeUnit::Days => vortex_bail!("Timestamp does not support Days time unit"),
264+
})
260265
}
261266
}
262267

@@ -282,6 +287,17 @@ mod tests {
282287
Ok(())
283288
}
284289

290+
#[test]
291+
fn validate_timestamp_scalar_beyond_jiff_range() -> VortexResult<()> {
292+
let dtype = DType::Extension(Timestamp::new(TimeUnit::Microseconds, Nullable).erased());
293+
Scalar::try_new(
294+
dtype,
295+
Some(ScalarValue::Primitive(PValue::I64(253_402_214_400_000_000))),
296+
)?;
297+
298+
Ok(())
299+
}
300+
285301
#[cfg_attr(miri, ignore)]
286302
#[test]
287303
fn reject_timestamp_with_invalid_timezone() {
@@ -321,6 +337,20 @@ mod tests {
321337
);
322338
}
323339

340+
#[cfg_attr(miri, ignore)]
341+
#[test]
342+
fn display_timestamp_scalar_beyond_jiff_range() {
343+
let dtype = DType::Extension(Timestamp::new(TimeUnit::Microseconds, Nullable).erased());
344+
let scalar = Scalar::new(
345+
dtype,
346+
Some(ScalarValue::Primitive(PValue::I64(253_402_214_400_000_000))),
347+
);
348+
assert_eq!(
349+
format!("{}", scalar.as_extension()),
350+
"253402214400000000µs since Unix epoch"
351+
);
352+
}
353+
324354
#[test]
325355
fn least_supertype_timestamp_units() {
326356
use crate::dtype::Nullability::NonNullable;
Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
#![expect(clippy::tests_outside_test_module)]
5+
6+
use std::sync::LazyLock;
7+
8+
use vortex_array::IntoArray;
9+
use vortex_array::arrays::ExtensionArray;
10+
use vortex_array::arrays::PrimitiveArray;
11+
use vortex_array::arrays::StructArray;
12+
use vortex_array::dtype::Nullability::NonNullable;
13+
use vortex_array::extension::datetime::TimeUnit;
14+
use vortex_array::extension::datetime::Timestamp;
15+
use vortex_error::VortexResult;
16+
use vortex_file::WriteOptionsSessionExt;
17+
use vortex_io::session::RuntimeSession;
18+
use vortex_layout::session::LayoutSession;
19+
use vortex_session::VortexSession;
20+
21+
mod common;
22+
23+
use common::enable_all_registered_array_encodings;
24+
25+
static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
26+
let session = vortex_array::array_session()
27+
.with::<LayoutSession>()
28+
.with::<RuntimeSession>();
29+
vortex_file::register_default_encodings(&session);
30+
enable_all_registered_array_encodings(&session);
31+
session
32+
});
33+
34+
async fn write_micros(value: i64) -> VortexResult<()> {
35+
let ext_dtype = Timestamp::new(TimeUnit::Microseconds, NonNullable).erased();
36+
let storage = PrimitiveArray::from_iter([value]).into_array();
37+
let ts = ExtensionArray::try_new(ext_dtype, storage)?.into_array();
38+
let data = StructArray::from_fields(&[("ts", ts)])?.into_array();
39+
40+
let mut bytes = Vec::new();
41+
SESSION
42+
.write_options()
43+
.write(&mut bytes, data.to_array_stream())
44+
.await?;
45+
Ok(())
46+
}
47+
48+
#[tokio::test]
49+
async fn write_timestamp_last_day_of_9999_does_not_panic() -> VortexResult<()> {
50+
// 9999-12-31T00:00:00Z is past jiff's max of 9999-12-30T22:00:00Z
51+
write_micros(253_402_214_400_000_000).await
52+
}
53+
54+
#[tokio::test]
55+
async fn write_timestamp_in_jiff_range() -> VortexResult<()> {
56+
// 9999-12-30T22:00:00Z jiff's maximum
57+
write_micros(253_402_207_200_000_000).await
58+
}

0 commit comments

Comments
 (0)