Skip to content

Commit 7943b4e

Browse files
committed
refactor(datetime-parts): clarify timestamp part bounds
Document the component widths, signed remainders, and bias-and-OR days range check. Name the range-check state explicitly and constrain scalar part reads to integer types. Compare the bit check against checked integer conversion and assert the remainder bounds at both timestamp storage extremes. Signed-off-by: Connor Tsui <[email protected]>
1 parent b15c451 commit 7943b4e

4 files changed

Lines changed: 69 additions & 27 deletions

File tree

‎encodings/datetime-parts/src/compress.rs‎

Lines changed: 41 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ pub fn split_temporal(array: TemporalArray, ctx: &mut ExecutionCtx) -> VortexRes
8484
let seconds_ptr = &mut seconds.spare_capacity_mut()[..length];
8585

8686
if matches!(time_unit, TimeUnit::Seconds) {
87-
let all_fit = split_slice_seconds(days_ptr, seconds_ptr, timestamps);
87+
let all_days_fit_i32 = split_slice_seconds(days_ptr, seconds_ptr, timestamps);
8888

8989
// SAFETY: `split_slice_seconds` initialized all `length` elements of both buffers.
9090
unsafe {
@@ -97,7 +97,7 @@ pub fn split_temporal(array: TemporalArray, ctx: &mut ExecutionCtx) -> VortexRes
9797
PrimitiveArray::new(days.freeze(), temporal_values.validity()?),
9898
timestamps,
9999
time_unit,
100-
all_fit,
100+
all_days_fit_i32,
101101
ctx,
102102
)?,
103103
seconds: seconds.into_array(),
@@ -108,7 +108,7 @@ pub fn split_temporal(array: TemporalArray, ctx: &mut ExecutionCtx) -> VortexRes
108108
let mut subseconds: BufferMut<i32> = BufferMut::with_capacity(timestamps.len());
109109
let subseconds_ptr = &mut subseconds.spare_capacity_mut()[..length];
110110

111-
let all_fit = match time_unit {
111+
let all_days_fit_i32 = match time_unit {
112112
TimeUnit::Nanoseconds => {
113113
split_slice::<1_000_000_000>(days_ptr, seconds_ptr, subseconds_ptr, timestamps)
114114
}
@@ -133,7 +133,7 @@ pub fn split_temporal(array: TemporalArray, ctx: &mut ExecutionCtx) -> VortexRes
133133
PrimitiveArray::new(days.freeze(), temporal_values.validity()?),
134134
timestamps,
135135
time_unit,
136-
all_fit,
136+
all_days_fit_i32,
137137
ctx,
138138
)?,
139139
seconds: seconds.into_array(),
@@ -142,14 +142,17 @@ pub fn split_temporal(array: TemporalArray, ctx: &mut ExecutionCtx) -> VortexRes
142142
}
143143

144144
/// Keeps widened days only when a non-null day exceeds the `i32` range.
145+
///
146+
/// The split loops may truncate days while writing the candidate `i32` buffer. Keep that buffer
147+
/// only if all days fit or the non-null bounds confirm that only null slots were truncated.
145148
fn widen_days_if_needed(
146149
days: PrimitiveArray,
147150
timestamps: &[i64],
148151
time_unit: TimeUnit,
149-
all_fit: bool,
152+
all_days_fit_i32: bool,
150153
ctx: &mut ExecutionCtx,
151154
) -> VortexResult<ArrayRef> {
152-
if all_fit {
155+
if all_days_fit_i32 {
153156
return Ok(days.into_array());
154157
}
155158

@@ -177,8 +180,9 @@ fn widen_days_if_needed(
177180

178181
/// Adds 2^31 to the days and converts to unsigned, placing in-range days below 2^32.
179182
///
180-
/// An out-of-range value sets a bit above bit 31. OR-ing the offsets preserves those bits, so the
181-
/// loops can check the whole slice without a branch for each timestamp.
183+
/// Days below `i32::MIN` become large unsigned values; days above `i32::MAX` reach at least 2^32.
184+
/// Both set a bit above bit 31. The input must come from splitting an `i64` timestamp, whose days
185+
/// leave enough room for the bias addition in `i64`.
182186
#[inline]
183187
fn day_offset(days: i64) -> u64 {
184188
(days - i64::from(i32::MIN)).cast_unsigned()
@@ -194,7 +198,9 @@ fn split_slice<const DIVISOR: i64>(
194198
subseconds: &mut [MaybeUninit<i32>],
195199
timestamps: &[i64],
196200
) -> bool {
197-
let mut day_offsets = 0u64;
201+
// OR preserves any high bits from `day_offset`, so the final high bits are zero only if every
202+
// day fits i32. `widen_days_if_needed` checks this result before keeping the narrow buffer.
203+
let mut day_range_bits = 0u64;
198204

199205
// Computing chunks of 4 elements lets LLVM optimize stores into
200206
// a 16-byte vector store per chunk.
@@ -211,7 +217,7 @@ fn split_slice<const DIVISOR: i64>(
211217
let mut subsecond_buf = [0i32; 4];
212218
for k in 0..4 {
213219
let parts = timestamp::split_with_divisor::<DIVISOR>(timestamp[k]);
214-
day_offsets |= day_offset(parts.days);
220+
day_range_bits |= day_offset(parts.days);
215221
day_buf[k] = parts.days.as_();
216222
second_buf[k] = parts.seconds;
217223
subsecond_buf[k] = parts.subseconds;
@@ -230,13 +236,13 @@ fn split_slice<const DIVISOR: i64>(
230236
.zip(subseconds_rem);
231237
for (((&ts, day), second), subsecond) in remainder {
232238
let parts = timestamp::split_with_divisor::<DIVISOR>(ts);
233-
day_offsets |= day_offset(parts.days);
239+
day_range_bits |= day_offset(parts.days);
234240
day.write(parts.days.as_());
235241
second.write(parts.seconds);
236242
subsecond.write(parts.subseconds);
237243
}
238244

239-
day_offsets >> 32 == 0
245+
day_range_bits >> 32 == 0
240246
}
241247

242248
/// Initializes the days and seconds slices for second timestamps.
@@ -248,15 +254,17 @@ fn split_slice_seconds(
248254
seconds: &mut [MaybeUninit<i32>],
249255
timestamps: &[i64],
250256
) -> bool {
251-
let mut day_offsets = 0u64;
257+
// OR preserves any high bits from `day_offset`, so a later in-range day cannot hide overflow.
258+
// `widen_days_if_needed` checks this result before keeping the narrow buffer.
259+
let mut day_range_bits = 0u64;
252260
for ((day, second), ts) in days.iter_mut().zip(seconds).zip(timestamps) {
253261
let parts = timestamp::split_with_divisor::<1>(*ts);
254-
day_offsets |= day_offset(parts.days);
262+
day_range_bits |= day_offset(parts.days);
255263
day.write(parts.days.as_());
256264
second.write(parts.seconds);
257265
}
258266

259-
day_offsets >> 32 == 0
267+
day_range_bits >> 32 == 0
260268
}
261269

262270
#[cfg(test)]
@@ -283,6 +291,7 @@ mod tests {
283291
use vortex_session::VortexSession;
284292
use vortex_session::registry::ReadContext;
285293

294+
use super::day_offset;
286295
use crate::DateTimeParts;
287296
use crate::TemporalParts;
288297
use crate::array::DateTimePartsArraySlotsExt;
@@ -295,6 +304,23 @@ mod tests {
295304
session
296305
});
297306

307+
#[rstest]
308+
#[case::empty(&[])]
309+
#[case::zero(&[0])]
310+
#[case::boundaries(&[i64::from(i32::MIN), i64::from(i32::MAX)])]
311+
#[case::below_min(&[i64::from(i32::MIN) - 1])]
312+
#[case::above_max(&[i64::from(i32::MAX) + 1])]
313+
#[case::both_overflows(&[i64::from(i32::MIN) - 1, 0, i64::from(i32::MAX) + 1, 0])]
314+
fn test_day_range_check(#[case] days: &[i64]) -> VortexResult<()> {
315+
let day_range_bits = days.iter().fold(0u64, |bits, &day| bits | day_offset(day));
316+
assert_eq!(
317+
day_range_bits >> 32 == 0,
318+
days.iter().all(|&day| i32::try_from(day).is_ok())
319+
);
320+
321+
Ok(())
322+
}
323+
298324
#[rstest]
299325
#[case(Validity::NonNullable)]
300326
#[case(Validity::AllValid)]

‎encodings/datetime-parts/src/compute/rules.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,8 @@ impl ArrayParentReduceRule<DateTimeParts> for DTPComparisonPushDownRule {
140140
// the same result dtype as the original.
141141
let target_dtype = days.dtype().with_nullability(c.dtype().nullability());
142142

143-
// Narrowed days, such as u16, cannot hold every literal's days.
143+
// A valid timestamp literal's days can fall outside the compressed child's range.
144+
// Decline the rewrite so the comparison kernel can handle that bound.
144145
let Ok(days_scalar) = Scalar::from(days_value).cast(&target_dtype) else {
145146
return Ok(None);
146147
};

‎encodings/datetime-parts/src/ops.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ use vortex_array::ArrayView;
55
use vortex_array::ExecutionCtx;
66
use vortex_array::ProbeState;
77
use vortex_array::dtype::DType;
8-
use vortex_array::dtype::NativePType;
8+
use vortex_array::dtype::IntegerPType;
99
use vortex_array::extension::datetime::Timestamp;
1010
use vortex_array::scalar::Scalar;
1111
use vortex_array::vtable::OperationsVTable;
@@ -78,7 +78,7 @@ impl OperationsVTable<DateTimeParts> for DateTimeParts {
7878

7979
/// Reads one timestamp part out of `slot`, through the probe so a repeated read keeps the
8080
/// child's preparation.
81-
fn part_at<T: NativePType>(
81+
fn part_at<T: IntegerPType>(
8282
state: &mut ProbeState<'_, DateTimeParts>,
8383
slot: usize,
8484
name: &'static str,

‎encodings/datetime-parts/src/timestamp.rs‎

Lines changed: 24 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33

44
//! Splits and combines individual timestamp counts.
55
//!
6-
//! Days use `i64` so every timestamp can be represented. Seconds and subseconds fit `i32`. The
7-
//! array splitter can narrow the days component when the values fit.
6+
//! [`TimestampParts`] preserves the full storage range of a timestamp. Array compression can
7+
//! narrow the days component when the values fit.
88
99
use num_traits::AsPrimitive;
1010
use vortex_array::extension::datetime::TimeUnit;
@@ -15,12 +15,21 @@ use vortex_error::vortex_err;
1515
pub const SECONDS_PER_DAY: i64 = 86_400; // 24 * 60 * 60
1616

1717
/// Components of a Unix timestamp relative to 1970-01-01 00:00:00 UTC.
18+
///
19+
/// [`split`] divides toward zero, so seconds and subseconds are signed remainders, including for
20+
/// timestamps before the epoch.
1821
pub struct TimestampParts {
1922
/// Days since the Unix epoch.
23+
///
24+
/// Uses `i64` because splitting an `i64` timestamp in seconds or milliseconds can exceed the
25+
/// `i32` days range.
2026
pub days: i64,
21-
/// Seconds within the day, with magnitude less than [`SECONDS_PER_DAY`].
27+
/// Seconds within the day. [`split`] produces values from -86,399 to 86,399, which fit `i32`.
2228
pub seconds: i32,
2329
/// Subseconds within the second, in the units of the original timestamp.
30+
///
31+
/// [`split`] produces a magnitude less than the number of units per second. Even at nanosecond
32+
/// precision, this is less than 1,000,000,000, so `i32` suffices.
2433
pub subseconds: i32,
2534
}
2635

@@ -170,13 +179,19 @@ mod tests {
170179
}
171180

172181
#[rstest]
173-
#[case::seconds(TimeUnit::Seconds)]
174-
#[case::milliseconds(TimeUnit::Milliseconds)]
175-
#[case::microseconds(TimeUnit::Microseconds)]
176-
#[case::nanoseconds(TimeUnit::Nanoseconds)]
177-
fn test_split_storage_extremes(#[case] unit: TimeUnit) -> VortexResult<()> {
182+
#[case::seconds(TimeUnit::Seconds, 1)]
183+
#[case::milliseconds(TimeUnit::Milliseconds, 1_000)]
184+
#[case::microseconds(TimeUnit::Microseconds, 1_000_000)]
185+
#[case::nanoseconds(TimeUnit::Nanoseconds, 1_000_000_000)]
186+
fn test_split_storage_extremes(
187+
#[case] unit: TimeUnit,
188+
#[case] ticks_per_second: i64,
189+
) -> VortexResult<()> {
178190
for ts in [i64::MAX, i64::MIN] {
179-
assert_eq!(combine(split(ts, unit)?, unit)?, ts);
191+
let parts = split(ts, unit)?;
192+
assert!(i64::from(parts.seconds).abs() < SECONDS_PER_DAY);
193+
assert!(i64::from(parts.subseconds).abs() < ticks_per_second);
194+
assert_eq!(combine(parts, unit)?, ts);
180195
}
181196

182197
Ok(())

0 commit comments

Comments
 (0)