Skip to content

Commit d6e73bd

Browse files
committed
refactor(datetime-parts): remove the cached-bounds split fast path
The default writer computes exact bounds, but CanonicalValidity rebuilds the arrays and drops those bounds before temporal compression. The writer probe supplied no cached bounds for either seconds or milliseconds, with or without nulls. Always check the computed days and remove the cache lookup, duplicate loop variants, and cache-only test and benchmark parameters. Keep the non-null day bounds check when widening is needed. Signed-off-by: Connor Tsui <[email protected]>
1 parent df5feb4 commit d6e73bd

3 files changed

Lines changed: 26 additions & 125 deletions

File tree

‎encodings/datetime-parts/benches/split_temporal.rs‎

Lines changed: 12 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,6 @@ use rand::SeedableRng as _;
1212
use rand::rngs::StdRng;
1313
use vortex_array::IntoArray;
1414
use vortex_array::VortexSessionExecute;
15-
use vortex_array::aggregate_fn::NumericalAggregateOpts;
16-
use vortex_array::aggregate_fn::fns::min_max::min_max;
1715
use vortex_array::arrays::PrimitiveArray;
1816
use vortex_array::arrays::TemporalArray;
1917
use vortex_array::extension::datetime::TimeUnit;
@@ -34,29 +32,16 @@ fn main() {
3432
static SESSION: LazyLock<VortexSession> =
3533
LazyLock::new(|| VortexSession::empty().with::<KernelSession>());
3634

37-
fn cache_bounds(array: &TemporalArray) {
38-
min_max(
39-
&array.clone().into_array(),
40-
&mut SESSION.create_execution_ctx(),
41-
NumericalAggregateOpts::default(),
42-
)
43-
.unwrap();
44-
}
45-
46-
const BENCH_ARGS: &[(usize, TimeUnit, bool)] = &[
47-
(65_536, TimeUnit::Seconds, false),
48-
(65_536, TimeUnit::Seconds, true),
49-
(65_536, TimeUnit::Milliseconds, false),
50-
(65_536, TimeUnit::Milliseconds, true),
51-
(65_536, TimeUnit::Microseconds, false),
52-
(65_536, TimeUnit::Microseconds, true),
53-
(65_536, TimeUnit::Nanoseconds, false),
54-
(65_536, TimeUnit::Nanoseconds, true),
35+
const BENCH_ARGS: &[(usize, TimeUnit)] = &[
36+
(65_536, TimeUnit::Seconds),
37+
(65_536, TimeUnit::Milliseconds),
38+
(65_536, TimeUnit::Microseconds),
39+
(65_536, TimeUnit::Nanoseconds),
5540
];
5641

5742
#[divan::bench(args = BENCH_ARGS)]
58-
fn split(bencher: Bencher, args: (usize, TimeUnit, bool)) {
59-
let (n, unit, cached_bounds) = args;
43+
fn split(bencher: Bencher, args: (usize, TimeUnit)) {
44+
let (n, unit) = args;
6045
let divisor: i64 = match unit {
6146
TimeUnit::Seconds => 1,
6247
TimeUnit::Milliseconds => 1_000,
@@ -74,25 +59,19 @@ fn split(bencher: Bencher, args: (usize, TimeUnit, bool)) {
7459
Some("UTC".into()),
7560
);
7661

77-
if cached_bounds {
78-
cache_bounds(&array);
79-
}
80-
8162
bencher
8263
.with_inputs(|| (array.clone(), SESSION.create_execution_ctx()))
8364
.bench_values(|(array, mut ctx)| split_temporal(array, &mut ctx).unwrap())
8465
}
8566

86-
const EXTREME_ARGS: &[(usize, TimeUnit, bool)] = &[
87-
(65_536, TimeUnit::Seconds, false),
88-
(65_536, TimeUnit::Seconds, true),
89-
(65_536, TimeUnit::Milliseconds, false),
90-
(65_536, TimeUnit::Milliseconds, true),
67+
const EXTREME_ARGS: &[(usize, TimeUnit)] = &[
68+
(65_536, TimeUnit::Seconds),
69+
(65_536, TimeUnit::Milliseconds),
9170
];
9271

9372
#[divan::bench(args = EXTREME_ARGS)]
94-
fn split_extreme(bencher: Bencher, args: (usize, TimeUnit, bool)) {
95-
let (n, unit, cached_bounds) = args;
73+
fn split_extreme(bencher: Bencher, args: (usize, TimeUnit)) {
74+
let (n, unit) = args;
9675
let timestamps = Buffer::from_iter((0..i64::try_from(n).unwrap()).map(|i| match i % 3 {
9776
0 => i64::MAX - i,
9877
1 => i64::MIN + i,
@@ -104,10 +83,6 @@ fn split_extreme(bencher: Bencher, args: (usize, TimeUnit, bool)) {
10483
Some("UTC".into()),
10584
);
10685

107-
if cached_bounds {
108-
cache_bounds(&array);
109-
}
110-
11186
bencher
11287
.with_inputs(|| (array.clone(), SESSION.create_execution_ctx()))
11388
.bench_values(|(array, mut ctx)| split_temporal(array, &mut ctx).unwrap())

‎encodings/datetime-parts/src/compress.rs‎

Lines changed: 14 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,9 @@
44
//! Splits temporal arrays into components for independent compression.
55
//!
66
//! The split keeps days in `i32` when possible and widens only that component to `i64` for extreme
7-
//! second or millisecond timestamps. Null slots do not restrict the range of an array.
7+
//! second or millisecond timestamps. The split checks the computed days as it writes the narrow
8+
//! buffer. If any day exceeds `i32`, it checks the non-null days before keeping a wide buffer.
9+
//! Null slots do not restrict the range of an array.
810
911
use std::mem::MaybeUninit;
1012

@@ -16,16 +18,12 @@ use vortex_array::aggregate_fn::NumericalAggregateOpts;
1618
use vortex_array::aggregate_fn::fns::min_max::MinMaxResult;
1719
use vortex_array::aggregate_fn::fns::min_max::min_max;
1820
use vortex_array::arrays::ConstantArray;
19-
use vortex_array::arrays::ExtensionArray;
2021
use vortex_array::arrays::PrimitiveArray;
2122
use vortex_array::arrays::TemporalArray;
2223
use vortex_array::builtins::ArrayBuiltins;
2324
use vortex_array::dtype::DType;
2425
use vortex_array::dtype::PType;
25-
use vortex_array::expr::stats::Stat;
26-
use vortex_array::expr::stats::StatsProvider;
2726
use vortex_array::extension::datetime::TimeUnit;
28-
use vortex_array::scalar::Scalar;
2927
use vortex_buffer::Buffer;
3028
use vortex_buffer::BufferMut;
3129
use vortex_error::VortexResult;
@@ -59,8 +57,6 @@ pub fn split_temporal(array: TemporalArray, ctx: &mut ExecutionCtx) -> VortexRes
5957
vortex_bail!("Cannot handle day-level data");
6058
}
6159

62-
let check_days = !cached_bounds(&array).is_some_and(|bounds| bounds_fit(&bounds, time_unit));
63-
6460
let temporal_values = array
6561
.temporal_values()
6662
.clone()
@@ -88,11 +84,7 @@ pub fn split_temporal(array: TemporalArray, ctx: &mut ExecutionCtx) -> VortexRes
8884
let seconds_ptr = &mut seconds.spare_capacity_mut()[..length];
8985

9086
if matches!(time_unit, TimeUnit::Seconds) {
91-
let all_fit = if check_days {
92-
split_slice_seconds::<true>(days_ptr, seconds_ptr, timestamps)
93-
} else {
94-
split_slice_seconds::<false>(days_ptr, seconds_ptr, timestamps)
95-
};
87+
let all_fit = split_slice_seconds(days_ptr, seconds_ptr, timestamps);
9688

9789
// SAFETY: `split_slice_seconds` initialized all `length` elements of both buffers.
9890
unsafe {
@@ -118,16 +110,13 @@ pub fn split_temporal(array: TemporalArray, ctx: &mut ExecutionCtx) -> VortexRes
118110

119111
let all_fit = match time_unit {
120112
TimeUnit::Nanoseconds => {
121-
split_slice::<1_000_000_000, false>(days_ptr, seconds_ptr, subseconds_ptr, timestamps)
113+
split_slice::<1_000_000_000>(days_ptr, seconds_ptr, subseconds_ptr, timestamps)
122114
}
123115
TimeUnit::Microseconds => {
124-
split_slice::<1_000_000, false>(days_ptr, seconds_ptr, subseconds_ptr, timestamps)
125-
}
126-
TimeUnit::Milliseconds if check_days => {
127-
split_slice::<1_000, true>(days_ptr, seconds_ptr, subseconds_ptr, timestamps)
116+
split_slice::<1_000_000>(days_ptr, seconds_ptr, subseconds_ptr, timestamps)
128117
}
129118
TimeUnit::Milliseconds => {
130-
split_slice::<1_000, false>(days_ptr, seconds_ptr, subseconds_ptr, timestamps)
119+
split_slice::<1_000>(days_ptr, seconds_ptr, subseconds_ptr, timestamps)
131120
}
132121
_ => unreachable!("Handled before"),
133122
};
@@ -186,33 +175,6 @@ fn widen_days_if_needed(
186175
Ok(if fits { days.into_array() } else { wide_days })
187176
}
188177

189-
/// Exact min and max cached on the array, which the layout writer computes before compressing.
190-
fn cached_bounds(array: &TemporalArray) -> Option<[Scalar; 2]> {
191-
let ext = ExtensionArray::from(array);
192-
let stats = ext.statistics();
193-
if let (Some(min), Some(max)) = (
194-
stats.get(Stat::Min).as_exact(),
195-
stats.get(Stat::Max).as_exact(),
196-
) {
197-
return Some([min, max].map(|bound| bound.as_extension().to_storage_scalar()));
198-
}
199-
200-
let stats = array.temporal_values().statistics();
201-
let min = stats.get(Stat::Min).as_exact()?;
202-
let max = stats.get(Stat::Max).as_exact()?;
203-
204-
Some([min, max])
205-
}
206-
207-
fn bounds_fit(bounds: &[Scalar; 2], time_unit: TimeUnit) -> bool {
208-
bounds.iter().all(|bound| {
209-
bound
210-
.as_primitive()
211-
.as_::<i64>()
212-
.is_some_and(|ts| timestamp::days_fit_i32(ts, time_unit))
213-
})
214-
}
215-
216178
/// Adds 2^31 to the days and converts to unsigned, placing in-range days below 2^32.
217179
///
218180
/// An out-of-range value sets a bit above bit 31. OR-ing the offsets preserves those bits, so the
@@ -224,9 +186,9 @@ fn day_offset(days: i64) -> u64 {
224186

225187
/// Initializes all three output slices and reports whether the days fit `i32`.
226188
///
227-
/// The slices must have the same length. With `CHECK_DAYS` disabled, always returns `true`.
189+
/// The slices must have the same length.
228190
#[inline]
229-
fn split_slice<const DIVISOR: i64, const CHECK_DAYS: bool>(
191+
fn split_slice<const DIVISOR: i64>(
230192
days: &mut [MaybeUninit<i32>],
231193
seconds: &mut [MaybeUninit<i32>],
232194
subseconds: &mut [MaybeUninit<i32>],
@@ -249,9 +211,7 @@ fn split_slice<const DIVISOR: i64, const CHECK_DAYS: bool>(
249211
let mut subsecond_buf = [0i32; 4];
250212
for k in 0..4 {
251213
let parts = timestamp::split_with_divisor::<DIVISOR>(timestamp[k]);
252-
if CHECK_DAYS {
253-
day_offsets |= day_offset(parts.days);
254-
}
214+
day_offsets |= day_offset(parts.days);
255215
day_buf[k] = parts.days.as_();
256216
second_buf[k] = parts.seconds;
257217
subsecond_buf[k] = parts.subseconds;
@@ -270,9 +230,7 @@ fn split_slice<const DIVISOR: i64, const CHECK_DAYS: bool>(
270230
.zip(subseconds_rem);
271231
for (((&ts, day), second), subsecond) in remainder {
272232
let parts = timestamp::split_with_divisor::<DIVISOR>(ts);
273-
if CHECK_DAYS {
274-
day_offsets |= day_offset(parts.days);
275-
}
233+
day_offsets |= day_offset(parts.days);
276234
day.write(parts.days.as_());
277235
second.write(parts.seconds);
278236
subsecond.write(parts.subseconds);
@@ -283,20 +241,17 @@ fn split_slice<const DIVISOR: i64, const CHECK_DAYS: bool>(
283241

284242
/// Initializes the days and seconds slices for second timestamps.
285243
///
286-
/// The slices must have the same length. With `CHECK_DAYS` enabled, reports whether all days fit
287-
/// `i32`. Otherwise returns `true`.
244+
/// The slices must have the same length. Returns whether all days fit `i32`.
288245
#[inline]
289-
fn split_slice_seconds<const CHECK_DAYS: bool>(
246+
fn split_slice_seconds(
290247
days: &mut [MaybeUninit<i32>],
291248
seconds: &mut [MaybeUninit<i32>],
292249
timestamps: &[i64],
293250
) -> bool {
294251
let mut day_offsets = 0u64;
295252
for ((day, second), ts) in days.iter_mut().zip(seconds).zip(timestamps) {
296253
let parts = timestamp::split_with_divisor::<1>(*ts);
297-
if CHECK_DAYS {
298-
day_offsets |= day_offset(parts.days);
299-
}
254+
day_offsets |= day_offset(parts.days);
300255
day.write(parts.days.as_());
301256
second.write(parts.seconds);
302257
}
@@ -313,8 +268,6 @@ mod tests {
313268
use vortex_array::ArrayContext;
314269
use vortex_array::IntoArray;
315270
use vortex_array::VortexSessionExecute;
316-
use vortex_array::aggregate_fn::NumericalAggregateOpts;
317-
use vortex_array::aggregate_fn::fns::min_max::min_max;
318271
use vortex_array::arrays::PrimitiveArray;
319272
use vortex_array::arrays::TemporalArray;
320273
use vortex_array::assert_arrays_eq;
@@ -330,7 +283,6 @@ mod tests {
330283
use vortex_session::VortexSession;
331284
use vortex_session::registry::ReadContext;
332285

333-
use super::cached_bounds;
334286
use crate::DateTimeParts;
335287
use crate::TemporalParts;
336288
use crate::array::DateTimePartsArraySlotsExt;
@@ -418,7 +370,6 @@ mod tests {
418370
fn test_extreme_timestamps_roundtrip(
419371
#[case] unit: TimeUnit,
420372
#[case] days_ptype: PType,
421-
#[values(false, true)] cache_bounds: bool,
422373
) -> VortexResult<()> {
423374
let mut ctx = SESSION.create_execution_ctx();
424375
let temporal = TemporalArray::new_timestamp(
@@ -433,10 +384,6 @@ mod tests {
433384
Some("UTC".into()),
434385
);
435386
let expected = temporal.clone().into_array();
436-
if cache_bounds {
437-
min_max(&expected, &mut ctx, NumericalAggregateOpts::default())?;
438-
}
439-
assert_eq!(cached_bounds(&temporal).is_some(), cache_bounds);
440387

441388
let parts = DateTimeParts::try_from_temporal(temporal, &mut ctx)?;
442389
assert_eq!(parts.days().dtype().as_ptype(), days_ptype);
@@ -479,7 +426,6 @@ mod tests {
479426
fn test_days_at_i32_bounds(
480427
#[case] unit: TimeUnit,
481428
#[case] ticks_per_second: i64,
482-
#[values(false, true)] cache_bounds: bool,
483429
) -> VortexResult<()> {
484430
let mut ctx = SESSION.create_execution_ctx();
485431
let ticks_per_day = SECONDS_PER_DAY * ticks_per_second;
@@ -490,13 +436,6 @@ mod tests {
490436
TemporalArray::new_timestamp(PrimitiveArray::from_iter(values).into_array(), unit, None)
491437
};
492438
let fits = temporal(vec![max, 0, min, max, min, 0, max, min, max]);
493-
if cache_bounds {
494-
min_max(
495-
&fits.clone().into_array(),
496-
&mut ctx,
497-
NumericalAggregateOpts::default(),
498-
)?;
499-
}
500439
let parts = DateTimeParts::try_from_temporal(fits.clone(), &mut ctx)?;
501440
assert_eq!(parts.days().dtype().as_ptype(), PType::I32);
502441
assert_arrays_eq!(parts, fits, &mut ctx);

‎encodings/datetime-parts/src/timestamp.rs‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -44,19 +44,6 @@ pub fn split(timestamp: i64, time_unit: TimeUnit) -> VortexResult<TimestampParts
4444
})
4545
}
4646

47-
/// Returns whether the timestamp's days can use the array splitter's `i32` buffer.
48-
pub(crate) fn days_fit_i32(timestamp: i64, time_unit: TimeUnit) -> bool {
49-
let ticks_per_second = match time_unit {
50-
TimeUnit::Nanoseconds => 1_000_000_000,
51-
TimeUnit::Microseconds => 1_000_000,
52-
TimeUnit::Milliseconds => 1_000,
53-
TimeUnit::Seconds => 1,
54-
TimeUnit::Days => return false,
55-
};
56-
57-
i32::try_from(timestamp / (SECONDS_PER_DAY * ticks_per_second)).is_ok()
58-
}
59-
6047
/// Splits a Unix timestamp into parts using a constant divisor.
6148
#[inline]
6249
pub(crate) fn split_with_divisor<const DIVISOR: i64>(timestamp: i64) -> TimestampParts {

0 commit comments

Comments
 (0)