Skip to content

Commit 9ade55e

Browse files
committed
perf: keep values-only truncation for safe non-UTC SECOND/MILLISECOND dictionaries
SECOND and MILLISECOND ignore the timezone, so when they do not wrap they need key masking only for a value near the lower bound. Previously a non-UTC dictionary fell through to the unconditional non-UTC masking branch and scanned every key.
1 parent da1d1ee commit 9ade55e

1 file changed

Lines changed: 46 additions & 10 deletions

File tree

‎native/spark-expr/src/kernels/temporal.rs‎

Lines changed: 46 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -731,7 +731,8 @@ pub(crate) fn timestamp_trunc_dyn(
731731
/// Keep key masking for fallible values and for coarse units with many distinct values, where
732732
/// masking can avoid expensive calendar work on unused entries. Fine arithmetic processes the
733733
/// physical values regardless of validity, so infallible fine units never need a key scan.
734-
/// MICROSECOND is always infallible; SECOND and MILLISECOND are only while they wrap.
734+
/// MICROSECOND is always infallible. SECOND and MILLISECOND ignore the timezone and, when they do
735+
/// not wrap, fail only near the lower bound, so only such a value needs the key scan.
735736
fn timestamp_trunc_dictionary_needs_mask(
736737
values: &dyn Array,
737738
keys_len: usize,
@@ -741,12 +742,25 @@ fn timestamp_trunc_dictionary_needs_mask(
741742
let Some(values) = values.as_any().downcast_ref::<TimestampMicrosecondArray>() else {
742743
return Ok(true);
743744
};
745+
// Truncation moves backwards by at most a year. Above this conservative lower-bound margin,
746+
// UTC/NTZ truncation cannot underflow i64 and all values can be evaluated without a key scan.
747+
const LOWER_SAFE_MICROS: i64 = i64::MIN + 370 * MICROS_PER_DAY;
748+
let has_value_near_lower_bound = || {
749+
values
750+
.iter()
751+
.flatten()
752+
.any(|micros| micros < LOWER_SAFE_MICROS)
753+
};
744754
let granularity = normalize_timestamp_trunc_format(format)?;
745755
if granularity == "microsecond"
746756
|| (wrap_second_millisecond_overflow && matches!(granularity, "millisecond" | "second"))
747757
{
748758
return Ok(false);
749759
}
760+
if matches!(granularity, "millisecond" | "second") {
761+
// These units ignore the timezone, so a non-UTC value needs no masking of its own.
762+
return Ok(has_value_near_lower_bound());
763+
}
750764
if values.timezone().is_some_and(|tz| !is_utc_timezone(tz)) {
751765
// Unused non-UTC values can also hit chrono boundary panics, not just Result errors.
752766
return Ok(true);
@@ -757,13 +771,7 @@ fn timestamp_trunc_dictionary_needs_mask(
757771
// Dense NULLs/high cardinality otherwise benefit from masking before truncation.
758772
return Ok(true);
759773
}
760-
// Truncation moves backwards by at most a year. Above this conservative lower-bound margin,
761-
// UTC/NTZ truncation cannot underflow i64 and all values can be evaluated without a key scan.
762-
const LOWER_SAFE_MICROS: i64 = i64::MIN + 370 * MICROS_PER_DAY;
763-
Ok(values
764-
.iter()
765-
.flatten()
766-
.any(|micros| micros < LOWER_SAFE_MICROS))
774+
Ok(has_value_near_lower_bound())
767775
}
768776

769777
/// Convert microseconds since epoch to NaiveDateTime
@@ -1448,8 +1456,8 @@ where
14481456
mod tests {
14491457
use super::{
14501458
naive_to_micros, normalize_timestamp_trunc_format, ntz_trunc_fn_for_format,
1451-
timestamp_trunc_coarse_micros, timestamp_trunc_ntz, MICROS_PER_DAY,
1452-
TIMESTAMP_TRUNC_ALIASES,
1459+
timestamp_trunc_coarse_micros, timestamp_trunc_dictionary_needs_mask, timestamp_trunc_ntz,
1460+
MICROS_PER_DAY, TIMESTAMP_TRUNC_ALIASES,
14531461
};
14541462
use crate::kernels::temporal::{
14551463
date_trunc, date_trunc_array_fmt_dyn, date_trunc_dyn, timestamp_trunc,
@@ -2204,6 +2212,34 @@ mod tests {
22042212
}
22052213
}
22062214

2215+
#[test]
2216+
fn test_timestamp_trunc_dictionary_fine_mask_decision() {
2217+
// SECOND/MILLISECOND ignore the timezone, so a dictionary of safe values is truncated
2218+
// through its values alone in either mode. Only a value that can fail needs the key scan.
2219+
for timezone in [None, Some("UTC"), Some("Asia/Tokyo")] {
2220+
let safe = TimestampMicrosecondArray::from(vec![
2221+
Some(instant_micros("2024-05-17T12:34:56Z")),
2222+
None,
2223+
])
2224+
.with_timezone_opt(timezone);
2225+
let near_min =
2226+
TimestampMicrosecondArray::from(vec![Some(i64::MIN)]).with_timezone_opt(timezone);
2227+
for format in ["SECOND", "MILLISECOND"] {
2228+
for wrap in [true, false] {
2229+
assert!(
2230+
!timestamp_trunc_dictionary_needs_mask(&safe, 8192, format, wrap).unwrap()
2231+
);
2232+
}
2233+
assert!(
2234+
!timestamp_trunc_dictionary_needs_mask(&near_min, 8192, format, true).unwrap()
2235+
);
2236+
assert!(
2237+
timestamp_trunc_dictionary_needs_mask(&near_min, 8192, format, false).unwrap()
2238+
);
2239+
}
2240+
}
2241+
}
2242+
22072243
#[test]
22082244
fn test_timestamp_trunc_dictionary_fine_unused_extremes() {
22092245
// Unused MIN entries must not raise, whether SECOND/MILLISECOND wrap (Spark before 4.2)

0 commit comments

Comments
 (0)