@@ -731,7 +731,8 @@ pub(crate) fn timestamp_trunc_dyn(
731731/// Keep key masking for fallible values and for coarse units with many distinct values, where
732732/// masking can avoid expensive calendar work on unused entries. Fine arithmetic processes the
733733/// physical values regardless of validity, so infallible fine units never need a key scan.
734- /// MICROSECOND is always infallible; SECOND and MILLISECOND are only while they wrap.
734+ /// MICROSECOND is always infallible. SECOND and MILLISECOND ignore the timezone and, when they do
735+ /// not wrap, fail only near the lower bound, so only such a value needs the key scan.
735736fn timestamp_trunc_dictionary_needs_mask (
736737 values : & dyn Array ,
737738 keys_len : usize ,
@@ -741,12 +742,25 @@ fn timestamp_trunc_dictionary_needs_mask(
741742 let Some ( values) = values. as_any ( ) . downcast_ref :: < TimestampMicrosecondArray > ( ) else {
742743 return Ok ( true ) ;
743744 } ;
745+ // Truncation moves backwards by at most a year. Above this conservative lower-bound margin,
746+ // UTC/NTZ truncation cannot underflow i64 and all values can be evaluated without a key scan.
747+ const LOWER_SAFE_MICROS : i64 = i64:: MIN + 370 * MICROS_PER_DAY ;
748+ let has_value_near_lower_bound = || {
749+ values
750+ . iter ( )
751+ . flatten ( )
752+ . any ( |micros| micros < LOWER_SAFE_MICROS )
753+ } ;
744754 let granularity = normalize_timestamp_trunc_format ( format) ?;
745755 if granularity == "microsecond"
746756 || ( wrap_second_millisecond_overflow && matches ! ( granularity, "millisecond" | "second" ) )
747757 {
748758 return Ok ( false ) ;
749759 }
760+ if matches ! ( granularity, "millisecond" | "second" ) {
761+ // These units ignore the timezone, so a non-UTC value needs no masking of its own.
762+ return Ok ( has_value_near_lower_bound ( ) ) ;
763+ }
750764 if values. timezone ( ) . is_some_and ( |tz| !is_utc_timezone ( tz) ) {
751765 // Unused non-UTC values can also hit chrono boundary panics, not just Result errors.
752766 return Ok ( true ) ;
@@ -757,13 +771,7 @@ fn timestamp_trunc_dictionary_needs_mask(
757771 // Dense NULLs/high cardinality otherwise benefit from masking before truncation.
758772 return Ok ( true ) ;
759773 }
760- // Truncation moves backwards by at most a year. Above this conservative lower-bound margin,
761- // UTC/NTZ truncation cannot underflow i64 and all values can be evaluated without a key scan.
762- const LOWER_SAFE_MICROS : i64 = i64:: MIN + 370 * MICROS_PER_DAY ;
763- Ok ( values
764- . iter ( )
765- . flatten ( )
766- . any ( |micros| micros < LOWER_SAFE_MICROS ) )
774+ Ok ( has_value_near_lower_bound ( ) )
767775}
768776
769777/// Convert microseconds since epoch to NaiveDateTime
@@ -1448,8 +1456,8 @@ where
14481456mod tests {
14491457 use super :: {
14501458 naive_to_micros, normalize_timestamp_trunc_format, ntz_trunc_fn_for_format,
1451- timestamp_trunc_coarse_micros, timestamp_trunc_ntz , MICROS_PER_DAY ,
1452- TIMESTAMP_TRUNC_ALIASES ,
1459+ timestamp_trunc_coarse_micros, timestamp_trunc_dictionary_needs_mask , timestamp_trunc_ntz ,
1460+ MICROS_PER_DAY , TIMESTAMP_TRUNC_ALIASES ,
14531461 } ;
14541462 use crate :: kernels:: temporal:: {
14551463 date_trunc, date_trunc_array_fmt_dyn, date_trunc_dyn, timestamp_trunc,
@@ -2204,6 +2212,34 @@ mod tests {
22042212 }
22052213 }
22062214
2215+ #[ test]
2216+ fn test_timestamp_trunc_dictionary_fine_mask_decision ( ) {
2217+ // SECOND/MILLISECOND ignore the timezone, so a dictionary of safe values is truncated
2218+ // through its values alone in either mode. Only a value that can fail needs the key scan.
2219+ for timezone in [ None , Some ( "UTC" ) , Some ( "Asia/Tokyo" ) ] {
2220+ let safe = TimestampMicrosecondArray :: from ( vec ! [
2221+ Some ( instant_micros( "2024-05-17T12:34:56Z" ) ) ,
2222+ None ,
2223+ ] )
2224+ . with_timezone_opt ( timezone) ;
2225+ let near_min =
2226+ TimestampMicrosecondArray :: from ( vec ! [ Some ( i64 :: MIN ) ] ) . with_timezone_opt ( timezone) ;
2227+ for format in [ "SECOND" , "MILLISECOND" ] {
2228+ for wrap in [ true , false ] {
2229+ assert ! (
2230+ !timestamp_trunc_dictionary_needs_mask( & safe, 8192 , format, wrap) . unwrap( )
2231+ ) ;
2232+ }
2233+ assert ! (
2234+ !timestamp_trunc_dictionary_needs_mask( & near_min, 8192 , format, true ) . unwrap( )
2235+ ) ;
2236+ assert ! (
2237+ timestamp_trunc_dictionary_needs_mask( & near_min, 8192 , format, false ) . unwrap( )
2238+ ) ;
2239+ }
2240+ }
2241+ }
2242+
22072243 #[ test]
22082244 fn test_timestamp_trunc_dictionary_fine_unused_extremes ( ) {
22092245 // Unused MIN entries must not raise, whether SECOND/MILLISECOND wrap (Spark before 4.2)
0 commit comments