|
23 | 23 | //! `hours` are plain floor division of the epoch value. `days` returns a date (Iceberg's |
24 | 24 | //! `DaysFunction.resultType()` is `DateType`), the other three return an int. |
25 | 25 | //! |
26 | | -//! The kernels work on the raw epoch values rather than going through Arrow's timezone-aware |
27 | | -//! `date_part`, which would otherwise shift a `TimestampType` column by the session offset. |
| 26 | +//! The kernels read the raw epoch values instead of Arrow's timezone-aware `date_part`. That is a |
| 27 | +//! defensive choice rather than a fix for an observed bug: Comet tags every `TimestampType` array |
| 28 | +//! `UTC` and the Iceberg writer casts each batch to a schema that tags `Timestamptz` as `+00:00`, |
| 29 | +//! so `date_part` would agree today. It only keeps agreeing for as long as that tagging holds, |
| 30 | +//! whereas the epoch arithmetic below is correct for any tag. |
| 31 | +//! |
| 32 | +//! The calendar split is integer arithmetic rather than a `chrono::NaiveDate`, which covers only |
| 33 | +//! about ±262k years. A Spark `DateType` is an `i32` epoch day (up to year 5881580) and Java's |
| 34 | +//! `LocalDate`, which Iceberg uses, covers all of it, so going through `chrono` would turn values |
| 35 | +//! the JVM handles into execution errors. |
28 | 36 |
|
29 | 37 | use super::{apply_unary, unsupported_type}; |
30 | 38 | use arrow::array::{ArrayRef, AsArray, Int32Array}; |
31 | 39 | use arrow::datatypes::{DataType, Date32Type, Int32Type, TimeUnit, TimestampMicrosecondType}; |
32 | | -use chrono::Datelike; |
33 | | -use datafusion::common::{utils::take_function_args, DataFusionError, Result}; |
| 40 | +use datafusion::common::{utils::take_function_args, Result}; |
34 | 41 | use datafusion::logical_expr::{ |
35 | 42 | ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, |
36 | 43 | }; |
@@ -67,46 +74,60 @@ impl TemporalUnit { |
67 | 74 | } |
68 | 75 | } |
69 | 76 |
|
70 | | -/// `DateTimeUtil.microsToDays`: floor division, so `-1` micros is day `-1`. |
| 77 | +/// `DateTimeUtil.microsToDays`: floor division, so `-1` micros is day `-1`. The quotient of the |
| 78 | +/// widest `i64` micros is about 1.07e8, so the narrowing is always exact here. |
71 | 79 | #[inline] |
72 | 80 | fn micros_to_days(micros: i64) -> i32 { |
73 | 81 | div_floor(micros, MICROS_PER_DAY) as i32 |
74 | 82 | } |
75 | 83 |
|
76 | | -/// `DateTimeUtil.microsToHours`. |
| 84 | +/// `DateTimeUtil.microsToHours`. Java narrows the hour count with a plain `(int)` cast, which |
| 85 | +/// wraps beyond about 7.7e18 micros; `as i32` truncates the same way. |
77 | 86 | #[inline] |
78 | 87 | fn micros_to_hours(micros: i64) -> i32 { |
79 | 88 | div_floor(micros, MICROS_PER_HOUR) as i32 |
80 | 89 | } |
81 | 90 |
|
82 | 91 | /// `DateTimeUtil.daysToYears`: whole calendar years between the epoch and the day, floored. |
83 | | -fn days_to_years(days: i32) -> Result<i32> { |
84 | | - Ok(civil_date(days)?.year() - UNIX_EPOCH_YEAR) |
| 92 | +fn days_to_years(days: i32) -> i32 { |
| 93 | + civil_from_days(days).0 - UNIX_EPOCH_YEAR |
85 | 94 | } |
86 | 95 |
|
87 | 96 | /// `DateTimeUtil.daysToMonths`: whole calendar months between the epoch and the day, floored. |
88 | | -fn days_to_months(days: i32) -> Result<i32> { |
89 | | - let date = civil_date(days)?; |
90 | | - Ok((date.year() - UNIX_EPOCH_YEAR) * 12 + date.month0() as i32) |
| 97 | +fn days_to_months(days: i32) -> i32 { |
| 98 | + let (year, month0) = civil_from_days(days); |
| 99 | + (year - UNIX_EPOCH_YEAR) * 12 + month0 |
91 | 100 | } |
92 | 101 |
|
93 | | -fn civil_date(days: i32) -> Result<chrono::NaiveDate> { |
94 | | - Date32Type::to_naive_date_opt(days).ok_or_else(|| { |
95 | | - DataFusionError::Execution(format!("day {days} is out of the supported date range")) |
96 | | - }) |
| 102 | +/// Splits an epoch day into its proleptic Gregorian `(year, month0)`, following Howard Hinnant's |
| 103 | +/// `civil_from_days`. The intermediates are `i64` so that every `i32` epoch day is in range, which |
| 104 | +/// is what `LocalDate` gives Iceberg; the widest results, at `i32::MIN` and `i32::MAX` days, are |
| 105 | +/// years -5877641 and 5881580, so both the year and the month count still fit in an `i32`. |
| 106 | +fn civil_from_days(days: i32) -> (i32, i32) { |
| 107 | + // Shift the epoch to 0000-03-01 so that the leap day falls at the end of the year. |
| 108 | + let shifted = days as i64 + 719_468; |
| 109 | + let era = shifted.div_euclid(146_097); |
| 110 | + let day_of_era = shifted.rem_euclid(146_097); // [0, 146096] |
| 111 | + let year_of_era = |
| 112 | + (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; // [0, 399] |
| 113 | + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); // [0, 365] |
| 114 | + let shifted_month = (5 * day_of_year + 2) / 153; // [0, 11], March is 0 |
| 115 | + let month = if shifted_month < 10 { |
| 116 | + shifted_month + 3 |
| 117 | + } else { |
| 118 | + shifted_month - 9 |
| 119 | + }; |
| 120 | + let year = era * 400 + year_of_era + i64::from(month <= 2); |
| 121 | + (year as i32, (month - 1) as i32) |
97 | 122 | } |
98 | 123 |
|
99 | 124 | /// Applies a calendar function of the epoch day to a date or timestamp column in one pass. |
100 | | -fn map_epoch_days( |
101 | | - fn_name: &str, |
102 | | - array: &ArrayRef, |
103 | | - f: impl Fn(i32) -> Result<i32>, |
104 | | -) -> Result<Int32Array> { |
| 125 | +fn map_epoch_days(fn_name: &str, array: &ArrayRef, f: impl Fn(i32) -> i32) -> Result<Int32Array> { |
105 | 126 | match array.data_type() { |
106 | | - DataType::Date32 => array.as_primitive::<Date32Type>().try_unary(f), |
107 | | - DataType::Timestamp(TimeUnit::Microsecond, _) => array |
| 127 | + DataType::Date32 => Ok(array.as_primitive::<Date32Type>().unary(f)), |
| 128 | + DataType::Timestamp(TimeUnit::Microsecond, _) => Ok(array |
108 | 129 | .as_primitive::<TimestampMicrosecondType>() |
109 | | - .try_unary(|micros| f(micros_to_days(micros))), |
| 130 | + .unary(|micros| f(micros_to_days(micros)))), |
110 | 131 | other => Err(unsupported_type(fn_name, other)), |
111 | 132 | } |
112 | 133 | } |
@@ -203,17 +224,25 @@ mod tests { |
203 | 224 | .unwrap() |
204 | 225 | } |
205 | 226 |
|
206 | | - // Boundaries around the epoch, as (epoch days, years, months). |
| 227 | + // Boundaries around the epoch, as (epoch days, years, months). The values past the epoch |
| 228 | + // block are outside `chrono::NaiveDate`'s range but well inside `LocalDate`'s; they come from |
| 229 | + // running Iceberg's `DateTimeUtil.convertDays` on a JDK 17 JVM. |
207 | 230 | const DAY_CASES: &[(i32, i32, i32)] = &[ |
208 | | - (17_486, 47, 574), // 2017-11-16, the Iceberg spec example |
209 | | - (0, 0, 0), // 1970-01-01 |
210 | | - (-1, -1, -1), // 1969-12-31 |
211 | | - (-365, -1, -12), // 1969-01-01 |
212 | | - (-366, -2, -13), // 1968-12-31 |
213 | | - (365, 1, 12), // 1971-01-01 |
214 | | - (364, 0, 11), // 1970-12-31 |
215 | | - (31, 0, 1), // 1970-02-01 |
216 | | - (30, 0, 0), // 1970-01-31 |
| 231 | + (17_486, 47, 574), // 2017-11-16, the Iceberg spec example |
| 232 | + (0, 0, 0), // 1970-01-01 |
| 233 | + (-1, -1, -1), // 1969-12-31 |
| 234 | + (-365, -1, -12), // 1969-01-01 |
| 235 | + (-366, -2, -13), // 1968-12-31 |
| 236 | + (365, 1, 12), // 1971-01-01 |
| 237 | + (364, 0, 11), // 1970-12-31 |
| 238 | + (31, 0, 1), // 1970-02-01 |
| 239 | + (30, 0, 0), // 1970-01-31 |
| 240 | + (100_000_000, 273_790, 3_285_488), // +275760-09-13 |
| 241 | + (-100_000_000, -273_791, -3_285_489), // -271821-04-20 |
| 242 | + (1_000_000_000, 2_737_907, 32_854_884), // +2739877-01-03 |
| 243 | + (-1_000_000_000, -2_737_908, -32_854_885), // -2735938-12-29 |
| 244 | + (i32::MAX, 5_879_610, 70_555_326), // +5881580-07-11 |
| 245 | + (i32::MIN, -5_879_611, -70_555_327), // -5877641-06-23 |
217 | 246 | ]; |
218 | 247 |
|
219 | 248 | #[test] |
@@ -266,6 +295,25 @@ mod tests { |
266 | 295 | (-MICROS_PER_DAY - 1, -1, -1, -2, -25), // 1969-12-30T23:59:59.999999 |
267 | 296 | (365 * MICROS_PER_DAY, 1, 12, 365, 8_760), // 1971-01-01T00:00:00 |
268 | 297 | (365 * MICROS_PER_DAY - 1, 0, 11, 364, 8_759), |
| 298 | + // The extremes of Spark's timestamp domain, from Iceberg's `DateTimeUtil` on a JVM. |
| 299 | + // The hour count is the one conversion that does not fit an `i32` there, and Java's |
| 300 | + // `(int)` narrowing wraps it exactly as `as i32` does. |
| 301 | + (i64::MAX, 292_277, 3_507_324, 106_751_991, -1_732_919_508), |
| 302 | + (i64::MIN, -292_278, -3_507_325, -106_751_992, 1_732_919_507), |
| 303 | + ( |
| 304 | + 8_000_000_000_000_000_000, |
| 305 | + 253_509, |
| 306 | + 3_042_118, |
| 307 | + 92_592_592, |
| 308 | + -2_072_745_074, |
| 309 | + ), |
| 310 | + ( |
| 311 | + -8_000_000_000_000_000_000, |
| 312 | + -253_510, |
| 313 | + -3_042_119, |
| 314 | + -92_592_593, |
| 315 | + 2_072_745_073, |
| 316 | + ), |
269 | 317 | ]; |
270 | 318 | let micros: Vec<Option<i64>> = cases.iter().map(|c| Some(c.0)).chain([None]).collect(); |
271 | 319 | // A non-UTC timezone tag must not change the result. |
@@ -318,6 +366,27 @@ mod tests { |
318 | 366 | } |
319 | 367 | } |
320 | 368 |
|
| 369 | + /// The pinned cases above check the endpoints of the domain; this checks the calendar split |
| 370 | + /// itself everywhere `chrono` can still represent the date, so the hand-rolled arithmetic |
| 371 | + /// cannot drift in between. |
| 372 | + #[test] |
| 373 | + fn civil_from_days_agrees_with_chrono() { |
| 374 | + use chrono::Datelike; |
| 375 | + // Every day of one full 400-year Gregorian cycle either side of the epoch, then a stride |
| 376 | + // over the rest of the `i32` domain (`to_naive_date_opt` returns `None` past chrono's |
| 377 | + // range, which is exactly the region the pinned cases cover). |
| 378 | + let dense = -146_097..=146_097; |
| 379 | + for days in dense.chain((i32::MIN..=i32::MAX).step_by(999_983)) { |
| 380 | + if let Some(date) = Date32Type::to_naive_date_opt(days) { |
| 381 | + assert_eq!( |
| 382 | + civil_from_days(days), |
| 383 | + (date.year(), date.month0() as i32), |
| 384 | + "day {days} ({date})" |
| 385 | + ); |
| 386 | + } |
| 387 | + } |
| 388 | + } |
| 389 | + |
321 | 390 | #[test] |
322 | 391 | fn rejects_unsupported_types() { |
323 | 392 | for unit in [ |
|
0 commit comments