Skip to content

Commit 71e03be

Browse files
committed
fix: cover the whole date domain in the Iceberg temporal kernels
`years` and `months` split the calendar with `chrono::NaiveDate`, whose range stops at about year 262143. A Spark `DateType` is an `i32` epoch day (up to year 5881580) and Iceberg's `DateTimeUtil` goes through `LocalDate`, which covers all of it, so `years(DATE)` on a far-future date raised an execution error where the JVM returns a value. Replace the `chrono` round trip with Howard Hinnant's `civil_from_days` in `i64` arithmetic, which is exact over the whole `i32` epoch-day domain and drops the fallible path from the kernels. The pinned expectations for the extremes come from running `DateTimeUtil.convertDays` / `convertMicros` on a JVM, and a new test checks the integer split against `chrono` everywhere `chrono` can represent the date.
1 parent 7de2708 commit 71e03be

1 file changed

Lines changed: 102 additions & 33 deletions

File tree

‎native/spark-expr/src/iceberg_funcs/temporal.rs‎

Lines changed: 102 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,21 @@
2323
//! `hours` are plain floor division of the epoch value. `days` returns a date (Iceberg's
2424
//! `DaysFunction.resultType()` is `DateType`), the other three return an int.
2525
//!
26-
//! The kernels work on the raw epoch values rather than going through Arrow's timezone-aware
27-
//! `date_part`, which would otherwise shift a `TimestampType` column by the session offset.
26+
//! The kernels read the raw epoch values instead of Arrow's timezone-aware `date_part`. That is a
27+
//! defensive choice rather than a fix for an observed bug: Comet tags every `TimestampType` array
28+
//! `UTC` and the Iceberg writer casts each batch to a schema that tags `Timestamptz` as `+00:00`,
29+
//! so `date_part` would agree today. It only keeps agreeing for as long as that tagging holds,
30+
//! whereas the epoch arithmetic below is correct for any tag.
31+
//!
32+
//! The calendar split is integer arithmetic rather than a `chrono::NaiveDate`, which covers only
33+
//! about ±262k years. A Spark `DateType` is an `i32` epoch day (up to year 5881580) and Java's
34+
//! `LocalDate`, which Iceberg uses, covers all of it, so going through `chrono` would turn values
35+
//! the JVM handles into execution errors.
2836
2937
use super::{apply_unary, unsupported_type};
3038
use arrow::array::{ArrayRef, AsArray, Int32Array};
3139
use arrow::datatypes::{DataType, Date32Type, Int32Type, TimeUnit, TimestampMicrosecondType};
32-
use chrono::Datelike;
33-
use datafusion::common::{utils::take_function_args, DataFusionError, Result};
40+
use datafusion::common::{utils::take_function_args, Result};
3441
use datafusion::logical_expr::{
3542
ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
3643
};
@@ -67,46 +74,60 @@ impl TemporalUnit {
6774
}
6875
}
6976

70-
/// `DateTimeUtil.microsToDays`: floor division, so `-1` micros is day `-1`.
77+
/// `DateTimeUtil.microsToDays`: floor division, so `-1` micros is day `-1`. The quotient of the
78+
/// widest `i64` micros is about 1.07e8, so the narrowing is always exact here.
7179
#[inline]
7280
fn micros_to_days(micros: i64) -> i32 {
7381
div_floor(micros, MICROS_PER_DAY) as i32
7482
}
7583

76-
/// `DateTimeUtil.microsToHours`.
84+
/// `DateTimeUtil.microsToHours`. Java narrows the hour count with a plain `(int)` cast, which
85+
/// wraps beyond about 7.7e18 micros; `as i32` truncates the same way.
7786
#[inline]
7887
fn micros_to_hours(micros: i64) -> i32 {
7988
div_floor(micros, MICROS_PER_HOUR) as i32
8089
}
8190

8291
/// `DateTimeUtil.daysToYears`: whole calendar years between the epoch and the day, floored.
83-
fn days_to_years(days: i32) -> Result<i32> {
84-
Ok(civil_date(days)?.year() - UNIX_EPOCH_YEAR)
92+
fn days_to_years(days: i32) -> i32 {
93+
civil_from_days(days).0 - UNIX_EPOCH_YEAR
8594
}
8695

8796
/// `DateTimeUtil.daysToMonths`: whole calendar months between the epoch and the day, floored.
88-
fn days_to_months(days: i32) -> Result<i32> {
89-
let date = civil_date(days)?;
90-
Ok((date.year() - UNIX_EPOCH_YEAR) * 12 + date.month0() as i32)
97+
fn days_to_months(days: i32) -> i32 {
98+
let (year, month0) = civil_from_days(days);
99+
(year - UNIX_EPOCH_YEAR) * 12 + month0
91100
}
92101

93-
fn civil_date(days: i32) -> Result<chrono::NaiveDate> {
94-
Date32Type::to_naive_date_opt(days).ok_or_else(|| {
95-
DataFusionError::Execution(format!("day {days} is out of the supported date range"))
96-
})
102+
/// Splits an epoch day into its proleptic Gregorian `(year, month0)`, following Howard Hinnant's
103+
/// `civil_from_days`. The intermediates are `i64` so that every `i32` epoch day is in range, which
104+
/// is what `LocalDate` gives Iceberg; the widest results, at `i32::MIN` and `i32::MAX` days, are
105+
/// years -5877641 and 5881580, so both the year and the month count still fit in an `i32`.
106+
fn civil_from_days(days: i32) -> (i32, i32) {
107+
// Shift the epoch to 0000-03-01 so that the leap day falls at the end of the year.
108+
let shifted = days as i64 + 719_468;
109+
let era = shifted.div_euclid(146_097);
110+
let day_of_era = shifted.rem_euclid(146_097); // [0, 146096]
111+
let year_of_era =
112+
(day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; // [0, 399]
113+
let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); // [0, 365]
114+
let shifted_month = (5 * day_of_year + 2) / 153; // [0, 11], March is 0
115+
let month = if shifted_month < 10 {
116+
shifted_month + 3
117+
} else {
118+
shifted_month - 9
119+
};
120+
let year = era * 400 + year_of_era + i64::from(month <= 2);
121+
(year as i32, (month - 1) as i32)
97122
}
98123

99124
/// Applies a calendar function of the epoch day to a date or timestamp column in one pass.
100-
fn map_epoch_days(
101-
fn_name: &str,
102-
array: &ArrayRef,
103-
f: impl Fn(i32) -> Result<i32>,
104-
) -> Result<Int32Array> {
125+
fn map_epoch_days(fn_name: &str, array: &ArrayRef, f: impl Fn(i32) -> i32) -> Result<Int32Array> {
105126
match array.data_type() {
106-
DataType::Date32 => array.as_primitive::<Date32Type>().try_unary(f),
107-
DataType::Timestamp(TimeUnit::Microsecond, _) => array
127+
DataType::Date32 => Ok(array.as_primitive::<Date32Type>().unary(f)),
128+
DataType::Timestamp(TimeUnit::Microsecond, _) => Ok(array
108129
.as_primitive::<TimestampMicrosecondType>()
109-
.try_unary(|micros| f(micros_to_days(micros))),
130+
.unary(|micros| f(micros_to_days(micros)))),
110131
other => Err(unsupported_type(fn_name, other)),
111132
}
112133
}
@@ -203,17 +224,25 @@ mod tests {
203224
.unwrap()
204225
}
205226

206-
// Boundaries around the epoch, as (epoch days, years, months).
227+
// Boundaries around the epoch, as (epoch days, years, months). The values past the epoch
228+
// block are outside `chrono::NaiveDate`'s range but well inside `LocalDate`'s; they come from
229+
// running Iceberg's `DateTimeUtil.convertDays` on a JDK 17 JVM.
207230
const DAY_CASES: &[(i32, i32, i32)] = &[
208-
(17_486, 47, 574), // 2017-11-16, the Iceberg spec example
209-
(0, 0, 0), // 1970-01-01
210-
(-1, -1, -1), // 1969-12-31
211-
(-365, -1, -12), // 1969-01-01
212-
(-366, -2, -13), // 1968-12-31
213-
(365, 1, 12), // 1971-01-01
214-
(364, 0, 11), // 1970-12-31
215-
(31, 0, 1), // 1970-02-01
216-
(30, 0, 0), // 1970-01-31
231+
(17_486, 47, 574), // 2017-11-16, the Iceberg spec example
232+
(0, 0, 0), // 1970-01-01
233+
(-1, -1, -1), // 1969-12-31
234+
(-365, -1, -12), // 1969-01-01
235+
(-366, -2, -13), // 1968-12-31
236+
(365, 1, 12), // 1971-01-01
237+
(364, 0, 11), // 1970-12-31
238+
(31, 0, 1), // 1970-02-01
239+
(30, 0, 0), // 1970-01-31
240+
(100_000_000, 273_790, 3_285_488), // +275760-09-13
241+
(-100_000_000, -273_791, -3_285_489), // -271821-04-20
242+
(1_000_000_000, 2_737_907, 32_854_884), // +2739877-01-03
243+
(-1_000_000_000, -2_737_908, -32_854_885), // -2735938-12-29
244+
(i32::MAX, 5_879_610, 70_555_326), // +5881580-07-11
245+
(i32::MIN, -5_879_611, -70_555_327), // -5877641-06-23
217246
];
218247

219248
#[test]
@@ -266,6 +295,25 @@ mod tests {
266295
(-MICROS_PER_DAY - 1, -1, -1, -2, -25), // 1969-12-30T23:59:59.999999
267296
(365 * MICROS_PER_DAY, 1, 12, 365, 8_760), // 1971-01-01T00:00:00
268297
(365 * MICROS_PER_DAY - 1, 0, 11, 364, 8_759),
298+
// The extremes of Spark's timestamp domain, from Iceberg's `DateTimeUtil` on a JVM.
299+
// The hour count is the one conversion that does not fit an `i32` there, and Java's
300+
// `(int)` narrowing wraps it exactly as `as i32` does.
301+
(i64::MAX, 292_277, 3_507_324, 106_751_991, -1_732_919_508),
302+
(i64::MIN, -292_278, -3_507_325, -106_751_992, 1_732_919_507),
303+
(
304+
8_000_000_000_000_000_000,
305+
253_509,
306+
3_042_118,
307+
92_592_592,
308+
-2_072_745_074,
309+
),
310+
(
311+
-8_000_000_000_000_000_000,
312+
-253_510,
313+
-3_042_119,
314+
-92_592_593,
315+
2_072_745_073,
316+
),
269317
];
270318
let micros: Vec<Option<i64>> = cases.iter().map(|c| Some(c.0)).chain([None]).collect();
271319
// A non-UTC timezone tag must not change the result.
@@ -318,6 +366,27 @@ mod tests {
318366
}
319367
}
320368

369+
/// The pinned cases above check the endpoints of the domain; this checks the calendar split
370+
/// itself everywhere `chrono` can still represent the date, so the hand-rolled arithmetic
371+
/// cannot drift in between.
372+
#[test]
373+
fn civil_from_days_agrees_with_chrono() {
374+
use chrono::Datelike;
375+
// Every day of one full 400-year Gregorian cycle either side of the epoch, then a stride
376+
// over the rest of the `i32` domain (`to_naive_date_opt` returns `None` past chrono's
377+
// range, which is exactly the region the pinned cases cover).
378+
let dense = -146_097..=146_097;
379+
for days in dense.chain((i32::MIN..=i32::MAX).step_by(999_983)) {
380+
if let Some(date) = Date32Type::to_naive_date_opt(days) {
381+
assert_eq!(
382+
civil_from_days(days),
383+
(date.year(), date.month0() as i32),
384+
"day {days} ({date})"
385+
);
386+
}
387+
}
388+
}
389+
321390
#[test]
322391
fn rejects_unsupported_types() {
323392
for unit in [

0 commit comments

Comments
 (0)