Skip to content

Commit e0ab0a6

Browse files
peterxclisunchao
andauthored
fix: support wide years in native make_date (#5443)
Co-authored-by: Chao Sun <[email protected]>
1 parent 082efba commit e0ab0a6

6 files changed

Lines changed: 21 additions & 49 deletions

File tree

‎native/spark-expr/src/conversion_funcs/mod.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,5 @@ mod string;
2222
mod temporal;
2323
pub(crate) mod trim;
2424
mod utils;
25+
26+
pub(crate) use string::ymd_to_epoch_day;

‎native/spark-expr/src/conversion_funcs/string.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1234,7 +1234,7 @@ fn is_leap_year(year: i64) -> bool {
12341234
/// Days since 1970-01-01 for a proleptic Gregorian year/month/day, or `None` when the
12351235
/// combination is not a real calendar date. Unlike `NaiveDate::from_ymd_opt`, this accepts
12361236
/// any year that fits in `i64`.
1237-
fn ymd_to_epoch_day(year: i64, month: i64, day: i64) -> Option<i64> {
1237+
pub(crate) fn ymd_to_epoch_day(year: i64, month: i64, day: i64) -> Option<i64> {
12381238
const DAYS_IN_MONTH: [i64; 12] = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
12391239
let mut max_day = *DAYS_IN_MONTH.get(usize::try_from(month.checked_sub(1)?).ok()?)?;
12401240
if month == 2 && is_leap_year(year) {

‎native/spark-expr/src/datetime_funcs/make_date.rs‎

Lines changed: 12 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,13 @@
1818
use arrow::array::{Array, Date32Array, Int32Array};
1919
use arrow::compute::cast;
2020
use arrow::datatypes::DataType;
21-
use chrono::NaiveDate;
2221
use datafusion::common::{utils::take_function_args, DataFusionError, Result};
2322
use datafusion::logical_expr::{
2423
ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
2524
};
2625
use std::sync::Arc;
2726

28-
use crate::SparkError;
27+
use crate::{conversion_funcs::ymd_to_epoch_day, SparkError};
2928

3029
/// Spark-compatible make_date function.
3130
/// Creates a date from year, month, and day columns.
@@ -98,18 +97,10 @@ fn cast_to_int32(arr: &Arc<dyn Array>) -> Result<Arc<dyn Array>> {
9897
}
9998

10099
/// Convert year, month, day to days since Unix epoch (1970-01-01).
101-
/// Returns None if the date is invalid.
100+
/// Returns None if the date is invalid or its epoch day does not fit Date32.
102101
fn make_date(year: i32, month: i32, day: i32) -> Option<i32> {
103-
// Validate month and day ranges first
104-
if !(1..=12).contains(&month) || !(1..=31).contains(&day) {
105-
return None;
106-
}
107-
108-
// Try to create a valid date
109-
NaiveDate::from_ymd_opt(year, month as u32, day as u32).map(|date| {
110-
date.signed_duration_since(NaiveDate::from_ymd_opt(1970, 1, 1).unwrap())
111-
.num_days() as i32
112-
})
102+
ymd_to_epoch_day(year.into(), month.into(), day.into())
103+
.and_then(|days| i32::try_from(days).ok())
113104
}
114105

115106
impl ScalarUDFImpl for SparkMakeDate {
@@ -246,33 +237,13 @@ mod tests {
246237
}
247238

248239
#[test]
249-
fn test_make_date_extreme_years() {
250-
// Spark supports dates from 0001-01-01 to 9999-12-31 (Proleptic Gregorian calendar)
251-
252-
// Minimum valid date in Spark: 0001-01-01
253-
assert!(make_date(1, 1, 1).is_some(), "Year 1 should be valid");
254-
255-
// Maximum valid date in Spark: 9999-12-31
256-
assert!(
257-
make_date(9999, 12, 31).is_some(),
258-
"Year 9999 should be valid"
259-
);
260-
261-
// Year 0 - In Proleptic Gregorian calendar, year 0 = 1 BCE
262-
// Spark returns NULL for year 0 in make_date
263-
// chrono supports year 0, but we should match Spark's behavior
264-
// For now, chrono allows it - this may need adjustment for full Spark compatibility
265-
let year_0_result = make_date(0, 1, 1);
266-
// chrono allows year 0 (1 BCE in proleptic Gregorian)
267-
assert!(year_0_result.is_some(), "chrono allows year 0");
268-
269-
// Negative years - Spark returns NULL for negative years
270-
// chrono supports negative years (BCE dates)
271-
let negative_year_result = make_date(-1, 1, 1);
272-
// chrono allows negative years
273-
assert!(
274-
negative_year_result.is_some(),
275-
"chrono allows negative years"
276-
);
240+
fn test_make_date_wide_year_range() {
241+
assert_eq!(make_date(0, 1, 1), Some(-719_528));
242+
assert_eq!(make_date(-1, 1, 1), Some(-719_893));
243+
assert_eq!(make_date(300_000, 6, 15), Some(108_853_388));
244+
assert_eq!(make_date(300_000, 2, 29), Some(108_853_281));
245+
246+
assert_eq!(make_date(5_881_580, 7, 11), Some(i32::MAX));
247+
assert_eq!(make_date(-5_877_641, 6, 23), Some(i32::MIN));
277248
}
278249
}

‎spark/src/main/scala/org/apache/comet/serde/datetime.scala‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -472,13 +472,6 @@ object CometMakeDate extends CometExpressionSerde[MakeDate] {
472472
* via the `ScalarFunc.fail_on_error` field.
473473
*/
474474

475-
override def getCompatibleNotes(): Seq[String] = Seq(
476-
"Native `make_date` is limited to chrono's year range `[-262143, 262142]`; Spark accepts" +
477-
" wider years (for example, `300000`), so Comet returns `NULL` or throws under ANSI mode" +
478-
" for dates Spark accepts, and may incorrectly report valid dates as invalid (for example," +
479-
" `300000-02-29` is falsely reported as not a leap year)" +
480-
" ([#5208](https://github.com/apache/datafusion-comet/issues/5208)).")
481-
482475
override def convert(expr: MakeDate, inputs: Seq[Attribute], binding: Boolean): Option[Expr] = {
483476
val childExpr = expr.children.map(exprToProtoInternal(_, inputs, binding))
484477
val optExpr = scalarFunctionExprToProtoWithReturnType(

‎spark/src/test/resources/sql-tests/expressions/datetime/make_date.sql‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,9 @@ SELECT make_date(0, 1, 1)
135135
query
136136
SELECT make_date(-1, 1, 1)
137137

138+
query
139+
SELECT make_date(300000, 6, 15), make_date(300000, 2, 29)
140+
138141
-- month boundaries - last day of each month
139142
query
140143
SELECT make_date(2023, 1, 31)

‎spark/src/test/resources/sql-tests/expressions/datetime/make_date_ansi.sql‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@
2828
query
2929
SELECT make_date(2024, 2, 28)
3030

31+
query
32+
SELECT make_date(300000, 6, 15), make_date(300000, 2, 29)
33+
3134
-- February 30 is not a valid date.
3235
query expect_error(Invalid date)
3336
SELECT make_date(2024, 2, 30)

0 commit comments

Comments
 (0)