|
18 | 18 | use arrow::array::{Array, Date32Array, Int32Array}; |
19 | 19 | use arrow::compute::cast; |
20 | 20 | use arrow::datatypes::DataType; |
21 | | -use chrono::NaiveDate; |
22 | 21 | use datafusion::common::{utils::take_function_args, DataFusionError, Result}; |
23 | 22 | use datafusion::logical_expr::{ |
24 | 23 | ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, |
25 | 24 | }; |
26 | 25 | use std::sync::Arc; |
27 | 26 |
|
28 | | -use crate::SparkError; |
| 27 | +use crate::{conversion_funcs::ymd_to_epoch_day, SparkError}; |
29 | 28 |
|
30 | 29 | /// Spark-compatible make_date function. |
31 | 30 | /// Creates a date from year, month, and day columns. |
@@ -98,18 +97,10 @@ fn cast_to_int32(arr: &Arc<dyn Array>) -> Result<Arc<dyn Array>> { |
98 | 97 | } |
99 | 98 |
|
100 | 99 | /// Convert year, month, day to days since Unix epoch (1970-01-01). |
101 | | -/// Returns None if the date is invalid. |
| 100 | +/// Returns None if the date is invalid or its epoch day does not fit Date32. |
102 | 101 | fn make_date(year: i32, month: i32, day: i32) -> Option<i32> { |
103 | | - // Validate month and day ranges first |
104 | | - if !(1..=12).contains(&month) || !(1..=31).contains(&day) { |
105 | | - return None; |
106 | | - } |
107 | | - |
108 | | - // Try to create a valid date |
109 | | - NaiveDate::from_ymd_opt(year, month as u32, day as u32).map(|date| { |
110 | | - date.signed_duration_since(NaiveDate::from_ymd_opt(1970, 1, 1).unwrap()) |
111 | | - .num_days() as i32 |
112 | | - }) |
| 102 | + ymd_to_epoch_day(year.into(), month.into(), day.into()) |
| 103 | + .and_then(|days| i32::try_from(days).ok()) |
113 | 104 | } |
114 | 105 |
|
115 | 106 | impl ScalarUDFImpl for SparkMakeDate { |
@@ -246,33 +237,13 @@ mod tests { |
246 | 237 | } |
247 | 238 |
|
248 | 239 | #[test] |
249 | | - fn test_make_date_extreme_years() { |
250 | | - // Spark supports dates from 0001-01-01 to 9999-12-31 (Proleptic Gregorian calendar) |
251 | | - |
252 | | - // Minimum valid date in Spark: 0001-01-01 |
253 | | - assert!(make_date(1, 1, 1).is_some(), "Year 1 should be valid"); |
254 | | - |
255 | | - // Maximum valid date in Spark: 9999-12-31 |
256 | | - assert!( |
257 | | - make_date(9999, 12, 31).is_some(), |
258 | | - "Year 9999 should be valid" |
259 | | - ); |
260 | | - |
261 | | - // Year 0 - In Proleptic Gregorian calendar, year 0 = 1 BCE |
262 | | - // Spark returns NULL for year 0 in make_date |
263 | | - // chrono supports year 0, but we should match Spark's behavior |
264 | | - // For now, chrono allows it - this may need adjustment for full Spark compatibility |
265 | | - let year_0_result = make_date(0, 1, 1); |
266 | | - // chrono allows year 0 (1 BCE in proleptic Gregorian) |
267 | | - assert!(year_0_result.is_some(), "chrono allows year 0"); |
268 | | - |
269 | | - // Negative years - Spark returns NULL for negative years |
270 | | - // chrono supports negative years (BCE dates) |
271 | | - let negative_year_result = make_date(-1, 1, 1); |
272 | | - // chrono allows negative years |
273 | | - assert!( |
274 | | - negative_year_result.is_some(), |
275 | | - "chrono allows negative years" |
276 | | - ); |
| 240 | + fn test_make_date_wide_year_range() { |
| 241 | + assert_eq!(make_date(0, 1, 1), Some(-719_528)); |
| 242 | + assert_eq!(make_date(-1, 1, 1), Some(-719_893)); |
| 243 | + assert_eq!(make_date(300_000, 6, 15), Some(108_853_388)); |
| 244 | + assert_eq!(make_date(300_000, 2, 29), Some(108_853_281)); |
| 245 | + |
| 246 | + assert_eq!(make_date(5_881_580, 7, 11), Some(i32::MAX)); |
| 247 | + assert_eq!(make_date(-5_877_641, 6, 23), Some(i32::MIN)); |
277 | 248 | } |
278 | 249 | } |
0 commit comments