Skip to content

Commit ef60197

Browse files
authored
fix: surface next_day and make_date ANSI errors as Spark exceptions (#5167)
1 parent 73ef55f commit ef60197

11 files changed

Lines changed: 131 additions & 30 deletions

File tree

‎docs/source/user-guide/latest/compatibility/index.md‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -121,10 +121,6 @@ divergence:
121121
raise raw Arrow errors that bypass `SparkErrorConverter` and surface as `CometNativeException`
122122
rather than `SparkArithmeticException` with the proper error class and query context
123123
([#5072](https://github.com/apache/datafusion-comet/issues/5072)).
124-
- `next_day` and `make_date` throw at the correct inputs but surface as `CometNativeException`
125-
instead of `SparkIllegalArgumentException [ILLEGAL_DAY_OF_WEEK]` /
126-
`SparkDateTimeException [DATETIME_FIELD_OUT_OF_BOUNDS.WITH_SUGGESTION]`
127-
([#5073](https://github.com/apache/datafusion-comet/issues/5073)).
128124
- Spark 4.2 introduced additional ANSI arithmetic overflow behavior differences that Comet does
129125
not yet track ([#4967](https://github.com/apache/datafusion-comet/issues/4967)).
130126

‎native/common/src/error.rs‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,12 @@ pub enum SparkError {
101101
#[error("[DATETIME_OVERFLOW] Datetime arithmetic overflow.")]
102102
DatetimeOverflow,
103103

104+
#[error("[ILLEGAL_DAY_OF_WEEK] Illegal input for day of week: {input}.")]
105+
IllegalDayOfWeek { input: String },
106+
107+
#[error("[DATETIME_FIELD_OUT_OF_BOUNDS] {range_message}. If necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error.")]
108+
DatetimeFieldOutOfBounds { range_message: String },
109+
104110
#[error("[INVALID_ARRAY_INDEX] The index {index_value} is out of bounds. The array has {array_size} elements. Use the SQL function get() to tolerate accessing element at invalid index and return NULL instead. If necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error.")]
105111
InvalidArrayIndex { index_value: i32, array_size: i32 },
106112

@@ -276,6 +282,8 @@ impl SparkError {
276282
"IntervalArithmeticOverflowWithoutSuggestion"
277283
}
278284
SparkError::DatetimeOverflow => "DatetimeOverflow",
285+
SparkError::IllegalDayOfWeek { .. } => "IllegalDayOfWeek",
286+
SparkError::DatetimeFieldOutOfBounds { .. } => "DatetimeFieldOutOfBounds",
279287
SparkError::InvalidArrayIndex { .. } => "InvalidArrayIndex",
280288
SparkError::InvalidElementAtIndex { .. } => "InvalidElementAtIndex",
281289
SparkError::InvalidBitmapPosition { .. } => "InvalidBitmapPosition",
@@ -458,6 +466,16 @@ impl SparkError {
458466
"suggestedFunc": suggested_func,
459467
})
460468
}
469+
SparkError::IllegalDayOfWeek { input } => {
470+
serde_json::json!({
471+
"string": input,
472+
})
473+
}
474+
SparkError::DatetimeFieldOutOfBounds { range_message } => {
475+
serde_json::json!({
476+
"rangeMessage": range_message,
477+
})
478+
}
461479
SparkError::InvalidFractionOfSecond { value } => {
462480
serde_json::json!({
463481
"value": value,
@@ -605,11 +623,17 @@ impl SparkError {
605623
// DateTimeException
606624
SparkError::InvalidInputInCastToDatetime { .. }
607625
| SparkError::CannotParseTimestamp { .. }
608-
| SparkError::InvalidFractionOfSecond { .. } => "org/apache/spark/SparkDateTimeException",
626+
| SparkError::InvalidFractionOfSecond { .. }
627+
| SparkError::DatetimeFieldOutOfBounds { .. } => {
628+
"org/apache/spark/SparkDateTimeException"
629+
}
609630

610631
// IllegalArgumentException
611632
SparkError::DatatypeCannotOrder { .. }
612-
| SparkError::InvalidUtf8String { .. } => "org/apache/spark/SparkIllegalArgumentException",
633+
| SparkError::InvalidUtf8String { .. }
634+
| SparkError::IllegalDayOfWeek { .. } => {
635+
"org/apache/spark/SparkIllegalArgumentException"
636+
}
613637

614638
// FileNotFound - will be converted to SparkFileNotFoundException by the shim
615639
SparkError::FileNotFound { .. } => "org/apache/spark/SparkException",
@@ -693,6 +717,8 @@ impl SparkError {
693717
// DateTime errors
694718
SparkError::CannotParseTimestamp { .. } => Some("CANNOT_PARSE_TIMESTAMP"),
695719
SparkError::InvalidFractionOfSecond { .. } => Some("INVALID_FRACTION_OF_SECOND"),
720+
SparkError::IllegalDayOfWeek { .. } => Some("ILLEGAL_DAY_OF_WEEK"),
721+
SparkError::DatetimeFieldOutOfBounds { .. } => Some("DATETIME_FIELD_OUT_OF_BOUNDS"),
696722

697723
// String/UTF8 errors
698724
SparkError::InvalidUtf8String { .. } => Some("INVALID_UTF8_STRING"),

‎native/spark-expr/src/datetime_funcs/make_date.rs‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ use datafusion::logical_expr::{
2525
};
2626
use std::sync::Arc;
2727

28+
use crate::SparkError;
29+
2830
/// Spark-compatible make_date function.
2931
/// Creates a date from year, month, and day columns.
3032
/// For an invalid `(year, month, day)` triple Spark returns NULL when `spark.sql.ansi.enabled` is
@@ -53,8 +55,8 @@ impl Default for SparkMakeDate {
5355

5456
/// Build the error message Spark surfaces for an invalid date under ANSI mode. Spark wraps the
5557
/// `java.time.DateTimeException` raised by `LocalDate.of` (via `ansiDateTimeArgumentOutOfRange` /
56-
/// `ansiDateTimeError`), so we reproduce `java.time`'s messages and validation order: month range,
57-
/// then day range, then the day-vs-month check.
58+
/// `ansiDateTimeError`), so we reproduce `java.time`'s messages and validation order: year range,
59+
/// month range, day range, then the day-vs-month check.
5860
fn invalid_date_message(year: i32, month: i32, day: i32) -> String {
5961
const MONTH_NAMES: [&str; 12] = [
6062
"JANUARY",
@@ -70,6 +72,9 @@ fn invalid_date_message(year: i32, month: i32, day: i32) -> String {
7072
"NOVEMBER",
7173
"DECEMBER",
7274
];
75+
if !(-999_999_999..=999_999_999).contains(&year) {
76+
return format!("Invalid value for Year (valid values -999999999 - 999999999): {year}");
77+
}
7378
if !(1..=12).contains(&month) {
7479
return format!("Invalid value for MonthOfYear (valid values 1 - 12): {month}");
7580
}
@@ -178,7 +183,10 @@ impl ScalarUDFImpl for SparkMakeDate {
178183
Some(days) => builder.append_value(days),
179184
None => {
180185
if self.fail_on_error {
181-
return Err(DataFusionError::Execution(invalid_date_message(y, m, d)));
186+
return Err(SparkError::DatetimeFieldOutOfBounds {
187+
range_message: invalid_date_message(y, m, d),
188+
}
189+
.into());
182190
}
183191
builder.append_null();
184192
}

‎native/spark-expr/src/datetime_funcs/next_day.rs‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ use datafusion::logical_expr::{
2525
};
2626
use std::sync::Arc;
2727

28+
use crate::SparkError;
29+
2830
/// Spark-compatible `next_day(start_date, day_of_week)` function.
2931
///
3032
/// Returns the first date which is later than `start_date` and named as `day_of_week`. Unlike the
@@ -146,9 +148,10 @@ impl ScalarUDFImpl for SparkNextDay {
146148
},
147149
None => {
148150
if self.fail_on_error {
149-
return Err(DataFusionError::Execution(format!(
150-
"Illegal input for day of week: {day_of_week}"
151-
)));
151+
return Err(SparkError::IllegalDayOfWeek {
152+
input: day_of_week.to_string(),
153+
}
154+
.into());
152155
}
153156
builder.append_null();
154157
}

‎spark/src/main/scala/org/apache/comet/serde/datetime.scala‎

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -446,12 +446,6 @@ object CometNextDay extends CometExpressionSerde[NextDay] {
446446
override def getIncompatibleReasons(): Seq[String] =
447447
DatetimeCollation.incompatibleReasons("next_day")
448448

449-
override def getCompatibleNotes(): Seq[String] = Seq(
450-
"Under ANSI mode, an invalid `dayOfWeek` surfaces as `CometNativeException` rather than" +
451-
" Spark's `SparkIllegalArgumentException` with error class `ILLEGAL_DAY_OF_WEEK`. The" +
452-
" throw/NULL decision is correct; only the exception class and error class differ" +
453-
" ([#5073](https://github.com/apache/datafusion-comet/issues/5073)).")
454-
455449
override def getSupportLevel(expr: NextDay): SupportLevel = {
456450
if (DatetimeCollation.hasNonDefaultCollation(expr)) {
457451
Incompatible(Some(collationReason))
@@ -479,11 +473,11 @@ object CometMakeDate extends CometExpressionSerde[MakeDate] {
479473
*/
480474

481475
override def getCompatibleNotes(): Seq[String] = Seq(
482-
"Under ANSI mode, an out-of-range `(year, month, day)` triple surfaces as" +
483-
" `CometNativeException` rather than Spark's `SparkDateTimeException` with error class" +
484-
" `DATETIME_FIELD_OUT_OF_BOUNDS.WITH_SUGGESTION`. The throw/NULL decision is correct;" +
485-
" only the exception class and error class differ" +
486-
" ([#5073](https://github.com/apache/datafusion-comet/issues/5073)).")
476+
"Native `make_date` is limited to chrono's year range `[-262143, 262142]`; Spark accepts" +
477+
" wider years (for example, `300000`), so Comet returns `NULL` or throws under ANSI mode" +
478+
" for dates Spark accepts, and may incorrectly report valid dates as invalid (for example," +
479+
" `300000-02-29` is falsely reported as not a leap year)" +
480+
" ([#5208](https://github.com/apache/datafusion-comet/issues/5208)).")
487481

488482
override def convert(expr: MakeDate, inputs: Seq[Attribute], binding: Boolean): Option[Expr] = {
489483
val childExpr = expr.children.map(exprToProtoInternal(_, inputs, binding))

‎spark/src/main/spark-3.4/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,16 @@ trait ShimSparkErrorConverter {
178178
Some(
179179
QueryExecutionErrors.ansiDateTimeParseError(new Exception(params("message").toString)))
180180

181+
case "IllegalDayOfWeek" =>
182+
Some(
183+
QueryExecutionErrors
184+
.ansiIllegalArgumentError(s"Illegal input for day of week: ${params("string")}"))
185+
186+
case "DatetimeFieldOutOfBounds" =>
187+
Some(
188+
QueryExecutionErrors.ansiDateTimeError(
189+
new java.time.DateTimeException(params("rangeMessage").toString)))
190+
181191
case "InvalidFractionOfSecond" =>
182192
Some(QueryExecutionErrors.invalidFractionOfSecondError())
183193

‎spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,16 @@ trait ShimSparkErrorConverter {
175175
Some(
176176
QueryExecutionErrors.ansiDateTimeParseError(new Exception(params("message").toString)))
177177

178+
case "IllegalDayOfWeek" =>
179+
Some(
180+
QueryExecutionErrors
181+
.ansiIllegalArgumentError(s"Illegal input for day of week: ${params("string")}"))
182+
183+
case "DatetimeFieldOutOfBounds" =>
184+
Some(
185+
QueryExecutionErrors.ansiDateTimeError(
186+
new java.time.DateTimeException(params("rangeMessage").toString)))
187+
178188
case "InvalidFractionOfSecond" =>
179189
Some(QueryExecutionErrors.invalidFractionOfSecondError())
180190

‎spark/src/main/spark-4.x/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,7 @@ import java.io.FileNotFoundException
2323

2424
import scala.util.matching.Regex
2525

26-
import org.apache.spark.QueryContext
27-
import org.apache.spark.SparkException
26+
import org.apache.spark.{QueryContext, SparkException, SparkIllegalArgumentException}
2827
import org.apache.spark.sql.errors.QueryExecutionErrors
2928
import org.apache.spark.sql.execution.datasources.SchemaColumnConvertNotSupportedException
3029
import org.apache.spark.sql.types._
@@ -201,6 +200,17 @@ trait ShimSparkErrorConverter {
201200
new Exception(params("message").toString),
202201
params("suggestedFunc").toString))
203202

203+
case "IllegalDayOfWeek" =>
204+
Some(
205+
new SparkIllegalArgumentException(
206+
errorClass = "ILLEGAL_DAY_OF_WEEK",
207+
messageParameters = Map("string" -> params("string").toString)))
208+
209+
case "DatetimeFieldOutOfBounds" =>
210+
Some(
211+
QueryExecutionErrors.ansiDateTimeArgumentOutOfRange(
212+
new java.time.DateTimeException(params("rangeMessage").toString)))
213+
204214
case "InvalidFractionOfSecond" =>
205215
Some(QueryExecutionErrors.invalidFractionOfSecondError(params("value").toString.toDouble))
206216

‎spark/src/test/resources/sql-tests/expressions/datetime/make_date_ansi.sql‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,11 @@
1616
-- under the License.
1717

1818
-- ANSI mode: Spark's MakeDate wraps the java.time.DateTimeException raised by LocalDate.of in
19-
-- ansiDateTimeArgumentOutOfRange (4.0, DATETIME_FIELD_OUT_OF_BOUNDS) / ansiDateTimeError (3.4/3.5)
20-
-- when spark.sql.ansi.enabled=true. Comet's native SparkMakeDate now throws the same
21-
-- java.time-style message under ANSI instead of returning NULL. The expect_error patterns below
22-
-- are substrings of that message and match across Spark versions.
19+
-- ansiDateTimeArgumentOutOfRange (4.x) / ansiDateTimeError (3.x) when
20+
-- spark.sql.ansi.enabled=true. The DATETIME_FIELD_OUT_OF_BOUNDS subclass and message parameters
21+
-- vary by Spark version. Comet's native SparkMakeDate throws the same java.time-style message
22+
-- under ANSI instead of returning NULL. The expect_error patterns below are substrings of that
23+
-- message and match across Spark versions.
2324
-- Config: spark.sql.ansi.enabled=true
2425

2526
-- Sentinel: a valid date must still execute natively under ANSI. This guards against the

‎spark/src/test/resources/sql-tests/expressions/datetime/next_day_ansi.sql‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@
1616
-- under the License.
1717

1818
-- ANSI mode: Spark's NextDay throws on a malformed dayOfWeek (SparkIllegalArgumentException /
19-
-- ILLEGAL_DAY_OF_WEEK on 3.5+, IllegalArgumentException on 3.4) when spark.sql.ansi.enabled=true.
19+
-- ILLEGAL_DAY_OF_WEEK on 4.0+, _LEGACY_ERROR_TEMP_2000 on 3.x) when
20+
-- spark.sql.ansi.enabled=true.
2021
-- Comet's native next_day now throws the same "Illegal input for day of week" message under ANSI
2122
-- instead of returning NULL.
2223
-- Config: spark.sql.ansi.enabled=true

0 commit comments

Comments
 (0)