Skip to content

Commit 76a4513

Browse files
authored
fix(spark): match Spark abs overflow errors (apache#24156)
## Which issue does this PR close? - Closes apache#24145. ## Rationale for this change When ANSI mode is enabled, `datafusion-spark` reports type-specific DataFusion errors for integral `abs` overflow. The scalar and array paths also produce different messages. Spark 4.2 uses one canonical `ARITHMETIC_OVERFLOW` message for both paths. ## What changes are included in this PR? - Add a Spark-local checked `abs` array kernel so Spark-specific error text does not change core DataFusion behavior. - Use the Spark 4.2 `ARITHMETIC_OVERFLOW` message for scalar and array overflows. - Update the existing Spark SQL logic tests for all signed integer widths and both input shapes. ## Are these changes tested? Yes. The following checks pass: - `cargo fmt --all -- --check` - `cargo clippy -p datafusion-spark --all-targets --all-features -- -D warnings` - `cargo test -p datafusion-spark --all-features` - `cargo test --profile=ci --test sqllogictests -- spark/math/abs.slt` - `cargo test --profile=ci --test sqllogictests -- math.slt` - `RUST_BACKTRACE=1 cargo test --profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --exclude datafusion-cli --workspace --lib --tests --bins --features avro,json,backtrace,extended_tests,recursive_protection,parquet_encryption` ## Are there any user-facing changes? Yes. With `datafusion.execution.enable_ansi_mode = true`, Spark `abs` overflow errors now use Spark 4.2's canonical `ARITHMETIC_OVERFLOW` message for scalar and array inputs. There are no public API changes. --------- Signed-off-by: goutamadwant <[email protected]>
1 parent 5c846af commit 76a4513

2 files changed

Lines changed: 40 additions & 33 deletions

File tree

  • datafusion
    • spark/src/function/math
    • sqllogictest/test_files/spark/math

‎datafusion/spark/src/function/math/abs.rs‎

Lines changed: 32 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,16 @@ use datafusion_expr::{
2424
Volatility,
2525
};
2626
use datafusion_functions::{
27-
downcast_named_arg, make_abs_function, make_try_abs_function,
28-
make_wrapping_abs_function,
27+
downcast_named_arg, make_abs_function, make_wrapping_abs_function,
2928
};
3029
use std::sync::Arc;
3130

31+
const ARITHMETIC_OVERFLOW_ERROR: &str = r#"[ARITHMETIC_OVERFLOW] overflow. If necessary set "spark.sql.ansi.enabled" to "false" to bypass this error. SQLSTATE: 22003"#;
32+
33+
fn arithmetic_overflow_error() -> ArrowError {
34+
ArrowError::ComputeError(ARITHMETIC_OVERFLOW_ERROR.to_string())
35+
}
36+
3237
/// Spark-compatible `abs` expression
3338
/// <https://spark.apache.org/docs/latest/api/sql/index.html#abs>
3439
///
@@ -91,13 +96,7 @@ impl ScalarUDFImpl for SparkAbs {
9196
macro_rules! scalar_compute_op {
9297
($ENABLE_ANSI_MODE:expr, $INPUT:ident, $SCALAR_TYPE:ident) => {{
9398
let result = if $ENABLE_ANSI_MODE {
94-
$INPUT.checked_abs().ok_or_else(|| {
95-
ArrowError::ComputeError(format!(
96-
"{} overflow on abs({:?})",
97-
stringify!($SCALAR_TYPE),
98-
$INPUT
99-
))
100-
})?
99+
$INPUT.checked_abs().ok_or_else(arithmetic_overflow_error)?
101100
} else {
102101
$INPUT.wrapping_abs()
103102
};
@@ -107,13 +106,7 @@ macro_rules! scalar_compute_op {
107106
}};
108107
($ENABLE_ANSI_MODE:expr, $INPUT:ident, $PRECISION:expr, $SCALE:expr, $SCALAR_TYPE:ident) => {{
109108
let result = if $ENABLE_ANSI_MODE {
110-
$INPUT.checked_abs().ok_or_else(|| {
111-
ArrowError::ComputeError(format!(
112-
"{} overflow on abs({:?})",
113-
stringify!($SCALAR_TYPE),
114-
$INPUT
115-
))
116-
})?
109+
$INPUT.checked_abs().ok_or_else(arithmetic_overflow_error)?
117110
} else {
118111
$INPUT.wrapping_abs()
119112
};
@@ -125,6 +118,18 @@ macro_rules! scalar_compute_op {
125118
}};
126119
}
127120

121+
macro_rules! make_try_spark_abs_function {
122+
($ARRAY_TYPE:ident) => {{
123+
|input: &ArrayRef| {
124+
let array = downcast_named_arg!(&input, "abs arg", $ARRAY_TYPE);
125+
let res: $ARRAY_TYPE = array
126+
.try_unary(|x| x.checked_abs().ok_or_else(arithmetic_overflow_error))
127+
.map(|v| v.with_data_type(input.data_type().clone()))?;
128+
Ok(Arc::new(res) as ArrayRef)
129+
}
130+
}};
131+
}
132+
128133
pub fn spark_abs(
129134
args: &[ColumnarValue],
130135
enable_ansi_mode: bool,
@@ -142,31 +147,31 @@ pub fn spark_abs(
142147
| DataType::UInt64 => Ok(args[0].clone()),
143148
DataType::Int8 => {
144149
let abs_fun = if enable_ansi_mode {
145-
make_try_abs_function!(Int8Array)
150+
make_try_spark_abs_function!(Int8Array)
146151
} else {
147152
make_wrapping_abs_function!(Int8Array)
148153
};
149154
abs_fun(array).map(ColumnarValue::Array)
150155
}
151156
DataType::Int16 => {
152157
let abs_fun = if enable_ansi_mode {
153-
make_try_abs_function!(Int16Array)
158+
make_try_spark_abs_function!(Int16Array)
154159
} else {
155160
make_wrapping_abs_function!(Int16Array)
156161
};
157162
abs_fun(array).map(ColumnarValue::Array)
158163
}
159164
DataType::Int32 => {
160165
let abs_fun = if enable_ansi_mode {
161-
make_try_abs_function!(Int32Array)
166+
make_try_spark_abs_function!(Int32Array)
162167
} else {
163168
make_wrapping_abs_function!(Int32Array)
164169
};
165170
abs_fun(array).map(ColumnarValue::Array)
166171
}
167172
DataType::Int64 => {
168173
let abs_fun = if enable_ansi_mode {
169-
make_try_abs_function!(Int64Array)
174+
make_try_spark_abs_function!(Int64Array)
170175
} else {
171176
make_wrapping_abs_function!(Int64Array)
172177
};
@@ -182,15 +187,15 @@ pub fn spark_abs(
182187
}
183188
DataType::Decimal128(_, _) => {
184189
let abs_fun = if enable_ansi_mode {
185-
make_try_abs_function!(Decimal128Array)
190+
make_try_spark_abs_function!(Decimal128Array)
186191
} else {
187192
make_wrapping_abs_function!(Decimal128Array)
188193
};
189194
abs_fun(array).map(ColumnarValue::Array)
190195
}
191196
DataType::Decimal256(_, _) => {
192197
let abs_fun = if enable_ansi_mode {
193-
make_try_abs_function!(Decimal256Array)
198+
make_try_spark_abs_function!(Decimal256Array)
194199
} else {
195200
make_wrapping_abs_function!(Decimal256Array)
196201
};
@@ -361,9 +366,11 @@ mod tests {
361366
let args = ColumnarValue::Array(Arc::new(input));
362367
match spark_abs(&[args], true) {
363368
Err(e) => {
364-
assert!(
365-
e.to_string().contains("overflow on abs"),
366-
"Error message did not match. Actual message: {e}"
369+
assert_eq!(
370+
e.strip_backtrace(),
371+
format!(
372+
"Arrow error: Compute error: {ARITHMETIC_OVERFLOW_ERROR}"
373+
)
367374
);
368375
}
369376
_ => unreachable!(),

‎datafusion/sqllogictest/test_files/spark/math/abs.slt‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -42,16 +42,16 @@ select abs((-128)::TINYINT), abs((-32768)::SMALLINT), abs((-2147483648)::INT), a
4242
statement ok
4343
set datafusion.execution.enable_ansi_mode = true;
4444

45-
query error DataFusion error: Arrow error: Compute error: Int8 overflow on abs\(\-128\)
45+
query error DataFusion error: Arrow error: Compute error: \[ARITHMETIC_OVERFLOW\] overflow\. If necessary set "spark\.sql\.ansi\.enabled" to "false" to bypass this error\. SQLSTATE: 22003
4646
select abs((-128)::TINYINT);
4747

48-
query error DataFusion error: Arrow error: Compute error: Int16 overflow on abs\(\-32768\)
48+
query error DataFusion error: Arrow error: Compute error: \[ARITHMETIC_OVERFLOW\] overflow\. If necessary set "spark\.sql\.ansi\.enabled" to "false" to bypass this error\. SQLSTATE: 22003
4949
select abs((-32768)::SMALLINT);
5050

51-
query error DataFusion error: Arrow error: Compute error: Int32 overflow on abs\(\-2147483648\)
51+
query error DataFusion error: Arrow error: Compute error: \[ARITHMETIC_OVERFLOW\] overflow\. If necessary set "spark\.sql\.ansi\.enabled" to "false" to bypass this error\. SQLSTATE: 22003
5252
select abs((-2147483648)::INT);
5353

54-
query error DataFusion error: Arrow error: Compute error: Int64 overflow on abs\(\-9223372036854775808\)
54+
query error DataFusion error: Arrow error: Compute error: \[ARITHMETIC_OVERFLOW\] overflow\. If necessary set "spark\.sql\.ansi\.enabled" to "false" to bypass this error\. SQLSTATE: 22003
5555
select abs((-9223372036854775808)::BIGINT);
5656

5757
statement ok
@@ -126,16 +126,16 @@ NULL
126126
statement ok
127127
set datafusion.execution.enable_ansi_mode = true;
128128

129-
query error DataFusion error: Arrow error: Compute error: Int8Array overflow on abs\(\-128\)
129+
query error DataFusion error: Arrow error: Compute error: \[ARITHMETIC_OVERFLOW\] overflow\. If necessary set "spark\.sql\.ansi\.enabled" to "false" to bypass this error\. SQLSTATE: 22003
130130
SELECT abs(a) FROM (VALUES (-127::TINYINT), ((-128)::TINYINT)) AS t(a);
131131

132-
query error DataFusion error: Arrow error: Compute error: Int16Array overflow on abs\(\-32768\)
132+
query error DataFusion error: Arrow error: Compute error: \[ARITHMETIC_OVERFLOW\] overflow\. If necessary set "spark\.sql\.ansi\.enabled" to "false" to bypass this error\. SQLSTATE: 22003
133133
select abs(a) FROM (VALUES (-32767::SMALLINT), ((-32768)::SMALLINT)) AS t(a);
134134

135-
query error DataFusion error: Arrow error: Compute error: Int32Array overflow on abs\(\-2147483648\)
135+
query error DataFusion error: Arrow error: Compute error: \[ARITHMETIC_OVERFLOW\] overflow\. If necessary set "spark\.sql\.ansi\.enabled" to "false" to bypass this error\. SQLSTATE: 22003
136136
select abs(a) FROM (VALUES (-2147483647::INT), ((-2147483648)::INT)) AS t(a);
137137

138-
query error DataFusion error: Arrow error: Compute error: Int64Array overflow on abs\(\-9223372036854775808\)
138+
query error DataFusion error: Arrow error: Compute error: \[ARITHMETIC_OVERFLOW\] overflow\. If necessary set "spark\.sql\.ansi\.enabled" to "false" to bypass this error\. SQLSTATE: 22003
139139
select abs(a) FROM (VALUES (-9223372036854775807::BIGINT), ((-9223372036854775808)::BIGINT)) AS t(a);
140140

141141
statement ok

0 commit comments

Comments
 (0)