Skip to content

Commit 07126ed

Browse files
committed
fix(spark): correct mod ANSI zero divisor and negative zero handling
In ANSI mode a floating-point zero divisor quietly returned NaN because Arrow's rem only reports division by zero for integer and decimal types, and a -0.0 divisor went unrecognised in both modes because Arrow's floating-point comparisons order totally, so -0.0 is distinct from 0.0. try_rem now detects zero divisors itself via an is_zero helper that counts -0.0 as zero, raises for a zero divisor of any numeric type in ANSI mode (masked by the validity of the dividend, since Spark's remainder is null intolerant), and substitutes NULL for zero divisors before calling Arrow's rem so the kernel never sees one. Closes apache#23894.
1 parent b9399dc commit 07126ed

3 files changed

Lines changed: 302 additions & 19 deletions

File tree

‎datafusion/spark/src/function/math/modulus.rs‎

Lines changed: 246 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -15,41 +15,77 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
use arrow::array::{Scalar, new_null_array};
18+
use arrow::array::{ArrayRef, BooleanArray, Scalar, new_null_array};
1919
use arrow::compute::kernels::numeric::add;
2020
use arrow::compute::kernels::{
21+
boolean::{and, is_not_null, or},
2122
cmp::{eq, lt},
22-
numeric::rem,
23+
numeric::{neg, rem},
2324
zip::zip,
2425
};
2526
use arrow::datatypes::DataType;
27+
use arrow::error::ArrowError;
2628
use datafusion_common::{Result, ScalarValue, assert_eq_or_internal_err};
2729
use datafusion_expr::{
2830
ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
2931
};
3032

33+
/// Returns a one element array holding negative zero, for the floating point
34+
/// types only.
35+
///
36+
/// Arrow's comparison kernels order floating point values totally, so `-0.0`
37+
/// compares as distinct from, and less than, `0.0`. Java, and therefore Spark,
38+
/// treats `-0.0` as equal to zero. The helper below uses this to restore the
39+
/// IEEE 754 answer.
40+
fn negative_zero(data_type: &DataType) -> Result<Option<ArrayRef>> {
41+
match data_type {
42+
DataType::Float16 | DataType::Float32 | DataType::Float64 => {
43+
let zero = ScalarValue::new_zero(data_type)?.to_array()?;
44+
Ok(Some(neg(zero.as_ref())?))
45+
}
46+
_ => Ok(None),
47+
}
48+
}
49+
50+
/// Rows of `values` that equal zero, counting `-0.0` as zero.
51+
fn is_zero(values: &ArrayRef) -> Result<BooleanArray> {
52+
let zero = ScalarValue::new_zero(values.data_type())?.to_array()?;
53+
let mask = eq(values, &Scalar::new(zero))?;
54+
match negative_zero(values.data_type())? {
55+
Some(negative_zero) => Ok(or(&mask, &eq(values, &Scalar::new(negative_zero))?)?),
56+
None => Ok(mask),
57+
}
58+
}
59+
3160
/// Computes `rem(left, right)` with divide-by-zero handling.
32-
/// In ANSI mode, any zero divisor causes an error.
33-
/// In legacy mode (ANSI off), zero divisors are replaced with NULL before
34-
/// computing the remainder, so those positions return NULL while others
35-
/// compute normally.
61+
/// In ANSI mode, a zero divisor of any numeric type causes an error, with
62+
/// `-0.0` counting as zero; a row whose dividend is NULL never raises, to
63+
/// match Spark's null-intolerant remainder. In legacy mode (ANSI off), zero
64+
/// divisors are replaced with NULL before computing the remainder, so those
65+
/// positions return NULL while others compute normally.
3666
fn try_rem(
37-
left: &arrow::array::ArrayRef,
38-
right: &arrow::array::ArrayRef,
67+
left: &ArrayRef,
68+
right: &ArrayRef,
3969
enable_ansi_mode: bool,
40-
) -> Result<arrow::array::ArrayRef> {
70+
) -> Result<ArrayRef> {
71+
let divisor_is_zero = is_zero(right)?;
72+
// Null out zero divisors so that the remainder kernels never see one:
73+
// division by zero then returns NULL instead of erroring (integers) or
74+
// returning NaN (floats). ANSI mode reports the error itself below, so
75+
// this substitution is harmless on rows that must raise.
76+
let null = Scalar::new(new_null_array(right.data_type(), 1));
77+
let safe_right = zip(&divisor_is_zero, &null, right)?;
4178
if enable_ansi_mode {
42-
Ok(rem(left, right)?)
43-
} else {
44-
// In legacy mode, null out zero divisors so that division by zero
45-
// returns NULL instead of erroring (integers) or returning NaN (floats).
46-
let zero = ScalarValue::new_zero(right.data_type())?.to_array()?;
47-
let zero = Scalar::new(zero);
48-
let null = Scalar::new(new_null_array(right.data_type(), 1));
49-
let is_zero = eq(right, &zero)?;
50-
let safe_right = zip(&is_zero, &null, right)?;
51-
Ok(rem(left, &safe_right)?)
79+
// Spark's remainder expressions are null intolerant, so a row whose
80+
// dividend is NULL evaluates to NULL and never raises, even when the
81+
// divisor on that row is zero. Mask the check by the validity of the
82+
// dividend to match.
83+
let raises = and(&divisor_is_zero, &is_not_null(left.as_ref())?)?;
84+
if raises.iter().flatten().any(|raises| raises) {
85+
return Err(ArrowError::DivideByZero.into());
86+
}
5287
}
88+
Ok(rem(left, &safe_right)?)
5389
}
5490

5591
/// Spark-compatible `mod` function
@@ -418,6 +454,101 @@ mod test {
418454
assert!(result.is_err());
419455
}
420456

457+
#[test]
458+
fn test_mod_zero_division_ansi_float() {
459+
// In ANSI mode a zero divisor of any numeric type must raise,
460+
// including floating point, where Arrow's `rem` follows IEEE 754
461+
// and quietly returns NaN (#23894)
462+
let left = Float64Array::from(vec![Some(10.5), Some(7.2)]);
463+
let right = Float64Array::from(vec![Some(0.0), Some(2.0)]);
464+
465+
let left_value = ColumnarValue::Array(Arc::new(left));
466+
let right_value = ColumnarValue::Array(Arc::new(right));
467+
468+
let result = spark_mod(&[left_value, right_value], true);
469+
assert!(result.is_err());
470+
}
471+
472+
#[test]
473+
fn test_mod_negative_zero_divisor_legacy() {
474+
// `-0.0` counts as a zero divisor, so it returns NULL in legacy
475+
// mode rather than NaN (#23894)
476+
let left = Float64Array::from(vec![Some(10.5), Some(7.5)]);
477+
let right = Float64Array::from(vec![Some(-0.0), Some(2.0)]);
478+
479+
let left_value = ColumnarValue::Array(Arc::new(left));
480+
let right_value = ColumnarValue::Array(Arc::new(right));
481+
482+
let result = spark_mod(&[left_value, right_value], false).unwrap();
483+
484+
if let ColumnarValue::Array(result_array) = result {
485+
let result_float64 = result_array
486+
.as_any()
487+
.downcast_ref::<Float64Array>()
488+
.unwrap();
489+
assert!(result_float64.is_null(0)); // 10.5 % -0.0 = NULL
490+
assert_eq!(result_float64.value(1), 1.5); // 7.5 % 2.0 = 1.5
491+
} else {
492+
panic!("Expected array result");
493+
}
494+
}
495+
496+
#[test]
497+
fn test_mod_negative_zero_divisor_ansi() {
498+
// `-0.0` counts as a zero divisor, so it raises in ANSI mode (#23894)
499+
let left = Float64Array::from(vec![Some(10.5)]);
500+
let right = Float64Array::from(vec![Some(-0.0)]);
501+
502+
let left_value = ColumnarValue::Array(Arc::new(left));
503+
let right_value = ColumnarValue::Array(Arc::new(right));
504+
505+
let result = spark_mod(&[left_value, right_value], true);
506+
assert!(result.is_err());
507+
}
508+
509+
#[test]
510+
fn test_mod_zero_division_ansi_null_dividend() {
511+
// Spark's remainder expressions are null intolerant: a NULL dividend
512+
// short-circuits to NULL before the divisor is validated, so a zero
513+
// divisor on such a row must not raise, even in ANSI mode (#23894)
514+
let left = Int32Array::from(vec![None, Some(10)]);
515+
let right = Int32Array::from(vec![Some(0), Some(3)]);
516+
517+
let left_value = ColumnarValue::Array(Arc::new(left));
518+
let right_value = ColumnarValue::Array(Arc::new(right));
519+
520+
let result = spark_mod(&[left_value, right_value], true).unwrap();
521+
522+
if let ColumnarValue::Array(result_array) = result {
523+
let result_int32 =
524+
result_array.as_any().downcast_ref::<Int32Array>().unwrap();
525+
assert!(result_int32.is_null(0)); // NULL % 0 = NULL (no error)
526+
assert_eq!(result_int32.value(1), 1); // 10 % 3 = 1
527+
} else {
528+
panic!("Expected array result");
529+
}
530+
531+
// Same for floating point
532+
let left = Float64Array::from(vec![None, Some(10.5)]);
533+
let right = Float64Array::from(vec![Some(0.0), Some(2.0)]);
534+
535+
let left_value = ColumnarValue::Array(Arc::new(left));
536+
let right_value = ColumnarValue::Array(Arc::new(right));
537+
538+
let result = spark_mod(&[left_value, right_value], true).unwrap();
539+
540+
if let ColumnarValue::Array(result_array) = result {
541+
let result_float64 = result_array
542+
.as_any()
543+
.downcast_ref::<Float64Array>()
544+
.unwrap();
545+
assert!(result_float64.is_null(0)); // NULL % 0.0 = NULL (no error)
546+
assert!((result_float64.value(1) - 0.5).abs() < f64::EPSILON); // 10.5 % 2.0 = 0.5
547+
} else {
548+
panic!("Expected array result");
549+
}
550+
}
551+
421552
// PMOD tests
422553
#[test]
423554
fn test_pmod_int32() {
@@ -645,6 +776,102 @@ mod test {
645776
assert!(result.is_err());
646777
}
647778

779+
#[test]
780+
fn test_pmod_zero_division_ansi_float() {
781+
// pmod routes through `try_rem` twice, so it needs the same coverage
782+
// as mod: in ANSI mode a zero divisor of any numeric type must
783+
// raise, including floating point, where Arrow's `rem` follows
784+
// IEEE 754 and quietly returns NaN (#23894)
785+
let left = Float64Array::from(vec![Some(10.5), Some(7.2)]);
786+
let right = Float64Array::from(vec![Some(0.0), Some(2.0)]);
787+
788+
let left_value = ColumnarValue::Array(Arc::new(left));
789+
let right_value = ColumnarValue::Array(Arc::new(right));
790+
791+
let result = spark_pmod(&[left_value, right_value], true);
792+
assert!(result.is_err());
793+
}
794+
795+
#[test]
796+
fn test_pmod_negative_zero_divisor_legacy() {
797+
// `-0.0` counts as a zero divisor, so it returns NULL in legacy
798+
// mode rather than NaN (#23894)
799+
let left = Float64Array::from(vec![Some(10.5), Some(-7.5)]);
800+
let right = Float64Array::from(vec![Some(-0.0), Some(2.0)]);
801+
802+
let left_value = ColumnarValue::Array(Arc::new(left));
803+
let right_value = ColumnarValue::Array(Arc::new(right));
804+
805+
let result = spark_pmod(&[left_value, right_value], false).unwrap();
806+
807+
if let ColumnarValue::Array(result_array) = result {
808+
let result_float64 = result_array
809+
.as_any()
810+
.downcast_ref::<Float64Array>()
811+
.unwrap();
812+
assert!(result_float64.is_null(0)); // 10.5 pmod -0.0 = NULL
813+
assert!((result_float64.value(1) - 0.5).abs() < f64::EPSILON); // -7.5 pmod 2.0 = 0.5
814+
} else {
815+
panic!("Expected array result");
816+
}
817+
}
818+
819+
#[test]
820+
fn test_pmod_negative_zero_divisor_ansi() {
821+
// `-0.0` counts as a zero divisor, so it raises in ANSI mode (#23894)
822+
let left = Float64Array::from(vec![Some(10.5)]);
823+
let right = Float64Array::from(vec![Some(-0.0)]);
824+
825+
let left_value = ColumnarValue::Array(Arc::new(left));
826+
let right_value = ColumnarValue::Array(Arc::new(right));
827+
828+
let result = spark_pmod(&[left_value, right_value], true);
829+
assert!(result.is_err());
830+
}
831+
832+
#[test]
833+
fn test_pmod_zero_division_ansi_null_dividend() {
834+
// Spark's remainder expressions are null intolerant: a NULL dividend
835+
// short-circuits to NULL before the divisor is validated, so a zero
836+
// divisor on such a row must not raise, even in ANSI mode (#23894)
837+
let left = Int32Array::from(vec![None, Some(10)]);
838+
let right = Int32Array::from(vec![Some(0), Some(3)]);
839+
840+
let left_value = ColumnarValue::Array(Arc::new(left));
841+
let right_value = ColumnarValue::Array(Arc::new(right));
842+
843+
let result = spark_pmod(&[left_value, right_value], true).unwrap();
844+
845+
if let ColumnarValue::Array(result_array) = result {
846+
let result_int32 =
847+
result_array.as_any().downcast_ref::<Int32Array>().unwrap();
848+
assert!(result_int32.is_null(0)); // NULL pmod 0 = NULL (no error)
849+
assert_eq!(result_int32.value(1), 1); // 10 pmod 3 = 1
850+
} else {
851+
panic!("Expected array result");
852+
}
853+
854+
// Same for floating point
855+
let left = Float64Array::from(vec![None, Some(10.5)]);
856+
let right = Float64Array::from(vec![Some(0.0), Some(2.0)]);
857+
858+
let left_value = ColumnarValue::Array(Arc::new(left));
859+
let right_value = ColumnarValue::Array(Arc::new(right));
860+
861+
let result = spark_pmod(&[left_value, right_value], true).unwrap();
862+
863+
if let ColumnarValue::Array(result_array) = result {
864+
let result_float64 = result_array
865+
.as_any()
866+
.downcast_ref::<Float64Array>()
867+
.unwrap();
868+
assert!(result_float64.is_null(0)); // NULL pmod 0.0 = NULL (no error)
869+
assert!((result_float64.value(1) - 0.5).abs() < f64::EPSILON); // 10.5 pmod 2.0 = 0.5
870+
} else {
871+
panic!("Expected array result");
872+
}
873+
}
874+
648875
#[test]
649876
fn test_pmod_negative_divisor() {
650877
// PMOD with negative divisor should still work like regular mod

‎datafusion/sqllogictest/test_files/spark/math/mod.slt‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,18 @@ SELECT MOD(10.5::float8, 0.0::float8) as mod_div_zero_float;
160160
----
161161
NULL
162162

163+
# A negative zero divisor counts as zero and returns NULL in legacy mode
164+
query R
165+
SELECT MOD(10.5::float8, -0.0::float8) as mod_div_zero_neg_float;
166+
----
167+
NULL
168+
169+
# A NULL dividend evaluates to NULL regardless of the divisor
170+
query I
171+
SELECT MOD(NULL::int, 0::int) as mod_null_dividend_legacy;
172+
----
173+
NULL
174+
163175
# Division by zero errors in ANSI mode
164176
statement ok
165177
set datafusion.execution.enable_ansi_mode = true;
@@ -170,6 +182,25 @@ SELECT MOD(10::int, 0::int);
170182
statement error DataFusion error: Arrow error: Divide by zero error
171183
SELECT MOD(-7::int, 0::int);
172184

185+
# A zero divisor of any numeric type raises, including floating point
186+
statement error DataFusion error: Arrow error: Divide by zero error
187+
SELECT MOD(10.5::float8, 0.0::float8);
188+
189+
# A negative zero divisor counts as zero and raises
190+
statement error DataFusion error: Arrow error: Divide by zero error
191+
SELECT MOD(10.5::float8, -0.0::float8);
192+
193+
# A NULL dividend short-circuits to NULL before the divisor is validated
194+
query I
195+
SELECT MOD(NULL::int, 0::int) as mod_null_dividend_ansi;
196+
----
197+
NULL
198+
199+
query R
200+
SELECT MOD(NULL::float8, 0.0::float8) as mod_null_dividend_ansi_float;
201+
----
202+
NULL
203+
173204
statement ok
174205
set datafusion.execution.enable_ansi_mode = false;
175206

‎datafusion/sqllogictest/test_files/spark/math/pmod.slt‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,12 @@ SELECT pmod(-7::int, 0::int) as pmod_zero_3;
7474
----
7575
NULL
7676

77+
# A negative zero divisor counts as zero and returns NULL in legacy mode
78+
query R
79+
SELECT pmod(10.5::float8, -0.0::float8) as pmod_div_zero_neg_float;
80+
----
81+
NULL
82+
7783
# Division by zero errors in ANSI mode
7884
statement ok
7985
set datafusion.execution.enable_ansi_mode = true;
@@ -84,6 +90,25 @@ SELECT pmod(10::int, 0::int);
8490
statement error DataFusion error: Arrow error: Divide by zero error
8591
SELECT pmod(-7::int, 0::int);
8692

93+
# A zero divisor of any numeric type raises, including floating point
94+
statement error DataFusion error: Arrow error: Divide by zero error
95+
SELECT pmod(10.5::float8, 0.0::float8);
96+
97+
# A negative zero divisor counts as zero and raises
98+
statement error DataFusion error: Arrow error: Divide by zero error
99+
SELECT pmod(10.5::float8, -0.0::float8);
100+
101+
# A NULL dividend short-circuits to NULL before the divisor is validated
102+
query I
103+
SELECT pmod(NULL::int, 0::int) as pmod_null_dividend_ansi;
104+
----
105+
NULL
106+
107+
query R
108+
SELECT pmod(NULL::float8, 0.0::float8) as pmod_null_dividend_ansi_float;
109+
----
110+
NULL
111+
87112
statement ok
88113
set datafusion.execution.enable_ansi_mode = false;
89114

0 commit comments

Comments
 (0)