Skip to content

Commit e7d7757

Browse files
committed
fix: compare a Remainder divisor with zero in Spark's ordering without ANSI
null_if_zero_primitive built Remainder's non-ANSI zero-divisor check as a plain BinaryExpr, so `x % -0.0` returned NULL only because DataFusion 55's apply_cmp folds signed zeros before Arrow's eq. The divisor normalization this PR removes used to hide that in Projects and Filters. The check now goes through spark_comparison, as Divide's guard does through the native comparison builder.
1 parent 46774ca commit e7d7757

2 files changed

Lines changed: 70 additions & 6 deletions

File tree

‎native/spark-expr/src/math_funcs/modulo_expr.rs‎

Lines changed: 66 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
use crate::{create_comet_physical_fun, IfExpr};
18+
use crate::{create_comet_physical_fun, spark_comparison, FloatOperands, IfExpr};
1919
use crate::{remainder_by_zero_error, Cast, EvalMode, SparkCastOptions};
2020
use arrow::array::{ArrayRef, ArrowNativeTypeOp, AsArray, PrimitiveArray};
2121
use arrow::compute::kernels::arity::try_binary;
@@ -27,7 +27,7 @@ use datafusion::common::{
2727
};
2828
use datafusion::config::ConfigOptions;
2929
use datafusion::execution::FunctionRegistry;
30-
use datafusion::physical_expr::expressions::{lit, BinaryExpr};
30+
use datafusion::physical_expr::expressions::lit;
3131
use datafusion::physical_expr::ScalarFunctionExpr;
3232
use datafusion::physical_expr_common::datum::{apply, apply_cmp_for_nested};
3333
use datafusion::{
@@ -232,12 +232,15 @@ fn null_if_zero_primitive(
232232

233233
// Create an expression like - `if (eval(expr) == Literal(0)) then NULL else eval(expr)`.
234234
// This expression evaluates to null for rows with zero values to prevent divide by zero
235-
// error.
236-
let eq_expr = Arc::new(BinaryExpr::new(
235+
// error. A float divisor is compared in Spark's ordering, where `-0.0` equals `0.0` as it
236+
// does in Spark's own check, while Arrow's `eq` tells the two zeros apart.
237+
let eq_expr = spark_comparison(
237238
Arc::<dyn PhysicalExpr>::clone(&expression),
238239
Operator::Eq,
239240
lit(zero),
240-
));
241+
input_schema,
242+
FloatOperands::Normalize,
243+
)?;
241244
let null_literal = lit(ScalarValue::try_new_null(&expr_data_type)?);
242245
let if_expr = Arc::new(IfExpr::new(eq_expr, null_literal, expression));
243246
Ok(if_expr)
@@ -286,12 +289,13 @@ fn create_modulo_scalar_function(
286289
#[cfg(test)]
287290
mod tests {
288291
use super::*;
292+
use crate::SparkComparison;
289293
use arrow::array::{
290294
Array, ArrayRef, Decimal128Array, Decimal128Builder, Float32Array, Float64Array,
291295
Int32Array, PrimitiveArray, RecordBatch,
292296
};
293297
use datafusion::logical_expr::ColumnarValue;
294-
use datafusion::physical_expr::expressions::{Column, Literal};
298+
use datafusion::physical_expr::expressions::{BinaryExpr, Column, Literal};
295299
use datafusion::prelude::SessionContext;
296300

297301
fn with_fail_on_error<F: Fn(bool)>(test_fn: F) {
@@ -593,6 +597,62 @@ mod tests {
593597
);
594598
}
595599

600+
#[test]
601+
fn test_modulo_divide_by_negative_zero_float_non_ansi() {
602+
// Spark treats `-0.0` as a zero divisor too, so without ANSI the result is null.
603+
run_float_modulo::<Float64Type>(
604+
DataType::Float64,
605+
Arc::new(Float64Array::from(vec![Some(1.0), Some(1.0), Some(1.0)])),
606+
Arc::new(Float64Array::from(vec![Some(-0.0), Some(0.0), Some(2.0)])),
607+
false,
608+
false,
609+
Some(Arc::new(Float64Array::from(vec![None, None, Some(1.0)]))),
610+
);
611+
run_float_modulo::<Float32Type>(
612+
DataType::Float32,
613+
Arc::new(Float32Array::from(vec![Some(1.0_f32), Some(1.0_f32)])),
614+
Arc::new(Float32Array::from(vec![Some(-0.0_f32), Some(2.0_f32)])),
615+
false,
616+
false,
617+
Some(Arc::new(Float32Array::from(vec![None, Some(1.0_f32)]))),
618+
);
619+
run_float_modulo_with_literal_divisor(
620+
Arc::new(Float64Array::from(vec![Some(1.0)])),
621+
ScalarValue::Float64(Some(-0.0)),
622+
false,
623+
false,
624+
Some(Arc::new(Float64Array::from(vec![None]))),
625+
);
626+
627+
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Float32, true)]));
628+
let lhs: ArrayRef = Arc::new(Float32Array::from(vec![Some(1.0_f32)]));
629+
let batch = RecordBatch::try_new(Arc::clone(&schema), vec![lhs]).unwrap();
630+
let modulo_expr = create_modulo_expr(
631+
Arc::new(Column::new("a", 0)),
632+
Arc::new(Literal::new(ScalarValue::Float32(Some(-0.0)))),
633+
DataType::Float32,
634+
schema,
635+
false,
636+
&SessionContext::new().state(),
637+
)
638+
.unwrap();
639+
let expected = Arc::new(Float32Array::from(vec![None]));
640+
verify_result(modulo_expr, batch, false, Some(expected));
641+
}
642+
643+
#[test]
644+
fn test_non_ansi_float_divisor_guard_compares_in_spark_ordering() {
645+
// DataFusion 55 folds `-0.0` into `0.0` before Arrow's `eq`, so the test above would pass
646+
// with a plain `BinaryExpr` guard too. The guard must not depend on that.
647+
let schema = Schema::new(vec![Field::new("b", DataType::Float64, true)]);
648+
let guard = null_if_zero_primitive(Arc::new(Column::new("b", 0)), &schema).unwrap();
649+
let condition = guard.children()[0];
650+
assert!(
651+
condition.downcast_ref::<SparkComparison>().is_some(),
652+
"unexpected guard condition: {condition}"
653+
);
654+
}
655+
596656
#[test]
597657
fn test_modulo_null_dividend_zero_divisor_float64_ansi() {
598658
// A null dividend must not raise even when the divisor is zero — Spark returns null

‎spark/src/test/resources/sql-tests/expressions/math/arithmetic.sql‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,3 +83,7 @@ SELECT a / b, a % b, d / 0.0 FROM test_div_zero
8383
-- `-0.0D` is a double literal with the sign bit set, and is a zero divisor too
8484
query
8585
SELECT d / -0.0D, d % -0.0D, d / double('-0.0'), d % double('-0.0') FROM test_div_zero
86+
87+
-- So is a column that holds `-0.0`, because Spark checks the divisor with `== 0`
88+
query
89+
SELECT 1.0D / (-d), 1.0D % (-d) FROM test_div_zero

0 commit comments

Comments
 (0)