Skip to content

Commit d336e0e

Browse files
committed
fix(spark): apply review feedback on pmod
Uses `decimal_coercion` directly at the call site and drops `pmod_computation_type`: only a decimal pair can reach that branch, since `Numeric(2)` already gives every other combination a common type. Returns a null scalar rather than building a null array for the null short-circuit. Removes comments that were wrong or unnecessary: a doc paragraph referencing `SparkPmod::coerce_types`, which this PR deletes; a claim that the computation type is wide enough by construction alongside a justification for overflow handling; and a pointer to an ANSI assertion "further down" that sits above it. Drops three sqllogictest cases that were duplicated in two places, and the value assertions that only restated the type assertions beside them.
1 parent ff84872 commit d336e0e

2 files changed

Lines changed: 19 additions & 62 deletions

File tree

‎datafusion/spark/src/function/math/modulus.rs‎

Lines changed: 18 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ use arrow::error::ArrowError;
3131
use datafusion_common::{Result, ScalarValue, assert_eq_or_internal_err, exec_err};
3232
use datafusion_expr::{
3333
Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature,
34-
TypeSignatureClass, Volatility, binary::binary_numeric_coercion,
34+
TypeSignatureClass, Volatility, binary::decimal_coercion,
3535
};
3636

3737
/// Returns a one element array holding negative zero, for the floating point
@@ -113,10 +113,7 @@ pub fn spark_mod(
113113
/// precision = min(p1 - s1, p2 - s2) + scale
114114
/// ```
115115
///
116-
/// The rule is applied to the *declared* argument types. Collapsing both
117-
/// arguments to a common decimal first would make the two precisions equal and
118-
/// the rule would degenerate to the input precision, which is why
119-
/// [`SparkPmod::coerce_types`] leaves decimal arguments intact.
116+
/// The rule is applied to the input argument types.
120117
fn pmod_decimal_result_type(p1: u8, s1: i8, p2: u8, s2: i8) -> DataType {
121118
let scale = s1.max(s2);
122119
let whole_digits = (i32::from(p1) - i32::from(s1)).min(i32::from(p2) - i32::from(s2));
@@ -125,19 +122,6 @@ fn pmod_decimal_result_type(p1: u8, s1: i8, p2: u8, s2: i8) -> DataType {
125122
DataType::Decimal128(precision as u8, scale)
126123
}
127124

128-
/// The type `pmod` computes in, which is not always the type it returns.
129-
///
130-
/// Spark's result type is narrower than the dividend, so the operands cannot be
131-
/// cast to it before the remainder is taken without overflowing the dividend.
132-
/// The computation therefore runs in a common type wide enough for both, and
133-
/// the result is narrowed afterwards.
134-
fn pmod_computation_type(lhs: &DataType, rhs: &DataType) -> Result<DataType> {
135-
match binary_numeric_coercion(lhs, rhs) {
136-
Some(computation_type) => Ok(computation_type),
137-
None => exec_err!("pmod does not support ({lhs}, {rhs})"),
138-
}
139-
}
140-
141125
/// Spark-compatible `pmod` function
142126
/// In ANSI mode, division by zero throws an error.
143127
/// In legacy mode, division by zero returns NULL (Spark behavior).
@@ -152,22 +136,28 @@ pub fn spark_pmod(
152136
// A null argument is passed through uncoerced by `Coercible` (#19458), so
153137
// it still carries `DataType::Null` here. Every operation below needs a
154138
// concrete numeric type, and the answer is null regardless.
139+
// Need to handle nulls separately as they are pass through by the signature
155140
if args.iter().any(|arg| arg.data_type() == &DataType::Null) {
156-
return Ok(ColumnarValue::Array(new_null_array(
141+
return Ok(ColumnarValue::Scalar(ScalarValue::try_new_null(
157142
result_type,
158-
args[0].len(),
159-
)));
143+
)?));
160144
}
161145

162146
let (left, right): (ArrayRef, ArrayRef) =
163147
if args[0].data_type() == args[1].data_type() {
164148
(Arc::clone(&args[0]), Arc::clone(&args[1]))
165149
} else {
166-
let computation_type =
167-
pmod_computation_type(args[0].data_type(), args[1].data_type())?;
168-
// The computation type is wide enough for both operands by
169-
// construction, so widening must not silently null on overflow the
170-
// way arrow's default (`safe: true`) cast would.
150+
// Only a decimal pair reaches here: `Numeric(2)` already gives
151+
// the other types a common type.
152+
let Some(computation_type) =
153+
decimal_coercion(args[0].data_type(), args[1].data_type())
154+
else {
155+
return exec_err!(
156+
"pmod does not support ({}, {})",
157+
args[0].data_type(),
158+
args[1].data_type()
159+
);
160+
};
171161
let widen = CastOptions {
172162
safe: false,
173163
..Default::default()
@@ -302,21 +292,15 @@ impl ScalarUDFImpl for SparkPmod {
302292
(DataType::Decimal128(p1, s1), DataType::Decimal128(p2, s2)) => {
303293
Ok(pmod_decimal_result_type(*p1, *s1, *p2, *s2))
304294
}
305-
// `Coercible` matches a null argument and passes it through
306-
// uncoerced (#19458), so an untyped NULL reaches here rather than
307-
// being folded by `Numeric(2)`. `mod` answers `Float64` for two
308-
// untyped nulls and the other side's type when only one is null;
309-
// `pmod` did too, so that behaviour is kept explicitly here.
295+
// Need to handle nulls explicitly, see: https://github.com/apache/datafusion/issues/19458
296+
// We align with the behaviour of `mod`
310297
(DataType::Null, DataType::Null) => Ok(DataType::Float64),
311298
(DataType::Null, other) | (other, DataType::Null) => Ok(other.clone()),
312-
// Arrow's rem function handles type promotion for the rest
313299
_ => Ok(arg_types[0].clone()),
314300
}
315301
}
316302

317303
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
318-
// The Spark result type was already derived in `return_type`, so it is
319-
// read back here rather than recomputed from the argument arrays.
320304
spark_pmod(
321305
&args.args,
322306
args.config_options.execution.enable_ansi_mode,

‎datafusion/sqllogictest/test_files/spark/math/pmod.slt‎

Lines changed: 1 addition & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -141,21 +141,11 @@ SELECT arrow_typeof(pmod(NULL, NULL));
141141
----
142142
Float64
143143

144-
query R
145-
SELECT pmod(NULL, NULL);
146-
----
147-
NULL
148-
149144
query T
150145
SELECT arrow_typeof(pmod(2.5::decimal(3,1), NULL));
151146
----
152147
Decimal128(3, 1)
153148

154-
query R
155-
SELECT pmod(2.5::decimal(3,1), NULL);
156-
----
157-
NULL
158-
159149
# An untyped NULL beside a typed non-decimal argument takes the Numeric path,
160150
# which cannot coerce the pair. `mod` rejects it the same way.
161151
statement error DataFusion error: Error during planning: Internal error: Function 'pmod' failed to match any signature
@@ -374,7 +364,7 @@ SELECT pmod(99.9::decimal(3,1), 2.5::decimal(2,1));
374364
# A remainder is bounded by the divisor, not by the result type, so a divisor
375365
# wider than the dividend can produce a value the result type cannot hold. Spark
376366
# wraps decimal arithmetic in CheckOverflow(nullOnOverflow = !ansiEnabled), so
377-
# this is NULL in legacy mode and an error under ANSI (asserted further down).
367+
# this is NULL in legacy mode; the ANSI error is asserted in the ANSI block above.
378368
query T
379369
SELECT arrow_typeof(pmod(-0.1::decimal(3,1), 9999.9::decimal(5,1)));
380370
----
@@ -385,23 +375,6 @@ SELECT pmod(-0.1::decimal(3,1), 9999.9::decimal(5,1));
385375
----
386376
NULL
387377

388-
# An untyped NULL pair falls back to Float64, as it did under Signature::numeric
389-
query T
390-
SELECT arrow_typeof(pmod(NULL, NULL));
391-
----
392-
Float64
393-
394-
query R
395-
SELECT pmod(NULL, NULL);
396-
----
397-
NULL
398-
399-
# A null divisor takes the dividend's type
400-
query T
401-
SELECT arrow_typeof(pmod(2.5::decimal(3,1), NULL));
402-
----
403-
Decimal128(3, 1)
404-
405378
# PMOD tests with different integer types
406379
query I
407380
SELECT pmod(10::int8, 3::int8) as pmod_int8_1;

0 commit comments

Comments
 (0)