Skip to content

Commit ff84872

Browse files
committed
fix(spark): use one_of signature and decide pmod's null result types explicitly
Replaces the hand-written `coerce_types` with the `one_of` signature suggested in review: Coercible([Decimal, Decimal]) Numeric(2) `Coercible` matches a null argument and passes it through uncoerced (#19458), so a decimal pair still reaches `return_type` with its declared precision and scale, while everything else falls to `Numeric(2)`. That drops `pmod_numeric_coercion`, which existed only to reproduce `Numeric`'s fold. The null cases the coercion API cannot express are handled in `return_type` instead: two untyped nulls answer `Float64`, and one untyped null keeps the other side's type. Both match what `mod` returns, verified directly: SELECT arrow_typeof(mod(NULL, NULL)); -- Float64 SELECT arrow_typeof(mod(2.5::decimal(3,1), NULL)); -- Decimal128(3, 1) Because the null is no longer coerced away, `spark_pmod` can now receive `DataType::Null` arrays, so it short-circuits to a null array of the result type rather than failing to build a zero scalar. `pmod(NULL, 3::int)` does not plan under `Numeric(2)`. That matches both `mod` today and `pmod` before this PR, which used `Signature::numeric(2)`, so it is covered by a `statement error` test rather than treated as a regression.
1 parent e363e12 commit ff84872

2 files changed

Lines changed: 62 additions & 62 deletions

File tree

‎datafusion/spark/src/function/math/modulus.rs‎

Lines changed: 33 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,10 @@ use arrow::compute::kernels::{
2828
};
2929
use arrow::datatypes::{DECIMAL128_MAX_PRECISION, DataType};
3030
use arrow::error::ArrowError;
31-
use datafusion_common::types::NativeType;
32-
use datafusion_common::{
33-
Result, ScalarValue, assert_eq_or_internal_err, exec_err, plan_err,
34-
};
31+
use datafusion_common::{Result, ScalarValue, assert_eq_or_internal_err, exec_err};
3532
use datafusion_expr::{
36-
ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
37-
binary::binary_numeric_coercion,
33+
Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature,
34+
TypeSignatureClass, Volatility, binary::binary_numeric_coercion,
3835
};
3936

4037
/// Returns a one element array holding negative zero, for the floating point
@@ -141,43 +138,6 @@ fn pmod_computation_type(lhs: &DataType, rhs: &DataType) -> Result<DataType> {
141138
}
142139
}
143140

144-
/// The coercion `Signature::numeric` applied before `pmod` moved to
145-
/// [`Signature::user_defined`], reproduced so that only the decimal pair below
146-
/// changes behaviour.
147-
///
148-
/// A null argument is skipped rather than coerced, and a call still typed null
149-
/// afterwards falls back to `Float64`; both match `TypeSignature::Numeric` in
150-
/// `datafusion_expr::type_coercion::functions`.
151-
fn pmod_numeric_coercion(lhs: &DataType, rhs: &DataType) -> Result<Vec<DataType>> {
152-
let mut valid_type = lhs.clone();
153-
154-
let rhs_native: NativeType = rhs.into();
155-
if rhs_native != NativeType::Null {
156-
if !rhs_native.is_numeric() {
157-
return plan_err!(
158-
"Function 'pmod' expects Numeric but received {rhs_native}"
159-
);
160-
}
161-
match binary_numeric_coercion(&valid_type, rhs) {
162-
Some(coerced_type) => valid_type = coerced_type,
163-
None => {
164-
return plan_err!(
165-
"For function 'pmod' {valid_type} and {rhs} are not coercible to a common numeric type"
166-
);
167-
}
168-
}
169-
}
170-
171-
let valid_native: NativeType = valid_type.clone().into();
172-
if valid_native == NativeType::Null {
173-
valid_type = DataType::Float64;
174-
} else if !valid_native.is_numeric() {
175-
return plan_err!("Function 'pmod' expects Numeric but received {valid_native}");
176-
}
177-
178-
Ok(vec![valid_type.clone(), valid_type])
179-
}
180-
181141
/// Spark-compatible `pmod` function
182142
/// In ANSI mode, division by zero throws an error.
183143
/// In legacy mode, division by zero returns NULL (Spark behavior).
@@ -189,6 +149,16 @@ pub fn spark_pmod(
189149
assert_eq_or_internal_err!(args.len(), 2, "pmod expects exactly two arguments");
190150
let args = ColumnarValue::values_to_arrays(args)?;
191151

152+
// A null argument is passed through uncoerced by `Coercible` (#19458), so
153+
// it still carries `DataType::Null` here. Every operation below needs a
154+
// concrete numeric type, and the answer is null regardless.
155+
if args.iter().any(|arg| arg.data_type() == &DataType::Null) {
156+
return Ok(ColumnarValue::Array(new_null_array(
157+
result_type,
158+
args[0].len(),
159+
)));
160+
}
161+
192162
let (left, right): (ArrayRef, ArrayRef) =
193163
if args[0].data_type() == args[1].data_type() {
194164
(Arc::clone(&args[0]), Arc::clone(&args[1]))
@@ -295,7 +265,19 @@ impl Default for SparkPmod {
295265
impl SparkPmod {
296266
pub fn new() -> Self {
297267
Self {
298-
signature: Signature::user_defined(Volatility::Immutable),
268+
signature: Signature::one_of(
269+
vec![
270+
// A decimal pair must reach `return_type` with the
271+
// precision and scale as written, since Spark defines
272+
// `Pmod.resultDecimalType` on the declared arguments.
273+
TypeSignature::Coercible(vec![
274+
Coercion::new_exact(TypeSignatureClass::Decimal),
275+
Coercion::new_exact(TypeSignatureClass::Decimal),
276+
]),
277+
TypeSignature::Numeric(2),
278+
],
279+
Volatility::Immutable,
280+
),
299281
}
300282
}
301283
}
@@ -320,29 +302,18 @@ impl ScalarUDFImpl for SparkPmod {
320302
(DataType::Decimal128(p1, s1), DataType::Decimal128(p2, s2)) => {
321303
Ok(pmod_decimal_result_type(*p1, *s1, *p2, *s2))
322304
}
305+
// `Coercible` matches a null argument and passes it through
306+
// uncoerced (#19458), so an untyped NULL reaches here rather than
307+
// being folded by `Numeric(2)`. `mod` answers `Float64` for two
308+
// untyped nulls and the other side's type when only one is null;
309+
// `pmod` did too, so that behaviour is kept explicitly here.
310+
(DataType::Null, DataType::Null) => Ok(DataType::Float64),
311+
(DataType::Null, other) | (other, DataType::Null) => Ok(other.clone()),
323312
// Arrow's rem function handles type promotion for the rest
324313
_ => Ok(arg_types[0].clone()),
325314
}
326315
}
327316

328-
fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
329-
if arg_types.len() != 2 {
330-
return plan_err!(
331-
"Function 'pmod' expects 2 arguments but received {}",
332-
arg_types.len()
333-
);
334-
}
335-
336-
match (&arg_types[0], &arg_types[1]) {
337-
// Spark applies resultDecimalType to the declared argument types, so
338-
// these are left alone; spark_pmod widens them for the computation.
339-
(DataType::Decimal128(_, _), DataType::Decimal128(_, _)) => {
340-
Ok(arg_types.to_vec())
341-
}
342-
(lhs, rhs) => pmod_numeric_coercion(lhs, rhs),
343-
}
344-
}
345-
346317
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
347318
// The Spark result type was already derived in `return_type`, so it is
348319
// read back here rather than recomputed from the argument arrays.

‎datafusion/sqllogictest/test_files/spark/math/pmod.slt‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,35 @@ SELECT pmod(NULL::int, NULL::int) as pmod_null_3;
132132
----
133133
NULL
134134

135+
# An untyped NULL matches the decimal signature and is passed through
136+
# uncoerced (apache/datafusion#19458), so these types are decided explicitly
137+
# rather than by coercion. `mod` answers Float64 for two untyped nulls and
138+
# keeps the other side's type when only one is null; pmod matches it.
139+
query T
140+
SELECT arrow_typeof(pmod(NULL, NULL));
141+
----
142+
Float64
143+
144+
query R
145+
SELECT pmod(NULL, NULL);
146+
----
147+
NULL
148+
149+
query T
150+
SELECT arrow_typeof(pmod(2.5::decimal(3,1), NULL));
151+
----
152+
Decimal128(3, 1)
153+
154+
query R
155+
SELECT pmod(2.5::decimal(3,1), NULL);
156+
----
157+
NULL
158+
159+
# An untyped NULL beside a typed non-decimal argument takes the Numeric path,
160+
# which cannot coerce the pair. `mod` rejects it the same way.
161+
statement error DataFusion error: Error during planning: Internal error: Function 'pmod' failed to match any signature
162+
SELECT pmod(NULL, 3::int);
163+
135164
# PMOD tests with large integers
136165
query I
137166
SELECT pmod(100::int, 30::int) as pmod_large_1;

0 commit comments

Comments
 (0)