Skip to content

Commit a34cc22

Browse files
KarpagamKarthikeyankosiew
authored andcommitted
feat: add Spark-compatible hypot function (apache#23774)
## Which issue does this PR close? - Closes apache#23770 - Part of #15914 ## Rationale for this change Spark provides [`hypot(expr1, expr2)`](https://spark.apache.org/docs/latest/api/sql/#hypot), which returns `sqrt(expr1^2 + expr2^2)` computed without intermediate overflow or underflow. It was not yet implemented in `datafusion-spark` — only an auto-generated test stub existed at `spark/math/hypot.slt` with its query commented out. ## What changes are included in this PR? - Add `SparkHypot` (implementing `ScalarUDFImpl`) in `datafusion/spark/src/function/math/hypot.rs`, backed by Rust's `f64::hypot` — the same overflow-safe algorithm as Java/Spark's `Math.hypot`. - Register it in `datafusion/spark/src/function/math/mod.rs`. - Enable the `hypot.slt` sqllogictest. The signature is `exact(Float64, Float64) -> Float64`, following the `datafusion-spark` convention of only accepting types Spark supports. Computation uses the Arrow `binary` kernel so NULL in either argument propagates to a NULL result, matching Spark. ## Are these changes tested? Yes — `datafusion/sqllogictest/test_files/spark/math/hypot.slt` covers: - scalar Pythagorean triples (`hypot(3, 4)` → 5, `hypot(5, 12)` → 13), - double inputs, - NULL propagation when either argument is NULL, - the array path (including a NULL row), - overflow-safety: `hypot(3e200, 4e200)` stays finite, whereas a naive `sqrt(a^2 + b^2)` would overflow to `Infinity`. ## Are there any user-facing changes? Yes — adds the Spark-compatible `hypot` scalar function to `datafusion-spark`. No breaking changes to public APIs.
1 parent 6407571 commit a34cc22

3 files changed

Lines changed: 200 additions & 4 deletions

File tree

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
use std::sync::Arc;
19+
20+
use arrow::array::{ArrayRef, AsArray, Float64Array};
21+
use arrow::compute::kernels::arity::binary;
22+
use arrow::datatypes::{DataType, Float64Type};
23+
use datafusion_common::Result;
24+
use datafusion_common::utils::take_function_args;
25+
use datafusion_expr::{
26+
ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
27+
};
28+
use datafusion_functions::utils::make_scalar_function;
29+
30+
/// Spark-compatible `hypot` function.
31+
///
32+
/// <https://spark.apache.org/docs/latest/api/sql/index.html#hypot>
33+
///
34+
/// Returns `sqrt(expr1^2 + expr2^2)` computed without intermediate overflow or
35+
/// underflow, matching Spark's use of `java.lang.Math.hypot`.
36+
#[derive(Debug, PartialEq, Eq, Hash)]
37+
pub struct SparkHypot {
38+
signature: Signature,
39+
}
40+
41+
impl Default for SparkHypot {
42+
fn default() -> Self {
43+
Self::new()
44+
}
45+
}
46+
47+
impl SparkHypot {
48+
pub fn new() -> Self {
49+
Self {
50+
// Spark only defines hypot over doubles
51+
signature: Signature::exact(
52+
vec![DataType::Float64, DataType::Float64],
53+
Volatility::Immutable,
54+
),
55+
}
56+
}
57+
}
58+
59+
impl ScalarUDFImpl for SparkHypot {
60+
fn name(&self) -> &str {
61+
"hypot"
62+
}
63+
64+
fn signature(&self) -> &Signature {
65+
&self.signature
66+
}
67+
68+
fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
69+
Ok(DataType::Float64)
70+
}
71+
72+
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
73+
make_scalar_function(spark_hypot, vec![])(&args.args)
74+
}
75+
}
76+
77+
fn spark_hypot(args: &[ArrayRef]) -> Result<ArrayRef> {
78+
let [x, y] = take_function_args("hypot", args)?;
79+
80+
let x = x.as_primitive::<Float64Type>();
81+
let y = y.as_primitive::<Float64Type>();
82+
let result: Float64Array = binary(x, y, |a, b| a.hypot(b))?;
83+
Ok(Arc::new(result))
84+
}

‎datafusion/spark/src/function/math/mod.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ pub mod expm1;
2222
pub mod factorial;
2323
pub mod floor;
2424
pub mod hex;
25+
pub mod hypot;
2526
pub mod modulus;
2627
pub mod negative;
2728
pub mod pow;
@@ -41,6 +42,7 @@ make_udf_function!(expm1::SparkExpm1, expm1);
4142
make_udf_function!(factorial::SparkFactorial, factorial);
4243
make_udf_function!(floor::SparkFloor, floor);
4344
make_udf_function!(hex::SparkHex, hex);
45+
make_udf_function!(hypot::SparkHypot, hypot);
4446
make_udf_function!(modulus::SparkMod, modulus);
4547
make_udf_function!(modulus::SparkPmod, pmod);
4648
make_udf_function!(pow::SparkPow, pow);
@@ -66,6 +68,7 @@ pub mod expr_fn {
6668
));
6769
export_functions!((floor, "Returns floor of expr.", arg1));
6870
export_functions!((hex, "Computes hex value of the given column.", arg1));
71+
export_functions!((hypot, "Returns sqrt(a^2 + b^2) without intermediate overflow or underflow.", arg1 arg2));
6972
export_functions!((modulus, "Returns the remainder of division of the first argument by the second argument.", arg1 arg2));
7073
export_functions!((pmod, "Returns the positive remainder of division of the first argument by the second argument.", arg1 arg2));
7174
export_functions!((
@@ -107,6 +110,7 @@ pub fn functions() -> Vec<Arc<ScalarUDF>> {
107110
factorial(),
108111
floor(),
109112
hex(),
113+
hypot(),
110114
modulus(),
111115
pmod(),
112116
pow(),

‎datafusion/sqllogictest/test_files/spark/math/hypot.slt‎

Lines changed: 112 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,115 @@
2121
# For more information, please see:
2222
# https://github.com/apache/datafusion/issues/15914
2323

24-
## Original Query: SELECT hypot(3, 4);
25-
## PySpark 3.5.5 Result: {'HYPOT(3, 4)': 5.0, 'typeof(HYPOT(3, 4))': 'double', 'typeof(3)': 'int', 'typeof(4)': 'int'}
26-
#query
27-
#SELECT hypot(3::int, 4::int);
24+
# Scalar: classic Pythagorean triples (3-4-5, 5-12-13)
25+
query R
26+
SELECT hypot(3, 4);
27+
----
28+
5
29+
30+
query R
31+
SELECT hypot(5, 12);
32+
----
33+
13
34+
35+
# Double inputs
36+
query R
37+
SELECT hypot(3.0::double, 4.0::double);
38+
----
39+
5
40+
41+
# NULL if either argument is NULL
42+
query R
43+
SELECT hypot(NULL::double, 4.0::double);
44+
----
45+
NULL
46+
47+
query R
48+
SELECT hypot(3.0::double, NULL::double);
49+
----
50+
NULL
51+
52+
# Array path, including a NULL row
53+
query R
54+
SELECT hypot(a, b) FROM (VALUES (3.0::double, 4.0::double), (6.0::double, 8.0::double), (NULL::double, 1.0::double)) AS t(a, b);
55+
----
56+
5
57+
10
58+
NULL
59+
60+
# Overflow-safe: naive sqrt(a*a + b*b) overflows to Infinity here; hypot stays finite (matches Spark's Math.hypot)
61+
query B
62+
SELECT hypot(3e200::double, 4e200::double) < 'Infinity'::double;
63+
----
64+
true
65+
66+
# any infinite input yields +Infinity, even when the other is NaN
67+
query R
68+
SELECT hypot('Infinity'::double, 4.0::double);
69+
----
70+
Infinity
71+
72+
query R
73+
SELECT hypot(4.0::double, '-Infinity'::double);
74+
----
75+
Infinity
76+
77+
query R
78+
SELECT hypot('Infinity'::double, 'NaN'::double);
79+
----
80+
Infinity
81+
82+
# NaN propagates when neither input is infinite
83+
query R
84+
SELECT hypot('NaN'::double, 4.0::double);
85+
----
86+
NaN
87+
88+
# signed zeros
89+
query RRR
90+
SELECT hypot(0.0::double, 0.0::double), hypot(-0.0::double, 0.0::double), hypot(3.0::double, -0.0::double);
91+
----
92+
0 0 3
93+
94+
# NULL propagates even when the other input is Infinity
95+
query R
96+
SELECT hypot(NULL::double, 'Infinity'::double);
97+
----
98+
NULL
99+
100+
# negative inputs yield the positive magnitude
101+
query RR
102+
SELECT hypot(-3.0::double, -4.0::double), hypot(-3.0::double, 4.0::double);
103+
----
104+
5 5
105+
106+
# Underflow-safe: naive sqrt(a*a + b*b) underflows to 0 for tiny inputs; hypot stays nonzero (matches Spark's Math.hypot)
107+
query B
108+
SELECT hypot(3e-200::double, 4e-200::double) > 0;
109+
----
110+
true
111+
112+
# Array path with special values (normal, +Infinity, NaN, NULL)
113+
query R
114+
SELECT hypot(a, b) FROM (VALUES
115+
(3.0::double, 4.0::double),
116+
('Infinity'::double, 1.0::double),
117+
('NaN'::double, 1.0::double),
118+
(NULL::double, 1.0::double)) AS t(a, b);
119+
----
120+
5
121+
Infinity
122+
NaN
123+
NULL
124+
125+
# both inputs NaN -> NaN
126+
query R
127+
SELECT hypot('NaN'::double, 'NaN'::double);
128+
----
129+
NaN
130+
131+
# both inputs infinite -> +Infinity
132+
query R
133+
SELECT hypot('Infinity'::double, '-Infinity'::double);
134+
----
135+
Infinity

0 commit comments

Comments
 (0)