Skip to content

Commit 0870b3d

Browse files
committed
review: name the variant in arrays_overlap errors and trim the version table
Use self.name() in the unsupported-type errors so spark_arrays_overlap_normalized reports itself, and test it on the array and scalar paths. Reduce the native docs to the two equality policies and point at CometArraysOverlap.nativeFunction for the version mapping. Keep one Spark version on each side of the boundary in the routing test and the convert assertion.
1 parent 4905804 commit 0870b3d

2 files changed

Lines changed: 31 additions & 20 deletions

File tree

‎native/spark-expr/src/array_funcs/arrays_overlap.rs‎

Lines changed: 29 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,10 @@
3030
//! - false if no overlap and neither array contains null elements
3131
//!
3232
//! Spark looks up flat elements in a hash set of boxed values, so every NaN is one value, while
33-
//! `-0.0` and `0.0` are two. Spark 4.2.0 normalizes the arguments in the plan (SPARK-54918), and
34-
//! 4.0.5, 4.1.4 and 4.2.1 normalize flat `FLOAT` and `DOUBLE` elements before the lookup
35-
//! (SPARK-59602), which makes the zeros one value too. `spark_arrays_overlap_normalized` matches
36-
//! them that way, and the serde picks it on those versions. Nested elements compare in Spark's SQL
37-
//! ordering on every version, where the zeros are equal.
33+
//! `-0.0` and `0.0` are two. `spark_arrays_overlap` matches that. `spark_arrays_overlap_normalized`
34+
//! normalizes flat `FLOAT` and `DOUBLE` elements before the lookup, which makes the zeros one value
35+
//! too. `CometArraysOverlap.nativeFunction` picks the variant for the Spark version. Nested
36+
//! elements compare in Spark's SQL ordering in both variants, where the zeros are equal.
3837
3938
use arrow::array::{
4039
make_comparator, Array, ArrayRef, AsArray, BooleanArray, GenericListArray, GenericStringArray,
@@ -85,7 +84,7 @@ impl SparkArraysOverlap {
8584
}
8685

8786
/// Normalizes flat `FLOAT` and `DOUBLE` elements before matching them, so `-0.0` and `0.0` are
88-
/// one value as well as every NaN.
87+
/// one value. Every NaN is one value in both variants.
8988
pub fn normalized() -> Self {
9089
Self {
9190
normalize_floats: true,
@@ -151,9 +150,7 @@ impl ScalarUDFImpl for SparkArraysOverlap {
151150
self.normalize_floats,
152151
)?,
153152
(l, r) => {
154-
return exec_err!(
155-
"spark_arrays_overlap does not support types '{l}' and '{r}'"
156-
)
153+
return exec_err!("{} does not support types '{l}' and '{r}'", self.name())
157154
}
158155
};
159156
Ok(ColumnarValue::Array(result))
@@ -174,9 +171,7 @@ impl ScalarUDFImpl for SparkArraysOverlap {
174171
self.normalize_floats,
175172
)?,
176173
(l, r) => {
177-
return exec_err!(
178-
"spark_arrays_overlap does not support types '{l}' and '{r}'"
179-
)
174+
return exec_err!("{} does not support types '{l}' and '{r}'", self.name())
180175
}
181176
};
182177
let scalar = ScalarValue::try_from_array(&result, 0)?;
@@ -916,6 +911,28 @@ mod tests {
916911
Ok(())
917912
}
918913

914+
#[test]
915+
fn test_unsupported_types_error_names_the_variant() {
916+
for udf in [SparkArraysOverlap::new(), SparkArraysOverlap::normalized()] {
917+
let column = Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef;
918+
let arrays = [
919+
ColumnarValue::Array(Arc::clone(&column)),
920+
ColumnarValue::Array(column),
921+
];
922+
let scalars = [
923+
ColumnarValue::Scalar(ScalarValue::Int32(Some(1))),
924+
ColumnarValue::Scalar(ScalarValue::Int32(Some(2))),
925+
];
926+
for (args, rows) in [(arrays, 2), (scalars, 1)] {
927+
let message = invoke(&udf, args, rows).unwrap_err().to_string();
928+
assert!(
929+
message.contains(&format!("{} does not support types", udf.name())),
930+
"{message}"
931+
);
932+
}
933+
}
934+
}
935+
919936
#[test]
920937
fn test_null_only_overlap() -> Result<()> {
921938
// [1, NULL] vs [NULL, 2] => null (no definite overlap, but nulls present)

‎spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -776,14 +776,8 @@ class CometArrayExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelp
776776
// normalizes, -0.0 and 0.0 are one value in a flat FLOAT or DOUBLE array, which no pinned
777777
// Spark profile can show: 4.2.0 normalizes the arguments in the plan, before the native
778778
// kernel sees them.
779-
Seq("3.4.3", "3.5.9", "4.0.4", "4.1.3").foreach { version =>
780-
assert(CometArraysOverlap.nativeFunction(version) == "spark_arrays_overlap", version)
781-
}
782-
Seq("4.0.5", "4.1.4", "4.2.0", "4.2.1").foreach { version =>
783-
assert(
784-
CometArraysOverlap.nativeFunction(version) == "spark_arrays_overlap_normalized",
785-
version)
786-
}
779+
assert(CometArraysOverlap.nativeFunction("4.1.3") == "spark_arrays_overlap")
780+
assert(CometArraysOverlap.nativeFunction("4.1.4") == "spark_arrays_overlap_normalized")
787781
// convert asks for the variant of the Spark version it runs on.
788782
val array = AttributeReference("a", ArrayType(DoubleType))()
789783
val proto =

0 commit comments

Comments
 (0)