Skip to content

Commit 68c1c07

Browse files
crm26claude
andcommitted
feat: add cosine_distance scalar function
Add cosine_distance (and list_cosine_distance alias) to compute cosine distance between two numeric arrays. Includes shared vector math primitives in vector_math.rs for reuse by follow-on functions. Part of apache#21536. Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>
1 parent 4389f14 commit 68c1c07

5 files changed

Lines changed: 537 additions & 0 deletions

File tree

Lines changed: 322 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,322 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
//! [`ScalarUDFImpl`] definitions for cosine_distance function.
19+
20+
use crate::utils::make_scalar_function;
21+
use crate::vector_math::{convert_to_f64_array, dot_product_f64, magnitude_f64};
22+
use arrow::array::{
23+
Array, ArrayRef, Float64Array, LargeListArray, ListArray, OffsetSizeTrait,
24+
};
25+
use arrow::datatypes::{
26+
DataType,
27+
DataType::{FixedSizeList, LargeList, List, Null},
28+
};
29+
use datafusion_common::cast::as_generic_list_array;
30+
use datafusion_common::utils::{ListCoercion, coerced_type_with_base_type_only};
31+
use datafusion_common::{Result, exec_err, plan_err, utils::take_function_args};
32+
use datafusion_expr::{
33+
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
34+
Volatility,
35+
};
36+
use datafusion_functions::downcast_arg;
37+
use datafusion_macros::user_doc;
38+
use itertools::Itertools;
39+
use std::sync::Arc;
40+
41+
make_udf_expr_and_func!(
42+
CosineDistance,
43+
cosine_distance,
44+
array1 array2,
45+
"returns the cosine distance between two numeric arrays.",
46+
cosine_distance_udf
47+
);
48+
49+
#[user_doc(
50+
doc_section(label = "Array Functions"),
51+
description = "Returns the cosine distance between two input arrays of equal length. The cosine distance is defined as 1 - cosine_similarity, i.e. `1 - dot(a,b) / (||a|| * ||b||)`. Returns NULL if either array is NULL or contains only zeros.",
52+
syntax_example = "cosine_distance(array1, array2)",
53+
sql_example = r#"```sql
54+
> select cosine_distance([1.0, 0.0], [0.0, 1.0]);
55+
+-----------------------------------------------+
56+
| cosine_distance(List([1.0,0.0]),List([0.0,1.0])) |
57+
+-----------------------------------------------+
58+
| 1.0 |
59+
+-----------------------------------------------+
60+
```"#,
61+
argument(
62+
name = "array1",
63+
description = "Array expression. Can be a constant, column, or function, and any combination of array operators."
64+
),
65+
argument(
66+
name = "array2",
67+
description = "Array expression. Can be a constant, column, or function, and any combination of array operators."
68+
)
69+
)]
70+
#[derive(Debug, PartialEq, Eq, Hash)]
71+
pub struct CosineDistance {
72+
signature: Signature,
73+
aliases: Vec<String>,
74+
}
75+
76+
impl Default for CosineDistance {
77+
fn default() -> Self {
78+
Self::new()
79+
}
80+
}
81+
82+
impl CosineDistance {
83+
pub fn new() -> Self {
84+
Self {
85+
signature: Signature::user_defined(Volatility::Immutable),
86+
aliases: vec!["list_cosine_distance".to_string()],
87+
}
88+
}
89+
}
90+
91+
impl ScalarUDFImpl for CosineDistance {
92+
fn name(&self) -> &str {
93+
"cosine_distance"
94+
}
95+
96+
fn signature(&self) -> &Signature {
97+
&self.signature
98+
}
99+
100+
fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
101+
Ok(DataType::Float64)
102+
}
103+
104+
fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
105+
let [_, _] = take_function_args(self.name(), arg_types)?;
106+
let coercion = Some(&ListCoercion::FixedSizedListToList);
107+
let arg_types = arg_types.iter().map(|arg_type| {
108+
if matches!(arg_type, Null | List(_) | LargeList(_) | FixedSizeList(..)) {
109+
Ok(coerced_type_with_base_type_only(
110+
arg_type,
111+
&DataType::Float64,
112+
coercion,
113+
))
114+
} else {
115+
plan_err!("{} does not support type {arg_type}", self.name())
116+
}
117+
});
118+
119+
arg_types.try_collect()
120+
}
121+
122+
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
123+
make_scalar_function(cosine_distance_inner)(&args.args)
124+
}
125+
126+
fn aliases(&self) -> &[String] {
127+
&self.aliases
128+
}
129+
130+
fn documentation(&self) -> Option<&Documentation> {
131+
self.doc()
132+
}
133+
}
134+
135+
fn cosine_distance_inner(args: &[ArrayRef]) -> Result<ArrayRef> {
136+
let [array1, array2] = take_function_args("cosine_distance", args)?;
137+
match (array1.data_type(), array2.data_type()) {
138+
(List(_), List(_)) => general_cosine_distance::<i32>(args),
139+
(LargeList(_), LargeList(_)) => general_cosine_distance::<i64>(args),
140+
(arg_type1, arg_type2) => {
141+
exec_err!(
142+
"cosine_distance does not support types {arg_type1} and {arg_type2}"
143+
)
144+
}
145+
}
146+
}
147+
148+
fn general_cosine_distance<O: OffsetSizeTrait>(arrays: &[ArrayRef]) -> Result<ArrayRef> {
149+
let list_array1 = as_generic_list_array::<O>(&arrays[0])?;
150+
let list_array2 = as_generic_list_array::<O>(&arrays[1])?;
151+
152+
let result = list_array1
153+
.iter()
154+
.zip(list_array2.iter())
155+
.map(|(arr1, arr2)| compute_cosine_distance(arr1, arr2))
156+
.collect::<Result<Float64Array>>()?;
157+
158+
Ok(Arc::new(result) as ArrayRef)
159+
}
160+
161+
/// Computes the cosine distance between two arrays: 1 - dot(a,b) / (||a|| * ||b||)
162+
fn compute_cosine_distance(
163+
arr1: Option<ArrayRef>,
164+
arr2: Option<ArrayRef>,
165+
) -> Result<Option<f64>> {
166+
let value1 = match arr1 {
167+
Some(arr) => arr,
168+
None => return Ok(None),
169+
};
170+
let value2 = match arr2 {
171+
Some(arr) => arr,
172+
None => return Ok(None),
173+
};
174+
175+
let mut value1 = value1;
176+
let mut value2 = value2;
177+
178+
loop {
179+
match value1.data_type() {
180+
List(_) => {
181+
if downcast_arg!(value1, ListArray).null_count() > 0 {
182+
return Ok(None);
183+
}
184+
value1 = downcast_arg!(value1, ListArray).value(0);
185+
}
186+
LargeList(_) => {
187+
if downcast_arg!(value1, LargeListArray).null_count() > 0 {
188+
return Ok(None);
189+
}
190+
value1 = downcast_arg!(value1, LargeListArray).value(0);
191+
}
192+
_ => break,
193+
}
194+
195+
match value2.data_type() {
196+
List(_) => {
197+
if downcast_arg!(value2, ListArray).null_count() > 0 {
198+
return Ok(None);
199+
}
200+
value2 = downcast_arg!(value2, ListArray).value(0);
201+
}
202+
LargeList(_) => {
203+
if downcast_arg!(value2, LargeListArray).null_count() > 0 {
204+
return Ok(None);
205+
}
206+
value2 = downcast_arg!(value2, LargeListArray).value(0);
207+
}
208+
_ => break,
209+
}
210+
}
211+
212+
if value1.null_count() != 0 || value2.null_count() != 0 {
213+
return Ok(None);
214+
}
215+
216+
let values1 = convert_to_f64_array(&value1)?;
217+
let values2 = convert_to_f64_array(&value2)?;
218+
219+
if values1.len() != values2.len() {
220+
return exec_err!("Both arrays must have the same length");
221+
}
222+
223+
let dot = dot_product_f64(&values1, &values2);
224+
let mag1 = magnitude_f64(&values1);
225+
let mag2 = magnitude_f64(&values2);
226+
227+
if mag1 == 0.0 || mag2 == 0.0 {
228+
return Ok(None);
229+
}
230+
231+
Ok(Some(1.0 - dot / (mag1 * mag2)))
232+
}
233+
234+
#[cfg(test)]
235+
mod tests {
236+
use super::*;
237+
use arrow::array::{Float64Array, ListArray};
238+
use arrow::buffer::OffsetBuffer;
239+
use arrow::datatypes::{DataType, Field};
240+
use std::sync::Arc;
241+
242+
fn make_f64_list_array(values: Vec<Option<Vec<Option<f64>>>>) -> ArrayRef {
243+
let mut flat: Vec<Option<f64>> = Vec::new();
244+
let mut offsets: Vec<i32> = vec![0];
245+
for v in &values {
246+
match v {
247+
Some(inner) => {
248+
flat.extend(inner);
249+
offsets.push(flat.len() as i32);
250+
}
251+
None => {
252+
offsets.push(flat.len() as i32);
253+
}
254+
}
255+
}
256+
let values_array = Arc::new(Float64Array::from(flat)) as ArrayRef;
257+
let field = Arc::new(Field::new_list_field(DataType::Float64, true));
258+
let offset_buffer = OffsetBuffer::new(offsets.into());
259+
let null_buffer = arrow::buffer::NullBuffer::from(
260+
values.iter().map(|v| v.is_some()).collect::<Vec<_>>(),
261+
);
262+
Arc::new(ListArray::new(
263+
field,
264+
offset_buffer,
265+
values_array,
266+
Some(null_buffer),
267+
))
268+
}
269+
270+
#[test]
271+
fn test_cosine_distance_orthogonal() {
272+
let arr1 = make_f64_list_array(vec![Some(vec![Some(1.0), Some(0.0)])]);
273+
let arr2 = make_f64_list_array(vec![Some(vec![Some(0.0), Some(1.0)])]);
274+
let result = cosine_distance_inner(&[arr1, arr2]).unwrap();
275+
let result = result.as_any().downcast_ref::<Float64Array>().unwrap();
276+
assert!((result.value(0) - 1.0).abs() < 1e-10);
277+
}
278+
279+
#[test]
280+
fn test_cosine_distance_identical() {
281+
let arr1 = make_f64_list_array(vec![Some(vec![Some(1.0), Some(2.0), Some(3.0)])]);
282+
let arr2 = make_f64_list_array(vec![Some(vec![Some(1.0), Some(2.0), Some(3.0)])]);
283+
let result = cosine_distance_inner(&[arr1, arr2]).unwrap();
284+
let result = result.as_any().downcast_ref::<Float64Array>().unwrap();
285+
assert!(result.value(0).abs() < 1e-10);
286+
}
287+
288+
#[test]
289+
fn test_cosine_distance_opposite() {
290+
let arr1 = make_f64_list_array(vec![Some(vec![Some(1.0), Some(0.0)])]);
291+
let arr2 = make_f64_list_array(vec![Some(vec![Some(-1.0), Some(0.0)])]);
292+
let result = cosine_distance_inner(&[arr1, arr2]).unwrap();
293+
let result = result.as_any().downcast_ref::<Float64Array>().unwrap();
294+
assert!((result.value(0) - 2.0).abs() < 1e-10);
295+
}
296+
297+
#[test]
298+
fn test_cosine_distance_null_array() {
299+
let arr1 = make_f64_list_array(vec![None]);
300+
let arr2 = make_f64_list_array(vec![Some(vec![Some(1.0), Some(2.0)])]);
301+
let result = cosine_distance_inner(&[arr1, arr2]).unwrap();
302+
let result = result.as_any().downcast_ref::<Float64Array>().unwrap();
303+
assert!(result.is_null(0));
304+
}
305+
306+
#[test]
307+
fn test_cosine_distance_mismatched_lengths() {
308+
let arr1 = make_f64_list_array(vec![Some(vec![Some(1.0), Some(2.0)])]);
309+
let arr2 = make_f64_list_array(vec![Some(vec![Some(1.0)])]);
310+
let result = cosine_distance_inner(&[arr1, arr2]);
311+
assert!(result.is_err());
312+
}
313+
314+
#[test]
315+
fn test_cosine_distance_zero_magnitude() {
316+
let arr1 = make_f64_list_array(vec![Some(vec![Some(0.0), Some(0.0)])]);
317+
let arr2 = make_f64_list_array(vec![Some(vec![Some(1.0), Some(0.0)])]);
318+
let result = cosine_distance_inner(&[arr1, arr2]).unwrap();
319+
let result = result.as_any().downcast_ref::<Float64Array>().unwrap();
320+
assert!(result.is_null(0));
321+
}
322+
}

‎datafusion/functions-nested/src/lib.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ pub mod array_has;
4141
pub mod arrays_zip;
4242
pub mod cardinality;
4343
pub mod concat;
44+
pub mod cosine_distance;
4445
pub mod dimension;
4546
pub mod distance;
4647
pub mod empty;
@@ -68,6 +69,7 @@ pub mod set_ops;
6869
pub mod sort;
6970
pub mod string;
7071
pub mod utils;
72+
pub mod vector_math;
7173

7274
use datafusion_common::Result;
7375
use datafusion_execution::FunctionRegistry;
@@ -85,6 +87,7 @@ pub mod expr_fn {
8587
pub use super::concat::array_append;
8688
pub use super::concat::array_concat;
8789
pub use super::concat::array_prepend;
90+
pub use super::cosine_distance::cosine_distance;
8891
pub use super::dimension::array_dims;
8992
pub use super::dimension::array_ndims;
9093
pub use super::distance::array_distance;
@@ -150,6 +153,7 @@ pub fn all_default_nested_functions() -> Vec<Arc<ScalarUDF>> {
150153
array_has::array_has_any_udf(),
151154
empty::array_empty_udf(),
152155
length::array_length_udf(),
156+
cosine_distance::cosine_distance_udf(),
153157
distance::array_distance_udf(),
154158
flatten::flatten_udf(),
155159
min_max::array_max_udf(),

0 commit comments

Comments
 (0)