|
18 | 18 | //! [`ScalarUDFImpl`] definitions for cosine_distance function. |
19 | 19 |
|
20 | 20 | use crate::utils::make_scalar_function; |
21 | | -use crate::vector_math::{convert_to_f64_array, dot_product_f64, magnitude_f64}; |
22 | | -use arrow::array::{ |
23 | | - Array, ArrayRef, Float64Array, LargeListArray, ListArray, OffsetSizeTrait, |
24 | | -}; |
| 21 | +use arrow::array::{Array, ArrayRef, Float64Array, OffsetSizeTrait}; |
25 | 22 | use arrow::datatypes::{ |
26 | 23 | DataType, |
27 | 24 | DataType::{FixedSizeList, LargeList, List, Null}, |
28 | 25 | }; |
29 | | -use datafusion_common::cast::as_generic_list_array; |
| 26 | +use datafusion_common::cast::{as_float64_array, as_generic_list_array}; |
30 | 27 | use datafusion_common::utils::{ListCoercion, coerced_type_with_base_type_only}; |
31 | 28 | use datafusion_common::{Result, exec_err, plan_err, utils::take_function_args}; |
32 | 29 | use datafusion_expr::{ |
33 | 30 | ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, |
34 | 31 | Volatility, |
35 | 32 | }; |
36 | | -use datafusion_functions::downcast_arg; |
37 | 33 | use datafusion_macros::user_doc; |
38 | 34 | use itertools::Itertools; |
39 | 35 | use std::sync::Arc; |
@@ -149,174 +145,58 @@ fn general_cosine_distance<O: OffsetSizeTrait>(arrays: &[ArrayRef]) -> Result<Ar |
149 | 145 | let list_array1 = as_generic_list_array::<O>(&arrays[0])?; |
150 | 146 | let list_array2 = as_generic_list_array::<O>(&arrays[1])?; |
151 | 147 |
|
152 | | - let result = list_array1 |
153 | | - .iter() |
154 | | - .zip(list_array2.iter()) |
155 | | - .map(|(arr1, arr2)| compute_cosine_distance(arr1, arr2)) |
156 | | - .collect::<Result<Float64Array>>()?; |
157 | | - |
158 | | - Ok(Arc::new(result) as ArrayRef) |
159 | | -} |
160 | | - |
161 | | -/// Computes the cosine distance between two arrays: 1 - dot(a,b) / (||a|| * ||b||) |
162 | | -fn compute_cosine_distance( |
163 | | - arr1: Option<ArrayRef>, |
164 | | - arr2: Option<ArrayRef>, |
165 | | -) -> Result<Option<f64>> { |
166 | | - let value1 = match arr1 { |
167 | | - Some(arr) => arr, |
168 | | - None => return Ok(None), |
169 | | - }; |
170 | | - let value2 = match arr2 { |
171 | | - Some(arr) => arr, |
172 | | - None => return Ok(None), |
173 | | - }; |
174 | | - |
175 | | - let mut value1 = value1; |
176 | | - let mut value2 = value2; |
| 148 | + let values1 = as_float64_array(list_array1.values())?; |
| 149 | + let values2 = as_float64_array(list_array2.values())?; |
| 150 | + let offsets1 = list_array1.value_offsets(); |
| 151 | + let offsets2 = list_array2.value_offsets(); |
177 | 152 |
|
178 | | - loop { |
179 | | - match value1.data_type() { |
180 | | - List(_) => { |
181 | | - if downcast_arg!(value1, ListArray).null_count() > 0 { |
182 | | - return Ok(None); |
183 | | - } |
184 | | - value1 = downcast_arg!(value1, ListArray).value(0); |
185 | | - } |
186 | | - LargeList(_) => { |
187 | | - if downcast_arg!(value1, LargeListArray).null_count() > 0 { |
188 | | - return Ok(None); |
189 | | - } |
190 | | - value1 = downcast_arg!(value1, LargeListArray).value(0); |
191 | | - } |
192 | | - _ => break, |
| 153 | + let mut builder = Float64Array::builder(list_array1.len()); |
| 154 | + for row in 0..list_array1.len() { |
| 155 | + if list_array1.is_null(row) || list_array2.is_null(row) { |
| 156 | + builder.append_null(); |
| 157 | + continue; |
193 | 158 | } |
194 | 159 |
|
195 | | - match value2.data_type() { |
196 | | - List(_) => { |
197 | | - if downcast_arg!(value2, ListArray).null_count() > 0 { |
198 | | - return Ok(None); |
199 | | - } |
200 | | - value2 = downcast_arg!(value2, ListArray).value(0); |
201 | | - } |
202 | | - LargeList(_) => { |
203 | | - if downcast_arg!(value2, LargeListArray).null_count() > 0 { |
204 | | - return Ok(None); |
205 | | - } |
206 | | - value2 = downcast_arg!(value2, LargeListArray).value(0); |
207 | | - } |
208 | | - _ => break, |
| 160 | + let start1 = offsets1[row].as_usize(); |
| 161 | + let end1 = offsets1[row + 1].as_usize(); |
| 162 | + let start2 = offsets2[row].as_usize(); |
| 163 | + let end2 = offsets2[row + 1].as_usize(); |
| 164 | + let len1 = end1 - start1; |
| 165 | + let len2 = end2 - start2; |
| 166 | + |
| 167 | + if len1 != len2 { |
| 168 | + return exec_err!( |
| 169 | + "cosine_distance requires both list inputs to have the same length, got {len1} and {len2}" |
| 170 | + ); |
209 | 171 | } |
210 | | - } |
211 | | - |
212 | | - if value1.null_count() != 0 || value2.null_count() != 0 { |
213 | | - return Ok(None); |
214 | | - } |
215 | | - |
216 | | - let values1 = convert_to_f64_array(&value1)?; |
217 | | - let values2 = convert_to_f64_array(&value2)?; |
218 | 172 |
|
219 | | - if values1.len() != values2.len() { |
220 | | - return exec_err!("Both arrays must have the same length"); |
221 | | - } |
222 | | - |
223 | | - let dot = dot_product_f64(&values1, &values2); |
224 | | - let mag1 = magnitude_f64(&values1); |
225 | | - let mag2 = magnitude_f64(&values2); |
226 | | - |
227 | | - if mag1 == 0.0 || mag2 == 0.0 { |
228 | | - return Ok(None); |
229 | | - } |
230 | | - |
231 | | - Ok(Some(1.0 - dot / (mag1 * mag2))) |
232 | | -} |
233 | | - |
234 | | -#[cfg(test)] |
235 | | -mod tests { |
236 | | - use super::*; |
237 | | - use arrow::array::{Float64Array, ListArray}; |
238 | | - use arrow::buffer::OffsetBuffer; |
239 | | - use arrow::datatypes::{DataType, Field}; |
240 | | - use std::sync::Arc; |
241 | | - |
242 | | - fn make_f64_list_array(values: Vec<Option<Vec<Option<f64>>>>) -> ArrayRef { |
243 | | - let mut flat: Vec<Option<f64>> = Vec::new(); |
244 | | - let mut offsets: Vec<i32> = vec![0]; |
245 | | - for v in &values { |
246 | | - match v { |
247 | | - Some(inner) => { |
248 | | - flat.extend(inner); |
249 | | - offsets.push(flat.len() as i32); |
250 | | - } |
251 | | - None => { |
252 | | - offsets.push(flat.len() as i32); |
253 | | - } |
254 | | - } |
| 173 | + let slice1 = values1.slice(start1, len1); |
| 174 | + let slice2 = values2.slice(start2, len2); |
| 175 | + if slice1.null_count() != 0 || slice2.null_count() != 0 { |
| 176 | + builder.append_null(); |
| 177 | + continue; |
255 | 178 | } |
256 | | - let values_array = Arc::new(Float64Array::from(flat)) as ArrayRef; |
257 | | - let field = Arc::new(Field::new_list_field(DataType::Float64, true)); |
258 | | - let offset_buffer = OffsetBuffer::new(offsets.into()); |
259 | | - let null_buffer = arrow::buffer::NullBuffer::from( |
260 | | - values.iter().map(|v| v.is_some()).collect::<Vec<_>>(), |
261 | | - ); |
262 | | - Arc::new(ListArray::new( |
263 | | - field, |
264 | | - offset_buffer, |
265 | | - values_array, |
266 | | - Some(null_buffer), |
267 | | - )) |
268 | | - } |
269 | | - |
270 | | - #[test] |
271 | | - fn test_cosine_distance_orthogonal() { |
272 | | - let arr1 = make_f64_list_array(vec![Some(vec![Some(1.0), Some(0.0)])]); |
273 | | - let arr2 = make_f64_list_array(vec![Some(vec![Some(0.0), Some(1.0)])]); |
274 | | - let result = cosine_distance_inner(&[arr1, arr2]).unwrap(); |
275 | | - let result = result.as_any().downcast_ref::<Float64Array>().unwrap(); |
276 | | - assert!((result.value(0) - 1.0).abs() < 1e-10); |
277 | | - } |
278 | 179 |
|
279 | | - #[test] |
280 | | - fn test_cosine_distance_identical() { |
281 | | - let arr1 = make_f64_list_array(vec![Some(vec![Some(1.0), Some(2.0), Some(3.0)])]); |
282 | | - let arr2 = make_f64_list_array(vec![Some(vec![Some(1.0), Some(2.0), Some(3.0)])]); |
283 | | - let result = cosine_distance_inner(&[arr1, arr2]).unwrap(); |
284 | | - let result = result.as_any().downcast_ref::<Float64Array>().unwrap(); |
285 | | - assert!(result.value(0).abs() < 1e-10); |
286 | | - } |
287 | | - |
288 | | - #[test] |
289 | | - fn test_cosine_distance_opposite() { |
290 | | - let arr1 = make_f64_list_array(vec![Some(vec![Some(1.0), Some(0.0)])]); |
291 | | - let arr2 = make_f64_list_array(vec![Some(vec![Some(-1.0), Some(0.0)])]); |
292 | | - let result = cosine_distance_inner(&[arr1, arr2]).unwrap(); |
293 | | - let result = result.as_any().downcast_ref::<Float64Array>().unwrap(); |
294 | | - assert!((result.value(0) - 2.0).abs() < 1e-10); |
295 | | - } |
296 | | - |
297 | | - #[test] |
298 | | - fn test_cosine_distance_null_array() { |
299 | | - let arr1 = make_f64_list_array(vec![None]); |
300 | | - let arr2 = make_f64_list_array(vec![Some(vec![Some(1.0), Some(2.0)])]); |
301 | | - let result = cosine_distance_inner(&[arr1, arr2]).unwrap(); |
302 | | - let result = result.as_any().downcast_ref::<Float64Array>().unwrap(); |
303 | | - assert!(result.is_null(0)); |
304 | | - } |
| 180 | + let vals1 = slice1.values(); |
| 181 | + let vals2 = slice2.values(); |
| 182 | + |
| 183 | + let mut dot = 0.0; |
| 184 | + let mut sq1 = 0.0; |
| 185 | + let mut sq2 = 0.0; |
| 186 | + for i in 0..len1 { |
| 187 | + let a = vals1[i]; |
| 188 | + let b = vals2[i]; |
| 189 | + dot += a * b; |
| 190 | + sq1 += a * a; |
| 191 | + sq2 += b * b; |
| 192 | + } |
305 | 193 |
|
306 | | - #[test] |
307 | | - fn test_cosine_distance_mismatched_lengths() { |
308 | | - let arr1 = make_f64_list_array(vec![Some(vec![Some(1.0), Some(2.0)])]); |
309 | | - let arr2 = make_f64_list_array(vec![Some(vec![Some(1.0)])]); |
310 | | - let result = cosine_distance_inner(&[arr1, arr2]); |
311 | | - assert!(result.is_err()); |
| 194 | + if sq1 == 0.0 || sq2 == 0.0 { |
| 195 | + builder.append_null(); |
| 196 | + } else { |
| 197 | + builder.append_value(1.0 - dot / (sq1.sqrt() * sq2.sqrt())); |
| 198 | + } |
312 | 199 | } |
313 | 200 |
|
314 | | - #[test] |
315 | | - fn test_cosine_distance_zero_magnitude() { |
316 | | - let arr1 = make_f64_list_array(vec![Some(vec![Some(0.0), Some(0.0)])]); |
317 | | - let arr2 = make_f64_list_array(vec![Some(vec![Some(1.0), Some(0.0)])]); |
318 | | - let result = cosine_distance_inner(&[arr1, arr2]).unwrap(); |
319 | | - let result = result.as_any().downcast_ref::<Float64Array>().unwrap(); |
320 | | - assert!(result.is_null(0)); |
321 | | - } |
| 201 | + Ok(Arc::new(builder.finish()) as ArrayRef) |
322 | 202 | } |
0 commit comments