Skip to content

Commit fc3ee90

Browse files
crm26claude
andcommitted
feat: rework cosine_distance per review feedback
Addresses review comments on apache#21542: - Iterate list offsets/values directly instead of per-row ArrayRef downcast - Remove nested-list unwrap loop (function does not support nested lists) - Drop convert_to_f64_array wrapper (coerce_types already guarantees Float64) - Remove duplicate Rust unit tests now covered by SLT - More descriptive error message for mismatched list lengths - Delete now-unused vector_math module; inline math into sole caller Adds SLT coverage for NULL-element-in-list behavior previously tested only in Rust unit tests. Part of apache#21536. Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
1 parent 68c1c07 commit fc3ee90

4 files changed

Lines changed: 53 additions & 236 deletions

File tree

‎datafusion/functions-nested/src/cosine_distance.rs‎

Lines changed: 46 additions & 166 deletions
Original file line numberDiff line numberDiff line change
@@ -18,22 +18,18 @@
1818
//! [`ScalarUDFImpl`] definitions for cosine_distance function.
1919
2020
use crate::utils::make_scalar_function;
21-
use crate::vector_math::{convert_to_f64_array, dot_product_f64, magnitude_f64};
22-
use arrow::array::{
23-
Array, ArrayRef, Float64Array, LargeListArray, ListArray, OffsetSizeTrait,
24-
};
21+
use arrow::array::{Array, ArrayRef, Float64Array, OffsetSizeTrait};
2522
use arrow::datatypes::{
2623
DataType,
2724
DataType::{FixedSizeList, LargeList, List, Null},
2825
};
29-
use datafusion_common::cast::as_generic_list_array;
26+
use datafusion_common::cast::{as_float64_array, as_generic_list_array};
3027
use datafusion_common::utils::{ListCoercion, coerced_type_with_base_type_only};
3128
use datafusion_common::{Result, exec_err, plan_err, utils::take_function_args};
3229
use datafusion_expr::{
3330
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
3431
Volatility,
3532
};
36-
use datafusion_functions::downcast_arg;
3733
use datafusion_macros::user_doc;
3834
use itertools::Itertools;
3935
use std::sync::Arc;
@@ -149,174 +145,58 @@ fn general_cosine_distance<O: OffsetSizeTrait>(arrays: &[ArrayRef]) -> Result<Ar
149145
let list_array1 = as_generic_list_array::<O>(&arrays[0])?;
150146
let list_array2 = as_generic_list_array::<O>(&arrays[1])?;
151147

152-
let result = list_array1
153-
.iter()
154-
.zip(list_array2.iter())
155-
.map(|(arr1, arr2)| compute_cosine_distance(arr1, arr2))
156-
.collect::<Result<Float64Array>>()?;
157-
158-
Ok(Arc::new(result) as ArrayRef)
159-
}
160-
161-
/// Computes the cosine distance between two arrays: 1 - dot(a,b) / (||a|| * ||b||)
162-
fn compute_cosine_distance(
163-
arr1: Option<ArrayRef>,
164-
arr2: Option<ArrayRef>,
165-
) -> Result<Option<f64>> {
166-
let value1 = match arr1 {
167-
Some(arr) => arr,
168-
None => return Ok(None),
169-
};
170-
let value2 = match arr2 {
171-
Some(arr) => arr,
172-
None => return Ok(None),
173-
};
174-
175-
let mut value1 = value1;
176-
let mut value2 = value2;
148+
let values1 = as_float64_array(list_array1.values())?;
149+
let values2 = as_float64_array(list_array2.values())?;
150+
let offsets1 = list_array1.value_offsets();
151+
let offsets2 = list_array2.value_offsets();
177152

178-
loop {
179-
match value1.data_type() {
180-
List(_) => {
181-
if downcast_arg!(value1, ListArray).null_count() > 0 {
182-
return Ok(None);
183-
}
184-
value1 = downcast_arg!(value1, ListArray).value(0);
185-
}
186-
LargeList(_) => {
187-
if downcast_arg!(value1, LargeListArray).null_count() > 0 {
188-
return Ok(None);
189-
}
190-
value1 = downcast_arg!(value1, LargeListArray).value(0);
191-
}
192-
_ => break,
153+
let mut builder = Float64Array::builder(list_array1.len());
154+
for row in 0..list_array1.len() {
155+
if list_array1.is_null(row) || list_array2.is_null(row) {
156+
builder.append_null();
157+
continue;
193158
}
194159

195-
match value2.data_type() {
196-
List(_) => {
197-
if downcast_arg!(value2, ListArray).null_count() > 0 {
198-
return Ok(None);
199-
}
200-
value2 = downcast_arg!(value2, ListArray).value(0);
201-
}
202-
LargeList(_) => {
203-
if downcast_arg!(value2, LargeListArray).null_count() > 0 {
204-
return Ok(None);
205-
}
206-
value2 = downcast_arg!(value2, LargeListArray).value(0);
207-
}
208-
_ => break,
160+
let start1 = offsets1[row].as_usize();
161+
let end1 = offsets1[row + 1].as_usize();
162+
let start2 = offsets2[row].as_usize();
163+
let end2 = offsets2[row + 1].as_usize();
164+
let len1 = end1 - start1;
165+
let len2 = end2 - start2;
166+
167+
if len1 != len2 {
168+
return exec_err!(
169+
"cosine_distance requires both list inputs to have the same length, got {len1} and {len2}"
170+
);
209171
}
210-
}
211-
212-
if value1.null_count() != 0 || value2.null_count() != 0 {
213-
return Ok(None);
214-
}
215-
216-
let values1 = convert_to_f64_array(&value1)?;
217-
let values2 = convert_to_f64_array(&value2)?;
218172

219-
if values1.len() != values2.len() {
220-
return exec_err!("Both arrays must have the same length");
221-
}
222-
223-
let dot = dot_product_f64(&values1, &values2);
224-
let mag1 = magnitude_f64(&values1);
225-
let mag2 = magnitude_f64(&values2);
226-
227-
if mag1 == 0.0 || mag2 == 0.0 {
228-
return Ok(None);
229-
}
230-
231-
Ok(Some(1.0 - dot / (mag1 * mag2)))
232-
}
233-
234-
#[cfg(test)]
235-
mod tests {
236-
use super::*;
237-
use arrow::array::{Float64Array, ListArray};
238-
use arrow::buffer::OffsetBuffer;
239-
use arrow::datatypes::{DataType, Field};
240-
use std::sync::Arc;
241-
242-
fn make_f64_list_array(values: Vec<Option<Vec<Option<f64>>>>) -> ArrayRef {
243-
let mut flat: Vec<Option<f64>> = Vec::new();
244-
let mut offsets: Vec<i32> = vec![0];
245-
for v in &values {
246-
match v {
247-
Some(inner) => {
248-
flat.extend(inner);
249-
offsets.push(flat.len() as i32);
250-
}
251-
None => {
252-
offsets.push(flat.len() as i32);
253-
}
254-
}
173+
let slice1 = values1.slice(start1, len1);
174+
let slice2 = values2.slice(start2, len2);
175+
if slice1.null_count() != 0 || slice2.null_count() != 0 {
176+
builder.append_null();
177+
continue;
255178
}
256-
let values_array = Arc::new(Float64Array::from(flat)) as ArrayRef;
257-
let field = Arc::new(Field::new_list_field(DataType::Float64, true));
258-
let offset_buffer = OffsetBuffer::new(offsets.into());
259-
let null_buffer = arrow::buffer::NullBuffer::from(
260-
values.iter().map(|v| v.is_some()).collect::<Vec<_>>(),
261-
);
262-
Arc::new(ListArray::new(
263-
field,
264-
offset_buffer,
265-
values_array,
266-
Some(null_buffer),
267-
))
268-
}
269-
270-
#[test]
271-
fn test_cosine_distance_orthogonal() {
272-
let arr1 = make_f64_list_array(vec![Some(vec![Some(1.0), Some(0.0)])]);
273-
let arr2 = make_f64_list_array(vec![Some(vec![Some(0.0), Some(1.0)])]);
274-
let result = cosine_distance_inner(&[arr1, arr2]).unwrap();
275-
let result = result.as_any().downcast_ref::<Float64Array>().unwrap();
276-
assert!((result.value(0) - 1.0).abs() < 1e-10);
277-
}
278179

279-
#[test]
280-
fn test_cosine_distance_identical() {
281-
let arr1 = make_f64_list_array(vec![Some(vec![Some(1.0), Some(2.0), Some(3.0)])]);
282-
let arr2 = make_f64_list_array(vec![Some(vec![Some(1.0), Some(2.0), Some(3.0)])]);
283-
let result = cosine_distance_inner(&[arr1, arr2]).unwrap();
284-
let result = result.as_any().downcast_ref::<Float64Array>().unwrap();
285-
assert!(result.value(0).abs() < 1e-10);
286-
}
287-
288-
#[test]
289-
fn test_cosine_distance_opposite() {
290-
let arr1 = make_f64_list_array(vec![Some(vec![Some(1.0), Some(0.0)])]);
291-
let arr2 = make_f64_list_array(vec![Some(vec![Some(-1.0), Some(0.0)])]);
292-
let result = cosine_distance_inner(&[arr1, arr2]).unwrap();
293-
let result = result.as_any().downcast_ref::<Float64Array>().unwrap();
294-
assert!((result.value(0) - 2.0).abs() < 1e-10);
295-
}
296-
297-
#[test]
298-
fn test_cosine_distance_null_array() {
299-
let arr1 = make_f64_list_array(vec![None]);
300-
let arr2 = make_f64_list_array(vec![Some(vec![Some(1.0), Some(2.0)])]);
301-
let result = cosine_distance_inner(&[arr1, arr2]).unwrap();
302-
let result = result.as_any().downcast_ref::<Float64Array>().unwrap();
303-
assert!(result.is_null(0));
304-
}
180+
let vals1 = slice1.values();
181+
let vals2 = slice2.values();
182+
183+
let mut dot = 0.0;
184+
let mut sq1 = 0.0;
185+
let mut sq2 = 0.0;
186+
for i in 0..len1 {
187+
let a = vals1[i];
188+
let b = vals2[i];
189+
dot += a * b;
190+
sq1 += a * a;
191+
sq2 += b * b;
192+
}
305193

306-
#[test]
307-
fn test_cosine_distance_mismatched_lengths() {
308-
let arr1 = make_f64_list_array(vec![Some(vec![Some(1.0), Some(2.0)])]);
309-
let arr2 = make_f64_list_array(vec![Some(vec![Some(1.0)])]);
310-
let result = cosine_distance_inner(&[arr1, arr2]);
311-
assert!(result.is_err());
194+
if sq1 == 0.0 || sq2 == 0.0 {
195+
builder.append_null();
196+
} else {
197+
builder.append_value(1.0 - dot / (sq1.sqrt() * sq2.sqrt()));
198+
}
312199
}
313200

314-
#[test]
315-
fn test_cosine_distance_zero_magnitude() {
316-
let arr1 = make_f64_list_array(vec![Some(vec![Some(0.0), Some(0.0)])]);
317-
let arr2 = make_f64_list_array(vec![Some(vec![Some(1.0), Some(0.0)])]);
318-
let result = cosine_distance_inner(&[arr1, arr2]).unwrap();
319-
let result = result.as_any().downcast_ref::<Float64Array>().unwrap();
320-
assert!(result.is_null(0));
321-
}
201+
Ok(Arc::new(builder.finish()) as ArrayRef)
322202
}

‎datafusion/functions-nested/src/lib.rs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,6 @@ pub mod set_ops;
6969
pub mod sort;
7070
pub mod string;
7171
pub mod utils;
72-
pub mod vector_math;
7372

7473
use datafusion_common::Result;
7574
use datafusion_execution::FunctionRegistry;

‎datafusion/functions-nested/src/vector_math.rs‎

Lines changed: 0 additions & 68 deletions
This file was deleted.

‎datafusion/sqllogictest/test_files/cosine_distance.slt‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,15 @@ select cosine_distance([0.0, 0.0], [1.0, 2.0]);
5656
NULL
5757

5858
# Mismatched lengths error
59-
query error Both arrays must have the same length
59+
query error cosine_distance requires both list inputs to have the same length
6060
select cosine_distance([1.0, 2.0], [1.0]);
6161

62+
# NULL element inside a list returns NULL for that row
63+
query R
64+
select cosine_distance([1.0, 2.0, NULL], [1.0, 2.0, 3.0]);
65+
----
66+
NULL
67+
6268
# LargeList support
6369
query R
6470
select cosine_distance(

0 commit comments

Comments
 (0)