Skip to content

Commit 044f1aa

Browse files
committed
feat: add array_add scalar function
Adds `array_add(array1, array2)` returning the element-wise sum of two numeric arrays. Aliased as `list_add`. Follows the per-function split pattern established by cosine_distance (apache#21542), inner_product (apache#21861), and array_normalize (apache#22013) per tracking issue apache#21536. Semantics: - NULL row in either input -> NULL row out - NULL element at position i in either input -> NULL element at i out (per-element propagation, divergent from inner_product which nulls the whole row; chosen because output is a list, not a scalar) - Length mismatch between rows -> exec_err - Empty arrays -> empty array Supports List, LargeList, and FixedSizeList inputs; numeric element types are coerced to Float64. If any input is LargeList, both sides are widened to LargeList for homogeneous runtime dispatch. Uses OffsetBufferBuilder + NullBufferBuilder per the pattern adopted in array_normalize round 1.
1 parent 4a41173 commit 044f1aa

4 files changed

Lines changed: 475 additions & 0 deletions

File tree

Lines changed: 230 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,230 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
//! [`ScalarUDFImpl`] definitions for array_add function.
19+
20+
use crate::utils::make_scalar_function;
21+
use arrow::array::{
22+
Array, ArrayRef, Float64Array, GenericListArray, NullBufferBuilder,
23+
OffsetBufferBuilder, OffsetSizeTrait,
24+
};
25+
use arrow::datatypes::{
26+
DataType,
27+
DataType::{FixedSizeList, LargeList, List, Null},
28+
Field,
29+
};
30+
use datafusion_common::cast::{as_float64_array, as_generic_list_array};
31+
use datafusion_common::utils::{ListCoercion, coerced_type_with_base_type_only};
32+
use datafusion_common::{
33+
Result, exec_err, internal_err, plan_err, utils::take_function_args,
34+
};
35+
use datafusion_expr::{
36+
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
37+
Volatility,
38+
};
39+
use datafusion_macros::user_doc;
40+
use std::sync::Arc;
41+
42+
make_udf_expr_and_func!(
43+
ArrayAdd,
44+
array_add,
45+
array1 array2,
46+
"returns the element-wise sum of two numeric arrays.",
47+
array_add_udf
48+
);
49+
50+
#[user_doc(
51+
doc_section(label = "Array Functions"),
52+
description = "Returns the element-wise sum of two input arrays of equal length, computed as `[array1[i] + array2[i]]`. Returns NULL if either input row is NULL. If a NULL element appears in either input at position `i`, the result element at position `i` is NULL. Returns an error if the two arrays have different lengths in the same row. Returns an empty array for two empty input arrays.",
53+
syntax_example = "array_add(array1, array2)",
54+
sql_example = r#"```sql
55+
> select array_add([1.0, 2.0, 3.0], [4.0, 5.0, 6.0]);
56+
+-------------------------------------------------+
57+
| array_add(List([1.0,2.0,3.0]),List([4.0,5.0,6.0])) |
58+
+-------------------------------------------------+
59+
| [5.0, 7.0, 9.0] |
60+
+-------------------------------------------------+
61+
```"#,
62+
argument(
63+
name = "array1",
64+
description = "Array expression. Can be a constant, column, or function, and any combination of array operators."
65+
),
66+
argument(
67+
name = "array2",
68+
description = "Array expression. Can be a constant, column, or function, and any combination of array operators."
69+
)
70+
)]
71+
#[derive(Debug, PartialEq, Eq, Hash)]
72+
pub struct ArrayAdd {
73+
signature: Signature,
74+
aliases: Vec<String>,
75+
}
76+
77+
impl Default for ArrayAdd {
78+
fn default() -> Self {
79+
Self::new()
80+
}
81+
}
82+
83+
impl ArrayAdd {
84+
pub fn new() -> Self {
85+
Self {
86+
signature: Signature::user_defined(Volatility::Immutable),
87+
aliases: vec!["list_add".to_string()],
88+
}
89+
}
90+
}
91+
92+
impl ScalarUDFImpl for ArrayAdd {
93+
fn name(&self) -> &str {
94+
"array_add"
95+
}
96+
97+
fn signature(&self) -> &Signature {
98+
&self.signature
99+
}
100+
101+
fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
102+
// After `coerce_types`, both args are List(Float64) or both LargeList(Float64).
103+
Ok(arg_types[0].clone())
104+
}
105+
106+
fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
107+
let [_, _] = take_function_args(self.name(), arg_types)?;
108+
let coercion = Some(&ListCoercion::FixedSizedListToList);
109+
110+
for arg_type in arg_types {
111+
if !matches!(arg_type, Null | List(_) | LargeList(_) | FixedSizeList(..)) {
112+
return plan_err!("{} does not support type {arg_type}", self.name());
113+
}
114+
}
115+
116+
// If any input is `LargeList`, both sides must be widened to `LargeList`
117+
// so the runtime dispatch in `array_add_inner` sees a homogeneous pair.
118+
// Follows the pattern in `InnerProduct::coerce_types`.
119+
let any_large_list = arg_types.iter().any(|t| matches!(t, LargeList(_)));
120+
121+
let coerced = arg_types
122+
.iter()
123+
.map(|arg_type| {
124+
if matches!(arg_type, Null) {
125+
let field = Arc::new(Field::new_list_field(DataType::Float64, true));
126+
return if any_large_list {
127+
LargeList(field)
128+
} else {
129+
List(field)
130+
};
131+
}
132+
let coerced = coerced_type_with_base_type_only(
133+
arg_type,
134+
&DataType::Float64,
135+
coercion,
136+
);
137+
match coerced {
138+
List(field) if any_large_list => LargeList(field),
139+
other => other,
140+
}
141+
})
142+
.collect();
143+
144+
Ok(coerced)
145+
}
146+
147+
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
148+
make_scalar_function(array_add_inner)(&args.args)
149+
}
150+
151+
fn aliases(&self) -> &[String] {
152+
&self.aliases
153+
}
154+
155+
fn documentation(&self) -> Option<&Documentation> {
156+
self.doc()
157+
}
158+
}
159+
160+
fn array_add_inner(args: &[ArrayRef]) -> Result<ArrayRef> {
161+
let [array1, array2] = take_function_args("array_add", args)?;
162+
match (array1.data_type(), array2.data_type()) {
163+
(List(_), List(_)) => general_array_add::<i32>(args),
164+
(LargeList(_), LargeList(_)) => general_array_add::<i64>(args),
165+
(arg_type1, arg_type2) => internal_err!(
166+
"array_add received unexpected types after coercion: {arg_type1} and {arg_type2}"
167+
),
168+
}
169+
}
170+
171+
fn general_array_add<O: OffsetSizeTrait>(arrays: &[ArrayRef]) -> Result<ArrayRef> {
172+
let list_array1 = as_generic_list_array::<O>(&arrays[0])?;
173+
let list_array2 = as_generic_list_array::<O>(&arrays[1])?;
174+
175+
let values1 = as_float64_array(list_array1.values())?;
176+
let values2 = as_float64_array(list_array2.values())?;
177+
let offsets1 = list_array1.value_offsets();
178+
let offsets2 = list_array2.value_offsets();
179+
180+
let mut value_builder = Float64Array::builder(values1.len().min(values2.len()));
181+
let mut new_offsets = OffsetBufferBuilder::<O>::new(list_array1.len());
182+
let mut row_nulls = NullBufferBuilder::new(list_array1.len());
183+
184+
for row in 0..list_array1.len() {
185+
if list_array1.is_null(row) || list_array2.is_null(row) {
186+
row_nulls.append_null();
187+
new_offsets.push_length(0);
188+
continue;
189+
}
190+
191+
let start1 = offsets1[row].as_usize();
192+
let end1 = offsets1[row + 1].as_usize();
193+
let start2 = offsets2[row].as_usize();
194+
let end2 = offsets2[row + 1].as_usize();
195+
let len1 = end1 - start1;
196+
let len2 = end2 - start2;
197+
198+
if len1 != len2 {
199+
return exec_err!(
200+
"array_add requires both list inputs to have the same length per row, got {len1} and {len2}"
201+
);
202+
}
203+
204+
let slice1 = values1.slice(start1, len1);
205+
let slice2 = values2.slice(start2, len2);
206+
207+
// Per-element add with per-element NULL propagation. A NULL on either side
208+
// yields a NULL at the same position in the output.
209+
for i in 0..len1 {
210+
if slice1.is_null(i) || slice2.is_null(i) {
211+
value_builder.append_null();
212+
} else {
213+
value_builder.append_value(slice1.value(i) + slice2.value(i));
214+
}
215+
}
216+
217+
row_nulls.append_non_null();
218+
new_offsets.push_length(len1);
219+
}
220+
221+
let values_array = Arc::new(value_builder.finish());
222+
let field = Arc::new(Field::new_list_field(DataType::Float64, true));
223+
224+
Ok(Arc::new(GenericListArray::<O>::try_new(
225+
field,
226+
new_offsets.finish(),
227+
values_array,
228+
row_nulls.finish(),
229+
)?))
230+
}

‎datafusion/functions-nested/src/lib.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ pub mod macros_lambda;
4343
pub mod array_any_match;
4444
pub(crate) mod lambda_utils;
4545

46+
pub mod array_add;
4647
pub mod array_compact;
4748
pub mod array_filter;
4849
pub mod array_has;
@@ -89,6 +90,7 @@ use std::sync::Arc;
8990

9091
/// Fluent-style API for creating `Expr`s
9192
pub mod expr_fn {
93+
pub use super::array_add::array_add;
9294
pub use super::array_any_match::array_any_match;
9395
pub use super::array_compact::array_compact;
9496
pub use super::array_filter::array_filter;
@@ -170,6 +172,7 @@ pub fn all_default_nested_functions() -> Vec<Arc<ScalarUDF>> {
170172
array_has::array_has_any_udf(),
171173
empty::array_empty_udf(),
172174
length::array_length_udf(),
175+
array_add::array_add_udf(),
173176
array_normalize::array_normalize_udf(),
174177
cosine_distance::cosine_distance_udf(),
175178
inner_product::inner_product_udf(),

0 commit comments

Comments
 (0)