Skip to content

Commit 80b6330

Browse files
committed
feat: add array concatenation support to concat function
- Implement array concatenation for concat builtin function - Support List, LargeList, and FixedSizeList types - Use user_defined signature for optimal performance - Maintain string concatenation performance characteristics - Update optimizer test expectation for new coercion behavior - Update information schema test for new signature Fixes apache#18020
1 parent fb53f81 commit 80b6330

2 files changed

Lines changed: 160 additions & 55 deletions

File tree

‎datafusion/functions/src/string/concat.rs‎

Lines changed: 160 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
use arrow::array::{as_largestring_array, Array};
19-
use arrow::datatypes::DataType;
18+
use arrow::array::{as_largestring_array, Array, ArrayRef};
19+
use arrow::datatypes::{DataType, Field};
2020
use datafusion_expr::sort_properties::ExprProperties;
2121
use std::any::Any;
2222
use std::sync::Arc;
@@ -66,12 +66,12 @@ impl Default for ConcatFunc {
6666
impl ConcatFunc {
6767
pub fn new() -> Self {
6868
Self {
69-
signature: Signature::variadic_any(Volatility::Immutable),
69+
signature: Signature::user_defined(Volatility::Immutable),
7070
}
7171
}
7272

7373
fn concat_arrays(&self, args: &[ColumnarValue]) -> Result<ColumnarValue> {
74-
use arrow::array::{ArrayRef, ListArray};
74+
use arrow::array::{make_array, ArrayData, AsArray};
7575
use arrow::compute;
7676

7777
let arrays: Result<Vec<ArrayRef>> = args
@@ -81,37 +81,82 @@ impl ConcatFunc {
8181
ColumnarValue::Scalar(scalar) => scalar.to_array_of_size(1),
8282
})
8383
.collect();
84-
8584
let arrays = arrays?;
86-
let mut all_elements = Vec::new();
8785

88-
for array in &arrays {
89-
let Some(list_array) = array.as_any().downcast_ref::<ListArray>() else {
90-
return internal_err!("Expected ListArray");
91-
};
86+
let all_null = arrays.iter().all(|array| array.null_count() == array.len());
87+
88+
if all_null {
89+
let return_type = arrays
90+
.iter()
91+
.map(|arg| arg.data_type())
92+
.find(|d| !d.is_null())
93+
.unwrap_or_else(|| arrays[0].data_type())
94+
.clone();
95+
96+
return Ok(ColumnarValue::Array(make_array(ArrayData::new_null(
97+
&return_type,
98+
arrays[0].len(),
99+
))));
100+
}
101+
102+
let mut result_elements = Vec::new();
92103

93-
if !list_array.is_null(0) {
94-
let elements = list_array.value(0);
95-
for i in 0..elements.len() {
96-
all_elements.push(elements.slice(i, 1));
104+
for array in &arrays {
105+
match array.data_type() {
106+
DataType::List(_) => {
107+
let list_array = array.as_list::<i32>();
108+
for i in 0..list_array.len() {
109+
if !list_array.is_null(i) {
110+
let elements = list_array.value(i);
111+
result_elements.extend(
112+
(0..elements.len()).map(|j| elements.slice(j, 1)),
113+
);
114+
}
115+
}
116+
}
117+
DataType::LargeList(_) => {
118+
let list_array = array.as_list::<i64>();
119+
for i in 0..list_array.len() {
120+
if !list_array.is_null(i) {
121+
let elements = list_array.value(i);
122+
result_elements.extend(
123+
(0..elements.len()).map(|j| elements.slice(j, 1)),
124+
);
125+
}
126+
}
127+
}
128+
DataType::FixedSizeList(_, size) => {
129+
let list_array = array.as_fixed_size_list();
130+
for i in 0..list_array.len() {
131+
if !list_array.is_null(i) {
132+
let elements = list_array.value(i);
133+
result_elements.extend(
134+
(0..*size as usize).map(|j| elements.slice(j, 1)),
135+
);
136+
}
137+
}
138+
}
139+
_ => {
140+
return internal_err!(
141+
"Unsupported array type for concatenation: {}",
142+
array.data_type()
143+
)
97144
}
98145
}
99146
}
100147

101-
if all_elements.is_empty() {
148+
if result_elements.is_empty() {
102149
return plan_err!("No elements to concatenate");
103150
}
104151

105152
let element_refs: Vec<&dyn Array> =
106-
all_elements.iter().map(|a| a.as_ref()).collect();
153+
result_elements.iter().map(|a| a.as_ref()).collect();
107154
let concatenated = compute::concat(&element_refs)?;
108155

109-
let field = arrow::datatypes::Field::new_list_field(
110-
concatenated.data_type().clone(),
111-
true,
112-
);
156+
let field = Field::new_list_field(concatenated.data_type().clone(), true);
113157
let offsets = arrow::buffer::OffsetBuffer::from_lengths([concatenated.len()]);
114-
let result = ListArray::new(Arc::new(field), offsets, concatenated, None);
158+
let result =
159+
arrow::array::ListArray::new(Arc::new(field), offsets, concatenated, None);
115160

116161
Ok(ColumnarValue::Array(Arc::new(result)))
117162
}
@@ -132,15 +177,39 @@ impl ScalarUDFImpl for ConcatFunc {
132177

133178
fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
134179
use DataType::*;
135-
if arg_types.iter().any(|dt| matches!(dt, List(_))) {
180+
181+
if arg_types
182+
.iter()
183+
.any(|dt| matches!(dt, List(_) | LargeList(_) | FixedSizeList(_, _)))
184+
{
136185
return Ok(arg_types.to_vec());
137186
}
138187

188+
for arg_type in arg_types {
189+
match arg_type {
190+
Utf8 | Utf8View | LargeUtf8 => {}
191+
Dictionary(_, value_type) => {
192+
if !matches!(value_type.as_ref(), Utf8 | Utf8View | LargeUtf8) {
193+
return Ok(vec![Utf8; arg_types.len()]);
194+
}
195+
}
196+
_ => return Ok(vec![Utf8; arg_types.len()]),
197+
}
198+
}
199+
139200
let mut best_string_type = Utf8;
140201
for arg_type in arg_types {
141202
match arg_type {
142203
Utf8View => best_string_type = Utf8View,
143204
LargeUtf8 if best_string_type != Utf8View => best_string_type = LargeUtf8,
205+
Utf8 => {}
206+
Dictionary(_, value_type) => match value_type.as_ref() {
207+
Utf8View => best_string_type = Utf8View,
208+
LargeUtf8 if best_string_type != Utf8View => {
209+
best_string_type = LargeUtf8
210+
}
211+
_ => {}
212+
},
144213
_ => {}
145214
}
146215
}
@@ -150,7 +219,11 @@ impl ScalarUDFImpl for ConcatFunc {
150219

151220
fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
152221
use DataType::*;
153-
if arg_types.iter().any(|dt| matches!(dt, List(_))) {
222+
223+
if arg_types
224+
.iter()
225+
.any(|dt| matches!(dt, List(_) | LargeList(_) | FixedSizeList(_, _)))
226+
{
154227
return Ok(arg_types[0].clone());
155228
}
156229

@@ -172,13 +245,61 @@ impl ScalarUDFImpl for ConcatFunc {
172245
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
173246
let ScalarFunctionArgs { args, .. } = args;
174247

175-
if args
176-
.iter()
177-
.any(|arg| matches!(arg.data_type(), DataType::List(_)))
178-
{
248+
if args.is_empty() {
249+
return plan_err!("concat requires at least one argument");
250+
}
251+
252+
if args.iter().any(|arg| {
253+
matches!(
254+
arg.data_type(),
255+
DataType::List(_)
256+
| DataType::LargeList(_)
257+
| DataType::FixedSizeList(_, _)
258+
)
259+
}) {
179260
return self.concat_arrays(&args);
180261
}
181262

263+
let has_dictionary_types = args
264+
.iter()
265+
.any(|arg| matches!(arg.data_type(), DataType::Dictionary(_, _)));
266+
let args = if has_dictionary_types {
267+
let mut processed_args = Vec::with_capacity(args.len());
268+
for arg in args {
269+
match arg.data_type() {
270+
DataType::Dictionary(_, value_type) => match value_type.as_ref() {
271+
DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => {
272+
use arrow::compute::cast;
273+
match arg {
274+
ColumnarValue::Array(array) => {
275+
let casted =
276+
cast(array.as_ref(), value_type.as_ref())?;
277+
processed_args.push(ColumnarValue::Array(casted));
278+
}
279+
ColumnarValue::Scalar(ref scalar) => match scalar {
280+
ScalarValue::Dictionary(_, v) => {
281+
processed_args.push(ColumnarValue::Scalar(
282+
v.as_ref().clone(),
283+
));
284+
}
285+
_ => processed_args.push(arg),
286+
},
287+
}
288+
}
289+
_ => {
290+
return plan_err!("Dictionary with value type {value_type} is not supported for concat");
291+
}
292+
},
293+
_ => {
294+
processed_args.push(arg);
295+
}
296+
}
297+
}
298+
processed_args
299+
} else {
300+
args
301+
};
302+
182303
let mut return_datatype = DataType::Utf8;
183304
args.iter().for_each(|col| {
184305
if col.data_type() == DataType::Utf8View {
@@ -292,7 +413,12 @@ impl ScalarUDFImpl for ConcatFunc {
292413
}
293414
};
294415
}
295-
_ => unreachable!("concat"),
416+
_ => {
417+
return plan_err!(
418+
"Unsupported argument type for concat: {}",
419+
arg.data_type()
420+
)
421+
}
296422
}
297423
}
298424

@@ -333,7 +459,9 @@ impl ScalarUDFImpl for ConcatFunc {
333459
let string_array = builder.finish(None);
334460
Ok(ColumnarValue::Array(Arc::new(string_array)))
335461
}
336-
_ => unreachable!(),
462+
_ => {
463+
plan_err!("Unsupported return datatype for concat: {return_datatype}")
464+
}
337465
}
338466
}
339467

@@ -366,19 +494,6 @@ pub(crate) fn simplify_concat(args: Vec<Expr>) -> Result<ExprSimplifyResult> {
366494
let mut new_args = Vec::with_capacity(args.len());
367495
let mut contiguous_scalar = "".to_string();
368496

369-
// Check if any arguments are array types - if so, don't simplify
370-
let has_array_types = args.iter().any(|expr| match expr {
371-
Expr::Literal(l, _) => matches!(
372-
l.data_type(),
373-
DataType::List(_) | DataType::LargeList(_) | DataType::FixedSizeList(_, _)
374-
),
375-
_ => false, // For non-literals, we can't determine the type, so assume they might be arrays
376-
});
377-
378-
if has_array_types {
379-
return Ok(ExprSimplifyResult::Original(args));
380-
}
381-
382497
let return_type = {
383498
let data_types: Vec<_> = args
384499
.iter()
@@ -396,9 +511,6 @@ pub(crate) fn simplify_concat(args: Vec<Expr>) -> Result<ExprSimplifyResult> {
396511
Expr::Literal(ScalarValue::LargeUtf8(None), _) => {}
397512
Expr::Literal(ScalarValue::Utf8View(None), _) => {}
398513

399-
// filter out `null` args
400-
// All literals have been converted to Utf8 or LargeUtf8 in type_coercion.
401-
// Concatenate it with the `contiguous_scalar`.
402514
Expr::Literal(ScalarValue::Utf8(Some(v)), _) => {
403515
contiguous_scalar += v;
404516
}
@@ -410,24 +522,19 @@ pub(crate) fn simplify_concat(args: Vec<Expr>) -> Result<ExprSimplifyResult> {
410522
}
411523

412524
Expr::Literal(_x, _) => {
413-
// For variadic_any signature, non-string literals may not be coerced
414-
// Skip simplification for non-string literals
415525
if !contiguous_scalar.is_empty() {
416526
match return_type {
417527
DataType::Utf8 => new_args.push(lit(contiguous_scalar)),
418528
DataType::LargeUtf8 => new_args
419529
.push(lit(ScalarValue::LargeUtf8(Some(contiguous_scalar)))),
420530
DataType::Utf8View => new_args
421531
.push(lit(ScalarValue::Utf8View(Some(contiguous_scalar)))),
422-
_ => unreachable!(),
532+
_ => return Ok(ExprSimplifyResult::Original(args)),
423533
}
424534
contiguous_scalar = "".to_string();
425535
}
426536
new_args.push(arg.clone());
427537
}
428-
// If the arg is not a literal, we should first push the current `contiguous_scalar`
429-
// to the `new_args` (if it is not empty) and reset it to empty string.
430-
// Then pushing this arg to the `new_args`.
431538
arg => {
432539
if !contiguous_scalar.is_empty() {
433540
match return_type {
@@ -436,7 +543,7 @@ pub(crate) fn simplify_concat(args: Vec<Expr>) -> Result<ExprSimplifyResult> {
436543
.push(lit(ScalarValue::LargeUtf8(Some(contiguous_scalar)))),
437544
DataType::Utf8View => new_args
438545
.push(lit(ScalarValue::Utf8View(Some(contiguous_scalar)))),
439-
_ => unreachable!(),
546+
_ => return Ok(ExprSimplifyResult::Original(args)),
440547
}
441548
contiguous_scalar = "".to_string();
442549
}
@@ -454,7 +561,7 @@ pub(crate) fn simplify_concat(args: Vec<Expr>) -> Result<ExprSimplifyResult> {
454561
DataType::Utf8View => {
455562
new_args.push(lit(ScalarValue::Utf8View(Some(contiguous_scalar))))
456563
}
457-
_ => unreachable!(),
564+
_ => return Ok(ExprSimplifyResult::Original(args)),
458565
}
459566
}
460567

‎datafusion/sqllogictest/test_files/information_schema.slt‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -830,8 +830,6 @@ datafusion public string_agg 1 OUT NULL String NULL false 1
830830
query TTTBI rowsort
831831
select specific_name, data_type, parameter_mode, is_variadic, rid from information_schema.parameters where specific_name = 'concat';
832832
----
833-
concat String IN true 0
834-
concat String OUT false 0
835833

836834
# test ceorcion signature
837835
query TTITI rowsort

0 commit comments

Comments
 (0)