@@ -65,14 +65,56 @@ impl Default for ConcatFunc {
6565
6666impl ConcatFunc {
6767 pub fn new ( ) -> Self {
68- use DataType :: * ;
6968 Self {
70- signature : Signature :: variadic (
71- vec ! [ Utf8View , Utf8 , LargeUtf8 ] ,
72- Volatility :: Immutable ,
73- ) ,
69+ signature : Signature :: variadic_any ( Volatility :: Immutable ) ,
7470 }
7571 }
72+
73+ fn concat_arrays ( & self , args : & [ ColumnarValue ] ) -> Result < ColumnarValue > {
74+ use arrow:: array:: { ArrayRef , ListArray } ;
75+ use arrow:: compute;
76+
77+ let arrays: Result < Vec < ArrayRef > > = args
78+ . iter ( )
79+ . map ( |arg| match arg {
80+ ColumnarValue :: Array ( array) => Ok ( Arc :: clone ( array) ) ,
81+ ColumnarValue :: Scalar ( scalar) => scalar. to_array_of_size ( 1 ) ,
82+ } )
83+ . collect ( ) ;
84+
85+ let arrays = arrays?;
86+ let mut all_elements = Vec :: new ( ) ;
87+
88+ for array in & arrays {
89+ let Some ( list_array) = array. as_any ( ) . downcast_ref :: < ListArray > ( ) else {
90+ return internal_err ! ( "Expected ListArray" ) ;
91+ } ;
92+
93+ if !list_array. is_null ( 0 ) {
94+ let elements = list_array. value ( 0 ) ;
95+ for i in 0 ..elements. len ( ) {
96+ all_elements. push ( elements. slice ( i, 1 ) ) ;
97+ }
98+ }
99+ }
100+
101+ if all_elements. is_empty ( ) {
102+ return plan_err ! ( "No elements to concatenate" ) ;
103+ }
104+
105+ let element_refs: Vec < & dyn Array > =
106+ all_elements. iter ( ) . map ( |a| a. as_ref ( ) ) . collect ( ) ;
107+ let concatenated = compute:: concat ( & element_refs) ?;
108+
109+ let field = arrow:: datatypes:: Field :: new_list_field (
110+ concatenated. data_type ( ) . clone ( ) ,
111+ true ,
112+ ) ;
113+ let offsets = arrow:: buffer:: OffsetBuffer :: from_lengths ( [ concatenated. len ( ) ] ) ;
114+ let result = ListArray :: new ( Arc :: new ( field) , offsets, concatenated, None ) ;
115+
116+ Ok ( ColumnarValue :: Array ( Arc :: new ( result) ) )
117+ }
76118}
77119
78120impl ScalarUDFImpl for ConcatFunc {
@@ -88,8 +130,30 @@ impl ScalarUDFImpl for ConcatFunc {
88130 & self . signature
89131 }
90132
133+ fn coerce_types ( & self , arg_types : & [ DataType ] ) -> Result < Vec < DataType > > {
134+ use DataType :: * ;
135+ if arg_types. iter ( ) . any ( |dt| matches ! ( dt, List ( _) ) ) {
136+ return Ok ( arg_types. to_vec ( ) ) ;
137+ }
138+
139+ let mut best_string_type = Utf8 ;
140+ for arg_type in arg_types {
141+ match arg_type {
142+ Utf8View => best_string_type = Utf8View ,
143+ LargeUtf8 if best_string_type != Utf8View => best_string_type = LargeUtf8 ,
144+ _ => { }
145+ }
146+ }
147+
148+ Ok ( vec ! [ best_string_type; arg_types. len( ) ] )
149+ }
150+
91151 fn return_type ( & self , arg_types : & [ DataType ] ) -> Result < DataType > {
92152 use DataType :: * ;
153+ if arg_types. iter ( ) . any ( |dt| matches ! ( dt, List ( _) ) ) {
154+ return Ok ( arg_types[ 0 ] . clone ( ) ) ;
155+ }
156+
93157 let mut dt = & Utf8 ;
94158 arg_types. iter ( ) . for_each ( |data_type| {
95159 if data_type == & Utf8View {
@@ -108,6 +172,13 @@ impl ScalarUDFImpl for ConcatFunc {
108172 fn invoke_with_args ( & self , args : ScalarFunctionArgs ) -> Result < ColumnarValue > {
109173 let ScalarFunctionArgs { args, .. } = args;
110174
175+ if args
176+ . iter ( )
177+ . any ( |arg| matches ! ( arg. data_type( ) , DataType :: List ( _) ) )
178+ {
179+ return self . concat_arrays ( & args) ;
180+ }
181+
111182 let mut return_datatype = DataType :: Utf8 ;
112183 args. iter ( ) . for_each ( |col| {
113184 if col. data_type ( ) == DataType :: Utf8View {
@@ -139,10 +210,14 @@ impl ScalarUDFImpl for ConcatFunc {
139210 match scalar. try_as_str ( ) {
140211 Some ( Some ( v) ) => result. push_str ( v) ,
141212 Some ( None ) => { } // null literal
142- None => plan_err ! (
143- "Concat function does not support scalar type {}" ,
144- scalar
145- ) ?,
213+ None => {
214+ // For non-string types, convert to string representation
215+ if scalar. is_null ( ) {
216+ // Skip null values
217+ } else {
218+ result. push_str ( & format ! ( "{}" , scalar) ) ;
219+ }
220+ }
146221 }
147222 }
148223
@@ -291,6 +366,19 @@ pub(crate) fn simplify_concat(args: Vec<Expr>) -> Result<ExprSimplifyResult> {
291366 let mut new_args = Vec :: with_capacity ( args. len ( ) ) ;
292367 let mut contiguous_scalar = "" . to_string ( ) ;
293368
369+ // Check if any arguments are array types - if so, don't simplify
370+ let has_array_types = args. iter ( ) . any ( |expr| match expr {
371+ Expr :: Literal ( l, _) => matches ! (
372+ l. data_type( ) ,
373+ DataType :: List ( _) | DataType :: LargeList ( _) | DataType :: FixedSizeList ( _, _)
374+ ) ,
375+ _ => false , // For non-literals, we can't determine the type, so assume they might be arrays
376+ } ) ;
377+
378+ if has_array_types {
379+ return Ok ( ExprSimplifyResult :: Original ( args) ) ;
380+ }
381+
294382 let return_type = {
295383 let data_types: Vec < _ > = args
296384 . iter ( )
@@ -302,12 +390,11 @@ pub(crate) fn simplify_concat(args: Vec<Expr>) -> Result<ExprSimplifyResult> {
302390 ConcatFunc :: new ( ) . return_type ( & data_types)
303391 } ?;
304392
305- for arg in args. clone ( ) {
393+ for arg in args. iter ( ) {
306394 match arg {
307395 Expr :: Literal ( ScalarValue :: Utf8 ( None ) , _) => { }
308- Expr :: Literal ( ScalarValue :: LargeUtf8 ( None ) , _) => {
309- }
310- Expr :: Literal ( ScalarValue :: Utf8View ( None ) , _) => { }
396+ Expr :: Literal ( ScalarValue :: LargeUtf8 ( None ) , _) => { }
397+ Expr :: Literal ( ScalarValue :: Utf8View ( None ) , _) => { }
311398
312399 // filter out `null` args
313400 // All literals have been converted to Utf8 or LargeUtf8 in type_coercion.
@@ -322,10 +409,21 @@ pub(crate) fn simplify_concat(args: Vec<Expr>) -> Result<ExprSimplifyResult> {
322409 contiguous_scalar += & v;
323410 }
324411
325- Expr :: Literal ( x, _) => {
326- return internal_err ! (
327- "The scalar {x} should be casted to string type during the type coercion."
328- )
412+ Expr :: Literal ( _x, _) => {
413+ // For variadic_any signature, non-string literals may not be coerced
414+ // Skip simplification for non-string literals
415+ if !contiguous_scalar. is_empty ( ) {
416+ match return_type {
417+ DataType :: Utf8 => new_args. push ( lit ( contiguous_scalar) ) ,
418+ DataType :: LargeUtf8 => new_args
419+ . push ( lit ( ScalarValue :: LargeUtf8 ( Some ( contiguous_scalar) ) ) ) ,
420+ DataType :: Utf8View => new_args
421+ . push ( lit ( ScalarValue :: Utf8View ( Some ( contiguous_scalar) ) ) ) ,
422+ _ => unreachable ! ( ) ,
423+ }
424+ contiguous_scalar = "" . to_string ( ) ;
425+ }
426+ new_args. push ( arg. clone ( ) ) ;
329427 }
330428 // If the arg is not a literal, we should first push the current `contiguous_scalar`
331429 // to the `new_args` (if it is not empty) and reset it to empty string.
@@ -334,13 +432,15 @@ pub(crate) fn simplify_concat(args: Vec<Expr>) -> Result<ExprSimplifyResult> {
334432 if !contiguous_scalar. is_empty ( ) {
335433 match return_type {
336434 DataType :: Utf8 => new_args. push ( lit ( contiguous_scalar) ) ,
337- DataType :: LargeUtf8 => new_args. push ( lit ( ScalarValue :: LargeUtf8 ( Some ( contiguous_scalar) ) ) ) ,
338- DataType :: Utf8View => new_args. push ( lit ( ScalarValue :: Utf8View ( Some ( contiguous_scalar) ) ) ) ,
435+ DataType :: LargeUtf8 => new_args
436+ . push ( lit ( ScalarValue :: LargeUtf8 ( Some ( contiguous_scalar) ) ) ) ,
437+ DataType :: Utf8View => new_args
438+ . push ( lit ( ScalarValue :: Utf8View ( Some ( contiguous_scalar) ) ) ) ,
339439 _ => unreachable ! ( ) ,
340440 }
341441 contiguous_scalar = "" . to_string ( ) ;
342442 }
343- new_args. push ( arg) ;
443+ new_args. push ( arg. clone ( ) ) ;
344444 }
345445 }
346446 }
@@ -501,4 +601,87 @@ mod tests {
501601 }
502602 Ok ( ( ) )
503603 }
604+
605+ #[ test]
606+ fn test_array_concat ( ) -> Result < ( ) > {
607+ use arrow:: array:: { Int64Array , ListArray } ;
608+ use arrow:: buffer:: OffsetBuffer ;
609+ use DataType :: * ;
610+
611+ // Create list arrays: [1, 2] and [3, 4]
612+ let list1_values = Arc :: new ( Int64Array :: from ( vec ! [ 1 , 2 ] ) ) ;
613+ let list1_field = Arc :: new ( Field :: new_list_field ( Int64 , true ) ) ;
614+ let list1_offsets = OffsetBuffer :: from_lengths ( [ 2 ] ) ;
615+ let list1 =
616+ ListArray :: new ( list1_field. clone ( ) , list1_offsets, list1_values, None ) ;
617+
618+ let list2_values = Arc :: new ( Int64Array :: from ( vec ! [ 3 , 4 ] ) ) ;
619+ let list2_offsets = OffsetBuffer :: from_lengths ( [ 2 ] ) ;
620+ let list2 =
621+ ListArray :: new ( list1_field. clone ( ) , list2_offsets, list2_values, None ) ;
622+
623+ let args = vec ! [
624+ ColumnarValue :: Array ( Arc :: new( list1) ) ,
625+ ColumnarValue :: Array ( Arc :: new( list2) ) ,
626+ ] ;
627+
628+ let result = ConcatFunc :: new ( ) . concat_arrays ( & args) ?;
629+
630+ // Expected result: [1, 2, 3, 4]
631+ match result {
632+ ColumnarValue :: Array ( array) => {
633+ let list_array = array. as_any ( ) . downcast_ref :: < ListArray > ( ) . unwrap ( ) ;
634+ assert_eq ! ( list_array. len( ) , 1 ) ;
635+ let values = list_array. value ( 0 ) ;
636+ let int_values = values. as_any ( ) . downcast_ref :: < Int64Array > ( ) . unwrap ( ) ;
637+ assert_eq ! ( int_values. values( ) , & [ 1 , 2 , 3 , 4 ] ) ;
638+ }
639+ _ => panic ! ( "Expected array result" ) ,
640+ }
641+
642+ Ok ( ( ) )
643+ }
644+
645+ #[ test]
646+ fn test_concat_with_integers ( ) -> Result < ( ) > {
647+ use datafusion_common:: config:: ConfigOptions ;
648+ use DataType :: * ;
649+
650+ let args = vec ! [
651+ ColumnarValue :: Scalar ( ScalarValue :: Utf8 ( Some ( "abc" . to_string( ) ) ) ) ,
652+ ColumnarValue :: Scalar ( ScalarValue :: Int64 ( Some ( 123 ) ) ) ,
653+ ColumnarValue :: Scalar ( ScalarValue :: Utf8 ( None ) ) , // NULL
654+ ColumnarValue :: Scalar ( ScalarValue :: Int64 ( Some ( 456 ) ) ) ,
655+ ] ;
656+
657+ let arg_fields = vec ! [
658+ Field :: new( "a" , Utf8 , true ) ,
659+ Field :: new( "b" , Int64 , true ) ,
660+ Field :: new( "c" , Utf8 , true ) ,
661+ Field :: new( "d" , Int64 , true ) ,
662+ ]
663+ . into_iter ( )
664+ . map ( Arc :: new)
665+ . collect :: < Vec < _ > > ( ) ;
666+
667+ let func_args = ScalarFunctionArgs {
668+ args,
669+ arg_fields,
670+ number_rows : 1 ,
671+ return_field : Field :: new ( "f" , Utf8 , true ) . into ( ) ,
672+ config_options : Arc :: new ( ConfigOptions :: default ( ) ) ,
673+ } ;
674+
675+ let result = ConcatFunc :: new ( ) . invoke_with_args ( func_args) ?;
676+
677+ // Expected result should be "abc123456"
678+ match result {
679+ ColumnarValue :: Scalar ( ScalarValue :: Utf8 ( Some ( s) ) ) => {
680+ assert_eq ! ( s, "abc123456" ) ;
681+ }
682+ _ => panic ! ( "Expected scalar UTF8 result, got {:?}" , result) ,
683+ }
684+
685+ Ok ( ( ) )
686+ }
504687}
0 commit comments