1515// specific language governing permissions and limitations
1616// under the License.
1717
18- use arrow:: array:: { as_largestring_array, Array } ;
19- use arrow:: datatypes:: DataType ;
18+ use arrow:: array:: { as_largestring_array, Array , ArrayRef } ;
19+ use arrow:: datatypes:: { DataType , Field } ;
2020use datafusion_expr:: sort_properties:: ExprProperties ;
2121use std:: any:: Any ;
2222use std:: sync:: Arc ;
@@ -66,12 +66,12 @@ impl Default for ConcatFunc {
6666impl ConcatFunc {
6767 pub fn new ( ) -> Self {
6868 Self {
69- signature : Signature :: variadic_any ( Volatility :: Immutable ) ,
69+ signature : Signature :: user_defined ( Volatility :: Immutable ) ,
7070 }
7171 }
7272
7373 fn concat_arrays ( & self , args : & [ ColumnarValue ] ) -> Result < ColumnarValue > {
74- use arrow:: array:: { ArrayRef , ListArray } ;
74+ use arrow:: array:: { make_array , ArrayData , AsArray } ;
7575 use arrow:: compute;
7676
7777 let arrays: Result < Vec < ArrayRef > > = args
@@ -81,37 +81,82 @@ impl ConcatFunc {
8181 ColumnarValue :: Scalar ( scalar) => scalar. to_array_of_size ( 1 ) ,
8282 } )
8383 . collect ( ) ;
84-
8584 let arrays = arrays?;
86- let mut all_elements = Vec :: new ( ) ;
8785
88- for array in & arrays {
89- let Some ( list_array) = array. as_any ( ) . downcast_ref :: < ListArray > ( ) else {
90- return internal_err ! ( "Expected ListArray" ) ;
91- } ;
86+ let all_null = arrays. iter ( ) . all ( |array| array. null_count ( ) == array. len ( ) ) ;
87+
88+ if all_null {
89+ let return_type = arrays
90+ . iter ( )
91+ . map ( |arg| arg. data_type ( ) )
92+ . find ( |d| !d. is_null ( ) )
93+ . unwrap_or_else ( || arrays[ 0 ] . data_type ( ) )
94+ . clone ( ) ;
95+
96+ return Ok ( ColumnarValue :: Array ( make_array ( ArrayData :: new_null (
97+ & return_type,
98+ arrays[ 0 ] . len ( ) ,
99+ ) ) ) ) ;
100+ }
101+
102+ let mut result_elements = Vec :: new ( ) ;
92103
93- if !list_array. is_null ( 0 ) {
94- let elements = list_array. value ( 0 ) ;
95- for i in 0 ..elements. len ( ) {
96- all_elements. push ( elements. slice ( i, 1 ) ) ;
104+ for array in & arrays {
105+ match array. data_type ( ) {
106+ DataType :: List ( _) => {
107+ let list_array = array. as_list :: < i32 > ( ) ;
108+ for i in 0 ..list_array. len ( ) {
109+ if !list_array. is_null ( i) {
110+ let elements = list_array. value ( i) ;
111+ result_elements. extend (
112+ ( 0 ..elements. len ( ) ) . map ( |j| elements. slice ( j, 1 ) ) ,
113+ ) ;
114+ }
115+ }
116+ }
117+ DataType :: LargeList ( _) => {
118+ let list_array = array. as_list :: < i64 > ( ) ;
119+ for i in 0 ..list_array. len ( ) {
120+ if !list_array. is_null ( i) {
121+ let elements = list_array. value ( i) ;
122+ result_elements. extend (
123+ ( 0 ..elements. len ( ) ) . map ( |j| elements. slice ( j, 1 ) ) ,
124+ ) ;
125+ }
126+ }
127+ }
128+ DataType :: FixedSizeList ( _, size) => {
129+ let list_array = array. as_fixed_size_list ( ) ;
130+ for i in 0 ..list_array. len ( ) {
131+ if !list_array. is_null ( i) {
132+ let elements = list_array. value ( i) ;
133+ result_elements. extend (
134+ ( 0 ..* size as usize ) . map ( |j| elements. slice ( j, 1 ) ) ,
135+ ) ;
136+ }
137+ }
138+ }
139+ _ => {
140+ return internal_err ! (
141+ "Unsupported array type for concatenation: {}" ,
142+ array. data_type( )
143+ )
97144 }
98145 }
99146 }
100147
101- if all_elements . is_empty ( ) {
148+ if result_elements . is_empty ( ) {
102149 return plan_err ! ( "No elements to concatenate" ) ;
103150 }
104151
105152 let element_refs: Vec < & dyn Array > =
106- all_elements . iter ( ) . map ( |a| a. as_ref ( ) ) . collect ( ) ;
153+ result_elements . iter ( ) . map ( |a| a. as_ref ( ) ) . collect ( ) ;
107154 let concatenated = compute:: concat ( & element_refs) ?;
108155
109- let field = arrow:: datatypes:: Field :: new_list_field (
110- concatenated. data_type ( ) . clone ( ) ,
111- true ,
112- ) ;
156+ let field = Field :: new_list_field ( concatenated. data_type ( ) . clone ( ) , true ) ;
113157 let offsets = arrow:: buffer:: OffsetBuffer :: from_lengths ( [ concatenated. len ( ) ] ) ;
114- let result = ListArray :: new ( Arc :: new ( field) , offsets, concatenated, None ) ;
158+ let result =
159+ arrow:: array:: ListArray :: new ( Arc :: new ( field) , offsets, concatenated, None ) ;
115160
116161 Ok ( ColumnarValue :: Array ( Arc :: new ( result) ) )
117162 }
@@ -132,15 +177,39 @@ impl ScalarUDFImpl for ConcatFunc {
132177
133178 fn coerce_types ( & self , arg_types : & [ DataType ] ) -> Result < Vec < DataType > > {
134179 use DataType :: * ;
135- if arg_types. iter ( ) . any ( |dt| matches ! ( dt, List ( _) ) ) {
180+
181+ if arg_types
182+ . iter ( )
183+ . any ( |dt| matches ! ( dt, List ( _) | LargeList ( _) | FixedSizeList ( _, _) ) )
184+ {
136185 return Ok ( arg_types. to_vec ( ) ) ;
137186 }
138187
188+ for arg_type in arg_types {
189+ match arg_type {
190+ Utf8 | Utf8View | LargeUtf8 => { }
191+ Dictionary ( _, value_type) => {
192+ if !matches ! ( value_type. as_ref( ) , Utf8 | Utf8View | LargeUtf8 ) {
193+ return Ok ( vec ! [ Utf8 ; arg_types. len( ) ] ) ;
194+ }
195+ }
196+ _ => return Ok ( vec ! [ Utf8 ; arg_types. len( ) ] ) ,
197+ }
198+ }
199+
139200 let mut best_string_type = Utf8 ;
140201 for arg_type in arg_types {
141202 match arg_type {
142203 Utf8View => best_string_type = Utf8View ,
143204 LargeUtf8 if best_string_type != Utf8View => best_string_type = LargeUtf8 ,
205+ Utf8 => { }
206+ Dictionary ( _, value_type) => match value_type. as_ref ( ) {
207+ Utf8View => best_string_type = Utf8View ,
208+ LargeUtf8 if best_string_type != Utf8View => {
209+ best_string_type = LargeUtf8
210+ }
211+ _ => { }
212+ } ,
144213 _ => { }
145214 }
146215 }
@@ -150,7 +219,11 @@ impl ScalarUDFImpl for ConcatFunc {
150219
151220 fn return_type ( & self , arg_types : & [ DataType ] ) -> Result < DataType > {
152221 use DataType :: * ;
153- if arg_types. iter ( ) . any ( |dt| matches ! ( dt, List ( _) ) ) {
222+
223+ if arg_types
224+ . iter ( )
225+ . any ( |dt| matches ! ( dt, List ( _) | LargeList ( _) | FixedSizeList ( _, _) ) )
226+ {
154227 return Ok ( arg_types[ 0 ] . clone ( ) ) ;
155228 }
156229
@@ -172,13 +245,61 @@ impl ScalarUDFImpl for ConcatFunc {
172245 fn invoke_with_args ( & self , args : ScalarFunctionArgs ) -> Result < ColumnarValue > {
173246 let ScalarFunctionArgs { args, .. } = args;
174247
175- if args
176- . iter ( )
177- . any ( |arg| matches ! ( arg. data_type( ) , DataType :: List ( _) ) )
178- {
248+ if args. is_empty ( ) {
249+ return plan_err ! ( "concat requires at least one argument" ) ;
250+ }
251+
252+ if args. iter ( ) . any ( |arg| {
253+ matches ! (
254+ arg. data_type( ) ,
255+ DataType :: List ( _)
256+ | DataType :: LargeList ( _)
257+ | DataType :: FixedSizeList ( _, _)
258+ )
259+ } ) {
179260 return self . concat_arrays ( & args) ;
180261 }
181262
263+ let has_dictionary_types = args
264+ . iter ( )
265+ . any ( |arg| matches ! ( arg. data_type( ) , DataType :: Dictionary ( _, _) ) ) ;
266+ let args = if has_dictionary_types {
267+ let mut processed_args = Vec :: with_capacity ( args. len ( ) ) ;
268+ for arg in args {
269+ match arg. data_type ( ) {
270+ DataType :: Dictionary ( _, value_type) => match value_type. as_ref ( ) {
271+ DataType :: Utf8 | DataType :: LargeUtf8 | DataType :: Utf8View => {
272+ use arrow:: compute:: cast;
273+ match arg {
274+ ColumnarValue :: Array ( array) => {
275+ let casted =
276+ cast ( array. as_ref ( ) , value_type. as_ref ( ) ) ?;
277+ processed_args. push ( ColumnarValue :: Array ( casted) ) ;
278+ }
279+ ColumnarValue :: Scalar ( ref scalar) => match scalar {
280+ ScalarValue :: Dictionary ( _, v) => {
281+ processed_args. push ( ColumnarValue :: Scalar (
282+ v. as_ref ( ) . clone ( ) ,
283+ ) ) ;
284+ }
285+ _ => processed_args. push ( arg) ,
286+ } ,
287+ }
288+ }
289+ _ => {
290+ return plan_err ! ( "Dictionary with value type {value_type} is not supported for concat" ) ;
291+ }
292+ } ,
293+ _ => {
294+ processed_args. push ( arg) ;
295+ }
296+ }
297+ }
298+ processed_args
299+ } else {
300+ args
301+ } ;
302+
182303 let mut return_datatype = DataType :: Utf8 ;
183304 args. iter ( ) . for_each ( |col| {
184305 if col. data_type ( ) == DataType :: Utf8View {
@@ -292,7 +413,12 @@ impl ScalarUDFImpl for ConcatFunc {
292413 }
293414 } ;
294415 }
295- _ => unreachable ! ( "concat" ) ,
416+ _ => {
417+ return plan_err ! (
418+ "Unsupported argument type for concat: {}" ,
419+ arg. data_type( )
420+ )
421+ }
296422 }
297423 }
298424
@@ -333,7 +459,9 @@ impl ScalarUDFImpl for ConcatFunc {
333459 let string_array = builder. finish ( None ) ;
334460 Ok ( ColumnarValue :: Array ( Arc :: new ( string_array) ) )
335461 }
336- _ => unreachable ! ( ) ,
462+ _ => {
463+ plan_err ! ( "Unsupported return datatype for concat: {return_datatype}" )
464+ }
337465 }
338466 }
339467
@@ -366,19 +494,6 @@ pub(crate) fn simplify_concat(args: Vec<Expr>) -> Result<ExprSimplifyResult> {
366494 let mut new_args = Vec :: with_capacity ( args. len ( ) ) ;
367495 let mut contiguous_scalar = "" . to_string ( ) ;
368496
369- // Check if any arguments are array types - if so, don't simplify
370- let has_array_types = args. iter ( ) . any ( |expr| match expr {
371- Expr :: Literal ( l, _) => matches ! (
372- l. data_type( ) ,
373- DataType :: List ( _) | DataType :: LargeList ( _) | DataType :: FixedSizeList ( _, _)
374- ) ,
375- _ => false , // For non-literals, we can't determine the type, so assume they might be arrays
376- } ) ;
377-
378- if has_array_types {
379- return Ok ( ExprSimplifyResult :: Original ( args) ) ;
380- }
381-
382497 let return_type = {
383498 let data_types: Vec < _ > = args
384499 . iter ( )
@@ -396,9 +511,6 @@ pub(crate) fn simplify_concat(args: Vec<Expr>) -> Result<ExprSimplifyResult> {
396511 Expr :: Literal ( ScalarValue :: LargeUtf8 ( None ) , _) => { }
397512 Expr :: Literal ( ScalarValue :: Utf8View ( None ) , _) => { }
398513
399- // filter out `null` args
400- // All literals have been converted to Utf8 or LargeUtf8 in type_coercion.
401- // Concatenate it with the `contiguous_scalar`.
402514 Expr :: Literal ( ScalarValue :: Utf8 ( Some ( v) ) , _) => {
403515 contiguous_scalar += v;
404516 }
@@ -410,24 +522,19 @@ pub(crate) fn simplify_concat(args: Vec<Expr>) -> Result<ExprSimplifyResult> {
410522 }
411523
412524 Expr :: Literal ( _x, _) => {
413- // For variadic_any signature, non-string literals may not be coerced
414- // Skip simplification for non-string literals
415525 if !contiguous_scalar. is_empty ( ) {
416526 match return_type {
417527 DataType :: Utf8 => new_args. push ( lit ( contiguous_scalar) ) ,
418528 DataType :: LargeUtf8 => new_args
419529 . push ( lit ( ScalarValue :: LargeUtf8 ( Some ( contiguous_scalar) ) ) ) ,
420530 DataType :: Utf8View => new_args
421531 . push ( lit ( ScalarValue :: Utf8View ( Some ( contiguous_scalar) ) ) ) ,
422- _ => unreachable ! ( ) ,
532+ _ => return Ok ( ExprSimplifyResult :: Original ( args ) ) ,
423533 }
424534 contiguous_scalar = "" . to_string ( ) ;
425535 }
426536 new_args. push ( arg. clone ( ) ) ;
427537 }
428- // If the arg is not a literal, we should first push the current `contiguous_scalar`
429- // to the `new_args` (if it is not empty) and reset it to empty string.
430- // Then pushing this arg to the `new_args`.
431538 arg => {
432539 if !contiguous_scalar. is_empty ( ) {
433540 match return_type {
@@ -436,7 +543,7 @@ pub(crate) fn simplify_concat(args: Vec<Expr>) -> Result<ExprSimplifyResult> {
436543 . push ( lit ( ScalarValue :: LargeUtf8 ( Some ( contiguous_scalar) ) ) ) ,
437544 DataType :: Utf8View => new_args
438545 . push ( lit ( ScalarValue :: Utf8View ( Some ( contiguous_scalar) ) ) ) ,
439- _ => unreachable ! ( ) ,
546+ _ => return Ok ( ExprSimplifyResult :: Original ( args ) ) ,
440547 }
441548 contiguous_scalar = "" . to_string ( ) ;
442549 }
@@ -454,7 +561,7 @@ pub(crate) fn simplify_concat(args: Vec<Expr>) -> Result<ExprSimplifyResult> {
454561 DataType :: Utf8View => {
455562 new_args. push ( lit ( ScalarValue :: Utf8View ( Some ( contiguous_scalar) ) ) )
456563 }
457- _ => unreachable ! ( ) ,
564+ _ => return Ok ( ExprSimplifyResult :: Original ( args ) ) ,
458565 }
459566 }
460567
0 commit comments