@@ -77,134 +77,6 @@ impl ConcatFunc {
7777 }
7878 }
7979
80- /// Extract array elements from a single row of arguments for array concatenation
81- fn extract_arrays_from_row (
82- & self ,
83- args : & [ ColumnarValue ] ,
84- row_idx : usize ,
85- ) -> Result < Vec < Arc < dyn Array > > > {
86- let mut inner_arrays = Vec :: new ( ) ;
87-
88- for arg in args {
89- match arg {
90- ColumnarValue :: Array ( arr) => {
91- if let Some ( array) = self . extract_list_value ( arr, row_idx) ? {
92- inner_arrays. push ( array) ;
93- }
94- }
95- ColumnarValue :: Scalar ( scalar) => {
96- if let Some ( array) = self . extract_scalar_list_value ( scalar) ? {
97- inner_arrays. push ( array) ;
98- }
99- }
100- }
101- }
102-
103- Ok ( inner_arrays)
104- }
105-
106- /// Extract list value from array at given row index, handling all list types
107- fn extract_list_value (
108- & self ,
109- arr : & Arc < dyn Array > ,
110- row_idx : usize ,
111- ) -> Result < Option < Arc < dyn Array > > > {
112- use arrow:: array:: * ;
113-
114- if arr. is_null ( row_idx) {
115- return Ok ( None ) ;
116- }
117-
118- match arr. data_type ( ) {
119- DataType :: List ( _) => {
120- let list_array =
121- arr. as_any ( ) . downcast_ref :: < ListArray > ( ) . ok_or_else ( || {
122- datafusion_common:: DataFusionError :: Plan (
123- "Failed to downcast to ListArray" . to_string ( ) ,
124- )
125- } ) ?;
126- Ok ( Some ( list_array. value ( row_idx) ) )
127- }
128- DataType :: LargeList ( _) => {
129- let list_array = arr
130- . as_any ( )
131- . downcast_ref :: < LargeListArray > ( )
132- . ok_or_else ( || {
133- datafusion_common:: DataFusionError :: Plan (
134- "Failed to downcast to LargeListArray" . to_string ( ) ,
135- )
136- } ) ?;
137- Ok ( Some ( list_array. value ( row_idx) ) )
138- }
139- DataType :: FixedSizeList ( _, _) => {
140- let list_array = arr
141- . as_any ( )
142- . downcast_ref :: < FixedSizeListArray > ( )
143- . ok_or_else ( || {
144- datafusion_common:: DataFusionError :: Plan (
145- "Failed to downcast to FixedSizeListArray" . to_string ( ) ,
146- )
147- } ) ?;
148- Ok ( Some ( list_array. value ( row_idx) ) )
149- }
150- _ => plan_err ! ( "Expected list array, got {}" , arr. data_type( ) ) ,
151- }
152- }
153-
154- /// Extract list value from scalar, handling all list scalar types
155- fn extract_scalar_list_value (
156- & self ,
157- scalar : & ScalarValue ,
158- ) -> Result < Option < Arc < dyn Array > > > {
159- use arrow:: array:: * ;
160-
161- if scalar. is_null ( ) {
162- return Ok ( None ) ;
163- }
164-
165- match scalar {
166- ScalarValue :: List ( arr) => {
167- let list_array =
168- arr. as_any ( ) . downcast_ref :: < ListArray > ( ) . ok_or_else ( || {
169- datafusion_common:: DataFusionError :: Plan (
170- "Failed to downcast scalar List to ListArray" . to_string ( ) ,
171- )
172- } ) ?;
173- if !list_array. is_null ( 0 ) {
174- Ok ( Some ( list_array. value ( 0 ) ) )
175- } else {
176- Ok ( None )
177- }
178- }
179- ScalarValue :: LargeList ( arr) => {
180- let list_array = arr
181- . as_any ( )
182- . downcast_ref :: < LargeListArray > ( )
183- . ok_or_else ( || {
184- datafusion_common:: DataFusionError :: Plan (
185- "Failed to downcast scalar LargeList to LargeListArray"
186- . to_string ( ) ,
187- )
188- } ) ?;
189- if !list_array. is_null ( 0 ) {
190- Ok ( Some ( list_array. value ( 0 ) ) )
191- } else {
192- Ok ( None )
193- }
194- }
195- ScalarValue :: FixedSizeList ( arr) => {
196- let list_array = arr. as_any ( ) . downcast_ref :: < FixedSizeListArray > ( )
197- . ok_or_else ( || datafusion_common:: DataFusionError :: Plan ( "Failed to downcast scalar FixedSizeList to FixedSizeListArray" . to_string ( ) ) ) ?;
198- if !list_array. is_null ( 0 ) {
199- Ok ( Some ( list_array. value ( 0 ) ) )
200- } else {
201- Ok ( None )
202- }
203- }
204- _ => Ok ( None ) ,
205- }
206- }
207-
20880 /// Get the string type with highest precedence: Utf8View > LargeUtf8 > Utf8
20981 fn get_string_type_precedence ( & self , arg_types : & [ DataType ] ) -> DataType {
21082 use DataType :: * ;
@@ -224,130 +96,136 @@ impl ConcatFunc {
22496 Utf8
22597 }
22698
227- /// Determine element type from first valid array argument
228- fn determine_element_type ( & self , args : & [ ColumnarValue ] ) -> Result < DataType > {
229- for arg in args {
230- match arg {
231- ColumnarValue :: Array ( arr) => match arr. data_type ( ) {
232- DataType :: List ( field)
233- | DataType :: LargeList ( field)
234- | DataType :: FixedSizeList ( field, _) => {
235- return Ok ( field. data_type ( ) . clone ( ) ) ;
236- }
237- _ => { }
238- } ,
239- ColumnarValue :: Scalar ( scalar) => match scalar {
240- ScalarValue :: List ( arr) => {
241- if let DataType :: List ( field) = arr. data_type ( ) {
242- return Ok ( field. data_type ( ) . clone ( ) ) ;
243- }
244- }
245- ScalarValue :: LargeList ( arr) => {
246- if let DataType :: LargeList ( field) = arr. data_type ( ) {
247- return Ok ( field. data_type ( ) . clone ( ) ) ;
248- }
249- }
250- ScalarValue :: FixedSizeList ( arr) => {
251- if let DataType :: FixedSizeList ( field, _) = arr. data_type ( ) {
252- return Ok ( field. data_type ( ) . clone ( ) ) ;
253- }
254- }
255- _ => { }
256- } ,
257- }
258- }
259- plan_err ! ( "No valid array arguments found. Expected at least one array argument for array concatenation." )
260- }
261-
262- /// Concatenate array arguments into a single array using runtime delegation to Arrow's compute::concat
99+ /// Concatenate array arguments using full array concatenation logic
263100 fn concat_arrays ( & self , args : & [ ColumnarValue ] ) -> Result < ColumnarValue > {
264101 use arrow:: array:: * ;
265- use arrow:: buffer:: OffsetBuffer ;
266102
267103 if args. is_empty ( ) {
268104 return plan_err ! ( "concat requires at least one argument" ) ;
269105 }
270106
271- let array_len = args
107+ // Convert ColumnarValue arguments to ArrayRef
108+ let array_refs: Result < Vec < Arc < dyn Array > > > = args
272109 . iter ( )
273- . find_map ( |arg| match arg {
274- ColumnarValue :: Array ( array ) => Some ( array . len ( ) ) ,
275- _ => None ,
110+ . map ( |arg| match arg {
111+ ColumnarValue :: Array ( arr ) => Ok ( Arc :: clone ( arr ) ) ,
112+ ColumnarValue :: Scalar ( scalar ) => scalar . to_array_of_size ( 1 ) ,
276113 } )
277- . unwrap_or ( 1 ) ;
114+ . collect ( ) ;
278115
279- if array_len == 1 {
280- // Single row case
281- let inner_arrays = self . extract_arrays_from_row ( args, 0 ) ?;
116+ let arrays = array_refs?;
282117
283- if inner_arrays. is_empty ( ) {
284- return plan_err ! ( "No valid arrays to concatenate. All array arguments were null or empty." ) ;
118+ // Check if all arrays are null
119+ let mut all_null = true ;
120+ let mut large_list = false ;
121+ for arg in & arrays {
122+ match arg. data_type ( ) {
123+ DataType :: Null => continue ,
124+ DataType :: LargeList ( _) => large_list = true ,
125+ _ => ( ) ,
285126 }
127+ if arg. null_count ( ) < arg. len ( ) {
128+ all_null = false ;
129+ }
130+ }
286131
287- let array_refs: Vec < & dyn Array > =
288- inner_arrays. iter ( ) . map ( |a| a. as_ref ( ) ) . collect ( ) ;
289- let concatenated = compute:: concat ( & array_refs) ?;
290-
291- let field = Arc :: new ( arrow:: datatypes:: Field :: new_list_field (
292- concatenated. data_type ( ) . clone ( ) ,
293- true ,
294- ) ) ;
295- let offsets = OffsetBuffer :: from_lengths ( [ concatenated. len ( ) ] ) ;
296- let result_list = ListArray :: new ( field, offsets, concatenated, None ) ;
132+ if all_null {
133+ // For concat function, if all arrays are null (even if they have types),
134+ // we return an error since there are no valid arrays to concatenate
135+ return plan_err ! ( "No valid arrays to concatenate" ) ;
136+ }
297137
298- Ok ( ColumnarValue :: Array ( Arc :: new ( result_list) ) )
138+ // Full implementation supporting multi-row arrays
139+ if large_list {
140+ self . concat_arrays_internal :: < i64 > ( & arrays)
299141 } else {
300- // Multi-row case
301- let mut result_arrays = Vec :: with_capacity ( array_len) ;
142+ self . concat_arrays_internal :: < i32 > ( & arrays)
143+ }
144+ }
302145
303- for row_idx in 0 ..array_len {
304- let row_inner_arrays = self . extract_arrays_from_row ( args, row_idx) ?;
146+ /// Internal array concatenation implementation supporting different offset types
147+ fn concat_arrays_internal < O : arrow:: array:: OffsetSizeTrait > (
148+ & self ,
149+ arrays : & [ Arc < dyn Array > ] ,
150+ ) -> Result < ColumnarValue >
151+ where
152+ i64 : TryInto < O > ,
153+ {
154+ use arrow:: array:: * ;
155+ use arrow:: buffer:: OffsetBuffer ;
156+ use arrow:: datatypes:: Field ;
157+ use datafusion_common:: cast:: as_generic_list_array;
305158
306- if row_inner_arrays. is_empty ( ) {
307- result_arrays. push ( None ) ;
308- } else {
309- let array_refs: Vec < & dyn Array > =
310- row_inner_arrays. iter ( ) . map ( |a| a. as_ref ( ) ) . collect ( ) ;
311- let concatenated = compute:: concat ( & array_refs) ?;
312- result_arrays. push ( Some ( concatenated) ) ;
313- }
314- }
159+ let list_arrays: Result < Vec < _ > > = arrays
160+ . iter ( )
161+ . map ( |arg| as_generic_list_array :: < O > ( arg) )
162+ . collect ( ) ;
163+ let list_arrays = list_arrays?;
315164
316- let element_type = self . determine_element_type ( args) ?;
317- let field = Arc :: new ( arrow:: datatypes:: Field :: new_list_field (
318- element_type. clone ( ) ,
319- true ,
320- ) ) ;
165+ // Assume number of rows is the same for all arrays
166+ let row_count = list_arrays[ 0 ] . len ( ) ;
321167
322- // Build final ListArray with proper offsets
323- let mut values_vec = Vec :: new ( ) ;
324- let mut lengths = Vec :: with_capacity ( array_len ) ;
168+ let mut array_lengths = vec ! [ ] ;
169+ let mut result_arrays = vec ! [ ] ;
170+ let mut valid = NullBufferBuilder :: new ( row_count ) ;
325171
326- for result in result_arrays {
327- match result {
328- Some ( array) => {
329- lengths. push ( array. len ( ) ) ;
330- values_vec. push ( array) ;
331- }
332- None => {
333- lengths. push ( 0 ) ;
334- }
172+ for i in 0 ..row_count {
173+ let nulls: Vec < bool > = list_arrays. iter ( ) . map ( |arr| arr. is_null ( i) ) . collect ( ) ;
174+
175+ // If all the arrays are null, the concatenated array is null
176+ let is_null = nulls. iter ( ) . all ( |& x| x) ;
177+ if is_null {
178+ array_lengths. push ( 0 ) ;
179+ valid. append_null ( ) ;
180+ } else {
181+ // Get all the arrays on i-th row
182+ let values: Vec < _ > = list_arrays
183+ . iter ( )
184+ . enumerate ( )
185+ . filter_map ( |( idx, arr) | {
186+ if !nulls[ idx] {
187+ Some ( arr. value ( i) )
188+ } else {
189+ None
190+ }
191+ } )
192+ . collect ( ) ;
193+
194+ if values. is_empty ( ) {
195+ array_lengths. push ( 0 ) ;
196+ valid. append_null ( ) ;
197+ } else {
198+ let elements: Vec < & dyn Array > =
199+ values. iter ( ) . map ( |a| a. as_ref ( ) ) . collect ( ) ;
200+
201+ // Concatenated array on i-th row
202+ let concatenated_array = compute:: concat ( & elements) ?;
203+ array_lengths. push ( concatenated_array. len ( ) ) ;
204+ result_arrays. push ( concatenated_array) ;
205+ valid. append_non_null ( ) ;
335206 }
336207 }
208+ }
337209
338- let values = if values_vec. is_empty ( ) {
339- new_empty_array ( & element_type)
340- } else {
341- let array_refs: Vec < & dyn Array > =
342- values_vec. iter ( ) . map ( |a| a. as_ref ( ) ) . collect ( ) ;
343- compute:: concat ( & array_refs) ?
344- } ;
210+ // Assume all arrays have the same data type
211+ let data_type = list_arrays[ 0 ] . value_type ( ) ;
345212
346- let offsets = OffsetBuffer :: from_lengths ( lengths) ;
347- let result_list = ListArray :: new ( field, offsets, values, None ) ;
213+ let values = if result_arrays. is_empty ( ) {
214+ new_empty_array ( & data_type)
215+ } else {
216+ let elements: Vec < & dyn Array > =
217+ result_arrays. iter ( ) . map ( |a| a. as_ref ( ) ) . collect ( ) ;
218+ compute:: concat ( & elements) ?
219+ } ;
348220
349- Ok ( ColumnarValue :: Array ( Arc :: new ( result_list) ) )
350- }
221+ let list_arr = GenericListArray :: < O > :: new (
222+ Arc :: new ( Field :: new_list_field ( data_type, true ) ) ,
223+ OffsetBuffer :: from_lengths ( array_lengths) ,
224+ values,
225+ valid. finish ( ) ,
226+ ) ;
227+
228+ Ok ( ColumnarValue :: Array ( Arc :: new ( list_arr) ) )
351229 }
352230}
353231
0 commit comments