Skip to content

Commit e20a48b

Browse files
committed
feat: delegate array concatenation from concat to array_concat logic
Fixes apache#18020
1 parent e5513c0 commit e20a48b

3 files changed

Lines changed: 105 additions & 227 deletions

File tree

‎datafusion/core/"/var/folders/83/9s6d5qh13z9fj_8mvgv7v5s00000gn/T/.tmprimaOK/target.csv"‎

Whitespace-only changes.

‎datafusion/core/"/var/folders/83/9s6d5qh13z9fj_8mvgv7v5s00000gn/T/.tmptK7aeT/target.csv"‎

Whitespace-only changes.

‎datafusion/functions/src/string/concat.rs‎

Lines changed: 105 additions & 227 deletions
Original file line numberDiff line numberDiff line change
@@ -77,134 +77,6 @@ impl ConcatFunc {
7777
}
7878
}
7979

80-
/// Extract array elements from a single row of arguments for array concatenation
81-
fn extract_arrays_from_row(
82-
&self,
83-
args: &[ColumnarValue],
84-
row_idx: usize,
85-
) -> Result<Vec<Arc<dyn Array>>> {
86-
let mut inner_arrays = Vec::new();
87-
88-
for arg in args {
89-
match arg {
90-
ColumnarValue::Array(arr) => {
91-
if let Some(array) = self.extract_list_value(arr, row_idx)? {
92-
inner_arrays.push(array);
93-
}
94-
}
95-
ColumnarValue::Scalar(scalar) => {
96-
if let Some(array) = self.extract_scalar_list_value(scalar)? {
97-
inner_arrays.push(array);
98-
}
99-
}
100-
}
101-
}
102-
103-
Ok(inner_arrays)
104-
}
105-
106-
/// Extract list value from array at given row index, handling all list types
107-
fn extract_list_value(
108-
&self,
109-
arr: &Arc<dyn Array>,
110-
row_idx: usize,
111-
) -> Result<Option<Arc<dyn Array>>> {
112-
use arrow::array::*;
113-
114-
if arr.is_null(row_idx) {
115-
return Ok(None);
116-
}
117-
118-
match arr.data_type() {
119-
DataType::List(_) => {
120-
let list_array =
121-
arr.as_any().downcast_ref::<ListArray>().ok_or_else(|| {
122-
datafusion_common::DataFusionError::Plan(
123-
"Failed to downcast to ListArray".to_string(),
124-
)
125-
})?;
126-
Ok(Some(list_array.value(row_idx)))
127-
}
128-
DataType::LargeList(_) => {
129-
let list_array = arr
130-
.as_any()
131-
.downcast_ref::<LargeListArray>()
132-
.ok_or_else(|| {
133-
datafusion_common::DataFusionError::Plan(
134-
"Failed to downcast to LargeListArray".to_string(),
135-
)
136-
})?;
137-
Ok(Some(list_array.value(row_idx)))
138-
}
139-
DataType::FixedSizeList(_, _) => {
140-
let list_array = arr
141-
.as_any()
142-
.downcast_ref::<FixedSizeListArray>()
143-
.ok_or_else(|| {
144-
datafusion_common::DataFusionError::Plan(
145-
"Failed to downcast to FixedSizeListArray".to_string(),
146-
)
147-
})?;
148-
Ok(Some(list_array.value(row_idx)))
149-
}
150-
_ => plan_err!("Expected list array, got {}", arr.data_type()),
151-
}
152-
}
153-
154-
/// Extract list value from scalar, handling all list scalar types
155-
fn extract_scalar_list_value(
156-
&self,
157-
scalar: &ScalarValue,
158-
) -> Result<Option<Arc<dyn Array>>> {
159-
use arrow::array::*;
160-
161-
if scalar.is_null() {
162-
return Ok(None);
163-
}
164-
165-
match scalar {
166-
ScalarValue::List(arr) => {
167-
let list_array =
168-
arr.as_any().downcast_ref::<ListArray>().ok_or_else(|| {
169-
datafusion_common::DataFusionError::Plan(
170-
"Failed to downcast scalar List to ListArray".to_string(),
171-
)
172-
})?;
173-
if !list_array.is_null(0) {
174-
Ok(Some(list_array.value(0)))
175-
} else {
176-
Ok(None)
177-
}
178-
}
179-
ScalarValue::LargeList(arr) => {
180-
let list_array = arr
181-
.as_any()
182-
.downcast_ref::<LargeListArray>()
183-
.ok_or_else(|| {
184-
datafusion_common::DataFusionError::Plan(
185-
"Failed to downcast scalar LargeList to LargeListArray"
186-
.to_string(),
187-
)
188-
})?;
189-
if !list_array.is_null(0) {
190-
Ok(Some(list_array.value(0)))
191-
} else {
192-
Ok(None)
193-
}
194-
}
195-
ScalarValue::FixedSizeList(arr) => {
196-
let list_array = arr.as_any().downcast_ref::<FixedSizeListArray>()
197-
.ok_or_else(|| datafusion_common::DataFusionError::Plan("Failed to downcast scalar FixedSizeList to FixedSizeListArray".to_string()))?;
198-
if !list_array.is_null(0) {
199-
Ok(Some(list_array.value(0)))
200-
} else {
201-
Ok(None)
202-
}
203-
}
204-
_ => Ok(None),
205-
}
206-
}
207-
20880
/// Get the string type with highest precedence: Utf8View > LargeUtf8 > Utf8
20981
fn get_string_type_precedence(&self, arg_types: &[DataType]) -> DataType {
21082
use DataType::*;
@@ -224,130 +96,136 @@ impl ConcatFunc {
22496
Utf8
22597
}
22698

227-
/// Determine element type from first valid array argument
228-
fn determine_element_type(&self, args: &[ColumnarValue]) -> Result<DataType> {
229-
for arg in args {
230-
match arg {
231-
ColumnarValue::Array(arr) => match arr.data_type() {
232-
DataType::List(field)
233-
| DataType::LargeList(field)
234-
| DataType::FixedSizeList(field, _) => {
235-
return Ok(field.data_type().clone());
236-
}
237-
_ => {}
238-
},
239-
ColumnarValue::Scalar(scalar) => match scalar {
240-
ScalarValue::List(arr) => {
241-
if let DataType::List(field) = arr.data_type() {
242-
return Ok(field.data_type().clone());
243-
}
244-
}
245-
ScalarValue::LargeList(arr) => {
246-
if let DataType::LargeList(field) = arr.data_type() {
247-
return Ok(field.data_type().clone());
248-
}
249-
}
250-
ScalarValue::FixedSizeList(arr) => {
251-
if let DataType::FixedSizeList(field, _) = arr.data_type() {
252-
return Ok(field.data_type().clone());
253-
}
254-
}
255-
_ => {}
256-
},
257-
}
258-
}
259-
plan_err!("No valid array arguments found. Expected at least one array argument for array concatenation.")
260-
}
261-
262-
/// Concatenate array arguments into a single array using runtime delegation to Arrow's compute::concat
99+
/// Concatenate array arguments using full array concatenation logic
263100
fn concat_arrays(&self, args: &[ColumnarValue]) -> Result<ColumnarValue> {
264101
use arrow::array::*;
265-
use arrow::buffer::OffsetBuffer;
266102

267103
if args.is_empty() {
268104
return plan_err!("concat requires at least one argument");
269105
}
270106

271-
let array_len = args
107+
// Convert ColumnarValue arguments to ArrayRef
108+
let array_refs: Result<Vec<Arc<dyn Array>>> = args
272109
.iter()
273-
.find_map(|arg| match arg {
274-
ColumnarValue::Array(array) => Some(array.len()),
275-
_ => None,
110+
.map(|arg| match arg {
111+
ColumnarValue::Array(arr) => Ok(Arc::clone(arr)),
112+
ColumnarValue::Scalar(scalar) => scalar.to_array_of_size(1),
276113
})
277-
.unwrap_or(1);
114+
.collect();
278115

279-
if array_len == 1 {
280-
// Single row case
281-
let inner_arrays = self.extract_arrays_from_row(args, 0)?;
116+
let arrays = array_refs?;
282117

283-
if inner_arrays.is_empty() {
284-
return plan_err!("No valid arrays to concatenate. All array arguments were null or empty.");
118+
// Check if all arrays are null
119+
let mut all_null = true;
120+
let mut large_list = false;
121+
for arg in &arrays {
122+
match arg.data_type() {
123+
DataType::Null => continue,
124+
DataType::LargeList(_) => large_list = true,
125+
_ => (),
285126
}
127+
if arg.null_count() < arg.len() {
128+
all_null = false;
129+
}
130+
}
286131

287-
let array_refs: Vec<&dyn Array> =
288-
inner_arrays.iter().map(|a| a.as_ref()).collect();
289-
let concatenated = compute::concat(&array_refs)?;
290-
291-
let field = Arc::new(arrow::datatypes::Field::new_list_field(
292-
concatenated.data_type().clone(),
293-
true,
294-
));
295-
let offsets = OffsetBuffer::from_lengths([concatenated.len()]);
296-
let result_list = ListArray::new(field, offsets, concatenated, None);
132+
if all_null {
133+
// For concat function, if all arrays are null (even if they have types),
134+
// we return an error since there are no valid arrays to concatenate
135+
return plan_err!("No valid arrays to concatenate");
136+
}
297137

298-
Ok(ColumnarValue::Array(Arc::new(result_list)))
138+
// Full implementation supporting multi-row arrays
139+
if large_list {
140+
self.concat_arrays_internal::<i64>(&arrays)
299141
} else {
300-
// Multi-row case
301-
let mut result_arrays = Vec::with_capacity(array_len);
142+
self.concat_arrays_internal::<i32>(&arrays)
143+
}
144+
}
302145

303-
for row_idx in 0..array_len {
304-
let row_inner_arrays = self.extract_arrays_from_row(args, row_idx)?;
146+
/// Internal array concatenation implementation supporting different offset types
147+
fn concat_arrays_internal<O: arrow::array::OffsetSizeTrait>(
148+
&self,
149+
arrays: &[Arc<dyn Array>],
150+
) -> Result<ColumnarValue>
151+
where
152+
i64: TryInto<O>,
153+
{
154+
use arrow::array::*;
155+
use arrow::buffer::OffsetBuffer;
156+
use arrow::datatypes::Field;
157+
use datafusion_common::cast::as_generic_list_array;
305158

306-
if row_inner_arrays.is_empty() {
307-
result_arrays.push(None);
308-
} else {
309-
let array_refs: Vec<&dyn Array> =
310-
row_inner_arrays.iter().map(|a| a.as_ref()).collect();
311-
let concatenated = compute::concat(&array_refs)?;
312-
result_arrays.push(Some(concatenated));
313-
}
314-
}
159+
let list_arrays: Result<Vec<_>> = arrays
160+
.iter()
161+
.map(|arg| as_generic_list_array::<O>(arg))
162+
.collect();
163+
let list_arrays = list_arrays?;
315164

316-
let element_type = self.determine_element_type(args)?;
317-
let field = Arc::new(arrow::datatypes::Field::new_list_field(
318-
element_type.clone(),
319-
true,
320-
));
165+
// Assume number of rows is the same for all arrays
166+
let row_count = list_arrays[0].len();
321167

322-
// Build final ListArray with proper offsets
323-
let mut values_vec = Vec::new();
324-
let mut lengths = Vec::with_capacity(array_len);
168+
let mut array_lengths = vec![];
169+
let mut result_arrays = vec![];
170+
let mut valid = NullBufferBuilder::new(row_count);
325171

326-
for result in result_arrays {
327-
match result {
328-
Some(array) => {
329-
lengths.push(array.len());
330-
values_vec.push(array);
331-
}
332-
None => {
333-
lengths.push(0);
334-
}
172+
for i in 0..row_count {
173+
let nulls: Vec<bool> = list_arrays.iter().map(|arr| arr.is_null(i)).collect();
174+
175+
// If all the arrays are null, the concatenated array is null
176+
let is_null = nulls.iter().all(|&x| x);
177+
if is_null {
178+
array_lengths.push(0);
179+
valid.append_null();
180+
} else {
181+
// Get all the arrays on i-th row
182+
let values: Vec<_> = list_arrays
183+
.iter()
184+
.enumerate()
185+
.filter_map(|(idx, arr)| {
186+
if !nulls[idx] {
187+
Some(arr.value(i))
188+
} else {
189+
None
190+
}
191+
})
192+
.collect();
193+
194+
if values.is_empty() {
195+
array_lengths.push(0);
196+
valid.append_null();
197+
} else {
198+
let elements: Vec<&dyn Array> =
199+
values.iter().map(|a| a.as_ref()).collect();
200+
201+
// Concatenated array on i-th row
202+
let concatenated_array = compute::concat(&elements)?;
203+
array_lengths.push(concatenated_array.len());
204+
result_arrays.push(concatenated_array);
205+
valid.append_non_null();
335206
}
336207
}
208+
}
337209

338-
let values = if values_vec.is_empty() {
339-
new_empty_array(&element_type)
340-
} else {
341-
let array_refs: Vec<&dyn Array> =
342-
values_vec.iter().map(|a| a.as_ref()).collect();
343-
compute::concat(&array_refs)?
344-
};
210+
// Assume all arrays have the same data type
211+
let data_type = list_arrays[0].value_type();
345212

346-
let offsets = OffsetBuffer::from_lengths(lengths);
347-
let result_list = ListArray::new(field, offsets, values, None);
213+
let values = if result_arrays.is_empty() {
214+
new_empty_array(&data_type)
215+
} else {
216+
let elements: Vec<&dyn Array> =
217+
result_arrays.iter().map(|a| a.as_ref()).collect();
218+
compute::concat(&elements)?
219+
};
348220

349-
Ok(ColumnarValue::Array(Arc::new(result_list)))
350-
}
221+
let list_arr = GenericListArray::<O>::new(
222+
Arc::new(Field::new_list_field(data_type, true)),
223+
OffsetBuffer::from_lengths(array_lengths),
224+
values,
225+
valid.finish(),
226+
);
227+
228+
Ok(ColumnarValue::Array(Arc::new(list_arr)))
351229
}
352230
}
353231

0 commit comments

Comments
 (0)