Skip to content

Commit b1d3f05

Browse files
committed
Use concat_elements_dyn from arrow-rs instead of a custom implementation of the same function
1 parent 1fd29c9 commit b1d3f05

2 files changed

Lines changed: 3 additions & 125 deletions

File tree

‎datafusion/physical-expr/src/expressions/binary.rs‎

Lines changed: 3 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,7 @@ use std::sync::Arc;
2424

2525
use arrow::array::*;
2626
use arrow::compute::kernels::boolean::{and_kleene, or_kleene};
27-
use arrow::compute::kernels::concat_elements::{
28-
concat_element_binary, concat_elements_utf8,
29-
};
27+
use arrow::compute::kernels::concat_elements::concat_elements_dyn;
3028
use arrow::compute::{SlicesIterator, cast, filter_record_batch};
3129
use arrow::datatypes::*;
3230
use arrow::error::ArrowError;
@@ -50,8 +48,7 @@ use kernels::{
5048
bitwise_and_dyn, bitwise_and_dyn_scalar, bitwise_or_dyn, bitwise_or_dyn_scalar,
5149
bitwise_shift_left_dyn, bitwise_shift_left_dyn_scalar, bitwise_shift_right_dyn,
5250
bitwise_shift_right_dyn_scalar, bitwise_xor_dyn, bitwise_xor_dyn_scalar,
53-
concat_elements_binary_view_array, concat_elements_utf8view, regex_match_dyn,
54-
regex_match_dyn_scalar,
51+
regex_match_dyn, regex_match_dyn_scalar,
5552
};
5653

5754
/// Binary expression
@@ -1054,37 +1051,7 @@ fn pre_selection_scatter(
10541051
}
10551052

10561053
fn concat_elements(left: &ArrayRef, right: &ArrayRef) -> Result<ArrayRef> {
1057-
Ok(match left.data_type() {
1058-
DataType::Utf8 => Arc::new(concat_elements_utf8(
1059-
left.as_string::<i32>(),
1060-
right.as_string::<i32>(),
1061-
)?),
1062-
DataType::LargeUtf8 => Arc::new(concat_elements_utf8(
1063-
left.as_string::<i64>(),
1064-
right.as_string::<i64>(),
1065-
)?),
1066-
DataType::Utf8View => Arc::new(concat_elements_utf8view(
1067-
left.as_string_view(),
1068-
right.as_string_view(),
1069-
)?),
1070-
DataType::Binary => Arc::new(concat_element_binary::<i32>(
1071-
left.as_binary(),
1072-
right.as_binary(),
1073-
)?),
1074-
DataType::LargeBinary => Arc::new(concat_element_binary::<i64>(
1075-
left.as_binary(),
1076-
right.as_binary(),
1077-
)?),
1078-
DataType::BinaryView => Arc::new(concat_elements_binary_view_array(
1079-
left.as_binary_view(),
1080-
right.as_binary_view(),
1081-
)?),
1082-
other => {
1083-
return internal_err!(
1084-
"Data type {other:?} not supported for binary operation 'concat_elements' on string arrays"
1085-
);
1086-
}
1087-
})
1054+
concat_elements_dyn(left, right).map_err(|e| e.into())
10881055
}
10891056

10901057
/// Create a binary expression whose arguments are correctly coerced.

‎datafusion/physical-expr/src/expressions/binary/kernels.rs‎

Lines changed: 0 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
//! This module contains computation kernels that are specific to
1919
//! datafusion and not (yet) targeted to port upstream to arrow
2020
use arrow::array::*;
21-
use arrow::buffer::{MutableBuffer, NullBuffer};
2221
use arrow::compute::kernels::bitwise::{
2322
bitwise_and, bitwise_and_scalar, bitwise_or, bitwise_or_scalar, bitwise_shift_left,
2423
bitwise_shift_left_scalar, bitwise_shift_right, bitwise_shift_right_scalar,
@@ -27,7 +26,6 @@ use arrow::compute::kernels::bitwise::{
2726
use arrow::compute::kernels::boolean::not;
2827
use arrow::compute::kernels::comparison::{regexp_is_match, regexp_is_match_scalar};
2928
use arrow::datatypes::DataType;
30-
use arrow::error::ArrowError;
3129
use datafusion_common::{Result, ScalarValue};
3230
use datafusion_common::{internal_err, plan_err};
3331

@@ -161,93 +159,6 @@ create_left_integral_dyn_scalar_kernel!(
161159
bitwise_shift_left_scalar
162160
);
163161

164-
/// Concatenates two `StringViewArray`s element-wise.
165-
/// If either element is `Null`, the result element is also `Null`.
166-
///
167-
/// # Errors
168-
/// - Returns an error if the input arrays have different lengths.
169-
/// - Returns an error if any concatenated string exceeds `u32::MAX` (≈4 GB) in length.
170-
pub fn concat_elements_utf8view(
171-
left: &StringViewArray,
172-
right: &StringViewArray,
173-
) -> std::result::Result<StringViewArray, ArrowError> {
174-
if left.len() != right.len() {
175-
return Err(ArrowError::ComputeError(format!(
176-
"Arrays must have the same length: {} != {}",
177-
left.len(),
178-
right.len()
179-
)));
180-
}
181-
let mut result = StringViewBuilder::with_capacity(left.len());
182-
183-
// Avoid reallocations by writing to a reused buffer (note we could be even
184-
// more efficient by creating the view directly here and avoid the buffer
185-
// but that would be more complex)
186-
let mut buffer = String::new();
187-
188-
// Pre-compute combined null bitmap, so the per-row NULL check is more
189-
// efficient
190-
let nulls = NullBuffer::union(left.nulls(), right.nulls());
191-
192-
for i in 0..left.len() {
193-
if nulls.as_ref().is_some_and(|n| n.is_null(i)) {
194-
result.append_null();
195-
} else {
196-
let l = left.value(i);
197-
let r = right.value(i);
198-
buffer.clear();
199-
buffer.push_str(l);
200-
buffer.push_str(r);
201-
result.try_append_value(&buffer)?;
202-
}
203-
}
204-
Ok(result.finish())
205-
}
206-
207-
/// Concatenates two `BinaryViewArray`s element-wise.
208-
/// If either element is `Null`, the result element is also `Null`.
209-
///
210-
/// # Errors
211-
/// - Returns an error if the input arrays have different lengths.
212-
/// - Returns an error if any concatenated string exceeds `u32::MAX` in length.
213-
pub fn concat_elements_binary_view_array(
214-
left: &BinaryViewArray,
215-
right: &BinaryViewArray,
216-
) -> std::result::Result<BinaryViewArray, ArrowError> {
217-
if left.len() != right.len() {
218-
return Err(ArrowError::ComputeError(format!(
219-
"Arrays must have the same length: {} != {}",
220-
left.len(),
221-
right.len()
222-
)));
223-
}
224-
let mut result = BinaryViewBuilder::with_capacity(left.len());
225-
226-
// Avoid reallocations by writing to a reused buffer (note we could be even
227-
// more efficient by creating the view directly here and avoid the buffer
228-
// but that would be more complex)
229-
let mut buffer = MutableBuffer::new(0);
230-
231-
// Pre-compute combined null bitmap, so the per-row NULL check is more
232-
// efficient
233-
let nulls = NullBuffer::union(left.nulls(), right.nulls());
234-
235-
for i in 0..left.len() {
236-
if nulls.as_ref().is_some_and(|n| n.is_null(i)) {
237-
result.append_null();
238-
} else {
239-
let l = left.value(i);
240-
let r = right.value(i);
241-
buffer.clear();
242-
buffer.extend_from_slice(l);
243-
buffer.extend_from_slice(r);
244-
// No try-version of append_value
245-
result.try_append_value(&buffer)?;
246-
}
247-
}
248-
Ok(result.finish())
249-
}
250-
251162
/// Invoke a compute kernel on a pair of binary data arrays with flags
252163
macro_rules! regexp_is_match_flag {
253164
($LEFT:expr, $RIGHT:expr, $ARRAYTYPE:ident, $NOT:expr, $FLAG:expr) => {{

0 commit comments

Comments
 (0)