Skip to content

Commit b79d396

Browse files
authored
Use concat_elements_dyn from arrow-rs (apache#23211)
## Which issue does this PR close? - Closes apache#23210 ## Rationale for this change apache/arrow-rs#9876 added `ByteView` and `FixedSizeBinary` support to `concat_elements_dyn` in `arrow-rs`. As a consequence the extended implementation in DataFusion can now be replaced by a call to the `arrow-rs` implementation. ## What changes are included in this PR? - Remove the `kernels::concat_elements_utf8view` and `kernels::concat_elements_binary_view_array` - Replace implementation of `binary::concat_elements` with a call to `arrow::compute::kernels::concat_elements::concat_elements_dyn` ## Are these changes tested? - Usage is covered by existing tests - The kernels themselves are tested in `arrow-rs` ## Are there any user-facing changes? No, two `pub` functions have been removed from `kernel`, but `kernel` itself is not `pub`.
1 parent 1e77af8 commit b79d396

6 files changed

Lines changed: 63 additions & 169 deletions

File tree

‎datafusion/expr-common/src/type_coercion/binary.rs‎

Lines changed: 49 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,7 @@ impl<'a> BinaryTypeCoercer<'a> {
237237
})
238238
}
239239
StringConcat => {
240-
string_concat_coercion(lhs, rhs).map(Signature::uniform).ok_or_else(|| {
240+
string_concat_coercion(lhs, rhs).ok_or_else(|| {
241241
plan_datafusion_err!(
242242
"Cannot infer common string type for string concat operation {} {} {}", self.lhs, self.op, self.rhs
243243
)
@@ -1629,42 +1629,55 @@ fn ree_coercion(
16291629
/// 1. At least one side of lhs and rhs should be string type (Utf8 / LargeUtf8)
16301630
/// 2. Data type of the other side should be able to cast to string type
16311631
/// 3. Binary and string types cannot be mixed
1632-
fn string_concat_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataType> {
1632+
fn string_concat_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<Signature> {
16331633
use arrow::datatypes::DataType::*;
1634-
string_coercion(lhs_type, rhs_type).or_else(|| match (lhs_type, rhs_type) {
1635-
// Allow pure binary + binary
1636-
(
1637-
Binary | LargeBinary | BinaryView | FixedSizeBinary(_),
1638-
Binary | LargeBinary | BinaryView | FixedSizeBinary(_),
1639-
) => {
1640-
// Coerce fixed-sized binary to variable-sized `Binary` to make uniform signature
1641-
// with the `Binary` result
1642-
let lhs_type = match lhs_type {
1643-
FixedSizeBinary(_) => &Binary,
1644-
val => val,
1645-
};
1646-
let rhs_type = match rhs_type {
1647-
FixedSizeBinary(_) => &Binary,
1648-
val => val,
1649-
};
1650-
binary_coercion(lhs_type, rhs_type)
1651-
}
1652-
// Predicate-based coercion rules are following,
1653-
// including mixed binary + string combinations
1654-
(Utf8View, from_type) | (from_type, Utf8View) => {
1655-
string_concat_internal_coercion(from_type, &Utf8View)
1656-
}
1657-
(Utf8, from_type) | (from_type, Utf8) => {
1658-
string_concat_internal_coercion(from_type, &Utf8)
1659-
}
1660-
(LargeUtf8, from_type) | (from_type, LargeUtf8) => {
1661-
string_concat_internal_coercion(from_type, &LargeUtf8)
1662-
}
1663-
(Dictionary(_, lhs_value_type), Dictionary(_, rhs_value_type)) => {
1664-
string_coercion(lhs_value_type, rhs_value_type).or(None)
1665-
}
1666-
_ => None,
1667-
})
1634+
1635+
string_coercion(lhs_type, rhs_type)
1636+
.map(Signature::uniform)
1637+
.or_else(|| match (lhs_type, rhs_type) {
1638+
// Allow concatenation of mixed fixed size binary
1639+
(FixedSizeBinary(l), FixedSizeBinary(r)) => Some(Signature {
1640+
lhs: lhs_type.clone(),
1641+
rhs: rhs_type.clone(),
1642+
ret: FixedSizeBinary(l + r),
1643+
}),
1644+
// Allow pure binary + binary
1645+
(
1646+
Binary | LargeBinary | BinaryView | FixedSizeBinary(_),
1647+
Binary | LargeBinary | BinaryView | FixedSizeBinary(_),
1648+
) => {
1649+
// Coerce fixed-sized binary to variable-sized `Binary` to make uniform signature
1650+
// with the `Binary` result
1651+
let lhs_type = match lhs_type {
1652+
FixedSizeBinary(_) => &Binary,
1653+
val => val,
1654+
};
1655+
let rhs_type = match rhs_type {
1656+
FixedSizeBinary(_) => &Binary,
1657+
val => val,
1658+
};
1659+
binary_coercion(lhs_type, rhs_type).map(Signature::uniform)
1660+
}
1661+
// Predicate-based coercion rules are following,
1662+
// including mixed binary + string combinations
1663+
(Utf8View, from_type) | (from_type, Utf8View) => {
1664+
string_concat_internal_coercion(from_type, &Utf8View)
1665+
.map(Signature::uniform)
1666+
}
1667+
(Utf8, from_type) | (from_type, Utf8) => {
1668+
string_concat_internal_coercion(from_type, &Utf8).map(Signature::uniform)
1669+
}
1670+
(LargeUtf8, from_type) | (from_type, LargeUtf8) => {
1671+
string_concat_internal_coercion(from_type, &LargeUtf8)
1672+
.map(Signature::uniform)
1673+
}
1674+
(Dictionary(_, lhs_value_type), Dictionary(_, rhs_value_type)) => {
1675+
string_coercion(lhs_value_type, rhs_value_type)
1676+
.or(None)
1677+
.map(Signature::uniform)
1678+
}
1679+
_ => None,
1680+
})
16681681
}
16691682

16701683
fn array_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataType> {

‎datafusion/expr-common/src/type_coercion/binary/tests/comparison.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -971,7 +971,8 @@ fn test_string_concat_coercion() -> Result<()> {
971971
DataType::FixedSizeBinary(4),
972972
DataType::FixedSizeBinary(16),
973973
Operator::StringConcat,
974-
DataType::Binary
974+
DataType::FixedSizeBinary(4),
975+
DataType::FixedSizeBinary(16)
975976
);
976977
test_coercion_binary_rule!(
977978
DataType::FixedSizeBinary(4),

‎datafusion/expr-common/src/type_coercion/binary/tests/mod.rs‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,18 @@ use super::*;
2727
/// - op: The binary operator (e.g., "+", "-", etc.)
2828
/// - expected_type: The type both sides should be coerced to
2929
macro_rules! test_coercion_binary_rule {
30-
($LHS_TYPE:expr, $RHS_TYPE:expr, $OP:expr, $RESULT_TYPE:expr) => {{
30+
($LHS_TYPE:expr, $RHS_TYPE:expr, $OP:expr, $RESULT_TYPE:expr) => {
3131
let (lhs, rhs) =
3232
BinaryTypeCoercer::new(&$LHS_TYPE, &$OP, &$RHS_TYPE).get_input_types()?;
3333
assert_eq!(lhs, $RESULT_TYPE);
3434
assert_eq!(rhs, $RESULT_TYPE);
35-
}};
35+
};
36+
($LHS_TYPE:expr, $RHS_TYPE:expr, $OP:expr, $L_RESULT_TYPE:expr, $R_RESULT_TYPE:expr) => {
37+
let (lhs, rhs) =
38+
BinaryTypeCoercer::new(&$LHS_TYPE, &$OP, &$RHS_TYPE).get_input_types()?;
39+
assert_eq!(lhs, $L_RESULT_TYPE);
40+
assert_eq!(rhs, $R_RESULT_TYPE);
41+
};
3642
}
3743

3844
/// Tests that coercion for a binary operator between one type and multiple right-hand side types

‎datafusion/physical-expr/src/expressions/binary.rs‎

Lines changed: 3 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,7 @@ use std::sync::Arc;
2424

2525
use arrow::array::*;
2626
use arrow::compute::kernels::boolean::{and_kleene, or_kleene};
27-
use arrow::compute::kernels::concat_elements::{
28-
concat_element_binary, concat_elements_utf8,
29-
};
27+
use arrow::compute::kernels::concat_elements::concat_elements_dyn;
3028
use arrow::compute::{SlicesIterator, cast, filter_record_batch};
3129
use arrow::datatypes::*;
3230
use arrow::error::ArrowError;
@@ -50,8 +48,7 @@ use kernels::{
5048
bitwise_and_dyn, bitwise_and_dyn_scalar, bitwise_or_dyn, bitwise_or_dyn_scalar,
5149
bitwise_shift_left_dyn, bitwise_shift_left_dyn_scalar, bitwise_shift_right_dyn,
5250
bitwise_shift_right_dyn_scalar, bitwise_xor_dyn, bitwise_xor_dyn_scalar,
53-
concat_elements_binary_view_array, concat_elements_utf8view, regex_match_dyn,
54-
regex_match_dyn_scalar,
51+
regex_match_dyn, regex_match_dyn_scalar,
5552
};
5653

5754
/// Binary expression
@@ -833,7 +830,7 @@ impl BinaryExpr {
833830
BitwiseXor => bitwise_xor_dyn(left, right),
834831
BitwiseShiftRight => bitwise_shift_right_dyn(left, right),
835832
BitwiseShiftLeft => bitwise_shift_left_dyn(left, right),
836-
StringConcat => concat_elements(&left, &right),
833+
StringConcat => concat_elements_dyn(&left, &right).map_err(|e| e.into()),
837834
AtArrow | ArrowAt | Arrow | LongArrow | HashArrow | HashLongArrow | AtAt
838835
| HashMinus | AtQuestion | Question | QuestionAnd | QuestionPipe
839836
| IntegerDivide | Colon => {
@@ -1053,40 +1050,6 @@ fn pre_selection_scatter(
10531050
Ok(ColumnarValue::Array(Arc::new(boolean_result)))
10541051
}
10551052

1056-
fn concat_elements(left: &ArrayRef, right: &ArrayRef) -> Result<ArrayRef> {
1057-
Ok(match left.data_type() {
1058-
DataType::Utf8 => Arc::new(concat_elements_utf8(
1059-
left.as_string::<i32>(),
1060-
right.as_string::<i32>(),
1061-
)?),
1062-
DataType::LargeUtf8 => Arc::new(concat_elements_utf8(
1063-
left.as_string::<i64>(),
1064-
right.as_string::<i64>(),
1065-
)?),
1066-
DataType::Utf8View => Arc::new(concat_elements_utf8view(
1067-
left.as_string_view(),
1068-
right.as_string_view(),
1069-
)?),
1070-
DataType::Binary => Arc::new(concat_element_binary::<i32>(
1071-
left.as_binary(),
1072-
right.as_binary(),
1073-
)?),
1074-
DataType::LargeBinary => Arc::new(concat_element_binary::<i64>(
1075-
left.as_binary(),
1076-
right.as_binary(),
1077-
)?),
1078-
DataType::BinaryView => Arc::new(concat_elements_binary_view_array(
1079-
left.as_binary_view(),
1080-
right.as_binary_view(),
1081-
)?),
1082-
other => {
1083-
return internal_err!(
1084-
"Data type {other:?} not supported for binary operation 'concat_elements' on string arrays"
1085-
);
1086-
}
1087-
})
1088-
}
1089-
10901053
/// Create a binary expression whose arguments are correctly coerced.
10911054
/// This function errors if it is not possible to coerce the arguments
10921055
/// to computational types supported by the operator.

‎datafusion/physical-expr/src/expressions/binary/kernels.rs‎

Lines changed: 0 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
//! This module contains computation kernels that are specific to
1919
//! datafusion and not (yet) targeted to port upstream to arrow
2020
use arrow::array::*;
21-
use arrow::buffer::{MutableBuffer, NullBuffer};
2221
use arrow::compute::kernels::bitwise::{
2322
bitwise_and, bitwise_and_scalar, bitwise_or, bitwise_or_scalar, bitwise_shift_left,
2423
bitwise_shift_left_scalar, bitwise_shift_right, bitwise_shift_right_scalar,
@@ -27,7 +26,6 @@ use arrow::compute::kernels::bitwise::{
2726
use arrow::compute::kernels::boolean::not;
2827
use arrow::compute::kernels::comparison::{regexp_is_match, regexp_is_match_scalar};
2928
use arrow::datatypes::DataType;
30-
use arrow::error::ArrowError;
3129
use datafusion_common::{Result, ScalarValue};
3230
use datafusion_common::{internal_err, plan_err};
3331

@@ -161,93 +159,6 @@ create_left_integral_dyn_scalar_kernel!(
161159
bitwise_shift_left_scalar
162160
);
163161

164-
/// Concatenates two `StringViewArray`s element-wise.
165-
/// If either element is `Null`, the result element is also `Null`.
166-
///
167-
/// # Errors
168-
/// - Returns an error if the input arrays have different lengths.
169-
/// - Returns an error if any concatenated string exceeds `u32::MAX` (≈4 GB) in length.
170-
pub fn concat_elements_utf8view(
171-
left: &StringViewArray,
172-
right: &StringViewArray,
173-
) -> std::result::Result<StringViewArray, ArrowError> {
174-
if left.len() != right.len() {
175-
return Err(ArrowError::ComputeError(format!(
176-
"Arrays must have the same length: {} != {}",
177-
left.len(),
178-
right.len()
179-
)));
180-
}
181-
let mut result = StringViewBuilder::with_capacity(left.len());
182-
183-
// Avoid reallocations by writing to a reused buffer (note we could be even
184-
// more efficient by creating the view directly here and avoid the buffer
185-
// but that would be more complex)
186-
let mut buffer = String::new();
187-
188-
// Pre-compute combined null bitmap, so the per-row NULL check is more
189-
// efficient
190-
let nulls = NullBuffer::union(left.nulls(), right.nulls());
191-
192-
for i in 0..left.len() {
193-
if nulls.as_ref().is_some_and(|n| n.is_null(i)) {
194-
result.append_null();
195-
} else {
196-
let l = left.value(i);
197-
let r = right.value(i);
198-
buffer.clear();
199-
buffer.push_str(l);
200-
buffer.push_str(r);
201-
result.try_append_value(&buffer)?;
202-
}
203-
}
204-
Ok(result.finish())
205-
}
206-
207-
/// Concatenates two `BinaryViewArray`s element-wise.
208-
/// If either element is `Null`, the result element is also `Null`.
209-
///
210-
/// # Errors
211-
/// - Returns an error if the input arrays have different lengths.
212-
/// - Returns an error if any concatenated string exceeds `u32::MAX` in length.
213-
pub fn concat_elements_binary_view_array(
214-
left: &BinaryViewArray,
215-
right: &BinaryViewArray,
216-
) -> std::result::Result<BinaryViewArray, ArrowError> {
217-
if left.len() != right.len() {
218-
return Err(ArrowError::ComputeError(format!(
219-
"Arrays must have the same length: {} != {}",
220-
left.len(),
221-
right.len()
222-
)));
223-
}
224-
let mut result = BinaryViewBuilder::with_capacity(left.len());
225-
226-
// Avoid reallocations by writing to a reused buffer (note we could be even
227-
// more efficient by creating the view directly here and avoid the buffer
228-
// but that would be more complex)
229-
let mut buffer = MutableBuffer::new(0);
230-
231-
// Pre-compute combined null bitmap, so the per-row NULL check is more
232-
// efficient
233-
let nulls = NullBuffer::union(left.nulls(), right.nulls());
234-
235-
for i in 0..left.len() {
236-
if nulls.as_ref().is_some_and(|n| n.is_null(i)) {
237-
result.append_null();
238-
} else {
239-
let l = left.value(i);
240-
let r = right.value(i);
241-
buffer.clear();
242-
buffer.extend_from_slice(l);
243-
buffer.extend_from_slice(r);
244-
// No try-version of append_value
245-
result.try_append_value(&buffer)?;
246-
}
247-
}
248-
Ok(result.finish())
249-
}
250-
251162
/// Invoke a compute kernel on a pair of binary data arrays with flags
252163
macro_rules! regexp_is_match_flag {
253164
($LEFT:expr, $RIGHT:expr, $ARRAYTYPE:ident, $NOT:expr, $FLAG:expr) => {{

‎datafusion/sqllogictest/test_files/binary.slt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -347,7 +347,7 @@ SELECT x'636166c3a9' || arrow_cast(x'68656c6c6f', 'FixedSizeBinary(5)'), arrow_t
347347
query ?T
348348
SELECT arrow_cast(x'6361', 'FixedSizeBinary(2)') || arrow_cast(x'68656c6c6f', 'FixedSizeBinary(5)'), arrow_typeof(arrow_cast(x'6361', 'FixedSizeBinary(2)') || arrow_cast(x'68656c6c6f', 'FixedSizeBinary(5)'));
349349
----
350-
636168656c6c6f Binary
350+
636168656c6c6f FixedSizeBinary(7)
351351

352352
# Byte pipe operator is allowed for mixed binary and text
353353
query T

0 commit comments

Comments
 (0)