Skip to content

Commit 7a2ee99

Browse files
committed
perf: preserve dictionary encoding through regexp_replace
Keep Dictionary(_, Utf8/LargeUtf8/Utf8View) through regexp_replace instead of casting it to the inner string type. When the pattern, replacement, and flags args are all scalar the function runs on the dict values and re-attaches the keys; otherwise it falls back to a flat value-type output.
1 parent 334f293 commit 7a2ee99

2 files changed

Lines changed: 130 additions & 41 deletions

File tree

‎datafusion/functions/src/regex/regexpreplace.rs‎

Lines changed: 112 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,15 @@
1818
//! Regex expressions
1919
use memchr::memchr;
2020

21+
use crate::utils::transform_leaf_type_preserving_encoding;
2122
use arrow::array::ArrayDataBuilder;
2223
use arrow::array::BufferBuilder;
2324
use arrow::array::GenericStringArray;
2425
use arrow::array::StringViewBuilder;
2526
use arrow::array::{Array, ArrayRef, OffsetSizeTrait};
2627
use arrow::array::{ArrayAccessor, StringViewArray};
2728
use arrow::array::{ArrayIter, AsArray, new_null_array};
28-
use arrow::datatypes::DataType;
29+
use arrow::datatypes::{DataType, Field, FieldRef};
2930
use datafusion_common::ScalarValue;
3031
use datafusion_common::cast::{
3132
as_large_string_array, as_string_array, as_string_view_array,
@@ -35,10 +36,10 @@ use datafusion_common::plan_err;
3536
use datafusion_common::utils::offset_span_len;
3637
use datafusion_common::{Result, cast::as_generic_string_array, internal_err};
3738
use datafusion_expr::ColumnarValue;
38-
use datafusion_expr::TypeSignature;
3939
use datafusion_expr::function::Hint;
4040
use datafusion_expr::{
41-
Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
41+
Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature,
42+
Volatility,
4243
};
4344
use datafusion_macros::user_doc;
4445
use regex::{CaptureLocations, Regex};
@@ -95,16 +96,8 @@ impl Default for RegexpReplaceFunc {
9596

9697
impl RegexpReplaceFunc {
9798
pub fn new() -> Self {
98-
use DataType::*;
99-
use TypeSignature::*;
10099
Self {
101-
signature: Signature::one_of(
102-
vec![
103-
Uniform(3, vec![Utf8View, LargeUtf8, Utf8]),
104-
Uniform(4, vec![Utf8View, LargeUtf8, Utf8]),
105-
],
106-
Volatility::Immutable,
107-
),
100+
signature: Signature::user_defined(Volatility::Immutable),
108101
}
109102
}
110103
}
@@ -119,33 +112,90 @@ impl ScalarUDFImpl for RegexpReplaceFunc {
119112
}
120113

121114
fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
122-
use DataType::*;
123-
Ok(match &arg_types[0] {
124-
LargeUtf8 | LargeBinary => LargeUtf8,
125-
Utf8 | Binary => Utf8,
126-
Utf8View | BinaryView => Utf8View,
127-
Null => Null,
128-
Dictionary(_, t) => match **t {
129-
LargeUtf8 | LargeBinary => LargeUtf8,
130-
Utf8 | Binary => Utf8,
131-
Null => Null,
132-
_ => {
133-
return plan_err!(
134-
"the regexp_replace can only accept strings but got {:?}",
135-
**t
136-
);
137-
}
138-
},
139-
other => {
140-
return plan_err!(
141-
"The regexp_replace function can only accept strings. Got {other}"
142-
);
115+
transform_leaf_type_preserving_encoding(&arg_types[0], &regexp_replace_leaf_type)
116+
}
117+
118+
fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {
119+
let input = &args.arg_fields[0];
120+
let others_scalar = args.scalar_arguments.iter().skip(1).all(Option::is_some);
121+
let output_type = match input.data_type() {
122+
DataType::Dictionary(_, value_type) if !others_scalar => {
123+
regexp_replace_leaf_type(value_type.as_ref())?
143124
}
144-
})
125+
other => self.return_type(std::slice::from_ref(other))?,
126+
};
127+
Ok(Arc::new(Field::new(self.name(), output_type, true)))
128+
}
129+
130+
fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
131+
if !(3..=4).contains(&arg_types.len()) {
132+
return plan_err!(
133+
"regexp_replace expects 3 or 4 arguments, got {}",
134+
arg_types.len()
135+
);
136+
}
137+
let leaf = match &arg_types[0] {
138+
DataType::Dictionary(_, v) => v.as_ref(),
139+
other => other,
140+
};
141+
let is_string_leaf = matches!(
142+
leaf,
143+
DataType::Utf8
144+
| DataType::LargeUtf8
145+
| DataType::Utf8View
146+
| DataType::Binary
147+
| DataType::LargeBinary
148+
| DataType::BinaryView
149+
);
150+
let target = if is_string_leaf {
151+
regexp_replace_leaf_type(leaf)?
152+
} else {
153+
DataType::Utf8View
154+
};
155+
let first = match &arg_types[0] {
156+
DataType::Dictionary(k, _) if is_string_leaf => {
157+
DataType::Dictionary(k.clone(), Box::new(target.clone()))
158+
}
159+
_ => target.clone(),
160+
};
161+
let mut out = Vec::with_capacity(arg_types.len());
162+
out.push(first);
163+
for _ in 1..arg_types.len() {
164+
out.push(target.clone());
165+
}
166+
Ok(out)
145167
}
146168

147169
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
148-
let args = &args.args;
170+
let mut args = args.args;
171+
172+
let scalar_dict_key = if let Some(ColumnarValue::Scalar(
173+
ScalarValue::Dictionary(key_type, value),
174+
)) = args.first()
175+
{
176+
let key = key_type.clone();
177+
args[0] = ColumnarValue::Scalar(value.as_ref().clone());
178+
Some(key)
179+
} else {
180+
None
181+
};
182+
183+
if let ColumnarValue::Array(array) = &args[0]
184+
&& let DataType::Dictionary(_, value_type) = array.data_type().clone()
185+
{
186+
let others_scalar = args[1..]
187+
.iter()
188+
.all(|a| matches!(a, ColumnarValue::Scalar(_)));
189+
if others_scalar {
190+
let dict = array.as_any_dictionary();
191+
let mut inner: Vec<ColumnarValue> = Vec::with_capacity(args.len());
192+
inner.push(ColumnarValue::Array(Arc::clone(dict.values())));
193+
inner.extend_from_slice(&args[1..]);
194+
let values_out = regexp_replace_func(&inner)?;
195+
return Ok(ColumnarValue::Array(dict.with_values(values_out)));
196+
}
197+
args[0] = ColumnarValue::Array(arrow::compute::cast(array, &value_type)?);
198+
}
149199

150200
let len = args
151201
.iter()
@@ -155,21 +205,42 @@ impl ScalarUDFImpl for RegexpReplaceFunc {
155205
});
156206

157207
let is_scalar = len.is_none();
158-
let result = regexp_replace_func(args);
159-
if is_scalar {
160-
// If all inputs are scalar, keeps output as scalar
161-
let result = result.and_then(|arr| ScalarValue::try_from_array(&arr, 0));
162-
result.map(ColumnarValue::Scalar)
208+
let result = regexp_replace_func(&args)?;
209+
let out = if is_scalar {
210+
ColumnarValue::Scalar(ScalarValue::try_from_array(&result, 0)?)
163211
} else {
164-
result.map(ColumnarValue::Array)
212+
ColumnarValue::Array(result)
213+
};
214+
215+
if let (Some(key_type), ColumnarValue::Scalar(scalar)) = (scalar_dict_key, &out) {
216+
return Ok(ColumnarValue::Scalar(ScalarValue::Dictionary(
217+
key_type,
218+
Box::new(scalar.clone()),
219+
)));
165220
}
221+
Ok(out)
166222
}
167223

168224
fn documentation(&self) -> Option<&Documentation> {
169225
self.doc()
170226
}
171227
}
172228

229+
fn regexp_replace_leaf_type(dt: &DataType) -> Result<DataType> {
230+
use DataType::*;
231+
Ok(match dt {
232+
LargeUtf8 | LargeBinary => LargeUtf8,
233+
Utf8 | Binary => Utf8,
234+
Utf8View | BinaryView => Utf8View,
235+
Null => Null,
236+
other => {
237+
return plan_err!(
238+
"the regexp_replace can only accept strings but got {other:?}"
239+
);
240+
}
241+
})
242+
}
243+
173244
fn regexp_replace_func(args: &[ColumnarValue]) -> Result<ArrayRef> {
174245
match args[0].data_type() {
175246
DataType::Utf8 => specialize_regexp_replace::<i32>(args),

‎datafusion/sqllogictest/test_files/regexp/regexp_replace.slt‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,24 @@ SELECT regexp_replace(arrow_cast('foobar', 'Dictionary(Int32, Utf8)'), 'bar', 'x
138138
----
139139
fooxx
140140

141+
statement ok
142+
CREATE TABLE regexp_replace_dict AS
143+
SELECT arrow_cast(column1, 'Dictionary(Int32, Utf8)') AS url
144+
FROM (VALUES
145+
('https://www.example.com/path'),
146+
('http://test.org/foo/bar'),
147+
('https://example.com/')
148+
) AS t(column1);
149+
150+
query T
151+
SELECT DISTINCT arrow_typeof(regexp_replace(url, '^https?://(?:www\.)?([^/]+)/.*$', '\1'))
152+
FROM regexp_replace_dict;
153+
----
154+
Dictionary(Int32, Utf8)
155+
156+
statement ok
157+
DROP TABLE regexp_replace_dict;
158+
141159
query TTT
142160
select
143161
regexp_replace(col, NULL, 'c'),

0 commit comments

Comments
 (0)