Skip to content

Commit fb8abd5

Browse files
authored
perf: preserve dictionary encoding through regexp_replace (apache#26186)
## Which issue does this PR close? - Closes apache#25522. ## Rationale for this change `regexp_replace` currently casts dictionary input to its underlying string type and materializes every row. When the pattern, replacement, and flags are scalar, it can apply the regex to the dictionary values once and reuse the existing keys. Filtered batches and shared Parquet dictionaries can contain many unreferenced values, so oversized dictionaries are compacted before applying the regex. ## What changes are included in this PR? - Keep the dictionary wrapper through coercion and return type inference when the other arguments are scalar. - Apply `regexp_replace` to the dictionary values and reuse the keys. - Fall back to a flat string result when the pattern, replacement, or flags are not scalar. ## What is the testing strategy for this PR? The sqllogictest coverage checks dictionary inputs with `Utf8`, `LargeUtf8`, `Utf8View`, and `Binary` values, along with the non-literal fallback. The focused `regexp_replace` unit tests and sqllogictest pass. ## Are there any user-facing changes? Yes. `regexp_replace` now preserves dictionary encoding when the other arguments are scalar. The returned values are unchanged.
1 parent 6546f09 commit fb8abd5

4 files changed

Lines changed: 204 additions & 41 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/functions/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ name = "datafusion_functions"
6767
[dependencies]
6868
arrow = { workspace = true }
6969
arrow-buffer = { workspace = true }
70+
arrow-select = { workspace = true }
7071
base64 = { version = "0.23", optional = true }
7172
blake2 = { version = "^0.11.0", optional = true }
7273
blake3 = { version = "1.8", optional = true }

‎datafusion/functions/src/regex/regexpreplace.rs‎

Lines changed: 119 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,16 @@
1818
//! Regex expressions
1919
use memchr::memchr;
2020

21+
use crate::utils::transform_leaf_type_preserving_encoding;
2122
use arrow::array::ArrayDataBuilder;
2223
use arrow::array::BufferBuilder;
2324
use arrow::array::GenericStringArray;
2425
use arrow::array::StringViewBuilder;
2526
use arrow::array::{Array, ArrayRef, OffsetSizeTrait};
2627
use arrow::array::{ArrayAccessor, StringViewArray};
2728
use arrow::array::{ArrayIter, AsArray, new_null_array};
28-
use arrow::datatypes::DataType;
29+
use arrow::datatypes::{DataType, Field, FieldRef};
30+
use arrow_select::dictionary::garbage_collect_any_dictionary;
2931
use datafusion_common::ScalarValue;
3032
use datafusion_common::cast::{
3133
as_large_string_array, as_string_array, as_string_view_array,
@@ -35,10 +37,10 @@ use datafusion_common::plan_err;
3537
use datafusion_common::utils::offset_span_len;
3638
use datafusion_common::{Result, cast::as_generic_string_array, internal_err};
3739
use datafusion_expr::ColumnarValue;
38-
use datafusion_expr::TypeSignature;
3940
use datafusion_expr::function::Hint;
4041
use datafusion_expr::{
41-
Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
42+
Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature,
43+
Volatility,
4244
};
4345
use datafusion_macros::user_doc;
4446
use regex::{CaptureLocations, Regex};
@@ -95,16 +97,8 @@ impl Default for RegexpReplaceFunc {
9597

9698
impl RegexpReplaceFunc {
9799
pub fn new() -> Self {
98-
use DataType::*;
99-
use TypeSignature::*;
100100
Self {
101-
signature: Signature::one_of(
102-
vec![
103-
Uniform(3, vec![Utf8View, LargeUtf8, Utf8]),
104-
Uniform(4, vec![Utf8View, LargeUtf8, Utf8]),
105-
],
106-
Volatility::Immutable,
107-
),
101+
signature: Signature::user_defined(Volatility::Immutable),
108102
}
109103
}
110104
}
@@ -119,33 +113,96 @@ impl ScalarUDFImpl for RegexpReplaceFunc {
119113
}
120114

121115
fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
122-
use DataType::*;
123-
Ok(match &arg_types[0] {
124-
LargeUtf8 | LargeBinary => LargeUtf8,
125-
Utf8 | Binary => Utf8,
126-
Utf8View | BinaryView => Utf8View,
127-
Null => Null,
128-
Dictionary(_, t) => match **t {
129-
LargeUtf8 | LargeBinary => LargeUtf8,
130-
Utf8 | Binary => Utf8,
131-
Null => Null,
132-
_ => {
133-
return plan_err!(
134-
"the regexp_replace can only accept strings but got {:?}",
135-
**t
136-
);
137-
}
138-
},
139-
other => {
140-
return plan_err!(
141-
"The regexp_replace function can only accept strings. Got {other}"
142-
);
116+
transform_leaf_type_preserving_encoding(&arg_types[0], &regexp_replace_leaf_type)
117+
}
118+
119+
fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {
120+
let input = &args.arg_fields[0];
121+
let others_scalar = args.scalar_arguments.iter().skip(1).all(Option::is_some);
122+
let output_type = match input.data_type() {
123+
DataType::Dictionary(_, value_type) if !others_scalar => {
124+
regexp_replace_leaf_type(value_type.as_ref())?
143125
}
144-
})
126+
other => self.return_type(std::slice::from_ref(other))?,
127+
};
128+
Ok(Arc::new(Field::new(self.name(), output_type, true)))
129+
}
130+
131+
fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
132+
if !(3..=4).contains(&arg_types.len()) {
133+
return plan_err!(
134+
"regexp_replace expects 3 or 4 arguments, got {}",
135+
arg_types.len()
136+
);
137+
}
138+
let leaf = match &arg_types[0] {
139+
DataType::Dictionary(_, v) => v.as_ref(),
140+
other => other,
141+
};
142+
let is_string_leaf = matches!(
143+
leaf,
144+
DataType::Utf8
145+
| DataType::LargeUtf8
146+
| DataType::Utf8View
147+
| DataType::Binary
148+
| DataType::LargeBinary
149+
| DataType::BinaryView
150+
);
151+
let target = if is_string_leaf {
152+
regexp_replace_leaf_type(leaf)?
153+
} else {
154+
DataType::Utf8View
155+
};
156+
let first = match &arg_types[0] {
157+
DataType::Dictionary(k, _) if is_string_leaf => {
158+
DataType::Dictionary(k.clone(), Box::new(target.clone()))
159+
}
160+
_ => target.clone(),
161+
};
162+
let mut out = Vec::with_capacity(arg_types.len());
163+
out.push(first);
164+
for _ in 1..arg_types.len() {
165+
out.push(target.clone());
166+
}
167+
Ok(out)
145168
}
146169

147170
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
148-
let args = &args.args;
171+
let preserve_dictionary =
172+
matches!(args.return_field.data_type(), DataType::Dictionary(_, _));
173+
let mut args = args.args;
174+
175+
let scalar_dict_key = if let Some(ColumnarValue::Scalar(
176+
ScalarValue::Dictionary(key_type, value),
177+
)) = args.first()
178+
{
179+
let key = key_type.clone();
180+
args[0] = ColumnarValue::Scalar(value.as_ref().clone());
181+
preserve_dictionary.then_some(key)
182+
} else {
183+
None
184+
};
185+
186+
if let ColumnarValue::Array(array) = &args[0]
187+
&& let DataType::Dictionary(_, value_type) = array.data_type().clone()
188+
{
189+
if preserve_dictionary {
190+
let compacted;
191+
let mut dict = array.as_any_dictionary();
192+
if dict.values().len() > dict.keys().len() {
193+
// Filtered batches and shared Parquet dictionaries can have more
194+
// values than rows, so drop unreferenced values first.
195+
compacted = garbage_collect_any_dictionary(dict)?;
196+
dict = compacted.as_any_dictionary();
197+
}
198+
let mut inner: Vec<ColumnarValue> = Vec::with_capacity(args.len());
199+
inner.push(ColumnarValue::Array(Arc::clone(dict.values())));
200+
inner.extend_from_slice(&args[1..]);
201+
let values_out = regexp_replace_func(&inner)?;
202+
return Ok(ColumnarValue::Array(dict.with_values(values_out)));
203+
}
204+
args[0] = ColumnarValue::Array(arrow::compute::cast(array, &value_type)?);
205+
}
149206

150207
let len = args
151208
.iter()
@@ -155,21 +212,42 @@ impl ScalarUDFImpl for RegexpReplaceFunc {
155212
});
156213

157214
let is_scalar = len.is_none();
158-
let result = regexp_replace_func(args);
159-
if is_scalar {
160-
// If all inputs are scalar, keeps output as scalar
161-
let result = result.and_then(|arr| ScalarValue::try_from_array(&arr, 0));
162-
result.map(ColumnarValue::Scalar)
215+
let result = regexp_replace_func(&args)?;
216+
let out = if is_scalar {
217+
ColumnarValue::Scalar(ScalarValue::try_from_array(&result, 0)?)
163218
} else {
164-
result.map(ColumnarValue::Array)
219+
ColumnarValue::Array(result)
220+
};
221+
222+
if let (Some(key_type), ColumnarValue::Scalar(scalar)) = (scalar_dict_key, &out) {
223+
return Ok(ColumnarValue::Scalar(ScalarValue::Dictionary(
224+
key_type,
225+
Box::new(scalar.clone()),
226+
)));
165227
}
228+
Ok(out)
166229
}
167230

168231
fn documentation(&self) -> Option<&Documentation> {
169232
self.doc()
170233
}
171234
}
172235

236+
fn regexp_replace_leaf_type(dt: &DataType) -> Result<DataType> {
237+
use DataType::*;
238+
Ok(match dt {
239+
LargeUtf8 | LargeBinary => LargeUtf8,
240+
Utf8 | Binary => Utf8,
241+
Utf8View | BinaryView => Utf8View,
242+
Null => Null,
243+
other => {
244+
return plan_err!(
245+
"the regexp_replace can only accept strings but got {other:?}"
246+
);
247+
}
248+
})
249+
}
250+
173251
fn regexp_replace_func(args: &[ColumnarValue]) -> Result<ArrayRef> {
174252
match args[0].data_type() {
175253
DataType::Utf8 => specialize_regexp_replace::<i32>(args),

‎datafusion/sqllogictest/test_files/regexp/regexp_replace.slt‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,89 @@ SELECT regexp_replace(arrow_cast('foobar', 'Dictionary(Int32, Utf8)'), 'bar', 'x
138138
----
139139
fooxx
140140

141+
statement ok
142+
CREATE TABLE regexp_replace_dict AS
143+
SELECT arrow_cast(column1, 'Dictionary(Int32, Utf8)') AS url
144+
FROM (VALUES
145+
('https://www.example.com/path'),
146+
('http://test.org/foo/bar'),
147+
('https://example.com/')
148+
) AS t(column1);
149+
150+
query T
151+
SELECT DISTINCT arrow_typeof(regexp_replace(url, '^https?://(?:www\.)?([^/]+)/.*$', '\1'))
152+
FROM regexp_replace_dict;
153+
----
154+
Dictionary(Int32, Utf8)
155+
156+
statement ok
157+
DROP TABLE regexp_replace_dict;
158+
159+
statement ok
160+
CREATE TABLE regexp_replace_dict_runtime_scalars AS
161+
SELECT arrow_cast(column1, 'Dictionary(Int32, Utf8)') AS s,
162+
column2 AS p
163+
FROM (VALUES ('abc', 'a'), ('abd', 'b')) AS v;
164+
165+
query T
166+
SELECT regexp_replace(s, 'a', arrow_typeof(p))
167+
FROM regexp_replace_dict_runtime_scalars
168+
ORDER BY p;
169+
----
170+
Utf8bc
171+
Utf8bd
172+
173+
statement ok
174+
DROP TABLE regexp_replace_dict_runtime_scalars;
175+
176+
statement ok
177+
CREATE TABLE regexp_replace_dict_types AS SELECT
178+
arrow_cast(column1, 'Dictionary(Int32, LargeUtf8)') AS dl,
179+
arrow_cast(column1, 'Dictionary(Int32, Utf8View)') AS dv,
180+
arrow_cast(column1, 'Dictionary(Int32, Binary)') AS db,
181+
arrow_cast(column1, 'Dictionary(Int32, Utf8)') AS d8,
182+
column1 AS s
183+
FROM (VALUES ('abc'), ('abd'), (NULL)) AS v;
184+
185+
query TT
186+
SELECT arrow_typeof(regexp_replace(dl, 'a', 'x')), regexp_replace(dl, 'a', 'x')
187+
FROM regexp_replace_dict_types
188+
ORDER BY s NULLS LAST;
189+
----
190+
Dictionary(Int32, LargeUtf8) xbc
191+
Dictionary(Int32, LargeUtf8) xbd
192+
Dictionary(Int32, LargeUtf8) NULL
193+
194+
query TT
195+
SELECT arrow_typeof(regexp_replace(dv, 'a', 'x')), regexp_replace(dv, 'a', 'x')
196+
FROM regexp_replace_dict_types
197+
ORDER BY s NULLS LAST;
198+
----
199+
Dictionary(Int32, Utf8View) xbc
200+
Dictionary(Int32, Utf8View) xbd
201+
Dictionary(Int32, Utf8View) NULL
202+
203+
query TT
204+
SELECT arrow_typeof(regexp_replace(db, 'a', 'x')), regexp_replace(db, 'a', 'x')
205+
FROM regexp_replace_dict_types
206+
ORDER BY s NULLS LAST;
207+
----
208+
Dictionary(Int32, Utf8) xbc
209+
Dictionary(Int32, Utf8) xbd
210+
Dictionary(Int32, Utf8) NULL
211+
212+
query TT
213+
SELECT arrow_typeof(regexp_replace(d8, s, 'x')), regexp_replace(d8, s, 'x')
214+
FROM regexp_replace_dict_types
215+
ORDER BY s NULLS LAST;
216+
----
217+
Utf8 x
218+
Utf8 x
219+
Utf8 NULL
220+
221+
statement ok
222+
DROP TABLE regexp_replace_dict_types;
223+
141224
query TTT
142225
select
143226
regexp_replace(col, NULL, 'c'),

0 commit comments

Comments
 (0)