Skip to content

Commit 62e3ba4

Browse files
committed
perf: preserve dictionary encoding through regexp_replace
Preserves Dictionary encoding on the input column by running regexp_replace over the dict's unique values and reusing the keys, avoiding the full materialization that previously happened when the column was dict-encoded. Closes apache#25522.
1 parent 334f293 commit 62e3ba4

4 files changed

Lines changed: 179 additions & 47 deletions

File tree

‎datafusion/functions/src/regex/regexpreplace.rs‎

Lines changed: 157 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -18,27 +18,30 @@
1818
//! Regex expressions
1919
use memchr::memchr;
2020

21+
use crate::utils::transform_leaf_type_preserving_encoding;
2122
use arrow::array::ArrayDataBuilder;
2223
use arrow::array::BufferBuilder;
2324
use arrow::array::GenericStringArray;
2425
use arrow::array::StringViewBuilder;
2526
use arrow::array::{Array, ArrayRef, OffsetSizeTrait};
2627
use arrow::array::{ArrayAccessor, StringViewArray};
2728
use arrow::array::{ArrayIter, AsArray, new_null_array};
28-
use arrow::datatypes::DataType;
29+
use arrow::datatypes::{DataType, Field, FieldRef};
2930
use datafusion_common::ScalarValue;
3031
use datafusion_common::cast::{
3132
as_large_string_array, as_string_array, as_string_view_array,
3233
};
3334
use datafusion_common::exec_err;
3435
use datafusion_common::plan_err;
36+
use datafusion_common::types::logical_string;
3537
use datafusion_common::utils::offset_span_len;
3638
use datafusion_common::{Result, cast::as_generic_string_array, internal_err};
3739
use datafusion_expr::ColumnarValue;
3840
use datafusion_expr::TypeSignature;
3941
use datafusion_expr::function::Hint;
4042
use datafusion_expr::{
41-
Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
43+
Coercion, Documentation, EncodingPreservation, ReturnFieldArgs, ScalarFunctionArgs,
44+
ScalarUDFImpl, Signature, TypeSignatureClass, Volatility,
4245
};
4346
use datafusion_macros::user_doc;
4447
use regex::{CaptureLocations, Regex};
@@ -95,13 +98,36 @@ impl Default for RegexpReplaceFunc {
9598

9699
impl RegexpReplaceFunc {
97100
pub fn new() -> Self {
98-
use DataType::*;
99-
use TypeSignature::*;
101+
let string_with_dict = || {
102+
Coercion::new_exact(TypeSignatureClass::Native(logical_string()))
103+
.with_encoding_preservation(EncodingPreservation::dictionary())
104+
};
105+
let string = || Coercion::new_exact(TypeSignatureClass::Native(logical_string()));
100106
Self {
107+
// Coercible preserves dict encoding on arg[0]; the Uniform fallback
108+
// keeps backwards compatibility for non-string inputs that used to
109+
// coerce to Utf8 (e.g. Boolean) under the original Uniform signature.
101110
signature: Signature::one_of(
102111
vec![
103-
Uniform(3, vec![Utf8View, LargeUtf8, Utf8]),
104-
Uniform(4, vec![Utf8View, LargeUtf8, Utf8]),
112+
TypeSignature::Coercible(vec![
113+
string_with_dict(),
114+
string(),
115+
string(),
116+
]),
117+
TypeSignature::Coercible(vec![
118+
string_with_dict(),
119+
string(),
120+
string(),
121+
string(),
122+
]),
123+
TypeSignature::Uniform(
124+
3,
125+
vec![DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View],
126+
),
127+
TypeSignature::Uniform(
128+
4,
129+
vec![DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View],
130+
),
105131
],
106132
Volatility::Immutable,
107133
),
@@ -119,33 +145,54 @@ impl ScalarUDFImpl for RegexpReplaceFunc {
119145
}
120146

121147
fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
122-
use DataType::*;
123-
Ok(match &arg_types[0] {
124-
LargeUtf8 | LargeBinary => LargeUtf8,
125-
Utf8 | Binary => Utf8,
126-
Utf8View | BinaryView => Utf8View,
127-
Null => Null,
128-
Dictionary(_, t) => match **t {
129-
LargeUtf8 | LargeBinary => LargeUtf8,
130-
Utf8 | Binary => Utf8,
131-
Null => Null,
132-
_ => {
133-
return plan_err!(
134-
"the regexp_replace can only accept strings but got {:?}",
135-
**t
136-
);
137-
}
138-
},
139-
other => {
140-
return plan_err!(
141-
"The regexp_replace function can only accept strings. Got {other}"
142-
);
148+
transform_leaf_type_preserving_encoding(&arg_types[0], &regexp_leaf_type)
149+
}
150+
151+
fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {
152+
let input = &args.arg_fields[0];
153+
let others_all_scalar = args.scalar_arguments.iter().skip(1).all(Option::is_some);
154+
// Dict can only round-trip when the remaining args are scalars; otherwise
155+
// `invoke_with_args` materializes the dict and returns the value type.
156+
let output_type = match input.data_type() {
157+
DataType::Dictionary(_, value_type) if !others_all_scalar => {
158+
regexp_leaf_type(value_type.as_ref())?
143159
}
144-
})
160+
other => self.return_type(std::slice::from_ref(other))?,
161+
};
162+
Ok(Arc::new(Field::new(self.name(), output_type, true)))
145163
}
146164

147165
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
148-
let args = &args.args;
166+
// Unwrap a scalar dict input so dispatch sees a plain string type;
167+
// re-wrap the scalar result afterwards to keep the dict wrapper that
168+
// `return_type` promised.
169+
let (scalar_dict_key, mut args) = match args.args.first() {
170+
Some(ColumnarValue::Scalar(ScalarValue::Dictionary(key_type, value))) => {
171+
let key_type = key_type.clone();
172+
let mut inner = args.args.clone();
173+
inner[0] = ColumnarValue::Scalar(value.as_ref().clone());
174+
(Some(key_type), inner)
175+
}
176+
_ => (None, args.args),
177+
};
178+
179+
// If arg[0] is a dict array but any of args 1..N are arrays, we cannot
180+
// reuse the dict keys — flatten the dict to its value type.
181+
if let ColumnarValue::Array(array) = &args[0]
182+
&& let DataType::Dictionary(_, value_type) = array.data_type().clone()
183+
&& args[1..]
184+
.iter()
185+
.any(|a| matches!(a, ColumnarValue::Array(_)))
186+
{
187+
let flat = arrow::compute::cast(array, &value_type)?;
188+
args[0] = ColumnarValue::Array(flat);
189+
}
190+
191+
// The Coercible signature allows each arg to coerce independently, so
192+
// the string types across args 0..N may differ. The generic dispatch
193+
// in `specialize_regexp_replace` requires uniform string types, so cast
194+
// args 1..N to match arg[0]'s string type.
195+
unify_string_args(&mut args)?;
149196

150197
let len = args
151198
.iter()
@@ -155,32 +202,99 @@ impl ScalarUDFImpl for RegexpReplaceFunc {
155202
});
156203

157204
let is_scalar = len.is_none();
158-
let result = regexp_replace_func(args);
159-
if is_scalar {
205+
let result = regexp_replace_func(&args);
206+
let result = if is_scalar {
160207
// If all inputs are scalar, keeps output as scalar
161-
let result = result.and_then(|arr| ScalarValue::try_from_array(&arr, 0));
162-
result.map(ColumnarValue::Scalar)
208+
result
209+
.and_then(|arr| ScalarValue::try_from_array(&arr, 0))
210+
.map(ColumnarValue::Scalar)
163211
} else {
164212
result.map(ColumnarValue::Array)
213+
}?;
214+
215+
if let (Some(key_type), ColumnarValue::Scalar(scalar)) =
216+
(scalar_dict_key, &result)
217+
{
218+
return Ok(ColumnarValue::Scalar(ScalarValue::Dictionary(
219+
key_type,
220+
Box::new(scalar.clone()),
221+
)));
165222
}
223+
Ok(result)
166224
}
167225

168226
fn documentation(&self) -> Option<&Documentation> {
169227
self.doc()
170228
}
171229
}
172230

231+
/// Maps the input leaf type to the regexp_replace output leaf type. Shared
232+
/// between `return_type` and `return_field_from_args`.
233+
fn regexp_leaf_type(data_type: &DataType) -> Result<DataType> {
234+
use DataType::*;
235+
Ok(match data_type {
236+
LargeUtf8 | LargeBinary => LargeUtf8,
237+
Utf8 | Binary => Utf8,
238+
Utf8View | BinaryView => Utf8View,
239+
Null => Null,
240+
other => {
241+
return plan_err!(
242+
"The regexp_replace function can only accept strings. Got {other}"
243+
);
244+
}
245+
})
246+
}
247+
248+
/// Casts the pattern/replacement/flags args to match the source arg's string
249+
/// type. The generic dispatch in `specialize_regexp_replace` only handles
250+
/// uniform string types; without this step a mixed set like
251+
/// `(Utf8View, Utf8View, Utf8, Utf8View)` would fail to dispatch.
252+
fn unify_string_args(args: &mut [ColumnarValue]) -> Result<()> {
253+
let target = match args[0].data_type() {
254+
DataType::Dictionary(_, value_type) => value_type.as_ref().clone(),
255+
other => other,
256+
};
257+
for arg in args.iter_mut().skip(1) {
258+
if arg.data_type() == target {
259+
continue;
260+
}
261+
*arg = match arg {
262+
ColumnarValue::Array(a) => {
263+
ColumnarValue::Array(arrow::compute::cast(a, &target)?)
264+
}
265+
ColumnarValue::Scalar(s) => ColumnarValue::Scalar(s.cast_to(&target)?),
266+
};
267+
}
268+
Ok(())
269+
}
270+
173271
fn regexp_replace_func(args: &[ColumnarValue]) -> Result<ArrayRef> {
174272
match args[0].data_type() {
175273
DataType::Utf8 => specialize_regexp_replace::<i32>(args),
176274
DataType::LargeUtf8 => specialize_regexp_replace::<i64>(args),
177275
DataType::Utf8View => specialize_regexp_replace::<i32>(args),
276+
DataType::Dictionary(_, _) => regexp_replace_dictionary(args),
178277
other => {
179278
internal_err!("Unsupported data type {other:?} for function regexp_replace")
180279
}
181280
}
182281
}
183282

283+
/// Runs regexp_replace over the dict's unique values and reuses the keys,
284+
/// avoiding materializing the input column.
285+
fn regexp_replace_dictionary(args: &[ColumnarValue]) -> Result<ArrayRef> {
286+
let ColumnarValue::Array(array) = &args[0] else {
287+
return internal_err!(
288+
"regexp_replace: dictionary scalar input should be unwrapped by invoke_with_args"
289+
);
290+
};
291+
let dictionary = array.as_any_dictionary();
292+
let mut inner_args = args.to_vec();
293+
inner_args[0] = ColumnarValue::Array(Arc::clone(dictionary.values()));
294+
let converted = regexp_replace_func(&inner_args)?;
295+
Ok(dictionary.with_values(converted))
296+
}
297+
184298
/// replace POSIX capture groups (like \1 or \\1) with Rust Regex group (like ${1})
185299
/// used by regexp_replace
186300
/// Handles both single backslash (\1) and double backslash (\\1) which can occur
@@ -470,17 +584,15 @@ where
470584
/// Get the first argument from the given string array.
471585
///
472586
/// Note: If the array is empty or the first argument is null,
473-
/// then aborts early.
587+
/// then aborts early with a null array of `$OUTPUT_DT` (the input column's
588+
/// type). Using the input column's type keeps the return value consistent
589+
/// with `return_type`, which also keys off the input column.
474590
macro_rules! fetch_string_arg {
475-
($ARG:expr, $NAME:expr, $ARRAY_SIZE:expr) => {{
591+
($ARG:expr, $NAME:expr, $ARRAY_SIZE:expr, $OUTPUT_DT:expr) => {{
476592
let string_array_type = ($ARG).data_type();
477593
match string_array_type {
478-
dt if $ARG.len() == 0 || $ARG.is_null(0) => {
479-
// Mimicking the existing behavior of regexp_replace, if any of the scalar arguments
480-
// are actually null, then the result will be an array of the same size as the first argument with all nulls.
481-
//
482-
// Also acts like an early abort mechanism when the input array is empty.
483-
return Ok(new_null_array(dt, $ARRAY_SIZE));
594+
_ if $ARG.len() == 0 || $ARG.is_null(0) => {
595+
return Ok(new_null_array($OUTPUT_DT, $ARRAY_SIZE));
484596
}
485597
DataType::Utf8 => {
486598
let array = as_string_array($ARG)?;
@@ -511,11 +623,12 @@ fn regexp_replace_static_pattern_replace<T: OffsetSizeTrait>(
511623
args: &[ArrayRef],
512624
) -> Result<ArrayRef> {
513625
let array_size = args[0].len();
514-
let pattern = fetch_string_arg!(&args[1], "pattern", array_size);
515-
let replacement = fetch_string_arg!(&args[2], "replacement", array_size);
626+
let output_dt = args[0].data_type();
627+
let pattern = fetch_string_arg!(&args[1], "pattern", array_size, output_dt);
628+
let replacement = fetch_string_arg!(&args[2], "replacement", array_size, output_dt);
516629
let flags = match args.len() {
517630
3 => None,
518-
4 => Some(fetch_string_arg!(&args[3], "flags", array_size)),
631+
4 => Some(fetch_string_arg!(&args[3], "flags", array_size, output_dt)),
519632
other => {
520633
return exec_err!(
521634
"regexp_replace was called with {other} arguments. It requires at least 3 and at most 4."

‎datafusion/sqllogictest/test_files/clickbench.slt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -762,7 +762,7 @@ logical_plan
762762
01)Sort: l DESC NULLS FIRST, fetch=25
763763
02)--Projection: regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1")) AS k, avg(octet_length(hits.Referer)) AS l, count(Int64(1)) AS count(*) AS c, min(hits.Referer)
764764
03)----Filter: count(Int64(1)) > Int64(100000)
765-
04)------Aggregate: groupBy=[[regexp_replace(hits.Referer, Utf8View("^https?://(?:www\.)?([^/]+)/.*$"), Utf8View("\1")) AS regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))]], aggr=[[avg(CAST(octet_length(hits.Referer) AS Float64)), count(Int64(1)), min(hits.Referer)]]
765+
04)------Aggregate: groupBy=[[regexp_replace(hits.Referer, Utf8("^https?://(?:www\.)?([^/]+)/.*$"), Utf8("\1"))]], aggr=[[avg(CAST(octet_length(hits.Referer) AS Float64)), count(Int64(1)), min(hits.Referer)]]
766766
05)--------SubqueryAlias: hits
767767
06)----------Filter: hits_raw.Referer != Utf8View("")
768768
07)------------TableScan: hits_raw projection=[Referer], partial_filters=[hits_raw.Referer != Utf8View("")]
@@ -774,7 +774,7 @@ physical_plan
774774
05)--------AggregateExec: mode=FinalPartitioned, gby=[regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))@0 as regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))], aggr=[avg(octet_length(hits.Referer)), count(Int64(1)), min(hits.Referer)]
775775
06)----------RepartitionExec: partitioning=Hash([regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))@0], 4), input_partitions=4
776776
07)------------AggregateExec: mode=Partial, gby=[regexp_replace(Referer@0, ^https?://(?:www\.)?([^/]+)/.*$, \1) as regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))], aggr=[avg(octet_length(hits.Referer)), count(Int64(1)), min(hits.Referer)]
777-
08)--------------FilterExec: Referer@0 !=
777+
08)--------------FilterExec: Referer@0 !=
778778
09)----------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
779779
10)------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/clickbench_hits_10.parquet]]}, projection=[Referer], file_type=parquet, predicate=Referer@14 != , pruning_predicate=Referer_null_count@2 != row_count@3 AND (Referer_min@0 != OR != Referer_max@1), required_guarantees=[Referer not in ()]
780780

‎datafusion/sqllogictest/test_files/regexp/regexp_replace.slt‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,25 @@ SELECT regexp_replace(arrow_cast('foobar', 'Dictionary(Int32, Utf8)'), 'bar', 'x
138138
----
139139
fooxx
140140

141+
# Dict column with scalar pattern/replacement preserves dictionary encoding.
142+
statement ok
143+
CREATE TABLE regexp_replace_dict AS
144+
SELECT arrow_cast(column1, 'Dictionary(Int32, Utf8)') AS url
145+
FROM (VALUES
146+
('https://www.example.com/path'),
147+
('http://test.org/foo/bar'),
148+
('https://example.com/')
149+
) AS t(column1);
150+
151+
query T
152+
SELECT DISTINCT arrow_typeof(regexp_replace(url, '^https?://(?:www\.)?([^/]+)/.*$', '\1'))
153+
FROM regexp_replace_dict;
154+
----
155+
Dictionary(Int32, Utf8)
156+
157+
statement ok
158+
DROP TABLE regexp_replace_dict;
159+
141160
query TTT
142161
select
143162
regexp_replace(col, NULL, 'c'),

‎datafusion/sqllogictest/test_files/string/string_view.slt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -814,7 +814,7 @@ EXPLAIN SELECT
814814
FROM test;
815815
----
816816
logical_plan
817-
01)Projection: regexp_replace(test.column1_utf8view, Utf8View("^https?://(?:www\.)?([^/]+)/.*$"), Utf8View("\1")) AS k
817+
01)Projection: regexp_replace(test.column1_utf8view, Utf8("^https?://(?:www\.)?([^/]+)/.*$"), Utf8("\1")) AS k
818818
02)--TableScan: test projection=[column1_utf8view]
819819

820820
## Ensure no casts for REPEAT

0 commit comments

Comments
 (0)