1818//! Regex expressions
1919use memchr:: memchr;
2020
21+ use crate :: utils:: transform_leaf_type_preserving_encoding;
2122use arrow:: array:: ArrayDataBuilder ;
2223use arrow:: array:: BufferBuilder ;
2324use arrow:: array:: GenericStringArray ;
2425use arrow:: array:: StringViewBuilder ;
2526use arrow:: array:: { Array , ArrayRef , OffsetSizeTrait } ;
2627use arrow:: array:: { ArrayAccessor , StringViewArray } ;
2728use arrow:: array:: { ArrayIter , AsArray , new_null_array} ;
28- use arrow:: datatypes:: DataType ;
29+ use arrow:: datatypes:: { DataType , Field , FieldRef } ;
2930use datafusion_common:: ScalarValue ;
3031use datafusion_common:: cast:: {
3132 as_large_string_array, as_string_array, as_string_view_array,
3233} ;
3334use datafusion_common:: exec_err;
3435use datafusion_common:: plan_err;
36+ use datafusion_common:: types:: logical_string;
3537use datafusion_common:: utils:: offset_span_len;
3638use datafusion_common:: { Result , cast:: as_generic_string_array, internal_err} ;
3739use datafusion_expr:: ColumnarValue ;
3840use datafusion_expr:: TypeSignature ;
3941use datafusion_expr:: function:: Hint ;
4042use datafusion_expr:: {
41- Documentation , ScalarFunctionArgs , ScalarUDFImpl , Signature , Volatility ,
43+ Coercion , Documentation , EncodingPreservation , ReturnFieldArgs , ScalarFunctionArgs ,
44+ ScalarUDFImpl , Signature , TypeSignatureClass , Volatility ,
4245} ;
4346use datafusion_macros:: user_doc;
4447use regex:: { CaptureLocations , Regex } ;
@@ -95,13 +98,36 @@ impl Default for RegexpReplaceFunc {
9598
9699impl RegexpReplaceFunc {
97100 pub fn new ( ) -> Self {
98- use DataType :: * ;
99- use TypeSignature :: * ;
101+ let string_with_dict = || {
102+ Coercion :: new_exact ( TypeSignatureClass :: Native ( logical_string ( ) ) )
103+ . with_encoding_preservation ( EncodingPreservation :: dictionary ( ) )
104+ } ;
105+ let string = || Coercion :: new_exact ( TypeSignatureClass :: Native ( logical_string ( ) ) ) ;
100106 Self {
107+ // Coercible preserves dict encoding on arg[0]; the Uniform fallback
108+ // keeps backwards compatibility for non-string inputs that used to
109+ // coerce to Utf8 (e.g. Boolean) under the original Uniform signature.
101110 signature : Signature :: one_of (
102111 vec ! [
103- Uniform ( 3 , vec![ Utf8View , LargeUtf8 , Utf8 ] ) ,
104- Uniform ( 4 , vec![ Utf8View , LargeUtf8 , Utf8 ] ) ,
112+ TypeSignature :: Coercible ( vec![
113+ string_with_dict( ) ,
114+ string( ) ,
115+ string( ) ,
116+ ] ) ,
117+ TypeSignature :: Coercible ( vec![
118+ string_with_dict( ) ,
119+ string( ) ,
120+ string( ) ,
121+ string( ) ,
122+ ] ) ,
123+ TypeSignature :: Uniform (
124+ 3 ,
125+ vec![ DataType :: Utf8 , DataType :: LargeUtf8 , DataType :: Utf8View ] ,
126+ ) ,
127+ TypeSignature :: Uniform (
128+ 4 ,
129+ vec![ DataType :: Utf8 , DataType :: LargeUtf8 , DataType :: Utf8View ] ,
130+ ) ,
105131 ] ,
106132 Volatility :: Immutable ,
107133 ) ,
@@ -119,33 +145,54 @@ impl ScalarUDFImpl for RegexpReplaceFunc {
119145 }
120146
121147 fn return_type ( & self , arg_types : & [ DataType ] ) -> Result < DataType > {
122- use DataType :: * ;
123- Ok ( match & arg_types[ 0 ] {
124- LargeUtf8 | LargeBinary => LargeUtf8 ,
125- Utf8 | Binary => Utf8 ,
126- Utf8View | BinaryView => Utf8View ,
127- Null => Null ,
128- Dictionary ( _, t) => match * * t {
129- LargeUtf8 | LargeBinary => LargeUtf8 ,
130- Utf8 | Binary => Utf8 ,
131- Null => Null ,
132- _ => {
133- return plan_err ! (
134- "the regexp_replace can only accept strings but got {:?}" ,
135- * * t
136- ) ;
137- }
138- } ,
139- other => {
140- return plan_err ! (
141- "The regexp_replace function can only accept strings. Got {other}"
142- ) ;
148+ transform_leaf_type_preserving_encoding ( & arg_types[ 0 ] , & regexp_leaf_type)
149+ }
150+
151+ fn return_field_from_args ( & self , args : ReturnFieldArgs ) -> Result < FieldRef > {
152+ let input = & args. arg_fields [ 0 ] ;
153+ let others_all_scalar = args. scalar_arguments . iter ( ) . skip ( 1 ) . all ( Option :: is_some) ;
154+ // Dict can only round-trip when the remaining args are scalars; otherwise
155+ // `invoke_with_args` materializes the dict and returns the value type.
156+ let output_type = match input. data_type ( ) {
157+ DataType :: Dictionary ( _, value_type) if !others_all_scalar => {
158+ regexp_leaf_type ( value_type. as_ref ( ) ) ?
143159 }
144- } )
160+ other => self . return_type ( std:: slice:: from_ref ( other) ) ?,
161+ } ;
162+ Ok ( Arc :: new ( Field :: new ( self . name ( ) , output_type, true ) ) )
145163 }
146164
147165 fn invoke_with_args ( & self , args : ScalarFunctionArgs ) -> Result < ColumnarValue > {
148- let args = & args. args ;
166+ // Unwrap a scalar dict input so dispatch sees a plain string type;
167+ // re-wrap the scalar result afterwards to keep the dict wrapper that
168+ // `return_type` promised.
169+ let ( scalar_dict_key, mut args) = match args. args . first ( ) {
170+ Some ( ColumnarValue :: Scalar ( ScalarValue :: Dictionary ( key_type, value) ) ) => {
171+ let key_type = key_type. clone ( ) ;
172+ let mut inner = args. args . clone ( ) ;
173+ inner[ 0 ] = ColumnarValue :: Scalar ( value. as_ref ( ) . clone ( ) ) ;
174+ ( Some ( key_type) , inner)
175+ }
176+ _ => ( None , args. args ) ,
177+ } ;
178+
179+ // If arg[0] is a dict array but any of args 1..N are arrays, we cannot
180+ // reuse the dict keys — flatten the dict to its value type.
181+ if let ColumnarValue :: Array ( array) = & args[ 0 ]
182+ && let DataType :: Dictionary ( _, value_type) = array. data_type ( ) . clone ( )
183+ && args[ 1 ..]
184+ . iter ( )
185+ . any ( |a| matches ! ( a, ColumnarValue :: Array ( _) ) )
186+ {
187+ let flat = arrow:: compute:: cast ( array, & value_type) ?;
188+ args[ 0 ] = ColumnarValue :: Array ( flat) ;
189+ }
190+
191+ // The Coercible signature allows each arg to coerce independently, so
192+ // the string types across args 0..N may differ. The generic dispatch
193+ // in `specialize_regexp_replace` requires uniform string types, so cast
194+ // args 1..N to match arg[0]'s string type.
195+ unify_string_args ( & mut args) ?;
149196
150197 let len = args
151198 . iter ( )
@@ -155,32 +202,99 @@ impl ScalarUDFImpl for RegexpReplaceFunc {
155202 } ) ;
156203
157204 let is_scalar = len. is_none ( ) ;
158- let result = regexp_replace_func ( args) ;
159- if is_scalar {
205+ let result = regexp_replace_func ( & args) ;
206+ let result = if is_scalar {
160207 // If all inputs are scalar, keeps output as scalar
161- let result = result. and_then ( |arr| ScalarValue :: try_from_array ( & arr, 0 ) ) ;
162- result. map ( ColumnarValue :: Scalar )
208+ result
209+ . and_then ( |arr| ScalarValue :: try_from_array ( & arr, 0 ) )
210+ . map ( ColumnarValue :: Scalar )
163211 } else {
164212 result. map ( ColumnarValue :: Array )
213+ } ?;
214+
215+ if let ( Some ( key_type) , ColumnarValue :: Scalar ( scalar) ) =
216+ ( scalar_dict_key, & result)
217+ {
218+ return Ok ( ColumnarValue :: Scalar ( ScalarValue :: Dictionary (
219+ key_type,
220+ Box :: new ( scalar. clone ( ) ) ,
221+ ) ) ) ;
165222 }
223+ Ok ( result)
166224 }
167225
168226 fn documentation ( & self ) -> Option < & Documentation > {
169227 self . doc ( )
170228 }
171229}
172230
231+ /// Maps the input leaf type to the regexp_replace output leaf type. Shared
232+ /// between `return_type` and `return_field_from_args`.
233+ fn regexp_leaf_type ( data_type : & DataType ) -> Result < DataType > {
234+ use DataType :: * ;
235+ Ok ( match data_type {
236+ LargeUtf8 | LargeBinary => LargeUtf8 ,
237+ Utf8 | Binary => Utf8 ,
238+ Utf8View | BinaryView => Utf8View ,
239+ Null => Null ,
240+ other => {
241+ return plan_err ! (
242+ "The regexp_replace function can only accept strings. Got {other}"
243+ ) ;
244+ }
245+ } )
246+ }
247+
248+ /// Casts the pattern/replacement/flags args to match the source arg's string
249+ /// type. The generic dispatch in `specialize_regexp_replace` only handles
250+ /// uniform string types; without this step a mixed set like
251+ /// `(Utf8View, Utf8View, Utf8, Utf8View)` would fail to dispatch.
252+ fn unify_string_args ( args : & mut [ ColumnarValue ] ) -> Result < ( ) > {
253+ let target = match args[ 0 ] . data_type ( ) {
254+ DataType :: Dictionary ( _, value_type) => value_type. as_ref ( ) . clone ( ) ,
255+ other => other,
256+ } ;
257+ for arg in args. iter_mut ( ) . skip ( 1 ) {
258+ if arg. data_type ( ) == target {
259+ continue ;
260+ }
261+ * arg = match arg {
262+ ColumnarValue :: Array ( a) => {
263+ ColumnarValue :: Array ( arrow:: compute:: cast ( a, & target) ?)
264+ }
265+ ColumnarValue :: Scalar ( s) => ColumnarValue :: Scalar ( s. cast_to ( & target) ?) ,
266+ } ;
267+ }
268+ Ok ( ( ) )
269+ }
270+
173271fn regexp_replace_func ( args : & [ ColumnarValue ] ) -> Result < ArrayRef > {
174272 match args[ 0 ] . data_type ( ) {
175273 DataType :: Utf8 => specialize_regexp_replace :: < i32 > ( args) ,
176274 DataType :: LargeUtf8 => specialize_regexp_replace :: < i64 > ( args) ,
177275 DataType :: Utf8View => specialize_regexp_replace :: < i32 > ( args) ,
276+ DataType :: Dictionary ( _, _) => regexp_replace_dictionary ( args) ,
178277 other => {
179278 internal_err ! ( "Unsupported data type {other:?} for function regexp_replace" )
180279 }
181280 }
182281}
183282
283+ /// Runs regexp_replace over the dict's unique values and reuses the keys,
284+ /// avoiding materializing the input column.
285+ fn regexp_replace_dictionary ( args : & [ ColumnarValue ] ) -> Result < ArrayRef > {
286+ let ColumnarValue :: Array ( array) = & args[ 0 ] else {
287+ return internal_err ! (
288+ "regexp_replace: dictionary scalar input should be unwrapped by invoke_with_args"
289+ ) ;
290+ } ;
291+ let dictionary = array. as_any_dictionary ( ) ;
292+ let mut inner_args = args. to_vec ( ) ;
293+ inner_args[ 0 ] = ColumnarValue :: Array ( Arc :: clone ( dictionary. values ( ) ) ) ;
294+ let converted = regexp_replace_func ( & inner_args) ?;
295+ Ok ( dictionary. with_values ( converted) )
296+ }
297+
184298/// replace POSIX capture groups (like \1 or \\1) with Rust Regex group (like ${1})
185299/// used by regexp_replace
186300/// Handles both single backslash (\1) and double backslash (\\1) which can occur
@@ -470,17 +584,15 @@ where
470584/// Get the first argument from the given string array.
471585///
472586/// Note: If the array is empty or the first argument is null,
473- /// then aborts early.
587+ /// then aborts early with a null array of `$OUTPUT_DT` (the input column's
588+ /// type). Using the input column's type keeps the return value consistent
589+ /// with `return_type`, which also keys off the input column.
474590macro_rules! fetch_string_arg {
475- ( $ARG: expr, $NAME: expr, $ARRAY_SIZE: expr) => { {
591+ ( $ARG: expr, $NAME: expr, $ARRAY_SIZE: expr, $OUTPUT_DT : expr ) => { {
476592 let string_array_type = ( $ARG) . data_type( ) ;
477593 match string_array_type {
478- dt if $ARG. len( ) == 0 || $ARG. is_null( 0 ) => {
479- // Mimicking the existing behavior of regexp_replace, if any of the scalar arguments
480- // are actually null, then the result will be an array of the same size as the first argument with all nulls.
481- //
482- // Also acts like an early abort mechanism when the input array is empty.
483- return Ok ( new_null_array( dt, $ARRAY_SIZE) ) ;
594+ _ if $ARG. len( ) == 0 || $ARG. is_null( 0 ) => {
595+ return Ok ( new_null_array( $OUTPUT_DT, $ARRAY_SIZE) ) ;
484596 }
485597 DataType :: Utf8 => {
486598 let array = as_string_array( $ARG) ?;
@@ -511,11 +623,12 @@ fn regexp_replace_static_pattern_replace<T: OffsetSizeTrait>(
511623 args : & [ ArrayRef ] ,
512624) -> Result < ArrayRef > {
513625 let array_size = args[ 0 ] . len ( ) ;
514- let pattern = fetch_string_arg ! ( & args[ 1 ] , "pattern" , array_size) ;
515- let replacement = fetch_string_arg ! ( & args[ 2 ] , "replacement" , array_size) ;
626+ let output_dt = args[ 0 ] . data_type ( ) ;
627+ let pattern = fetch_string_arg ! ( & args[ 1 ] , "pattern" , array_size, output_dt) ;
628+ let replacement = fetch_string_arg ! ( & args[ 2 ] , "replacement" , array_size, output_dt) ;
516629 let flags = match args. len ( ) {
517630 3 => None ,
518- 4 => Some ( fetch_string_arg ! ( & args[ 3 ] , "flags" , array_size) ) ,
631+ 4 => Some ( fetch_string_arg ! ( & args[ 3 ] , "flags" , array_size, output_dt ) ) ,
519632 other => {
520633 return exec_err ! (
521634 "regexp_replace was called with {other} arguments. It requires at least 3 and at most 4."
0 commit comments