1818//! Regex expressions
1919use memchr:: memchr;
2020
21+ use crate :: utils:: transform_leaf_type_preserving_encoding;
2122use arrow:: array:: ArrayDataBuilder ;
2223use arrow:: array:: BufferBuilder ;
2324use arrow:: array:: GenericStringArray ;
2425use arrow:: array:: StringViewBuilder ;
2526use arrow:: array:: { Array , ArrayRef , OffsetSizeTrait } ;
2627use arrow:: array:: { ArrayAccessor , StringViewArray } ;
2728use arrow:: array:: { ArrayIter , AsArray , new_null_array} ;
28- use arrow:: datatypes:: DataType ;
29+ use arrow:: datatypes:: { DataType , Field , FieldRef } ;
2930use datafusion_common:: ScalarValue ;
3031use datafusion_common:: cast:: {
3132 as_large_string_array, as_string_array, as_string_view_array,
@@ -35,10 +36,10 @@ use datafusion_common::plan_err;
3536use datafusion_common:: utils:: offset_span_len;
3637use datafusion_common:: { Result , cast:: as_generic_string_array, internal_err} ;
3738use datafusion_expr:: ColumnarValue ;
38- use datafusion_expr:: TypeSignature ;
3939use datafusion_expr:: function:: Hint ;
4040use datafusion_expr:: {
41- Documentation , ScalarFunctionArgs , ScalarUDFImpl , Signature , Volatility ,
41+ Documentation , ReturnFieldArgs , ScalarFunctionArgs , ScalarUDFImpl , Signature ,
42+ Volatility ,
4243} ;
4344use datafusion_macros:: user_doc;
4445use regex:: { CaptureLocations , Regex } ;
@@ -95,16 +96,8 @@ impl Default for RegexpReplaceFunc {
9596
9697impl RegexpReplaceFunc {
9798 pub fn new ( ) -> Self {
98- use DataType :: * ;
99- use TypeSignature :: * ;
10099 Self {
101- signature : Signature :: one_of (
102- vec ! [
103- Uniform ( 3 , vec![ Utf8View , LargeUtf8 , Utf8 ] ) ,
104- Uniform ( 4 , vec![ Utf8View , LargeUtf8 , Utf8 ] ) ,
105- ] ,
106- Volatility :: Immutable ,
107- ) ,
100+ signature : Signature :: user_defined ( Volatility :: Immutable ) ,
108101 }
109102 }
110103}
@@ -119,33 +112,90 @@ impl ScalarUDFImpl for RegexpReplaceFunc {
119112 }
120113
121114 fn return_type ( & self , arg_types : & [ DataType ] ) -> Result < DataType > {
122- use DataType :: * ;
123- Ok ( match & arg_types[ 0 ] {
124- LargeUtf8 | LargeBinary => LargeUtf8 ,
125- Utf8 | Binary => Utf8 ,
126- Utf8View | BinaryView => Utf8View ,
127- Null => Null ,
128- Dictionary ( _, t) => match * * t {
129- LargeUtf8 | LargeBinary => LargeUtf8 ,
130- Utf8 | Binary => Utf8 ,
131- Null => Null ,
132- _ => {
133- return plan_err ! (
134- "the regexp_replace can only accept strings but got {:?}" ,
135- * * t
136- ) ;
137- }
138- } ,
139- other => {
140- return plan_err ! (
141- "The regexp_replace function can only accept strings. Got {other}"
142- ) ;
115+ transform_leaf_type_preserving_encoding ( & arg_types[ 0 ] , & regexp_replace_leaf_type)
116+ }
117+
118+ fn return_field_from_args ( & self , args : ReturnFieldArgs ) -> Result < FieldRef > {
119+ let input = & args. arg_fields [ 0 ] ;
120+ let others_scalar = args. scalar_arguments . iter ( ) . skip ( 1 ) . all ( Option :: is_some) ;
121+ let output_type = match input. data_type ( ) {
122+ DataType :: Dictionary ( _, value_type) if !others_scalar => {
123+ regexp_replace_leaf_type ( value_type. as_ref ( ) ) ?
143124 }
144- } )
125+ other => self . return_type ( std:: slice:: from_ref ( other) ) ?,
126+ } ;
127+ Ok ( Arc :: new ( Field :: new ( self . name ( ) , output_type, true ) ) )
128+ }
129+
130+ fn coerce_types ( & self , arg_types : & [ DataType ] ) -> Result < Vec < DataType > > {
131+ if !( 3 ..=4 ) . contains ( & arg_types. len ( ) ) {
132+ return plan_err ! (
133+ "regexp_replace expects 3 or 4 arguments, got {}" ,
134+ arg_types. len( )
135+ ) ;
136+ }
137+ let leaf = match & arg_types[ 0 ] {
138+ DataType :: Dictionary ( _, v) => v. as_ref ( ) ,
139+ other => other,
140+ } ;
141+ let is_string_leaf = matches ! (
142+ leaf,
143+ DataType :: Utf8
144+ | DataType :: LargeUtf8
145+ | DataType :: Utf8View
146+ | DataType :: Binary
147+ | DataType :: LargeBinary
148+ | DataType :: BinaryView
149+ ) ;
150+ let target = if is_string_leaf {
151+ regexp_replace_leaf_type ( leaf) ?
152+ } else {
153+ DataType :: Utf8View
154+ } ;
155+ let first = match & arg_types[ 0 ] {
156+ DataType :: Dictionary ( k, _) if is_string_leaf => {
157+ DataType :: Dictionary ( k. clone ( ) , Box :: new ( target. clone ( ) ) )
158+ }
159+ _ => target. clone ( ) ,
160+ } ;
161+ let mut out = Vec :: with_capacity ( arg_types. len ( ) ) ;
162+ out. push ( first) ;
163+ for _ in 1 ..arg_types. len ( ) {
164+ out. push ( target. clone ( ) ) ;
165+ }
166+ Ok ( out)
145167 }
146168
147169 fn invoke_with_args ( & self , args : ScalarFunctionArgs ) -> Result < ColumnarValue > {
148- let args = & args. args ;
170+ let mut args = args. args ;
171+
172+ let scalar_dict_key = if let Some ( ColumnarValue :: Scalar (
173+ ScalarValue :: Dictionary ( key_type, value) ,
174+ ) ) = args. first ( )
175+ {
176+ let key = key_type. clone ( ) ;
177+ args[ 0 ] = ColumnarValue :: Scalar ( value. as_ref ( ) . clone ( ) ) ;
178+ Some ( key)
179+ } else {
180+ None
181+ } ;
182+
183+ if let ColumnarValue :: Array ( array) = & args[ 0 ]
184+ && let DataType :: Dictionary ( _, value_type) = array. data_type ( ) . clone ( )
185+ {
186+ let others_scalar = args[ 1 ..]
187+ . iter ( )
188+ . all ( |a| matches ! ( a, ColumnarValue :: Scalar ( _) ) ) ;
189+ if others_scalar {
190+ let dict = array. as_any_dictionary ( ) ;
191+ let mut inner: Vec < ColumnarValue > = Vec :: with_capacity ( args. len ( ) ) ;
192+ inner. push ( ColumnarValue :: Array ( Arc :: clone ( dict. values ( ) ) ) ) ;
193+ inner. extend_from_slice ( & args[ 1 ..] ) ;
194+ let values_out = regexp_replace_func ( & inner) ?;
195+ return Ok ( ColumnarValue :: Array ( dict. with_values ( values_out) ) ) ;
196+ }
197+ args[ 0 ] = ColumnarValue :: Array ( arrow:: compute:: cast ( array, & value_type) ?) ;
198+ }
149199
150200 let len = args
151201 . iter ( )
@@ -155,21 +205,42 @@ impl ScalarUDFImpl for RegexpReplaceFunc {
155205 } ) ;
156206
157207 let is_scalar = len. is_none ( ) ;
158- let result = regexp_replace_func ( args) ;
159- if is_scalar {
160- // If all inputs are scalar, keeps output as scalar
161- let result = result. and_then ( |arr| ScalarValue :: try_from_array ( & arr, 0 ) ) ;
162- result. map ( ColumnarValue :: Scalar )
208+ let result = regexp_replace_func ( & args) ?;
209+ let out = if is_scalar {
210+ ColumnarValue :: Scalar ( ScalarValue :: try_from_array ( & result, 0 ) ?)
163211 } else {
164- result. map ( ColumnarValue :: Array )
212+ ColumnarValue :: Array ( result)
213+ } ;
214+
215+ if let ( Some ( key_type) , ColumnarValue :: Scalar ( scalar) ) = ( scalar_dict_key, & out) {
216+ return Ok ( ColumnarValue :: Scalar ( ScalarValue :: Dictionary (
217+ key_type,
218+ Box :: new ( scalar. clone ( ) ) ,
219+ ) ) ) ;
165220 }
221+ Ok ( out)
166222 }
167223
168224 fn documentation ( & self ) -> Option < & Documentation > {
169225 self . doc ( )
170226 }
171227}
172228
229+ fn regexp_replace_leaf_type ( dt : & DataType ) -> Result < DataType > {
230+ use DataType :: * ;
231+ Ok ( match dt {
232+ LargeUtf8 | LargeBinary => LargeUtf8 ,
233+ Utf8 | Binary => Utf8 ,
234+ Utf8View | BinaryView => Utf8View ,
235+ Null => Null ,
236+ other => {
237+ return plan_err ! (
238+ "the regexp_replace can only accept strings but got {other:?}"
239+ ) ;
240+ }
241+ } )
242+ }
243+
173244fn regexp_replace_func ( args : & [ ColumnarValue ] ) -> Result < ArrayRef > {
174245 match args[ 0 ] . data_type ( ) {
175246 DataType :: Utf8 => specialize_regexp_replace :: < i32 > ( args) ,
0 commit comments