1515// specific language governing permissions and limitations
1616// under the License.
1717
18+ use std:: collections:: HashMap ;
1819use std:: sync:: Arc ;
1920
2021use crate :: strings:: sub_view;
2122use crate :: utils:: make_scalar_function;
2223use arrow:: array:: {
23- Array , ArrayRef , AsArray , GenericStringArray , Int64Array , OffsetSizeTrait ,
24- StringArrayType , StringViewArray ,
24+ Array , ArrayRef , AsArray , DictionaryArray , GenericStringArray , Int64Array ,
25+ OffsetSizeTrait , PrimitiveBuilder , StringArrayType , StringViewArray , UInt64Array ,
2526} ;
2627use arrow:: buffer:: { NullBuffer , ScalarBuffer } ;
27- use arrow:: compute:: kernels:: cast:: cast;
28- use arrow:: datatypes:: DataType ;
28+ use arrow:: compute:: take;
29+ use arrow:: datatypes:: { ArrowDictionaryKeyType , ArrowNativeType , DataType } ;
30+ use arrow:: downcast_dictionary_array;
2931use datafusion_common:: cast:: as_int64_array;
3032use datafusion_common:: types:: {
3133 NativeType , logical_int32, logical_int64, logical_string,
@@ -162,10 +164,12 @@ fn substr(args: &[ArrayRef]) -> Result<ArrayRef> {
162164 let string_array = args[ 0 ] . as_string_view ( ) ;
163165 string_view_substr ( string_array, & args[ 1 ..] )
164166 }
165- DataType :: Dictionary ( _, value_type) => {
166- let mut flat_args = args. to_vec ( ) ;
167- flat_args[ 0 ] = cast ( & args[ 0 ] , value_type) ?;
168- Ok ( cast ( & substr ( & flat_args) ?, args[ 0 ] . data_type ( ) ) ?)
167+ DataType :: Dictionary ( _, _) => {
168+ let array = & args[ 0 ] ;
169+ downcast_dictionary_array ! {
170+ array => dictionary_substr( array, & args[ 1 ..] ) ,
171+ _ => unreachable!( "substr matched a dictionary array" ) ,
172+ }
169173 }
170174 other => exec_err ! (
171175 "Unsupported data type {other:?} for function substr,\
@@ -174,6 +178,69 @@ fn substr(args: &[ArrayRef]) -> Result<ArrayRef> {
174178 }
175179}
176180
181+ fn dictionary_substr < K : ArrowDictionaryKeyType > (
182+ dictionary : & DictionaryArray < K > ,
183+ args : & [ ArrayRef ] ,
184+ ) -> Result < ArrayRef > {
185+ let start_array = as_int64_array ( & args[ 0 ] ) ?;
186+ let count_array = args. get ( 1 ) . map ( |a| as_int64_array ( a) ) . transpose ( ) ?;
187+
188+ let mut combinations = HashMap :: new ( ) ;
189+ let mut value_indices = Vec :: new ( ) ;
190+ let mut starts = Vec :: new ( ) ;
191+ let mut counts = count_array. map ( |_| Vec :: new ( ) ) ;
192+ let mut keys = PrimitiveBuilder :: < K > :: with_capacity ( dictionary. len ( ) ) ;
193+
194+ for i in 0 ..dictionary. len ( ) {
195+ let Some ( value_index) = dictionary. key ( i) else {
196+ keys. append_null ( ) ;
197+ continue ;
198+ } ;
199+ if dictionary. values ( ) . is_null ( value_index)
200+ || start_array. is_null ( i)
201+ || count_array. is_some_and ( |a| a. is_null ( i) )
202+ {
203+ keys. append_null ( ) ;
204+ continue ;
205+ }
206+
207+ let start = start_array. value ( i) ;
208+ let count = count_array. map ( |a| a. value ( i) ) ;
209+ let combination = ( value_index, start, count) ;
210+ let next_key = combinations. len ( ) ;
211+ let key = * combinations. entry ( combination) . or_insert_with ( || {
212+ value_indices. push ( value_index as u64 ) ;
213+ starts. push ( start) ;
214+ if let ( Some ( counts) , Some ( count) ) = ( & mut counts, count) {
215+ counts. push ( count) ;
216+ }
217+ next_key
218+ } ) ;
219+ let key = K :: Native :: from_usize ( key) . ok_or_else ( || {
220+ datafusion_common:: DataFusionError :: Execution ( format ! (
221+ "substr result has too many dictionary values for key type {}" ,
222+ K :: DATA_TYPE
223+ ) )
224+ } ) ?;
225+ keys. append_value ( key) ;
226+ }
227+
228+ let values = take (
229+ dictionary. values ( ) . as_ref ( ) ,
230+ & UInt64Array :: from ( value_indices) ,
231+ None ,
232+ ) ?;
233+ let mut distinct_args = vec ! [ values, Arc :: new( Int64Array :: from( starts) ) as ArrayRef ] ;
234+ if let Some ( counts) = counts {
235+ distinct_args. push ( Arc :: new ( Int64Array :: from ( counts) ) ) ;
236+ }
237+ let values = substr ( & distinct_args) ?;
238+ Ok ( Arc :: new ( DictionaryArray :: < K > :: try_new (
239+ keys. finish ( ) ,
240+ values,
241+ ) ?) )
242+ }
243+
177244/// Convert the given `start` and `count` to valid byte indices within `input` string.
178245///
179246/// Input `start` and `count` are equivalent to PostgreSQL's `substr(s, start, count)`.
0 commit comments