@@ -27,9 +27,9 @@ use arrow::{
2727} ;
2828use datafusion_common:: { DataFusionError , Result } ;
2929use hashbrown:: HashMap ;
30- use std:: any:: type_name;
3130use std:: cmp:: Ordering ;
3231use std:: sync:: Arc ;
32+ use std:: { any:: type_name, cmp:: max} ;
3333use unicode_segmentation:: UnicodeSegmentation ;
3434
3535macro_rules! downcast_string_arg {
@@ -60,6 +60,7 @@ macro_rules! downcast_arg {
6060
6161/// Returns number of characters in the string.
6262/// character_length('josé') = 4
63+ /// The implementation counts UTF-8 code points to count the number of characters
6364pub fn character_length < T : ArrowPrimitiveType > ( args : & [ ArrayRef ] ) -> Result < ArrayRef >
6465where
6566 T :: Native : OffsetSizeTrait ,
7576 . iter ( )
7677 . map ( |string| {
7778 string. map ( |string : & str | {
78- T :: Native :: from_usize ( string. graphemes ( true ) . count ( ) ) . expect (
79- "should not fail as graphemes.count will always return integer" ,
80- )
79+ T :: Native :: from_usize ( string. chars ( ) . count ( ) )
80+ . expect ( "should not fail as string.chars will always return integer" )
8181 } )
8282 } )
8383 . collect :: < PrimitiveArray < T > > ( ) ;
8787
8888/// Returns first n characters in the string, or when n is negative, returns all but last |n| characters.
8989/// left('abcde', 2) = 'ab'
90+ /// The implementation uses UTF-8 code points as characters
9091pub fn left < T : OffsetSizeTrait > ( args : & [ ArrayRef ] ) -> Result < ArrayRef > {
9192 let string_array = downcast_string_arg ! ( args[ 0 ] , "string" , T ) ;
9293 let n_array = downcast_arg ! ( args[ 1 ] , "n" , Int64Array ) ;
@@ -96,19 +97,16 @@ pub fn left<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
9697 . map ( |( string, n) | match ( string, n) {
9798 ( Some ( string) , Some ( n) ) => match n. cmp ( & 0 ) {
9899 Ordering :: Less => {
99- let graphemes = string. graphemes ( true ) ;
100- let len = graphemes. clone ( ) . count ( ) as i64 ;
101- match n. abs ( ) . cmp ( & len) {
102- Ordering :: Less => {
103- Some ( graphemes. take ( ( len + n) as usize ) . collect :: < String > ( ) )
104- }
105- Ordering :: Equal => Some ( "" . to_string ( ) ) ,
106- Ordering :: Greater => Some ( "" . to_string ( ) ) ,
107- }
100+ let len = string. chars ( ) . count ( ) as i64 ;
101+ Some ( if n. abs ( ) < len {
102+ string. chars ( ) . take ( ( len + n) as usize ) . collect :: < String > ( )
103+ } else {
104+ "" . to_string ( )
105+ } )
108106 }
109107 Ordering :: Equal => Some ( "" . to_string ( ) ) ,
110108 Ordering :: Greater => {
111- Some ( string. graphemes ( true ) . take ( n as usize ) . collect :: < String > ( ) )
109+ Some ( string. chars ( ) . take ( n as usize ) . collect :: < String > ( ) )
112110 }
113111 } ,
114112 _ => None ,
@@ -139,11 +137,8 @@ pub fn lpad<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
139137 if length < graphemes. len ( ) {
140138 Some ( graphemes[ ..length] . concat ( ) )
141139 } else {
142- let mut s = string. to_string ( ) ;
143- s. insert_str (
144- 0 ,
145- " " . repeat ( length - graphemes. len ( ) ) . as_str ( ) ,
146- ) ;
140+ let mut s: String = " " . repeat ( length - graphemes. len ( ) ) ;
141+ s. push_str ( string) ;
147142 Some ( s)
148143 }
149144 }
@@ -209,21 +204,21 @@ pub fn lpad<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
209204
210205/// Reverses the order of the characters in the string.
211206/// reverse('abcde') = 'edcba'
207+ /// The implementation uses UTF-8 code points as characters
212208pub fn reverse < T : OffsetSizeTrait > ( args : & [ ArrayRef ] ) -> Result < ArrayRef > {
213209 let string_array = downcast_string_arg ! ( args[ 0 ] , "string" , T ) ;
214210
215211 let result = string_array
216212 . iter ( )
217- . map ( |string| {
218- string. map ( |string : & str | string. graphemes ( true ) . rev ( ) . collect :: < String > ( ) )
219- } )
213+ . map ( |string| string. map ( |string : & str | string. chars ( ) . rev ( ) . collect :: < String > ( ) ) )
220214 . collect :: < GenericStringArray < T > > ( ) ;
221215
222216 Ok ( Arc :: new ( result) as ArrayRef )
223217}
224218
225219/// Returns last n characters in the string, or when n is negative, returns all but first |n| characters.
226220/// right('abcde', 2) = 'de'
221+ /// The implementation uses UTF-8 code points as characters
227222pub fn right < T : OffsetSizeTrait > ( args : & [ ArrayRef ] ) -> Result < ArrayRef > {
228223 let string_array = downcast_string_arg ! ( args[ 0 ] , "string" , T ) ;
229224 let n_array = downcast_arg ! ( args[ 1 ] , "n" , Int64Array ) ;
@@ -233,33 +228,17 @@ pub fn right<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
233228 . zip ( n_array. iter ( ) )
234229 . map ( |( string, n) | match ( string, n) {
235230 ( Some ( string) , Some ( n) ) => match n. cmp ( & 0 ) {
236- Ordering :: Less => {
237- let graphemes = string. graphemes ( true ) . rev ( ) ;
238- let len = graphemes. clone ( ) . count ( ) as i64 ;
239- match n. abs ( ) . cmp ( & len) {
240- Ordering :: Less => Some (
241- graphemes
242- . take ( ( len + n) as usize )
243- . collect :: < Vec < & str > > ( )
244- . iter ( )
245- . rev ( )
246- . copied ( )
247- . collect :: < String > ( ) ,
248- ) ,
249- Ordering :: Equal => Some ( "" . to_string ( ) ) ,
250- Ordering :: Greater => Some ( "" . to_string ( ) ) ,
251- }
252- }
231+ Ordering :: Less => Some (
232+ string
233+ . chars ( )
234+ . skip ( n. unsigned_abs ( ) as usize )
235+ . collect :: < String > ( ) ,
236+ ) ,
253237 Ordering :: Equal => Some ( "" . to_string ( ) ) ,
254238 Ordering :: Greater => Some (
255239 string
256- . graphemes ( true )
257- . rev ( )
258- . take ( n as usize )
259- . collect :: < Vec < & str > > ( )
260- . iter ( )
261- . rev ( )
262- . copied ( )
240+ . chars ( )
241+ . skip ( max ( string. chars ( ) . count ( ) as i64 - n, 0 ) as usize )
263242 . collect :: < String > ( ) ,
264243 ) ,
265244 } ,
@@ -349,6 +328,7 @@ pub fn rpad<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
349328
350329/// Returns starting index of specified substring within string, or zero if it's not present. (Same as position(substring in string), but note the reversed argument order.)
351330/// strpos('high', 'ig') = 2
331+ /// The implementation uses UTF-8 code points as characters
352332pub fn strpos < T : ArrowPrimitiveType > ( args : & [ ArrayRef ] ) -> Result < ArrayRef >
353333where
354334 T :: Native : OffsetSizeTrait ,
@@ -374,28 +354,13 @@ where
374354 . zip ( substring_array. iter ( ) )
375355 . map ( |( string, substring) | match ( string, substring) {
376356 ( Some ( string) , Some ( substring) ) => {
377- // the rfind method returns the byte index of the substring which may or may not be the same as the character index due to UTF8 encoding
378- // this method first finds the matching byte using rfind
379- // then maps that to the character index by matching on the grapheme_index of the byte_index
380- Some (
381- T :: Native :: from_usize ( string. to_string ( ) . rfind ( substring) . map_or (
382- 0 ,
383- |byte_offset| {
384- string
385- . grapheme_indices ( true )
386- . collect :: < Vec < ( usize , & str ) > > ( )
387- . iter ( )
388- . enumerate ( )
389- . filter ( |( _, ( offset, _) ) | * offset == byte_offset)
390- . map ( |( index, _) | index)
391- . collect :: < Vec < usize > > ( )
392- . first ( )
393- . expect ( "should not fail as grapheme_indices and byte offsets are tightly coupled" )
394- . to_owned ( )
395- + 1
396- } ,
397- ) )
398- . expect ( "should not fail due to map_or default value" )
357+ // the find method returns the byte index of the substring
358+ // Next, we count the number of the chars until that byte
359+ T :: Native :: from_usize (
360+ string
361+ . find ( substring)
362+ . map ( |x| string[ ..x] . chars ( ) . count ( ) + 1 )
363+ . unwrap_or ( 0 ) ,
399364 )
400365 }
401366 _ => None ,
@@ -408,6 +373,7 @@ where
408373/// Extracts the substring of string starting at the start'th character, and extending for count characters if that is specified. (Same as substring(string from start for count).)
409374/// substr('alphabet', 3) = 'phabet'
410375/// substr('alphabet', 3, 2) = 'ph'
376+ /// The implementation uses UTF-8 code points as characters
411377pub fn substr < T : OffsetSizeTrait > ( args : & [ ArrayRef ] ) -> Result < ArrayRef > {
412378 match args. len ( ) {
413379 2 => {
@@ -422,13 +388,7 @@ pub fn substr<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
422388 if start <= 0 {
423389 Some ( string. to_string ( ) )
424390 } else {
425- let graphemes = string. graphemes ( true ) . collect :: < Vec < & str > > ( ) ;
426- let start_pos = start as usize - 1 ;
427- if graphemes. len ( ) < start_pos {
428- Some ( "" . to_string ( ) )
429- } else {
430- Some ( graphemes[ start_pos..] . concat ( ) )
431- }
391+ Some ( string. chars ( ) . skip ( start as usize - 1 ) . collect ( ) )
432392 }
433393 }
434394 _ => None ,
@@ -455,29 +415,9 @@ pub fn substr<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
455415 count
456416 ) ) )
457417 } else {
458- let graphemes = string. graphemes ( true ) . collect :: < Vec < & str > > ( ) ;
459- let ( start_pos, end_pos) = if start <= 0 {
460- let end_pos = start + count - 1 ;
461- (
462- 0_usize ,
463- if end_pos < 0 {
464- // we use 0 as workaround for usize to return empty string
465- 0
466- } else {
467- end_pos as usize
468- } ,
469- )
470- } else {
471- ( ( start - 1 ) as usize , ( start + count - 1 ) as usize )
472- } ;
473-
474- if end_pos == 0 || graphemes. len ( ) < start_pos {
475- Ok ( Some ( "" . to_string ( ) ) )
476- } else if graphemes. len ( ) < end_pos {
477- Ok ( Some ( graphemes[ start_pos..] . concat ( ) ) )
478- } else {
479- Ok ( Some ( graphemes[ start_pos..end_pos] . concat ( ) ) )
480- }
418+ let skip = max ( 0 , start - 1 ) ;
419+ let count = max ( 0 , count + ( if start < 1 { start - 1 } else { 0 } ) ) ;
420+ Ok ( Some ( string. chars ( ) . skip ( skip as usize ) . take ( count as usize ) . collect :: < String > ( ) ) )
481421 }
482422 }
483423 _ => Ok ( None ) ,
0 commit comments