File tree Expand file tree Collapse file tree
native/spark-expr/src/array_funcs Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -51,12 +51,6 @@ impl Utf8Collation {
5151 if matches ! ( self , Self :: Binary | Self :: BinaryRtrim ) {
5252 return left. cmp ( right) ;
5353 }
54- if left. is_ascii ( ) && right. is_ascii ( ) {
55- return left
56- . bytes ( )
57- . map ( |b| b. to_ascii_lowercase ( ) )
58- . cmp ( right. bytes ( ) . map ( |b| b. to_ascii_lowercase ( ) ) ) ;
59- }
6054 fn lower ( value : & str , unicode_version : u32 ) -> impl Iterator < Item = u32 > + ' _ {
6155 value. chars ( ) . flat_map ( move |c| {
6256 let cp = c as u32 ;
@@ -73,7 +67,18 @@ impl Utf8Collation {
7367 . chain ( ( second != 0 ) . then_some ( second) )
7468 } )
7569 }
76- lower ( left, unicode_version) . cmp ( lower ( right, unicode_version) )
70+ // Compare ASCII prefixes lazily so a retained long winner is not rescanned.
71+ for ( offset, ( l, r) ) in left. bytes ( ) . zip ( right. bytes ( ) ) . enumerate ( ) {
72+ if !l. is_ascii ( ) || !r. is_ascii ( ) {
73+ return lower ( & left[ offset..] , unicode_version)
74+ . cmp ( lower ( & right[ offset..] , unicode_version) ) ;
75+ }
76+ let ordering = l. to_ascii_lowercase ( ) . cmp ( & r. to_ascii_lowercase ( ) ) ;
77+ if ordering != Ordering :: Equal {
78+ return ordering;
79+ }
80+ }
81+ left. len ( ) . cmp ( & right. len ( ) )
7782 }
7883}
7984
Original file line number Diff line number Diff line change @@ -453,6 +453,10 @@ fn utf8_lcase_uses_spark_unicode_version_and_space_trimming() {
453453 ( "İ" , "i\u{307} " , Equal ) ,
454454 ( "ς" , "σ" , Equal ) ,
455455 ( "K" , "k" , Equal ) ,
456+ ( "abİx" , "ABi\u{307} y" , Less ) ,
457+ ( "abK" , "ABk" , Equal ) ,
458+ ( "ab" , "ABΣ" , Less ) ,
459+ ( "aΣ" , "B" , Less ) ,
456460 ( "\u{10400} " , "\u{10428} " , Equal ) ,
457461 ( "é" , "e" , Greater ) ,
458462 ( "A " , "a" , Greater ) ,
@@ -463,6 +467,10 @@ fn utf8_lcase_uses_spark_unicode_version_and_space_trimming() {
463467 ) ,
464468 ] {
465469 assert_eq ! ( Utf8Collation :: Lcase . compare( left, right, version) , expected) ;
470+ assert_eq ! (
471+ Utf8Collation :: Lcase . compare( right, left, version) ,
472+ expected. reverse( )
473+ ) ;
466474 }
467475 assert_eq ! ( Utf8Collation :: LcaseRtrim . compare( "A " , "a" , version) , Equal ) ;
468476 assert_eq ! (
You can’t perform that action at this time.
0 commit comments