Skip to content

Commit d44b19a

Browse files
committed
fix: compare array extrema ASCII prefixes lazily
1 parent 895838b commit d44b19a

2 files changed

Lines changed: 20 additions & 7 deletions

File tree

‎native/spark-expr/src/array_funcs/array_extrema.rs‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -51,12 +51,6 @@ impl Utf8Collation {
5151
if matches!(self, Self::Binary | Self::BinaryRtrim) {
5252
return left.cmp(right);
5353
}
54-
if left.is_ascii() && right.is_ascii() {
55-
return left
56-
.bytes()
57-
.map(|b| b.to_ascii_lowercase())
58-
.cmp(right.bytes().map(|b| b.to_ascii_lowercase()));
59-
}
6054
fn lower(value: &str, unicode_version: u32) -> impl Iterator<Item = u32> + '_ {
6155
value.chars().flat_map(move |c| {
6256
let cp = c as u32;
@@ -73,7 +67,18 @@ impl Utf8Collation {
7367
.chain((second != 0).then_some(second))
7468
})
7569
}
76-
lower(left, unicode_version).cmp(lower(right, unicode_version))
70+
// Compare ASCII prefixes lazily so a retained long winner is not rescanned.
71+
for (offset, (l, r)) in left.bytes().zip(right.bytes()).enumerate() {
72+
if !l.is_ascii() || !r.is_ascii() {
73+
return lower(&left[offset..], unicode_version)
74+
.cmp(lower(&right[offset..], unicode_version));
75+
}
76+
let ordering = l.to_ascii_lowercase().cmp(&r.to_ascii_lowercase());
77+
if ordering != Ordering::Equal {
78+
return ordering;
79+
}
80+
}
81+
left.len().cmp(&right.len())
7782
}
7883
}
7984

‎native/spark-expr/src/array_funcs/array_extrema/tests.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -453,6 +453,10 @@ fn utf8_lcase_uses_spark_unicode_version_and_space_trimming() {
453453
("İ", "i\u{307}", Equal),
454454
("ς", "σ", Equal),
455455
("K", "k", Equal),
456+
("abİx", "ABi\u{307}y", Less),
457+
("abK", "ABk", Equal),
458+
("ab", "ABΣ", Less),
459+
("aΣ", "B", Less),
456460
("\u{10400}", "\u{10428}", Equal),
457461
("é", "e", Greater),
458462
("A ", "a", Greater),
@@ -463,6 +467,10 @@ fn utf8_lcase_uses_spark_unicode_version_and_space_trimming() {
463467
),
464468
] {
465469
assert_eq!(Utf8Collation::Lcase.compare(left, right, version), expected);
470+
assert_eq!(
471+
Utf8Collation::Lcase.compare(right, left, version),
472+
expected.reverse()
473+
);
466474
}
467475
assert_eq!(Utf8Collation::LcaseRtrim.compare("A ", "a", version), Equal);
468476
assert_eq!(

0 commit comments

Comments
 (0)