Skip to content

Commit b5bb8f2

Browse files
committed
chore: address review feedback on upper and lower
- functions.slt: check `Utf8`, `LargeUtf8` and `Utf8View` in one query per function, and add a `lower` row with a sigma both inside and at the end of a word. - `append_case_converted`: document that the extra `is_ascii` read is cheap next to what ASCII values save, and why lowercasing needs the whole value. - Build the case-conversion benchmark inputs without cloning the values.
1 parent 5301c81 commit b5bb8f2

3 files changed

Lines changed: 60 additions & 91 deletions

File tree

‎datafusion/functions/benches/string_expressions/case_conversion_inputs.rs‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -137,15 +137,25 @@ pub fn inputs() -> Vec<(String, ArrayRef)> {
137137
let words = build_values(word_value);
138138
inputs.push((
139139
"non_ascii_words_utf8".to_string(),
140-
Arc::new(StringArray::from(words.clone())),
140+
Arc::new(words.iter().map(Option::as_deref).collect::<StringArray>()),
141141
));
142142
inputs.push((
143143
"non_ascii_words_large_utf8".to_string(),
144-
Arc::new(LargeStringArray::from(words.clone())),
144+
Arc::new(
145+
words
146+
.iter()
147+
.map(Option::as_deref)
148+
.collect::<LargeStringArray>(),
149+
),
145150
));
146151
inputs.push((
147152
"non_ascii_words_utf8view".to_string(),
148-
Arc::new(StringViewArray::from(words)),
153+
Arc::new(
154+
words
155+
.iter()
156+
.map(Option::as_deref)
157+
.collect::<StringViewArray>(),
158+
),
149159
));
150160

151161
for str_len in [8, 32, 128] {
@@ -163,11 +173,16 @@ pub fn inputs() -> Vec<(String, ArrayRef)> {
163173
] {
164174
inputs.push((
165175
format!("{shape}_utf8_len{str_len}"),
166-
Arc::new(StringArray::from(values.clone())),
176+
Arc::new(values.iter().map(Option::as_deref).collect::<StringArray>()),
167177
));
168178
inputs.push((
169179
format!("{shape}_utf8view_len{str_len}"),
170-
Arc::new(StringViewArray::from(values)),
180+
Arc::new(
181+
values
182+
.iter()
183+
.map(Option::as_deref)
184+
.collect::<StringViewArray>(),
185+
),
171186
));
172187
}
173188
}

‎datafusion/functions/src/string/common.rs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -394,7 +394,15 @@ fn unicode_case(s: &str, lower: bool) -> String {
394394
///
395395
/// An ASCII value is converted byte by byte straight into the builder. Other
396396
/// values go through `str::to_lowercase` / `str::to_uppercase`, which allocate
397-
/// a `String` but convert any leading run of ASCII bytes in bulk.
397+
/// a `String` but convert any leading run of ASCII bytes in bulk. Checking
398+
/// `is_ascii` first means a non-ASCII value is read up to its first non-ASCII
399+
/// byte before it is converted; that extra read is cheap compared with what an
400+
/// ASCII value saves by being written straight into the builder instead of
401+
/// through a temporary `String`.
402+
///
403+
/// Lowercasing needs the whole value: `Σ` becomes `ς` at the end of a word and
404+
/// `σ` elsewhere, so converting one character at a time, or converting the
405+
/// ASCII prefix separately, would give the wrong result.
398406
#[inline]
399407
fn append_case_converted<O: OffsetSizeTrait>(
400408
builder: &mut GenericStringArrayBuilder<O>,

‎datafusion/sqllogictest/test_files/functions.slt‎

Lines changed: 31 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -507,47 +507,19 @@ FROM (VALUES
507507
(9, 'ⱦx'),
508508
(10, '农历新年 new year'));
509509

510-
query T
511-
SELECT upper(utf8) FROM upper_mixed_test ORDER BY id
512-
----
513-
DATAFUSION
514-
A LONG ASCII-ONLY VALUE
515-
(empty)
516-
NULL
517-
ÉA
518-
ÜBER ALLES, THEN ASCII
519-
STRASSE
520-
A LONG VALUE ENDING IN SS
521-
ȾX
522-
农历新年 NEW YEAR
523-
524-
query T
525-
SELECT upper(large_utf8) FROM upper_mixed_test ORDER BY id
526-
----
527-
DATAFUSION
528-
A LONG ASCII-ONLY VALUE
529-
(empty)
530-
NULL
531-
ÉA
532-
ÜBER ALLES, THEN ASCII
533-
STRASSE
534-
A LONG VALUE ENDING IN SS
535-
ȾX
536-
农历新年 NEW YEAR
537-
538-
query T
539-
SELECT upper(utf8view) FROM upper_mixed_test ORDER BY id
540-
----
541-
DATAFUSION
542-
A LONG ASCII-ONLY VALUE
543-
(empty)
544-
NULL
545-
ÉA
546-
ÜBER ALLES, THEN ASCII
547-
STRASSE
548-
A LONG VALUE ENDING IN SS
549-
ȾX
550-
农历新年 NEW YEAR
510+
query TTT
511+
SELECT upper(utf8), upper(large_utf8), upper(utf8view) FROM upper_mixed_test ORDER BY id
512+
----
513+
DATAFUSION DATAFUSION DATAFUSION
514+
A LONG ASCII-ONLY VALUE A LONG ASCII-ONLY VALUE A LONG ASCII-ONLY VALUE
515+
(empty) (empty) (empty)
516+
NULL NULL NULL
517+
ÉA ÉA ÉA
518+
ÜBER ALLES, THEN ASCII ÜBER ALLES, THEN ASCII ÜBER ALLES, THEN ASCII
519+
STRASSE STRASSE STRASSE
520+
A LONG VALUE ENDING IN SS A LONG VALUE ENDING IN SS A LONG VALUE ENDING IN SS
521+
ȾX ȾX ȾX
522+
农历新年 NEW YEAR 农历新年 NEW YEAR 农历新年 NEW YEAR
551523

552524
statement ok
553525
DROP TABLE upper_mixed_test
@@ -776,8 +748,8 @@ DROP TABLE lower_dictionary_test
776748

777749
# The array is not all ASCII, so each row is converted on its own: ASCII rows,
778750
# and rows with non-ASCII characters at the start and end, including values
779-
# longer than the 12 bytes a Utf8View can store inline. A sigma at the end of a
780-
# word lowercases to ς.
751+
# longer than the 12 bytes a Utf8View can store inline. A sigma lowercases to ς
752+
# at the end of a word and to σ elsewhere.
781753
statement ok
782754
CREATE TABLE lower_mixed_test AS
783755
SELECT column1 AS id,
@@ -794,49 +766,23 @@ FROM (VALUES
794766
(7, 'A LONG VALUE ENDING IN ΟΔΟΣ'),
795767
(8, 'ABΣ'),
796768
(9, 'ȾX'),
797-
(10, '农历新年 NEW YEAR'));
769+
(10, '农历新年 NEW YEAR'),
770+
(11, 'ὈΔΥΣΣΕΎΣ'));
798771

799-
query T
800-
SELECT lower(utf8) FROM lower_mixed_test ORDER BY id
801-
----
802-
datafusion
803-
a long ascii-only value
804-
(empty)
805-
NULL
806-
éa
807-
über alles, then ascii
808-
a long value ending in οδος
809-
abς
810-
ⱦx
811-
农历新年 new year
812-
813-
query T
814-
SELECT lower(large_utf8) FROM lower_mixed_test ORDER BY id
815-
----
816-
datafusion
817-
a long ascii-only value
818-
(empty)
819-
NULL
820-
éa
821-
über alles, then ascii
822-
a long value ending in οδος
823-
abς
824-
ⱦx
825-
农历新年 new year
826-
827-
query T
828-
SELECT lower(utf8view) FROM lower_mixed_test ORDER BY id
829-
----
830-
datafusion
831-
a long ascii-only value
832-
(empty)
833-
NULL
834-
éa
835-
über alles, then ascii
836-
a long value ending in οδος
837-
abς
838-
ⱦx
839-
农历新年 new year
772+
query TTT
773+
SELECT lower(utf8), lower(large_utf8), lower(utf8view) FROM lower_mixed_test ORDER BY id
774+
----
775+
datafusion datafusion datafusion
776+
a long ascii-only value a long ascii-only value a long ascii-only value
777+
(empty) (empty) (empty)
778+
NULL NULL NULL
779+
éa éa éa
780+
über alles, then ascii über alles, then ascii über alles, then ascii
781+
a long value ending in οδος a long value ending in οδος a long value ending in οδος
782+
abς abς abς
783+
ⱦx ⱦx ⱦx
784+
农历新年 new year 农历新年 new year 农历新年 new year
785+
ὀδυσσεύς ὀδυσσεύς ὀδυσσεύς
840786

841787
statement ok
842788
DROP TABLE lower_mixed_test

0 commit comments

Comments
 (0)