Skip to content

Commit 88f6548

Browse files
authored
Use code points instead of grapheme clusters for string functions (#3054)
1 parent a4fa44f commit 88f6548

5 files changed

Lines changed: 59 additions & 102 deletions

File tree

‎datafusion/core/tests/sql/unicode.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ async fn test_unicode_expressions() -> Result<()> {
6161
test_expression!("lpad(NULL, 0)", "NULL");
6262
test_expression!("lpad(NULL, 5, 'xy')", "NULL");
6363
test_expression!("reverse('abcde')", "edcba");
64-
test_expression!("reverse('loẅks')", "skẅol");
64+
test_expression!("reverse('loẅks')", "sk̈wol"); // Compatible with PostgreSQL
6565
test_expression!("reverse(NULL)", "NULL");
6666
test_expression!("right('abcde', -2)", "cde");
6767
test_expression!("right('abcde', -200)", "");

‎datafusion/physical-expr/src/functions.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1645,7 +1645,7 @@ mod tests {
16451645
test_function!(
16461646
Reverse,
16471647
&[lit("loẅks")],
1648-
Ok(Some("skẅol")),
1648+
Ok(Some("sk̈wol")),
16491649
&str,
16501650
Utf8,
16511651
StringArray
@@ -1654,7 +1654,7 @@ mod tests {
16541654
test_function!(
16551655
Reverse,
16561656
&[lit("loẅks")],
1657-
Ok(Some("skẅol")),
1657+
Ok(Some("sk̈wol")),
16581658
&str,
16591659
Utf8,
16601660
StringArray

‎datafusion/physical-expr/src/unicode_expressions.rs‎

Lines changed: 38 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,9 @@ use arrow::{
2727
};
2828
use datafusion_common::{DataFusionError, Result};
2929
use hashbrown::HashMap;
30-
use std::any::type_name;
3130
use std::cmp::Ordering;
3231
use std::sync::Arc;
32+
use std::{any::type_name, cmp::max};
3333
use unicode_segmentation::UnicodeSegmentation;
3434

3535
macro_rules! downcast_string_arg {
@@ -60,6 +60,7 @@ macro_rules! downcast_arg {
6060

6161
/// Returns number of characters in the string.
6262
/// character_length('josé') = 4
63+
/// The implementation counts UTF-8 code points to count the number of characters
6364
pub fn character_length<T: ArrowPrimitiveType>(args: &[ArrayRef]) -> Result<ArrayRef>
6465
where
6566
T::Native: OffsetSizeTrait,
@@ -75,9 +76,8 @@ where
7576
.iter()
7677
.map(|string| {
7778
string.map(|string: &str| {
78-
T::Native::from_usize(string.graphemes(true).count()).expect(
79-
"should not fail as graphemes.count will always return integer",
80-
)
79+
T::Native::from_usize(string.chars().count())
80+
.expect("should not fail as string.chars will always return integer")
8181
})
8282
})
8383
.collect::<PrimitiveArray<T>>();
@@ -87,6 +87,7 @@ where
8787

8888
/// Returns first n characters in the string, or when n is negative, returns all but last |n| characters.
8989
/// left('abcde', 2) = 'ab'
90+
/// The implementation uses UTF-8 code points as characters
9091
pub fn left<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
9192
let string_array = downcast_string_arg!(args[0], "string", T);
9293
let n_array = downcast_arg!(args[1], "n", Int64Array);
@@ -96,19 +97,16 @@ pub fn left<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
9697
.map(|(string, n)| match (string, n) {
9798
(Some(string), Some(n)) => match n.cmp(&0) {
9899
Ordering::Less => {
99-
let graphemes = string.graphemes(true);
100-
let len = graphemes.clone().count() as i64;
101-
match n.abs().cmp(&len) {
102-
Ordering::Less => {
103-
Some(graphemes.take((len + n) as usize).collect::<String>())
104-
}
105-
Ordering::Equal => Some("".to_string()),
106-
Ordering::Greater => Some("".to_string()),
107-
}
100+
let len = string.chars().count() as i64;
101+
Some(if n.abs() < len {
102+
string.chars().take((len + n) as usize).collect::<String>()
103+
} else {
104+
"".to_string()
105+
})
108106
}
109107
Ordering::Equal => Some("".to_string()),
110108
Ordering::Greater => {
111-
Some(string.graphemes(true).take(n as usize).collect::<String>())
109+
Some(string.chars().take(n as usize).collect::<String>())
112110
}
113111
},
114112
_ => None,
@@ -139,11 +137,8 @@ pub fn lpad<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
139137
if length < graphemes.len() {
140138
Some(graphemes[..length].concat())
141139
} else {
142-
let mut s = string.to_string();
143-
s.insert_str(
144-
0,
145-
" ".repeat(length - graphemes.len()).as_str(),
146-
);
140+
let mut s: String = " ".repeat(length - graphemes.len());
141+
s.push_str(string);
147142
Some(s)
148143
}
149144
}
@@ -209,21 +204,21 @@ pub fn lpad<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
209204

210205
/// Reverses the order of the characters in the string.
211206
/// reverse('abcde') = 'edcba'
207+
/// The implementation uses UTF-8 code points as characters
212208
pub fn reverse<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
213209
let string_array = downcast_string_arg!(args[0], "string", T);
214210

215211
let result = string_array
216212
.iter()
217-
.map(|string| {
218-
string.map(|string: &str| string.graphemes(true).rev().collect::<String>())
219-
})
213+
.map(|string| string.map(|string: &str| string.chars().rev().collect::<String>()))
220214
.collect::<GenericStringArray<T>>();
221215

222216
Ok(Arc::new(result) as ArrayRef)
223217
}
224218

225219
/// Returns last n characters in the string, or when n is negative, returns all but first |n| characters.
226220
/// right('abcde', 2) = 'de'
221+
/// The implementation uses UTF-8 code points as characters
227222
pub fn right<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
228223
let string_array = downcast_string_arg!(args[0], "string", T);
229224
let n_array = downcast_arg!(args[1], "n", Int64Array);
@@ -233,33 +228,17 @@ pub fn right<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
233228
.zip(n_array.iter())
234229
.map(|(string, n)| match (string, n) {
235230
(Some(string), Some(n)) => match n.cmp(&0) {
236-
Ordering::Less => {
237-
let graphemes = string.graphemes(true).rev();
238-
let len = graphemes.clone().count() as i64;
239-
match n.abs().cmp(&len) {
240-
Ordering::Less => Some(
241-
graphemes
242-
.take((len + n) as usize)
243-
.collect::<Vec<&str>>()
244-
.iter()
245-
.rev()
246-
.copied()
247-
.collect::<String>(),
248-
),
249-
Ordering::Equal => Some("".to_string()),
250-
Ordering::Greater => Some("".to_string()),
251-
}
252-
}
231+
Ordering::Less => Some(
232+
string
233+
.chars()
234+
.skip(n.unsigned_abs() as usize)
235+
.collect::<String>(),
236+
),
253237
Ordering::Equal => Some("".to_string()),
254238
Ordering::Greater => Some(
255239
string
256-
.graphemes(true)
257-
.rev()
258-
.take(n as usize)
259-
.collect::<Vec<&str>>()
260-
.iter()
261-
.rev()
262-
.copied()
240+
.chars()
241+
.skip(max(string.chars().count() as i64 - n, 0) as usize)
263242
.collect::<String>(),
264243
),
265244
},
@@ -349,6 +328,7 @@ pub fn rpad<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
349328

350329
/// Returns starting index of specified substring within string, or zero if it's not present. (Same as position(substring in string), but note the reversed argument order.)
351330
/// strpos('high', 'ig') = 2
331+
/// The implementation uses UTF-8 code points as characters
352332
pub fn strpos<T: ArrowPrimitiveType>(args: &[ArrayRef]) -> Result<ArrayRef>
353333
where
354334
T::Native: OffsetSizeTrait,
@@ -374,28 +354,13 @@ where
374354
.zip(substring_array.iter())
375355
.map(|(string, substring)| match (string, substring) {
376356
(Some(string), Some(substring)) => {
377-
// the rfind method returns the byte index of the substring which may or may not be the same as the character index due to UTF8 encoding
378-
// this method first finds the matching byte using rfind
379-
// then maps that to the character index by matching on the grapheme_index of the byte_index
380-
Some(
381-
T::Native::from_usize(string.to_string().rfind(substring).map_or(
382-
0,
383-
|byte_offset| {
384-
string
385-
.grapheme_indices(true)
386-
.collect::<Vec<(usize, &str)>>()
387-
.iter()
388-
.enumerate()
389-
.filter(|(_, (offset, _))| *offset == byte_offset)
390-
.map(|(index, _)| index)
391-
.collect::<Vec<usize>>()
392-
.first()
393-
.expect("should not fail as grapheme_indices and byte offsets are tightly coupled")
394-
.to_owned()
395-
+ 1
396-
},
397-
))
398-
.expect("should not fail due to map_or default value")
357+
// the find method returns the byte index of the substring
358+
// Next, we count the number of the chars until that byte
359+
T::Native::from_usize(
360+
string
361+
.find(substring)
362+
.map(|x| string[..x].chars().count() + 1)
363+
.unwrap_or(0),
399364
)
400365
}
401366
_ => None,
@@ -408,6 +373,7 @@ where
408373
/// Extracts the substring of string starting at the start'th character, and extending for count characters if that is specified. (Same as substring(string from start for count).)
409374
/// substr('alphabet', 3) = 'phabet'
410375
/// substr('alphabet', 3, 2) = 'ph'
376+
/// The implementation uses UTF-8 code points as characters
411377
pub fn substr<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
412378
match args.len() {
413379
2 => {
@@ -422,13 +388,7 @@ pub fn substr<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
422388
if start <= 0 {
423389
Some(string.to_string())
424390
} else {
425-
let graphemes = string.graphemes(true).collect::<Vec<&str>>();
426-
let start_pos = start as usize - 1;
427-
if graphemes.len() < start_pos {
428-
Some("".to_string())
429-
} else {
430-
Some(graphemes[start_pos..].concat())
431-
}
391+
Some(string.chars().skip(start as usize - 1).collect())
432392
}
433393
}
434394
_ => None,
@@ -455,29 +415,9 @@ pub fn substr<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
455415
count
456416
)))
457417
} else {
458-
let graphemes = string.graphemes(true).collect::<Vec<&str>>();
459-
let (start_pos, end_pos) = if start <= 0 {
460-
let end_pos = start + count - 1;
461-
(
462-
0_usize,
463-
if end_pos < 0 {
464-
// we use 0 as workaround for usize to return empty string
465-
0
466-
} else {
467-
end_pos as usize
468-
},
469-
)
470-
} else {
471-
((start - 1) as usize, (start + count - 1) as usize)
472-
};
473-
474-
if end_pos == 0 || graphemes.len() < start_pos {
475-
Ok(Some("".to_string()))
476-
} else if graphemes.len() < end_pos {
477-
Ok(Some(graphemes[start_pos..].concat()))
478-
} else {
479-
Ok(Some(graphemes[start_pos..end_pos].concat()))
480-
}
418+
let skip = max(0, start - 1);
419+
let count = max(0, count + (if start < 1 {start - 1} else {0}));
420+
Ok(Some(string.chars().skip(skip as usize).take(count as usize).collect::<String>()))
481421
}
482422
}
483423
_ => Ok(None),
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
-- Licensed to the Apache Software Foundation (ASF) under one
2+
-- or more contributor license agreements. See the NOTICE file
3+
-- distributed with this work for additional information
4+
-- regarding copyright ownership. The ASF licenses this file
5+
-- to you under the Apache License, Version 2.0 (the
6+
-- "License"); you may not use this file except in compliance
7+
-- with the License. You may obtain a copy of the License at
8+
9+
-- http://www.apache.org/licenses/LICENSE-2.0
10+
11+
-- Unless required by applicable law or agreed to in writing, software
12+
-- distributed under the License is distributed on an "AS IS" BASIS,
13+
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
-- See the License for the specific language governing permissions and
15+
-- limitations under the License.
16+
17+
select length('ä');

‎integration-tests/test_psql_parity.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ def generate_csv_from_psql(fname: str):
8282

8383
class TestPsqlParity:
8484
def test_tests_count(self):
85-
assert len(test_files) == 21, "tests are missed"
85+
assert len(test_files) == 22, "tests are missed"
8686

8787
@pytest.mark.parametrize("fname", test_files)
8888
def test_sql_file(self, fname):

0 commit comments

Comments
 (0)