Skip to content

Commit f622ed1

Browse files
committed
address review: preserve LargeUtf8 output, drop unsafe, widen tests
- `quote`/`soundex` now preserve the input's `LargeUtf8` type instead of narrowing to `Utf8`, which had triggered a planner assertion. The `soundex.slt` and `quote.slt` files exercised only `Utf8` inputs, so the mismatch went unnoticed. - Drop the `from_utf8_unchecked` in `soundex`; the four-byte validation is cheap and no longer relies on an invariant that only tests could guard. - Unwrap the offset first/last directly, per @Jefffrey. - Add unit tests and sqllogictests for `LargeUtf8`, `Utf8View`, `NULL`, empty values, non-ASCII alphabetic first characters, sliced arrays, and multi-row batches.
1 parent 4a7577f commit f622ed1

4 files changed

Lines changed: 309 additions & 20 deletions

File tree

‎datafusion/spark/src/function/string/quote.rs‎

Lines changed: 89 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
use arrow::array::{ArrayRef, OffsetSizeTrait, StringBuilder};
18+
use arrow::array::{ArrayRef, GenericStringBuilder, OffsetSizeTrait};
1919
use arrow::datatypes::DataType;
2020
use datafusion::logical_expr::{Coercion, ColumnarValue, Signature, TypeSignatureClass};
2121
use datafusion_common::cast::{as_generic_string_array, as_string_view_array};
@@ -91,20 +91,21 @@ fn quote_array<T: OffsetSizeTrait>(array: &ArrayRef) -> Result<ArrayRef> {
9191
let str_array = as_generic_string_array::<T>(array)?;
9292
// Slicing an array keeps the whole value buffer and narrows only the
9393
// offsets, so measure the data through the offsets rather than through
94-
// `value_data()`.
94+
// `value_data()`. An offset buffer holds one more entry than the array has
95+
// rows, so it is never empty.
9596
let offsets = str_array.value_offsets();
96-
let data_len = match (offsets.first(), offsets.last()) {
97-
(Some(first), Some(last)) => last.as_usize() - first.as_usize(),
98-
_ => 0,
99-
};
100-
Ok(quote_impl(str_array.iter(), data_len))
97+
let data_len = offsets.last().unwrap().as_usize() - offsets[0].as_usize();
98+
Ok(quote_impl::<T, _>(str_array.iter(), data_len))
10199
}
102100

103101
fn quote_view(str_view: &ArrayRef) -> Result<ArrayRef> {
104102
let str_array = as_string_view_array(str_view)?;
105103
// `total_bytes_len` walks the (sliced) views and counts inlined values,
106104
// unlike the buffer capacities reported by `get_buffer_memory_size`.
107-
Ok(quote_impl(str_array.iter(), str_array.total_bytes_len()))
105+
Ok(quote_impl::<i32, _>(
106+
str_array.iter(),
107+
str_array.total_bytes_len(),
108+
))
108109
}
109110

110111
const QUOTE_CHAR: char = '\'';
@@ -115,12 +116,13 @@ const ESCAPED_QUOTE: &str = "\\'";
115116
///
116117
/// `data_capacity` is a hint for the total input byte length; the output adds two
117118
/// surrounding quotes per row plus one byte per escaped quote.
118-
fn quote_impl<'a>(
119-
input: impl Iterator<Item = Option<&'a str>>,
119+
fn quote_impl<'a, O: OffsetSizeTrait, I: Iterator<Item = Option<&'a str>>>(
120+
input: I,
120121
data_capacity: usize,
121122
) -> ArrayRef {
122123
let len = input.size_hint().0;
123-
let mut builder = StringBuilder::with_capacity(len, data_capacity + 2 * len);
124+
let mut builder =
125+
GenericStringBuilder::<O>::with_capacity(len, data_capacity + 2 * len);
124126
for value in input {
125127
match value {
126128
Some(value) => append_quoted(&mut builder, value),
@@ -135,7 +137,7 @@ fn quote_impl<'a>(
135137
/// Writes straight into the builder's buffer — finalized by the trailing
136138
/// `append_value("")` — so no intermediate `String` is allocated per row, and
137139
/// copies the runs between quotes rather than one character at a time.
138-
fn append_quoted(builder: &mut StringBuilder, s: &str) {
140+
fn append_quoted<O: OffsetSizeTrait>(builder: &mut GenericStringBuilder<O>, s: &str) {
139141
// `write_str` on a `GenericStringBuilder` is infallible.
140142
let mut runs = s.split(QUOTE_CHAR);
141143
builder.write_char(QUOTE_CHAR).unwrap();
@@ -148,3 +150,78 @@ fn append_quoted(builder: &mut StringBuilder, s: &str) {
148150
builder.write_char(QUOTE_CHAR).unwrap();
149151
builder.append_value("");
150152
}
153+
154+
#[cfg(test)]
155+
mod tests {
156+
use super::*;
157+
use arrow::array::{LargeStringArray, StringArray, StringViewArray};
158+
159+
fn quote(array: ArrayRef) -> ArrayRef {
160+
spark_quote_inner(&[array]).unwrap()
161+
}
162+
163+
fn as_strings(array: &ArrayRef) -> Vec<Option<&str>> {
164+
as_generic_string_array::<i32>(array)
165+
.unwrap()
166+
.iter()
167+
.collect()
168+
}
169+
170+
#[test]
171+
fn quote_preserves_the_offset_width() {
172+
let utf8 = quote(Arc::new(StringArray::from(vec!["it's"])) as ArrayRef);
173+
assert_eq!(utf8.data_type(), &DataType::Utf8);
174+
assert_eq!(as_strings(&utf8), vec![Some("'it\\'s'")]);
175+
176+
let large = quote(Arc::new(LargeStringArray::from(vec!["it's"])) as ArrayRef);
177+
assert_eq!(large.data_type(), &DataType::LargeUtf8);
178+
let large = as_generic_string_array::<i64>(&large).unwrap();
179+
assert_eq!(large.value(0), "'it\\'s'");
180+
181+
// A view input has no offsets to preserve, so it narrows to `Utf8`.
182+
let view = quote(Arc::new(StringViewArray::from(vec!["it's"])) as ArrayRef);
183+
assert_eq!(view.data_type(), &DataType::Utf8);
184+
assert_eq!(as_strings(&view), vec![Some("'it\\'s'")]);
185+
}
186+
187+
/// Slicing keeps the whole value buffer, so the capacity hint has to be
188+
/// measured through the offsets rather than through `value_data()`.
189+
#[test]
190+
fn quote_sliced_array() {
191+
let array = Arc::new(StringArray::from(vec![
192+
Some("a very long leading value that inflates value_data"),
193+
Some("it's"),
194+
None,
195+
Some(""),
196+
])) as ArrayRef;
197+
198+
let result = quote(array.slice(1, 3));
199+
assert_eq!(
200+
as_strings(&result),
201+
vec![Some("'it\\'s'"), None, Some("''")]
202+
);
203+
}
204+
205+
#[test]
206+
fn quote_sliced_view_array() {
207+
let array = Arc::new(StringViewArray::from(vec![
208+
// Longer than 12 bytes, so this one lives in a data buffer.
209+
Some("a very long leading value that inflates value_data"),
210+
Some("it's"),
211+
None,
212+
])) as ArrayRef;
213+
214+
let result = quote(array.slice(1, 2));
215+
assert_eq!(as_strings(&result), vec![Some("'it\\'s'"), None]);
216+
}
217+
218+
#[test]
219+
fn quote_empty_array() {
220+
let empty = quote(Arc::new(StringArray::from(Vec::<&str>::new())) as ArrayRef);
221+
assert_eq!(empty.len(), 0);
222+
223+
let empty_view =
224+
quote(Arc::new(StringViewArray::from(Vec::<&str>::new())) as ArrayRef);
225+
assert_eq!(empty_view.len(), 0);
226+
}
227+
}

‎datafusion/spark/src/function/string/soundex.rs‎

Lines changed: 106 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
use arrow::array::{ArrayRef, OffsetSizeTrait, StringBuilder};
18+
use arrow::array::{ArrayRef, GenericStringBuilder, OffsetSizeTrait};
1919
use arrow::datatypes::DataType;
2020
use datafusion_common::cast::{as_generic_string_array, as_string_view_array};
2121
use datafusion_common::utils::take_function_args;
@@ -80,17 +80,23 @@ fn spark_soundex_inner(arg: &[ArrayRef]) -> Result<ArrayRef> {
8080
}
8181

8282
fn soundex_array<T: OffsetSizeTrait>(array: &ArrayRef) -> Result<ArrayRef> {
83-
Ok(soundex_impl(as_generic_string_array::<T>(array)?.iter()))
83+
Ok(soundex_impl::<T, _>(
84+
as_generic_string_array::<T>(array)?.iter(),
85+
))
8486
}
8587

8688
fn soundex_view(str_view: &ArrayRef) -> Result<ArrayRef> {
87-
Ok(soundex_impl(as_string_view_array(str_view)?.iter()))
89+
Ok(soundex_impl::<i32, _>(
90+
as_string_view_array(str_view)?.iter(),
91+
))
8892
}
8993

90-
fn soundex_impl<'a>(input: impl Iterator<Item = Option<&'a str>>) -> ArrayRef {
94+
fn soundex_impl<'a, O: OffsetSizeTrait, I: Iterator<Item = Option<&'a str>>>(
95+
input: I,
96+
) -> ArrayRef {
9197
let len = input.size_hint().0;
9298
// A soundex code is always exactly 4 ASCII characters.
93-
let mut builder = StringBuilder::with_capacity(len, len * SOUNDEX_LEN);
99+
let mut builder = GenericStringBuilder::<O>::with_capacity(len, len * SOUNDEX_LEN);
94100
for value in input {
95101
match value {
96102
Some(value) => append_soundex(&mut builder, value),
@@ -123,7 +129,7 @@ const SOUNDEX_LEN: usize = 4;
123129
///
124130
/// Strings that do not start with an ASCII letter are passed through unchanged.
125131
/// Otherwise the code is built in a stack buffer, so no row allocates.
126-
fn append_soundex(builder: &mut StringBuilder, s: &str) {
132+
fn append_soundex<O: OffsetSizeTrait>(builder: &mut GenericStringBuilder<O>, s: &str) {
127133
let mut chars = s.chars();
128134

129135
let first_char = match chars.next() {
@@ -163,6 +169,98 @@ fn append_soundex(builder: &mut StringBuilder, s: &str) {
163169
}
164170
}
165171

166-
// SAFETY: `soundex_code` holds an ASCII letter followed by ASCII digits.
167-
builder.append_value(unsafe { std::str::from_utf8_unchecked(&soundex_code) });
172+
// `soundex_code` holds an ASCII letter followed by ASCII digits, so the
173+
// validation here is a four-byte check that never fails.
174+
builder
175+
.append_value(std::str::from_utf8(&soundex_code).expect("soundex code is ASCII"));
176+
}
177+
178+
#[cfg(test)]
179+
mod tests {
180+
use super::*;
181+
use arrow::array::{LargeStringArray, StringArray, StringViewArray};
182+
183+
fn soundex(array: ArrayRef) -> ArrayRef {
184+
spark_soundex_inner(&[array]).unwrap()
185+
}
186+
187+
fn as_strings(array: &ArrayRef) -> Vec<Option<&str>> {
188+
as_generic_string_array::<i32>(array)
189+
.unwrap()
190+
.iter()
191+
.collect()
192+
}
193+
194+
#[test]
195+
fn soundex_preserves_the_offset_width() {
196+
let utf8 = soundex(Arc::new(StringArray::from(vec!["Miller"])) as ArrayRef);
197+
assert_eq!(utf8.data_type(), &DataType::Utf8);
198+
assert_eq!(as_strings(&utf8), vec![Some("M460")]);
199+
200+
let large = soundex(Arc::new(LargeStringArray::from(vec!["Miller"])) as ArrayRef);
201+
assert_eq!(large.data_type(), &DataType::LargeUtf8);
202+
assert_eq!(
203+
as_generic_string_array::<i64>(&large).unwrap().value(0),
204+
"M460"
205+
);
206+
207+
// A view input has no offsets to preserve, so it narrows to `Utf8`.
208+
let view = soundex(Arc::new(StringViewArray::from(vec!["Miller"])) as ArrayRef);
209+
assert_eq!(view.data_type(), &DataType::Utf8);
210+
assert_eq!(as_strings(&view), vec![Some("M460")]);
211+
}
212+
213+
/// Values whose first character is not an ASCII letter are passed through
214+
/// unchanged, so the output is not always the four-byte code.
215+
#[test]
216+
fn soundex_multi_row_batch() {
217+
let array = Arc::new(StringArray::from(vec![
218+
Some("Miller"),
219+
None,
220+
Some(""),
221+
// Non-ASCII alphabetic first character: passthrough, not a code.
222+
Some("Ñoño"),
223+
Some("Éclair"),
224+
Some("123"),
225+
Some("Robert"),
226+
])) as ArrayRef;
227+
228+
assert_eq!(
229+
as_strings(&soundex(array)),
230+
vec![
231+
Some("M460"),
232+
None,
233+
Some(""),
234+
Some("Ñoño"),
235+
Some("Éclair"),
236+
Some("123"),
237+
Some("R163"),
238+
]
239+
);
240+
}
241+
242+
#[test]
243+
fn soundex_sliced_array() {
244+
let array = Arc::new(StringArray::from(vec![
245+
Some("Miller"),
246+
Some("Ñoño"),
247+
None,
248+
Some("Robert"),
249+
])) as ArrayRef;
250+
251+
assert_eq!(
252+
as_strings(&soundex(array.slice(1, 3))),
253+
vec![Some("Ñoño"), None, Some("R163")]
254+
);
255+
}
256+
257+
#[test]
258+
fn soundex_empty_array() {
259+
let empty = soundex(Arc::new(StringArray::from(Vec::<&str>::new())) as ArrayRef);
260+
assert_eq!(empty.len(), 0);
261+
262+
let empty_view =
263+
soundex(Arc::new(StringViewArray::from(Vec::<&str>::new())) as ArrayRef);
264+
assert_eq!(empty_view.len(), 0);
265+
}
168266
}

‎datafusion/sqllogictest/test_files/spark/string/quote.slt‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,3 +159,64 @@ query T
159159
SELECT quote('''');
160160
----
161161
'\''
162+
163+
query T
164+
SELECT quote(NULL);
165+
----
166+
NULL
167+
168+
query T
169+
SELECT quote('');
170+
----
171+
''
172+
173+
# LargeUtf8 and Utf8View inputs
174+
175+
query T
176+
SELECT quote(arrow_cast('it''s', 'LargeUtf8'));
177+
----
178+
'it\'s'
179+
180+
query T
181+
SELECT quote(arrow_cast('', 'LargeUtf8'));
182+
----
183+
''
184+
185+
query T
186+
SELECT quote(arrow_cast('it''s', 'Utf8View'));
187+
----
188+
'it\'s'
189+
190+
query T
191+
SELECT quote(arrow_cast('', 'Utf8View'));
192+
----
193+
''
194+
195+
# Multi-row batches, so the output builder is reused across rows
196+
197+
query T
198+
SELECT quote(c) FROM VALUES ('a'), ('it''s'), (NULL), (''), ('x''''y') AS t(c);
199+
----
200+
'a'
201+
'it\'s'
202+
NULL
203+
''
204+
'x\'\'y'
205+
206+
query T
207+
SELECT quote(arrow_cast(c, 'LargeUtf8')) FROM VALUES ('a'), ('it''s'), (NULL) AS t(c);
208+
----
209+
'a'
210+
'it\'s'
211+
NULL
212+
213+
# Mixes values that are inlined in the views (<= 12 bytes) with values held in a
214+
# data buffer, since the two are measured differently for the capacity hint.
215+
query T
216+
SELECT quote(arrow_cast(c, 'Utf8View'))
217+
FROM VALUES ('a'), ('it''s'), (NULL), ('a value that is longer than twelve bytes and has an '' embedded quote') AS t(c);
218+
----
219+
'a'
220+
'it\'s'
221+
NULL
222+
'a value that is longer than twelve bytes and has an \' embedded quote'

0 commit comments

Comments
 (0)