Skip to content

Commit 0f9be90

Browse files
committed
refactor: deduplicate split list construction
Extract finish_split_list while preserving the input validity buffers from #6953. Keep the LargeUtf8 output-type regression test and reuse the upstream array/array null-delimiter test. Closes #6983
1 parent 4a8a949 commit 0f9be90

1 file changed

Lines changed: 105 additions & 59 deletions

File tree

  • native/spark-expr/src/string_funcs

‎native/spark-expr/src/string_funcs/split.rs‎

Lines changed: 105 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ use arrow::array::{
1919
Array, ArrayBuilder, ArrayRef, GenericListArray, GenericStringArray, GenericStringBuilder,
2020
ListArray, OffsetSizeTrait,
2121
};
22-
use arrow::buffer::OffsetBuffer;
22+
use arrow::buffer::{NullBuffer, OffsetBuffer};
2323
use arrow::datatypes::{DataType, Field};
2424
use datafusion::common::{
2525
cast::as_generic_string_array, exec_err, DataFusionError, Result as DataFusionResult,
@@ -286,6 +286,32 @@ fn split_sql_scalar_array<O: OffsetSizeTrait>(
286286
}
287287
}
288288

289+
/// Build the list output shared by the four split functions: finish the per-row values builder
290+
/// into a Utf8/LargeUtf8 child array and wrap it in a list array with the collected offsets.
291+
///
292+
/// The caller owns the offsets and the null buffer, since those differ per split variant.
293+
fn finish_split_list<O: OffsetSizeTrait>(
294+
offsets: Vec<O>,
295+
mut values_builder: GenericStringBuilder<O>,
296+
nulls: Option<NullBuffer>,
297+
) -> ColumnarValue {
298+
let values_array = Arc::new(values_builder.finish()) as ArrayRef;
299+
let item_type = if O::IS_LARGE {
300+
DataType::LargeUtf8
301+
} else {
302+
DataType::Utf8
303+
};
304+
let field = Arc::new(Field::new("item", item_type, false));
305+
let list_array = GenericListArray::<O>::new(
306+
field,
307+
OffsetBuffer::new(offsets.into()),
308+
values_array,
309+
nulls,
310+
);
311+
312+
ColumnarValue::Array(Arc::new(list_array))
313+
}
314+
289315
fn split_generic<O: OffsetSizeTrait>(
290316
string_array: &GenericStringArray<O>,
291317
regex: &Regex,
@@ -307,21 +333,11 @@ fn split_generic<O: OffsetSizeTrait>(
307333
offsets.push(O::usize_as(values_builder.len()));
308334
}
309335

310-
let values_array = Arc::new(values_builder.finish()) as ArrayRef;
311-
let item_type = if O::IS_LARGE {
312-
DataType::LargeUtf8
313-
} else {
314-
DataType::Utf8
315-
};
316-
let field = Arc::new(Field::new("item", item_type, false));
317-
let list_array = GenericListArray::<O>::new(
318-
field,
319-
OffsetBuffer::new(offsets.into()),
320-
values_array,
336+
Ok(finish_split_list(
337+
offsets,
338+
values_builder,
321339
string_array.nulls().cloned(),
322-
);
323-
324-
Ok(ColumnarValue::Array(Arc::new(list_array)))
340+
))
325341
}
326342

327343
fn split_sql_generic_scalar<O: OffsetSizeTrait>(
@@ -340,21 +356,11 @@ fn split_sql_generic_scalar<O: OffsetSizeTrait>(
340356
offsets.push(O::usize_as(values_builder.len()));
341357
}
342358

343-
let values_array = Arc::new(values_builder.finish()) as ArrayRef;
344-
let item_type = if O::IS_LARGE {
345-
DataType::LargeUtf8
346-
} else {
347-
DataType::Utf8
348-
};
349-
let field = Arc::new(Field::new("item", item_type, false));
350-
let list_array = GenericListArray::<O>::new(
351-
field,
352-
OffsetBuffer::new(offsets.into()),
353-
values_array,
359+
Ok(finish_split_list(
360+
offsets,
361+
values_builder,
354362
string_array.nulls().cloned(),
355-
);
356-
357-
Ok(ColumnarValue::Array(Arc::new(list_array)))
363+
))
358364
}
359365

360366
fn split_sql_generic_scalar_array<O: OffsetSizeTrait, D: OffsetSizeTrait>(
@@ -373,21 +379,11 @@ fn split_sql_generic_scalar_array<O: OffsetSizeTrait, D: OffsetSizeTrait>(
373379
offsets.push(O::usize_as(values_builder.len()));
374380
}
375381

376-
let values_array = Arc::new(values_builder.finish()) as ArrayRef;
377-
let item_type = if O::IS_LARGE {
378-
DataType::LargeUtf8
379-
} else {
380-
DataType::Utf8
381-
};
382-
let field = Arc::new(Field::new("item", item_type, false));
383-
let list_array = GenericListArray::<O>::new(
384-
field,
385-
OffsetBuffer::new(offsets.into()),
386-
values_array,
382+
Ok(finish_split_list(
383+
offsets,
384+
values_builder,
387385
delimiter_array.nulls().cloned(),
388-
);
389-
390-
Ok(ColumnarValue::Array(Arc::new(list_array)))
386+
))
391387
}
392388

393389
fn split_sql_generic_array<O: OffsetSizeTrait, D: OffsetSizeTrait>(
@@ -410,21 +406,11 @@ fn split_sql_generic_array<O: OffsetSizeTrait, D: OffsetSizeTrait>(
410406
offsets.push(O::usize_as(values_builder.len()));
411407
}
412408

413-
let values_array = Arc::new(values_builder.finish()) as ArrayRef;
414-
let item_type = if O::IS_LARGE {
415-
DataType::LargeUtf8
416-
} else {
417-
DataType::Utf8
418-
};
419-
let field = Arc::new(Field::new("item", item_type, false));
420-
let list_array = GenericListArray::<O>::new(
421-
field,
422-
OffsetBuffer::new(offsets.into()),
423-
values_array,
424-
arrow::buffer::NullBuffer::union(string_array.nulls(), delimiter_array.nulls()),
425-
);
426-
427-
Ok(ColumnarValue::Array(Arc::new(list_array)))
409+
Ok(finish_split_list(
410+
offsets,
411+
values_builder,
412+
NullBuffer::union(string_array.nulls(), delimiter_array.nulls()),
413+
))
428414
}
429415

430416
/// Push the splits of `string` into `builder`. Avoids materializing an
@@ -600,7 +586,7 @@ fn new_null_list_array_value(len: usize) -> ListArray {
600586
#[cfg(test)]
601587
mod tests {
602588
use super::*;
603-
use arrow::array::StringArray;
589+
use arrow::array::{LargeListArray, LargeStringArray, StringArray};
604590

605591
fn run_split(args: &[ColumnarValue]) -> DataFusionResult<ColumnarValue> {
606592
spark_split(args, &PatternCache::new())
@@ -806,6 +792,58 @@ mod tests {
806792
}
807793
}
808794

795+
#[test]
796+
fn test_split_sql_array_scalar_delimiter_with_nulls() {
797+
let string_array =
798+
Arc::new(StringArray::from(vec![Some("a,b"), None, Some("c,")])) as ArrayRef;
799+
let delimiter = ColumnarValue::Scalar(ScalarValue::Utf8(Some(",".to_string())));
800+
801+
let result = spark_split_sql(&[ColumnarValue::Array(string_array), delimiter]).unwrap();
802+
803+
match result {
804+
ColumnarValue::Array(arr) => {
805+
let list_array = arr.as_any().downcast_ref::<ListArray>().unwrap();
806+
assert_eq!(list_array.len(), 3);
807+
assert_eq!(
808+
list_array.data_type(),
809+
&DataType::List(Arc::new(Field::new("item", DataType::Utf8, false)))
810+
);
811+
assert_list_value(list_array, 0, &["a", "b"]);
812+
assert!(list_array.is_null(1));
813+
assert_list_value(list_array, 2, &["c", ""]);
814+
}
815+
_ => panic!("Expected Array result"),
816+
}
817+
}
818+
819+
#[test]
820+
fn test_split_sql_large_utf8_scalar_string_array_delimiter() {
821+
let delimiter_array = Arc::new(StringArray::from(vec![Some(","), None])) as ArrayRef;
822+
823+
let result = spark_split_sql(&[
824+
ColumnarValue::Scalar(ScalarValue::LargeUtf8(Some("a,b".to_string()))),
825+
ColumnarValue::Array(delimiter_array),
826+
])
827+
.unwrap();
828+
829+
match result {
830+
ColumnarValue::Array(arr) => {
831+
let list_array = arr.as_any().downcast_ref::<LargeListArray>().unwrap();
832+
assert_eq!(list_array.len(), 2);
833+
// Output offsets and item type follow the input type, not the delimiter's.
834+
assert_eq!(list_array.value_offsets().len(), 3);
835+
assert_eq!(
836+
list_array.data_type(),
837+
&DataType::LargeList(Arc::new(Field::new("item", DataType::LargeUtf8, false)))
838+
);
839+
assert_eq!(list_array.values().data_type(), &DataType::LargeUtf8);
840+
assert_large_list_value(list_array, 0, &["a", "b"]);
841+
assert!(list_array.is_null(1));
842+
}
843+
_ => panic!("Expected Array result"),
844+
}
845+
}
846+
809847
/// One expression evaluates many batches; the pattern must compile once and results
810848
/// must stay correct on every batch.
811849
#[test]
@@ -850,6 +888,14 @@ mod tests {
850888
assert_eq!(actual, expected);
851889
}
852890

891+
fn assert_large_list_value(list_array: &LargeListArray, row: usize, expected: &[&str]) {
892+
let value = list_array.value(row);
893+
let strings = value.as_any().downcast_ref::<LargeStringArray>().unwrap();
894+
let actual = strings.iter().collect::<Vec<_>>();
895+
let expected = expected.iter().map(|s| Some(*s)).collect::<Vec<_>>();
896+
assert_eq!(actual, expected);
897+
}
898+
853899
fn assert_null_list_scalar(result: ColumnarValue) {
854900
match result {
855901
ColumnarValue::Scalar(ScalarValue::List(array)) => {

0 commit comments

Comments
 (0)