@@ -19,7 +19,7 @@ use arrow::array::{
1919 Array , ArrayBuilder , ArrayRef , GenericListArray , GenericStringArray , GenericStringBuilder ,
2020 ListArray , OffsetSizeTrait ,
2121} ;
22- use arrow:: buffer:: OffsetBuffer ;
22+ use arrow:: buffer:: { NullBuffer , OffsetBuffer } ;
2323use arrow:: datatypes:: { DataType , Field } ;
2424use datafusion:: common:: {
2525 cast:: as_generic_string_array, exec_err, DataFusionError , Result as DataFusionResult ,
@@ -286,6 +286,32 @@ fn split_sql_scalar_array<O: OffsetSizeTrait>(
286286 }
287287}
288288
289+ /// Build the list output shared by the four split functions: finish the per-row values builder
290+ /// into a Utf8/LargeUtf8 child array and wrap it in a list array with the collected offsets.
291+ ///
292+ /// The caller owns the offsets and the null buffer, since those differ per split variant.
293+ fn finish_split_list < O : OffsetSizeTrait > (
294+ offsets : Vec < O > ,
295+ mut values_builder : GenericStringBuilder < O > ,
296+ nulls : Option < NullBuffer > ,
297+ ) -> ColumnarValue {
298+ let values_array = Arc :: new ( values_builder. finish ( ) ) as ArrayRef ;
299+ let item_type = if O :: IS_LARGE {
300+ DataType :: LargeUtf8
301+ } else {
302+ DataType :: Utf8
303+ } ;
304+ let field = Arc :: new ( Field :: new ( "item" , item_type, false ) ) ;
305+ let list_array = GenericListArray :: < O > :: new (
306+ field,
307+ OffsetBuffer :: new ( offsets. into ( ) ) ,
308+ values_array,
309+ nulls,
310+ ) ;
311+
312+ ColumnarValue :: Array ( Arc :: new ( list_array) )
313+ }
314+
289315fn split_generic < O : OffsetSizeTrait > (
290316 string_array : & GenericStringArray < O > ,
291317 regex : & Regex ,
@@ -307,21 +333,11 @@ fn split_generic<O: OffsetSizeTrait>(
307333 offsets. push ( O :: usize_as ( values_builder. len ( ) ) ) ;
308334 }
309335
310- let values_array = Arc :: new ( values_builder. finish ( ) ) as ArrayRef ;
311- let item_type = if O :: IS_LARGE {
312- DataType :: LargeUtf8
313- } else {
314- DataType :: Utf8
315- } ;
316- let field = Arc :: new ( Field :: new ( "item" , item_type, false ) ) ;
317- let list_array = GenericListArray :: < O > :: new (
318- field,
319- OffsetBuffer :: new ( offsets. into ( ) ) ,
320- values_array,
336+ Ok ( finish_split_list (
337+ offsets,
338+ values_builder,
321339 string_array. nulls ( ) . cloned ( ) ,
322- ) ;
323-
324- Ok ( ColumnarValue :: Array ( Arc :: new ( list_array) ) )
340+ ) )
325341}
326342
327343fn split_sql_generic_scalar < O : OffsetSizeTrait > (
@@ -340,21 +356,11 @@ fn split_sql_generic_scalar<O: OffsetSizeTrait>(
340356 offsets. push ( O :: usize_as ( values_builder. len ( ) ) ) ;
341357 }
342358
343- let values_array = Arc :: new ( values_builder. finish ( ) ) as ArrayRef ;
344- let item_type = if O :: IS_LARGE {
345- DataType :: LargeUtf8
346- } else {
347- DataType :: Utf8
348- } ;
349- let field = Arc :: new ( Field :: new ( "item" , item_type, false ) ) ;
350- let list_array = GenericListArray :: < O > :: new (
351- field,
352- OffsetBuffer :: new ( offsets. into ( ) ) ,
353- values_array,
359+ Ok ( finish_split_list (
360+ offsets,
361+ values_builder,
354362 string_array. nulls ( ) . cloned ( ) ,
355- ) ;
356-
357- Ok ( ColumnarValue :: Array ( Arc :: new ( list_array) ) )
363+ ) )
358364}
359365
360366fn split_sql_generic_scalar_array < O : OffsetSizeTrait , D : OffsetSizeTrait > (
@@ -373,21 +379,11 @@ fn split_sql_generic_scalar_array<O: OffsetSizeTrait, D: OffsetSizeTrait>(
373379 offsets. push ( O :: usize_as ( values_builder. len ( ) ) ) ;
374380 }
375381
376- let values_array = Arc :: new ( values_builder. finish ( ) ) as ArrayRef ;
377- let item_type = if O :: IS_LARGE {
378- DataType :: LargeUtf8
379- } else {
380- DataType :: Utf8
381- } ;
382- let field = Arc :: new ( Field :: new ( "item" , item_type, false ) ) ;
383- let list_array = GenericListArray :: < O > :: new (
384- field,
385- OffsetBuffer :: new ( offsets. into ( ) ) ,
386- values_array,
382+ Ok ( finish_split_list (
383+ offsets,
384+ values_builder,
387385 delimiter_array. nulls ( ) . cloned ( ) ,
388- ) ;
389-
390- Ok ( ColumnarValue :: Array ( Arc :: new ( list_array) ) )
386+ ) )
391387}
392388
393389fn split_sql_generic_array < O : OffsetSizeTrait , D : OffsetSizeTrait > (
@@ -410,21 +406,11 @@ fn split_sql_generic_array<O: OffsetSizeTrait, D: OffsetSizeTrait>(
410406 offsets. push ( O :: usize_as ( values_builder. len ( ) ) ) ;
411407 }
412408
413- let values_array = Arc :: new ( values_builder. finish ( ) ) as ArrayRef ;
414- let item_type = if O :: IS_LARGE {
415- DataType :: LargeUtf8
416- } else {
417- DataType :: Utf8
418- } ;
419- let field = Arc :: new ( Field :: new ( "item" , item_type, false ) ) ;
420- let list_array = GenericListArray :: < O > :: new (
421- field,
422- OffsetBuffer :: new ( offsets. into ( ) ) ,
423- values_array,
424- arrow:: buffer:: NullBuffer :: union ( string_array. nulls ( ) , delimiter_array. nulls ( ) ) ,
425- ) ;
426-
427- Ok ( ColumnarValue :: Array ( Arc :: new ( list_array) ) )
409+ Ok ( finish_split_list (
410+ offsets,
411+ values_builder,
412+ NullBuffer :: union ( string_array. nulls ( ) , delimiter_array. nulls ( ) ) ,
413+ ) )
428414}
429415
430416/// Push the splits of `string` into `builder`. Avoids materializing an
@@ -600,7 +586,7 @@ fn new_null_list_array_value(len: usize) -> ListArray {
600586#[ cfg( test) ]
601587mod tests {
602588 use super :: * ;
603- use arrow:: array:: StringArray ;
589+ use arrow:: array:: { LargeListArray , LargeStringArray , StringArray } ;
604590
605591 fn run_split ( args : & [ ColumnarValue ] ) -> DataFusionResult < ColumnarValue > {
606592 spark_split ( args, & PatternCache :: new ( ) )
@@ -806,6 +792,58 @@ mod tests {
806792 }
807793 }
808794
795+ #[ test]
796+ fn test_split_sql_array_scalar_delimiter_with_nulls ( ) {
797+ let string_array =
798+ Arc :: new ( StringArray :: from ( vec ! [ Some ( "a,b" ) , None , Some ( "c," ) ] ) ) as ArrayRef ;
799+ let delimiter = ColumnarValue :: Scalar ( ScalarValue :: Utf8 ( Some ( "," . to_string ( ) ) ) ) ;
800+
801+ let result = spark_split_sql ( & [ ColumnarValue :: Array ( string_array) , delimiter] ) . unwrap ( ) ;
802+
803+ match result {
804+ ColumnarValue :: Array ( arr) => {
805+ let list_array = arr. as_any ( ) . downcast_ref :: < ListArray > ( ) . unwrap ( ) ;
806+ assert_eq ! ( list_array. len( ) , 3 ) ;
807+ assert_eq ! (
808+ list_array. data_type( ) ,
809+ & DataType :: List ( Arc :: new( Field :: new( "item" , DataType :: Utf8 , false ) ) )
810+ ) ;
811+ assert_list_value ( list_array, 0 , & [ "a" , "b" ] ) ;
812+ assert ! ( list_array. is_null( 1 ) ) ;
813+ assert_list_value ( list_array, 2 , & [ "c" , "" ] ) ;
814+ }
815+ _ => panic ! ( "Expected Array result" ) ,
816+ }
817+ }
818+
819+ #[ test]
820+ fn test_split_sql_large_utf8_scalar_string_array_delimiter ( ) {
821+ let delimiter_array = Arc :: new ( StringArray :: from ( vec ! [ Some ( "," ) , None ] ) ) as ArrayRef ;
822+
823+ let result = spark_split_sql ( & [
824+ ColumnarValue :: Scalar ( ScalarValue :: LargeUtf8 ( Some ( "a,b" . to_string ( ) ) ) ) ,
825+ ColumnarValue :: Array ( delimiter_array) ,
826+ ] )
827+ . unwrap ( ) ;
828+
829+ match result {
830+ ColumnarValue :: Array ( arr) => {
831+ let list_array = arr. as_any ( ) . downcast_ref :: < LargeListArray > ( ) . unwrap ( ) ;
832+ assert_eq ! ( list_array. len( ) , 2 ) ;
833+ // Output offsets and item type follow the input type, not the delimiter's.
834+ assert_eq ! ( list_array. value_offsets( ) . len( ) , 3 ) ;
835+ assert_eq ! (
836+ list_array. data_type( ) ,
837+ & DataType :: LargeList ( Arc :: new( Field :: new( "item" , DataType :: LargeUtf8 , false ) ) )
838+ ) ;
839+ assert_eq ! ( list_array. values( ) . data_type( ) , & DataType :: LargeUtf8 ) ;
840+ assert_large_list_value ( list_array, 0 , & [ "a" , "b" ] ) ;
841+ assert ! ( list_array. is_null( 1 ) ) ;
842+ }
843+ _ => panic ! ( "Expected Array result" ) ,
844+ }
845+ }
846+
809847 /// One expression evaluates many batches; the pattern must compile once and results
810848 /// must stay correct on every batch.
811849 #[ test]
@@ -850,6 +888,14 @@ mod tests {
850888 assert_eq ! ( actual, expected) ;
851889 }
852890
891+ fn assert_large_list_value ( list_array : & LargeListArray , row : usize , expected : & [ & str ] ) {
892+ let value = list_array. value ( row) ;
893+ let strings = value. as_any ( ) . downcast_ref :: < LargeStringArray > ( ) . unwrap ( ) ;
894+ let actual = strings. iter ( ) . collect :: < Vec < _ > > ( ) ;
895+ let expected = expected. iter ( ) . map ( |s| Some ( * s) ) . collect :: < Vec < _ > > ( ) ;
896+ assert_eq ! ( actual, expected) ;
897+ }
898+
853899 fn assert_null_list_scalar ( result : ColumnarValue ) {
854900 match result {
855901 ColumnarValue :: Scalar ( ScalarValue :: List ( array) ) => {
0 commit comments