1818//! Compare Comet's float `sort_array` and `array_remove` with DataFusion's `array_sort` and
1919//! `array_remove_all` on doubles without zeros or NaNs, where both return the same arrays, which
2020//! is checked before timing; special-value semantics belong in tests.
21+ //!
22+ //! Comet's `array_distinct` and `array_union` normalize their arguments and then call DataFusion's
23+ //! kernels. Their `_sliced` cases pass a slice of a longer list, as `explode` returns, to show
24+ //! that the normalization covers only the rows of the slice.
2125
2226use arrow:: array:: { Array , ArrayRef , Float64Array , ListArray } ;
2327use arrow:: buffer:: { NullBuffer , OffsetBuffer } ;
@@ -26,9 +30,12 @@ use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
2630use datafusion:: common:: config:: ConfigOptions ;
2731use datafusion:: common:: ScalarValue ;
2832use datafusion:: functions_nested:: remove:: array_remove_all_udf;
33+ use datafusion:: functions_nested:: set_ops:: { array_distinct_udf, array_union_udf} ;
2934use datafusion:: functions_nested:: sort:: array_sort_udf;
3035use datafusion:: logical_expr:: { ColumnarValue , ScalarFunctionArgs , ScalarUDF } ;
31- use datafusion_comet_spark_expr:: { SparkArrayRemove , SparkFloatArrayContains , SparkSortArray } ;
36+ use datafusion_comet_spark_expr:: {
37+ SparkArrayRemove , SparkArraySetOp , SparkFloatArrayContains , SparkSortArray ,
38+ } ;
3239use datafusion_spark:: function:: array:: array_contains:: SparkArrayContains ;
3340use std:: hint:: black_box;
3441use std:: sync:: Arc ;
@@ -41,22 +48,28 @@ fn value(i: usize) -> f64 {
4148 ( ( i * 7919 ) % 101 ) as f64 * 0.5 + 1.0
4249}
4350
44- /// `ROWS ` lists of `len` doubles, with every tenth list and every seventh element null if
51+ /// `rows ` lists of `len` doubles, with every tenth list and every seventh element null if
4552/// `nulls`.
46- fn lists ( len : usize , nulls : bool ) -> ArrayRef {
53+ fn lists ( rows : usize , len : usize , nulls : bool ) -> ArrayRef {
4754 let values = if nulls {
48- Float64Array :: from_iter ( ( 0 ..ROWS * len) . map ( |i| ( i % 7 != 0 ) . then ( || value ( i) ) ) )
55+ Float64Array :: from_iter ( ( 0 ..rows * len) . map ( |i| ( i % 7 != 0 ) . then ( || value ( i) ) ) )
4956 } else {
50- Float64Array :: from_iter_values ( ( 0 ..ROWS * len) . map ( value) )
57+ Float64Array :: from_iter_values ( ( 0 ..rows * len) . map ( value) )
5158 } ;
5259 Arc :: new ( ListArray :: new (
5360 Arc :: new ( Field :: new_list_field ( DataType :: Float64 , true ) ) ,
54- OffsetBuffer :: from_lengths ( std:: iter:: repeat_n ( len, ROWS ) ) ,
61+ OffsetBuffer :: from_lengths ( std:: iter:: repeat_n ( len, rows ) ) ,
5562 Arc :: new ( values) ,
56- nulls. then ( || NullBuffer :: from_iter ( ( 0 ..ROWS ) . map ( |row| row % 10 != 0 ) ) ) ,
63+ nulls. then ( || NullBuffer :: from_iter ( ( 0 ..rows ) . map ( |row| row % 10 != 0 ) ) ) ,
5764 ) )
5865}
5966
67+ /// `ROWS` lists sliced from the middle of 16 times as many. `explode` of a list of lists returns
68+ /// slices like this, one per output batch, which all keep the same child array.
69+ fn sliced_lists ( len : usize , nulls : bool ) -> ArrayRef {
70+ lists ( 16 * ROWS , len, nulls) . slice ( 8 * ROWS , ROWS )
71+ }
72+
6073fn args ( args : Vec < ColumnarValue > , list : & ArrayRef ) -> ScalarFunctionArgs {
6174 ScalarFunctionArgs {
6275 arg_fields : args
@@ -81,9 +94,12 @@ fn criterion_benchmark(c: &mut Criterion) {
8194 // The native path other element types take: array_has plus Spark's null semantics.
8295 let datafusion_contains = Arc :: new ( ScalarUDF :: new_from_impl ( SparkArrayContains :: default ( ) ) ) ;
8396 let ( datafusion_sort, datafusion_remove) = ( array_sort_udf ( ) , array_remove_all_udf ( ) ) ;
97+ let comet_distinct = ScalarUDF :: new_from_impl ( SparkArraySetOp :: distinct ( ) ) ;
98+ let comet_union = ScalarUDF :: new_from_impl ( SparkArraySetOp :: union ( ) ) ;
99+ let ( datafusion_distinct, datafusion_union) = ( array_distinct_udf ( ) , array_union_udf ( ) ) ;
84100 for len in [ 8 , 50 ] {
85101 for nulls in [ false , true ] {
86- let list = lists ( len, nulls) ;
102+ let list = lists ( ROWS , len, nulls) ;
87103 let data = format ! ( "len={len}_null={nulls}" ) ;
88104 let column = ColumnarValue :: Array ( Arc :: clone ( & list) ) ;
89105 let scalar = |value : ScalarValue | ColumnarValue :: Scalar ( value) ;
@@ -121,9 +137,17 @@ fn criterion_benchmark(c: &mut Criterion) {
121137 let row_values =
122138 Float64Array :: from_iter_values ( ( 0 ..ROWS ) . map ( |row| value ( row * len + 1 ) ) ) ;
123139 let contains_column_args = args (
124- vec ! [ column, ColumnarValue :: Array ( Arc :: new( row_values) ) ] ,
140+ vec ! [ column. clone ( ) , ColumnarValue :: Array ( Arc :: new( row_values) ) ] ,
125141 & list,
126142 ) ;
143+ let distinct_args = args ( vec ! [ column. clone( ) ] , & list) ;
144+ // A union of a list with itself: the right side adds no elements, but it is still
145+ // normalized and looked up.
146+ let union_args = args ( vec ! [ column. clone( ) , column] , & list) ;
147+ let sliced = sliced_lists ( len, nulls) ;
148+ let sliced_column = ColumnarValue :: Array ( Arc :: clone ( & sliced) ) ;
149+ let sliced_distinct_args = args ( vec ! [ sliced_column. clone( ) ] , & sliced) ;
150+ let sliced_union_args = args ( vec ! [ sliced_column. clone( ) , sliced_column] , & sliced) ;
127151 let cases = [
128152 (
129153 "sort_array" ,
@@ -160,6 +184,34 @@ fn criterion_benchmark(c: &mut Criterion) {
160184 & contains_column_args,
161185 & contains_column_args,
162186 ) ,
187+ (
188+ "array_distinct" ,
189+ & comet_distinct,
190+ & datafusion_distinct,
191+ & distinct_args,
192+ & distinct_args,
193+ ) ,
194+ (
195+ "array_distinct_sliced" ,
196+ & comet_distinct,
197+ & datafusion_distinct,
198+ & sliced_distinct_args,
199+ & sliced_distinct_args,
200+ ) ,
201+ (
202+ "array_union" ,
203+ & comet_union,
204+ & datafusion_union,
205+ & union_args,
206+ & union_args,
207+ ) ,
208+ (
209+ "array_union_sliced" ,
210+ & comet_union,
211+ & datafusion_union,
212+ & sliced_union_args,
213+ & sliced_union_args,
214+ ) ,
163215 ] ;
164216 for ( op, comet, datafusion, comet_args, datafusion_args) in cases {
165217 let evaluate = |udf : & ScalarUDF , args : & ScalarFunctionArgs | {
0 commit comments