@@ -31,20 +31,20 @@ use arrow::array::{
3131 cast:: AsArray ,
3232} ;
3333use arrow:: array:: {
34- BooleanArray , Datum , GenericListArray , Int32Array , Int64Array , MutableArrayData ,
35- Scalar , make_array,
34+ ArrowPrimitiveType , BooleanArray , Datum , GenericListArray , Int32Array , Int64Array , MutableArrayData , PrimitiveArray , Scalar , make_array
3635} ;
3736use arrow:: buffer:: OffsetBuffer ;
3837use arrow:: compute:: kernels:: cmp:: neq;
3938use arrow:: compute:: kernels:: length:: length;
4039use arrow:: compute:: kernels:: zip:: zip;
4140use arrow:: compute:: { SortColumn , SortOptions , partition} ;
42- use arrow:: datatypes:: { DataType , Field , SchemaRef } ;
41+ use arrow:: datatypes:: { ArrowNativeType , DataType , Field , Int32Type , Int64Type , SchemaRef } ;
4342#[ cfg( feature = "sql" ) ]
4443use sqlparser:: { ast:: Ident , dialect:: GenericDialect , parser:: Parser } ;
4544use std:: borrow:: { Borrow , Cow } ;
4645use std:: cmp:: { Ordering , min} ;
4746use std:: collections:: HashSet ;
47+ use std:: iter:: repeat_n;
4848use std:: num:: NonZero ;
4949use std:: ops:: Range ;
5050use std:: sync:: { Arc , LazyLock } ;
@@ -1141,6 +1141,62 @@ fn truncate_list_nulls<O: OffsetSizeTrait>(
11411141 Ok ( list. clone ( ) )
11421142}
11431143
1144+ /// If `array` is a contiguos list, returns a new array of the same length as it's inner values
1145+ /// where each value is the 1-based index of the sublist it's contained. Example:
1146+ ///
1147+ /// `[[1], [2, 3], [4, 5, 6]] => [1, 2, 2, 3, 3, 3]`
1148+ ///
1149+ /// If it's not a contiguos list, return an error
1150+ pub fn list_values_row_number ( array : & dyn Array ) -> Result < ArrayRef > {
1151+ match array. data_type ( ) {
1152+ DataType :: List ( _) => Ok ( Arc :: new ( list_array_values_row_number :: < Int32Type > (
1153+ array. as_list ( ) . offsets ( ) ,
1154+ ) ) ) ,
1155+ DataType :: LargeList ( _) => Ok ( Arc :: new (
1156+ list_array_values_row_number :: < Int64Type > ( array. as_list ( ) . offsets ( ) ) ,
1157+ ) ) ,
1158+ DataType :: FixedSizeList ( _, _) => {
1159+ let fixed_size_list = array. as_fixed_size_list ( ) ;
1160+
1161+ Ok ( Arc :: new ( fsl_values_row_number (
1162+ fixed_size_list. value_length ( ) ,
1163+ fixed_size_list. len ( ) ,
1164+ ) ?) )
1165+ }
1166+ other => _exec_err ! ( "expected list, got {other}" ) ,
1167+ }
1168+ }
1169+
1170+ /// [0, 2, 2, 5, 6] -> [0, 0, 2, 2, 2, 3]
1171+ fn list_array_values_row_number < T : ArrowPrimitiveType > (
1172+ offsets : & OffsetBuffer < T :: Native > ,
1173+ ) -> PrimitiveArray < T > {
1174+ let mut rows_number = Vec :: with_capacity (
1175+ offsets[ offsets. len ( ) - 1 ] . to_usize ( ) . unwrap ( ) - offsets[ 0 ] . to_usize ( ) . unwrap ( ) ,
1176+ ) ;
1177+
1178+ for ( i, len) in offsets. lengths ( ) . enumerate ( ) {
1179+ rows_number. extend ( repeat_n ( T :: Native :: usize_as ( i) , len) ) ;
1180+ }
1181+
1182+ PrimitiveArray :: new ( rows_number. into ( ) , None )
1183+ }
1184+
1185+ /// (2, 3) -> [0, 0, 1, 1, 2, 2]
1186+ fn fsl_values_row_number ( list_size : i32 , array_len : usize ) -> Result < Int32Array > {
1187+ let list_size = list_size. to_usize ( ) . ok_or_else ( || {
1188+ _exec_datafusion_err ! ( "fsl_values_index: invalid list_size {list_size}" )
1189+ } ) ?;
1190+
1191+ let mut rows_number = Vec :: with_capacity ( list_size * array_len) ;
1192+
1193+ for i in 0 ..array_len {
1194+ rows_number. extend ( repeat_n ( i as i32 , list_size) ) ;
1195+ }
1196+
1197+ Ok ( PrimitiveArray :: new ( rows_number. into ( ) , None ) )
1198+ }
1199+
11441200#[ cfg( test) ]
11451201mod tests {
11461202 use std:: sync:: Arc ;
@@ -1614,4 +1670,77 @@ mod tests {
16141670 expected. as_fixed_size_list( ) . values( )
16151671 ) ;
16161672 }
1673+
1674+ #[ test]
1675+ fn test_list_array_values_row_number ( ) {
1676+ assert_eq ! (
1677+ list_array_values_row_number:: <Int32Type >( & OffsetBuffer :: from_lengths( [
1678+ 1 , 3 , 0 , 2 ,
1679+ ] ) ) ,
1680+ Int32Array :: from( vec![ 0 , 1 , 1 , 1 , 3 , 3 ] )
1681+ ) ;
1682+
1683+ assert_eq ! (
1684+ list_array_values_row_number:: <Int32Type >( & OffsetBuffer :: from_lengths( [ ] ) ) ,
1685+ Int32Array :: new_null( 0 )
1686+ ) ;
1687+
1688+ assert_eq ! (
1689+ list_array_values_row_number:: <Int32Type >( & OffsetBuffer :: from_lengths( [ 0 ] ) ) ,
1690+ Int32Array :: new_null( 0 )
1691+ ) ;
1692+
1693+ assert_eq ! (
1694+ list_array_values_row_number:: <Int32Type >( & OffsetBuffer :: from_lengths( [
1695+ 0 , 0
1696+ ] ) ) ,
1697+ Int32Array :: new_null( 0 )
1698+ ) ;
1699+
1700+ assert_eq ! (
1701+ list_array_values_row_number:: <Int32Type >( & OffsetBuffer :: from_lengths( [ 1 ] ) ) ,
1702+ Int32Array :: from( vec![ 0 ] )
1703+ ) ;
1704+
1705+ assert_eq ! (
1706+ list_array_values_row_number:: <Int32Type >( & OffsetBuffer :: from_lengths( [ 2 ] ) ) ,
1707+ Int32Array :: from( vec![ 0 , 0 ] )
1708+ ) ;
1709+ }
1710+
1711+ #[ test]
1712+ fn test_fsl_values_row_number ( ) {
1713+ assert_eq ! (
1714+ fsl_values_row_number( 2 , 3 ) . unwrap( ) ,
1715+ Int32Array :: from( vec![ 0 , 0 , 1 , 1 , 2 , 2 ] )
1716+ ) ;
1717+
1718+ assert_eq ! (
1719+ fsl_values_row_number( 1 , 3 ) . unwrap( ) ,
1720+ Int32Array :: from( vec![ 0 , 1 , 2 ] )
1721+ ) ;
1722+
1723+ assert_eq ! (
1724+ fsl_values_row_number( 2 , 1 ) . unwrap( ) ,
1725+ Int32Array :: from( vec![ 0 , 0 ] )
1726+ ) ;
1727+
1728+ assert_eq ! (
1729+ fsl_values_row_number( 2 , 0 ) . unwrap( ) ,
1730+ Int32Array :: new_null( 0 ) ,
1731+ ) ;
1732+
1733+ assert_eq ! (
1734+ fsl_values_row_number( 0 , 2 ) . unwrap( ) ,
1735+ Int32Array :: new_null( 0 ) ,
1736+ ) ;
1737+
1738+ assert_eq ! (
1739+ fsl_values_row_number( 0 , 0 ) . unwrap( ) ,
1740+ Int32Array :: new_null( 0 ) ,
1741+ ) ;
1742+
1743+ fsl_values_row_number ( -1 , 2 ) . unwrap_err ( ) ;
1744+ fsl_values_row_number ( -1 , 0 ) . unwrap_err ( ) ;
1745+ }
16171746}
0 commit comments