1717use std:: sync:: Arc ;
1818
1919use arrow_schema:: DataType ;
20- use datafusion_expr:: statistics:: { new_generic_from_binary_op, ColumnStatisticsNew , ProbabilityDistribution , TableStatistics } ;
20+ use datafusion_expr:: statistics:: { new_generic_from_binary_op, ColumnStatistics , ProbabilityDistribution , TableStatistics } ;
2121use datafusion_expr:: Operator ;
2222use futures:: { Stream , StreamExt } ;
2323use datafusion_common:: ScalarValue ;
@@ -46,9 +46,9 @@ pub async fn get_statistics_with_limit(
4646 // - zero for summations, and
4747 // - neutral element for extreme points.
4848 let size = file_schema. fields ( ) . len ( ) ;
49- let mut col_stats_set = vec ! [ ColumnStatisticsNew :: new_unknown( ) ?; size] ;
50- let mut num_rows = ProbabilityDistribution :: new_generic_unknown ( & DataType :: UInt64 ) ?;
51- let mut total_byte_size = ProbabilityDistribution :: new_generic_unknown ( & DataType :: UInt64 ) ?;
49+ let mut col_stats_set = vec ! [ ColumnStatistics :: new_unknown( ) ?; size] ;
50+ let mut num_rows = ProbabilityDistribution :: new_unknown ( & DataType :: UInt64 ) ?;
51+ let mut total_byte_size = ProbabilityDistribution :: new_unknown ( & DataType :: UInt64 ) ?;
5252 let limit_value = limit. map ( |v| v as _ ) . unwrap_or ( u64:: MAX ) ;
5353 let limit = ScalarValue :: UInt64 ( Some ( limit_value) ) ;
5454
@@ -76,12 +76,8 @@ pub async fn get_statistics_with_limit(
7676 // files. This only applies when we know the number of rows. It also
7777 // currently ignores tables that have no statistics regarding the
7878 // number of rows.
79- let conservative_num_rows = if num_rows. as_ref ( ) . is_null ( ) {
80- ScalarValue :: UInt64 ( Some ( u64:: MIN ) )
81- } else {
82- num_rows. as_ref ( ) . clone ( )
83- } ;
84- if conservative_num_rows <= limit {
79+ let conservative_num_rows = num_rows. get_value ( ) . unwrap_or ( & ScalarValue :: UInt64 ( Some ( u64:: MIN ) ) ) ;
80+ if conservative_num_rows <= & limit {
8581 while let Some ( current) = all_files. next ( ) . await {
8682 let ( mut file, file_stats) = current?;
8783 file. statistics = Some ( file_stats. as_ref ( ) . clone ( ) ) ;
@@ -94,25 +90,25 @@ pub async fn get_statistics_with_limit(
9490 // counts across all the files in question. If any file does not
9591 // provide any information or provides an inexact value, we demote
9692 // the statistic precision to inexact.
97- num_rows = add_row_stats ( file_stats. num_rows . clone ( ) , num_rows) ?;
93+ num_rows = new_generic_from_binary_op ( & Operator :: Plus , & file_stats. num_rows , & num_rows) ?;
9894
9995 total_byte_size =
100- add_row_stats ( file_stats. total_byte_size . clone ( ) , total_byte_size) ?;
96+ new_generic_from_binary_op ( & Operator :: Plus , & file_stats. total_byte_size , & total_byte_size) ?;
10197
10298 for ( file_col_stats, col_stats) in file_stats
10399 . column_statistics
104100 . iter ( )
105101 . zip ( col_stats_set. iter_mut ( ) )
106102 {
107- let ColumnStatisticsNew {
103+ let ColumnStatistics {
108104 null_count : file_nc,
109105 max_value : file_max,
110106 min_value : file_min,
111107 sum_value : file_sum,
112108 distinct_count : _,
113109 } = file_col_stats;
114110
115- col_stats. null_count = add_row_stats ( file_nc. clone ( ) , col_stats. null_count . clone ( ) ) ?;
111+ col_stats. null_count = new_generic_from_binary_op ( & Operator :: Plus , & file_nc, & col_stats. null_count ) ?;
116112 set_max_if_greater ( file_max, & mut col_stats. max_value ) ;
117113 set_min_if_lesser ( file_min, & mut col_stats. min_value ) ;
118114 col_stats. sum_value = new_generic_from_binary_op ( & Operator :: Plus , file_sum, & col_stats. sum_value ) ?;
@@ -122,11 +118,7 @@ pub async fn get_statistics_with_limit(
122118 // files. This only applies when we know the number of rows. It also
123119 // currently ignores tables that have no statistics regarding the
124120 // number of rows.
125- let num_rows = if num_rows. as_ref ( ) . is_null ( ) {
126- & ScalarValue :: UInt64 ( Some ( u64:: MIN ) )
127- } else {
128- num_rows. as_ref ( )
129- } ;
121+ let num_rows = num_rows. get_value ( ) . unwrap_or ( & ScalarValue :: UInt64 ( Some ( u64:: MIN ) ) ) ;
130122 if num_rows > & limit {
131123 break ;
132124 }
@@ -149,16 +141,6 @@ pub async fn get_statistics_with_limit(
149141 Ok ( ( result_files, statistics) )
150142}
151143
152- fn add_row_stats (
153- file_num_rows : ProbabilityDistribution ,
154- num_row : ProbabilityDistribution ,
155- ) -> Result < ProbabilityDistribution > {
156- match ( file_num_rows, & num_row) {
157- ( lhs, rhs) => new_generic_from_binary_op ( & Operator :: Plus , & lhs, & rhs) ,
158- _ => unimplemented ! ( ) ,
159- }
160- }
161-
162144/// If the given value is numerically greater than the original maximum value,
163145/// return the new maximum value with appropriate exactness information.
164146fn set_max_if_greater (
@@ -188,7 +170,9 @@ fn set_max_if_greater(
188170 // }
189171 // _ => {}
190172 // }
191- todo ! ( )
173+ if max_nominee. get_value ( ) . unwrap_or ( & ScalarValue :: Null ) < max_value. get_value ( ) . unwrap_or ( & ScalarValue :: Null ) {
174+ * max_value = max_nominee. clone ( ) ;
175+ }
192176}
193177
194178/// If the given value is numerically lesser than the original minimum value,
@@ -220,5 +204,7 @@ fn set_min_if_lesser(
220204 // }
221205 // _ => {}
222206 // }
223- todo ! ( )
207+ if min_nominee. get_value ( ) . unwrap_or ( & ScalarValue :: Null ) > min_value. get_value ( ) . unwrap_or ( & ScalarValue :: Null ) {
208+ * min_value = min_nominee. clone ( ) ;
209+ }
224210}
0 commit comments