1818use std:: mem;
1919use std:: sync:: Arc ;
2020
21+ use arrow_schema:: DataType ;
22+ use datafusion_expr:: interval_arithmetic:: Interval ;
23+ use datafusion_expr:: statistics:: { new_generic_from_binary_op, ColumnStatisticsNew , Distribution , StatisticsNew } ;
24+ use datafusion_expr:: Operator ;
2125use futures:: { Stream , StreamExt } ;
2226
2327use datafusion_common:: stats:: Precision ;
@@ -35,11 +39,11 @@ use super::listing::PartitionedFile;
3539/// `ListingTable`. If it is false we only construct bare statistics and skip a potentially expensive
3640/// call to `multiunzip` for constructing file level summary statistics.
3741pub async fn get_statistics_with_limit (
38- all_files : impl Stream < Item = Result < ( PartitionedFile , Arc < Statistics > ) > > ,
42+ all_files : impl Stream < Item = Result < ( PartitionedFile , Arc < StatisticsNew > ) > > ,
3943 file_schema : SchemaRef ,
4044 limit : Option < usize > ,
4145 collect_stats : bool ,
42- ) -> Result < ( Vec < PartitionedFile > , Statistics ) > {
46+ ) -> Result < ( Vec < PartitionedFile > , StatisticsNew ) > {
4347 let mut result_files = vec ! [ ] ;
4448 // These statistics can be calculated as long as at least one file provides
4549 // useful information. If none of the files provides any information, then
@@ -48,9 +52,23 @@ pub async fn get_statistics_with_limit(
4852 // - zero for summations, and
4953 // - neutral element for extreme points.
5054 let size = file_schema. fields ( ) . len ( ) ;
51- let mut col_stats_set = vec ! [ ColumnStatistics :: default ( ) ; size] ;
52- let mut num_rows = Precision :: < usize > :: Absent ;
53- let mut total_byte_size = Precision :: < usize > :: Absent ;
55+ let mut col_stats_set = vec ! [ ColumnStatisticsNew :: new_unknown( ) ?; size] ;
56+ let mut num_rows = Distribution :: new_generic (
57+ ScalarValue :: Null ,
58+ ScalarValue :: Null ,
59+ ScalarValue :: Null ,
60+ Interval :: make_zero ( & DataType :: UInt64 ) ?,
61+ ) ?;
62+ let mut total_byte_size = Distribution :: new_generic (
63+ ScalarValue :: Null ,
64+ ScalarValue :: Null ,
65+ ScalarValue :: Null ,
66+ Interval :: make_zero ( & DataType :: UInt64 ) ?,
67+ ) ?;
68+ let limit = match limit {
69+ Some ( v) => ScalarValue :: UInt64 ( Some ( v as u64 ) ) ,
70+ None => ScalarValue :: UInt64 ( Some ( u64:: MAX ) ) ,
71+ } ;
5472
5573 // Fusing the stream allows us to call next safely even once it is finished.
5674 let mut all_files = Box :: pin ( all_files. fuse ( ) ) ;
@@ -77,10 +95,10 @@ pub async fn get_statistics_with_limit(
7795 // currently ignores tables that have no statistics regarding the
7896 // number of rows.
7997 let conservative_num_rows = match num_rows {
80- Precision :: Exact ( nr ) => nr ,
81- _ => usize :: MIN ,
98+ Distribution :: Uniform ( ref dist ) => dist . get_value ( ) . unwrap_or ( & ScalarValue :: UInt64 ( Some ( u64 :: MIN ) ) ) . clone ( ) ,
99+ _ => ScalarValue :: UInt64 ( Some ( u64 :: MIN ) ) ,
82100 } ;
83- if conservative_num_rows <= limit. unwrap_or ( usize :: MAX ) {
101+ if conservative_num_rows <= limit {
84102 while let Some ( current) = all_files. next ( ) . await {
85103 let ( mut file, file_stats) = current?;
86104 file. statistics = Some ( file_stats. as_ref ( ) . clone ( ) ) ;
@@ -103,7 +121,7 @@ pub async fn get_statistics_with_limit(
103121 . iter ( )
104122 . zip ( col_stats_set. iter_mut ( ) )
105123 {
106- let ColumnStatistics {
124+ let ColumnStatisticsNew {
107125 null_count : file_nc,
108126 max_value : file_max,
109127 min_value : file_min,
@@ -114,23 +132,21 @@ pub async fn get_statistics_with_limit(
114132 col_stats. null_count = add_row_stats ( * file_nc, col_stats. null_count ) ;
115133 set_max_if_greater ( file_max, & mut col_stats. max_value ) ;
116134 set_min_if_lesser ( file_min, & mut col_stats. min_value ) ;
117- col_stats. sum_value = file_sum . add ( & col_stats. sum_value ) ;
135+ col_stats. sum_value = new_generic_from_binary_op ( & Operator :: Plus , file_sum , & col_stats. sum_value ) ? ;
118136 }
119137
120138 // If the number of rows exceeds the limit, we can stop processing
121139 // files. This only applies when we know the number of rows. It also
122140 // currently ignores tables that have no statistics regarding the
123141 // number of rows.
124- if num_rows. get_value ( ) . unwrap_or ( & usize:: MIN )
125- > & limit. unwrap_or ( usize:: MAX )
126- {
142+ if num_rows. get_value ( ) . unwrap_or ( & ScalarValue :: UInt64 ( Some ( u64:: MIN ) ) ) > & limit {
127143 break ;
128144 }
129145 }
130146 }
131147 } ;
132148
133- let mut statistics = Statistics {
149+ let mut statistics = StatisticsNew {
134150 num_rows,
135151 total_byte_size,
136152 column_statistics : col_stats_set,
@@ -139,81 +155,84 @@ pub async fn get_statistics_with_limit(
139155 // If we still have files in the stream, it means that the limit kicked
140156 // in, and the statistic could have been different had we processed the
141157 // files in a different order.
142- statistics = statistics. to_inexact ( )
158+ statistics = statistics. to_inexact ( ) ?
143159 }
144160
145161 Ok ( ( result_files, statistics) )
146162}
147163
148164fn add_row_stats (
149- file_num_rows : Precision < usize > ,
150- num_rows : Precision < usize > ,
151- ) -> Precision < usize > {
152- match ( file_num_rows, & num_rows) {
153- ( Precision :: Absent , _) => num_rows. to_inexact ( ) ,
154- ( lhs, Precision :: Absent ) => lhs. to_inexact ( ) ,
155- ( lhs, rhs) => lhs. add ( rhs) ,
156- }
165+ file_num_rows : Distribution ,
166+ num_row : Distribution ,
167+ ) -> Distribution {
168+ // match (file_num_rows, &num_rows) {
169+ // (Precision::Absent, _) => num_rows.to_inexact(),
170+ // (lhs, Precision::Absent) => lhs.to_inexact(),
171+ // (lhs, rhs) => lhs.add(rhs),
172+ // }
173+ todo ! ( )
157174}
158175
159176/// If the given value is numerically greater than the original maximum value,
160177/// return the new maximum value with appropriate exactness information.
161178fn set_max_if_greater (
162- max_nominee : & Precision < ScalarValue > ,
163- max_value : & mut Precision < ScalarValue > ,
179+ max_nominee : & Distribution ,
180+ max_value : & mut Distribution ,
164181) {
165- match ( & max_value, max_nominee) {
166- ( Precision :: Exact ( val1) , Precision :: Exact ( val2) ) if val1 < val2 => {
167- * max_value = max_nominee. clone ( ) ;
168- }
169- ( Precision :: Exact ( val1) , Precision :: Inexact ( val2) )
170- | ( Precision :: Inexact ( val1) , Precision :: Inexact ( val2) )
171- | ( Precision :: Inexact ( val1) , Precision :: Exact ( val2) )
172- if val1 < val2 =>
173- {
174- * max_value = max_nominee. clone ( ) . to_inexact ( ) ;
175- }
176- ( Precision :: Exact ( _) , Precision :: Absent ) => {
177- let exact_max = mem:: take ( max_value) ;
178- * max_value = exact_max. to_inexact ( ) ;
179- }
180- ( Precision :: Absent , Precision :: Exact ( _) ) => {
181- * max_value = max_nominee. clone ( ) . to_inexact ( ) ;
182- }
183- ( Precision :: Absent , Precision :: Inexact ( _) ) => {
184- * max_value = max_nominee. clone ( ) ;
185- }
186- _ => { }
187- }
182+ // match (&max_value, max_nominee) {
183+ // (Precision::Exact(val1), Precision::Exact(val2)) if val1 < val2 => {
184+ // *max_value = max_nominee.clone();
185+ // }
186+ // (Precision::Exact(val1), Precision::Inexact(val2))
187+ // | (Precision::Inexact(val1), Precision::Inexact(val2))
188+ // | (Precision::Inexact(val1), Precision::Exact(val2))
189+ // if val1 < val2 =>
190+ // {
191+ // *max_value = max_nominee.clone().to_inexact();
192+ // }
193+ // (Precision::Exact(_), Precision::Absent) => {
194+ // let exact_max = mem::take(max_value);
195+ // *max_value = exact_max.to_inexact();
196+ // }
197+ // (Precision::Absent, Precision::Exact(_)) => {
198+ // *max_value = max_nominee.clone().to_inexact();
199+ // }
200+ // (Precision::Absent, Precision::Inexact(_)) => {
201+ // *max_value = max_nominee.clone();
202+ // }
203+ // _ => {}
204+ // }
205+ todo ! ( )
188206}
189207
190208/// If the given value is numerically lesser than the original minimum value,
191209/// return the new minimum value with appropriate exactness information.
192210fn set_min_if_lesser (
193- min_nominee : & Precision < ScalarValue > ,
194- min_value : & mut Precision < ScalarValue > ,
211+ min_nominee : & Distribution ,
212+ min_value : & mut Distribution ,
195213) {
196- match ( & min_value, min_nominee) {
197- ( Precision :: Exact ( val1) , Precision :: Exact ( val2) ) if val1 > val2 => {
198- * min_value = min_nominee. clone ( ) ;
199- }
200- ( Precision :: Exact ( val1) , Precision :: Inexact ( val2) )
201- | ( Precision :: Inexact ( val1) , Precision :: Inexact ( val2) )
202- | ( Precision :: Inexact ( val1) , Precision :: Exact ( val2) )
203- if val1 > val2 =>
204- {
205- * min_value = min_nominee. clone ( ) . to_inexact ( ) ;
206- }
207- ( Precision :: Exact ( _) , Precision :: Absent ) => {
208- let exact_min = mem:: take ( min_value) ;
209- * min_value = exact_min. to_inexact ( ) ;
210- }
211- ( Precision :: Absent , Precision :: Exact ( _) ) => {
212- * min_value = min_nominee. clone ( ) . to_inexact ( ) ;
213- }
214- ( Precision :: Absent , Precision :: Inexact ( _) ) => {
215- * min_value = min_nominee. clone ( ) ;
216- }
217- _ => { }
218- }
214+ // match (&min_value, min_nominee) {
215+ // (Precision::Exact(val1), Precision::Exact(val2)) if val1 > val2 => {
216+ // *min_value = min_nominee.clone();
217+ // }
218+ // (Precision::Exact(val1), Precision::Inexact(val2))
219+ // | (Precision::Inexact(val1), Precision::Inexact(val2))
220+ // | (Precision::Inexact(val1), Precision::Exact(val2))
221+ // if val1 > val2 =>
222+ // {
223+ // *min_value = min_nominee.clone().to_inexact();
224+ // }
225+ // (Precision::Exact(_), Precision::Absent) => {
226+ // let exact_min = mem::take(min_value);
227+ // *min_value = exact_min.to_inexact();
228+ // }
229+ // (Precision::Absent, Precision::Exact(_)) => {
230+ // *min_value = min_nominee.clone().to_inexact();
231+ // }
232+ // (Precision::Absent, Precision::Inexact(_)) => {
233+ // *min_value = min_nominee.clone();
234+ // }
235+ // _ => {}
236+ // }
237+ todo ! ( )
219238}
0 commit comments