@@ -26,10 +26,10 @@ use crate::PartitionedFile;
2626use crate :: file_groups:: FileGroup ;
2727
2828use arrow:: array:: RecordBatch ;
29- use arrow:: compute:: SortColumn ;
30- use arrow:: datatypes:: SchemaRef ;
29+ use arrow:: compute:: { SortColumn , SortOptions } ;
30+ use arrow:: datatypes:: { Field , SchemaRef } ;
3131use arrow:: row:: { Row , Rows } ;
32- use datafusion_common:: stats:: NdvFallback ;
32+ use datafusion_common:: stats:: { NdvFallback , Precision } ;
3333use datafusion_common:: {
3434 DataFusionError , Result , ScalarValue , plan_datafusion_err, plan_err,
3535} ;
@@ -83,16 +83,46 @@ impl MinMaxStatistics {
8383 } ;
8484
8585 // Helper function to get min/max statistics for a given column of projected_schema
86- let get_min_max = |i : usize | -> Result < ( Vec < ScalarValue > , Vec < ScalarValue > ) > {
86+ let get_min_max = |i : usize ,
87+ field : & Field ,
88+ options : SortOptions |
89+ -> Result < ( Vec < ScalarValue > , Vec < ScalarValue > ) > {
90+ let null = ScalarValue :: try_new_null ( field. data_type ( ) ) ?;
8791 Ok ( statistics_and_partition_values
8892 . iter ( )
8993 . map ( |( s, pv) | {
9094 if i < s. column_statistics . len ( ) {
91- s. column_statistics [ i]
92- . min_value
93- . get_value ( )
94- . cloned ( )
95- . zip ( s. column_statistics [ i] . max_value . get_value ( ) . cloned ( ) )
95+ let column_statistics = & s. column_statistics [ i] ;
96+ let all_null = matches ! (
97+ ( s. num_rows, column_statistics. null_count) ,
98+ ( Precision :: Exact ( rows) , Precision :: Exact ( nulls) )
99+ if rows > 0 && rows == nulls
100+ ) ;
101+ let may_have_nulls = match column_statistics. null_count {
102+ Precision :: Exact ( nulls) => nulls > 0 ,
103+ _ => field. is_nullable ( ) ,
104+ } ;
105+ // Self::new swaps the bounds for descending sort keys.
106+ let nulls_on_min = options. nulls_first != options. descending ;
107+ let min = if all_null || ( may_have_nulls && nulls_on_min) {
108+ Some ( null. clone ( ) )
109+ } else {
110+ column_statistics
111+ . min_value
112+ . get_value ( )
113+ . filter ( |value| !value. is_null ( ) )
114+ . cloned ( )
115+ } ;
116+ let max = if all_null || ( may_have_nulls && !nulls_on_min) {
117+ Some ( null. clone ( ) )
118+ } else {
119+ column_statistics
120+ . max_value
121+ . get_value ( )
122+ . filter ( |value| !value. is_null ( ) )
123+ . cloned ( )
124+ } ;
125+ min. zip ( max)
96126 . ok_or_else ( || plan_datafusion_err ! ( "statistics not found" ) )
97127 } else if let Some ( partition_value) =
98128 pv. get ( i - s. column_statistics . len ( ) )
@@ -135,17 +165,20 @@ impl MinMaxStatistics {
135165
136166 let ( min_values, max_values) : ( Vec < _ > , Vec < _ > ) = sort_columns
137167 . iter ( )
138- . map ( |c| {
168+ . zip ( projected_sort_order. iter ( ) )
169+ . map ( |( c, sort_expr) | {
139170 // Reverse the projection to get the index of the column in the full statistics
140171 // The file statistics contains _every_ column , but the sort column's index()
141172 // refers to the index in projected_schema
142173 let i = projection
143174 . map ( |p| p[ c. index ( ) ] )
144175 . unwrap_or_else ( || c. index ( ) ) ;
145176
146- let ( min, max) = get_min_max ( i) . map_err ( |e| {
147- e. context ( format ! ( "get min/max for column: '{}'" , c. name( ) ) )
148- } ) ?;
177+ let ( min, max) =
178+ get_min_max ( i, projected_schema. field ( c. index ( ) ) , sort_expr. options )
179+ . map_err ( |e| {
180+ e. context ( format ! ( "get min/max for column: '{}'" , c. name( ) ) )
181+ } ) ?;
149182 Ok ( (
150183 ScalarValue :: iter_to_array ( min) ?,
151184 ScalarValue :: iter_to_array ( max) ?,
0 commit comments