@@ -27,9 +27,9 @@ use crate::file_groups::FileGroup;
2727
2828use arrow:: array:: RecordBatch ;
2929use arrow:: compute:: SortColumn ;
30- use arrow:: datatypes:: SchemaRef ;
30+ use arrow:: datatypes:: { DataType , SchemaRef } ;
3131use arrow:: row:: { Row , Rows } ;
32- use datafusion_common:: stats:: NdvFallback ;
32+ use datafusion_common:: stats:: { NdvFallback , Precision } ;
3333use datafusion_common:: {
3434 DataFusionError , Result , ScalarValue , plan_datafusion_err, plan_err,
3535} ;
@@ -83,17 +83,28 @@ impl MinMaxStatistics {
8383 } ;
8484
8585 // Helper function to get min/max statistics for a given column of projected_schema
86- let get_min_max = |i : usize | -> Result < ( Vec < ScalarValue > , Vec < ScalarValue > ) > {
86+ let get_min_max = |i : usize ,
87+ data_type : & DataType |
88+ -> Result < ( Vec < ScalarValue > , Vec < ScalarValue > ) > {
8789 Ok ( statistics_and_partition_values
8890 . iter ( )
8991 . map ( |( s, pv) | {
9092 if i < s. column_statistics . len ( ) {
91- s. column_statistics [ i]
92- . min_value
93- . get_value ( )
94- . cloned ( )
95- . zip ( s. column_statistics [ i] . max_value . get_value ( ) . cloned ( ) )
96- . ok_or_else ( || plan_datafusion_err ! ( "statistics not found" ) )
93+ let col_stats = & s. column_statistics [ i] ;
94+ match (
95+ col_stats. min_value . get_value ( ) ,
96+ col_stats. max_value . get_value ( ) ,
97+ ) {
98+ ( Some ( min) , Some ( max) ) => Ok ( ( min. clone ( ) , max. clone ( ) ) ) ,
99+ // An all-null column, e.g. a null partition value, has a null min and max.
100+ _ if matches ! ( s. num_rows, Precision :: Exact ( _) )
101+ && col_stats. null_count == s. num_rows =>
102+ {
103+ let null = ScalarValue :: try_new_null ( data_type) ?;
104+ Ok ( ( null. clone ( ) , null) )
105+ }
106+ _ => Err ( plan_datafusion_err ! ( "statistics not found" ) ) ,
107+ }
97108 } else if let Some ( partition_value) =
98109 pv. get ( i - s. column_statistics . len ( ) )
99110 {
@@ -143,7 +154,8 @@ impl MinMaxStatistics {
143154 . map ( |p| p[ c. index ( ) ] )
144155 . unwrap_or_else ( || c. index ( ) ) ;
145156
146- let ( min, max) = get_min_max ( i) . map_err ( |e| {
157+ let data_type = projected_schema. field ( c. index ( ) ) . data_type ( ) ;
158+ let ( min, max) = get_min_max ( i, data_type) . map_err ( |e| {
147159 e. context ( format ! ( "get min/max for column: '{}'" , c. name( ) ) )
148160 } ) ?;
149161 Ok ( (
@@ -384,8 +396,7 @@ mod tests {
384396 use super :: * ;
385397 use crate :: PartitionedFile ;
386398 use crate :: file_groups:: FileGroup ;
387- use arrow:: datatypes:: { DataType , Field , Schema } ;
388- use datafusion_common:: stats:: Precision ;
399+ use arrow:: datatypes:: { Field , Schema } ;
389400 use datafusion_physical_plan:: ColumnStatistics ;
390401
391402 fn test_schema ( ) -> SchemaRef {
@@ -502,4 +513,56 @@ mod tests {
502513 "unexpected error: {err:?}"
503514 ) ;
504515 }
516+
517+ #[ test]
518+ fn min_max_statistics_null_partition_value ( ) -> Result < ( ) > {
519+ let schema = Arc :: new ( Schema :: new ( vec ! [
520+ Field :: new( "a" , DataType :: Int64 , true ) ,
521+ Field :: new( "d" , DataType :: Date32 , true ) ,
522+ ] ) ) ;
523+ let sort_order =
524+ [ PhysicalSortExpr :: new_default ( Arc :: new ( Column :: new ( "d" , 1 ) ) ) ] . into ( ) ;
525+ let file = |path : & str , d : Option < i32 > | {
526+ let mut pf = PartitionedFile :: new ( path, 1 ) ;
527+ pf. partition_values = vec ! [ ScalarValue :: Date32 ( d) ] ;
528+ pf. with_statistics ( Arc :: new (
529+ Statistics :: new_unknown ( & test_schema ( ) )
530+ . with_num_rows ( Precision :: Exact ( 1 ) ) ,
531+ ) )
532+ } ;
533+ let files = [ file ( "d=1/f.parquet" , Some ( 1 ) ) , file ( "null/f.parquet" , None ) ] ;
534+
535+ let stats =
536+ MinMaxStatistics :: new_from_files ( & sort_order, & schema, None , files. iter ( ) ) ?;
537+
538+ let sorted = stats. min_values_sorted ( ) ;
539+ assert_eq ! ( sorted. iter( ) . map( |( i, _) | * i) . collect:: <Vec <_>>( ) , [ 1 , 0 ] ) ;
540+ assert ! ( !stats. is_sorted( ) ) ;
541+
542+ Ok ( ( ) )
543+ }
544+
545+ #[ test]
546+ fn min_max_statistics_null_partition_value_unknown_num_rows ( ) {
547+ let schema = Arc :: new ( Schema :: new ( vec ! [
548+ Field :: new( "a" , DataType :: Int64 , true ) ,
549+ Field :: new( "d" , DataType :: Date32 , true ) ,
550+ ] ) ) ;
551+ let sort_order =
552+ [ PhysicalSortExpr :: new_default ( Arc :: new ( Column :: new ( "d" , 1 ) ) ) ] . into ( ) ;
553+ let mut pf = PartitionedFile :: new ( "null/f.parquet" , 1 ) ;
554+ pf. partition_values = vec ! [ ScalarValue :: Date32 ( None ) ] ;
555+ let files =
556+ [ pf. with_statistics ( Arc :: new ( Statistics :: new_unknown ( & test_schema ( ) ) ) ) ] ;
557+
558+ let Err ( err) =
559+ MinMaxStatistics :: new_from_files ( & sort_order, & schema, None , files. iter ( ) )
560+ else {
561+ panic ! ( "expected missing statistics error" )
562+ } ;
563+ assert ! (
564+ err. to_string( ) . contains( "statistics not found" ) ,
565+ "unexpected error: {err:?}"
566+ ) ;
567+ }
505568}
0 commit comments