@@ -14,6 +14,7 @@ use datafusion_common::ColumnStatistics;
1414use datafusion_common:: DataFusionError ;
1515use datafusion_common:: GetExt ;
1616use datafusion_common:: Result as DFResult ;
17+ use datafusion_common:: ScalarValue as DFScalarValue ;
1718use datafusion_common:: Statistics ;
1819use datafusion_common:: config:: ConfigField ;
1920use datafusion_common:: config_namespace;
@@ -60,13 +61,15 @@ use vortex::file::VORTEX_FILE_EXTENSION;
6061use vortex:: io:: object_store:: ObjectStoreReadAt ;
6162use vortex:: io:: session:: RuntimeSessionExt ;
6263use vortex:: scalar:: Scalar ;
64+ use vortex:: scalar:: ScalarValue as VortexScalarValue ;
6365use vortex:: session:: VortexSession ;
6466
6567use super :: cache:: CachedVortexMetadata ;
6668use super :: sink:: VortexSink ;
6769use super :: source:: VortexSource ;
6870use crate :: PrecisionExt as _;
6971use crate :: convert:: TryToDataFusion ;
72+ use crate :: convert:: stats:: is_constant_to_distinct_count;
7073
7174const DEFAULT_FOOTER_INITIAL_READ_SIZE_BYTES : usize = MAX_POSTSCRIPT_SIZE as usize + EOF_SIZE ;
7275
@@ -500,11 +503,13 @@ impl FileFormat for VortexFormat {
500503 . vortex_expect ( "Row count overflow" ) ,
501504 ) ,
502505 total_byte_size : Precision :: Absent ,
503- column_statistics : vec ! [ ColumnStatistics :: default ( ) ; struct_dtype. nfields( ) ] ,
506+ column_statistics : vec ! [
507+ ColumnStatistics :: default ( ) ;
508+ table_schema. fields( ) . len( )
509+ ] ,
504510 } ) ;
505511 } ;
506512
507- let mut sum_of_column_byte_sizes = stats:: Precision :: exact ( 0_usize ) ;
508513 let mut column_statistics = Vec :: with_capacity ( table_schema. fields ( ) . len ( ) ) ;
509514
510515 for field in table_schema. fields ( ) . iter ( ) {
@@ -518,55 +523,22 @@ impl FileFormat for VortexFormat {
518523 let ( stats_set, stats_dtype) = file_stats. get ( col_idx) ;
519524
520525 // Update the total size in bytes.
521- let column_size = stats_set
522- . get_as :: < usize > ( Stat :: UncompressedSizeInBytes , & PType :: U64 . into ( ) )
523- . unwrap_or_else ( || stats:: Precision :: inexact ( 0_usize ) ) ;
524- sum_of_column_byte_sizes = sum_of_column_byte_sizes
525- . zip ( column_size)
526- . map ( |( acc, size) | acc + size) ;
527-
528- // TODO(connor): There's a lot that can go wrong here, should probably handle this
529- // more gracefully...
530- // Find the min statistic.
531- let min = stats_set. get ( Stat :: Min ) . and_then ( |pstat_val| {
532- pstat_val
533- . map ( |stat_val| {
534- // Because of DataFusion's Schema evolution, it is possible that the
535- // type of the min/max stat has changed. Thus we construct the stat as
536- // the file datatype first and only then do we cast accordingly.
537- Scalar :: try_new (
538- Stat :: Min
539- . dtype ( stats_dtype)
540- . vortex_expect ( "must have a valid dtype" ) ,
541- Some ( stat_val) ,
542- )
543- . vortex_expect ( "`Stat::Min` somehow had an incompatible `DType`" )
544- . cast ( & DType :: from_arrow ( field. as_ref ( ) ) )
545- . vortex_expect ( "Unable to cast to target type that DataFusion wants" )
546- . try_to_df ( )
547- . ok ( )
548- } )
549- . transpose ( )
550- } ) ;
551-
552- // Find the max statistic.
553- let max = stats_set. get ( Stat :: Max ) . and_then ( |pstat_val| {
554- pstat_val
555- . map ( |stat_val| {
556- Scalar :: try_new (
557- Stat :: Max
558- . dtype ( stats_dtype)
559- . vortex_expect ( "must have a valid dtype" ) ,
560- Some ( stat_val) ,
561- )
562- . vortex_expect ( "`Stat::Max` somehow had an incompatible `DType`" )
563- . cast ( & DType :: from_arrow ( field. as_ref ( ) ) )
564- . vortex_expect ( "Unable to cast to target type that DataFusion wants" )
565- . try_to_df ( )
566- . ok ( )
567- } )
568- . transpose ( )
569- } ) ;
526+ let column_size =
527+ stats_set. get_as :: < usize > ( Stat :: UncompressedSizeInBytes , & PType :: U64 . into ( ) ) ;
528+
529+ let target_dtype = DType :: from_arrow ( field. as_ref ( ) ) ;
530+ let min = scalar_stat_to_df (
531+ Stat :: Min ,
532+ stats_set. get ( Stat :: Min ) ,
533+ stats_dtype,
534+ & target_dtype,
535+ ) ;
536+ let max = scalar_stat_to_df (
537+ Stat :: Max ,
538+ stats_set. get ( Stat :: Max ) ,
539+ stats_dtype,
540+ & target_dtype,
541+ ) ;
570542
571543 let null_count = stats_set. get_as :: < usize > ( Stat :: NullCount , & PType :: U64 . into ( ) ) ;
572544
@@ -575,16 +547,19 @@ impl FileFormat for VortexFormat {
575547 min_value : min. to_df ( ) ,
576548 max_value : max. to_df ( ) ,
577549 sum_value : Precision :: Absent ,
578- distinct_count : stats_set
579- . get_as :: < bool > ( Stat :: IsConstant , & DType :: Bool ( Nullability :: NonNullable ) )
580- . and_then ( |is_constant| is_constant. as_exact ( ) . map ( |_| Precision :: Exact ( 1 ) ) )
581- . unwrap_or ( Precision :: Absent ) ,
582- // TODO(connor): Is this correct?
550+ distinct_count : is_constant_to_distinct_count (
551+ stats_set. get_as :: < bool > (
552+ Stat :: IsConstant ,
553+ & DType :: Bool ( Nullability :: NonNullable ) ,
554+ ) ,
555+ ) ,
583556 byte_size : column_size. to_df ( ) ,
584557 } )
585558 }
586559
587- let total_byte_size = sum_of_column_byte_sizes. to_df ( ) ;
560+ let total_byte_size = column_statistics
561+ . iter ( )
562+ . fold ( Precision :: Exact ( 0 ) , |acc, cs| acc. add ( & cs. byte_size ) ) ;
588563
589564 Ok ( Statistics {
590565 num_rows : Precision :: Exact (
@@ -651,6 +626,23 @@ impl FileFormat for VortexFormat {
651626 }
652627}
653628
629+ fn scalar_stat_to_df (
630+ stat : Stat ,
631+ value : Option < stats:: Precision < VortexScalarValue > > ,
632+ stats_dtype : & DType ,
633+ target_dtype : & DType ,
634+ ) -> Option < stats:: Precision < DFScalarValue > > {
635+ let stat_dtype = stat. dtype ( stats_dtype) ?;
636+
637+ value?
638+ . try_map ( |stat_value| {
639+ Scalar :: try_new ( stat_dtype, Some ( stat_value) ) ?
640+ . cast ( target_dtype) ?
641+ . try_to_df ( )
642+ } )
643+ . ok ( )
644+ }
645+
654646#[ cfg( test) ]
655647mod tests {
656648
0 commit comments