11// SPDX-License-Identifier: Apache-2.0
22// SPDX-FileCopyrightText: Copyright the Vortex contributors
33
4+ //! Collects chunk statistics and combines them into file statistics.
5+ //!
6+ //! Sums retain their accumulator state so overflow survives across chunks. Other statistics use
7+ //! intermediate columns, including truncation markers for variable-length bounds.
8+
49use std:: future;
510use std:: marker:: PhantomData ;
611use std:: sync:: Arc ;
@@ -72,7 +77,7 @@ pub fn accumulate_stats(
7277/// Accumulates write-time statistics for a single file column.
7378struct StatsAccumulator {
7479 builders : Vec < Box < dyn StatsArrayBuilder > > ,
75- /// Keep overflow in the partial state instead of treating its null final as an invalid row .
80+ /// Retains overflow that a nullable column of finalized chunk sums would discard .
7681 sum : Option < Accumulator < Sum > > ,
7782 length : usize ,
7883}
@@ -89,8 +94,9 @@ impl StatsAccumulator {
8994
9095 let sum = ( stats. contains ( & Stat :: Sum ) && Stat :: Sum . dtype ( dtype) . is_some ( ) ) . then ( || {
9196 Accumulator :: try_new ( Sum , NumericalAggregateOpts :: skip_nans ( ) , dtype. clone ( ) )
92- . vortex_expect ( "supported Sum dtype must have an accumulator " )
97+ . vortex_expect ( "StatsAccumulator::new checked that the input dtype supports Sum " )
9398 } ) ;
99+
94100 let builders = stats
95101 . iter ( )
96102 . filter ( |& & stat| stat != Stat :: Sum )
@@ -121,26 +127,31 @@ impl StatsAccumulator {
121127 array. dtype ( ) . clone ( ) ,
122128 ) ?;
123129 chunk_sum. accumulate ( array, ctx) ?;
130+
124131 let result = chunk_sum. final_scalar ( ) ?;
125132 let is_nan = result
126133 . as_primitive_opt ( )
127134 . is_some_and ( |value| value. is_nan ( ) ) ;
128135 if let Some ( value) = result. into_value ( ) {
129136 array. statistics ( ) . set ( Stat :: Sum , Precision :: Exact ( value) ) ;
130137 }
138+
131139 // The file writer's NaN-skipping policy also applies to chunk finals.
132140 if !is_nan {
133141 sum. merge_from ( & mut chunk_sum) ?;
134142 }
135143 }
144+
136145 for builder in & mut self . builders {
137146 if let Some ( value) = array. statistics ( ) . compute_stat ( builder. stat ( ) , ctx) ? {
138147 builder. append_scalar ( value. cast ( & value. dtype ( ) . as_nullable ( ) ) ?) ?;
139148 } else {
140149 builder. append_null ( ) ;
141150 }
142151 }
152+
143153 self . length += 1 ;
154+
144155 Ok ( ( ) )
145156 }
146157
@@ -175,6 +186,7 @@ impl StatsAccumulator {
175186 /// Returns an aggregated stats set for the table.
176187 fn as_stats_set ( & mut self , stats : & [ Stat ] , ctx : & mut ExecutionCtx ) -> VortexResult < StatsSet > {
177188 let mut stats_set = StatsSet :: default ( ) ;
189+
178190 if self . length != 0
179191 && stats. contains ( & Stat :: Sum )
180192 && let Some ( sum) = & self . sum
@@ -183,6 +195,7 @@ impl StatsAccumulator {
183195 // The legacy stats set cannot store a null, so an overflowed sum remains absent.
184196 stats_set. set ( Stat :: Sum , Precision :: Exact ( value) ) ;
185197 }
198+
186199 let Some ( stats_table) = self . as_array ( ctx) ? else {
187200 return Ok ( stats_set) ;
188201 } ;
@@ -578,6 +591,7 @@ mod tests {
578591 use vortex_array:: builders:: VarBinViewBuilder ;
579592 #[ cfg( target_pointer_width = "64" ) ]
580593 use vortex_array:: dtype:: DecimalDType ;
594+ #[ cfg( target_pointer_width = "64" ) ]
581595 use vortex_array:: expr:: stats:: StatsProvider ;
582596 #[ cfg( target_pointer_width = "64" ) ]
583597 use vortex_array:: scalar:: DecimalValue ;
@@ -586,6 +600,17 @@ mod tests {
586600
587601 use super :: * ;
588602
603+ fn accumulate_sum_stats ( dtype : & DType , chunks : & [ ArrayRef ] ) -> VortexResult < StatsSet > {
604+ let mut ctx = array_session ( ) . create_execution_ctx ( ) ;
605+ let mut acc = StatsAccumulator :: new ( dtype, & [ Stat :: Sum ] , 64 ) ;
606+
607+ for chunk in chunks {
608+ acc. push_chunk ( chunk, & mut ctx) ?;
609+ }
610+
611+ acc. as_stats_set ( & [ Stat :: Sum ] , & mut ctx)
612+ }
613+
589614 #[ rstest]
590615 #[ case( DType :: Utf8 ( Nullability :: NonNullable ) ) ]
591616 #[ case( DType :: Binary ( Nullability :: NonNullable ) ) ]
@@ -686,27 +711,26 @@ mod tests {
686711 }
687712
688713 #[ rstest]
689- #[ case:: overflow_then_positive( vec![ i64 :: MAX , 1 ] , vec![ 2 ] , None ) ]
690- #[ case:: overflow_then_zero( vec![ i64 :: MAX , 1 ] , vec![ 0 ] , None ) ]
691- #[ case:: all_chunks_overflow( vec![ i64 :: MAX , 1 ] , vec![ i64 :: MAX , 1 ] , None ) ]
692- #[ case:: overflow_across_chunks( vec![ i64 :: MAX ] , vec![ 1 ] , None ) ]
693- #[ case:: valid_sum( vec![ 3 , 4 ] , vec![ 5 ] , Some ( 12 ) ) ]
694- fn sum_keeps_overflow_across_chunks (
695- #[ case] first : Vec < i64 > ,
696- #[ case] second : Vec < i64 > ,
714+ #[ case:: overflow_across_chunks( vec![ vec![ Some ( i64 :: MAX ) ] , vec![ Some ( 1 ) ] ] , None ) ]
715+ #[ case:: nullable_chunks( vec![ vec![ None , Some ( 3 ) ] , vec![ Some ( 4 ) , None ] ] , Some ( 7 ) ) ]
716+ #[ case:: no_chunks( vec![ ] , None ) ]
717+ #[ case:: empty_chunk( vec![ vec![ ] ] , Some ( 0 ) ) ]
718+ fn sum_combines_chunk_states (
719+ #[ case] chunks : Vec < Vec < Option < i64 > > > ,
697720 #[ case] expected : Option < i64 > ,
698721 ) -> VortexResult < ( ) > {
699- let first = PrimitiveArray :: from_iter ( first ) . into_array ( ) ;
700- let second = PrimitiveArray :: from_iter ( second ) . into_array ( ) ;
701- let mut ctx = array_session ( ) . create_execution_ctx ( ) ;
702- let mut acc = StatsAccumulator :: new ( first . dtype ( ) , & [ Stat :: Sum ] , 64 ) ;
703- acc . push_chunk ( & first , & mut ctx ) ? ;
704- acc . push_chunk ( & second , & mut ctx ) ? ;
705- let stats = acc . as_stats_set ( & [ Stat :: Sum ] , & mut ctx ) ?;
722+ let dtype = DType :: Primitive ( PType :: I64 , Nullability :: Nullable ) ;
723+ let chunks = chunks
724+ . into_iter ( )
725+ . map ( |values| PrimitiveArray :: from_option_iter ( values ) . into_array ( ) )
726+ . collect :: < Vec < _ > > ( ) ;
727+
728+ let stats = accumulate_sum_stats ( & dtype , & chunks ) ?;
706729 assert_eq ! (
707730 stats. get( Stat :: Sum ) ,
708731 expected. map_or( Precision :: Absent , Precision :: exact)
709732 ) ;
733+
710734 Ok ( ( ) )
711735 }
712736
@@ -718,12 +742,15 @@ mod tests {
718742 DecimalDType :: new ( 1 , 0 ) ,
719743 Nullability :: NonNullable ,
720744 ) ;
721- let first = ConstantArray :: new ( value. clone ( ) , 10_000_000_000 ) . into_array ( ) ;
722- let second = ConstantArray :: new ( value, 10_000_000_000 ) . into_array ( ) ;
723- let mut ctx = array_session ( ) . create_execution_ctx ( ) ;
724- let mut acc = StatsAccumulator :: new ( first. dtype ( ) , & [ Stat :: Sum ] , 64 ) ;
725- for chunk in [ & first, & second] {
726- acc. push_chunk ( chunk, & mut ctx) ?;
745+ let chunks = [
746+ ConstantArray :: new ( value. clone ( ) , 10_000_000_000 ) . into_array ( ) ,
747+ ConstantArray :: new ( value, 10_000_000_000 ) . into_array ( ) ,
748+ ] ;
749+
750+ let stats = accumulate_sum_stats ( chunks[ 0 ] . dtype ( ) , & chunks) ?;
751+ assert_eq ! ( stats. get( Stat :: Sum ) , Precision :: Absent ) ;
752+
753+ for chunk in & chunks {
727754 assert_eq ! (
728755 chunk. statistics( ) . get( Stat :: Sum ) ,
729756 Precision :: Exact ( Scalar :: decimal(
@@ -733,115 +760,41 @@ mod tests {
733760 ) )
734761 ) ;
735762 }
736- let stats = acc. as_stats_set ( & [ Stat :: Sum ] , & mut ctx) ?;
737- assert_eq ! ( stats. get( Stat :: Sum ) , Precision :: Absent ) ;
738- Ok ( ( ) )
739- }
740763
741- #[ rstest]
742- #[ case:: non_null( vec![ 3 , 4 ] , Some ( 7 ) ) ]
743- #[ case:: overflow( vec![ i64 :: MAX , 1 ] , None ) ]
744- fn sum_preserves_chunk_cache_publication (
745- #[ case] values : Vec < i64 > ,
746- #[ case] expected : Option < i64 > ,
747- ) -> VortexResult < ( ) > {
748- let array = PrimitiveArray :: from_iter ( values) . into_array ( ) ;
749- let mut ctx = array_session ( ) . create_execution_ctx ( ) ;
750- let mut acc = StatsAccumulator :: new ( array. dtype ( ) , & [ Stat :: Sum ] , 64 ) ;
751- acc. push_chunk ( & array, & mut ctx) ?;
752- assert_eq ! (
753- array. statistics( ) . get( Stat :: Sum ) ,
754- expected. map_or( Precision :: Absent , |value| {
755- Precision :: Exact ( Scalar :: primitive( value, Nullability :: Nullable ) )
756- } )
757- ) ;
758764 Ok ( ( ) )
759765 }
760766
761767 #[ rstest]
762- #[ case:: arithmetic_nan_then_finite( vec![ 1.0 ] , 1.0 ) ]
763- #[ case:: all_nan_finals( vec![ f64 :: INFINITY , f64 :: NEG_INFINITY ] , 0.0 ) ]
764- fn sum_skips_nan_chunk_finals (
768+ #[ case:: nan_chunk_then_finite( vec![ f64 :: INFINITY , f64 :: NEG_INFINITY ] , vec![ 1.0 ] , 1.0 ) ]
769+ #[ case:: all_nan_chunks(
770+ vec![ f64 :: INFINITY , f64 :: NEG_INFINITY ] ,
771+ vec![ f64 :: INFINITY , f64 :: NEG_INFINITY ] ,
772+ 0.0
773+ ) ]
774+ #[ case:: nan_from_merge( vec![ f64 :: INFINITY ] , vec![ f64 :: NEG_INFINITY ] , f64 :: NAN ) ]
775+ #[ case:: chunk_grouping( vec![ 1e16 ] , vec![ -1e16 , 1.0 ] , 0.0 ) ]
776+ fn sum_preserves_float_chunk_semantics (
777+ #[ case] first : Vec < f64 > ,
765778 #[ case] second : Vec < f64 > ,
766779 #[ case] expected : f64 ,
767- #[ values( false , true ) ] cached : bool ,
768780 ) -> VortexResult < ( ) > {
769- let first = PrimitiveArray :: from_iter ( [ f64:: INFINITY , f64:: NEG_INFINITY ] ) . into_array ( ) ;
770- let second = PrimitiveArray :: from_iter ( second) . into_array ( ) ;
771- let mut ctx = array_session ( ) . create_execution_ctx ( ) ;
772- if cached {
773- sum ( & first, & mut ctx) ?;
774- sum ( & second, & mut ctx) ?;
781+ let chunks = [
782+ PrimitiveArray :: from_iter ( first) . into_array ( ) ,
783+ PrimitiveArray :: from_iter ( second) . into_array ( ) ,
784+ ] ;
785+
786+ let stats = accumulate_sum_stats ( chunks[ 0 ] . dtype ( ) , & chunks) ?;
787+ if expected. is_nan ( ) {
788+ assert ! (
789+ stats
790+ . get( Stat :: Sum )
791+ . as_exact( )
792+ . is_some_and( |value| value. as_primitive( ) . is_nan( ) )
793+ ) ;
794+ } else {
795+ assert_eq ! ( stats. get( Stat :: Sum ) , Precision :: exact( expected) ) ;
775796 }
776- let mut acc = StatsAccumulator :: new ( first. dtype ( ) , & [ Stat :: Sum ] , 64 ) ;
777- acc. push_chunk ( & first, & mut ctx) ?;
778- acc. push_chunk ( & second, & mut ctx) ?;
779- assert ! (
780- first
781- . statistics( )
782- . get( Stat :: Sum )
783- . as_exact( )
784- . is_some_and( |value| value. as_primitive( ) . is_nan( ) )
785- ) ;
786- let stats = acc. as_stats_set ( & [ Stat :: Sum ] , & mut ctx) ?;
787- assert_eq ! ( stats. get( Stat :: Sum ) , Precision :: exact( expected) ) ;
788- Ok ( ( ) )
789- }
790797
791- #[ test]
792- fn sum_keeps_nan_from_merging_chunk_finals ( ) -> VortexResult < ( ) > {
793- let first = PrimitiveArray :: from_iter ( [ f64:: INFINITY ] ) . into_array ( ) ;
794- let second = PrimitiveArray :: from_iter ( [ f64:: NEG_INFINITY ] ) . into_array ( ) ;
795- let mut ctx = array_session ( ) . create_execution_ctx ( ) ;
796- let mut acc = StatsAccumulator :: new ( first. dtype ( ) , & [ Stat :: Sum ] , 64 ) ;
797- acc. push_chunk ( & first, & mut ctx) ?;
798- acc. push_chunk ( & second, & mut ctx) ?;
799- let stats = acc. as_stats_set ( & [ Stat :: Sum ] , & mut ctx) ?;
800- assert ! (
801- stats
802- . get( Stat :: Sum )
803- . as_exact( )
804- . is_some_and( |value| value. as_primitive( ) . is_nan( ) )
805- ) ;
806- Ok ( ( ) )
807- }
808-
809- #[ test]
810- fn sum_keeps_chunk_float_grouping ( ) -> VortexResult < ( ) > {
811- let first = PrimitiveArray :: from_iter ( [ 1e16f64 ] ) . into_array ( ) ;
812- let second = PrimitiveArray :: from_iter ( [ -1e16f64 , 1.0 ] ) . into_array ( ) ;
813- let mut ctx = array_session ( ) . create_execution_ctx ( ) ;
814- let mut acc = StatsAccumulator :: new ( first. dtype ( ) , & [ Stat :: Sum ] , 64 ) ;
815- acc. push_chunk ( & first, & mut ctx) ?;
816- acc. push_chunk ( & second, & mut ctx) ?;
817- let stats = acc. as_stats_set ( & [ Stat :: Sum ] , & mut ctx) ?;
818- assert_eq ! ( stats. get( Stat :: Sum ) , Precision :: exact( 0.0f64 ) ) ;
819- Ok ( ( ) )
820- }
821-
822- #[ rstest]
823- #[ case:: no_chunks( vec![ ] , None ) ]
824- #[ case:: empty_chunk( vec![ vec![ ] ] , Some ( 0 ) ) ]
825- #[ case:: all_null_chunk( vec![ vec![ None , None ] ] , Some ( 0 ) ) ]
826- #[ case:: nullable_chunks( vec![ vec![ None , Some ( 3 ) ] , vec![ Some ( 4 ) , None ] ] , Some ( 7 ) ) ]
827- fn sum_distinguishes_missing_input_from_zero (
828- #[ case] chunks : Vec < Vec < Option < i64 > > > ,
829- #[ case] expected : Option < i64 > ,
830- ) -> VortexResult < ( ) > {
831- let mut ctx = array_session ( ) . create_execution_ctx ( ) ;
832- let dtype = DType :: Primitive ( PType :: I64 , Nullability :: Nullable ) ;
833- let mut acc = StatsAccumulator :: new ( & dtype, & [ Stat :: Sum ] , 64 ) ;
834- for chunk in chunks {
835- acc. push_chunk (
836- & PrimitiveArray :: from_option_iter ( chunk) . into_array ( ) ,
837- & mut ctx,
838- ) ?;
839- }
840- let stats = acc. as_stats_set ( & [ Stat :: Sum ] , & mut ctx) ?;
841- assert_eq ! (
842- stats. get( Stat :: Sum ) ,
843- expected. map_or( Precision :: Absent , Precision :: exact)
844- ) ;
845798 Ok ( ( ) )
846799 }
847800}
0 commit comments