22// SPDX-FileCopyrightText: Copyright the Vortex contributors
33
44use std:: future;
5+ use std:: iter;
56use std:: sync:: Arc ;
67
78use futures:: StreamExt ;
@@ -204,11 +205,16 @@ struct StatsAccumulator {
204205}
205206
206207impl StatsAccumulator {
208+ /// An accumulator that records nothing.
209+ fn empty ( ) -> Self {
210+ Self {
211+ aggregates : Vec :: new ( ) ,
212+ }
213+ }
214+
207215 fn new ( dtype : & DType , stats : Option < & [ AggregateFnRef ] > ) -> Self {
208216 if !supports_file_stats ( dtype) {
209- return Self {
210- aggregates : Vec :: new ( ) ,
211- } ;
217+ return Self :: empty ( ) ;
212218 }
213219
214220 // When no explicit list is requested, each leaf picks its own default the way
@@ -334,6 +340,9 @@ fn default_pruning_aggregate_fns(dtype: &DType) -> Vec<AggregateFnRef> {
334340/// [`DType::Variant`], are skipped entirely: no entry is emitted for them or anything beneath
335341/// them.
336342///
343+ /// Entries beneath a nullable struct are still emitted, but the writer records no aggregates for
344+ /// them (see [`StatsNode::Unrecorded`]).
345+ ///
337346/// This function is the single source of truth for the number and order of stats entries, used
338347/// both when accumulating stats at write time and when reconstructing them from the footer at
339348/// read time, so no paths or dtypes need to be persisted in the flatbuffer itself.
@@ -364,6 +373,18 @@ enum StatsNode {
364373 Leaf ( StatsAccumulator ) ,
365374 /// A dtype that does not support file stats (e.g. [`DType::Variant`]); contributes no entries.
366375 Skipped ,
376+ /// A subtree beneath a nullable struct, which contributes `entries` entries that record no
377+ /// aggregates.
378+ ///
379+ /// Struct fields are stored unmasked, so a row where the struct is null holds an unspecified
380+ /// value in each child that the child's own validity may report as valid. Stats over those
381+ /// children would count that value, e.g. undercount nulls, which can prune a file that
382+ /// `is_null(s.a)` matches. Until nulls are propagated from structs into their children, these
383+ /// entries are left empty. See <https://github.com/vortex-data/vortex/issues/9205>.
384+ ///
385+ /// The entries are kept rather than dropped so the footer layout from
386+ /// [`postorder_stats_layout`] doesn't depend on whether they're recorded.
387+ Unrecorded { entries : usize } ,
367388 /// A dtype with addressable children. Currently only built for `DType::Struct`.
368389 Container {
369390 /// One child per addressable sub-dtype, in declaration order (one per struct field,
@@ -382,15 +403,25 @@ impl StatsNode {
382403 fn build ( dtype : & DType , stats : Option < & [ AggregateFnRef ] > ) -> Self {
383404 match dtype. as_struct_fields_opt ( ) {
384405 Some ( struct_fields) => {
406+ let nullable = dtype. nullability ( ) == Nullability :: Nullable ;
385407 let children = struct_fields
386408 . names ( )
387409 . iter ( )
388410 . zip ( struct_fields. fields ( ) )
389411 . map ( |( name, field_dtype) | {
390- ( Field :: Name ( name. clone ( ) ) , Self :: build ( & field_dtype, stats) )
412+ let child = if nullable {
413+ Self :: Unrecorded {
414+ entries : postorder_stats_layout ( & field_dtype) . len ( ) ,
415+ }
416+ } else {
417+ Self :: build ( & field_dtype, stats)
418+ } ;
419+ ( Field :: Name ( name. clone ( ) ) , child)
391420 } )
392421 . collect ( ) ;
393- let own = if dtype. nullability ( ) == Nullability :: Nullable {
422+ // Only reached through non-nullable ancestors, so the struct's own validity is
423+ // its full logical validity.
424+ let own = if nullable {
394425 Self :: Leaf ( StatsAccumulator :: new ( dtype, stats) )
395426 } else {
396427 Self :: Skipped
@@ -407,7 +438,7 @@ impl StatsNode {
407438
408439 fn push_chunk ( & mut self , array : & ArrayRef , ctx : & mut ExecutionCtx ) -> VortexResult < ( ) > {
409440 match self {
410- Self :: Skipped => Ok ( ( ) ) ,
441+ Self :: Skipped | Self :: Unrecorded { .. } => Ok ( ( ) ) ,
411442 Self :: Leaf ( acc) => acc. push_chunk ( array, ctx) ,
412443 Self :: Container { .. } => {
413444 let struct_array = array. clone ( ) . execute :: < StructArray > ( ctx) ?;
@@ -460,6 +491,10 @@ impl StatsNode {
460491 ) -> VortexResult < ( ) > {
461492 match self {
462493 Self :: Skipped => Ok ( ( ) ) ,
494+ Self :: Unrecorded { entries } => {
495+ out. extend ( iter:: repeat_n ( AggregateStats :: default ( ) , * entries) ) ;
496+ Ok ( ( ) )
497+ }
463498 Self :: Leaf ( acc) => {
464499 out. push ( acc. aggregate_stats ( allowed) ?) ;
465500 Ok ( ( ) )
@@ -509,6 +544,13 @@ impl FileStatsAccumulator {
509544
510545 let legacy = Arc :: new ( Mutex :: new ( if write_legacy_stats {
511546 match dtype. as_struct_fields_opt ( ) {
547+ // The fields of a nullable root are recorded empty, like `StatsNode::Unrecorded`.
548+ Some ( struct_fields) if dtype. nullability ( ) == Nullability :: Nullable => {
549+ struct_fields
550+ . fields ( )
551+ . map ( |_| StatsAccumulator :: empty ( ) )
552+ . collect ( )
553+ }
512554 Some ( struct_fields) => struct_fields
513555 . fields ( )
514556 . map ( |field_dtype| StatsAccumulator :: new ( & field_dtype, stats. as_deref ( ) ) )
@@ -855,10 +897,11 @@ mod tests {
855897 }
856898
857899 #[ test]
858- fn three_level_nested_struct_accumulates_stats_at_each_level ( ) -> VortexResult < ( ) > {
900+ fn three_level_nested_struct_records_only_entries_without_nullable_ancestors ( )
901+ -> VortexResult < ( ) > {
859902 // Same shape as `postorder_layout_three_level_nesting_with_mixed_nullability`, but
860- // exercises actual accumulation: each nullable level along the path should independently
861- // contribute its own null-count entry, in post-order .
903+ // exercises actual accumulation. The root is nullable, so every entry beneath it is left
904+ // empty; only the root's own null-count entry is recorded .
862905 let mut ctx = array_session ( ) . create_execution_ctx ( ) ;
863906
864907 let leaf_c = buffer ! [ 10i32 , 20 , 30 ] . into_array ( ) ;
@@ -882,28 +925,69 @@ mod tests {
882925
883926 let stats_sets = node_stats_sets ( & node) ?;
884927
885- // `a.b.c`'s own stats come first (post-order), then `a.b`'s null-count entry, then the
886- // root's own null-count entry.
928+ // Post-order: `a.b.c`, then `a.b`'s null-count entry, then the root's.
887929 assert_eq ! ( stats_sets. len( ) , 3 ) ;
930+ assert ! ( stats_sets[ 0 ] . is_empty( ) ) ;
931+ assert ! ( stats_sets[ 1 ] . is_empty( ) ) ;
888932 assert_eq ! (
889- stats_sets[ 0 ] . get( Stat :: Min ) . as_exact( ) ,
890- Some ( ScalarValue :: from( 10i32 ) )
933+ stats_sets[ 2 ] . get( Stat :: NullCount ) . as_exact( ) ,
934+ Some ( ScalarValue :: from( 1u64 ) )
891935 ) ;
936+ Ok ( ( ) )
937+ }
938+
939+ #[ test]
940+ fn children_of_nullable_struct_are_unrecorded ( ) -> VortexResult < ( ) > {
941+ // https://github.com/vortex-data/vortex/issues/9205
942+ let mut ctx = array_session ( ) . create_execution_ctx ( ) ;
943+
944+ let a = buffer ! [ 1i32 , 999 , 3 ] . into_array ( ) ;
945+ let s_validity = Validity :: Array ( BoolArray :: from_iter ( [ true , false , true ] ) . into_array ( ) ) ;
946+ let s = StructArray :: new ( FieldNames :: from ( [ "a" ] ) , [ a] , 3 , s_validity) . into_array ( ) ;
947+ let root =
948+ StructArray :: new ( FieldNames :: from ( [ "s" ] ) , [ s] , 3 , Validity :: NonNullable ) . into_array ( ) ;
949+
950+ let requested = [ agg ( Stat :: NullCount ) , agg ( Stat :: Min ) , agg ( Stat :: Max ) ] ;
951+ let mut node = StatsNode :: build ( root. dtype ( ) , Some ( & requested) ) ;
952+ node. push_chunk ( & root, & mut ctx) ?;
953+
954+ let mut aggregates = Vec :: new ( ) ;
955+ node. collect_aggregate_stats ( & |_| true , & mut aggregates) ?;
956+
957+ // `s.a`, then `s`'s own null-count entry.
958+ assert_eq ! ( aggregates. len( ) , 2 ) ;
959+ assert ! ( aggregates[ 0 ] . is_empty( ) ) ;
892960 assert_eq ! (
893- stats_sets [ 0 ] . get( Stat :: Max ) . as_exact ( ) ,
894- Some ( ScalarValue :: from( 30i32 ) )
961+ aggregates [ 1 ] . get( & agg ( Stat :: NullCount ) ) ,
962+ Precision :: exact ( Scalar :: from( 1u64 ) )
895963 ) ;
964+ Ok ( ( ) )
965+ }
966+
967+ #[ test]
968+ fn nullable_children_of_non_nullable_struct_are_recorded ( ) -> VortexResult < ( ) > {
969+ // `s` is non-nullable, so `s.a`'s own validity is its full logical validity.
970+ let mut ctx = array_session ( ) . create_execution_ctx ( ) ;
971+
972+ let a = PrimitiveArray :: from_option_iter ( [ Some ( 1i32 ) , None , Some ( 3 ) ] ) . into_array ( ) ;
973+ let s =
974+ StructArray :: new ( FieldNames :: from ( [ "a" ] ) , [ a] , 3 , Validity :: NonNullable ) . into_array ( ) ;
975+ let root =
976+ StructArray :: new ( FieldNames :: from ( [ "s" ] ) , [ s] , 3 , Validity :: NonNullable ) . into_array ( ) ;
977+
978+ let requested = [ agg ( Stat :: NullCount ) , agg ( Stat :: Max ) ] ;
979+ let mut node = StatsNode :: build ( root. dtype ( ) , Some ( & requested) ) ;
980+ node. push_chunk ( & root, & mut ctx) ?;
981+
982+ let stats_sets = node_stats_sets ( & node) ?;
983+ assert_eq ! ( stats_sets. len( ) , 1 ) ;
896984 assert_eq ! (
897985 stats_sets[ 0 ] . get( Stat :: NullCount ) . as_exact( ) ,
898- Some ( ScalarValue :: from( 0u64 ) )
899- ) ;
900- assert_eq ! (
901- stats_sets[ 1 ] . get( Stat :: NullCount ) . as_exact( ) ,
902986 Some ( ScalarValue :: from( 1u64 ) )
903987 ) ;
904988 assert_eq ! (
905- stats_sets[ 2 ] . get( Stat :: NullCount ) . as_exact( ) ,
906- Some ( ScalarValue :: from( 1u64 ) )
989+ stats_sets[ 0 ] . get( Stat :: Max ) . as_exact( ) ,
990+ Some ( ScalarValue :: from( 3i32 ) )
907991 ) ;
908992 Ok ( ( ) )
909993 }
@@ -975,6 +1059,32 @@ mod tests {
9751059 Ok ( ( ) )
9761060 }
9771061
1062+ #[ test]
1063+ fn legacy_aggregate_stats_of_nullable_root_are_unrecorded ( ) -> VortexResult < ( ) > {
1064+ let session = array_session ( ) ;
1065+ let dtype = DType :: struct_ ( [ ( "a" , i32_dtype ( ) ) ] , Nullability :: Nullable ) ;
1066+
1067+ let acc = FileStatsAccumulator :: try_new (
1068+ & dtype,
1069+ Some ( Arc :: from ( [ agg ( Stat :: NullCount ) , agg ( Stat :: Max ) ] ) ) ,
1070+ & session,
1071+ true ,
1072+ & LayoutWriterContext :: new ( ArrayContext :: empty ( ) ) ,
1073+ ) ?;
1074+
1075+ let a = buffer ! [ 1i32 , 999 , 3 ] . into_array ( ) ;
1076+ let validity = Validity :: Array ( BoolArray :: from_iter ( [ true , false , true ] ) . into_array ( ) ) ;
1077+ let root = StructArray :: new ( FieldNames :: from ( [ "a" ] ) , [ a] , 3 , validity) . into_array ( ) ;
1078+
1079+ let ( mut ptr, _eof) = SequenceId :: root ( ) . split ( ) ;
1080+ acc. process ( Ok ( ( ptr. advance ( ) , root) ) ) ?;
1081+
1082+ let legacy = acc. legacy_aggregate_stats ( ) ?;
1083+ assert_eq ! ( legacy. len( ) , 1 ) ;
1084+ assert ! ( legacy[ 0 ] . is_empty( ) ) ;
1085+ Ok ( ( ) )
1086+ }
1087+
9781088 #[ rstest]
9791089 #[ case( DType :: struct_( [ ( "a" , i32_dtype( ) ) ] , Nullability :: NonNullable ) , StructArray :: new( FieldNames :: from( [ "a" ] ) , [ buffer![ 1i32 , 2 , 3 ] . into_array( ) ] , 3 , Validity :: NonNullable ) . into_array( ) ) ]
9801090 #[ case( i32_dtype( ) , buffer![ 1i32 , 2 , 3 ] . into_array( ) ) ]
0 commit comments