@@ -12,11 +12,15 @@ use flatbuffers::FlatBufferBuilder;
1212use flatbuffers:: WIPOffset ;
1313use itertools:: Itertools ;
1414use vortex_array:: dtype:: DType ;
15+ use vortex_array:: dtype:: FieldPath ;
1516use vortex_array:: flatbuffers:: FlatBufferRoot ;
1617use vortex_array:: flatbuffers:: WriteFlatBuffer ;
18+ use vortex_array:: flatbuffers:: array:: ArrayStats ;
1719use vortex_array:: stats:: StatsSet ;
20+ use vortex_error:: VortexExpect ;
1821use vortex_error:: VortexResult ;
1922use vortex_error:: vortex_ensure_eq;
23+ use vortex_layout:: layouts:: file_stats:: postorder_stats_layout;
2024use vortex_session:: VortexSession ;
2125
2226use crate :: flatbuffers:: footer as fb;
@@ -32,72 +36,97 @@ pub struct FileStatistics {
3236 stats : Arc < [ StatsSet ] > ,
3337 /// An array of `DType`s, one for each field or column in the file.
3438 dtypes : Arc < [ DType ] > ,
39+ /// An array of field paths, one for each field or column in the file. Parallel to `stats` and
40+ /// `dtypes`. For files written before nested field stats, every path has depth 1 (or is the
41+ /// root path, for a non-struct file dtype).
42+ paths : Arc < [ FieldPath ] > ,
3543}
3644
3745impl FileStatistics {
38- /// Creates a new [`FileStatistics`] from the given statistics and data types.
46+ /// Creates a new [`FileStatistics`] from the given statistics, data types, and field paths .
3947 ///
4048 /// # Panics
4149 ///
42- /// Panics if `stats` and `dtypes ` have different lengths.
43- pub fn new ( stats : Arc < [ StatsSet ] > , dtypes : Arc < [ DType ] > ) -> Self {
50+ /// Panics if `stats`, `dtypes`, and `paths ` have different lengths.
51+ pub fn new ( stats : Arc < [ StatsSet ] > , dtypes : Arc < [ DType ] > , paths : Arc < [ FieldPath ] > ) -> Self {
4452 assert_eq ! (
4553 stats. len( ) ,
4654 dtypes. len( ) ,
4755 "stats and dtypes must have the same length"
4856 ) ;
57+ assert_eq ! (
58+ stats. len( ) ,
59+ paths. len( ) ,
60+ "stats and paths must have the same length"
61+ ) ;
4962
50- Self { stats, dtypes }
63+ Self {
64+ stats,
65+ dtypes,
66+ paths,
67+ }
5168 }
5269
5370 /// Creates a new [`FileStatistics`] from the given statistics and file dtype.
5471 ///
55- /// If the [`DType`] of the file is a [`DType::Struct`], then there must be the same number of
56- /// stats as struct fields. Otherwise, there must be only 1 statistic .
72+ /// `stats` must follow the post-order nested-struct layout produced by
73+ /// [`postorder_stats_layout`] for `file_dtype` .
5774 ///
5875 /// # Panics
5976 ///
6077 /// Panics if the number of stats doesn't match the expected number based on the dtype.
6178 pub fn new_with_dtype ( stats : Arc < [ StatsSet ] > , file_dtype : & DType ) -> Self {
62- if let DType :: Struct ( struct_fields , _ ) = file_dtype {
63- assert_eq ! (
64- stats. len( ) ,
65- struct_fields . nfields ( ) ,
66- "stats length must match number of struct fields "
67- ) ;
79+ let layout = postorder_stats_layout ( file_dtype) ;
80+ assert_eq ! (
81+ stats. len( ) ,
82+ layout . len ( ) ,
83+ "stats length must match the post-order stats layout for the file dtype "
84+ ) ;
6885
69- let dtypes = struct_fields . fields ( ) . collect ( ) ;
86+ let ( paths , dtypes ) : ( Vec < FieldPath > , Vec < DType > ) = layout . into_iter ( ) . unzip ( ) ;
7087
71- Self { stats, dtypes }
72- } else {
73- assert_eq ! (
74- stats. len( ) ,
75- 1 ,
76- "non-struct dtype must have exactly 1 statistic"
77- ) ;
78-
79- Self {
80- stats,
81- dtypes : Arc :: new ( [ file_dtype. clone ( ) ] ) ,
82- }
88+ Self {
89+ stats,
90+ dtypes : dtypes. into ( ) ,
91+ paths : paths. into ( ) ,
8392 }
8493 }
8594
8695 /// Creates [`FileStatistics`] from a flatbuffers [`fb::FileStatistics<'a>`].
87- ///
88- /// If the [`DType`] of the file is a [`DType::Struct`], then there must be the same number of
89- /// file stats in the flatbuffer. Otherwise, there must be only 1 statistic.
9096 pub fn from_flatbuffer < ' a > (
9197 fb : & fb:: FileStatistics < ' a > ,
9298 file_dtype : & DType ,
9399 session : & VortexSession ,
94100 ) -> VortexResult < Self > {
95101 let field_stats = fb. field_stats ( ) . unwrap_or_default ( ) ;
102+ let mut array_stats: Vec < ArrayStats > = field_stats. iter ( ) . collect ( ) ;
96103
104+ if fb. is_nested ( ) {
105+ let layout = postorder_stats_layout ( file_dtype) ;
106+ vortex_ensure_eq ! ( array_stats. len( ) , layout. len( ) ) ;
107+
108+ let mut stats_sets = Vec :: with_capacity ( array_stats. len ( ) ) ;
109+ let mut dtypes = Vec :: with_capacity ( layout. len ( ) ) ;
110+ let mut paths = Vec :: with_capacity ( layout. len ( ) ) ;
111+ for ( array_stat, ( path, dtype) ) in array_stats. into_iter ( ) . zip ( layout) {
112+ stats_sets. push ( StatsSet :: from_flatbuffer ( & array_stat, & dtype, session) ?) ;
113+ dtypes. push ( dtype) ;
114+ paths. push ( path) ;
115+ }
116+
117+ return Ok ( Self {
118+ stats : stats_sets. into ( ) ,
119+ dtypes : dtypes. into ( ) ,
120+ paths : paths. into ( ) ,
121+ } ) ;
122+ }
123+
124+ // Legacy (pre-nested-stats) layout: top-level struct fields only, or a single entry for a
125+ // non-struct root dtype.
97126 if let DType :: Struct ( struct_fields, _) = file_dtype {
98- vortex_ensure_eq ! ( field_stats . len( ) , struct_fields. nfields( ) ) ;
127+ vortex_ensure_eq ! ( array_stats . len( ) , struct_fields. nfields( ) ) ;
99128
100- let stats_sets: Arc < [ StatsSet ] > = field_stats
129+ let stats_sets: Arc < [ StatsSet ] > = array_stats
101130 . into_iter ( )
102131 . zip ( struct_fields. fields ( ) )
103132 . map ( |( array_stat, field_dtype) | {
@@ -106,20 +135,29 @@ impl FileStatistics {
106135 . try_collect ( ) ?;
107136
108137 let dtypes = struct_fields. fields ( ) . collect ( ) ;
138+ let paths = struct_fields
139+ . names ( )
140+ . iter ( )
141+ . map ( |name| FieldPath :: from_name ( name. clone ( ) ) )
142+ . collect ( ) ;
109143
110144 Ok ( Self {
111145 stats : stats_sets,
112146 dtypes,
147+ paths,
113148 } )
114149 } else {
115- vortex_ensure_eq ! ( field_stats . len( ) , 1 ) ;
150+ vortex_ensure_eq ! ( array_stats . len( ) , 1 ) ;
116151
117- let array_stat = field_stats. get ( 0 ) ;
152+ let array_stat = array_stats
153+ . pop ( )
154+ . vortex_expect ( "we just checked that there was 1 field" ) ;
118155 let stats_set = StatsSet :: from_flatbuffer ( & array_stat, file_dtype, session) ?;
119156
120157 Ok ( Self {
121158 stats : Arc :: new ( [ stats_set] ) ,
122159 dtypes : Arc :: new ( [ file_dtype. clone ( ) ] ) ,
160+ paths : Arc :: new ( [ FieldPath :: root ( ) ] ) ,
123161 } )
124162 }
125163 }
@@ -134,6 +172,11 @@ impl FileStatistics {
134172 & self . dtypes
135173 }
136174
175+ /// Returns a reference to the field paths.
176+ pub fn paths ( & self ) -> & Arc < [ FieldPath ] > {
177+ & self . paths
178+ }
179+
137180 /// Returns the statistics and data type for a specific field.
138181 ///
139182 /// # Panics
@@ -142,6 +185,14 @@ impl FileStatistics {
142185 pub fn get ( & self , field_idx : usize ) -> ( & StatsSet , & DType ) {
143186 ( & self . stats [ field_idx] , & self . dtypes [ field_idx] )
144187 }
188+
189+ /// Returns the statistics and data type for the field at the given path, if present.
190+ pub fn get_by_path ( & self , path : & FieldPath ) -> Option < ( & StatsSet , & DType ) > {
191+ self . paths
192+ . iter ( )
193+ . position ( |p| p == path)
194+ . map ( |idx| ( & self . stats [ idx] , & self . dtypes [ idx] ) )
195+ }
145196}
146197
147198impl < ' a > IntoIterator for & ' a FileStatistics {
@@ -173,7 +224,101 @@ impl WriteFlatBuffer for FileStatistics {
173224 fbb,
174225 & fb:: FileStatisticsArgs {
175226 field_stats : Some ( field_stats) ,
227+ is_nested : true ,
176228 } ,
177229 ) )
178230 }
179231}
232+
233+ #[ cfg( test) ]
234+ mod tests {
235+ use flatbuffers:: FlatBufferBuilder ;
236+ use vortex_array:: array_session;
237+ use vortex_array:: dtype:: FieldPath ;
238+ use vortex_array:: dtype:: Nullability ;
239+ use vortex_array:: dtype:: PType ;
240+ use vortex_array:: expr:: stats:: Precision ;
241+ use vortex_array:: expr:: stats:: Stat ;
242+ use vortex_array:: flatbuffers:: WriteFlatBufferExt ;
243+ use vortex_array:: scalar:: ScalarValue ;
244+
245+ use super :: * ;
246+
247+ fn i32_dtype ( ) -> DType {
248+ DType :: Primitive ( PType :: I32 , Nullability :: NonNullable )
249+ }
250+
251+ #[ test]
252+ fn nested_round_trip_resolves_by_path ( ) -> VortexResult < ( ) > {
253+ let session = array_session ( ) ;
254+ let inner = DType :: struct_ ( [ ( "b" , i32_dtype ( ) ) ] , Nullability :: Nullable ) ;
255+ let file_dtype = DType :: struct_ ( [ ( "a" , inner) ] , Nullability :: NonNullable ) ;
256+
257+ // Layout: [a.b, a] (a's own null-count entry trails its child).
258+ let mut b_stats = StatsSet :: default ( ) ;
259+ b_stats. set ( Stat :: Min , Precision :: exact ( ScalarValue :: from ( 1i32 ) ) ) ;
260+ let mut a_stats = StatsSet :: default ( ) ;
261+ a_stats. set ( Stat :: NullCount , Precision :: exact ( ScalarValue :: from ( 1u64 ) ) ) ;
262+
263+ let file_stats = FileStatistics :: new_with_dtype ( Arc :: from ( [ b_stats, a_stats] ) , & file_dtype) ;
264+
265+ let bytes = file_stats. write_flatbuffer_bytes ( ) ?;
266+ let fb = flatbuffers:: root :: < fb:: FileStatistics > ( bytes. as_ref ( ) )
267+ . vortex_expect ( "valid flatbuffer" ) ;
268+ assert ! ( fb. is_nested( ) ) ;
269+
270+ let read_back = FileStatistics :: from_flatbuffer ( & fb, & file_dtype, & session) ?;
271+
272+ let ( b, _) = read_back
273+ . get_by_path ( & FieldPath :: from_name ( "a" ) . push ( "b" ) )
274+ . expect ( "a.b stats" ) ;
275+ assert_eq ! ( b. get( Stat :: Min ) . as_exact( ) , Some ( ScalarValue :: from( 1i32 ) ) ) ;
276+
277+ let ( a, _) = read_back
278+ . get_by_path ( & FieldPath :: from_name ( "a" ) )
279+ . expect ( "a's own null-count stats" ) ;
280+ assert_eq ! (
281+ a. get( Stat :: NullCount ) . as_exact( ) ,
282+ Some ( ScalarValue :: from( 1u64 ) )
283+ ) ;
284+
285+ assert ! ( read_back. get_by_path( & FieldPath :: root( ) ) . is_none( ) ) ;
286+
287+ Ok ( ( ) )
288+ }
289+
290+ #[ test]
291+ fn legacy_non_nested_footer_still_parses ( ) -> VortexResult < ( ) > {
292+ // Simulates a footer written before nested field stats existed: `is_nested` is absent
293+ // (defaults to false), and `field_stats` holds one entry per top-level struct field.
294+ let session = array_session ( ) ;
295+ let file_dtype = DType :: struct_ ( [ ( "col" , i32_dtype ( ) ) ] , Nullability :: NonNullable ) ;
296+
297+ let mut stats = StatsSet :: default ( ) ;
298+ stats. set ( Stat :: Min , Precision :: exact ( ScalarValue :: from ( 7i32 ) ) ) ;
299+
300+ let mut fbb = FlatBufferBuilder :: new ( ) ;
301+ let array_stats = stats. write_flatbuffer ( & mut fbb) ?;
302+ let field_stats = fbb. create_vector ( & [ array_stats] ) ;
303+ let root = fb:: FileStatistics :: create (
304+ & mut fbb,
305+ & fb:: FileStatisticsArgs {
306+ field_stats : Some ( field_stats) ,
307+ is_nested : false ,
308+ } ,
309+ ) ;
310+ fbb. finish_minimal ( root) ;
311+ let bytes = fbb. finished_data ( ) . to_vec ( ) ;
312+
313+ let fb = flatbuffers:: root :: < fb:: FileStatistics > ( & bytes) . vortex_expect ( "valid flatbuffer" ) ;
314+ assert ! ( !fb. is_nested( ) ) ;
315+
316+ let read_back = FileStatistics :: from_flatbuffer ( & fb, & file_dtype, & session) ?;
317+ let ( col, _) = read_back
318+ . get_by_path ( & FieldPath :: from_name ( "col" ) )
319+ . expect ( "col stats" ) ;
320+ assert_eq ! ( col. get( Stat :: Min ) . as_exact( ) , Some ( ScalarValue :: from( 7i32 ) ) ) ;
321+
322+ Ok ( ( ) )
323+ }
324+ }
0 commit comments