Skip to content

Commit 06064c2

Browse files
committed
Support file stats on nested struct fields (vortex-data#6389)
Store file-level statistics via a post-order walk of the DType tree, so nested struct fields get whole-file pruning instead of only top-level columns. Each leaf gets a stats entry, and each nullable struct gets a trailing null-count entry. A new `is_nested` footer flag keeps old files readable via the legacy top-level-only layout. Signed-off-by: "Thor" <[email protected]> Signed-off-by: Thor <[email protected]>
1 parent c11856c commit 06064c2

7 files changed

Lines changed: 521 additions & 136 deletions

File tree

‎vortex-file/flatbuffers/vortex-file/footer.fbs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,15 @@ table PostscriptSegment {
5858
table FileStatistics {
5959
/// Statistics for each field in the root schema. If the root schema is not a struct, there will
6060
/// be a single entry in this array.
61+
///
62+
/// When `is_nested` is true, entries follow a post-order walk of the root `DType` tree: each
63+
/// leaf field gets an entry, and each nullable struct additionally gets a trailing null-count
64+
/// entry inserted after its children's entries.
6165
field_stats: [ArrayStats];
66+
/// Whether `field_stats` follows the post-order nested-struct layout (true) or the legacy
67+
/// top-level-fields-only layout (false). Defaults to false for backward compatibility with
68+
/// files written before nested field stats were supported.
69+
is_nested: bool = false;
6270
}
6371

6472
/// The `Registry` object stores dictionary-encoded configuration for segments,

‎vortex-file/src/file.rs‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -227,19 +227,14 @@ impl VortexFile {
227227
/// Row-count-aware pruning predicates are evaluated with the file's total
228228
/// row count as their scope.
229229
pub fn can_prune(&self, filter: &Expression) -> VortexResult<bool> {
230-
let Some((stats, fields)) = self
231-
.footer
232-
.statistics()
233-
.zip(self.footer.dtype().as_struct_fields_opt())
234-
else {
230+
let Some(stats) = self.footer.statistics() else {
235231
return Ok(false);
236232
};
237233

238234
can_prune_file_stats(
239235
&filter.bind(self.footer.dtype())?,
240236
self.footer.row_count(),
241237
stats,
242-
fields,
243238
&self.session,
244239
)
245240
}

‎vortex-file/src/footer/file_statistics.rs‎

Lines changed: 177 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,15 @@ use flatbuffers::FlatBufferBuilder;
1212
use flatbuffers::WIPOffset;
1313
use itertools::Itertools;
1414
use vortex_array::dtype::DType;
15+
use vortex_array::dtype::FieldPath;
1516
use vortex_array::flatbuffers::FlatBufferRoot;
1617
use vortex_array::flatbuffers::WriteFlatBuffer;
18+
use vortex_array::flatbuffers::array::ArrayStats;
1719
use vortex_array::stats::StatsSet;
20+
use vortex_error::VortexExpect;
1821
use vortex_error::VortexResult;
1922
use vortex_error::vortex_ensure_eq;
23+
use vortex_layout::layouts::file_stats::postorder_stats_layout;
2024
use vortex_session::VortexSession;
2125

2226
use crate::flatbuffers::footer as fb;
@@ -32,72 +36,97 @@ pub struct FileStatistics {
3236
stats: Arc<[StatsSet]>,
3337
/// An array of `DType`s, one for each field or column in the file.
3438
dtypes: Arc<[DType]>,
39+
/// An array of field paths, one for each field or column in the file. Parallel to `stats` and
40+
/// `dtypes`. For files written before nested field stats, every path has depth 1 (or is the
41+
/// root path, for a non-struct file dtype).
42+
paths: Arc<[FieldPath]>,
3543
}
3644

3745
impl FileStatistics {
38-
/// Creates a new [`FileStatistics`] from the given statistics and data types.
46+
/// Creates a new [`FileStatistics`] from the given statistics, data types, and field paths.
3947
///
4048
/// # Panics
4149
///
42-
/// Panics if `stats` and `dtypes` have different lengths.
43-
pub fn new(stats: Arc<[StatsSet]>, dtypes: Arc<[DType]>) -> Self {
50+
/// Panics if `stats`, `dtypes`, and `paths` have different lengths.
51+
pub fn new(stats: Arc<[StatsSet]>, dtypes: Arc<[DType]>, paths: Arc<[FieldPath]>) -> Self {
4452
assert_eq!(
4553
stats.len(),
4654
dtypes.len(),
4755
"stats and dtypes must have the same length"
4856
);
57+
assert_eq!(
58+
stats.len(),
59+
paths.len(),
60+
"stats and paths must have the same length"
61+
);
4962

50-
Self { stats, dtypes }
63+
Self {
64+
stats,
65+
dtypes,
66+
paths,
67+
}
5168
}
5269

5370
/// Creates a new [`FileStatistics`] from the given statistics and file dtype.
5471
///
55-
/// If the [`DType`] of the file is a [`DType::Struct`], then there must be the same number of
56-
/// stats as struct fields. Otherwise, there must be only 1 statistic.
72+
/// `stats` must follow the post-order nested-struct layout produced by
73+
/// [`postorder_stats_layout`] for `file_dtype`.
5774
///
5875
/// # Panics
5976
///
6077
/// Panics if the number of stats doesn't match the expected number based on the dtype.
6178
pub fn new_with_dtype(stats: Arc<[StatsSet]>, file_dtype: &DType) -> Self {
62-
if let DType::Struct(struct_fields, _) = file_dtype {
63-
assert_eq!(
64-
stats.len(),
65-
struct_fields.nfields(),
66-
"stats length must match number of struct fields"
67-
);
79+
let layout = postorder_stats_layout(file_dtype);
80+
assert_eq!(
81+
stats.len(),
82+
layout.len(),
83+
"stats length must match the post-order stats layout for the file dtype"
84+
);
6885

69-
let dtypes = struct_fields.fields().collect();
86+
let (paths, dtypes): (Vec<FieldPath>, Vec<DType>) = layout.into_iter().unzip();
7087

71-
Self { stats, dtypes }
72-
} else {
73-
assert_eq!(
74-
stats.len(),
75-
1,
76-
"non-struct dtype must have exactly 1 statistic"
77-
);
78-
79-
Self {
80-
stats,
81-
dtypes: Arc::new([file_dtype.clone()]),
82-
}
88+
Self {
89+
stats,
90+
dtypes: dtypes.into(),
91+
paths: paths.into(),
8392
}
8493
}
8594

8695
/// Creates [`FileStatistics`] from a flatbuffers [`fb::FileStatistics<'a>`].
87-
///
88-
/// If the [`DType`] of the file is a [`DType::Struct`], then there must be the same number of
89-
/// file stats in the flatbuffer. Otherwise, there must be only 1 statistic.
9096
pub fn from_flatbuffer<'a>(
9197
fb: &fb::FileStatistics<'a>,
9298
file_dtype: &DType,
9399
session: &VortexSession,
94100
) -> VortexResult<Self> {
95101
let field_stats = fb.field_stats().unwrap_or_default();
102+
let mut array_stats: Vec<ArrayStats> = field_stats.iter().collect();
96103

104+
if fb.is_nested() {
105+
let layout = postorder_stats_layout(file_dtype);
106+
vortex_ensure_eq!(array_stats.len(), layout.len());
107+
108+
let mut stats_sets = Vec::with_capacity(array_stats.len());
109+
let mut dtypes = Vec::with_capacity(layout.len());
110+
let mut paths = Vec::with_capacity(layout.len());
111+
for (array_stat, (path, dtype)) in array_stats.into_iter().zip(layout) {
112+
stats_sets.push(StatsSet::from_flatbuffer(&array_stat, &dtype, session)?);
113+
dtypes.push(dtype);
114+
paths.push(path);
115+
}
116+
117+
return Ok(Self {
118+
stats: stats_sets.into(),
119+
dtypes: dtypes.into(),
120+
paths: paths.into(),
121+
});
122+
}
123+
124+
// Legacy (pre-nested-stats) layout: top-level struct fields only, or a single entry for a
125+
// non-struct root dtype.
97126
if let DType::Struct(struct_fields, _) = file_dtype {
98-
vortex_ensure_eq!(field_stats.len(), struct_fields.nfields());
127+
vortex_ensure_eq!(array_stats.len(), struct_fields.nfields());
99128

100-
let stats_sets: Arc<[StatsSet]> = field_stats
129+
let stats_sets: Arc<[StatsSet]> = array_stats
101130
.into_iter()
102131
.zip(struct_fields.fields())
103132
.map(|(array_stat, field_dtype)| {
@@ -106,20 +135,29 @@ impl FileStatistics {
106135
.try_collect()?;
107136

108137
let dtypes = struct_fields.fields().collect();
138+
let paths = struct_fields
139+
.names()
140+
.iter()
141+
.map(|name| FieldPath::from_name(name.clone()))
142+
.collect();
109143

110144
Ok(Self {
111145
stats: stats_sets,
112146
dtypes,
147+
paths,
113148
})
114149
} else {
115-
vortex_ensure_eq!(field_stats.len(), 1);
150+
vortex_ensure_eq!(array_stats.len(), 1);
116151

117-
let array_stat = field_stats.get(0);
152+
let array_stat = array_stats
153+
.pop()
154+
.vortex_expect("we just checked that there was 1 field");
118155
let stats_set = StatsSet::from_flatbuffer(&array_stat, file_dtype, session)?;
119156

120157
Ok(Self {
121158
stats: Arc::new([stats_set]),
122159
dtypes: Arc::new([file_dtype.clone()]),
160+
paths: Arc::new([FieldPath::root()]),
123161
})
124162
}
125163
}
@@ -134,6 +172,11 @@ impl FileStatistics {
134172
&self.dtypes
135173
}
136174

175+
/// Returns a reference to the field paths.
176+
pub fn paths(&self) -> &Arc<[FieldPath]> {
177+
&self.paths
178+
}
179+
137180
/// Returns the statistics and data type for a specific field.
138181
///
139182
/// # Panics
@@ -142,6 +185,14 @@ impl FileStatistics {
142185
pub fn get(&self, field_idx: usize) -> (&StatsSet, &DType) {
143186
(&self.stats[field_idx], &self.dtypes[field_idx])
144187
}
188+
189+
/// Returns the statistics and data type for the field at the given path, if present.
190+
pub fn get_by_path(&self, path: &FieldPath) -> Option<(&StatsSet, &DType)> {
191+
self.paths
192+
.iter()
193+
.position(|p| p == path)
194+
.map(|idx| (&self.stats[idx], &self.dtypes[idx]))
195+
}
145196
}
146197

147198
impl<'a> IntoIterator for &'a FileStatistics {
@@ -173,7 +224,101 @@ impl WriteFlatBuffer for FileStatistics {
173224
fbb,
174225
&fb::FileStatisticsArgs {
175226
field_stats: Some(field_stats),
227+
is_nested: true,
176228
},
177229
))
178230
}
179231
}
232+
233+
#[cfg(test)]
234+
mod tests {
235+
use flatbuffers::FlatBufferBuilder;
236+
use vortex_array::array_session;
237+
use vortex_array::dtype::FieldPath;
238+
use vortex_array::dtype::Nullability;
239+
use vortex_array::dtype::PType;
240+
use vortex_array::expr::stats::Precision;
241+
use vortex_array::expr::stats::Stat;
242+
use vortex_array::flatbuffers::WriteFlatBufferExt;
243+
use vortex_array::scalar::ScalarValue;
244+
245+
use super::*;
246+
247+
fn i32_dtype() -> DType {
248+
DType::Primitive(PType::I32, Nullability::NonNullable)
249+
}
250+
251+
#[test]
252+
fn nested_round_trip_resolves_by_path() -> VortexResult<()> {
253+
let session = array_session();
254+
let inner = DType::struct_([("b", i32_dtype())], Nullability::Nullable);
255+
let file_dtype = DType::struct_([("a", inner)], Nullability::NonNullable);
256+
257+
// Layout: [a.b, a] (a's own null-count entry trails its child).
258+
let mut b_stats = StatsSet::default();
259+
b_stats.set(Stat::Min, Precision::exact(ScalarValue::from(1i32)));
260+
let mut a_stats = StatsSet::default();
261+
a_stats.set(Stat::NullCount, Precision::exact(ScalarValue::from(1u64)));
262+
263+
let file_stats = FileStatistics::new_with_dtype(Arc::from([b_stats, a_stats]), &file_dtype);
264+
265+
let bytes = file_stats.write_flatbuffer_bytes()?;
266+
let fb = flatbuffers::root::<fb::FileStatistics>(bytes.as_ref())
267+
.vortex_expect("valid flatbuffer");
268+
assert!(fb.is_nested());
269+
270+
let read_back = FileStatistics::from_flatbuffer(&fb, &file_dtype, &session)?;
271+
272+
let (b, _) = read_back
273+
.get_by_path(&FieldPath::from_name("a").push("b"))
274+
.expect("a.b stats");
275+
assert_eq!(b.get(Stat::Min).as_exact(), Some(ScalarValue::from(1i32)));
276+
277+
let (a, _) = read_back
278+
.get_by_path(&FieldPath::from_name("a"))
279+
.expect("a's own null-count stats");
280+
assert_eq!(
281+
a.get(Stat::NullCount).as_exact(),
282+
Some(ScalarValue::from(1u64))
283+
);
284+
285+
assert!(read_back.get_by_path(&FieldPath::root()).is_none());
286+
287+
Ok(())
288+
}
289+
290+
#[test]
291+
fn legacy_non_nested_footer_still_parses() -> VortexResult<()> {
292+
// Simulates a footer written before nested field stats existed: `is_nested` is absent
293+
// (defaults to false), and `field_stats` holds one entry per top-level struct field.
294+
let session = array_session();
295+
let file_dtype = DType::struct_([("col", i32_dtype())], Nullability::NonNullable);
296+
297+
let mut stats = StatsSet::default();
298+
stats.set(Stat::Min, Precision::exact(ScalarValue::from(7i32)));
299+
300+
let mut fbb = FlatBufferBuilder::new();
301+
let array_stats = stats.write_flatbuffer(&mut fbb)?;
302+
let field_stats = fbb.create_vector(&[array_stats]);
303+
let root = fb::FileStatistics::create(
304+
&mut fbb,
305+
&fb::FileStatisticsArgs {
306+
field_stats: Some(field_stats),
307+
is_nested: false,
308+
},
309+
);
310+
fbb.finish_minimal(root);
311+
let bytes = fbb.finished_data().to_vec();
312+
313+
let fb = flatbuffers::root::<fb::FileStatistics>(&bytes).vortex_expect("valid flatbuffer");
314+
assert!(!fb.is_nested());
315+
316+
let read_back = FileStatistics::from_flatbuffer(&fb, &file_dtype, &session)?;
317+
let (col, _) = read_back
318+
.get_by_path(&FieldPath::from_name("col"))
319+
.expect("col stats");
320+
assert_eq!(col.get(Stat::Min).as_exact(), Some(ScalarValue::from(7i32)));
321+
322+
Ok(())
323+
}
324+
}

‎vortex-file/src/pruning.rs‎

Lines changed: 3 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ use vortex_array::arrays::ConstantArray;
99
use vortex_array::arrays::NullArray;
1010
use vortex_array::dtype::DType;
1111
use vortex_array::dtype::FieldPath;
12-
use vortex_array::dtype::StructFields;
1312
use vortex_array::expr::BoundExpression;
1413
use vortex_array::expr::bound::lit;
1514
use vortex_array::expr::stats::Stat;
@@ -29,17 +28,13 @@ pub(crate) fn can_prune_file_stats(
2928
expr: &BoundExpression,
3029
row_count: u64,
3130
file_stats: &FileStatistics,
32-
struct_fields: &StructFields,
3331
session: &VortexSession,
3432
) -> VortexResult<bool> {
3533
let Some(pruning_expr) = expr.falsify(session)? else {
3634
return Ok(false);
3735
};
3836

39-
let binder = FileStatsBinder {
40-
file_stats,
41-
struct_fields,
42-
};
37+
let binder = FileStatsBinder { file_stats };
4338
let pruning_expr = bind_stats(pruning_expr, &binder)?;
4439

4540
if let Some(result) = pruning_expr.as_opt::<Literal>() {
@@ -62,7 +57,6 @@ pub(crate) fn can_prune_file_stats(
6257

6358
struct FileStatsBinder<'a> {
6459
file_stats: &'a FileStatistics,
65-
struct_fields: &'a StructFields,
6660
}
6761

6862
impl StatBinder for FileStatsBinder<'_> {
@@ -84,18 +78,10 @@ impl StatBinder for FileStatsBinder<'_> {
8478

8579
impl FileStatsBinder<'_> {
8680
fn stat_ref(&self, field_path: &FieldPath, stat: Stat) -> Option<BoundExpression> {
87-
// FileStats currently only holds top-level field statistics.
88-
if field_path.parts().len() != 1 {
89-
return None;
90-
}
91-
92-
let field_name = field_path.parts()[0].as_name()?;
93-
let field_idx = self.struct_fields.find(field_name)?;
94-
let field_stats = self.file_stats.stats_sets().get(field_idx)?;
81+
let (field_stats, field_dtype) = self.file_stats.get_by_path(field_path)?;
9582

9683
let stat_value = field_stats.get(stat).as_exact()?;
97-
let field_dtype = self.struct_fields.field_by_index(field_idx)?;
98-
let stat_dtype = stat.dtype(&field_dtype)?;
84+
let stat_dtype = stat.dtype(field_dtype)?;
9985
let stat_scalar = Scalar::try_new(stat_dtype, Some(stat_value)).ok()?;
10086

10187
Some(lit(stat_scalar))

0 commit comments

Comments
 (0)