Skip to content

Commit 9452e32

Browse files
committed
Support file stats on nested struct fields (vortex-data#6389)
Store file-level statistics via a post-order walk of the DType tree, so nested struct fields get whole-file pruning instead of only top-level columns. Each leaf gets a stats entry, and each nullable struct gets a trailing null-count entry. A new `is_nested` footer flag keeps old files readable via the legacy top-level-only layout. Signed-off-by: "Thor" <[email protected]> Signed-off-by: Thor <[email protected]>
1 parent 4224390 commit 9452e32

8 files changed

Lines changed: 545 additions & 136 deletions

File tree

‎vortex-file/flatbuffers/vortex-file/footer.fbs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,15 @@ table PostscriptSegment {
5858
table FileStatistics {
5959
/// Statistics for each field in the root schema. If the root schema is not a struct, there will
6060
/// be a single entry in this array.
61+
///
62+
/// When `is_nested` is true, entries follow a post-order walk of the root `DType` tree: each
63+
/// leaf field gets an entry, and each nullable struct additionally gets a trailing null-count
64+
/// entry inserted after its children's entries.
6165
field_stats: [ArrayStats];
66+
/// Whether `field_stats` follows the post-order nested-struct layout (true) or the legacy
67+
/// top-level-fields-only layout (false). Defaults to false for backward compatibility with
68+
/// files written before nested field stats were supported.
69+
is_nested: bool = false;
6270
}
6371

6472
/// The `Registry` object stores dictionary-encoded configuration for segments,

‎vortex-file/src/file.rs‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -227,19 +227,14 @@ impl VortexFile {
227227
/// Row-count-aware pruning predicates are evaluated with the file's total
228228
/// row count as their scope.
229229
pub fn can_prune(&self, filter: &Expression) -> VortexResult<bool> {
230-
let Some((stats, fields)) = self
231-
.footer
232-
.statistics()
233-
.zip(self.footer.dtype().as_struct_fields_opt())
234-
else {
230+
let Some(stats) = self.footer.statistics() else {
235231
return Ok(false);
236232
};
237233

238234
can_prune_file_stats(
239235
&filter.bind(self.footer.dtype())?,
240236
self.footer.row_count(),
241237
stats,
242-
fields,
243238
&self.session,
244239
)
245240
}

‎vortex-file/src/flatbuffers/generated/footer.rs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -816,6 +816,7 @@ impl<'a> ::flatbuffers::Follow<'a> for FileStatistics<'a> {
816816

817817
impl<'a> FileStatistics<'a> {
818818
pub const VT_FIELD_STATS: ::flatbuffers::VOffsetT = 4;
819+
pub const VT_IS_NESTED: ::flatbuffers::VOffsetT = 6;
819820

820821
#[inline]
821822
pub unsafe fn init_from_table(table: ::flatbuffers::Table<'a>) -> Self {
@@ -828,19 +829,34 @@ impl<'a> FileStatistics<'a> {
828829
) -> ::flatbuffers::WIPOffset<FileStatistics<'bldr>> {
829830
let mut builder = FileStatisticsBuilder::new(_fbb);
830831
if let Some(x) = args.field_stats { builder.add_field_stats(x); }
832+
builder.add_is_nested(args.is_nested);
831833
builder.finish()
832834
}
833835

834836

835837
/// Statistics for each field in the root schema. If the root schema is not a struct, there will
836838
/// be a single entry in this array.
839+
///
840+
/// When `is_nested` is true, entries follow a post-order walk of the root `DType` tree: each
841+
/// leaf field gets an entry, and each nullable struct additionally gets a trailing null-count
842+
/// entry inserted after its children's entries.
837843
#[inline]
838844
pub fn field_stats(&self) -> Option<::flatbuffers::Vector<'a, ::flatbuffers::ForwardsUOffset<ArrayStats<'a>>>> {
839845
// Safety:
840846
// Created from valid Table for this object
841847
// which contains a valid value in this slot
842848
unsafe { self._tab.get::<::flatbuffers::ForwardsUOffset<::flatbuffers::Vector<'a, ::flatbuffers::ForwardsUOffset<ArrayStats>>>>(FileStatistics::VT_FIELD_STATS, None)}
843849
}
850+
/// Whether `field_stats` follows the post-order nested-struct layout (true) or the legacy
851+
/// top-level-fields-only layout (false). Defaults to false for backward compatibility with
852+
/// files written before nested field stats were supported.
853+
#[inline]
854+
pub fn is_nested(&self) -> bool {
855+
// Safety:
856+
// Created from valid Table for this object
857+
// which contains a valid value in this slot
858+
unsafe { self._tab.get::<bool>(FileStatistics::VT_IS_NESTED, Some(false)).unwrap()}
859+
}
844860
}
845861

846862
impl ::flatbuffers::Verifiable for FileStatistics<'_> {
@@ -850,18 +866,21 @@ impl ::flatbuffers::Verifiable for FileStatistics<'_> {
850866
) -> Result<(), ::flatbuffers::InvalidFlatbuffer> {
851867
v.visit_table(pos)?
852868
.visit_field::<::flatbuffers::ForwardsUOffset<::flatbuffers::Vector<'_, ::flatbuffers::ForwardsUOffset<ArrayStats>>>>("field_stats", Self::VT_FIELD_STATS, false)?
869+
.visit_field::<bool>("is_nested", Self::VT_IS_NESTED, false)?
853870
.finish();
854871
Ok(())
855872
}
856873
}
857874
pub struct FileStatisticsArgs<'a> {
858875
pub field_stats: Option<::flatbuffers::WIPOffset<::flatbuffers::Vector<'a, ::flatbuffers::ForwardsUOffset<ArrayStats<'a>>>>>,
876+
pub is_nested: bool,
859877
}
860878
impl<'a> Default for FileStatisticsArgs<'a> {
861879
#[inline]
862880
fn default() -> Self {
863881
FileStatisticsArgs {
864882
field_stats: None,
883+
is_nested: false,
865884
}
866885
}
867886
}
@@ -876,6 +895,10 @@ impl<'a: 'b, 'b, A: ::flatbuffers::Allocator + 'a> FileStatisticsBuilder<'a, 'b,
876895
self.fbb_.push_slot_always::<::flatbuffers::WIPOffset<_>>(FileStatistics::VT_FIELD_STATS, field_stats);
877896
}
878897
#[inline]
898+
pub fn add_is_nested(&mut self, is_nested: bool) {
899+
self.fbb_.push_slot::<bool>(FileStatistics::VT_IS_NESTED, is_nested, false);
900+
}
901+
#[inline]
879902
pub fn new(_fbb: &'b mut ::flatbuffers::FlatBufferBuilder<'a, A>) -> FileStatisticsBuilder<'a, 'b, A> {
880903
let start = _fbb.start_table();
881904
FileStatisticsBuilder {
@@ -894,6 +917,7 @@ impl ::core::fmt::Debug for FileStatistics<'_> {
894917
fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
895918
let mut ds = f.debug_struct("FileStatistics");
896919
ds.field("field_stats", &self.field_stats());
920+
ds.field("is_nested", &self.is_nested());
897921
ds.finish()
898922
}
899923
}

0 commit comments

Comments
 (0)