Skip to content

Commit 22ecef2

Browse files
committed
Support file stats on nested struct fields (vortex-data#6389)
Store file-level statistics via a post-order walk of the DType tree, so nested struct fields get whole-file pruning instead of only top-level columns. Each leaf gets a stats entry, and each nullable struct gets a trailing null-count entry. A new `is_nested` footer flag keeps old files readable via the legacy top-level-only layout. Signed-off-by: "Thor" <[email protected]> Signed-off-by: Thor <[email protected]>
1 parent 3cc1a5f commit 22ecef2

22 files changed

Lines changed: 2716 additions & 766 deletions

File tree

‎vortex-array/src/stats/mod.rs‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -33,16 +33,6 @@ use vortex_error::VortexExpect;
3333

3434
use crate::expr::stats::Stat;
3535

36-
/// Statistics that are used for pruning files (i.e., we want to ensure they are computed when compressing/writing).
37-
/// Sum is included for boolean arrays.
38-
pub const PRUNING_STATS: &[Stat] = &[
39-
Stat::Min,
40-
Stat::Max,
41-
Stat::Sum,
42-
Stat::NullCount,
43-
Stat::NaNCount,
44-
];
45-
4636
pub fn as_stat_bitset_bytes(stats: &[Stat]) -> Vec<u8> {
4737
let max_stat = u8::from(last::<Stat>().vortex_expect("last stat")) as usize + 1;
4838
// TODO(ngates): use vortex-buffer::BitBuffer

‎vortex-datafusion/src/persistent/format.rs‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,12 @@ use futures::stream;
4545
use object_store::ObjectMeta;
4646
use object_store::ObjectStore;
4747
use vortex::VortexSessionDefault;
48+
use vortex::aggregate_fn::AggregateFnVTableExt;
49+
use vortex::aggregate_fn::EmptyOptions;
50+
use vortex::aggregate_fn::fns::is_constant::IsConstant;
4851
use vortex::array::memory::MemorySessionExt;
4952
use vortex::dtype::DType;
53+
use vortex::dtype::FieldPath;
5054
use vortex::dtype::PType;
5155
use vortex::error::VortexExpect;
5256
use vortex::error::VortexResult;
@@ -605,7 +609,7 @@ impl FileFormat for VortexFormat {
605609
})
606610
});
607611

608-
let (dtype, file_stats, row_count) = match cached_metadata {
612+
let (_dtype, file_stats, row_count) = match cached_metadata {
609613
Some(metadata) => metadata,
610614
None => {
611615
// Not entry - open the file
@@ -642,10 +646,6 @@ impl FileFormat for VortexFormat {
642646
}
643647
};
644648

645-
let struct_dtype = dtype
646-
.as_struct_fields_opt()
647-
.vortex_expect("dtype is not a struct");
648-
649649
// Evaluate the statistics for each column that we are able to return to DataFusion.
650650
let Some(file_stats) = file_stats else {
651651
// If the file has no column stats, the best we can do is return a row count.
@@ -668,12 +668,23 @@ impl FileFormat for VortexFormat {
668668
for field in table_schema.fields().iter() {
669669
// If the column does not exist, continue. This can happen if the schema has evolved
670670
// but we have not yet updated the Vortex file.
671-
let Some(col_idx) = struct_dtype.find(field.name()) else {
671+
let Some((aggregates, stats_dtype)) =
672+
file_stats.get_by_path(&FieldPath::from_name(field.name().as_str()))
673+
else {
672674
// The default sets all statistics to `Precision<Absent>`.
673675
column_statistics.push(ColumnStatistics::default());
674676
continue;
675677
};
676-
let (stats_set, stats_dtype) = file_stats.get(col_idx);
678+
let mut stats_set = aggregates.to_stats_set();
679+
// `IsConstant` has no aggregate behind its stat slot, so `to_stats_set` omits it.
680+
if let Some(is_constant) = aggregates
681+
.get(&IsConstant.bind(EmptyOptions))
682+
.as_exact()
683+
.and_then(|is_constant| bool::try_from(&is_constant).ok())
684+
{
685+
stats_set.set(Stat::IsConstant, Precision::exact(is_constant));
686+
}
687+
let stats_set = &stats_set;
677688

678689
// Update the total size in bytes.
679690
let column_size =

‎vortex-duckdb/cpp/copy_function.cpp‎

Lines changed: 7 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
#include "duckdb/main/capi/capi_internal.hpp"
1414
#include "duckdb/main/client_context.hpp"
1515
#include "duckdb/main/connection.hpp"
16-
#include "duckdb/parser/keyword_helper.hpp"
1716
#include "duckdb/parser/parsed_data/create_copy_function_info.hpp"
1817

1918
unique_ptr<FunctionData> copy_to_bind(ClientContext &,
@@ -42,7 +41,7 @@ unique_ptr<FunctionData> copy_to_bind(ClientContext &,
4241
throw BinderException(IntoErrString(error_out));
4342
}
4443
auto cdata = unique_ptr<CData>(reinterpret_cast<CData *>(ffi_bind_data));
45-
return make_uniq<VortexCopyBindData>(std::move(cdata), names);
44+
return make_uniq<VortexCopyBindData>(std::move(cdata));
4645
}
4746

4847
unique_ptr<GlobalFunctionData>
@@ -88,7 +87,7 @@ void copy_to_get_written_statistics(ClientContext &,
8887
gstate.Cast<VortexCopyGlobalState>().written_stats = &statistics;
8988
}
9089

91-
void copy_to_finalize(ClientContext &, FunctionData &bind_data, GlobalFunctionData &gstate) {
90+
void copy_to_finalize(ClientContext &, FunctionData &, GlobalFunctionData &gstate) {
9291
auto &global = gstate.Cast<VortexCopyGlobalState>();
9392
void *const ffi_global = global.ffi_global->DataPtr();
9493

@@ -101,23 +100,16 @@ void copy_to_finalize(ClientContext &, FunctionData &bind_data, GlobalFunctionDa
101100
if (!global.written_stats) {
102101
return;
103102
}
104-
const auto &names = bind_data.Cast<VortexCopyBindData>().column_names;
105103
duckdb_vx_written_file_statistics file_stats;
106104
if (!duckdb_copy_function_get_written_file_statistics(ffi_global, &file_stats)) {
107105
throw InternalException("vortex COPY: written statistics were requested but not produced");
108106
}
109-
if (file_stats.num_columns != names.size()) {
110-
throw InternalException("vortex COPY: %llu statistics columns for %llu written columns",
111-
file_stats.num_columns,
112-
names.size());
113-
}
114107
D_ASSERT(global.written_stats != nullptr);
115108
global.written_stats->row_count = file_stats.row_count;
116109
global.written_stats->file_size_bytes = file_stats.file_size_bytes;
117110
global.written_stats->footer_size_bytes = Value::UBIGINT(file_stats.footer_size_bytes);
118-
// Keyed by top-level column name only. The vortex footer reports one statistics set per
119-
// top-level field, so nested struct/list leaf columns get no statistics here (unlike parquet,
120-
// which recurses to leaf paths). Flat tables are fully covered.
111+
// One entry per leaf column: nested struct fields are reported at their full path and structs
112+
// get no entry of their own, as in parquet.
121113
for (idx_t i = 0; i < file_stats.num_columns; i++) {
122114
duckdb_vx_written_column_statistics col_stats {};
123115
duckdb_vx_error col_error = nullptr;
@@ -146,9 +138,9 @@ void copy_to_finalize(ClientContext &, FunctionData &bind_data, GlobalFunctionDa
146138
if (col_stats.has_nan_stat) {
147139
column["has_nan"] = Value::BOOLEAN(col_stats.contains_nan);
148140
}
149-
150-
string quoted = SQLQuotedIdentifier::ToString(names[i]);
151-
global.written_stats->column_statistics.emplace(std::move(quoted), std::move(column));
141+
auto column_key = reinterpret_cast<Value *>(col_stats.column_key)->ToString();
142+
duckdb_destroy_value(&col_stats.column_key);
143+
global.written_stats->column_statistics.emplace(std::move(column_key), std::move(column));
152144
}
153145
}
154146

‎vortex-duckdb/cpp/include/copy_function.hpp‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,9 @@
88
using namespace duckdb;
99

1010
struct VortexCopyBindData final : TableFunctionData {
11-
VortexCopyBindData(unique_ptr<CData> ffi_bind, vector<Identifier> column_names)
12-
: ffi_bind(std::move(ffi_bind)), column_names(std::move(column_names)) {
11+
VortexCopyBindData(unique_ptr<CData> ffi_bind) : ffi_bind(std::move(ffi_bind)) {
1312
}
1413
unique_ptr<CData> ffi_bind;
15-
vector<Identifier> column_names;
1614
};
1715

1816
struct VortexCopyGlobalState final : GlobalFunctionData {

‎vortex-duckdb/cpp/include/table_function.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,9 @@ typedef struct {
8383

8484
// Per-column statistics of a written Vortex file.
8585
typedef struct {
86+
// Owned VARCHAR value holding the column's quoted, dot-separated path (e.g. `"s"."b"`), the key
87+
// format DuckLake expects; the caller must destroy it.
88+
duckdb_value column_key;
8689
// Owned values, null if absent; the caller must destroy them.
8790
duckdb_value min;
8891
duckdb_value max;

‎vortex-duckdb/src/column_statistics.rs‎

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,22 @@
11
// SPDX-License-Identifier: Apache-2.0
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

4+
use vortex::aggregate_fn::AggregateFnRef;
5+
use vortex::aggregate_fn::AggregateFnVTableExt;
6+
use vortex::aggregate_fn::EmptyOptions;
7+
use vortex::aggregate_fn::NumericalAggregateOpts;
8+
use vortex::aggregate_fn::fns::max::Max;
9+
use vortex::aggregate_fn::fns::min::Min;
10+
use vortex::aggregate_fn::fns::nan_count::NanCount;
11+
use vortex::aggregate_fn::fns::null_count::NullCount;
12+
use vortex::aggregate_fn::fns::uncompressed_size_in_bytes::UncompressedSizeInBytes;
13+
use vortex::aggregate_fn::fns::uncompressed_size_in_bytes::UncompressedSizeOpts;
414
use vortex::array::match_each_float_ptype;
5-
use vortex::array::stats::StatsSet;
615
use vortex::dtype::DType;
7-
use vortex::dtype::PType;
816
use vortex::error::VortexExpect as _;
917
use vortex::error::VortexResult;
1018
use vortex::expr::stats::Precision;
11-
use vortex::expr::stats::Stat;
19+
use vortex::layout::layouts::file_stats::AggregateStats;
1220
use vortex::scalar::Scalar;
1321
use vortex::scalar::ScalarValue;
1422

@@ -74,33 +82,25 @@ pub struct ColumnStatisticsAggregate {
7482
}
7583

7684
impl ColumnStatisticsAggregate {
77-
pub fn new(stats: &StatsSet) -> Self {
78-
let min = match stats.get(Stat::Min) {
79-
Precision::Exact(min) => Some(min),
80-
_ => None,
81-
};
82-
let max = match stats.get(Stat::Max) {
83-
Precision::Exact(max) => Some(max),
85+
pub fn new(aggregates: &AggregateStats) -> Self {
86+
let exact = |aggregate_fn: AggregateFnRef| match aggregates.get(&aggregate_fn) {
87+
Precision::Exact(value) => value.into_value(),
8488
_ => None,
8589
};
8690

87-
let max_string_length =
88-
if let Precision::Exact(value) = stats.get(Stat::UncompressedSizeInBytes) {
89-
// DuckDB's string length is u32
90-
#[allow(clippy::cast_possible_truncation)]
91-
Some(value.as_primitive().as_u64().vortex_expect("not a u64") as u32)
92-
} else {
93-
None
94-
};
91+
let min = exact(Min.bind(NumericalAggregateOpts::skip_nans()));
92+
let max = exact(Max.bind(NumericalAggregateOpts::skip_nans()));
9593

96-
let has_null = match stats.get(Stat::NullCount) {
97-
Precision::Exact(cnt) => cnt.as_primitive().as_u64().vortex_expect("not a u64") > 0,
98-
_ => true,
99-
};
94+
// DuckDB's string length is u32
95+
#[allow(clippy::cast_possible_truncation)]
96+
let max_string_length = exact(UncompressedSizeInBytes.bind(UncompressedSizeOpts::exact()))
97+
.map(|value| value.as_primitive().as_u64().vortex_expect("not a u64") as u32);
98+
99+
let has_null = exact(NullCount.bind(EmptyOptions))
100+
.is_none_or(|count| count.as_primitive().as_u64().vortex_expect("not a u64") > 0);
100101

101-
let nan_count = stats
102-
.get_as::<u64>(Stat::NaNCount, &PType::U64.into())
103-
.as_exact();
102+
let nan_count = exact(NanCount.bind(EmptyOptions))
103+
.map(|count| count.as_primitive().as_u64().vortex_expect("not a u64"));
104104

105105
Self {
106106
min,

0 commit comments

Comments
 (0)