Skip to content

Commit 5b899f1

Browse files
committed
Store finalized aggregate results in file footers
Signed-off-by: Connor Tsui <[email protected]>
1 parent 1643e97 commit 5b899f1

16 files changed

Lines changed: 721 additions & 186 deletions

File tree

‎vortex-array/src/expr/stats/mod.rs‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -281,7 +281,7 @@ impl Display for Stat {
281281
}
282282

283283
#[cfg(test)]
284-
mod test {
284+
mod tests {
285285
use enum_iterator::all;
286286
use rstest::rstest;
287287

@@ -316,9 +316,9 @@ mod test {
316316
}
317317

318318
#[rstest]
319-
#[case(Stat::Min, Min.bind(NumericalAggregateOpts::include_nans()))]
320-
#[case(Stat::Max, Max.bind(NumericalAggregateOpts::include_nans()))]
321-
#[case(Stat::Sum, Sum.bind(NumericalAggregateOpts::include_nans()))]
319+
#[case::min(Stat::Min, Min.bind(NumericalAggregateOpts::include_nans()))]
320+
#[case::max(Stat::Max, Max.bind(NumericalAggregateOpts::include_nans()))]
321+
#[case::sum(Stat::Sum, Sum.bind(NumericalAggregateOpts::include_nans()))]
322322
fn finalized_numerical_keys_skip_nans(
323323
#[case] stat: Stat,
324324
#[case] include_nans: crate::aggregate_fn::AggregateFnRef,
@@ -331,9 +331,9 @@ mod test {
331331
}
332332

333333
#[rstest]
334-
#[case(Stat::IsConstant)]
335-
#[case(Stat::IsSorted)]
336-
#[case(Stat::IsStrictSorted)]
334+
#[case::constant(Stat::IsConstant)]
335+
#[case::non_strict_sorted(Stat::IsSorted)]
336+
#[case::strict_sorted(Stat::IsStrictSorted)]
337337
fn finalized_flags_do_not_supply_partial_requests(#[case] stat: Stat) {
338338
let key = stat.finalized_aggregate_fn();
339339
assert!(stat.aggregate_fn().is_none());
Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
use flatbuffers::FlatBufferBuilder;
5+
use flatbuffers::WIPOffset;
6+
use vortex_error::VortexResult;
7+
use vortex_error::vortex_bail;
8+
use vortex_error::vortex_ensure;
9+
use vortex_error::vortex_err;
10+
use vortex_session::VortexSession;
11+
12+
use crate::dtype::DType;
13+
use crate::dtype::PType;
14+
use crate::expr::stats::Precision;
15+
use crate::expr::stats::Stat;
16+
use crate::flatbuffers::array as fba;
17+
use crate::scalar::Scalar;
18+
use crate::scalar::ScalarValue;
19+
use crate::stats::AggregateResults;
20+
use crate::stats::StatsSet;
21+
22+
/// Convert legacy writer output without reading array values or computing aggregates.
23+
///
24+
/// Only values present in the legacy set are supplied. The legacy set cannot contain nulls.
25+
pub fn legacy_stats_to_results(
26+
input_dtype: &DType,
27+
stats: &StatsSet,
28+
) -> VortexResult<AggregateResults> {
29+
let mut entries = Vec::new();
30+
for stat in Stat::all() {
31+
let value = stats.get(stat);
32+
if value.is_absent() {
33+
continue;
34+
}
35+
let dtype = historical_dtype(stat, input_dtype)
36+
.ok_or_else(|| vortex_err!("File statistic {stat} does not support {input_dtype}"))?;
37+
let value = value
38+
.map(|value| Scalar::try_new(dtype, Some(value)))
39+
.transpose()?;
40+
entries.push((stat.finalized_aggregate_fn().clone(), value));
41+
}
42+
Ok(AggregateResults::from_validated(entries))
43+
}
44+
45+
/// Decode finalized results using historical field types.
46+
///
47+
/// Present nullable nulls retain their precision. Missing fields are absent. Nulls in non-nullable
48+
/// fields and invalid precision tags are rejected. Numerical keys use NaN-skipping options.
49+
pub fn read_summary(
50+
fb: &fba::ArrayStats<'_>,
51+
input_dtype: &DType,
52+
session: &VortexSession,
53+
) -> VortexResult<AggregateResults> {
54+
let mut entries = Vec::new();
55+
for stat in Stat::all() {
56+
let Some(dtype) = historical_dtype(stat, input_dtype) else {
57+
continue;
58+
};
59+
let value = match stat {
60+
Stat::Min | Stat::Max | Stat::Sum => {
61+
let bytes = match stat {
62+
Stat::Min => fb.min(),
63+
Stat::Max => fb.max(),
64+
_ => fb.sum(),
65+
};
66+
let Some(bytes) = bytes else { continue };
67+
let value = ScalarValue::from_proto_bytes(bytes.bytes(), &dtype, session)?;
68+
let value = Scalar::try_new(dtype, value)?;
69+
let precision = match stat {
70+
Stat::Min => fb.min_precision(),
71+
Stat::Max => fb.max_precision(),
72+
_ => fba::Precision::Exact,
73+
};
74+
match precision {
75+
fba::Precision::Exact => Precision::Exact(value),
76+
fba::Precision::Inexact => Precision::Inexact(value),
77+
other => vortex_bail!("Corrupted {stat} precision: {other:?}"),
78+
}
79+
}
80+
Stat::IsConstant | Stat::IsSorted | Stat::IsStrictSorted => {
81+
let value = match stat {
82+
Stat::IsConstant => fb.is_constant(),
83+
Stat::IsSorted => fb.is_sorted(),
84+
_ => fb.is_strict_sorted(),
85+
};
86+
let Some(value) = value else { continue };
87+
Precision::Exact(Scalar::try_new(dtype, Some(value.into()))?)
88+
}
89+
Stat::NullCount | Stat::NaNCount | Stat::UncompressedSizeInBytes => {
90+
let value = match stat {
91+
Stat::NullCount => fb.null_count(),
92+
Stat::NaNCount => fb.nan_count(),
93+
_ => fb.uncompressed_size_in_bytes(),
94+
};
95+
let Some(value) = value else { continue };
96+
Precision::Exact(Scalar::try_new(dtype, Some(value.into()))?)
97+
}
98+
};
99+
entries.push((stat.finalized_aggregate_fn().clone(), value));
100+
}
101+
Ok(AggregateResults::from_validated(entries))
102+
}
103+
104+
/// Write finalized results into the existing footer fields.
105+
///
106+
/// Only fixed historical functions and options are supported. Only extrema can be inexact.
107+
/// Scalar types must match the historical field, allowing a change to root nullability. Nulls
108+
/// require a nullable historical field. Legacy `StatsSet` decoding omits those null results.
109+
pub fn write_summary<'fb>(
110+
results: &AggregateResults,
111+
input_dtype: &DType,
112+
fbb: &mut FlatBufferBuilder<'fb>,
113+
) -> VortexResult<WIPOffset<fba::ArrayStats<'fb>>> {
114+
for (aggregate, _) in results.iter() {
115+
vortex_ensure!(
116+
Stat::all().any(|stat| stat.finalized_aggregate_fn() == aggregate),
117+
"File footer cannot represent aggregate {aggregate}"
118+
);
119+
}
120+
121+
let mut args = fba::ArrayStatsArgs::default();
122+
// Preserve the legacy serializer's scalar-vector order so unchanged writer output has the same
123+
// bytes. The table's optional numeric fields do not allocate vectors.
124+
for stat in [
125+
Stat::Min,
126+
Stat::Max,
127+
Stat::Sum,
128+
Stat::IsConstant,
129+
Stat::IsSorted,
130+
Stat::IsStrictSorted,
131+
Stat::NullCount,
132+
Stat::UncompressedSizeInBytes,
133+
Stat::NaNCount,
134+
] {
135+
let value = results.get_result(stat.finalized_aggregate_fn());
136+
let Some(scalar) = value.as_ref().into_inner() else {
137+
continue;
138+
};
139+
let dtype = historical_dtype(stat, input_dtype)
140+
.ok_or_else(|| vortex_err!("File statistic {stat} does not support {input_dtype}"))?;
141+
vortex_ensure!(
142+
scalar.dtype().as_nullable() == dtype.as_nullable(),
143+
"File statistic {stat} requires result dtype {dtype}, got {}",
144+
scalar.dtype()
145+
);
146+
let scalar = Scalar::try_new(dtype, scalar.value().cloned())?;
147+
vortex_ensure!(
148+
value.is_exact() || matches!(stat, Stat::Min | Stat::Max),
149+
"File footer cannot represent an inexact {stat} result"
150+
);
151+
match stat {
152+
Stat::Min | Stat::Max | Stat::Sum => {
153+
let bytes = ScalarValue::to_proto_bytes::<Vec<u8>>(scalar.value());
154+
let bytes = Some(fbb.create_vector(&bytes));
155+
let precision = if value.is_exact() {
156+
fba::Precision::Exact
157+
} else {
158+
fba::Precision::Inexact
159+
};
160+
match stat {
161+
Stat::Min => {
162+
args.min = bytes;
163+
args.min_precision = precision;
164+
}
165+
Stat::Max => {
166+
args.max = bytes;
167+
args.max_precision = precision;
168+
}
169+
_ => args.sum = bytes,
170+
}
171+
}
172+
Stat::IsConstant => args.is_constant = Some(bool::try_from(&scalar)?),
173+
Stat::IsSorted => args.is_sorted = Some(bool::try_from(&scalar)?),
174+
Stat::IsStrictSorted => args.is_strict_sorted = Some(bool::try_from(&scalar)?),
175+
Stat::NullCount => args.null_count = Some(u64::try_from(&scalar)?),
176+
Stat::NaNCount => args.nan_count = Some(u64::try_from(&scalar)?),
177+
Stat::UncompressedSizeInBytes => {
178+
args.uncompressed_size_in_bytes = Some(u64::try_from(&scalar)?);
179+
}
180+
}
181+
}
182+
Ok(fba::ArrayStats::create(fbb, &args))
183+
}
184+
185+
fn historical_dtype(stat: Stat, input_dtype: &DType) -> Option<DType> {
186+
match stat {
187+
// These wire fields exist for every input type, even where a current kernel declines.
188+
Stat::NullCount | Stat::NaNCount | Stat::UncompressedSizeInBytes => Some(PType::U64.into()),
189+
Stat::Sum => stat.dtype(input_dtype).or_else(|| {
190+
// Older writers also summed extension storage values.
191+
if let DType::Extension(ext) = input_dtype {
192+
historical_dtype(stat, ext.storage_dtype())
193+
} else {
194+
None
195+
}
196+
}),
197+
_ => stat.dtype(input_dtype),
198+
}
199+
}
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
//! Finalized results in historical file footer fields.
5+
//!
6+
//! The codec preserves the wire schema and historical scalar types, including types that current
7+
//! aggregate kernels cannot compute. A present nullable null is a result. A missing field is
8+
//! absent.
9+
10+
mod footer;
11+
pub use footer::legacy_stats_to_results;
12+
pub use footer::read_summary;
13+
pub use footer::write_summary;
14+
15+
#[cfg(test)]
16+
mod tests;

0 commit comments

Comments
 (0)