Skip to content

Commit a59294e

Browse files
authored
Expose aggregate statistics via Scan API (vortex-data#10838)
Returns arbitrary aggregate function partials for a given field path. This allows callers to probe for any statistics. To consider: * Exposing this as a batch endpoint? (Vec<AggregateFnRef> -> Vec<Precision<Scalar>>) * Deprecate the field_statistics function --------- Signed-off-by: Nicholas Gates <[email protected]>
1 parent 967e2f5 commit a59294e

5 files changed

Lines changed: 144 additions & 4 deletions

File tree

‎vortex-file/src/file.rs‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ use vortex_utils::aliases::hash_map::HashMap;
3232
use crate::FileStatistics;
3333
use crate::footer::Footer;
3434
use crate::pruning::can_prune_file_stats;
35+
use crate::source::FileDataSource;
3536
use crate::v2::FileStatsLayoutReader;
3637

3738
/// Represents a Vortex file, providing access to its metadata and content.
@@ -237,13 +238,14 @@ impl VortexFile {
237238
/// Create a [`DataSource`](vortex_scan::DataSource) from this file for scanning.
238239
///
239240
/// Wraps the file's layout reader with [`FileStatsLayoutReader`] (when file-level
240-
/// statistics are available) and [`LayoutReaderDataSource`].
241+
/// statistics are available) and [`LayoutReaderDataSource`]. Stored footer aggregates are
242+
/// available through the source statistics methods without reading payloads.
241243
pub fn data_source(&self) -> VortexResult<DataSourceRef> {
242244
let reader = self.layout_reader()?;
243245

244-
Ok(Arc::new(LayoutReaderDataSource::new(
245-
reader,
246-
self.session.clone(),
246+
Ok(Arc::new(FileDataSource::new(
247+
LayoutReaderDataSource::new(reader, self.session.clone()),
248+
self.file_stats().cloned(),
247249
)))
248250
}
249251

‎vortex-file/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@ mod pruning;
113113
mod read;
114114
/// Segment sources, caches, and sinks used by file readers and writers.
115115
pub mod segments;
116+
mod source;
116117
mod strategy;
117118
#[cfg(test)]
118119
mod tests;

‎vortex-file/src/source.rs‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
use async_trait::async_trait;
5+
use vortex_array::aggregate_fn::AggregateFnRef;
6+
use vortex_array::dtype::DType;
7+
use vortex_array::dtype::FieldPath;
8+
use vortex_array::expr::stats::Precision;
9+
use vortex_array::scalar::Scalar;
10+
use vortex_array::stats::StatsSet;
11+
use vortex_error::VortexResult;
12+
use vortex_layout::scan::layout::LayoutReaderDataSource;
13+
use vortex_scan::DataSource;
14+
use vortex_scan::DataSourceScanRef;
15+
use vortex_scan::PartitionRef;
16+
use vortex_scan::ScanRequest;
17+
use vortex_session::VortexSession;
18+
19+
use crate::FileStatistics;
20+
21+
pub(crate) struct FileDataSource {
22+
source: LayoutReaderDataSource,
23+
statistics: Option<FileStatistics>,
24+
}
25+
26+
impl FileDataSource {
27+
pub(crate) fn new(source: LayoutReaderDataSource, statistics: Option<FileStatistics>) -> Self {
28+
Self { source, statistics }
29+
}
30+
}
31+
32+
#[async_trait]
33+
impl DataSource for FileDataSource {
34+
fn dtype(&self) -> &DType {
35+
self.source.dtype()
36+
}
37+
38+
fn row_count(&self) -> Precision<u64> {
39+
self.source.row_count()
40+
}
41+
42+
fn byte_size(&self) -> Precision<u64> {
43+
self.source.byte_size()
44+
}
45+
46+
fn deserialize_partition(
47+
&self,
48+
data: &[u8],
49+
session: &VortexSession,
50+
) -> VortexResult<PartitionRef> {
51+
self.source.deserialize_partition(data, session)
52+
}
53+
54+
async fn scan(&self, request: ScanRequest) -> VortexResult<DataSourceScanRef> {
55+
self.source.scan(request).await
56+
}
57+
58+
async fn field_statistics(&self, field_path: &FieldPath) -> VortexResult<StatsSet> {
59+
Ok(self
60+
.statistics
61+
.as_ref()
62+
.and_then(|statistics| statistics.get_by_path(field_path))
63+
.map(|(statistics, _)| statistics.to_stats_set())
64+
.unwrap_or_default())
65+
}
66+
67+
async fn field_statistic(
68+
&self,
69+
field_path: &FieldPath,
70+
aggregate_fn: &AggregateFnRef,
71+
) -> VortexResult<Precision<Scalar>> {
72+
let Some((statistics, _)) = self
73+
.statistics
74+
.as_ref()
75+
.and_then(|statistics| statistics.get_by_path(field_path))
76+
else {
77+
return Ok(Precision::Absent);
78+
};
79+
Ok(statistics
80+
.iter()
81+
.find(|statistic| statistic.aggregate_fn() == aggregate_fn)
82+
.and_then(|statistic| statistic.partial())
83+
.cloned()
84+
.map_or(Precision::Absent, Precision::Exact))
85+
}
86+
}

‎vortex-file/src/tests.rs‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@ use vortex_array::ArrayRef;
2121
use vortex_array::IntoArray;
2222
use vortex_array::VortexSessionExecute;
2323
use vortex_array::aggregate_fn::AggregateFnRef;
24+
use vortex_array::aggregate_fn::AggregateFnVTableExt;
25+
use vortex_array::aggregate_fn::NumericalAggregateOpts;
26+
use vortex_array::aggregate_fn::fns::sum_v2::SumV2;
2427
use vortex_array::arrays::BoolArray;
2528
use vortex_array::arrays::ChunkedArray;
2629
use vortex_array::arrays::ConstantArray;
@@ -2288,6 +2291,34 @@ async fn unreadable_file_statistics_do_not_fail_the_open() -> VortexResult<()> {
22882291
Ok(())
22892292
}
22902293

2294+
#[tokio::test]
2295+
async fn file_data_source_returns_mergeable_partials() -> VortexResult<()> {
2296+
let sum = SumV2.bind(NumericalAggregateOpts::default());
2297+
let array =
2298+
StructArray::from_fields(&[("numbers", buffer![1_i32, 2, 3].into_array())])?.into_array();
2299+
let mut buf = ByteBufferMut::empty();
2300+
SESSION
2301+
.write_options()
2302+
.disable_editions()
2303+
.with_file_statistics(vec![sum.clone()])
2304+
.write(&mut buf, array.to_array_stream())
2305+
.await?;
2306+
let source = SESSION.open_options().open_buffer(buf)?.data_source()?;
2307+
let partial = source
2308+
.field_statistic(&field_path!(numbers), &sum)
2309+
.await?
2310+
.as_exact()
2311+
.expect("stored partial");
2312+
let mut accumulator = sum.accumulator(&I32.into())?;
2313+
accumulator.combine_partials(&partial)?;
2314+
accumulator.combine_partials(&partial)?;
2315+
assert_eq!(
2316+
accumulator.final_scalar()?,
2317+
Scalar::primitive(12_i64, Nullability::Nullable)
2318+
);
2319+
Ok(())
2320+
}
2321+
22912322
#[tokio::test]
22922323
async fn test_writer_with_statistics() -> VortexResult<()> {
22932324
let array = StructArray::from_fields(&[("numbers", buffer![1u32, 2, 3, 4, 5].into_array())])?

‎vortex-scan/src/lib.rs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,13 @@ use std::sync::Arc;
3434
use async_trait::async_trait;
3535
use futures::stream::BoxStream;
3636
use selection::Selection;
37+
use vortex_array::aggregate_fn::AggregateFnRef;
3738
use vortex_array::dtype::DType;
3839
use vortex_array::dtype::FieldPath;
3940
use vortex_array::expr::Expression;
4041
use vortex_array::expr::root;
4142
use vortex_array::expr::stats::Precision;
43+
use vortex_array::scalar::Scalar;
4244
use vortex_array::stats::StatsSet;
4345
use vortex_array::stream::SendableArrayStream;
4446
use vortex_error::VortexResult;
@@ -121,6 +123,24 @@ pub trait DataSource: 'static + Send + Sync {
121123

122124
/// Returns the statistics for a given field.
123125
async fn field_statistics(&self, field_path: &FieldPath) -> VortexResult<StatsSet>;
126+
127+
/// Returns a stored aggregate partial over the unfiltered source field.
128+
///
129+
/// The scalar has the aggregate's `partial_dtype`; callers merge partials and finalize
130+
/// through the requested aggregate. Identity includes bound options. Missing partials,
131+
/// incomplete coverage, and incompatible aggregates return [`Precision::Absent`], as
132+
/// does the default implementation. Finalized legacy statistics are not partials.
133+
///
134+
/// Lookup must not scan payloads. Precision describes the partial state; consumers must
135+
/// respect the aggregate's approximation contract. Later scan filters and limits do not
136+
/// change this source-level partial. A stored null partial is distinct from absence.
137+
async fn field_statistic(
138+
&self,
139+
_field_path: &FieldPath,
140+
_aggregate_fn: &AggregateFnRef,
141+
) -> VortexResult<Precision<Scalar>> {
142+
Ok(Precision::Absent)
143+
}
124144
}
125145

126146
/// A request to scan a data source.

0 commit comments

Comments
 (0)