Skip to content

Commit d4f40a9

Browse files
authored
Add statistic expression (#7854)
Per the epic #7707 This PR adds an expression that returns the requested statistic, blown out for each row. When run against a single array, this returns a ConstantArray. When run against ChunkedArray, it returns RunEnd stats. It will be used by pluggable Falsify/Verify rewrite rules to allow for the workflow of adding custom aggregate functions to zone maps for falsification checks, e.g. bloom filters. --------- Signed-off-by: Nicholas Gates <[email protected]>
1 parent 59fcd14 commit d4f40a9

7 files changed

Lines changed: 532 additions & 0 deletions

File tree

‎vortex-array/public-api.lock‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17874,6 +17874,84 @@ pub fn vortex_array::scalar_fn::fns::select::Select::stat_falsification(&self, &
1787417874

1787517875
pub fn vortex_array::scalar_fn::fns::select::Select::validity(&self, &Self::Options, &vortex_array::expr::Expression) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
1787617876

17877+
pub mod vortex_array::scalar_fn::fns::stat
17878+
17879+
pub struct vortex_array::scalar_fn::fns::stat::StatFn
17880+
17881+
impl core::clone::Clone for vortex_array::scalar_fn::fns::stat::StatFn
17882+
17883+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::clone(&self) -> vortex_array::scalar_fn::fns::stat::StatFn
17884+
17885+
impl vortex_array::scalar_fn::ScalarFnVTable for vortex_array::scalar_fn::fns::stat::StatFn
17886+
17887+
pub type vortex_array::scalar_fn::fns::stat::StatFn::Options = vortex_array::scalar_fn::fns::stat::StatOptions
17888+
17889+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::arity(&self, &Self::Options) -> vortex_array::scalar_fn::Arity
17890+
17891+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::child_name(&self, &Self::Options, usize) -> vortex_array::scalar_fn::ChildName
17892+
17893+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::coerce_args(&self, &Self::Options, &[vortex_array::dtype::DType]) -> vortex_error::VortexResult<alloc::vec::Vec<vortex_array::dtype::DType>>
17894+
17895+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::deserialize(&self, &[u8], &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Options>
17896+
17897+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::execute(&self, &Self::Options, &dyn vortex_array::scalar_fn::ExecutionArgs, &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ArrayRef>
17898+
17899+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::fmt_sql(&self, &Self::Options, &vortex_array::expr::Expression, &mut core::fmt::Formatter<'_>) -> core::fmt::Result
17900+
17901+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::id(&self) -> vortex_array::scalar_fn::ScalarFnId
17902+
17903+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::is_fallible(&self, &Self::Options) -> bool
17904+
17905+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::is_null_sensitive(&self, &Self::Options) -> bool
17906+
17907+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::reduce(&self, &Self::Options, &dyn vortex_array::scalar_fn::ReduceNode, &dyn vortex_array::scalar_fn::ReduceCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::scalar_fn::ReduceNodeRef>>
17908+
17909+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::return_dtype(&self, &Self::Options, &[vortex_array::dtype::DType]) -> vortex_error::VortexResult<vortex_array::dtype::DType>
17910+
17911+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::serialize(&self, &Self::Options) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
17912+
17913+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::simplify(&self, &Self::Options, &vortex_array::expr::Expression, &dyn vortex_array::scalar_fn::SimplifyCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
17914+
17915+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::simplify_untyped(&self, &Self::Options, &vortex_array::expr::Expression) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
17916+
17917+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::stat_expression(&self, &Self::Options, &vortex_array::expr::Expression, vortex_array::expr::stats::Stat, &dyn vortex_array::expr::pruning::StatsCatalog) -> core::option::Option<vortex_array::expr::Expression>
17918+
17919+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::stat_falsification(&self, &Self::Options, &vortex_array::expr::Expression, &dyn vortex_array::expr::pruning::StatsCatalog) -> core::option::Option<vortex_array::expr::Expression>
17920+
17921+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::validity(&self, &Self::Options, &vortex_array::expr::Expression) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
17922+
17923+
pub struct vortex_array::scalar_fn::fns::stat::StatOptions
17924+
17925+
impl vortex_array::scalar_fn::fns::stat::StatOptions
17926+
17927+
pub fn vortex_array::scalar_fn::fns::stat::StatOptions::aggregate_fn(&self) -> &vortex_array::aggregate_fn::AggregateFnRef
17928+
17929+
pub fn vortex_array::scalar_fn::fns::stat::StatOptions::new(vortex_array::aggregate_fn::AggregateFnRef) -> Self
17930+
17931+
impl core::clone::Clone for vortex_array::scalar_fn::fns::stat::StatOptions
17932+
17933+
pub fn vortex_array::scalar_fn::fns::stat::StatOptions::clone(&self) -> vortex_array::scalar_fn::fns::stat::StatOptions
17934+
17935+
impl core::cmp::Eq for vortex_array::scalar_fn::fns::stat::StatOptions
17936+
17937+
impl core::cmp::PartialEq for vortex_array::scalar_fn::fns::stat::StatOptions
17938+
17939+
pub fn vortex_array::scalar_fn::fns::stat::StatOptions::eq(&self, &vortex_array::scalar_fn::fns::stat::StatOptions) -> bool
17940+
17941+
impl core::fmt::Debug for vortex_array::scalar_fn::fns::stat::StatOptions
17942+
17943+
pub fn vortex_array::scalar_fn::fns::stat::StatOptions::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result
17944+
17945+
impl core::fmt::Display for vortex_array::scalar_fn::fns::stat::StatOptions
17946+
17947+
pub fn vortex_array::scalar_fn::fns::stat::StatOptions::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result
17948+
17949+
impl core::hash::Hash for vortex_array::scalar_fn::fns::stat::StatOptions
17950+
17951+
pub fn vortex_array::scalar_fn::fns::stat::StatOptions::hash<__H: core::hash::Hasher>(&self, &mut __H)
17952+
17953+
impl core::marker::StructuralPartialEq for vortex_array::scalar_fn::fns::stat::StatOptions
17954+
1787717955
pub mod vortex_array::scalar_fn::fns::zip
1787817956

1787917957
pub struct vortex_array::scalar_fn::fns::zip::Zip
@@ -19140,6 +19218,44 @@ pub fn vortex_array::scalar_fn::fns::select::Select::stat_falsification(&self, &
1914019218

1914119219
pub fn vortex_array::scalar_fn::fns::select::Select::validity(&self, &Self::Options, &vortex_array::expr::Expression) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
1914219220

19221+
impl vortex_array::scalar_fn::ScalarFnVTable for vortex_array::scalar_fn::fns::stat::StatFn
19222+
19223+
pub type vortex_array::scalar_fn::fns::stat::StatFn::Options = vortex_array::scalar_fn::fns::stat::StatOptions
19224+
19225+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::arity(&self, &Self::Options) -> vortex_array::scalar_fn::Arity
19226+
19227+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::child_name(&self, &Self::Options, usize) -> vortex_array::scalar_fn::ChildName
19228+
19229+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::coerce_args(&self, &Self::Options, &[vortex_array::dtype::DType]) -> vortex_error::VortexResult<alloc::vec::Vec<vortex_array::dtype::DType>>
19230+
19231+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::deserialize(&self, &[u8], &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Options>
19232+
19233+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::execute(&self, &Self::Options, &dyn vortex_array::scalar_fn::ExecutionArgs, &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ArrayRef>
19234+
19235+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::fmt_sql(&self, &Self::Options, &vortex_array::expr::Expression, &mut core::fmt::Formatter<'_>) -> core::fmt::Result
19236+
19237+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::id(&self) -> vortex_array::scalar_fn::ScalarFnId
19238+
19239+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::is_fallible(&self, &Self::Options) -> bool
19240+
19241+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::is_null_sensitive(&self, &Self::Options) -> bool
19242+
19243+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::reduce(&self, &Self::Options, &dyn vortex_array::scalar_fn::ReduceNode, &dyn vortex_array::scalar_fn::ReduceCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::scalar_fn::ReduceNodeRef>>
19244+
19245+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::return_dtype(&self, &Self::Options, &[vortex_array::dtype::DType]) -> vortex_error::VortexResult<vortex_array::dtype::DType>
19246+
19247+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::serialize(&self, &Self::Options) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
19248+
19249+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::simplify(&self, &Self::Options, &vortex_array::expr::Expression, &dyn vortex_array::scalar_fn::SimplifyCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
19250+
19251+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::simplify_untyped(&self, &Self::Options, &vortex_array::expr::Expression) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
19252+
19253+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::stat_expression(&self, &Self::Options, &vortex_array::expr::Expression, vortex_array::expr::stats::Stat, &dyn vortex_array::expr::pruning::StatsCatalog) -> core::option::Option<vortex_array::expr::Expression>
19254+
19255+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::stat_falsification(&self, &Self::Options, &vortex_array::expr::Expression, &dyn vortex_array::expr::pruning::StatsCatalog) -> core::option::Option<vortex_array::expr::Expression>
19256+
19257+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::validity(&self, &Self::Options, &vortex_array::expr::Expression) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
19258+
1914319259
impl vortex_array::scalar_fn::ScalarFnVTable for vortex_array::scalar_fn::fns::zip::Zip
1914419260

1914519261
pub type vortex_array::scalar_fn::fns::zip::Zip::Options = vortex_array::scalar_fn::EmptyOptions
@@ -19524,6 +19640,86 @@ pub type vortex_array::session::ArrayRegistry = vortex_session::registry::Regist
1952419640

1952519641
pub mod vortex_array::stats
1952619642

19643+
pub mod vortex_array::stats::expr
19644+
19645+
pub struct vortex_array::stats::expr::StatFn
19646+
19647+
impl core::clone::Clone for vortex_array::scalar_fn::fns::stat::StatFn
19648+
19649+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::clone(&self) -> vortex_array::scalar_fn::fns::stat::StatFn
19650+
19651+
impl vortex_array::scalar_fn::ScalarFnVTable for vortex_array::scalar_fn::fns::stat::StatFn
19652+
19653+
pub type vortex_array::scalar_fn::fns::stat::StatFn::Options = vortex_array::scalar_fn::fns::stat::StatOptions
19654+
19655+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::arity(&self, &Self::Options) -> vortex_array::scalar_fn::Arity
19656+
19657+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::child_name(&self, &Self::Options, usize) -> vortex_array::scalar_fn::ChildName
19658+
19659+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::coerce_args(&self, &Self::Options, &[vortex_array::dtype::DType]) -> vortex_error::VortexResult<alloc::vec::Vec<vortex_array::dtype::DType>>
19660+
19661+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::deserialize(&self, &[u8], &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Options>
19662+
19663+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::execute(&self, &Self::Options, &dyn vortex_array::scalar_fn::ExecutionArgs, &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ArrayRef>
19664+
19665+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::fmt_sql(&self, &Self::Options, &vortex_array::expr::Expression, &mut core::fmt::Formatter<'_>) -> core::fmt::Result
19666+
19667+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::id(&self) -> vortex_array::scalar_fn::ScalarFnId
19668+
19669+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::is_fallible(&self, &Self::Options) -> bool
19670+
19671+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::is_null_sensitive(&self, &Self::Options) -> bool
19672+
19673+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::reduce(&self, &Self::Options, &dyn vortex_array::scalar_fn::ReduceNode, &dyn vortex_array::scalar_fn::ReduceCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::scalar_fn::ReduceNodeRef>>
19674+
19675+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::return_dtype(&self, &Self::Options, &[vortex_array::dtype::DType]) -> vortex_error::VortexResult<vortex_array::dtype::DType>
19676+
19677+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::serialize(&self, &Self::Options) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
19678+
19679+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::simplify(&self, &Self::Options, &vortex_array::expr::Expression, &dyn vortex_array::scalar_fn::SimplifyCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
19680+
19681+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::simplify_untyped(&self, &Self::Options, &vortex_array::expr::Expression) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
19682+
19683+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::stat_expression(&self, &Self::Options, &vortex_array::expr::Expression, vortex_array::expr::stats::Stat, &dyn vortex_array::expr::pruning::StatsCatalog) -> core::option::Option<vortex_array::expr::Expression>
19684+
19685+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::stat_falsification(&self, &Self::Options, &vortex_array::expr::Expression, &dyn vortex_array::expr::pruning::StatsCatalog) -> core::option::Option<vortex_array::expr::Expression>
19686+
19687+
pub fn vortex_array::scalar_fn::fns::stat::StatFn::validity(&self, &Self::Options, &vortex_array::expr::Expression) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
19688+
19689+
pub struct vortex_array::stats::expr::StatOptions
19690+
19691+
impl vortex_array::scalar_fn::fns::stat::StatOptions
19692+
19693+
pub fn vortex_array::scalar_fn::fns::stat::StatOptions::aggregate_fn(&self) -> &vortex_array::aggregate_fn::AggregateFnRef
19694+
19695+
pub fn vortex_array::scalar_fn::fns::stat::StatOptions::new(vortex_array::aggregate_fn::AggregateFnRef) -> Self
19696+
19697+
impl core::clone::Clone for vortex_array::scalar_fn::fns::stat::StatOptions
19698+
19699+
pub fn vortex_array::scalar_fn::fns::stat::StatOptions::clone(&self) -> vortex_array::scalar_fn::fns::stat::StatOptions
19700+
19701+
impl core::cmp::Eq for vortex_array::scalar_fn::fns::stat::StatOptions
19702+
19703+
impl core::cmp::PartialEq for vortex_array::scalar_fn::fns::stat::StatOptions
19704+
19705+
pub fn vortex_array::scalar_fn::fns::stat::StatOptions::eq(&self, &vortex_array::scalar_fn::fns::stat::StatOptions) -> bool
19706+
19707+
impl core::fmt::Debug for vortex_array::scalar_fn::fns::stat::StatOptions
19708+
19709+
pub fn vortex_array::scalar_fn::fns::stat::StatOptions::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result
19710+
19711+
impl core::fmt::Display for vortex_array::scalar_fn::fns::stat::StatOptions
19712+
19713+
pub fn vortex_array::scalar_fn::fns::stat::StatOptions::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result
19714+
19715+
impl core::hash::Hash for vortex_array::scalar_fn::fns::stat::StatOptions
19716+
19717+
pub fn vortex_array::scalar_fn::fns::stat::StatOptions::hash<__H: core::hash::Hasher>(&self, &mut __H)
19718+
19719+
impl core::marker::StructuralPartialEq for vortex_array::scalar_fn::fns::stat::StatOptions
19720+
19721+
pub fn vortex_array::stats::expr::stat(vortex_array::expr::Expression, vortex_array::aggregate_fn::AggregateFnRef) -> vortex_array::expr::Expression
19722+
1952719723
pub mod vortex_array::stats::flatbuffers
1952819724

1952919725
pub struct vortex_array::stats::ArrayStats
@@ -19756,6 +19952,8 @@ pub const vortex_array::stats::PRUNING_STATS: &[vortex_array::expr::stats::Stat]
1975619952

1975719953
pub fn vortex_array::stats::as_stat_bitset_bytes(&[vortex_array::expr::stats::Stat]) -> alloc::vec::Vec<u8>
1975819954

19955+
pub fn vortex_array::stats::stat(vortex_array::expr::Expression, vortex_array::aggregate_fn::AggregateFnRef) -> vortex_array::expr::Expression
19956+
1975919957
pub fn vortex_array::stats::stats_from_bitset_bytes(&[u8]) -> alloc::vec::Vec<vortex_array::expr::stats::Stat>
1976019958

1976119959
pub type vortex_array::stats::StatsArray = [(vortex_array::expr::stats::Stat, vortex_array::expr::stats::Precision<vortex_array::scalar::ScalarValue>); 4]

‎vortex-array/src/scalar_fn/fns/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,4 +20,5 @@ pub mod operators;
2020
pub mod pack;
2121
pub mod root;
2222
pub mod select;
23+
pub mod stat;
2324
pub mod zip;

0 commit comments

Comments
 (0)