Skip to content

Commit 4c58853

Browse files
committed
Bind pruning facts through aggregate requests
Pruning should use the same aggregate requests as cached results and zone partials. Bind exact footer results through the explicit partial contract, and keep direct field facts separate from computed inputs while preserving built-in cast proofs. Signed-off-by: Connor Tsui <[email protected]>
1 parent e7c69fb commit 4c58853

4 files changed

Lines changed: 382 additions & 92 deletions

File tree

‎vortex-array/src/stats/rewrite/builtins.rs‎

Lines changed: 55 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

44
use std::sync::Arc;
5+
use std::sync::LazyLock;
56

67
use vortex_array::stats::rewrite::falsify;
78
use vortex_error::VortexExpect;
@@ -15,6 +16,14 @@ use crate::aggregate_fn::fns::all_nan::AllNan;
1516
use crate::aggregate_fn::fns::all_non_nan::AllNonNan;
1617
use crate::aggregate_fn::fns::all_non_null::AllNonNull;
1718
use crate::aggregate_fn::fns::all_null::AllNull;
19+
use crate::aggregate_fn::fns::max::MAX_SKIP_NANS;
20+
use crate::aggregate_fn::fns::max::Max;
21+
use crate::aggregate_fn::fns::min::MIN_SKIP_NANS;
22+
use crate::aggregate_fn::fns::min::Min;
23+
use crate::aggregate_fn::fns::nan_count::NAN_COUNT;
24+
use crate::aggregate_fn::fns::nan_count::NanCount;
25+
use crate::aggregate_fn::fns::null_count::NULL_COUNT;
26+
use crate::aggregate_fn::fns::null_count::NullCount;
1827
use crate::dtype::DType;
1928
use crate::expr::BoundExpression;
2029
use crate::expr::bound::and;
@@ -30,7 +39,6 @@ use crate::expr::bound::lt;
3039
use crate::expr::bound::lt_eq;
3140
use crate::expr::bound::or;
3241
use crate::expr::bound::or_collect;
33-
use crate::expr::stats::Stat;
3442
use crate::scalar::StringLike;
3543
use crate::scalar_fn::EmptyOptions;
3644
use crate::scalar_fn::ScalarFnId;
@@ -55,6 +63,13 @@ use crate::stats::bound::stat;
5563
use crate::stats::rewrite::StatsRewriteRule;
5664
use crate::stats::session::StatsSession;
5765

66+
static ALL_NULL: LazyLock<AggregateFnRef> = LazyLock::new(|| AllNull.bind(AggregateEmptyOptions));
67+
static ALL_NON_NULL: LazyLock<AggregateFnRef> =
68+
LazyLock::new(|| AllNonNull.bind(AggregateEmptyOptions));
69+
static ALL_NAN: LazyLock<AggregateFnRef> = LazyLock::new(|| AllNan.bind(AggregateEmptyOptions));
70+
static ALL_NON_NAN: LazyLock<AggregateFnRef> =
71+
LazyLock::new(|| AllNonNan.bind(AggregateEmptyOptions));
72+
5873
/// Register built-in stats rewrite rules.
5974
pub(crate) fn register_builtins(session: &StatsSession) {
6075
session.register_rewrite(BinaryNanCountStatsRewrite);
@@ -523,7 +538,7 @@ fn list_contains_falsify<P: NonNanProof>(
523538
let list = expr.child(0);
524539
let needle = expr.child(1);
525540

526-
let Some(list_scalar) = literal_stat(list, Stat::Min) else {
541+
let Some(list_scalar) = literal_stat(list, &MIN_SKIP_NANS) else {
527542
return Ok(None);
528543
};
529544
let elements = list_scalar
@@ -618,35 +633,35 @@ fn dynamic_comparison_falsify<P: NonNanProof>(
618633
}
619634

620635
fn min(expr: &BoundExpression) -> Option<BoundExpression> {
621-
stat_expr(expr, Stat::Min)
636+
stat_expr(expr, &MIN_SKIP_NANS)
622637
}
623638

624639
fn max(expr: &BoundExpression) -> Option<BoundExpression> {
625-
stat_expr(expr, Stat::Max)
640+
stat_expr(expr, &MAX_SKIP_NANS)
626641
}
627642

628643
fn null_count(expr: &BoundExpression) -> Option<BoundExpression> {
629-
stat_expr(expr, Stat::NullCount)
644+
stat_expr(expr, &NULL_COUNT)
630645
}
631646

632647
fn nan_count(expr: &BoundExpression) -> Option<BoundExpression> {
633-
stat_expr(expr, Stat::NaNCount)
648+
stat_expr(expr, &NAN_COUNT)
634649
}
635650

636651
fn all_null(expr: &BoundExpression) -> BoundExpression {
637-
stat_fn(expr.clone(), AllNull.bind(AggregateEmptyOptions))
652+
stat_fn(expr.clone(), ALL_NULL.clone())
638653
}
639654

640655
fn all_non_null(expr: &BoundExpression) -> BoundExpression {
641-
stat_fn(expr.clone(), AllNonNull.bind(AggregateEmptyOptions))
656+
stat_fn(expr.clone(), ALL_NON_NULL.clone())
642657
}
643658

644659
fn all_nan(expr: &BoundExpression) -> BoundExpression {
645-
stat_fn(expr.clone(), AllNan.bind(AggregateEmptyOptions))
660+
stat_fn(expr.clone(), ALL_NAN.clone())
646661
}
647662

648663
fn all_non_nan(expr: &BoundExpression) -> BoundExpression {
649-
stat_fn(expr.clone(), AllNonNan.bind(AggregateEmptyOptions))
664+
stat_fn(expr.clone(), ALL_NON_NAN.clone())
650665
}
651666

652667
enum NanCheck {
@@ -667,7 +682,7 @@ impl NonNanProof for NanCountProof {
667682
const EMIT_UNGUARDED_REWRITES: bool = true;
668683

669684
fn check(expr: &BoundExpression) -> VortexResult<NanCheck> {
670-
non_nan_check(expr, |expr| match stat_expr(expr, Stat::NaNCount) {
685+
non_nan_check(expr, |expr| match stat_expr(expr, &NAN_COUNT) {
671686
Some(nan_count) => NanCheck::Check(eq(nan_count, lit(0u64))),
672687
None => NanCheck::Unavailable,
673688
})
@@ -681,7 +696,7 @@ impl NonNanProof for AllNonNanProof {
681696

682697
fn check(expr: &BoundExpression) -> VortexResult<NanCheck> {
683698
non_nan_check(expr, |expr| {
684-
NanCheck::Check(stat_fn(expr.clone(), AllNonNan.bind(AggregateEmptyOptions)))
699+
NanCheck::Check(stat_fn(expr.clone(), ALL_NON_NAN.clone()))
685700
})
686701
}
687702
}
@@ -723,8 +738,8 @@ fn has_nans(dtype: &DType) -> bool {
723738
dtype.is_float()
724739
}
725740

726-
fn stat_expr(expr: &BoundExpression, stat: Stat) -> Option<BoundExpression> {
727-
if let Some(literal) = literal_stat(expr, stat) {
741+
fn stat_expr(expr: &BoundExpression, aggregate: &AggregateFnRef) -> Option<BoundExpression> {
742+
if let Some(literal) = literal_stat(expr, aggregate) {
728743
return Some(literal);
729744
}
730745

@@ -736,17 +751,16 @@ fn stat_expr(expr: &BoundExpression, stat: Stat) -> Option<BoundExpression> {
736751
}
737752

738753
if let Some(dtype) = expr.as_opt::<Cast>() {
739-
return cast_stat(expr.child(0), dtype, stat);
754+
return cast_stat(expr.child(0), dtype, aggregate);
740755
}
741756

742-
let aggregate_fn = stat.aggregate_fn()?;
743757
// The aggregate may not support the expression's dtype, e.g. min/max over structs,
744758
// even when the predicate itself is well-typed. Such stats cannot be lowered later,
745759
// so do not reference them in the rewrite.
746-
aggregate_fn
760+
aggregate
747761
.return_dtype(expr.dtype())
748762
.is_some()
749-
.then(|| stat_fn(expr.clone(), aggregate_fn))
763+
.then(|| stat_fn(expr.clone(), aggregate.clone()))
750764
}
751765

752766
fn with_non_nan_guards<'a, P: NonNanProof>(
@@ -773,32 +787,34 @@ fn with_non_nan_guards<'a, P: NonNanProof>(
773787
})
774788
}
775789

776-
fn literal_stat(expr: &BoundExpression, stat: Stat) -> Option<BoundExpression> {
790+
fn literal_stat(expr: &BoundExpression, aggregate: &AggregateFnRef) -> Option<BoundExpression> {
777791
let scalar = expr.as_opt::<Literal>()?;
778-
match stat {
779-
Stat::Min | Stat::Max => Some(lit(scalar.clone())),
780-
Stat::NullCount => Some(lit(if scalar.is_null() { 1u64 } else { 0u64 })),
781-
Stat::NaNCount => {
782-
let value = scalar.as_primitive_opt()?;
783-
if !value.ptype().is_float() {
784-
return None;
785-
}
786-
787-
Some(lit(if value.is_nan() { 1u64 } else { 0u64 }))
792+
if aggregate.is::<Min>() || aggregate.is::<Max>() {
793+
return Some(lit(scalar.clone()));
794+
}
795+
if aggregate.is::<NullCount>() {
796+
return Some(lit(if scalar.is_null() { 1u64 } else { 0u64 }));
797+
}
798+
if aggregate.is::<NanCount>() {
799+
let value = scalar.as_primitive_opt()?;
800+
if value.ptype().is_float() {
801+
return Some(lit(if value.is_nan() { 1u64 } else { 0u64 }));
788802
}
789-
Stat::IsConstant
790-
| Stat::IsSorted
791-
| Stat::IsStrictSorted
792-
| Stat::Sum
793-
| Stat::UncompressedSizeInBytes => None,
794803
}
804+
None
795805
}
796806

797-
fn cast_stat(expr: &BoundExpression, dtype: &DType, stat: Stat) -> Option<BoundExpression> {
798-
match stat {
799-
Stat::Min | Stat::Max => stat_expr(expr, stat).map(|stat| cast(stat, dtype.clone())),
800-
Stat::NaNCount | Stat::Sum | Stat::UncompressedSizeInBytes => stat_expr(expr, stat),
801-
Stat::NullCount | Stat::IsConstant | Stat::IsSorted | Stat::IsStrictSorted => None,
807+
fn cast_stat(
808+
expr: &BoundExpression,
809+
dtype: &DType,
810+
aggregate: &AggregateFnRef,
811+
) -> Option<BoundExpression> {
812+
if aggregate.is::<Min>() || aggregate.is::<Max>() {
813+
stat_expr(expr, aggregate).map(|stat| cast(stat, dtype.clone()))
814+
} else if aggregate.is::<NanCount>() {
815+
stat_expr(expr, aggregate)
816+
} else {
817+
None
802818
}
803819
}
804820

‎vortex-file/src/pruning.rs‎

Lines changed: 118 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,7 @@ use vortex_array::dtype::FieldPath;
1212
use vortex_array::dtype::StructFields;
1313
use vortex_array::expr::BoundExpression;
1414
use vortex_array::expr::bound::lit;
15-
use vortex_array::expr::stats::Stat;
16-
use vortex_array::scalar_fn::fns::cast::Cast;
15+
use vortex_array::scalar::Scalar;
1716
use vortex_array::scalar_fn::fns::get_item::GetItem;
1817
use vortex_array::scalar_fn::fns::literal::Literal;
1918
use vortex_array::scalar_fn::internal::row_count::substitute_row_count;
@@ -69,35 +68,62 @@ impl StatBinder for FileStatsBinder<'_> {
6968
&self,
7069
input: &BoundExpression,
7170
aggregate_fn: &AggregateFnRef,
72-
_stat_dtype: &DType,
71+
stat_dtype: &DType,
7372
) -> VortexResult<Option<BoundExpression>> {
74-
let Some(stat) = Stat::from_aggregate_fn(aggregate_fn) else {
75-
return Ok(None);
76-
};
7773
let Some(field_path) = direct_field_path(input) else {
7874
return Ok(None);
7975
};
80-
Ok(self.stat_ref(&field_path, stat))
76+
self.stat_ref(&field_path, aggregate_fn, stat_dtype)
8177
}
8278
}
8379

8480
impl FileStatsBinder<'_> {
85-
fn stat_ref(&self, field_path: &FieldPath, stat: Stat) -> Option<BoundExpression> {
81+
fn stat_ref(
82+
&self,
83+
field_path: &FieldPath,
84+
aggregate: &AggregateFnRef,
85+
stat_dtype: &DType,
86+
) -> VortexResult<Option<BoundExpression>> {
8687
// FileStats currently only holds top-level field statistics.
8788
if field_path.parts().len() != 1 {
88-
return None;
89+
return Ok(None);
8990
}
9091

91-
let field_name = field_path.parts()[0].as_name()?;
92-
let field_idx = self.struct_fields.find(field_name)?;
93-
let field_stats = self.file_stats.results().get(field_idx)?;
94-
95-
let aggregate = stat.aggregate_fn()?;
96-
let scalar = field_stats.get_result(&aggregate).as_exact()?;
97-
if scalar.is_null() {
98-
return None;
92+
let Some(field_idx) = field_path.parts()[0]
93+
.as_name()
94+
.and_then(|name| self.struct_fields.find(name))
95+
else {
96+
return Ok(None);
97+
};
98+
let Some(field_stats) = self.file_stats.results().get(field_idx) else {
99+
return Ok(None);
100+
};
101+
let input_dtype = &self.file_stats.dtypes()[field_idx];
102+
let Some(result) = field_stats.get_result(aggregate).as_exact() else {
103+
return Ok(None);
104+
};
105+
if result.is_null() {
106+
return Ok(None);
107+
}
108+
let Some(return_dtype) = aggregate.return_dtype(input_dtype) else {
109+
return Ok(None);
110+
};
111+
if result.dtype().as_nullable() != return_dtype.as_nullable() {
112+
return Ok(None);
113+
}
114+
// Historical footer extrema use the input's nullability. Normalize the lookup value only.
115+
let result = if result.dtype() == &return_dtype {
116+
result
117+
} else {
118+
Scalar::try_new(return_dtype, result.into_value())?
119+
};
120+
let Some(partial) = aggregate.partial_from_result(input_dtype, &result)? else {
121+
return Ok(None);
122+
};
123+
if partial.is_null() || partial.dtype().as_nullable() != stat_dtype.as_nullable() {
124+
return Ok(None);
99125
}
100-
Some(lit(scalar))
126+
Ok(Some(lit(partial)))
101127
}
102128
}
103129

@@ -106,10 +132,80 @@ fn direct_field_path(expr: &BoundExpression) -> Option<FieldPath> {
106132
return Some(FieldPath::root());
107133
}
108134

109-
if expr.is::<Cast>() {
110-
return direct_field_path(expr.child(0));
111-
}
112-
113135
let field_name = expr.as_opt::<GetItem>()?;
114136
direct_field_path(expr.child(0)).map(|path| path.push(field_name.clone()))
115137
}
138+
139+
#[cfg(test)]
140+
mod tests {
141+
use std::sync::Arc;
142+
143+
use vortex_array::aggregate_fn::AggregateFnVTableExt;
144+
use vortex_array::aggregate_fn::NumericalAggregateOpts;
145+
use vortex_array::aggregate_fn::fns::count::Count;
146+
use vortex_array::aggregate_fn::fns::max::Max;
147+
use vortex_array::dtype::Nullability;
148+
use vortex_array::dtype::PType;
149+
use vortex_array::expr::cast;
150+
use vortex_array::expr::col;
151+
use vortex_array::expr::gt;
152+
use vortex_array::expr::lit;
153+
use vortex_array::expr::stats::Precision;
154+
use vortex_array::stats::AggregateResults;
155+
156+
use super::*;
157+
158+
#[test]
159+
fn computed_inputs_do_not_bind_source_field_results() -> VortexResult<()> {
160+
let dtype = DType::from(PType::I32);
161+
let fields = StructFields::from_iter([("col", dtype.clone())]);
162+
let scope = DType::Struct(fields.clone(), Nullability::NonNullable);
163+
let aggregate = Count.bind(NumericalAggregateOpts::include_nans());
164+
let results =
165+
AggregateResults::try_new(&dtype, [(aggregate.clone(), Precision::exact(3u64))])?;
166+
let file_stats = FileStatistics::new(Arc::new([results]), Arc::new([dtype]));
167+
let binder = FileStatsBinder {
168+
file_stats: &file_stats,
169+
struct_fields: &fields,
170+
};
171+
let field = col("col").bind(&scope)?;
172+
let computed = cast(col("col"), DType::from(PType::I64)).bind(&scope)?;
173+
let stat_dtype = DType::Primitive(PType::U64, Nullability::Nullable);
174+
assert!(
175+
binder
176+
.bind_aggregate(&field, &aggregate, &stat_dtype)?
177+
.is_some()
178+
);
179+
assert!(
180+
binder
181+
.bind_aggregate(&computed, &aggregate, &stat_dtype)?
182+
.is_none()
183+
);
184+
Ok(())
185+
}
186+
187+
#[test]
188+
fn builtin_safe_cast_keeps_file_extrema() -> VortexResult<()> {
189+
let dtype = DType::from(PType::I32);
190+
let fields = StructFields::from_iter([("col", dtype.clone())]);
191+
let scope = DType::Struct(fields.clone(), Nullability::NonNullable);
192+
let aggregate = Max.bind(NumericalAggregateOpts::skip_nans());
193+
let results = AggregateResults::try_new(
194+
&dtype,
195+
[(
196+
aggregate,
197+
Precision::exact(Scalar::primitive(3i32, Nullability::Nullable)),
198+
)],
199+
)?;
200+
let file_stats = FileStatistics::new(Arc::new([results]), Arc::new([dtype]));
201+
let predicate = gt(cast(col("col"), DType::from(PType::I64)), lit(10i64)).bind(&scope)?;
202+
assert!(can_prune_file_stats(
203+
&predicate,
204+
3,
205+
&file_stats,
206+
&fields,
207+
&vortex_array::array_session()
208+
)?);
209+
Ok(())
210+
}
211+
}

0 commit comments

Comments
 (0)