Skip to content

Commit 7f0edbc

Browse files
Merge branch 'develop' into cwirt/upstream-scan-split-concurrency
2 parents 843c42e + 807a0c8 commit 7f0edbc

21 files changed

Lines changed: 1995 additions & 57 deletions

File tree

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
//! Aggregate pushdown into the integer most significant part of [`DecimalByteParts`].
5+
//!
6+
//! With no lower parts, the MSP holds the unscaled decimal values. Integer aggregates can reuse
7+
//! child kernels, but their results must preserve decimal scale, precision, and state nullability.
8+
9+
use vortex_array::ArrayRef;
10+
use vortex_array::ExecutionCtx;
11+
use vortex_array::aggregate_fn::AggregateFnRef;
12+
use vortex_array::aggregate_fn::fns::is_sorted::IsSorted;
13+
use vortex_array::aggregate_fn::fns::is_sorted::is_sorted;
14+
use vortex_array::aggregate_fn::fns::is_sorted::is_strict_sorted;
15+
use vortex_array::aggregate_fn::fns::max::Max;
16+
use vortex_array::aggregate_fn::fns::min::Min;
17+
use vortex_array::aggregate_fn::fns::min_max::MinMax;
18+
use vortex_array::aggregate_fn::fns::min_max::min_max;
19+
use vortex_array::aggregate_fn::fns::sum::Sum;
20+
use vortex_array::aggregate_fn::fns::sum::sum;
21+
use vortex_array::aggregate_fn::kernels::DynAggregateKernel;
22+
use vortex_array::dtype::DType;
23+
use vortex_array::expr::stats::Stat;
24+
use vortex_array::scalar::DecimalValue;
25+
use vortex_array::scalar::Scalar;
26+
use vortex_error::VortexExpect;
27+
use vortex_error::VortexResult;
28+
29+
use crate::DecimalByteParts;
30+
use crate::decimal_byte_parts::DecimalBytePartsArraySlotsExt;
31+
32+
/// Computes decimal partials from the MSP without decoding the DBP batch.
33+
#[derive(Debug)]
34+
pub(crate) struct DecimalBytePartsAggregateKernel;
35+
36+
impl DynAggregateKernel for DecimalBytePartsAggregateKernel {
37+
fn aggregate(
38+
&self,
39+
aggregate_fn: &AggregateFnRef,
40+
batch: &ArrayRef,
41+
ctx: &mut ExecutionCtx,
42+
) -> VortexResult<Option<Scalar>> {
43+
let Some(array) = batch.as_opt::<DecimalByteParts>() else {
44+
return Ok(None);
45+
};
46+
47+
if !array.lower_parts().is_empty() {
48+
return Ok(None);
49+
}
50+
51+
if let Some(options) = aggregate_fn.as_opt::<IsSorted>() {
52+
return Ok(Some(is_sorted_partial(
53+
batch,
54+
array.msp(),
55+
options.strict,
56+
ctx,
57+
)?));
58+
}
59+
60+
if aggregate_fn.is::<Sum>() {
61+
return sum_partial(aggregate_fn, batch, array.msp(), ctx);
62+
}
63+
64+
extrema_partial(aggregate_fn, batch, array.msp(), ctx)
65+
}
66+
}
67+
68+
fn is_sorted_partial(
69+
batch: &ArrayRef,
70+
msp: &ArrayRef,
71+
strict: bool,
72+
ctx: &mut ExecutionCtx,
73+
) -> VortexResult<Scalar> {
74+
let sorted = if strict {
75+
is_strict_sorted(msp, ctx)?
76+
} else {
77+
is_sorted(msp, ctx)?
78+
};
79+
80+
IsSorted::make_partial(batch, sorted, strict, ctx)
81+
}
82+
83+
/// Returns `None` to decline, so the accumulator falls back to decoding the batch.
84+
fn sum_partial(
85+
aggregate_fn: &AggregateFnRef,
86+
batch: &ArrayRef,
87+
msp: &ArrayRef,
88+
ctx: &mut ExecutionCtx,
89+
) -> VortexResult<Option<Scalar>> {
90+
// Only the Stat::Sum path sums each batch from zero. Other variants add values to the
91+
// running sum and can overflow at different points than a batch total.
92+
if Stat::from_aggregate_fn(aggregate_fn) != Some(Stat::Sum) {
93+
return Ok(None);
94+
}
95+
96+
// TODO(connor): Predict integer overflow from dtypes and batch length using Widen's
97+
// is_exact_sum bound in a later refinement. For now, retry decimal execution on overflow.
98+
let total = sum(msp, ctx)?;
99+
if total.is_null() {
100+
return Ok(None);
101+
}
102+
103+
let dtype = aggregate_fn
104+
.state_dtype(batch.dtype())
105+
.vortex_expect("Sum accepts the DBP decimal dtype");
106+
let value = DecimalValue::from(i64::try_from(&total)?);
107+
let decimal_dtype = *dtype
108+
.as_decimal_opt()
109+
.vortex_expect("the Sum state dtype of a decimal input is a decimal");
110+
let partial = if value.fits_in_precision(decimal_dtype) {
111+
Scalar::try_decimal(value, decimal_dtype, dtype.nullability())?
112+
} else {
113+
Scalar::null(dtype)
114+
};
115+
116+
Ok(Some(partial))
117+
}
118+
119+
/// Returns `None` to decline, so the accumulator falls back to decoding the batch.
120+
fn extrema_partial(
121+
aggregate_fn: &AggregateFnRef,
122+
batch: &ArrayRef,
123+
msp: &ArrayRef,
124+
ctx: &mut ExecutionCtx,
125+
) -> VortexResult<Option<Scalar>> {
126+
let options = aggregate_fn
127+
.as_opt::<MinMax>()
128+
.or_else(|| aggregate_fn.as_opt::<Min>())
129+
.or_else(|| aggregate_fn.as_opt::<Max>());
130+
let Some(options) = options else {
131+
return Ok(None);
132+
};
133+
134+
let dtype = aggregate_fn
135+
.state_dtype(batch.dtype())
136+
.vortex_expect("extrema aggregates accept the DBP decimal dtype");
137+
let Some(extrema) = min_max(msp, ctx, *options)? else {
138+
return Ok(Some(Scalar::null(dtype)));
139+
};
140+
141+
let element_dtype = batch.dtype().as_nonnullable();
142+
let min = relabel(&extrema.min, &element_dtype)?;
143+
let max = relabel(&extrema.max, &element_dtype)?;
144+
let partial = if aggregate_fn.is::<MinMax>() {
145+
Scalar::struct_(dtype, vec![min, max])
146+
} else if aggregate_fn.is::<Min>() {
147+
min.into_nullable()
148+
} else {
149+
max.into_nullable()
150+
};
151+
152+
Ok(Some(partial))
153+
}
154+
155+
/// Preserves the unscaled integer value, unlike an integer-to-decimal scalar cast.
156+
fn relabel(value: &Scalar, dtype: &DType) -> VortexResult<Scalar> {
157+
Scalar::try_decimal(
158+
DecimalValue::from(i64::try_from(value)?),
159+
*dtype
160+
.as_decimal_opt()
161+
.vortex_expect("DBP has a decimal dtype"),
162+
dtype.nullability(),
163+
)
164+
}
165+
166+
#[cfg(test)]
167+
mod tests;

0 commit comments

Comments
 (0)