Skip to content

Commit 000e896

Browse files
authored
Aggregates: accumulate columnar (#6839)
Changes the AggregateFnVTable::accumulate to take columnar instead of canonical. Might as well make the function define the optimized path over constants the same way we do for scalar functions.
1 parent 216cc12 commit 000e896

5 files changed

Lines changed: 106 additions & 17 deletions

File tree

‎vortex-array/public-api.lock‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ pub type vortex_array::aggregate_fn::fns::sum::Sum::Options = vortex_array::aggr
5858

5959
pub type vortex_array::aggregate_fn::fns::sum::Sum::Partial = vortex_array::aggregate_fn::fns::sum::SumPartial
6060

61-
pub fn vortex_array::aggregate_fn::fns::sum::Sum::accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::Canonical, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()>
61+
pub fn vortex_array::aggregate_fn::fns::sum::Sum::accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::Columnar, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()>
6262

6363
pub fn vortex_array::aggregate_fn::fns::sum::Sum::combine_partials(&self, partial: &mut Self::Partial, other: vortex_array::scalar::Scalar) -> vortex_error::VortexResult<()>
6464

@@ -286,7 +286,7 @@ pub type vortex_array::aggregate_fn::AggregateFnVTable::Options: 'static + core:
286286

287287
pub type vortex_array::aggregate_fn::AggregateFnVTable::Partial: 'static + core::marker::Send
288288

289-
pub fn vortex_array::aggregate_fn::AggregateFnVTable::accumulate(&self, state: &mut Self::Partial, batch: &vortex_array::Canonical, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()>
289+
pub fn vortex_array::aggregate_fn::AggregateFnVTable::accumulate(&self, state: &mut Self::Partial, batch: &vortex_array::Columnar, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()>
290290

291291
pub fn vortex_array::aggregate_fn::AggregateFnVTable::combine_partials(&self, partial: &mut Self::Partial, other: vortex_array::scalar::Scalar) -> vortex_error::VortexResult<()>
292292

@@ -316,7 +316,7 @@ pub type vortex_array::aggregate_fn::fns::sum::Sum::Options = vortex_array::aggr
316316

317317
pub type vortex_array::aggregate_fn::fns::sum::Sum::Partial = vortex_array::aggregate_fn::fns::sum::SumPartial
318318

319-
pub fn vortex_array::aggregate_fn::fns::sum::Sum::accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::Canonical, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()>
319+
pub fn vortex_array::aggregate_fn::fns::sum::Sum::accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::Columnar, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()>
320320

321321
pub fn vortex_array::aggregate_fn::fns::sum::Sum::combine_partials(&self, partial: &mut Self::Partial, other: vortex_array::scalar::Scalar) -> vortex_error::VortexResult<()>
322322

‎vortex-array/src/aggregate_fn/accumulator.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ use vortex_session::VortexSession;
77

88
use crate::AnyCanonical;
99
use crate::ArrayRef;
10-
use crate::Canonical;
10+
use crate::Columnar;
1111
use crate::DynArray;
1212
use crate::VortexSessionExecute;
1313
use crate::aggregate_fn::AggregateFn;
@@ -131,11 +131,11 @@ impl<V: AggregateFnVTable> DynAccumulator for Accumulator<V> {
131131
batch = batch.execute(&mut ctx)?;
132132
}
133133

134-
// Otherwise, execute the batch until it is canonical and accumulate it into the state.
135-
let canonical = batch.execute::<Canonical>(&mut ctx)?;
134+
// Otherwise, execute the batch until it is columnar and accumulate it into the state.
135+
let columnar = batch.execute::<Columnar>(&mut ctx)?;
136136

137137
self.vtable
138-
.accumulate(&mut self.partial, &canonical, &mut ctx)
138+
.accumulate(&mut self.partial, &columnar, &mut ctx)
139139
}
140140

141141
fn is_saturated(&self) -> bool {

‎vortex-array/src/aggregate_fn/accumulator_grouped.rs‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ use vortex_session::VortexSession;
1414
use crate::AnyCanonical;
1515
use crate::ArrayRef;
1616
use crate::Canonical;
17+
use crate::Columnar;
1718
use crate::DynArray;
1819
use crate::ExecutionCtx;
1920
use crate::IntoArray;
@@ -121,7 +122,11 @@ impl<V: AggregateFnVTable> DynGroupedAccumulator for GroupedAccumulator<V> {
121122

122123
// We first execute the groups until it is a ListView or FixedSizeList, since we only
123124
// dispatch the aggregate kernel over the elements of these arrays.
124-
match groups.clone().execute::<Canonical>(&mut ctx)? {
125+
let canonical = match groups.clone().execute::<Columnar>(&mut ctx)? {
126+
Columnar::Canonical(c) => c,
127+
Columnar::Constant(c) => c.into_array().execute::<Canonical>(&mut ctx)?,
128+
};
129+
match canonical {
125130
Canonical::List(groups) => self.accumulate_list_view(&groups, &mut ctx),
126131
Canonical::FixedSizeList(groups) => self.accumulate_fixed_size_list(&groups, &mut ctx),
127132
_ => vortex_panic!("We checked the DType above, so this should never happen"),
@@ -192,7 +197,7 @@ impl<V: AggregateFnVTable> GroupedAccumulator<V> {
192197
}
193198

194199
// Otherwise, we iterate the offsets and sizes and accumulate each group one by one.
195-
let elements = elements.execute::<Canonical>(ctx)?.into_array();
200+
let elements = elements.execute::<Columnar>(ctx)?.into_array();
196201
let offsets = groups.offsets();
197202
let sizes = groups.sizes().cast(offsets.dtype().clone())?;
198203
let validity = groups.validity().to_mask(offsets.len());
@@ -279,7 +284,7 @@ impl<V: AggregateFnVTable> GroupedAccumulator<V> {
279284
}
280285

281286
// Otherwise, we iterate the offsets and sizes and accumulate each group one by one.
282-
let elements = elements.execute::<Canonical>(ctx)?.into_array();
287+
let elements = elements.execute::<Columnar>(ctx)?.into_array();
283288
let validity = groups.validity().to_mask(groups.len());
284289

285290
let mut accumulator = Accumulator::try_new(

‎vortex-array/src/aggregate_fn/fns/sum.rs‎

Lines changed: 89 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,13 @@ use vortex_mask::AllOr;
1414

1515
use crate::ArrayRef;
1616
use crate::Canonical;
17+
use crate::Columnar;
1718
use crate::ExecutionCtx;
1819
use crate::aggregate_fn::AggregateFnId;
1920
use crate::aggregate_fn::AggregateFnVTable;
2021
use crate::aggregate_fn::EmptyOptions;
2122
use crate::arrays::BoolArray;
23+
use crate::arrays::ConstantArray;
2224
use crate::arrays::DecimalArray;
2325
use crate::arrays::PrimitiveArray;
2426
use crate::dtype::DType;
@@ -149,7 +151,7 @@ impl AggregateFnVTable for Sum {
149151
fn accumulate(
150152
&self,
151153
partial: &mut Self::Partial,
152-
batch: &Canonical,
154+
batch: &Columnar,
153155
_ctx: &mut ExecutionCtx,
154156
) -> VortexResult<()> {
155157
let mut inner = match partial.current.take() {
@@ -158,10 +160,13 @@ impl AggregateFnVTable for Sum {
158160
};
159161

160162
let result = match batch {
161-
Canonical::Primitive(p) => accumulate_primitive(&mut inner, p),
162-
Canonical::Bool(b) => accumulate_bool(&mut inner, b),
163-
Canonical::Decimal(d) => accumulate_decimal(&mut inner, d),
164-
_ => vortex_bail!("Unsupported canonical type for sum: {}", batch.dtype()),
163+
Columnar::Canonical(c) => match c {
164+
Canonical::Primitive(p) => accumulate_primitive(&mut inner, p),
165+
Canonical::Bool(b) => accumulate_bool(&mut inner, b),
166+
Canonical::Decimal(d) => accumulate_decimal(&mut inner, d),
167+
_ => vortex_bail!("Unsupported canonical type for sum: {}", batch.dtype()),
168+
},
169+
Columnar::Constant(c) => accumulate_constant(&mut inner, c),
165170
};
166171

167172
match result {
@@ -349,6 +354,85 @@ fn accumulate_bool(inner: &mut SumState, b: &BoolArray) -> VortexResult<bool> {
349354
Ok(checked_add_u64(acc, true_count))
350355
}
351356

357+
/// Accumulate a constant array into the sum state.
358+
/// Computes `scalar * len` and adds to the accumulator.
359+
/// Returns Ok(true) if saturated (overflow), Ok(false) if not.
360+
fn accumulate_constant(inner: &mut SumState, c: &ConstantArray) -> VortexResult<bool> {
361+
let scalar = c.scalar();
362+
if scalar.is_null() || c.is_empty() {
363+
return Ok(false);
364+
}
365+
let len = c.len();
366+
367+
match scalar.dtype() {
368+
DType::Bool(_) => {
369+
let SumState::Unsigned(acc) = inner else {
370+
vortex_panic!("expected unsigned sum state for bool input");
371+
};
372+
let val = scalar
373+
.as_bool()
374+
.value()
375+
.ok_or_else(|| vortex_err!("Expected non-null bool scalar for sum"))?;
376+
if val {
377+
Ok(checked_add_u64(acc, len as u64))
378+
} else {
379+
Ok(false)
380+
}
381+
}
382+
DType::Primitive(..) => {
383+
let pvalue = scalar
384+
.as_primitive()
385+
.pvalue()
386+
.ok_or_else(|| vortex_err!("Expected non-null primitive scalar for sum"))?;
387+
match inner {
388+
SumState::Unsigned(acc) => {
389+
let val = pvalue.cast::<u64>()?;
390+
match val.checked_mul(len as u64) {
391+
Some(product) => Ok(checked_add_u64(acc, product)),
392+
None => Ok(true),
393+
}
394+
}
395+
SumState::Signed(acc) => {
396+
let val = pvalue.cast::<i64>()?;
397+
match i64::try_from(len).ok().and_then(|l| val.checked_mul(l)) {
398+
Some(product) => Ok(checked_add_i64(acc, product)),
399+
None => Ok(true),
400+
}
401+
}
402+
SumState::Float(acc) => {
403+
let val = pvalue.cast::<f64>()?;
404+
*acc += val * len as f64;
405+
Ok(false)
406+
}
407+
SumState::Decimal(_) => {
408+
vortex_panic!("decimal sum state with primitive input")
409+
}
410+
}
411+
}
412+
DType::Decimal(..) => {
413+
let SumState::Decimal(acc) = inner else {
414+
vortex_panic!("expected decimal sum state for decimal input");
415+
};
416+
let val = scalar
417+
.as_decimal()
418+
.decimal_value()
419+
.ok_or_else(|| vortex_err!("Expected non-null decimal scalar for sum"))?;
420+
let len_decimal = DecimalValue::from(len as i128);
421+
match val.checked_mul(&len_decimal) {
422+
Some(product) => match acc.checked_add(&product) {
423+
Some(r) => {
424+
*acc = r;
425+
Ok(false)
426+
}
427+
None => Ok(true),
428+
},
429+
None => Ok(true),
430+
}
431+
}
432+
_ => vortex_bail!("Unsupported constant type for sum: {}", scalar.dtype()),
433+
}
434+
}
435+
352436
/// Accumulate a decimal array into the sum state.
353437
/// Returns Ok(true) if saturated (overflow), Ok(false) if not.
354438
fn accumulate_decimal(inner: &mut SumState, d: &DecimalArray) -> VortexResult<bool> {

‎vortex-array/src/aggregate_fn/vtable.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ use vortex_error::vortex_bail;
1212
use vortex_session::VortexSession;
1313

1414
use crate::ArrayRef;
15-
use crate::Canonical;
15+
use crate::Columnar;
1616
use crate::DynArray;
1717
use crate::ExecutionCtx;
1818
use crate::IntoArray;
@@ -95,7 +95,7 @@ pub trait AggregateFnVTable: 'static + Sized + Clone + Send + Sync {
9595
fn accumulate(
9696
&self,
9797
state: &mut Self::Partial,
98-
batch: &Canonical,
98+
batch: &Columnar,
9999
ctx: &mut ExecutionCtx,
100100
) -> VortexResult<()>;
101101

0 commit comments

Comments
 (0)