Skip to content

Commit 00d28df

Browse files
Revert "SQL style null-on-empty semantics for Sum and Mean (#9113)"
This reverts commit 329bba2.
1 parent 4b37237 commit 00d28df

27 files changed

Lines changed: 370 additions & 1901 deletions

File tree

‎vortex-array/benches/aggregate_grouped.rs‎

Lines changed: 8 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,12 @@ use rand::RngExt;
1111
use rand::SeedableRng;
1212
use rand::rngs::StdRng;
1313
use vortex_array::ArrayRef;
14-
use vortex_array::Canonical;
1514
use vortex_array::IntoArray;
1615
use vortex_array::VortexSessionExecute;
1716
use vortex_array::aggregate_fn::AggregateFnVTable;
1817
use vortex_array::aggregate_fn::DynGroupedAccumulator;
1918
use vortex_array::aggregate_fn::GroupedAccumulator;
19+
use vortex_array::aggregate_fn::NumericalAggregateOpts;
2020
use vortex_array::aggregate_fn::fns::count::Count;
2121
use vortex_array::aggregate_fn::fns::sum::Sum;
2222
use vortex_array::arrays::ListViewArray;
@@ -154,12 +154,14 @@ fn list_element_dtype(list_view: &ArrayRef) -> DType {
154154

155155
fn grouped_accumulator<V>(list_view: &ArrayRef, vtable: V) -> ArrayRef
156156
where
157-
V: AggregateFnVTable + Clone,
158-
V::Options: Default,
157+
V: AggregateFnVTable<Options = NumericalAggregateOpts> + Clone,
159158
{
160-
let mut acc =
161-
GroupedAccumulator::try_new(vtable, V::Options::default(), list_element_dtype(list_view))
162-
.unwrap();
159+
let mut acc = GroupedAccumulator::try_new(
160+
vtable,
161+
NumericalAggregateOpts::default(),
162+
list_element_dtype(list_view),
163+
)
164+
.unwrap();
163165
acc.accumulate_list(list_view, &mut SESSION.create_execution_ctx())
164166
.unwrap();
165167
divan::black_box(acc.finish().unwrap())
@@ -197,59 +199,6 @@ fn sum_f64_clustered_nulls(bencher: Bencher) {
197199
.bench_refs(|input| grouped_accumulator(input, Sum));
198200
}
199201

200-
/// Like [`grouped_accumulator`], but executes the lazy finalize result to canonical so the
201-
/// bench measures the full cost of producing usable sums.
202-
fn grouped_accumulator_canonical<V>(list_view: &ArrayRef, vtable: V) -> ArrayRef
203-
where
204-
V: AggregateFnVTable + Clone,
205-
V::Options: Default,
206-
{
207-
let mut acc =
208-
GroupedAccumulator::try_new(vtable, V::Options::default(), list_element_dtype(list_view))
209-
.unwrap();
210-
let mut ctx = SESSION.create_execution_ctx();
211-
acc.accumulate_list(list_view, &mut ctx).unwrap();
212-
let result = acc
213-
.finish()
214-
.unwrap()
215-
.execute::<Canonical>(&mut ctx)
216-
.unwrap()
217-
.into_array();
218-
divan::black_box(result)
219-
}
220-
221-
#[divan::bench]
222-
fn canonical_sum_i32_nullable_all_valid(bencher: Bencher) {
223-
let input = i32_nullable_all_valid_input();
224-
bencher
225-
.with_inputs(|| &input)
226-
.bench_refs(|input| grouped_accumulator_canonical(input, Sum));
227-
}
228-
229-
#[divan::bench]
230-
fn canonical_sum_i32_clustered_nulls(bencher: Bencher) {
231-
let input = i32_clustered_nulls_input();
232-
bencher
233-
.with_inputs(|| &input)
234-
.bench_refs(|input| grouped_accumulator_canonical(input, Sum));
235-
}
236-
237-
#[divan::bench]
238-
fn canonical_sum_f64_all_valid(bencher: Bencher) {
239-
let input = f64_all_valid_input();
240-
bencher
241-
.with_inputs(|| &input)
242-
.bench_refs(|input| grouped_accumulator_canonical(input, Sum));
243-
}
244-
245-
#[divan::bench]
246-
fn canonical_sum_f64_clustered_nulls(bencher: Bencher) {
247-
let input = f64_clustered_nulls_input();
248-
bencher
249-
.with_inputs(|| &input)
250-
.bench_refs(|input| grouped_accumulator_canonical(input, Sum));
251-
}
252-
253202
#[divan::bench]
254203
fn count_i32_clustered_nulls(bencher: Bencher) {
255204
let input = i32_clustered_nulls_input();

‎vortex-array/src/aggregate_fn/accumulator.rs‎

Lines changed: 10 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -119,18 +119,9 @@ impl<V: AggregateFnVTable> DynAccumulator for Accumulator<V> {
119119
batch.dtype()
120120
);
121121

122-
// Stat::Sum stores a finalized scalar rather than a partial. Give Sum's vtable the first
123-
// chance to consume that cache before dispatching an encoding kernel.
124-
let checked_cached_sum = Stat::from_aggregate_fn(&self.aggregate_fn) == Some(Stat::Sum)
125-
&& batch.statistics().get(Stat::Sum).is_exact();
126-
if checked_cached_sum && self.vtable.try_accumulate(&mut self.partial, batch, ctx)? {
127-
return Ok(());
128-
}
129-
130-
// 0. Cached stats bridge: consume an exact partial from the aggregate's Stat slot before
131-
// kernel dispatch or decode. Sum is handled above because its cache stores a result.
122+
// 0. Legacy stats bridge: if this aggregate is still cached under a legacy Stat slot,
123+
// consume that exact stat before kernel dispatch or decode.
132124
if let Some(stat) = Stat::from_aggregate_fn(&self.aggregate_fn)
133-
&& stat != Stat::Sum
134125
&& let Precision::Exact(partial) = batch.statistics().get(stat)
135126
{
136127
let partial = if partial.dtype() == &self.partial_dtype {
@@ -176,7 +167,7 @@ impl<V: AggregateFnVTable> DynAccumulator for Accumulator<V> {
176167
}
177168

178169
// 2. Allow the vtable to short-circuit on the raw array before decompression.
179-
if !checked_cached_sum && self.vtable.try_accumulate(&mut self.partial, batch, ctx)? {
170+
if self.vtable.try_accumulate(&mut self.partial, batch, ctx)? {
180171
return Ok(());
181172
}
182173

@@ -289,7 +280,6 @@ mod tests {
289280
use crate::aggregate_fn::combined::PairOptions;
290281
use crate::aggregate_fn::fns::mean::Mean;
291282
use crate::aggregate_fn::fns::sum::Sum;
292-
use crate::aggregate_fn::fns::sum::SumAggregateOpts;
293283
use crate::aggregate_fn::kernels::DynAggregateKernel;
294284
use crate::aggregate_fn::session::AggregateFnSession;
295285
use crate::array::VTable;
@@ -298,10 +288,7 @@ mod tests {
298288
use crate::dtype::DType;
299289
use crate::dtype::Nullability;
300290
use crate::dtype::PType;
301-
use crate::expr::stats::Precision;
302-
use crate::expr::stats::Stat;
303291
use crate::scalar::Scalar;
304-
use crate::scalar::ScalarValue;
305292

306293
/// Mean partial sentinel `{sum: 42.0, count: 1}` — distinguishable from the
307294
/// natural fan-out result `{sum: 7.0, count: 1}` that `Combined::try_accumulate`
@@ -333,7 +320,7 @@ mod tests {
333320
}
334321
}
335322

336-
/// Sum partial sentinel `{sum: 42.0, is_overflow: false, is_empty: false}` — distinguishable from the natural Sum of
323+
/// Sum partial sentinel `42.0` — distinguishable from the natural Sum of
337324
/// `dict_of_seven()` which is `7.0`.
338325
#[derive(Debug)]
339326
struct SentinelSumPartialKernel;
@@ -344,7 +331,7 @@ mod tests {
344331
_batch: &ArrayRef,
345332
_ctx: &mut ExecutionCtx,
346333
) -> VortexResult<Option<Scalar>> {
347-
Ok(Some(sum_partial(42.0)))
334+
Ok(Some(Scalar::primitive(42.0f64, Nullability::Nullable)))
348335
}
349336
}
350337

@@ -363,7 +350,7 @@ mod tests {
363350
Accumulator::try_new(
364351
Mean::combined(),
365352
PairOptions(
366-
SumAggregateOpts::default(),
353+
NumericalAggregateOpts::default(),
367354
NumericalAggregateOpts::default(),
368355
),
369356
dtype,
@@ -372,24 +359,11 @@ mod tests {
372359

373360
fn sentinel_partial() -> Scalar {
374361
let acc = mean_f64_accumulator().expect("build accumulator");
375-
let sum = sum_partial(42.0);
362+
let sum = Scalar::primitive(42.0f64, Nullability::Nullable);
376363
let count = Scalar::primitive(1u64, Nullability::NonNullable);
377364
Scalar::struct_(acc.partial_dtype, vec![sum, count])
378365
}
379366

380-
fn sum_partial(value: f64) -> Scalar {
381-
let dtype = DType::Primitive(PType::F64, Nullability::NonNullable);
382-
let acc = Accumulator::try_new(Sum, SumAggregateOpts::default(), dtype).expect("sum");
383-
Scalar::struct_(
384-
acc.partial_dtype,
385-
vec![
386-
Scalar::primitive(value, Nullability::NonNullable),
387-
Scalar::bool(false, Nullability::NonNullable),
388-
Scalar::bool(false, Nullability::NonNullable),
389-
],
390-
)
391-
}
392-
393367
/// Kernel registered for `(Dict, Combined<Mean>)` fires in preference to
394368
/// `Combined::try_accumulate`'s fan-out path — proves the dispatch reorder.
395369
#[test]
@@ -407,13 +381,7 @@ mod tests {
407381

408382
let s = partial.as_struct();
409383
assert_eq!(
410-
s.field("sum")
411-
.unwrap()
412-
.as_struct()
413-
.field("sum")
414-
.unwrap()
415-
.as_primitive()
416-
.as_::<f64>(),
384+
s.field("sum").unwrap().as_primitive().as_::<f64>(),
417385
Some(42.0)
418386
);
419387
assert_eq!(
@@ -440,13 +408,7 @@ mod tests {
440408

441409
let s = partial.as_struct();
442410
assert_eq!(
443-
s.field("sum")
444-
.unwrap()
445-
.as_struct()
446-
.field("sum")
447-
.unwrap()
448-
.as_primitive()
449-
.as_::<f64>(),
411+
s.field("sum").unwrap().as_primitive().as_::<f64>(),
450412
Some(7.0)
451413
);
452414
assert_eq!(
@@ -477,13 +439,7 @@ mod tests {
477439
// via `Combined<Mean>`'s fan-out. `Count`'s native `try_accumulate` reads the
478440
// batch's valid_count, so count is the real 1.
479441
assert_eq!(
480-
s.field("sum")
481-
.unwrap()
482-
.as_struct()
483-
.field("sum")
484-
.unwrap()
485-
.as_primitive()
486-
.as_::<f64>(),
442+
s.field("sum").unwrap().as_primitive().as_::<f64>(),
487443
Some(42.0)
488444
);
489445
assert_eq!(
@@ -492,26 +448,4 @@ mod tests {
492448
);
493449
Ok(())
494450
}
495-
496-
#[test]
497-
fn cached_sum_precedes_encoding_kernel() -> VortexResult<()> {
498-
static KERNEL: SentinelSumPartialKernel = SentinelSumPartialKernel;
499-
let session = fresh_session();
500-
session
501-
.get::<AggregateFnSession>()
502-
.register_aggregate_kernel(Dict.id(), Some(Sum.id()), &KERNEL);
503-
let mut ctx = session.create_execution_ctx();
504-
505-
let batch = dict_of_seven();
506-
batch
507-
.statistics()
508-
.set(Stat::Sum, Precision::Exact(ScalarValue::from(11.0f64)));
509-
510-
let dtype = DType::Primitive(PType::F64, Nullability::NonNullable);
511-
let mut acc = Accumulator::try_new(Sum, SumAggregateOpts::default(), dtype)?;
512-
acc.accumulate(&batch, &mut ctx)?;
513-
514-
assert_eq!(acc.finish()?.as_primitive().as_::<f64>(), Some(11.0));
515-
Ok(())
516-
}
517451
}

‎vortex-array/src/aggregate_fn/accumulator_grouped.rs‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -273,10 +273,7 @@ impl<V: AggregateFnVTable> DynGroupedAccumulator for GroupedAccumulator<V> {
273273
}
274274

275275
fn flush(&mut self) -> VortexResult<ArrayRef> {
276-
let mut states = std::mem::take(&mut self.partials);
277-
if states.len() == 1 {
278-
return Ok(states.pop().vortex_expect("checked one partial"));
279-
}
276+
let states = std::mem::take(&mut self.partials);
280277
Ok(ChunkedArray::try_new(states, self.partial_dtype.clone())?.into_array())
281278
}
282279

‎vortex-array/src/aggregate_fn/fns/mean/mod.rs‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ use crate::aggregate_fn::combined::CombinedOptions;
1919
use crate::aggregate_fn::combined::PairOptions;
2020
use crate::aggregate_fn::fns::count::Count;
2121
use crate::aggregate_fn::fns::sum::Sum;
22-
use crate::aggregate_fn::fns::sum::SumAggregateOpts;
2322
use crate::aggregate_fn::fns::sum::sum_decimal_dtype;
2423
use crate::arrays::ConstantArray;
2524
use crate::builtins::ArrayBuiltins;
@@ -41,7 +40,7 @@ pub fn mean(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Scalar> {
4140
let mut acc = Accumulator::try_new(
4241
Mean::combined(),
4342
PairOptions(
44-
SumAggregateOpts::default(),
43+
NumericalAggregateOpts::default(),
4544
NumericalAggregateOpts::default(),
4645
),
4746
array.dtype().clone(),
@@ -320,7 +319,7 @@ mod tests {
320319
let keep_nans = NumericalAggregateOpts::include_nans();
321320
let mut acc = Accumulator::try_new(
322321
Mean::combined(),
323-
PairOptions(keep_nans.into(), keep_nans),
322+
PairOptions(keep_nans, keep_nans),
324323
array.dtype().clone(),
325324
)?;
326325
acc.accumulate(&array, &mut ctx)?;
@@ -412,7 +411,7 @@ mod tests {
412411
let mut acc = Accumulator::try_new(
413412
Mean::combined(),
414413
PairOptions(
415-
SumAggregateOpts::default(),
414+
NumericalAggregateOpts::default(),
416415
NumericalAggregateOpts::default(),
417416
),
418417
dtype,
@@ -447,7 +446,7 @@ mod tests {
447446
let mut acc = Accumulator::try_new(
448447
Mean::combined(),
449448
PairOptions(
450-
SumAggregateOpts::default(),
449+
NumericalAggregateOpts::default(),
451450
NumericalAggregateOpts::default(),
452451
),
453452
DType::Primitive(PType::F64, Nullability::Nullable),
@@ -475,7 +474,7 @@ mod tests {
475474
let mut acc = GroupedAccumulator::try_new(
476475
Mean::combined(),
477476
PairOptions(
478-
SumAggregateOpts::default(),
477+
NumericalAggregateOpts::default(),
479478
NumericalAggregateOpts::default(),
480479
),
481480
DType::Primitive(PType::F64, Nullability::Nullable),

0 commit comments

Comments
 (0)