Skip to content

Commit 9ef3d47

Browse files
authored
fix: Address avg return type overflow for decimals (apache#24371)
## Which issue does this PR close? - Closes apache#24369 ## Rationale for this change `AVG(Decimal(p, s))` derives a result with precision and scale increased by four. For example, AVG(Decimal32(9, 0)) returned Decimal32(9, 4), although the rescaled average can require 13 digits. This caused valid averages near the input precision limit to fail with an overflow even though Decimal64(13, 4) can represent the result. ## What changes are included in this PR? - Preserve the input decimal variant when the derived result fits, and widen it when necessary, while still clamping Decimal256 at MAX. - Allow decimal AVG accumulators to use different native types for the input, intermediate sum, and output. - Update SQL type assertions and affected logical-plan expectations. ## Are these changes tested? Regression tests and adjusted SLT-based tests. ## Are there any user-facing changes? Yes. The Arrow result type of AVG(decimal) may now be wider when the derived precision does not fit the input variant. --------- Signed-off-by: Adam Gutglick <[email protected]>
1 parent 15c097f commit 9ef3d47

4 files changed

Lines changed: 376 additions & 155 deletions

File tree

‎datafusion/functions-aggregate-common/src/aggregate/avg_distinct/decimal.rs‎

Lines changed: 23 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -32,22 +32,28 @@ use crate::utils::DecimalAverager;
3232
/// Generic implementation of `AVG DISTINCT` for Decimal types.
3333
/// Handles both all Arrow decimal types (32, 64, 128 and 256 bits).
3434
///
35-
/// The distinct values are stored in the input type `I`; only the intermediate
36-
/// sum is computed in the (never narrower) sum type `S` so it cannot overflow
37-
/// `I`'s native type.
35+
/// The distinct values are stored in the input type `I`, the result is returned
36+
/// as the same or wider type `O`, and the intermediate sum is computed in the
37+
/// (never narrower) sum type `S` so it cannot overflow `I`'s native type.
3838
#[derive(Debug)]
3939
pub struct DecimalDistinctAvgAccumulator<
4040
I: DecimalType + Debug,
4141
S: DecimalType + Debug = I,
42+
O: DecimalType + Debug = I,
4243
> {
4344
sum_accumulator: DistinctSumAccumulator<I>,
4445
sum_scale: i8,
4546
target_precision: u8,
4647
target_scale: i8,
47-
_sum_type: PhantomData<S>,
48+
_types: PhantomData<(S, O)>,
4849
}
4950

50-
impl<I: DecimalType + Debug, S: DecimalType + Debug> DecimalDistinctAvgAccumulator<I, S> {
51+
impl<I, S, O> DecimalDistinctAvgAccumulator<I, S, O>
52+
where
53+
I: DecimalType + Debug,
54+
S: DecimalType + Debug,
55+
O: DecimalType + Debug,
56+
{
5157
pub fn with_decimal_params(
5258
sum_scale: i8,
5359
target_precision: u8,
@@ -60,7 +66,7 @@ impl<I: DecimalType + Debug, S: DecimalType + Debug> DecimalDistinctAvgAccumulat
6066
sum_scale,
6167
target_precision,
6268
target_scale,
63-
_sum_type: PhantomData,
69+
_types: PhantomData,
6470
}
6571
}
6672
}
@@ -78,12 +84,14 @@ where
7884
sum.add_wrapping(value.into())
7985
}
8086

81-
impl<I, S> Accumulator for DecimalDistinctAvgAccumulator<I, S>
87+
impl<I, S, O> Accumulator for DecimalDistinctAvgAccumulator<I, S, O>
8288
where
8389
I: DecimalType + ArrowNumericType + Debug,
8490
S: DecimalType + ArrowNumericType + Debug,
91+
O: DecimalType + ArrowNumericType + Debug,
8592
I::Native: Into<S::Native> + DecimalCast,
8693
S::Native: DecimalCast,
94+
O::Native: DecimalCast,
8795
{
8896
fn state(&mut self) -> Result<Vec<ScalarValue>> {
8997
self.sum_accumulator.state()
@@ -98,10 +106,10 @@ where
98106
}
99107

100108
fn evaluate(&mut self) -> Result<ScalarValue> {
101-
let out_type = I::TYPE_CONSTRUCTOR(self.target_precision, self.target_scale);
109+
let out_type = O::TYPE_CONSTRUCTOR(self.target_precision, self.target_scale);
102110
let count = self.sum_accumulator.distinct_count();
103111
if count == 0 {
104-
return ScalarValue::new_primitive::<I>(None, &out_type);
112+
return ScalarValue::new_primitive::<O>(None, &out_type);
105113
}
106114

107115
// Sum the distinct input values in the wider `S` so the total cannot
@@ -123,18 +131,17 @@ where
123131
self.target_precision,
124132
self.target_scale,
125133
)?;
126-
// Narrowing the average back to the (never wider) output type cannot
127-
// fail in practice: `DecimalAverager::avg` validates the average
128-
// against the output precision, whose bound fits the output's native
129-
// type by construction
134+
// Converting the average to the output type cannot fail in practice:
135+
// `DecimalAverager::avg` validates it against the output precision,
136+
// whose bound fits the output's native type by construction.
130137
let avg =
131-
I::Native::from_decimal(averager.avg(sum, count)?).ok_or_else(|| {
138+
O::Native::from_decimal(averager.avg(sum, count)?).ok_or_else(|| {
132139
exec_datafusion_err!(
133140
"Arithmetic overflow in avg: the computed average does not fit \
134-
the output type"
141+
the output type"
135142
)
136143
})?;
137-
ScalarValue::new_primitive::<I>(Some(avg), &out_type)
144+
ScalarValue::new_primitive::<O>(Some(avg), &out_type)
138145
}
139146

140147
fn size(&self) -> usize {

0 commit comments

Comments
 (0)