Skip to content

Commit e8e41ae

Browse files
authored
Fix panic from avg(x ORDER BY y) (and more) (#25711)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123. --> - Closes #. ## Rationale for this change Several aggregate UDFs keep the default implementation of `order_sensitivity()` but panic when the `ORDER BY` column is passed in. <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. Please explain the problem you are trying to solve in terms of the user-visible behavior, rather than the implementation. For example, "The code in `foo.rs` doesn't handle nulls" is a symptom of the implementation. "COUNT(DISTINCT) returns wrong results when the column contains nulls" is the user-visible problem. --> ## What changes are included in this PR? This change updates these UDFs to return `AggregateOrderSensitivity::Insensitive` so that the `ORDER BY` columns are ignored. <!-- There is no need to duplicate the description in the issue here, but it is sometimes worth providing a summary of the individual changes in this PR. --> ## What is the testing strategy for this PR? I added sqllogictests that fail without the fix. <!-- We typically require tests for all PRs in order to: 1. Prevent the code from being accidentally broken by subsequent changes 2. Serve as another way to document the expected behavior of the code Briefly describe how this PR is tested, and point to the specific tests you added. For example: 'This new feature is covered by the `sqllogictest` cases added in `foo.slt`'. If this PR does not add tests, explain why. For example, if the change is already covered by existing tests, please mention it. You should also check the `codecov` bot reply on this PR to confirm the changed code is exercised. --> ## Are there any user-facing changes? No <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. If there are any breaking changes to public APIs, please add the `api change` label. -->
1 parent aece5fb commit e8e41ae

11 files changed

Lines changed: 218 additions & 8 deletions

File tree

‎datafusion/functions-aggregate/src/approx_distinct.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ use datafusion_common::{
4040
};
4141
use datafusion_expr::DistinctHandling;
4242
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
43-
use datafusion_expr::utils::format_state_name;
43+
use datafusion_expr::utils::{AggregateOrderSensitivity, format_state_name};
4444
use datafusion_expr::{
4545
Accumulator, AggregateUDFImpl, Documentation, EmitTo, GroupsAccumulator, Signature,
4646
Volatility,
@@ -868,6 +868,10 @@ impl AggregateUDFImpl for ApproxDistinct {
868868
}
869869
}
870870

871+
fn order_sensitivity(&self) -> AggregateOrderSensitivity {
872+
AggregateOrderSensitivity::Insensitive
873+
}
874+
871875
fn documentation(&self) -> Option<&Documentation> {
872876
self.doc()
873877
}

‎datafusion/functions-aggregate/src/average.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ use datafusion_common::{
3636
Result, ScalarValue, exec_datafusion_err, exec_err, internal_err, not_impl_err,
3737
};
3838
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
39-
use datafusion_expr::utils::format_state_name;
39+
use datafusion_expr::utils::{AggregateOrderSensitivity, format_state_name};
4040
use datafusion_expr::{
4141
Accumulator, AggregateUDFImpl, Coercion, Documentation, EmitTo, Expr, GroupSelection,
4242
GroupsAccumulator, ReversedUDAF, Signature, TypeSignature, TypeSignatureClass,
@@ -492,6 +492,10 @@ impl AggregateUDFImpl for Avg {
492492
ReversedUDAF::Identical
493493
}
494494

495+
fn order_sensitivity(&self) -> AggregateOrderSensitivity {
496+
AggregateOrderSensitivity::Insensitive
497+
}
498+
495499
fn documentation(&self) -> Option<&Documentation> {
496500
self.doc()
497501
}

‎datafusion/functions-aggregate/src/bit_and_or_xor.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ use datafusion_common::cast::as_list_array;
3333
use datafusion_common::{Result, ScalarValue, not_impl_err};
3434
use datafusion_expr::DistinctHandling;
3535
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
36-
use datafusion_expr::utils::format_state_name;
36+
use datafusion_expr::utils::{AggregateOrderSensitivity, format_state_name};
3737
use datafusion_expr::{
3838
Accumulator, AggregateUDFImpl, Coercion, Documentation, GroupsAccumulator,
3939
ReversedUDAF, Signature, TypeSignatureClass, Volatility,
@@ -316,6 +316,10 @@ impl AggregateUDFImpl for BitwiseOperation {
316316
ReversedUDAF::Identical
317317
}
318318

319+
fn order_sensitivity(&self) -> AggregateOrderSensitivity {
320+
AggregateOrderSensitivity::Insensitive
321+
}
322+
319323
fn documentation(&self) -> Option<&Documentation> {
320324
Some(self.documentation)
321325
}

‎datafusion/functions-aggregate/src/correlation.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ use datafusion_common::{Result, ScalarValue};
4242
use datafusion_expr::{
4343
Accumulator, AggregateUDFImpl, Documentation, Signature, Volatility,
4444
function::{AccumulatorArgs, StateFieldsArgs},
45-
utils::format_state_name,
45+
utils::{AggregateOrderSensitivity, format_state_name},
4646
};
4747
use datafusion_functions_aggregate_common::stats::StatsType;
4848
use datafusion_macros::user_doc;
@@ -131,6 +131,10 @@ impl AggregateUDFImpl for Correlation {
131131
.collect())
132132
}
133133

134+
fn order_sensitivity(&self) -> AggregateOrderSensitivity {
135+
AggregateOrderSensitivity::Insensitive
136+
}
137+
134138
fn documentation(&self) -> Option<&Documentation> {
135139
self.doc()
136140
}

‎datafusion/functions-aggregate/src/median.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ use datafusion_common::Result;
2828
use datafusion_common::types::logical_float64;
2929
use datafusion_expr::GroupsAccumulator;
3030
use datafusion_expr::function::StateFieldsArgs;
31+
use datafusion_expr::utils::AggregateOrderSensitivity;
3132
use datafusion_expr::{
3233
Accumulator, AggregateUDFImpl, Coercion, Documentation, Signature, TypeSignature,
3334
TypeSignatureClass, Volatility, function::AccumulatorArgs,
@@ -143,6 +144,10 @@ impl AggregateUDFImpl for Median {
143144
)
144145
}
145146

147+
fn order_sensitivity(&self) -> AggregateOrderSensitivity {
148+
AggregateOrderSensitivity::Insensitive
149+
}
150+
146151
fn documentation(&self) -> Option<&Documentation> {
147152
self.doc()
148153
}

‎datafusion/functions-aggregate/src/stddev.rs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ use arrow::{array::ArrayRef, datatypes::DataType, datatypes::Field};
2828
use datafusion_common::ScalarValue;
2929
use datafusion_common::{Result, internal_err, not_impl_err};
3030
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
31-
use datafusion_expr::utils::format_state_name;
31+
use datafusion_expr::utils::{AggregateOrderSensitivity, format_state_name};
3232
use datafusion_expr::{
3333
Accumulator, AggregateUDFImpl, Documentation, GroupSelection, GroupsAccumulator,
3434
Signature, Volatility,
@@ -137,6 +137,10 @@ impl AggregateUDFImpl for Stddev {
137137
Ok(Box::new(StddevGroupsAccumulator::new(StatsType::Sample)))
138138
}
139139

140+
fn order_sensitivity(&self) -> AggregateOrderSensitivity {
141+
AggregateOrderSensitivity::Insensitive
142+
}
143+
140144
fn documentation(&self) -> Option<&Documentation> {
141145
self.doc()
142146
}
@@ -243,6 +247,10 @@ impl AggregateUDFImpl for StddevPop {
243247
)))
244248
}
245249

250+
fn order_sensitivity(&self) -> AggregateOrderSensitivity {
251+
AggregateOrderSensitivity::Insensitive
252+
}
253+
246254
fn documentation(&self) -> Option<&Documentation> {
247255
self.doc()
248256
}

‎datafusion/functions-aggregate/src/variance.rs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ use datafusion_expr::{
3030
Accumulator, AggregateUDFImpl, Documentation, GroupSelection, GroupsAccumulator,
3131
Signature, Volatility,
3232
function::{AccumulatorArgs, StateFieldsArgs},
33-
utils::format_state_name,
33+
utils::{AggregateOrderSensitivity, format_state_name},
3434
};
3535
use datafusion_functions_aggregate_common::utils::GenericDistinctBuffer;
3636
use datafusion_functions_aggregate_common::{
@@ -147,6 +147,10 @@ impl AggregateUDFImpl for VarianceSample {
147147
Ok(Box::new(VarianceGroupsAccumulator::new(StatsType::Sample)))
148148
}
149149

150+
fn order_sensitivity(&self) -> AggregateOrderSensitivity {
151+
AggregateOrderSensitivity::Insensitive
152+
}
153+
150154
fn documentation(&self) -> Option<&Documentation> {
151155
self.doc()
152156
}
@@ -252,6 +256,10 @@ impl AggregateUDFImpl for VariancePopulation {
252256
)))
253257
}
254258

259+
fn order_sensitivity(&self) -> AggregateOrderSensitivity {
260+
AggregateOrderSensitivity::Insensitive
261+
}
262+
255263
fn documentation(&self) -> Option<&Documentation> {
256264
self.doc()
257265
}

‎datafusion/spark/src/function/aggregate/avg.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ use arrow::datatypes::{DataType, Field, FieldRef};
2727
use datafusion_common::types::{NativeType, logical_float64};
2828
use datafusion_common::{Result, ScalarValue, not_impl_err};
2929
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
30-
use datafusion_expr::utils::format_state_name;
30+
use datafusion_expr::utils::{AggregateOrderSensitivity, format_state_name};
3131
use datafusion_expr::{
3232
Accumulator, AggregateUDFImpl, Coercion, EmitTo, GroupsAccumulator, ReversedUDAF,
3333
Signature, TypeSignatureClass, Volatility,
@@ -117,6 +117,10 @@ impl AggregateUDFImpl for SparkAvg {
117117
ReversedUDAF::Identical
118118
}
119119

120+
fn order_sensitivity(&self) -> AggregateOrderSensitivity {
121+
AggregateOrderSensitivity::Insensitive
122+
}
123+
120124
fn groups_accumulator_supported(&self, args: AccumulatorArgs) -> bool {
121125
!args.is_distinct
122126
}

‎datafusion/sqllogictest/test_files/aggregate.slt‎

Lines changed: 89 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9951,7 +9951,9 @@ SELECT sum(s) FROM (SELECT sum(column2) OVER () AS s FROM nested_agg_t);
99519951
statement ok
99529952
DROP TABLE nested_agg_t;
99539953

9954-
# Tests for COUNT(... ORDER BY ...)
9954+
# Tests for aggregates that ignore ORDER BY
9955+
9956+
# COUNT(... ORDER BY ...)
99559957
statement ok
99569958
CREATE TABLE count_order_by_t (a INT, b INT, c INT) AS VALUES
99579959
(1, NULL, 10),
@@ -9993,3 +9995,89 @@ physical_plan
99939995

99949996
statement ok
99959997
DROP TABLE count_order_by_t;
9998+
9999+
# Aggregates that keep the default `order_sensitivity` receive the ORDER BY
10000+
# columns as extra arguments, which their GroupsAccumulator must ignore.
10001+
statement ok
10002+
CREATE TABLE agg_order_by_t (g INT, v DOUBLE, i BIGINT) AS VALUES
10003+
(1, 1.0, 1),
10004+
(1, 3.0, 3),
10005+
(2, 5.0, 5),
10006+
(2, 7.0, 7),
10007+
(2, 9.0, 9);
10008+
10009+
# Same aggregates without ORDER BY, for comparison
10010+
query IRIR
10011+
SELECT g, avg(v), bit_and(i), var(v) FROM agg_order_by_t GROUP BY g ORDER BY g;
10012+
----
10013+
1 2 1 2
10014+
2 7 1 4
10015+
10016+
query IR
10017+
SELECT g, avg(v ORDER BY i) FROM agg_order_by_t GROUP BY g ORDER BY g;
10018+
----
10019+
1 2
10020+
2 7
10021+
10022+
query II
10023+
SELECT g, bit_and(i ORDER BY v) FROM agg_order_by_t GROUP BY g ORDER BY g;
10024+
----
10025+
1 1
10026+
2 1
10027+
10028+
query II
10029+
SELECT g, bit_or(i ORDER BY v) FROM agg_order_by_t GROUP BY g ORDER BY g;
10030+
----
10031+
1 3
10032+
2 15
10033+
10034+
query II
10035+
SELECT g, bit_xor(i ORDER BY v) FROM agg_order_by_t GROUP BY g ORDER BY g;
10036+
----
10037+
1 2
10038+
2 11
10039+
10040+
query IR
10041+
SELECT g, var(v ORDER BY i) FROM agg_order_by_t GROUP BY g ORDER BY g;
10042+
----
10043+
1 2
10044+
2 4
10045+
10046+
query IR
10047+
SELECT g, var_pop(v ORDER BY i) FROM agg_order_by_t GROUP BY g ORDER BY g;
10048+
----
10049+
1 1
10050+
2 2.666666666667
10051+
10052+
query IR
10053+
SELECT g, stddev(v ORDER BY i) FROM agg_order_by_t GROUP BY g ORDER BY g;
10054+
----
10055+
1 1.414213562373
10056+
2 2
10057+
10058+
query IR
10059+
SELECT g, stddev_pop(v ORDER BY i) FROM agg_order_by_t GROUP BY g ORDER BY g;
10060+
----
10061+
1 1
10062+
2 1.632993161855
10063+
10064+
# DISTINCT uses a different Accumulator
10065+
query IRR
10066+
SELECT g, avg(DISTINCT v ORDER BY v), var(DISTINCT v ORDER BY v) FROM agg_order_by_t GROUP BY g ORDER BY g;
10067+
----
10068+
1 2 2
10069+
2 7 4
10070+
10071+
# Without GROUP BY the Accumulator is used instead of the GroupsAccumulator
10072+
query R
10073+
SELECT avg(v ORDER BY i) FROM agg_order_by_t;
10074+
----
10075+
5
10076+
10077+
query RR
10078+
SELECT avg(DISTINCT v ORDER BY v), var(DISTINCT v ORDER BY v) FROM agg_order_by_t;
10079+
----
10080+
5 10
10081+
10082+
statement ok
10083+
DROP TABLE agg_order_by_t;

‎datafusion/sqllogictest/test_files/aggregate_skip_partial.slt‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -823,3 +823,74 @@ reset datafusion.execution.skip_partial_aggregation_probe_rows_threshold;
823823

824824
statement ok
825825
DROP TABLE aggregate_test_100_bool
826+
827+
# `convert_to_state` receives every argument, so aggregates that ignore
828+
# ORDER BY must not pass the ORDER BY columns through to it
829+
statement ok
830+
set datafusion.execution.skip_partial_aggregation_probe_rows_threshold = 0;
831+
832+
statement ok
833+
set datafusion.execution.skip_partial_aggregation_probe_ratio_threshold = 0.0;
834+
835+
statement ok
836+
set datafusion.execution.target_partitions = 2;
837+
838+
statement ok
839+
set datafusion.execution.batch_size = 1;
840+
841+
statement ok
842+
CREATE TABLE agg_order_by_t (g INT, v DOUBLE, i BIGINT) AS VALUES
843+
(1, 1.0, 1),
844+
(1, 3.0, 3),
845+
(2, 5.0, 5),
846+
(2, 7.0, 7),
847+
(2, 9.0, 9);
848+
849+
# median without ORDER BY, for comparison
850+
query IR
851+
SELECT g, median(v) FROM agg_order_by_t GROUP BY g ORDER BY g;
852+
----
853+
1 2
854+
2 7
855+
856+
query IR
857+
SELECT g, median(v ORDER BY i) FROM agg_order_by_t GROUP BY g ORDER BY g;
858+
----
859+
1 2
860+
2 7
861+
862+
query II
863+
SELECT g, approx_distinct(i ORDER BY v) FROM agg_order_by_t GROUP BY g ORDER BY g;
864+
----
865+
1 2
866+
2 3
867+
868+
query IR
869+
SELECT g, corr(v, i ORDER BY i) FROM agg_order_by_t GROUP BY g ORDER BY g;
870+
----
871+
1 1
872+
2 1
873+
874+
query IR
875+
SELECT g, var(v ORDER BY i) FROM agg_order_by_t GROUP BY g ORDER BY g;
876+
----
877+
1 2
878+
2 4
879+
880+
statement ok
881+
DROP TABLE agg_order_by_t;
882+
883+
# Config reset
884+
statement ok
885+
reset datafusion.execution.batch_size;
886+
887+
# The SLT runner sets `target_partitions` to 4 instead of using the default, so
888+
# reset it explicitly.
889+
statement ok
890+
set datafusion.execution.target_partitions = 4;
891+
892+
statement ok
893+
reset datafusion.execution.skip_partial_aggregation_probe_ratio_threshold;
894+
895+
statement ok
896+
reset datafusion.execution.skip_partial_aggregation_probe_rows_threshold;

0 commit comments

Comments
 (0)