Skip to content

Commit 5610e58

Browse files
authored
fix: evaluate grouped aggregate arguments after FILTER (#24444)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123. --> - Closes #24443. ## Rationale for this change <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. Please explain the problem you are trying to solve in terms of the user-visible behavior, rather than the implementation. For example, "The code in `foo.rs` doesn't handle nulls" is a symptom of the implementation. "COUNT(DISTINCT) returns wrong results when the column contains nulls" is the user-visible problem. --> Grouped aggregates evaluate their arguments before applying `FILTER`, which can cause errors on rows that should not participate in the aggregate. ## What changes are included in this PR? <!-- There is no need to duplicate the description in the issue here, but it is sometimes worth providing a summary of the individual changes in this PR. --> Evaluate aggregate filters first and use the resulting filters as selections when evaluating arguments. This covers migrated hash and ordered aggregation, legacy grouped hash aggregation, and both skip-partial paths. ## Are these changes tested? <!-- We typically require tests for all PRs in order to: 1. Prevent the code from being accidentally broken by subsequent changes 2. Serve as another way to document the expected behavior of the code If tests are not included in your PR, please explain why (for example, are they covered by existing tests)? --> Yes. slts and unit tests cover migrated, legacy, ordered, and skip-partial execution. ## Are there any user-facing changes? <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. If there are any breaking changes to public APIs, please add the `api change` label. --> No public APIs are changed. Grouped aggregate arguments are no longer evaluated for rows rejected by `FILTER`.
1 parent f951020 commit 5610e58

2 files changed

Lines changed: 118 additions & 14 deletions

File tree

‎datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -179,9 +179,7 @@ impl<AggrMode> AggregateHashTable<AggrMode> {
179179

180180
let timer = self.group_by_metrics.aggregate_arguments_time.timer();
181181
// The evaluated args for each accumulator
182-
let accumulator_args = self
183-
.state
184-
.building()
182+
let accumulator_args = state
185183
.accumulators
186184
.iter()
187185
.enumerate()
@@ -562,23 +560,16 @@ impl HashAggregateAccumulator {
562560

563561
/// Evaluate aggregate arguments and filter for one input batch.
564562
///
565-
/// For example, `AVG(x + 1) FILTER (WHERE x > 0)` evaluates both `x + 1`
566-
/// and `x > 0`.
563+
/// For example, `AVG(2 / x) FILTER (WHERE x > 0)` evaluates `x > 0`
564+
/// first, then evaluates `2 / x` only for selected rows.
565+
/// Filtered rows will be evaluated to `NULL`, and won't trigger errors
566+
/// such as divide by zero.
567567
///
568568
/// These arrays can be passed directly to [`GroupsAccumulator`] next.
569569
pub(super) fn evaluate_acc_args(
570570
&self,
571571
batch: &RecordBatch,
572572
) -> Result<EvaluatedAccumulatorArgs> {
573-
let arguments = self
574-
.arguments
575-
.iter()
576-
.map(|expr| {
577-
expr.evaluate(batch)
578-
.and_then(|value| value.into_array(batch.num_rows()))
579-
})
580-
.collect::<Result<_>>()?;
581-
582573
let filter = self
583574
.filter
584575
.as_ref()
@@ -588,6 +579,19 @@ impl HashAggregateAccumulator {
588579
.and_then(|value| value.into_array(batch.num_rows()))
589580
})
590581
.transpose()?;
582+
let selection = filter.as_ref().map(|filter| filter.as_boolean());
583+
let arguments = self
584+
.arguments
585+
.iter()
586+
.map(|expr| {
587+
selection
588+
.map_or_else(
589+
|| expr.evaluate(batch),
590+
|selection| expr.evaluate_selection(batch, selection),
591+
)
592+
.and_then(|value| value.into_array(batch.num_rows()))
593+
})
594+
.collect::<Result<_>>()?;
591595

592596
Ok(EvaluatedAccumulatorArgs { arguments, filter })
593597
}
Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# Aggregate FILTER must select rows before evaluating fallible arguments.
19+
20+
statement ok
21+
CREATE TABLE aggregate_filter_selection (g INT, v BIGINT);
22+
23+
statement ok
24+
INSERT INTO aggregate_filter_selection VALUES
25+
(1, 0),
26+
(2, 2),
27+
(3, 0),
28+
(4, 5);
29+
30+
statement ok
31+
SET datafusion.execution.skip_partial_aggregation_probe_rows_threshold = 100000;
32+
33+
statement ok
34+
SET datafusion.execution.enable_migration_aggregate = true;
35+
36+
query II
37+
SELECT g, SUM(10 / v) FILTER (WHERE v <> 0)
38+
FROM aggregate_filter_selection
39+
GROUP BY g
40+
ORDER BY g;
41+
----
42+
1 NULL
43+
2 5
44+
3 NULL
45+
4 2
46+
47+
statement ok
48+
SET datafusion.execution.target_partitions = 2;
49+
50+
statement ok
51+
SET datafusion.execution.batch_size = 1;
52+
53+
statement ok
54+
SET datafusion.execution.skip_partial_aggregation_probe_rows_threshold = 0;
55+
56+
statement ok
57+
SET datafusion.execution.skip_partial_aggregation_probe_ratio_threshold = 0.0;
58+
59+
statement ok
60+
CREATE TABLE aggregate_filter_selection_skip (g INT, v BIGINT);
61+
62+
statement ok
63+
INSERT INTO aggregate_filter_selection_skip VALUES
64+
(1, 2),
65+
(3, 5),
66+
(2, 0);
67+
68+
statement ok
69+
SET datafusion.execution.enable_migration_aggregate = true;
70+
71+
query II
72+
SELECT g, SUM(10 / v) FILTER (WHERE v <> 0)
73+
FROM aggregate_filter_selection_skip
74+
GROUP BY g
75+
ORDER BY g;
76+
----
77+
1 5
78+
2 NULL
79+
3 2
80+
81+
statement ok
82+
RESET datafusion.execution.enable_migration_aggregate;
83+
84+
statement ok
85+
RESET datafusion.execution.skip_partial_aggregation_probe_ratio_threshold;
86+
87+
statement ok
88+
RESET datafusion.execution.skip_partial_aggregation_probe_rows_threshold;
89+
90+
statement ok
91+
RESET datafusion.execution.batch_size;
92+
93+
statement ok
94+
SET datafusion.execution.target_partitions = 4;
95+
96+
statement ok
97+
DROP TABLE aggregate_filter_selection_skip;
98+
99+
statement ok
100+
DROP TABLE aggregate_filter_selection;

0 commit comments

Comments
 (0)