Skip to content

Commit 05fec9c

Browse files
committed
fix: keep constant filters above grand totals
A global aggregate or empty grouping set produces a row even on empty input. Pushing a false predicate below it can return that extra row. CLOSES apache#26545
1 parent 0a363f6 commit 05fec9c

6 files changed

Lines changed: 60 additions & 40 deletions

File tree

‎datafusion/optimizer/src/decorrelate.rs‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ use std::sync::Arc;
99

1010
use crate::push_down_filter::lr_is_preserved;
1111
use crate::simplify_expressions::ExprSimplifier;
12-
use crate::utils::replace_qualified_name;
12+
use crate::utils::{aggregate_emits_on_empty_input, replace_qualified_name};
1313

1414
use datafusion_common::nested_struct::is_injective_cast;
1515
use datafusion_common::tree_node::{
@@ -889,10 +889,7 @@ impl PullUpCorrelatedExpr {
889889
return GroupingSetsPullUp::NoColumnsToAdd;
890890
}
891891

892-
let has_empty_set = grouping_sets.iter().any(|grouping_set| match grouping_set {
893-
GroupingSet::Rollup(_) | GroupingSet::Cube(_) => true,
894-
GroupingSet::GroupingSets(sets) => sets.iter().any(|set| set.is_empty()),
895-
});
892+
let has_empty_set = aggregate_emits_on_empty_input(group_expr);
896893
// `GROUPING`/`GROUPING_ID` are resolved to bit tests on `__grouping_id`
897894
// above the aggregate by `ResolveGroupingFunction` before this rule
898895
// runs, so `aggregate.aggr_expr` never holds a `GROUPING` call by the

‎datafusion/optimizer/src/propagate_empty_relation.rs‎

Lines changed: 3 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,10 @@ use datafusion_common::JoinType;
1010
use datafusion_common::tree_node::Transformed;
1111
use datafusion_common::{Column, DFSchemaRef, Result, ScalarValue, plan_err};
1212
use datafusion_expr::logical_plan::LogicalPlan;
13-
use datafusion_expr::{EmptyRelation, Expr, GroupingSet, Projection, Union, cast, lit};
13+
use datafusion_expr::{EmptyRelation, Expr, Projection, Union, cast, lit};
1414

1515
use crate::optimizer::ApplyOrder;
16+
use crate::utils::aggregate_emits_on_empty_input;
1617
use crate::{OptimizerConfig, OptimizerRule};
1718

1819
/// Optimization rule that bottom-up to eliminate plan by propagating empty_relation.
@@ -157,13 +158,7 @@ impl OptimizerRule for PropagateEmptyRelation {
157158
}
158159
}
159160
LogicalPlan::Aggregate(ref agg) => {
160-
// An aggregate over an empty input can be eliminated only when
161-
// there is no empty grouping set. An empty grouping set `()`
162-
// (from `GROUPING SETS(())`, `ROLLUP(...)`, or `CUBE(...)`)
163-
// always produces exactly one row even on empty input, so it
164-
// must not be replaced by an empty relation.
165-
if !agg.group_expr.is_empty()
166-
&& !has_empty_grouping_set(&agg.group_expr)
161+
if !aggregate_emits_on_empty_input(&agg.group_expr)
167162
&& let Some(empty_plan) = empty_child(&plan)?
168163
{
169164
return Ok(Transformed::yes(empty_plan));
@@ -314,30 +309,6 @@ fn build_null_padded_projection(
314309
)?))
315310
}
316311

317-
/// Returns `true` if any grouping set in the list of GROUP BY expressions is
318-
/// the empty set `()`.
319-
///
320-
/// An empty grouping set acts as a "grand total" group: the aggregate must
321-
/// always produce **exactly one row** for it, even when the input is empty.
322-
/// This means an aggregate with an empty grouping set cannot be replaced by
323-
/// an empty relation.
324-
///
325-
/// The three forms that can contain an empty grouping set:
326-
/// - `GROUPING SETS (…, (), …)` — explicitly listed.
327-
/// - `ROLLUP(exprs)` — always expands to include `()`.
328-
/// - `CUBE(exprs)` — always expands to include `()`.
329-
fn has_empty_grouping_set(group_expr: &[Expr]) -> bool {
330-
match group_expr.first() {
331-
Some(Expr::GroupingSet(GroupingSet::GroupingSets(groups))) => {
332-
groups.iter().any(|g| g.is_empty())
333-
}
334-
// Both ROLLUP and CUBE always include the empty grouping set ().
335-
Some(Expr::GroupingSet(GroupingSet::Rollup(_)))
336-
| Some(Expr::GroupingSet(GroupingSet::Cube(_))) => true,
337-
_ => false,
338-
}
339-
}
340-
341312
#[cfg(test)]
342313
mod tests {
343314

‎datafusion/optimizer/src/push_down_filter.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,8 @@ use crate::extract_leaf_expressions::is_pure_extraction_projection;
5858
use crate::optimizer::ApplyOrder;
5959
use crate::simplify_expressions::{reorder_predicates, simplify_predicates};
6060
use crate::utils::{
61-
ColumnReference, has_all_column_refs, is_restrict_null_predicate, schema_columns,
61+
ColumnReference, aggregate_emits_on_empty_input, has_all_column_refs,
62+
is_restrict_null_predicate, schema_columns,
6263
};
6364
use crate::{OptimizerConfig, OptimizerRule};
6465
use datafusion_expr::ExpressionPlacement;
@@ -1112,6 +1113,7 @@ impl OptimizerRule for PushDownFilter {
11121113
.cloned()
11131114
.collect();
11141115
let group_expr_columns = expr_columns(&non_volatile_group_exprs);
1116+
let emits_on_empty = aggregate_emits_on_empty_input(&agg.group_expr);
11151117

11161118
// As for plan Filter: Column(a+b) > 0 -- Agg: groupby:[Column(a)+Column(b)]
11171119
// After push, we need to replace `a+b` with Column(a)+Column(b)
@@ -1132,7 +1134,9 @@ impl OptimizerRule for PushDownFilter {
11321134
let mut push_predicates = vec![];
11331135
for expr in predicates {
11341136
let cols = expr.column_refs();
1137+
// A grand total emits a row even when the predicate empties its input.
11351138
if !expr.is_volatile()
1139+
&& !emits_on_empty
11361140
&& cols.iter().all(|c| group_expr_columns.contains(c))
11371141
{
11381142
push_predicates.push(replace_cols_by_column(expr, &replace_map)?);

‎datafusion/optimizer/src/utils.rs‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ use datafusion_expr::expr::{Exists, InSubquery, SetComparison};
1818
use datafusion_expr::expr_rewriter::replace_col;
1919
use datafusion_expr::physical_planning_context::PhysicalPlanningContext;
2020
use datafusion_expr::{
21-
ColumnarValue, DistinctHandling, Expr, WriteOp, logical_plan::LogicalPlan,
21+
ColumnarValue, DistinctHandling, Expr, GroupingSet, WriteOp,
22+
logical_plan::LogicalPlan,
2223
};
2324
use datafusion_physical_expr::create_physical_expr;
2425
use log::{debug, trace};
@@ -46,6 +47,19 @@ pub(crate) fn is_repeatable(expr: &Expr) -> bool {
4647
.expect("expression traversal is infallible")
4748
}
4849

50+
/// A global aggregate or an empty grouping set produces a grand total even on empty input.
51+
pub(crate) fn aggregate_emits_on_empty_input(group_expr: &[Expr]) -> bool {
52+
match group_expr.first() {
53+
None | Some(Expr::GroupingSet(GroupingSet::Rollup(_) | GroupingSet::Cube(_))) => {
54+
true
55+
}
56+
Some(Expr::GroupingSet(GroupingSet::GroupingSets(groups))) => {
57+
groups.iter().any(Vec::is_empty)
58+
}
59+
_ => false,
60+
}
61+
}
62+
4963
/// Whether an aggregate can safely ignore repeated input rows. This holds when
5064
/// the function declares [`DistinctHandling::Insensitive`], or when it is
5165
/// called with `DISTINCT` and declares [`DistinctHandling::Sensitive`], so that

‎datafusion/sqllogictest/test_files/push_down_filter_regression.slt‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,40 @@
44

55
# Test push down filter
66

7+
# Regression test for https://github.com/apache/datafusion/issues/26545
8+
# Pushing the filter below COUNT(*) would return one row containing zero.
9+
query I
10+
SELECT c FROM (
11+
SELECT 'all' AS k, COUNT(*) AS c FROM (VALUES (1)) AS t(a)
12+
) AS s WHERE k = 'none';
13+
----
14+
15+
# The empty grouping set in ROLLUP also produces a grand total on empty input.
16+
query II
17+
SELECT a, c FROM (
18+
SELECT a, 'all' AS k, COUNT(*) AS c FROM (VALUES (1)) AS t(a)
19+
GROUP BY ROLLUP(a)
20+
) AS s WHERE k = 'none';
21+
----
22+
23+
# A HAVING predicate without column references must not filter the
24+
# aggregate's input either.
25+
query I
26+
SELECT COUNT(*) FROM (VALUES (1)) AS t(a)
27+
HAVING (SELECT max(b) FROM (VALUES (5)) AS u(b)) > 100;
28+
----
29+
30+
# The filter reaches each UNION ALL branch; the 'one' branch must not
31+
# leave a row with a count of zero.
32+
query I
33+
SELECT c FROM (
34+
SELECT 'one' AS k, COUNT(*) AS c FROM (VALUES (1), (2)) AS t(a) WHERE a = 1
35+
UNION ALL
36+
SELECT 'all' AS k, COUNT(*) AS c FROM (VALUES (1), (2)) AS t(a)
37+
) AS s WHERE k = 'all';
38+
----
39+
2
40+
741
# Regression test for https://github.com/apache/datafusion/issues/17188
842
query I
943
COPY (select i as k, i as v from generate_series(1, 10000000) as t(i))

‎datafusion/substrait/tests/cases/roundtrip_logical_plan.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -971,7 +971,7 @@ async fn roundtrip_scalar_subquery_projection_with_join_rewrite() -> Result<()>
971971
Left Join:
972972
TableScan: data projection=[a]
973973
Aggregate: groupBy=[[]], aggr=[[max(data2.b)]]
974-
TableScan: data2 projection=[b], partial_filters=[Boolean(true)]
974+
TableScan: data2 projection=[b]
975975
"
976976
);
977977

0 commit comments

Comments
 (0)