Skip to content

Commit a203cf2

Browse files
committed
fix: keep volatile filters above aggregates
Evaluate volatile predicates once per group so filtering cannot change the number of input rows counted within a group. CLOSES apache#27074
1 parent a616831 commit a203cf2

2 files changed

Lines changed: 21 additions & 4 deletions

File tree

‎datafusion/optimizer/src/push_down_filter.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1132,7 +1132,9 @@ impl OptimizerRule for PushDownFilter {
11321132
let mut push_predicates = vec![];
11331133
for expr in predicates {
11341134
let cols = expr.column_refs();
1135-
if cols.iter().all(|c| group_expr_columns.contains(c)) {
1135+
if !expr.is_volatile()
1136+
&& cols.iter().all(|c| group_expr_columns.contains(c))
1137+
{
11361138
push_predicates.push(replace_cols_by_column(expr, &replace_map)?);
11371139
} else {
11381140
keep_predicates.push(expr);

‎datafusion/sqllogictest/test_files/push_down_filter_regression.slt‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -717,14 +717,29 @@ SELECT evaluations();
717717
----
718718
10
719719

720+
statement ok
721+
set datafusion.explain.logical_plan_only = true;
722+
723+
# Regression for #27074: evaluate random() once per group, while pushing
724+
# the deterministic predicate on the grouping key below the aggregate.
725+
query TT
726+
EXPLAIN SELECT * FROM (
727+
SELECT column1, COUNT(*) AS c FROM (VALUES (1), (1), (2)) GROUP BY column1
728+
)
729+
WHERE column1 = 1 AND random() < 0.5;
730+
----
731+
logical_plan
732+
01)Projection: column1, count(Int64(1)) AS c
733+
02)--Filter: random() < Float64(0.5)
734+
03)----Aggregate: groupBy=[[column1]], aggr=[[count(Int64(1))]]
735+
04)------Filter: column1 = Int64(1)
736+
05)--------Values: (Int64(1)), (Int64(1)), (Int64(2))
737+
720738
# Window filter pushdown over expression PARTITION BY keys.
721739
#
722740
# A predicate that reads only the partition keys is constant within a partition,
723741
# so it is pushed below the window; anything else stays above it.
724742

725-
statement ok
726-
set datafusion.explain.logical_plan_only = true;
727-
728743
statement ok
729744
create table window_expr_key(k varchar, v int) as values
730745
('a', 1), ('a', 2), ('a', 3),

0 commit comments

Comments
 (0)