Skip to content

Commit d1d6ba4

Browse files
authored
fix: Find SQL aggregates by their function name, and raise on nested aggregate calls (#29751)
1 parent 5c6a473 commit d1d6ba4

7 files changed

Lines changed: 509 additions & 556 deletions

File tree

‎crates/polars-sql/src/context.rs‎

Lines changed: 89 additions & 550 deletions
Large diffs are not rendered by default.

‎crates/polars-sql/src/functions.rs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1212,6 +1212,30 @@ impl PolarsSQLFunctions {
12121212
},
12131213
})
12141214
}
1215+
1216+
/// Whether `call`, the parsed call of `function` without OVER, aggregates the rows of a
1217+
/// group: a SQL aggregate, or a user-defined function that returns one value.
1218+
pub(crate) fn is_aggregate_call(
1219+
function: &SQLFunction,
1220+
ctx: &SQLContext,
1221+
call: &Expr,
1222+
) -> PolarsResult<bool> {
1223+
use PolarsSQLFunctions::*;
1224+
Ok(match Self::try_from_sql(function, ctx)? {
1225+
#[cfg(feature = "approx_quantile")]
1226+
ApproxQuantile => true,
1227+
ArrayAgg | Avg | Corr | Count | CovarPop | CovarSamp | First | Last | Max | Median
1228+
| Min | QuantileCont | QuantileDisc | StdDev | StringAgg | Sum | Total | Variance => {
1229+
true
1230+
},
1231+
// Without OVER, FIRST_VALUE is FIRST.
1232+
FirstValue => true,
1233+
Udf(_) => {
1234+
matches!(call, Expr::AnonymousFunction { options, .. } if options.returns_scalar())
1235+
},
1236+
_ => false,
1237+
})
1238+
}
12151239
}
12161240

12171241
impl SQLFunctionVisitor<'_> {
Lines changed: 306 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,306 @@
1+
//! The group context of a SELECT block: which parts of its expressions are computed per
2+
//! group, and which run on the aggregated rows.
3+
4+
use std::sync::LazyLock;
5+
6+
use polars_core::prelude::*;
7+
use polars_lazy::prelude::*;
8+
use polars_plan::plans::visitor::{TreeWalker, VisitRecursion, Visitor};
9+
use polars_plan::prelude::*;
10+
use polars_utils::aliases::PlHashSet;
11+
use polars_utils::format_pl_smallstr;
12+
13+
use crate::context::is_correlated_result_col;
14+
use crate::unique_column_name;
15+
16+
/// The expressions computed in the group context, without aggregate marks.
17+
pub(crate) struct AggregateOutputs {
18+
exprs: Vec<Expr>,
19+
}
20+
21+
impl AggregateOutputs {
22+
pub(crate) fn with_capacity(capacity: usize) -> Self {
23+
Self {
24+
exprs: Vec::with_capacity(capacity),
25+
}
26+
}
27+
28+
pub(crate) fn push(&mut self, expr: Expr) {
29+
self.exprs.push(strip_aggregate_marks(expr));
30+
}
31+
32+
pub(crate) fn iter(&self) -> impl Iterator<Item = &Expr> {
33+
self.exprs.iter()
34+
}
35+
36+
pub(crate) fn into_exprs(self) -> Vec<Expr> {
37+
self.exprs
38+
}
39+
40+
/// Reference to the output computing `expr`, reusing an existing aggregate with
41+
/// the same expression (from SELECT, or the same aggregate repeated in a window).
42+
fn get_or_insert_hoisted(&mut self, expr: Expr) -> Expr {
43+
let expr = strip_aggregate_marks(expr);
44+
let existing = self.exprs.iter().find_map(|agg| match agg {
45+
Expr::Alias(inner, name) if **inner == expr => Some(name.clone()),
46+
_ => None,
47+
});
48+
let name = existing.unwrap_or_else(|| {
49+
let name = format_pl_smallstr!("__POLARS_HOISTED_AGG_{}", unique_column_name());
50+
self.exprs.push(expr.alias(name.clone()));
51+
name
52+
});
53+
col(name)
54+
}
55+
}
56+
57+
/// Splits post-aggregation expressions from the aggregates they contain.
58+
pub(crate) struct GroupContextSplitter<'a> {
59+
pub(crate) keys: &'a Schema,
60+
/// Each group key without its alias, and the column holding it after aggregation.
61+
pub(crate) key_exprs: Vec<(Expr, PlSmallStr)>,
62+
pub(crate) whole_frame_partition: Option<&'a PlSmallStr>,
63+
/// Columns holding the result of a scalar subquery.
64+
pub(crate) subquery_names: &'a PlHashSet<PlSmallStr>,
65+
pub(crate) aggregates: AggregateOutputs,
66+
}
67+
68+
impl GroupContextSplitter<'_> {
69+
/// Whether `expr` has one value per group without being an aggregate call: a scalar
70+
/// subquery, or a correlated column.
71+
fn is_group_value(&self, expr: &Expr) -> bool {
72+
matches!(expr, Expr::Agg(AggExpr::First(inner)) if matches!(inner.as_ref(), Expr::Column(name)
73+
if self.subquery_names.contains(name) || is_correlated_result_col(name)))
74+
}
75+
76+
/// Whether a SELECT projection must be processed in the group context rather
77+
/// than passed through as a group key: it contains an aggregate, a window, or a
78+
/// function over a non-key column.
79+
pub(crate) fn requires_group_processing(&self, expr: &Expr) -> bool {
80+
has_expr(expr, |e| match e {
81+
_ if is_marked_aggregate(e) => true,
82+
Expr::Agg(_) | Expr::Len | Expr::Over { .. } => true,
83+
#[cfg(feature = "dynamic_group_by")]
84+
Expr::Rolling { .. } => true,
85+
Expr::AnonymousFunction { options, .. } => options.returns_scalar(),
86+
Expr::Function { function: func, .. }
87+
if !matches!(func, FunctionExpr::StructExpr(_)) =>
88+
{
89+
has_expr(
90+
e,
91+
|e| matches!(e, Expr::Column(name) if !self.keys.contains(name)),
92+
)
93+
},
94+
_ => false,
95+
})
96+
}
97+
98+
/// Whether `expr` must run after aggregation: it holds a window, or combines
99+
/// an aggregate with a grouped key.
100+
pub(crate) fn needs_post_aggregation(&self, expr: &Expr) -> bool {
101+
struct Finder<'a, 'b> {
102+
splitter: &'a GroupContextSplitter<'b>,
103+
has_window: bool,
104+
has_group_key: bool,
105+
has_aggregate: bool,
106+
}
107+
impl Visitor for Finder<'_, '_> {
108+
type Node = Expr;
109+
type Arena = ();
110+
111+
fn pre_visit(&mut self, node: &Expr, _: &()) -> PolarsResult<VisitRecursion> {
112+
// Siblings are still visited so that no aggregate goes unnoticed.
113+
Ok(match node {
114+
Expr::Over { .. } => {
115+
self.has_window = true;
116+
VisitRecursion::Skip
117+
},
118+
Expr::Column(name) => {
119+
self.has_group_key |= self.splitter.keys.contains(name);
120+
VisitRecursion::Skip
121+
},
122+
_ if is_marked_aggregate(node) || self.splitter.is_group_value(node) => {
123+
self.has_aggregate = true;
124+
VisitRecursion::Skip
125+
},
126+
_ => VisitRecursion::Continue,
127+
})
128+
}
129+
}
130+
let mut finder = Finder {
131+
splitter: self,
132+
has_window: false,
133+
has_group_key: false,
134+
has_aggregate: false,
135+
};
136+
let _ = expr.visit(&mut finder, &());
137+
finder.has_window || (finder.has_group_key && finder.has_aggregate)
138+
}
139+
140+
/// Replace every aggregate in `expr` with a reference to a hoisted aggregation output,
141+
/// recorded in `self.aggregates`. A window runs on the aggregated rows; its inputs are
142+
/// bound by `bind_window_input`.
143+
pub(crate) fn hoist(&mut self, expr: Expr) -> PolarsResult<Expr> {
144+
match expr {
145+
Expr::Over {
146+
function,
147+
partition_by,
148+
order_by,
149+
mapping,
150+
} => Ok(Expr::Over {
151+
function: Arc::new(self.bind_window_input(Arc::unwrap_or_clone(function))?),
152+
partition_by: partition_by
153+
.into_iter()
154+
.map(|e| match e {
155+
Expr::Column(name) if Some(&name) == self.whole_frame_partition => {
156+
Ok(lit(1))
157+
},
158+
e => self.bind_window_input(e),
159+
})
160+
.collect::<PolarsResult<_>>()?,
161+
order_by: order_by
162+
.map(|(e, options)| {
163+
let e = self.bind_window_input(Arc::unwrap_or_clone(e))?;
164+
PolarsResult::Ok((Arc::new(e), options))
165+
})
166+
.transpose()?,
167+
mapping,
168+
}),
169+
e if is_marked_aggregate(&e) || self.is_group_value(&e) => {
170+
Ok(self.aggregates.get_or_insert_hoisted(e))
171+
},
172+
e => e.map_children(&mut |c, _| self.hoist(c), &mut ()),
173+
}
174+
}
175+
176+
/// Bind an input of a window function to the aggregated rows: an aggregate reads its
177+
/// hoisted output and a group key reads its key column.
178+
fn bind_window_input(&mut self, expr: Expr) -> PolarsResult<Expr> {
179+
if let Some((_, name)) = self.key_exprs.iter().find(|(key, _)| *key == expr) {
180+
return Ok(col(name.clone()));
181+
}
182+
match expr {
183+
e if is_marked_aggregate(&e) || self.is_group_value(&e) => {
184+
Ok(self.aggregates.get_or_insert_hoisted(e))
185+
},
186+
Expr::Column(name) if self.subquery_names.contains(&name) => {
187+
Ok(self.aggregates.get_or_insert_hoisted(col(name).first()))
188+
},
189+
Expr::Column(name) => {
190+
polars_ensure!(
191+
self.keys.contains(&name),
192+
SQLSyntax: "'{}' should participate in the GROUP BY clause or an aggregate function", name
193+
);
194+
Ok(Expr::Column(name))
195+
},
196+
e => e.map_children(&mut |c, _| self.bind_window_input(c), &mut ()),
197+
}
198+
}
199+
}
200+
201+
/// The names that QUALIFY and ORDER BY can read besides the input columns: columns renamed
202+
/// by `SELECT * RENAME`, then SELECT aliases. Each stands for an expression over the input
203+
/// columns, of a type that is known if it could be inferred.
204+
pub(crate) struct OutputNames {
205+
names: PlHashMap<PlSmallStr, (Expr, Option<DataType>)>,
206+
}
207+
208+
impl OutputNames {
209+
/// The output names of `projections` and `renames` over the input `schema`. An input
210+
/// column comes before an output name of the same name. `typed_schema` is `schema` with
211+
/// the columns that the projections read before the block resolves them.
212+
pub(crate) fn new(
213+
projections: &[Expr],
214+
renames: &PlHashMap<PlSmallStr, PlSmallStr>,
215+
schema: &Schema,
216+
typed_schema: &Schema,
217+
) -> Self {
218+
let mut names = PlHashMap::new();
219+
for (before, after) in renames {
220+
if !schema.contains(after) {
221+
names
222+
.entry(after.clone())
223+
.or_insert_with(|| (col(before.clone()), schema.get(before).cloned()));
224+
}
225+
}
226+
for projection in projections {
227+
if let Expr::Alias(inner, name) = projection
228+
&& !schema.contains(name)
229+
&& !names.contains_key(name)
230+
{
231+
let dtype = projection.to_field(typed_schema).ok().map(|f| f.dtype);
232+
names.insert(name.clone(), (inner.as_ref().clone(), dtype));
233+
}
234+
}
235+
Self { names }
236+
}
237+
238+
/// `schema` with the output names of a known type, to parse an expression that can
239+
/// read them.
240+
pub(crate) fn extend_schema(&self, schema: &Schema) -> Schema {
241+
let mut schema = schema.clone();
242+
for (name, (_, dtype)) in &self.names {
243+
if let Some(dtype) = dtype {
244+
schema.with_column(name.clone(), dtype.clone());
245+
}
246+
}
247+
schema
248+
}
249+
250+
/// Replace the output names in `expr` by their expressions.
251+
pub(crate) fn resolve(&self, expr: Expr) -> Expr {
252+
expr.map_expr(|e| match e {
253+
Expr::Column(name) => match self.names.get(&name) {
254+
Some((expr, _)) => expr.clone(),
255+
None => Expr::Column(name),
256+
},
257+
e => e,
258+
})
259+
}
260+
}
261+
262+
/// Whether a block without GROUP BY has both windows and aggregates, as in
263+
/// `SELECT SUM(x), COUNT(*) OVER () FROM t`.
264+
pub(crate) fn has_windows_over_aggregates(projections: &[Expr]) -> bool {
265+
projections
266+
.iter()
267+
.any(|e| has_expr(e, |e| matches!(e, Expr::Over { .. })))
268+
&& projections.iter().any(has_marked_aggregate)
269+
}
270+
271+
/// Marks the call of an aggregate function: a SQL aggregate, or a user-defined function that
272+
/// returns one value. The parser sets it in the clauses that run in the group context of a
273+
/// block (the SELECT list, QUALIFY, HAVING and the aggregates of ORDER BY), and in the inputs
274+
/// of window functions, as on `COUNT(*)` in `SUM(COUNT(*)) OVER ()`. A window function is not
275+
/// marked, nor are the reductions that its lowering adds.
276+
///
277+
/// The mark is a rename that keeps the name, by a callback that only this module holds, so no
278+
/// alias written in the query is taken for it. `GroupContextSplitter` computes each marked
279+
/// aggregate per group, and a block without GROUP BY or windows removes the marks, so none
280+
/// reach the plan.
281+
static AGGREGATE_MARK: LazyLock<PlanCallback<PlSmallStr, PlSmallStr>> =
282+
LazyLock::new(|| PlanCallback::new(Ok));
283+
284+
pub(crate) fn mark_aggregate(expr: Expr) -> Expr {
285+
Expr::RenameAlias {
286+
function: RenameAliasFn::Map(AGGREGATE_MARK.clone()),
287+
expr: Arc::new(expr),
288+
}
289+
}
290+
291+
pub(crate) fn is_marked_aggregate(expr: &Expr) -> bool {
292+
matches!(expr, Expr::RenameAlias { function: RenameAliasFn::Map(f), .. } if *f == *AGGREGATE_MARK)
293+
}
294+
295+
/// Whether `expr` holds a marked aggregate.
296+
pub(crate) fn has_marked_aggregate(expr: &Expr) -> bool {
297+
has_expr(expr, is_marked_aggregate)
298+
}
299+
300+
/// `expr` without aggregate marks.
301+
pub(crate) fn strip_aggregate_marks(expr: Expr) -> Expr {
302+
expr.map_expr(|e| match e {
303+
Expr::RenameAlias { expr, .. } if is_marked_aggregate(&e) => Arc::unwrap_or_clone(expr),
304+
e => e,
305+
})
306+
}

‎crates/polars-sql/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
mod context;
55
pub mod function_registry;
66
mod functions;
7+
mod group_context;
78
mod grouping_sets;
89
pub mod keywords;
910
mod literal_folding;

‎crates/polars-sql/src/sql_expr.rs‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,8 @@ use sqlparser::keywords;
3737
use sqlparser::parser::{Parser, ParserOptions};
3838
use sqlparser::tokenizer::Token;
3939

40-
use crate::functions::SQLFunctionVisitor;
40+
use crate::functions::{PolarsSQLFunctions, SQLFunctionVisitor};
41+
use crate::group_context::{has_marked_aggregate, mark_aggregate};
4142
use crate::literal_folding::{
4243
decimal_lit, fold_scalar, parse_exact_literal, try_fold_decimal_arithmetic,
4344
};
@@ -1120,11 +1121,15 @@ impl SQLExprVisitor<'_> {
11201121
self.ctx.group_scope.in_window = in_window;
11211122
let expr = expr?;
11221123

1123-
if in_window && !is_window {
1124-
let empty = Schema::default();
1125-
return self
1126-
.ctx
1127-
.mark_aggregate_call(expr, self.active_schema.unwrap_or(&empty));
1124+
if !is_window
1125+
&& (in_window || self.ctx.group_scope.mark_aggregates)
1126+
&& PolarsSQLFunctions::is_aggregate_call(function, self.ctx, &expr)?
1127+
{
1128+
polars_ensure!(
1129+
!has_marked_aggregate(&expr),
1130+
SQLSyntax: "aggregate function calls cannot be nested"
1131+
);
1132+
return Ok(mark_aggregate(expr));
11281133
}
11291134
Ok(expr)
11301135
}

0 commit comments

Comments
 (0)