Skip to content

Commit 3e48274

Browse files
adriangbclaude
andcommitted
fix: count all volatile calls and outer references in the evaluation sites invariant
Use Expr::is_volatile_node so volatile aggregate, window and higher-order calls are counted, key OuterReferenceColumn like Column, add recursion protection to the plan walk, collect subqueries in the same walk, and correct the module docs about KeepInPlace calls. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 592f8ae commit 3e48274

1 file changed

Lines changed: 65 additions & 63 deletions

File tree

‎datafusion/optimizer/src/evaluation_sites.rs‎

Lines changed: 65 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -45,19 +45,20 @@
4545
//!
4646
//! # What is counted
4747
//!
48-
//! A *site* is one volatile [`Expr::ScalarFunction`] node. Volatile calls
49-
//! only: two evaluations of a volatile call give two different answers, so a
50-
//! new site is always a wrong results bug.
48+
//! A *site* is one volatile function call: an [`Expr`] for which
49+
//! [`Expr::is_volatile_node`] holds, so a scalar, aggregate, window or higher
50+
//! order function. Volatile calls only: two evaluations of a volatile call
51+
//! give two different answers, so a new site is always a wrong results bug.
5152
//!
52-
//! A new evaluation site of an
53+
//! A new evaluation site of a call that is not volatile, such as an
5354
//! [`ExpressionPlacement::KeepInPlace`](datafusion_expr_common::placement::ExpressionPlacement::KeepInPlace)
54-
//! call is not counted, because it is not an invariant: pushing a predicate
55+
//! one, is not counted, because it is not an invariant: pushing a predicate
5556
//! through a projection that computes such an expression one time is a decided
5657
//! trade-off, since the predicate can then reach the scan. See the decisions
5758
//! table of <https://github.com/apache/datafusion/issues/25459>.
5859
//!
59-
//! Sites are keyed by a structural form of the call, in which every column is
60-
//! reduced to its unqualified name. A rule that duplicates a call and
60+
//! Sites are keyed by a structural form of the call, in which every column,
61+
//! outer reference columns included, is reduced to its unqualified name. A rule that duplicates a call and
6162
//! re-qualifies its columns, for example from `f(a)` to `f(test.a)`, must not
6263
//! escape the check. Only a call that is already in the input plan is checked.
6364
//! A rule is free to rewrite `concat_ws` into `concat`, or one Spark function
@@ -82,12 +83,11 @@ use std::collections::HashMap;
8283
use std::sync::Arc;
8384

8485
use datafusion_common::{
85-
Result, internal_err,
86-
tree_node::{TreeNode, TreeNodeRecursion},
86+
Column, Result, internal_err,
87+
tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion},
8788
};
88-
use datafusion_expr::expr::{Exists, InSubquery, ScalarFunction, SetComparison};
89-
use datafusion_expr::expr_rewriter::unnormalize_col;
90-
use datafusion_expr::{Expr, LogicalPlan, Volatility};
89+
use datafusion_expr::expr::{Exists, InSubquery, SetComparison};
90+
use datafusion_expr::{Expr, LogicalPlan};
9191

9292
/// Report a volatile call that the optimizer run made the plan evaluate at
9393
/// more sites than the input plan does.
@@ -197,8 +197,9 @@ impl EvaluationSites {
197197
}
198198

199199
/// `own_sites(plan) + max over inputs of count_plan(input)`.
200+
#[cfg_attr(feature = "recursive_protection", recursive::recursive)]
200201
fn count_plan(plan: &LogicalPlan, acc: &mut Counts) {
201-
count_own(plan, acc);
202+
let subqueries = count_own(plan, acc);
202203

203204
let mut branch_max = Counts::new();
204205
let mut branch = Counts::new();
@@ -207,7 +208,7 @@ fn count_plan(plan: &LogicalPlan, acc: &mut Counts) {
207208
}
208209
// A subquery runs over its own rows, so it is another lineage, not a
209210
// continuation of this one.
210-
for subquery in subquery_plans(plan) {
211+
for subquery in subqueries {
211212
merge_branch(&subquery, &mut branch, &mut branch_max);
212213
}
213214

@@ -227,68 +228,57 @@ fn merge_branch(branch_plan: &LogicalPlan, branch: &mut Counts, branch_max: &mut
227228
}
228229
}
229230

230-
/// The evaluation sites in the expressions of `plan` itself.
231-
fn count_own(plan: &LogicalPlan, acc: &mut Counts) {
231+
/// Count the evaluation sites in the expressions of `plan` itself into `acc`,
232+
/// and return the plans of the subqueries that those expressions reference.
233+
fn count_own(plan: &LogicalPlan, acc: &mut Counts) -> Vec<Arc<LogicalPlan>> {
232234
// `TableScan::filters` is a copy of a predicate that the source evaluates.
233235
// Unless the source reports `Exact` support the `Filter` node stays above
234236
// the scan, so counting both would report every filter pushdown.
235-
if matches!(plan, LogicalPlan::TableScan(_)) {
236-
return;
237-
}
237+
let count_sites = !matches!(plan, LogicalPlan::TableScan(_));
238+
let mut subqueries = Vec::new();
238239
let _ = plan.apply_expressions(|expr| {
239-
count_expr(expr, acc);
240-
Ok(TreeNodeRecursion::Continue)
241-
});
242-
}
243-
244-
fn count_expr(expr: &Expr, acc: &mut Counts) {
245-
let _ = expr.apply(|expr| {
246-
if let Expr::ScalarFunction(func) = expr
247-
&& is_site(func)
248-
{
249-
*acc.entry(structural_key(expr)).or_insert(0) += 1;
250-
}
251-
Ok(TreeNodeRecursion::Continue)
252-
});
253-
}
254-
255-
/// True while a call needs a guard.
256-
///
257-
/// A volatile call is a site whatever its arguments are. `random()` takes no
258-
/// argument at all, and the constant folder does not fold it.
259-
fn is_site(func: &ScalarFunction) -> bool {
260-
func.func.signature().volatility == Volatility::Volatile
261-
}
262-
263-
/// The key of a call: its text, with every column reduced to its unqualified
264-
/// name.
265-
///
266-
/// A rule may re-qualify the columns of a call while it moves it, so the
267-
/// printed text of a call is not stable enough to key by. `f(a)` and
268-
/// `f(test.a)` are the same call.
269-
fn structural_key(expr: &Expr) -> String {
270-
unnormalize_col(expr.clone()).to_string()
271-
}
272-
273-
/// The plans of the subqueries referenced by the expressions of `plan`.
274-
fn subquery_plans(plan: &LogicalPlan) -> Vec<Arc<LogicalPlan>> {
275-
let mut plans = Vec::new();
276-
let _ = plan.apply_expressions(|expr| {
277-
let _ = expr.apply(|expr| {
240+
expr.apply(|expr| {
278241
match expr {
279242
Expr::Exists(Exists { subquery, .. })
280243
| Expr::InSubquery(InSubquery { subquery, .. })
281244
| Expr::SetComparison(SetComparison { subquery, .. })
282245
| Expr::ScalarSubquery(subquery) => {
283-
plans.push(Arc::clone(&subquery.subquery));
246+
subqueries.push(Arc::clone(&subquery.subquery));
247+
}
248+
// A volatile call is a site whatever its arguments are.
249+
// `random()` takes no argument at all, and the constant folder
250+
// does not fold it.
251+
_ if count_sites && expr.is_volatile_node() => {
252+
*acc.entry(structural_key(expr)).or_insert(0) += 1;
284253
}
285254
_ => {}
286255
}
287256
Ok(TreeNodeRecursion::Continue)
288-
});
289-
Ok(TreeNodeRecursion::Continue)
257+
})
290258
});
291-
plans
259+
subqueries
260+
}
261+
262+
/// The key of a call: its text, with every column, outer reference columns
263+
/// included, reduced to its unqualified name.
264+
///
265+
/// A rule may re-qualify the columns of a call while it moves it, so the
266+
/// printed text of a call is not stable enough to key by. `f(a)` and
267+
/// `f(test.a)` are the same call. Decorrelating a subquery turns
268+
/// `f(outer_ref(t.a))` into `f(t.a)`, which is the same call too.
269+
fn structural_key(expr: &Expr) -> String {
270+
expr.clone()
271+
.transform(|expr| {
272+
Ok(match expr {
273+
Expr::Column(c) | Expr::OuterReferenceColumn(_, c) => {
274+
Transformed::yes(Expr::Column(Column::new_unqualified(c.name)))
275+
}
276+
_ => Transformed::no(expr),
277+
})
278+
})
279+
.data()
280+
.expect("structural_key is infallible")
281+
.to_string()
292282
}
293283

294284
#[cfg(test)]
@@ -297,9 +287,11 @@ mod tests {
297287
use crate::test::udfs::PlacementTestUDF;
298288
use crate::test::{test_table_scan, test_table_scan_with_name};
299289
use crate::{Optimizer, OptimizerContext};
290+
use arrow::datatypes::DataType;
300291
use datafusion_common::Result;
301292
use datafusion_expr::{
302-
ExpressionPlacement, JoinType, LogicalPlanBuilder, ScalarUDF, col, exists, lit,
293+
ExpressionPlacement, JoinType, LogicalPlanBuilder, ScalarUDF, Volatility, col,
294+
exists, lit, out_ref_col,
303295
};
304296

305297
/// `leaf_udf(arg)`, made volatile. Its placement says that it may move to
@@ -425,6 +417,16 @@ mod tests {
425417
);
426418
}
427419

420+
/// Decorrelating a subquery turns an outer reference into a plain column,
421+
/// which must not change the key of the call that reads it.
422+
#[test]
423+
fn the_key_ignores_outer_references() {
424+
assert_eq!(
425+
structural_key(&volatile(out_ref_col(DataType::UInt32, "test.a"))),
426+
structural_key(&volatile(col("a")))
427+
);
428+
}
429+
428430
/// Inlining the definition of a column into a node below it is the shape
429431
/// the invariant reports. The call is re-qualified on the way, which a key
430432
/// of printed text would not survive.

0 commit comments

Comments
 (0)