4545//!
4646//! # What is counted
4747//!
48- //! A *site* is one volatile [`Expr::ScalarFunction`] node. Volatile calls
49- //! only: two evaluations of a volatile call give two different answers, so a
50- //! new site is always a wrong results bug.
48+ //! A *site* is one volatile function call: an [`Expr`] for which
49+ //! [`Expr::is_volatile_node`] holds, so a scalar, aggregate, window or higher
50+ //! order function. Volatile calls only: two evaluations of a volatile call
51+ //! give two different answers, so a new site is always a wrong results bug.
5152//!
52- //! A new evaluation site of an
53+ //! A new evaluation site of a call that is not volatile, such as an
5354//! [`ExpressionPlacement::KeepInPlace`](datafusion_expr_common::placement::ExpressionPlacement::KeepInPlace)
54- //! call is not counted, because it is not an invariant: pushing a predicate
55+ //! one, is not counted, because it is not an invariant: pushing a predicate
5556//! through a projection that computes such an expression one time is a decided
5657//! trade-off, since the predicate can then reach the scan. See the decisions
5758//! table of <https://github.com/apache/datafusion/issues/25459>.
5859//!
59- //! Sites are keyed by a structural form of the call, in which every column is
60- //! reduced to its unqualified name. A rule that duplicates a call and
60+ //! Sites are keyed by a structural form of the call, in which every column,
61+ //! outer reference columns included, is reduced to its unqualified name. A rule that duplicates a call and
6162//! re-qualifies its columns, for example from `f(a)` to `f(test.a)`, must not
6263//! escape the check. Only a call that is already in the input plan is checked.
6364//! A rule is free to rewrite `concat_ws` into `concat`, or one Spark function
@@ -82,12 +83,11 @@ use std::collections::HashMap;
8283use std:: sync:: Arc ;
8384
8485use datafusion_common:: {
85- Result , internal_err,
86- tree_node:: { TreeNode , TreeNodeRecursion } ,
86+ Column , Result , internal_err,
87+ tree_node:: { Transformed , TransformedResult , TreeNode , TreeNodeRecursion } ,
8788} ;
88- use datafusion_expr:: expr:: { Exists , InSubquery , ScalarFunction , SetComparison } ;
89- use datafusion_expr:: expr_rewriter:: unnormalize_col;
90- use datafusion_expr:: { Expr , LogicalPlan , Volatility } ;
89+ use datafusion_expr:: expr:: { Exists , InSubquery , SetComparison } ;
90+ use datafusion_expr:: { Expr , LogicalPlan } ;
9191
9292/// Report a volatile call that the optimizer run made the plan evaluate at
9393/// more sites than the input plan does.
@@ -197,8 +197,9 @@ impl EvaluationSites {
197197}
198198
199199/// `own_sites(plan) + max over inputs of count_plan(input)`.
200+ #[ cfg_attr( feature = "recursive_protection" , recursive:: recursive) ]
200201fn count_plan ( plan : & LogicalPlan , acc : & mut Counts ) {
201- count_own ( plan, acc) ;
202+ let subqueries = count_own ( plan, acc) ;
202203
203204 let mut branch_max = Counts :: new ( ) ;
204205 let mut branch = Counts :: new ( ) ;
@@ -207,7 +208,7 @@ fn count_plan(plan: &LogicalPlan, acc: &mut Counts) {
207208 }
208209 // A subquery runs over its own rows, so it is another lineage, not a
209210 // continuation of this one.
210- for subquery in subquery_plans ( plan ) {
211+ for subquery in subqueries {
211212 merge_branch ( & subquery, & mut branch, & mut branch_max) ;
212213 }
213214
@@ -227,68 +228,57 @@ fn merge_branch(branch_plan: &LogicalPlan, branch: &mut Counts, branch_max: &mut
227228 }
228229}
229230
230- /// The evaluation sites in the expressions of `plan` itself.
231- fn count_own ( plan : & LogicalPlan , acc : & mut Counts ) {
231+ /// Count the evaluation sites in the expressions of `plan` itself into `acc`,
232+ /// and return the plans of the subqueries that those expressions reference.
233+ fn count_own ( plan : & LogicalPlan , acc : & mut Counts ) -> Vec < Arc < LogicalPlan > > {
232234 // `TableScan::filters` is a copy of a predicate that the source evaluates.
233235 // Unless the source reports `Exact` support the `Filter` node stays above
234236 // the scan, so counting both would report every filter pushdown.
235- if matches ! ( plan, LogicalPlan :: TableScan ( _) ) {
236- return ;
237- }
237+ let count_sites = !matches ! ( plan, LogicalPlan :: TableScan ( _) ) ;
238+ let mut subqueries = Vec :: new ( ) ;
238239 let _ = plan. apply_expressions ( |expr| {
239- count_expr ( expr, acc) ;
240- Ok ( TreeNodeRecursion :: Continue )
241- } ) ;
242- }
243-
244- fn count_expr ( expr : & Expr , acc : & mut Counts ) {
245- let _ = expr. apply ( |expr| {
246- if let Expr :: ScalarFunction ( func) = expr
247- && is_site ( func)
248- {
249- * acc. entry ( structural_key ( expr) ) . or_insert ( 0 ) += 1 ;
250- }
251- Ok ( TreeNodeRecursion :: Continue )
252- } ) ;
253- }
254-
255- /// True while a call needs a guard.
256- ///
257- /// A volatile call is a site whatever its arguments are. `random()` takes no
258- /// argument at all, and the constant folder does not fold it.
259- fn is_site ( func : & ScalarFunction ) -> bool {
260- func. func . signature ( ) . volatility == Volatility :: Volatile
261- }
262-
263- /// The key of a call: its text, with every column reduced to its unqualified
264- /// name.
265- ///
266- /// A rule may re-qualify the columns of a call while it moves it, so the
267- /// printed text of a call is not stable enough to key by. `f(a)` and
268- /// `f(test.a)` are the same call.
269- fn structural_key ( expr : & Expr ) -> String {
270- unnormalize_col ( expr. clone ( ) ) . to_string ( )
271- }
272-
273- /// The plans of the subqueries referenced by the expressions of `plan`.
274- fn subquery_plans ( plan : & LogicalPlan ) -> Vec < Arc < LogicalPlan > > {
275- let mut plans = Vec :: new ( ) ;
276- let _ = plan. apply_expressions ( |expr| {
277- let _ = expr. apply ( |expr| {
240+ expr. apply ( |expr| {
278241 match expr {
279242 Expr :: Exists ( Exists { subquery, .. } )
280243 | Expr :: InSubquery ( InSubquery { subquery, .. } )
281244 | Expr :: SetComparison ( SetComparison { subquery, .. } )
282245 | Expr :: ScalarSubquery ( subquery) => {
283- plans. push ( Arc :: clone ( & subquery. subquery ) ) ;
246+ subqueries. push ( Arc :: clone ( & subquery. subquery ) ) ;
247+ }
248+ // A volatile call is a site whatever its arguments are.
249+ // `random()` takes no argument at all, and the constant folder
250+ // does not fold it.
251+ _ if count_sites && expr. is_volatile_node ( ) => {
252+ * acc. entry ( structural_key ( expr) ) . or_insert ( 0 ) += 1 ;
284253 }
285254 _ => { }
286255 }
287256 Ok ( TreeNodeRecursion :: Continue )
288- } ) ;
289- Ok ( TreeNodeRecursion :: Continue )
257+ } )
290258 } ) ;
291- plans
259+ subqueries
260+ }
261+
262+ /// The key of a call: its text, with every column, outer reference columns
263+ /// included, reduced to its unqualified name.
264+ ///
265+ /// A rule may re-qualify the columns of a call while it moves it, so the
266+ /// printed text of a call is not stable enough to key by. `f(a)` and
267+ /// `f(test.a)` are the same call. Decorrelating a subquery turns
268+ /// `f(outer_ref(t.a))` into `f(t.a)`, which is the same call too.
269+ fn structural_key ( expr : & Expr ) -> String {
270+ expr. clone ( )
271+ . transform ( |expr| {
272+ Ok ( match expr {
273+ Expr :: Column ( c) | Expr :: OuterReferenceColumn ( _, c) => {
274+ Transformed :: yes ( Expr :: Column ( Column :: new_unqualified ( c. name ) ) )
275+ }
276+ _ => Transformed :: no ( expr) ,
277+ } )
278+ } )
279+ . data ( )
280+ . expect ( "structural_key is infallible" )
281+ . to_string ( )
292282}
293283
294284#[ cfg( test) ]
@@ -297,9 +287,11 @@ mod tests {
297287 use crate :: test:: udfs:: PlacementTestUDF ;
298288 use crate :: test:: { test_table_scan, test_table_scan_with_name} ;
299289 use crate :: { Optimizer , OptimizerContext } ;
290+ use arrow:: datatypes:: DataType ;
300291 use datafusion_common:: Result ;
301292 use datafusion_expr:: {
302- ExpressionPlacement , JoinType , LogicalPlanBuilder , ScalarUDF , col, exists, lit,
293+ ExpressionPlacement , JoinType , LogicalPlanBuilder , ScalarUDF , Volatility , col,
294+ exists, lit, out_ref_col,
303295 } ;
304296
305297 /// `leaf_udf(arg)`, made volatile. Its placement says that it may move to
@@ -425,6 +417,16 @@ mod tests {
425417 ) ;
426418 }
427419
420+ /// Decorrelating a subquery turns an outer reference into a plain column,
421+ /// which must not change the key of the call that reads it.
422+ #[ test]
423+ fn the_key_ignores_outer_references ( ) {
424+ assert_eq ! (
425+ structural_key( & volatile( out_ref_col( DataType :: UInt32 , "test.a" ) ) ) ,
426+ structural_key( & volatile( col( "a" ) ) )
427+ ) ;
428+ }
429+
428430 /// Inlining the definition of a column into a node below it is the shape
429431 /// the invariant reports. The call is re-qualified on the way, which a key
430432 /// of printed text would not survive.
0 commit comments