1818//!
1919//! # An end of run invariant, by design
2020//!
21- //! [`EvaluationSites::check_no_new_evaluations`] runs one time, at the end of
22- //! [`crate::Optimizer::optimize`], and compares the final plan against the
23- //! plan the optimizer was given. Intermediate plans are not checked, because
21+ //! [`check_no_new_volatile_evaluations`] compares the final plan of an
22+ //! optimizer run against the plan the optimizer was given. Tests call it, see
23+ //! `datafusion/core/tests/sql/volatile_evaluations.rs`. Intermediate plans are
24+ //! not checked, because
2425//! rules cooperate: `simplify_expressions` expands `x BETWEEN a AND b` into
2526//! two references to `x`, and `CommonSubexprEliminate` hoists the duplicate
2627//! back out in the same pass. Only the plan that the user pays for matters,
2728//! so only that plan is checked.
2829//!
29- //! The price is that the invariant does not name the rule at fault. That is a
30- //! debugging aid, not a gate, and the before and after site counts in the
31- //! message are enough to start from.
30+ //! The price is that the check does not name the rule at fault. The before
31+ //! and after site counts in the message are enough to start from.
3232//!
3333//! # What is counted
3434//!
@@ -76,34 +76,6 @@ use datafusion_common::{
7676use datafusion_expr:: expr:: { Exists , InSubquery , SetComparison } ;
7777use datafusion_expr:: { Expr , LogicalPlan } ;
7878
79- /// Report a volatile call that the optimizer run made the plan evaluate at
80- /// more sites than the input plan does.
81- ///
82- /// Off, because one shape still fails on `main`:
83- ///
84- /// - `random() BETWEEN 0.0 AND 1.0` in `expr.slt`, which is
85- /// <https://github.com/apache/datafusion/issues/25457>, a wrong results bug
86- /// with a fix of its own
87- ///
88- /// Turn it on once that is fixed. One other shape is worth recording here,
89- /// because it is what an end of run check buys: `file_row_index()` re-inlined
90- /// by `optimize_projections` in `file_row_index.slt`. A per rule check reports
91- /// it, this check does not, because `CommonSubexprEliminate` hoists the call
92- /// and `optimize_projections` puts it back, which leaves the final plan with
93- /// the site count the input plan has. The classification order that lets a
94- /// volatile call be inlined at all is fixed by
95- /// <https://github.com/apache/datafusion/pull/25456>.
96- const ENFORCE_NO_NEW_VOLATILE_EVALUATIONS : bool = false ;
97-
98- /// True while the invariant is checked.
99- ///
100- /// Debug builds only. The check walks the plan two times per optimizer run,
101- /// which is cheap but not free, and the other plan invariants are checked
102- /// there too.
103- const fn enabled ( ) -> bool {
104- cfg ! ( debug_assertions) && ENFORCE_NO_NEW_VOLATILE_EVALUATIONS
105- }
106-
10779/// The number of evaluation sites of one call, per row lineage, keyed by
10880/// [`structural_key`].
10981type Counts = HashMap < String , usize > ;
@@ -118,16 +90,6 @@ pub(crate) struct EvaluationSites {
11890
11991impl EvaluationSites {
12092 /// Count the evaluation sites of `plan`.
121- ///
122- /// Empty, and free, while the invariant is off.
123- pub ( crate ) fn of ( plan : & LogicalPlan ) -> Self {
124- if !enabled ( ) {
125- return Self :: default ( ) ;
126- }
127- Self :: count ( plan)
128- }
129-
130- /// Count the evaluation sites of `plan`, whatever the switch says.
13193 fn count ( plan : & LogicalPlan ) -> Self {
13294 let mut counts = Counts :: new ( ) ;
13395 count_plan ( plan, & mut counts) ;
@@ -137,15 +99,6 @@ impl EvaluationSites {
13799 /// Fail if `plan` evaluates a volatile call more times than `self` does.
138100 ///
139101 /// `self` must be the count of the plan that the optimizer was given.
140- pub ( crate ) fn check_no_new_evaluations ( & self , plan : & LogicalPlan ) -> Result < ( ) > {
141- if !enabled ( ) {
142- return Ok ( ( ) ) ;
143- }
144- self . check ( plan)
145- }
146-
147- /// The body of [`Self::check_no_new_evaluations`], without the switch, so
148- /// that the machinery has a test while the switch is off.
149102 fn check ( & self , plan : & LogicalPlan ) -> Result < ( ) > {
150103 let after = Self :: count ( plan) ;
151104 let mut offenders: Vec < ( & String , usize , usize ) > = after
@@ -183,6 +136,17 @@ impl EvaluationSites {
183136 }
184137}
185138
139+ /// Fail if `optimized` evaluates a volatile call at more sites than `input`
140+ /// does. `input` is the plan the optimizer was given.
141+ ///
142+ /// See the [module documentation](self) for what is counted.
143+ pub fn check_no_new_volatile_evaluations (
144+ input : & LogicalPlan ,
145+ optimized : & LogicalPlan ,
146+ ) -> Result < ( ) > {
147+ EvaluationSites :: count ( input) . check ( optimized)
148+ }
149+
186150/// `own_sites(plan) + max over inputs of count_plan(input)`.
187151#[ cfg_attr( feature = "recursive_protection" , recursive:: recursive) ]
188152fn count_plan ( plan : & LogicalPlan , acc : & mut Counts ) {
@@ -460,7 +424,7 @@ mod tests {
460424 Ok ( ( ) )
461425 }
462426
463- /// The machinery against a real optimizer run, with the switch off : a
427+ /// The machinery against a real optimizer run: a
464428 /// filter on a volatile column defined by a projection must not be pushed
465429 /// below it, so the optimized plan keeps the site count of the input.
466430 #[ test]
@@ -477,27 +441,4 @@ mod tests {
477441 before. check ( & optimized) ?;
478442 Ok ( ( ) )
479443 }
480-
481- /// <https://github.com/apache/datafusion/issues/25457>, pinned so that the
482- /// test fails, and must be flipped to `?`, once that is fixed.
483- #[ test]
484- fn reports_between_expansion_of_a_kept_volatile_call ( ) -> Result < ( ) > {
485- let udf = ScalarUDF :: new_from_impl (
486- PlacementTestUDF :: new ( )
487- . with_placement ( ExpressionPlacement :: KeepInPlace )
488- . with_volatility ( Volatility :: Volatile ) ,
489- ) ;
490- let input = LogicalPlanBuilder :: from ( test_table_scan ( ) ?)
491- . filter ( udf. call ( vec ! [ col( "a" ) ] ) . between ( lit ( 1u32 ) , lit ( 5u32 ) ) ) ?
492- . build ( ) ?;
493- let before = EvaluationSites :: count ( & input) ;
494- let optimized =
495- Optimizer :: new ( ) . optimize ( input, & OptimizerContext :: new ( ) , |_, _| { } ) ?;
496- let error = before. check ( & optimized) . unwrap_err ( ) . to_string ( ) ;
497- assert ! (
498- error. contains( "is evaluated at 2 sites, against 1" ) ,
499- "{error}"
500- ) ;
501- Ok ( ( ) )
502- }
503444}
0 commit comments