Skip to content

Commit 2713a63

Browse files
adriangbclaude
andcommitted
fix: review findings on the combined leaf pushdown branch
- get_preimage: skip the rewrite when an argument is volatile. floor(random() * 10) = 3 became two random() calls (wrong results, and an internal error with the invariant on). Bug on main, exposed by the switch. Two EXPLAIN cases in floor_preimage.slt. - extract_leaf_expressions (merge of #25456 and #26871): move the respelling into pinned_for_merge so all three pin checks give the inliner the same spelling, and remove a dangling comment. - route_to_inputs: route direct_columns by index, with the name match as a fallback (#25456 with #26871). - reports_renamed_column_merge_as_a_new_site: also pass in builds without debug_assertions. - Doc of ENFORCE_NO_NEW_VOLATILE_EVALUATIONS: name the preimage fix and the known renamed-column false positive. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 0251916 commit 2713a63

4 files changed

Lines changed: 92 additions & 38 deletions

File tree

‎datafusion/optimizer/src/evaluation_sites.rs‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -79,12 +79,17 @@ use datafusion_expr::{Expr, LogicalPlan};
7979
/// Report a volatile call that the optimizer run made the plan evaluate at
8080
/// more sites than the input plan does.
8181
///
82-
/// On. The last shape that failed on `main`, `random() BETWEEN 0.0 AND 1.0`
83-
/// (<https://github.com/apache/datafusion/issues/25457>), is fixed by
84-
/// <https://github.com/apache/datafusion/pull/25476>.
82+
/// On. `random() BETWEEN 0.0 AND 1.0`
83+
/// (<https://github.com/apache/datafusion/issues/25457>) is fixed by
84+
/// <https://github.com/apache/datafusion/pull/25476>, and a preimage rewrite
85+
/// such as `floor(random() * 10) = 3` no longer duplicates its volatile
86+
/// argument.
8587
///
86-
/// One other shape is worth recording here,
87-
/// because it is what an end of run check buys: `file_row_index()` re-inlined
88+
/// Known false positive: a volatile call with column arguments over a
89+
/// renamed column, see `reports_renamed_column_merge_as_a_new_site`.
90+
///
91+
/// One other shape is worth recording here, because it is what an end of run
92+
/// check buys: `file_row_index()` re-inlined
8893
/// by `optimize_projections` in `file_row_index.slt`. A per rule check reports
8994
/// it, this check does not, because `CommonSubexprEliminate` hoists the call
9095
/// and `optimize_projections` puts it back, which leaves the final plan with
@@ -515,10 +520,18 @@ mod tests {
515520
.project(vec![col("a"), col("a").alias("y")])?
516521
.project(vec![udf.call(vec![col("a")]), udf.call(vec![col("y")])])?
517522
.build()?;
518-
let error = Optimizer::new()
519-
.optimize(input, &OptimizerContext::new(), |_, _| {})
520-
.unwrap_err()
521-
.to_string();
523+
let before = EvaluationSites::count(&input);
524+
// The optimizer runs the check in debug builds only, so check the
525+
// optimized plan here when it did not.
526+
let error = match Optimizer::new().optimize(
527+
input,
528+
&OptimizerContext::new(),
529+
|_, _| {},
530+
) {
531+
Ok(optimized) => before.check(&optimized).unwrap_err(),
532+
Err(error) => error,
533+
}
534+
.to_string();
522535
assert!(
523536
error.contains("_udf(a)` is evaluated at 2 sites, against 1"),
524537
"{placement:?}: {error:?}"

‎datafusion/optimizer/src/extract_leaf_expressions.rs‎

Lines changed: 49 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -180,13 +180,27 @@ fn resolve_against(schema: &DFSchema, col: &Column) -> Option<Column> {
180180
///
181181
/// `inlined` are the expressions that the merge resolves, and
182182
/// `direct_columns` the columns that the nodes above reference directly.
183+
///
184+
/// The inliner and the merge must agree on which output column a reference
185+
/// names, so the references are checked in the canonical spelling of
186+
/// `schema`, the output schema of the projection (see [`column_index`]).
183187
fn pinned_for_merge<'a>(
184188
inliner: &ProjectionInliner<'_>,
189+
schema: &DFSchema,
185190
inlined: impl IntoIterator<Item = &'a Expr>,
186191
direct_columns: &IndexSet<Column>,
187-
) -> PinnedDefinitions {
188-
let direct: Vec<Expr> = direct_columns.iter().cloned().map(Expr::Column).collect();
189-
inliner.pinned_with_unknown_consumers(inlined, &direct, DuplicationCost::Free)
192+
) -> Result<PinnedDefinitions> {
193+
let inlined = inlined
194+
.into_iter()
195+
.map(|expr| respell_cols(expr.clone(), schema, schema))
196+
.collect::<Result<Vec<_>>>()?;
197+
let direct: Vec<Expr> = direct_columns
198+
.iter()
199+
.map(|col| {
200+
Expr::Column(resolve_against(schema, col).unwrap_or_else(|| col.clone()))
201+
})
202+
.collect();
203+
Ok(inliner.pinned_with_unknown_consumers(&inlined, &direct, DuplicationCost::Free))
190204
}
191205

192206
/// Extracts `MoveTowardsLeafNodes` sub-expressions from non-projection nodes
@@ -359,7 +373,7 @@ fn extract_from_plan(
359373
LeafExpressionExtractor::new(schema.as_ref(), alias_generator)
360374
.with_input_projection(projection.as_ref(), &node_exprs, &IndexSet::new())
361375
})
362-
.collect();
376+
.collect::<Result<_>>()?;
363377

364378
// Build per-input column sets for routing expressions to the correct input
365379
let input_column_sets: Vec<HashSet<ColumnReference>> = input_schemas
@@ -691,13 +705,18 @@ impl<'a> LeafExpressionExtractor<'a> {
691705
projection: Option<&'a Projection>,
692706
consumers: impl IntoIterator<Item = &'e Expr>,
693707
direct_columns: &IndexSet<Column>,
694-
) -> Self {
708+
) -> Result<Self> {
695709
if let Some(projection) = projection {
696710
let inliner = ProjectionInliner::new(projection);
697-
self.pinned = pinned_for_merge(&inliner, consumers, direct_columns);
711+
self.pinned = pinned_for_merge(
712+
&inliner,
713+
&projection.schema,
714+
consumers,
715+
direct_columns,
716+
)?;
698717
self.input_inliner = Some(inliner);
699718
}
700-
self
719+
Ok(self)
701720
}
702721

703722
/// Returns `true` if `expr` can be extracted into the input. See
@@ -788,22 +807,11 @@ fn build_extraction_projection_impl(
788807
target_schema: &DFSchema,
789808
) -> Result<Option<Projection>> {
790809
if let LogicalPlan::Projection(existing) = target.as_ref() {
791-
// Resolves column references through the existing projection
792-
// The inliner and the merge below must agree on which output column a
793-
// reference names, so check the references in the canonical spelling
794-
// of the output schema (see [`column_index`]).
795810
let inliner = ProjectionInliner::new(existing);
796-
let extracted = extracted_exprs
797-
.iter()
798-
.map(|(e, _)| respell_cols(e.clone(), &existing.schema, &existing.schema))
799-
.collect::<Result<Vec<_>>>()?;
800-
let direct_columns: IndexSet<Column> = direct_columns
801-
.iter()
802-
.map(|col| {
803-
resolve_against(&existing.schema, col).unwrap_or_else(|| col.clone())
804-
})
805-
.collect();
806-
if !pinned_for_merge(&inliner, &extracted, &direct_columns).is_empty() {
811+
let extracted = extracted_exprs.iter().map(|(e, _)| e);
812+
if !pinned_for_merge(&inliner, &existing.schema, extracted, direct_columns)?
813+
.is_empty()
814+
{
807815
return Ok(None);
808816
}
809817

@@ -1062,7 +1070,7 @@ fn split_and_push_projection(
10621070
input_projection,
10631071
consumers,
10641072
known_direct_columns.unwrap_or(&no_direct_columns),
1065-
);
1073+
)?;
10661074
let mut extractors = vec![extractor];
10671075
let input_column_sets = vec![schema_columns(input_schema.as_ref())];
10681076

@@ -1155,9 +1163,10 @@ fn split_and_push_projection(
11551163
if let Some(inliner) = &extractor.input_inliner
11561164
&& !pinned_for_merge(
11571165
inliner,
1166+
input_schema.as_ref(),
11581167
extraction_pairs.iter().map(|(expr, _)| expr),
11591168
&direct_columns,
1160-
)
1169+
)?
11611170
.is_empty()
11621171
{
11631172
return Ok(None);
@@ -1407,11 +1416,16 @@ fn route_to_inputs(
14071416
None => return Ok(None), // Ambiguous column — bail out
14081417
}
14091418
}
1410-
// A direct reference counts for every input that can own it.
1419+
// A direct reference counts for every input that can own it: by
1420+
// name, or resolved like the merge resolves it (see [`column_index`]).
14111421
for col in &target.direct_columns {
14121422
let col_ref = ColumnReference::new(col.relation.as_ref(), col.name());
1413-
for (idx, cols) in input_column_sets.iter().enumerate() {
1414-
if cols.contains(&col_ref) {
1423+
for (idx, (cols, schema)) in
1424+
input_column_sets.iter().zip(input_schemas).enumerate()
1425+
{
1426+
if let Some(resolved) = resolve_against(schema, col) {
1427+
per_input[idx].direct_columns.insert(resolved);
1428+
} else if cols.contains(&col_ref) {
14151429
per_input[idx].direct_columns.insert(col.clone());
14161430
}
14171431
}
@@ -4119,8 +4133,14 @@ mod tests {
41194133
)?;
41204134
let inliner = ProjectionInliner::new(&child);
41214135
let blocks = |pairs: &[(Expr, String)], columns: &IndexSet<Column>| {
4122-
!pinned_for_merge(&inliner, pairs.iter().map(|(expr, _)| expr), columns)
4123-
.is_empty()
4136+
!pinned_for_merge(
4137+
&inliner,
4138+
&child.schema,
4139+
pairs.iter().map(|(expr, _)| expr),
4140+
columns,
4141+
)
4142+
.unwrap()
4143+
.is_empty()
41244144
};
41254145
let no_columns = IndexSet::new();
41264146

‎datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2141,6 +2141,12 @@ fn get_preimage(
21412141
if func.signature().volatility != Volatility::Immutable {
21422142
return Ok(PreimageResult::None);
21432143
}
2144+
// The rewrite can reference the argument more than one time, for example
2145+
// `floor(x) = 3` becomes `x >= 3 AND x < 4`. A volatile argument such as
2146+
// `random() * 10` must not be evaluated twice.
2147+
if args.iter().any(Expr::is_volatile) {
2148+
return Ok(PreimageResult::None);
2149+
}
21442150
func.preimage(args, right_expr, info)
21452151
}
21462152

‎datafusion/sqllogictest/test_files/floor_preimage.slt‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,21 @@ logical_plan
147147
01)Filter: test_data.float_val >= Float64(5) AND test_data.float_val < Float64(6)
148148
02)--TableScan: test_data projection=[id, float_val, int_val, decimal_val]
149149

150+
# A volatile argument is not rewritten: the rewrite would evaluate it twice
151+
query TT
152+
EXPLAIN SELECT id FROM test_data WHERE floor(random() * 10) = arrow_cast(3, 'Float64');
153+
----
154+
logical_plan
155+
01)Filter: floor(random() * Float64(10)) = Float64(3)
156+
02)--TableScan: test_data projection=[id]
157+
158+
query TT
159+
EXPLAIN SELECT id FROM test_data WHERE floor(random() * 10) IN (arrow_cast(3, 'Float64'), arrow_cast(4, 'Float64'));
160+
----
161+
logical_plan
162+
01)Filter: floor(random() * Float64(10)) IN ([Float64(3), Float64(4)])
163+
02)--TableScan: test_data projection=[id]
164+
150165
# 5. IS NOT DISTINCT FROM - adds IS NOT NULL
151166
query TT
152167
EXPLAIN SELECT * FROM test_data WHERE floor(float_val) IS NOT DISTINCT FROM arrow_cast(5, 'Float64');

0 commit comments

Comments
 (0)