Skip to content

Commit 7ffdb3e

Browse files
committed
fix(optimizer): count bare references when guarding KeepInPlace merges
Correctness fixes to the extract-leaf merge guard (folds into apache#23691 on squash-merge): - count pass-through / pre-existing-alias inputs (`columns_needed`), not just `extraction_pairs`, when deciding whether merging would duplicate a compute-once `KeepInPlace` expression. For `SELECT f(c)['a'], f(c)` the bare `f(c)` lives in `columns_needed`, so the pairs-only count saw one use, merged, and re-inlined the struct UDF into a second evaluation (apache#23655) - restrict the "kept" set to `ExpressionPlacement::KeepInPlace` instead of `!should_push_to_leaves()`, which also matched `Literal`; literals are cheap to duplicate and should never block a merge Test: regression for the bare-reference shape, asserting the UDF is hoisted once into `__common_expr_1` with no re-inlined `get_field_like(f(c))`.
1 parent 5e55eac commit 7ffdb3e

1 file changed

Lines changed: 72 additions & 12 deletions

File tree

‎datafusion/optimizer/src/extract_leaf_expressions.rs‎

Lines changed: 72 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -757,37 +757,43 @@ fn try_push_input(
757757
split_and_push_projection(proj, alias_generator)
758758
}
759759

760-
/// Returns `true` if merging the extracted `pairs` into `child` would inline a
761-
/// `KeepInPlace` definition into more than one extracted expression — e.g. a
762-
/// struct-returning UDF `f(c)` behind both `f(c)['a']` and `f(c)['b']`.
760+
/// Returns `true` if merging the extracted `pairs` (and the pass-through
761+
/// `columns_needed`) into `child` would inline a `KeepInPlace` definition into
762+
/// more than one reference site — e.g. a struct-returning UDF `f(c)` behind both
763+
/// `f(c)['a']` and `f(c)['b']`, or behind `f(c)['a']` and a bare `f(c)`.
763764
///
764765
/// `CommonSubexprEliminate` hoists such an expression into a shared column so it
765766
/// runs once; [`build_extraction_projection_impl`] then resolves that column
766767
/// back to its definition and re-duplicates it. Mirrors the
767768
/// `optimize_projections` merge guard (#8296): a compute-once expression used
768769
/// more than once stays in place.
769-
fn merge_would_duplicate_kept_expr(pairs: &[(Expr, String)], child: &Projection) -> bool {
770+
fn merge_would_duplicate_kept_expr(
771+
pairs: &[(Expr, String)],
772+
columns_needed: &IndexSet<Column>,
773+
child: &Projection,
774+
) -> bool {
770775
// Columns whose `KeepInPlace` definition must stay put; inlining one would
771-
// duplicate a compute-once expression. Plain and pushable definitions are
772-
// cheap to duplicate.
776+
// duplicate a compute-once expression. Plain columns, pushable exprs, and
777+
// literals are all cheap to duplicate, so only `KeepInPlace` counts.
773778
let kept_columns: std::collections::HashSet<String> = child
774779
.schema
775780
.iter()
776781
.zip(child.expr.iter())
777782
.filter_map(|((qualifier, field), expr)| {
778-
if expr.placement().should_push_to_leaves() {
779-
None
780-
} else {
783+
if matches!(expr.placement(), ExpressionPlacement::KeepInPlace) {
781784
Some(Column::from((qualifier, field)).flat_name())
785+
} else {
786+
None
782787
}
783788
})
784789
.collect();
785790
if kept_columns.is_empty() {
786791
return false;
787792
}
788793

789-
// A pair that references the same kept column twice still inlines one copy,
790-
// so count each pair at most once per column.
794+
// Count every site that references a kept column. A pair that references the
795+
// same kept column twice still inlines one copy, so count each pair at most
796+
// once per column.
791797
let mut usage: HashMap<String, usize> = HashMap::new();
792798
for (expr, _alias) in pairs {
793799
let mut seen: std::collections::HashSet<String> =
@@ -799,6 +805,15 @@ fn merge_would_duplicate_kept_expr(pairs: &[(Expr, String)], child: &Projection)
799805
}
800806
}
801807
}
808+
// Pass-through / pre-existing-alias inputs are additional reference sites:
809+
// a bare `f(c)` alongside `f(c)['a']` also pins the compute-once column, so
810+
// counting only `pairs` under-counted and re-duplicated it (#23655).
811+
for col in columns_needed {
812+
let name = col.flat_name();
813+
if kept_columns.contains(&name) {
814+
*usage.entry(name).or_insert(0) += 1;
815+
}
816+
}
802817
usage.values().any(|&count| count > 1)
803818
}
804819

@@ -947,7 +962,7 @@ fn split_and_push_projection(
947962
// Merging here would re-inline an expression `CommonSubexprEliminate` hoisted
948963
// to run once, undoing it (issue #23655); leave the projection in place.
949964
if let LogicalPlan::Projection(child) = input.as_ref()
950-
&& merge_would_duplicate_kept_expr(&extraction_pairs, child)
965+
&& merge_would_duplicate_kept_expr(&extraction_pairs, columns_needed, child)
951966
{
952967
return Ok(None);
953968
}
@@ -3270,4 +3285,49 @@ mod tests {
32703285

32713286
Ok(())
32723287
}
3288+
3289+
#[test]
3290+
fn test_struct_returning_udf_evaluated_once_with_bare_reference() -> Result<()> {
3291+
// Regression for the `SELECT f(c)['a'], f(c)` shape: the struct UDF is
3292+
// referenced once through a field access and once as a bare value. The
3293+
// bare reference is tracked in `columns_needed`, not `extraction_pairs`,
3294+
// so a guard that counted only pairs would re-inline `f(c)` and evaluate
3295+
// it twice (#23655).
3296+
let udf = ScalarUDF::new_from_impl(
3297+
PlacementTestUDF::new().with_placement(ExpressionPlacement::KeepInPlace),
3298+
);
3299+
let f_c = udf.call(vec![col("c")]);
3300+
let plan = LogicalPlanBuilder::from(test_table_scan()?)
3301+
.project(vec![get_field_like(f_c.clone(), "a"), f_c])?
3302+
.build()?;
3303+
3304+
let ctx = OptimizerContext::new();
3305+
let optimizer = Optimizer::with_rules(vec![
3306+
Arc::new(CommonSubexprEliminate::new()),
3307+
Arc::new(ExtractLeafExpressions::new()),
3308+
Arc::new(PushDownLeafProjections::new()),
3309+
Arc::new(OptimizeProjections::new()),
3310+
]);
3311+
let optimized = optimizer.optimize(plan, &ctx, |_, _| {})?;
3312+
3313+
// The field access must read the UDF back through `__common_expr_1`, not
3314+
// wrap a fresh `keep_in_place_udf(...)` call. Asserted explicitly so a
3315+
// careless snapshot re-accept can't quietly restore the double evaluation.
3316+
let formatted = format!("{optimized}");
3317+
assert_eq!(
3318+
formatted
3319+
.matches("get_field_like(keep_in_place_udf")
3320+
.count(),
3321+
0,
3322+
"struct-returning UDF must not be re-evaluated inside a field access; got:\n{formatted}"
3323+
);
3324+
3325+
insta::assert_snapshot!(formatted, @r#"
3326+
Projection: get_field_like(__common_expr_1 AS keep_in_place_udf(test.c), Utf8("a")), __common_expr_1 AS keep_in_place_udf(test.c)
3327+
Projection: keep_in_place_udf(test.c) AS __common_expr_1
3328+
TableScan: test projection=[c]
3329+
"#);
3330+
3331+
Ok(())
3332+
}
32733333
}

0 commit comments

Comments
 (0)