Skip to content

Commit 77cdc9d

Browse files
goutamadwantkosiew
authored andcommitted
perf: reuse column normalization context across expressions
1 parent d44a009 commit 77cdc9d

2 files changed

Lines changed: 146 additions & 32 deletions

File tree

‎datafusion/expr/src/expr_rewriter/mod.rs‎

Lines changed: 140 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ use std::sync::Arc;
2424

2525
use crate::expr::{Alias, Sort, Unnest};
2626
use crate::logical_plan::Projection;
27-
use crate::{Expr, ExprSchemable, LogicalPlan, LogicalPlanBuilder};
27+
use crate::{Expr, ExprSchemable, LogicalPlan};
2828

2929
use datafusion_common::TableReference;
3030
use datafusion_common::config::ConfigOptions;
@@ -64,20 +64,54 @@ pub trait FunctionRewrite: Debug {
6464
) -> Result<Transformed<Expr>>;
6565
}
6666

67-
/// Recursively call `LogicalPlanBuilder::normalize` on all [`Column`] expressions
68-
/// in the `expr` expression tree.
67+
/// Recursively normalize all [`Column`] expressions in the `expr` expression tree.
6968
pub fn normalize_col(expr: Expr, plan: &LogicalPlan) -> Result<Expr> {
70-
expr.transform(|expr| {
71-
Ok({
72-
if let Expr::Column(c) = expr {
73-
let col = LogicalPlanBuilder::normalize(plan, c)?;
74-
Transformed::yes(Expr::Column(col))
75-
} else {
76-
Transformed::no(expr)
77-
}
69+
ColumnNormalizer::new(plan).normalize(expr)
70+
}
71+
72+
/// Reuses the plan's normalization context across expressions. Initialize it
73+
/// lazily so literals and already-qualified columns need no plan traversal.
74+
pub(crate) struct ColumnNormalizer<'a> {
75+
plan: &'a LogicalPlan,
76+
context: Option<(Vec<&'a DFSchema>, Vec<HashSet<Column>>)>,
77+
}
78+
79+
impl<'a> ColumnNormalizer<'a> {
80+
pub(crate) fn new(plan: &'a LogicalPlan) -> Self {
81+
Self {
82+
plan,
83+
context: None,
84+
}
85+
}
86+
87+
#[inline]
88+
pub(crate) fn normalize_column(&mut self, column: Column) -> Result<Column> {
89+
if column.relation.is_some() {
90+
return Ok(column);
91+
}
92+
93+
let (fallback_schemas, using_columns) = match &mut self.context {
94+
Some(context) => context,
95+
context @ None => context.insert((
96+
self.plan.fallback_normalize_schemas(),
97+
self.plan.using_columns()?,
98+
)),
99+
};
100+
column.normalize_with_schemas_and_ambiguity_check(
101+
&[&[self.plan.schema()], fallback_schemas],
102+
using_columns,
103+
)
104+
}
105+
106+
pub(crate) fn normalize(&mut self, expr: Expr) -> Result<Expr> {
107+
expr.transform(|expr| match expr {
108+
Expr::Column(column) => self
109+
.normalize_column(column)
110+
.map(|column| Transformed::yes(Expr::Column(column))),
111+
_ => Ok(Transformed::no(expr)),
78112
})
79-
})
80-
.data()
113+
.data()
114+
}
81115
}
82116

83117
/// See [`Column::normalize_with_schemas_and_ambiguity_check`] for usage
@@ -118,21 +152,24 @@ pub fn normalize_cols(
118152
exprs: impl IntoIterator<Item = impl Into<Expr>>,
119153
plan: &LogicalPlan,
120154
) -> Result<Vec<Expr>> {
155+
let mut normalizer = ColumnNormalizer::new(plan);
121156
exprs
122157
.into_iter()
123-
.map(|e| normalize_col(e.into(), plan))
158+
.map(|e| normalizer.normalize(e.into()))
124159
.collect()
125160
}
126161

127162
pub fn normalize_sorts(
128163
sorts: impl IntoIterator<Item = impl Into<Sort>>,
129164
plan: &LogicalPlan,
130165
) -> Result<Vec<Sort>> {
166+
let mut normalizer = ColumnNormalizer::new(plan);
131167
sorts
132168
.into_iter()
133169
.map(|e| {
134170
let sort = e.into();
135-
normalize_col(sort.expr, plan)
171+
normalizer
172+
.normalize(sort.expr)
136173
.map(|expr| Sort::new(expr, sort.asc, sort.nulls_first))
137174
})
138175
.collect()
@@ -382,7 +419,7 @@ mod test {
382419

383420
use super::*;
384421
use crate::literal::lit_with_metadata;
385-
use crate::{Cast, col, lit};
422+
use crate::{Cast, LogicalPlanBuilder, col, lit};
386423
use arrow::datatypes::{DataType, Field, Schema};
387424
use datafusion_common::ScalarValue;
388425
use datafusion_common::tree_node::TreeNodeRewriter;
@@ -491,6 +528,93 @@ mod test {
491528
assert_eq!(error, expected);
492529
}
493530

531+
#[test]
532+
fn normalize_batch_schema_precedence() -> Result<()> {
533+
let schema = Schema::new(vec![
534+
Field::new("a", DataType::Int32, false),
535+
Field::new("b", DataType::Int32, false),
536+
]);
537+
let plan = crate::logical_plan::table_scan(Some("t"), &schema, None)?
538+
.project([col("t.a").alias("b")])?
539+
.build()?;
540+
let exprs = vec![col("b") + col("a"), col("other.missing"), lit(1)];
541+
let expected = vec![col("b") + col("t.a"), col("other.missing"), lit(1)];
542+
assert_eq!(super::normalize_cols(exprs.clone(), &plan)?, expected);
543+
assert_eq!(normalize_col(exprs[0].clone(), &plan)?, expected[0]);
544+
let sorts = exprs.into_iter().map(|e| e.sort(false, true));
545+
assert_eq!(
546+
normalize_sorts(sorts, &plan)?,
547+
expected
548+
.into_iter()
549+
.map(|e| e.sort(false, true))
550+
.collect::<Vec<_>>()
551+
);
552+
Ok(())
553+
}
554+
555+
#[test]
556+
fn normalize_batch_using_join() -> Result<()> {
557+
let schema = Schema::new(vec![
558+
Field::new("a", DataType::Int32, false),
559+
Field::new("b", DataType::Int32, false),
560+
]);
561+
let right =
562+
crate::logical_plan::table_scan(Some("right"), &schema, None)?.build()?;
563+
let plan = crate::logical_plan::table_scan(Some("left"), &schema, None)?
564+
.join_using(right, crate::JoinType::Inner, vec![Column::from_name("a")])?
565+
.build()?;
566+
assert_eq!(
567+
super::normalize_cols([col("a"), col("a") + col("right.a")], &plan)?,
568+
vec![col("left.a"), col("left.a") + col("right.a")]
569+
);
570+
let projected = LogicalPlanBuilder::from(plan.clone())
571+
.project([col("a").alias("key"), col("left.b").alias("value")])?
572+
.build()?;
573+
assert_eq!(
574+
projected.expressions(),
575+
vec![col("left.a").alias("key"), col("left.b").alias("value")]
576+
);
577+
let err = super::normalize_cols([col("a"), col("b"), col("missing")], &plan)
578+
.unwrap_err();
579+
assert!(
580+
err.strip_backtrace()
581+
.contains("Ambiguous reference to unqualified field b")
582+
);
583+
Ok(())
584+
}
585+
586+
#[test]
587+
fn normalize_batch_skips_unused_plan_context() -> Result<()> {
588+
let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
589+
let right =
590+
crate::logical_plan::table_scan(Some("right"), &schema, None)?.build()?;
591+
let mut plan = crate::logical_plan::table_scan(Some("left"), &schema, None)?
592+
.join_using(right, crate::JoinType::Inner, vec![Column::from_name("a")])?
593+
.build()?;
594+
// Invalid USING keys must only be inspected when an unqualified column
595+
// needs normalization, just as with a single expression.
596+
let LogicalPlan::Join(join) = &mut plan else {
597+
unreachable!()
598+
};
599+
join.on[0].0 = lit(1);
600+
assert!(plan.using_columns().is_err());
601+
assert!(super::normalize_cols(Vec::<Expr>::new(), &plan)?.is_empty());
602+
let exprs = vec![lit(1), col("left.a"), col("other.missing")];
603+
assert_eq!(super::normalize_cols(exprs.clone(), &plan)?, exprs);
604+
assert_eq!(normalize_col(col("left.a"), &plan)?, col("left.a"));
605+
assert_eq!(
606+
normalize_sorts([col("left.a").sort(true, false)], &plan)?,
607+
vec![col("left.a").sort(true, false)]
608+
);
609+
assert!(super::normalize_cols([col("left.a"), col("a")], &plan).is_err());
610+
let projected = LogicalPlanBuilder::from(plan.clone())
611+
.project([col("left.a")])?
612+
.build()?;
613+
assert_eq!(projected.expressions(), vec![col("left.a")]);
614+
assert!(LogicalPlanBuilder::from(plan).project([col("a")]).is_err());
615+
Ok(())
616+
}
617+
494618
#[test]
495619
fn unnormalize_cols() {
496620
let expr = col("tableA.a") + col("tableB.b");

‎datafusion/expr/src/logical_plan/builder.rs‎

Lines changed: 6 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ use std::sync::Arc;
2626
use crate::dml::CopyTo;
2727
use crate::expr::{Alias, PlannedReplaceSelectItem, Sort as SortExpr};
2828
use crate::expr_rewriter::{
29-
coerce_plan_expr_for_schema, normalize_col,
29+
ColumnNormalizer, coerce_plan_expr_for_schema, normalize_col,
3030
normalize_col_with_schemas_and_ambiguity_check, normalize_cols, normalize_sorts,
3131
rewrite_sort_cols_by_aggs,
3232
};
@@ -1070,18 +1070,7 @@ impl LogicalPlanBuilder {
10701070
}
10711071

10721072
pub(crate) fn normalize(plan: &LogicalPlan, column: Column) -> Result<Column> {
1073-
if column.relation.is_some() {
1074-
// column is already normalized
1075-
return Ok(column);
1076-
}
1077-
1078-
let schema = plan.schema();
1079-
let fallback_schemas = plan.fallback_normalize_schemas();
1080-
let using_columns = plan.using_columns()?;
1081-
column.normalize_with_schemas_and_ambiguity_check(
1082-
&[&[schema], &fallback_schemas],
1083-
&using_columns,
1084-
)
1073+
ColumnNormalizer::new(plan).normalize_column(column)
10851074
}
10861075

10871076
/// Apply a join with on constraint and specified null equality.
@@ -2033,6 +2022,7 @@ fn project_with_validation(
20332022
) -> Result<LogicalPlan> {
20342023
let mut projected_expr = vec![];
20352024
let mut has_wildcard = false;
2025+
let mut normalizer = ColumnNormalizer::new(&plan);
20362026
for (e, validate) in expr {
20372027
let e = e.into();
20382028
match e {
@@ -2051,7 +2041,7 @@ fn project_with_validation(
20512041
for e in expanded {
20522042
if validate {
20532043
projected_expr
2054-
.push(columnize_expr(normalize_col(e, &plan)?, &plan)?)
2044+
.push(columnize_expr(normalizer.normalize(e)?, &plan)?)
20552045
} else {
20562046
projected_expr.push(e)
20572047
}
@@ -2073,15 +2063,15 @@ fn project_with_validation(
20732063
for e in expanded {
20742064
if validate {
20752065
projected_expr
2076-
.push(columnize_expr(normalize_col(e, &plan)?, &plan)?)
2066+
.push(columnize_expr(normalizer.normalize(e)?, &plan)?)
20772067
} else {
20782068
projected_expr.push(e)
20792069
}
20802070
}
20812071
}
20822072
SelectExpr::Expression(e) => {
20832073
if validate {
2084-
projected_expr.push(columnize_expr(normalize_col(e, &plan)?, &plan)?)
2074+
projected_expr.push(columnize_expr(normalizer.normalize(e)?, &plan)?)
20852075
} else {
20862076
projected_expr.push(e)
20872077
}

0 commit comments

Comments
 (0)