@@ -24,7 +24,7 @@ use std::sync::Arc;
2424
2525use crate :: expr:: { Alias , Sort , Unnest } ;
2626use crate :: logical_plan:: Projection ;
27- use crate :: { Expr , ExprSchemable , LogicalPlan , LogicalPlanBuilder } ;
27+ use crate :: { Expr , ExprSchemable , LogicalPlan } ;
2828
2929use datafusion_common:: TableReference ;
3030use datafusion_common:: config:: ConfigOptions ;
@@ -64,20 +64,54 @@ pub trait FunctionRewrite: Debug {
6464 ) -> Result < Transformed < Expr > > ;
6565}
6666
67- /// Recursively call `LogicalPlanBuilder::normalize` on all [`Column`] expressions
68- /// in the `expr` expression tree.
67+ /// Recursively normalize all [`Column`] expressions in the `expr` expression tree.
6968pub fn normalize_col ( expr : Expr , plan : & LogicalPlan ) -> Result < Expr > {
70- expr. transform ( |expr| {
71- Ok ( {
72- if let Expr :: Column ( c) = expr {
73- let col = LogicalPlanBuilder :: normalize ( plan, c) ?;
74- Transformed :: yes ( Expr :: Column ( col) )
75- } else {
76- Transformed :: no ( expr)
77- }
69+ ColumnNormalizer :: new ( plan) . normalize ( expr)
70+ }
71+
72+ /// Reuses the plan's normalization context across expressions. Initialize it
73+ /// lazily so literals and already-qualified columns need no plan traversal.
74+ pub ( crate ) struct ColumnNormalizer < ' a > {
75+ plan : & ' a LogicalPlan ,
76+ context : Option < ( Vec < & ' a DFSchema > , Vec < HashSet < Column > > ) > ,
77+ }
78+
79+ impl < ' a > ColumnNormalizer < ' a > {
80+ pub ( crate ) fn new ( plan : & ' a LogicalPlan ) -> Self {
81+ Self {
82+ plan,
83+ context : None ,
84+ }
85+ }
86+
87+ #[ inline]
88+ pub ( crate ) fn normalize_column ( & mut self , column : Column ) -> Result < Column > {
89+ if column. relation . is_some ( ) {
90+ return Ok ( column) ;
91+ }
92+
93+ let ( fallback_schemas, using_columns) = match & mut self . context {
94+ Some ( context) => context,
95+ context @ None => context. insert ( (
96+ self . plan . fallback_normalize_schemas ( ) ,
97+ self . plan . using_columns ( ) ?,
98+ ) ) ,
99+ } ;
100+ column. normalize_with_schemas_and_ambiguity_check (
101+ & [ & [ self . plan . schema ( ) ] , fallback_schemas] ,
102+ using_columns,
103+ )
104+ }
105+
106+ pub ( crate ) fn normalize ( & mut self , expr : Expr ) -> Result < Expr > {
107+ expr. transform ( |expr| match expr {
108+ Expr :: Column ( column) => self
109+ . normalize_column ( column)
110+ . map ( |column| Transformed :: yes ( Expr :: Column ( column) ) ) ,
111+ _ => Ok ( Transformed :: no ( expr) ) ,
78112 } )
79- } )
80- . data ( )
113+ . data ( )
114+ }
81115}
82116
83117/// See [`Column::normalize_with_schemas_and_ambiguity_check`] for usage
@@ -118,21 +152,24 @@ pub fn normalize_cols(
118152 exprs : impl IntoIterator < Item = impl Into < Expr > > ,
119153 plan : & LogicalPlan ,
120154) -> Result < Vec < Expr > > {
155+ let mut normalizer = ColumnNormalizer :: new ( plan) ;
121156 exprs
122157 . into_iter ( )
123- . map ( |e| normalize_col ( e. into ( ) , plan ) )
158+ . map ( |e| normalizer . normalize ( e. into ( ) ) )
124159 . collect ( )
125160}
126161
127162pub fn normalize_sorts (
128163 sorts : impl IntoIterator < Item = impl Into < Sort > > ,
129164 plan : & LogicalPlan ,
130165) -> Result < Vec < Sort > > {
166+ let mut normalizer = ColumnNormalizer :: new ( plan) ;
131167 sorts
132168 . into_iter ( )
133169 . map ( |e| {
134170 let sort = e. into ( ) ;
135- normalize_col ( sort. expr , plan)
171+ normalizer
172+ . normalize ( sort. expr )
136173 . map ( |expr| Sort :: new ( expr, sort. asc , sort. nulls_first ) )
137174 } )
138175 . collect ( )
@@ -382,7 +419,7 @@ mod test {
382419
383420 use super :: * ;
384421 use crate :: literal:: lit_with_metadata;
385- use crate :: { Cast , col, lit} ;
422+ use crate :: { Cast , LogicalPlanBuilder , col, lit} ;
386423 use arrow:: datatypes:: { DataType , Field , Schema } ;
387424 use datafusion_common:: ScalarValue ;
388425 use datafusion_common:: tree_node:: TreeNodeRewriter ;
@@ -491,6 +528,93 @@ mod test {
491528 assert_eq ! ( error, expected) ;
492529 }
493530
531+ #[ test]
532+ fn normalize_batch_schema_precedence ( ) -> Result < ( ) > {
533+ let schema = Schema :: new ( vec ! [
534+ Field :: new( "a" , DataType :: Int32 , false ) ,
535+ Field :: new( "b" , DataType :: Int32 , false ) ,
536+ ] ) ;
537+ let plan = crate :: logical_plan:: table_scan ( Some ( "t" ) , & schema, None ) ?
538+ . project ( [ col ( "t.a" ) . alias ( "b" ) ] ) ?
539+ . build ( ) ?;
540+ let exprs = vec ! [ col( "b" ) + col( "a" ) , col( "other.missing" ) , lit( 1 ) ] ;
541+ let expected = vec ! [ col( "b" ) + col( "t.a" ) , col( "other.missing" ) , lit( 1 ) ] ;
542+ assert_eq ! ( super :: normalize_cols( exprs. clone( ) , & plan) ?, expected) ;
543+ assert_eq ! ( normalize_col( exprs[ 0 ] . clone( ) , & plan) ?, expected[ 0 ] ) ;
544+ let sorts = exprs. into_iter ( ) . map ( |e| e. sort ( false , true ) ) ;
545+ assert_eq ! (
546+ normalize_sorts( sorts, & plan) ?,
547+ expected
548+ . into_iter( )
549+ . map( |e| e. sort( false , true ) )
550+ . collect:: <Vec <_>>( )
551+ ) ;
552+ Ok ( ( ) )
553+ }
554+
555+ #[ test]
556+ fn normalize_batch_using_join ( ) -> Result < ( ) > {
557+ let schema = Schema :: new ( vec ! [
558+ Field :: new( "a" , DataType :: Int32 , false ) ,
559+ Field :: new( "b" , DataType :: Int32 , false ) ,
560+ ] ) ;
561+ let right =
562+ crate :: logical_plan:: table_scan ( Some ( "right" ) , & schema, None ) ?. build ( ) ?;
563+ let plan = crate :: logical_plan:: table_scan ( Some ( "left" ) , & schema, None ) ?
564+ . join_using ( right, crate :: JoinType :: Inner , vec ! [ Column :: from_name( "a" ) ] ) ?
565+ . build ( ) ?;
566+ assert_eq ! (
567+ super :: normalize_cols( [ col( "a" ) , col( "a" ) + col( "right.a" ) ] , & plan) ?,
568+ vec![ col( "left.a" ) , col( "left.a" ) + col( "right.a" ) ]
569+ ) ;
570+ let projected = LogicalPlanBuilder :: from ( plan. clone ( ) )
571+ . project ( [ col ( "a" ) . alias ( "key" ) , col ( "left.b" ) . alias ( "value" ) ] ) ?
572+ . build ( ) ?;
573+ assert_eq ! (
574+ projected. expressions( ) ,
575+ vec![ col( "left.a" ) . alias( "key" ) , col( "left.b" ) . alias( "value" ) ]
576+ ) ;
577+ let err = super :: normalize_cols ( [ col ( "a" ) , col ( "b" ) , col ( "missing" ) ] , & plan)
578+ . unwrap_err ( ) ;
579+ assert ! (
580+ err. strip_backtrace( )
581+ . contains( "Ambiguous reference to unqualified field b" )
582+ ) ;
583+ Ok ( ( ) )
584+ }
585+
586+ #[ test]
587+ fn normalize_batch_skips_unused_plan_context ( ) -> Result < ( ) > {
588+ let schema = Schema :: new ( vec ! [ Field :: new( "a" , DataType :: Int32 , false ) ] ) ;
589+ let right =
590+ crate :: logical_plan:: table_scan ( Some ( "right" ) , & schema, None ) ?. build ( ) ?;
591+ let mut plan = crate :: logical_plan:: table_scan ( Some ( "left" ) , & schema, None ) ?
592+ . join_using ( right, crate :: JoinType :: Inner , vec ! [ Column :: from_name( "a" ) ] ) ?
593+ . build ( ) ?;
594+ // Invalid USING keys must only be inspected when an unqualified column
595+ // needs normalization, just as with a single expression.
596+ let LogicalPlan :: Join ( join) = & mut plan else {
597+ unreachable ! ( )
598+ } ;
599+ join. on [ 0 ] . 0 = lit ( 1 ) ;
600+ assert ! ( plan. using_columns( ) . is_err( ) ) ;
601+ assert ! ( super :: normalize_cols( Vec :: <Expr >:: new( ) , & plan) ?. is_empty( ) ) ;
602+ let exprs = vec ! [ lit( 1 ) , col( "left.a" ) , col( "other.missing" ) ] ;
603+ assert_eq ! ( super :: normalize_cols( exprs. clone( ) , & plan) ?, exprs) ;
604+ assert_eq ! ( normalize_col( col( "left.a" ) , & plan) ?, col( "left.a" ) ) ;
605+ assert_eq ! (
606+ normalize_sorts( [ col( "left.a" ) . sort( true , false ) ] , & plan) ?,
607+ vec![ col( "left.a" ) . sort( true , false ) ]
608+ ) ;
609+ assert ! ( super :: normalize_cols( [ col( "left.a" ) , col( "a" ) ] , & plan) . is_err( ) ) ;
610+ let projected = LogicalPlanBuilder :: from ( plan. clone ( ) )
611+ . project ( [ col ( "left.a" ) ] ) ?
612+ . build ( ) ?;
613+ assert_eq ! ( projected. expressions( ) , vec![ col( "left.a" ) ] ) ;
614+ assert ! ( LogicalPlanBuilder :: from( plan) . project( [ col( "a" ) ] ) . is_err( ) ) ;
615+ Ok ( ( ) )
616+ }
617+
494618 #[ test]
495619 fn unnormalize_cols ( ) {
496620 let expr = col ( "tableA.a" ) + col ( "tableB.b" ) ;
0 commit comments