1717
1818use crate :: logical_plan:: consumer:: SubstraitConsumer ;
1919use datafusion:: arrow:: datatypes:: Field ;
20- use datafusion:: common:: { DFSchema , JoinType , NullEquality , not_impl_err, substrait_err} ;
20+ use datafusion:: common:: { JoinType , NullEquality , not_impl_err, substrait_err} ;
2121use datafusion:: logical_expr:: {
22- Cast , Expr , LogicalPlan , LogicalPlanBuilder , Projection , projection_schema,
23- requalify_sides_if_needed,
22+ Cast , Expr , LogicalPlan , LogicalPlanBuilder , requalify_sides_if_needed,
2423} ;
2524use std:: sync:: Arc ;
2625use substrait:: proto:: set_rel:: SetOp ;
@@ -113,44 +112,22 @@ async fn intersect_rels(
113112/// Joining against distinct right rows keeps each left row at most once, as the
114113/// semi join does.
115114///
116- /// Differing metadata does not change which path is taken, as it is no part of
117- /// nullability. The result describes the left input's metadata exactly at the
118- /// field level, and does so at the schema level for the plan this function
119- /// returns, though not necessarily after later rewrites; see the two notes
120- /// below.
115+ /// Differing metadata does not change which path is taken. A column read from
116+ /// the right is cast to an explicit target field carrying the left field's
117+ /// metadata; an explicit-field cast target's metadata is used exactly as
118+ /// given rather than merged into the source's, so this drops a key only the
119+ /// right field carried and keeps the left field's metadata exactly. A
120+ /// `Field`'s own metadata travels with it through any later rewrite, so this
121+ /// holds for the plan this function returns and for any rewrite of it.
121122///
122- /// At the field level: a column read from the right is cast to an explicit
123- /// target field carrying the left field's metadata, and an explicit-field
124- /// cast target replaces the source's metadata outright rather than merging
125- /// into it, so a key only the right input's field carries is dropped rather
126- /// than surviving into the result. A column read from the left is already
127- /// exactly the left input's own field, so it needs no such treatment. This is
128- /// robust: a `Field`'s own metadata travels with it through any later
129- /// rewrite, since a rebuilt schema is still assembled from the same `Field`s.
130- ///
131- /// At the schema level: [`build_join_schema`](datafusion::logical_expr::build_join_schema)
132- /// chains the two inputs' schema metadata maps (right's entries, then
133- /// left's), so a conflicting key resolves to the left's value but a key
134- /// present only on the right's schema survives, and [`projection_schema`]
135- /// copies that merged map verbatim onto a projection built directly on top
136- /// of the join. The projection this function returns is therefore built with
137- /// an explicit schema instead, whose metadata is the left input's own schema
138- /// metadata, unmodified - correct for the plan as returned here, and for a
139- /// caller that reads its schema without executing it.
140- ///
141- /// That schema-level fix is not preserved through arbitrary later rewrites,
142- /// unlike the field-level one. `LogicalPlan::recompute_schema` rebuilds a
143- /// `Join`'s or a `Projection`'s schema-level metadata from its (possibly
144- /// rewritten) children's *current* schema-level metadata, discarding
145- /// whatever this function set. DataFusion's optimizer calls it whenever it
146- /// judges anything in the plan changed, including something unrelated
147- /// elsewhere in the same query, so by the time this plan reaches physical
148- /// planning its schema-level metadata may again be the join's merged one
149- /// rather than the left input's. This holds even for the simplest two-input,
150- /// non-correlated case, with no join filter or `INTERSECTION_PRIMARY` union
151- /// involved. There is currently no supported way from this crate to opt a
152- /// join's or a projection's schema-level metadata out of that
153- /// recomputation.
123+ /// Schema-level (as opposed to per-field) metadata gets no such treatment:
124+ /// [`build_join_schema`](datafusion::logical_expr::build_join_schema) merges
125+ /// the two inputs' schema metadata with the left's value winning a
126+ /// conflicting key, and that merged map is what the projection built on top
127+ /// of the join reports - the same as the semi join
128+ /// [`LogicalPlanBuilder::intersect`] takes for the cases this function
129+ /// doesn't rewrite. A key only the right input's schema carries can
130+ /// therefore still appear in the result's schema-level metadata.
154131///
155132/// [Set Operation rules]: https://substrait.io/relations/logical_relations/#set-operation
156133fn intersect_rel (
@@ -222,10 +199,6 @@ fn intersect_rel(
222199 } )
223200 . collect :: < Vec < _ > > ( ) ;
224201
225- // Captured before `left` is consumed by `join_detailed` below: this is
226- // the metadata the projection's schema must end up with, not the join's.
227- let left_schema_metadata = left. schema ( ) . metadata ( ) . clone ( ) ;
228-
229202 let joined = left
230203 . join_detailed (
231204 right,
@@ -236,43 +209,7 @@ fn intersect_rel(
236209 ) ?
237210 . build ( ) ?;
238211
239- // `projection_schema` would copy the join's own schema metadata, which
240- // (per `build_join_schema`) can carry a key that only the right side's
241- // schema has. Build the fields the normal way, then swap in the left
242- // input's schema metadata so the projection's schema, like its fields,
243- // describes the left input exactly.
244- //
245- // This is correct for the plan `intersect_rel` returns, but it is not
246- // preserved through arbitrary later rewrites: `LogicalPlan::Join` and
247- // `LogicalPlan::Projection` both recompute their schema's metadata from
248- // their (possibly rewritten) children whenever `recompute_schema` runs on
249- // them (see its match arms in `datafusion/expr/src/logical_plan/plan.rs`),
250- // which happens whenever the surrounding optimizer decides anything in
251- // the plan changed - even something unrelated elsewhere in the same
252- // query. There is no supported way from this crate to make a join's or a
253- // projection's schema-level metadata (as opposed to per-field metadata,
254- // which is carried on the `Field`s themselves and does survive) opt out
255- // of that recomputation. See the discussion on this PR for confirmation
256- // this is a general limitation, not specific to this construction.
257- let projected_schema = projection_schema ( & joined, & exprs) ?;
258- let schema = Arc :: new (
259- DFSchema :: new_with_metadata (
260- projected_schema
261- . iter ( )
262- . map ( |( qualifier, field) | ( qualifier. cloned ( ) , Arc :: clone ( field) ) )
263- . collect ( ) ,
264- left_schema_metadata,
265- ) ?
266- . with_functional_dependencies (
267- projected_schema. functional_dependencies ( ) . clone ( ) ,
268- ) ?,
269- ) ;
270-
271- Ok ( LogicalPlan :: Projection ( Projection :: try_new_with_schema (
272- exprs,
273- Arc :: new ( joined) ,
274- schema,
275- ) ?) )
212+ LogicalPlanBuilder :: from ( joined) . project ( exprs) ?. build ( )
276213}
277214
278215async fn except_rels (
0 commit comments