|
17 | 17 |
|
18 | 18 | use crate::logical_plan::consumer::SubstraitConsumer; |
19 | 19 | use datafusion::arrow::datatypes::Field; |
20 | | -use datafusion::common::{JoinType, NullEquality, not_impl_err, substrait_err}; |
| 20 | +use datafusion::common::{DFSchema, JoinType, NullEquality, not_impl_err, substrait_err}; |
21 | 21 | use datafusion::logical_expr::{ |
22 | | - Cast, Expr, LogicalPlan, LogicalPlanBuilder, requalify_sides_if_needed, |
| 22 | + Cast, Expr, LogicalPlan, LogicalPlanBuilder, Projection, projection_schema, |
| 23 | + requalify_sides_if_needed, |
23 | 24 | }; |
24 | 25 | use std::sync::Arc; |
25 | 26 | use substrait::proto::set_rel::SetOp; |
@@ -113,15 +114,43 @@ async fn intersect_rels( |
113 | 114 | /// semi join does. |
114 | 115 | /// |
115 | 116 | /// Differing metadata does not change which path is taken, as it is no part of |
116 | | -/// nullability. The result should describe the left input, exactly as |
117 | | -/// [`LogicalPlanBuilder::intersect`] does: a column read from the right is cast |
118 | | -/// to an explicit target field carrying the left field's metadata, and an |
119 | | -/// explicit-field cast target replaces the source's metadata outright rather |
120 | | -/// than merging into it, so a key only the right input carries is dropped |
121 | | -/// rather than surviving into the result. A column read from the left is |
122 | | -/// already exactly the left input's own field, in the logical and in the |
123 | | -/// physical plan alike, so it needs no such treatment. The physical plan and |
124 | | -/// the batches carry the same metadata as the logical plan. |
| 117 | +/// nullability. The result describes the left input's metadata exactly at the |
| 118 | +/// field level, and does so at the schema level for the plan this function |
| 119 | +/// returns, though not necessarily after later rewrites; see the two notes |
| 120 | +/// below. |
| 121 | +/// |
| 122 | +/// At the field level: a column read from the right is cast to an explicit |
| 123 | +/// target field carrying the left field's metadata, and an explicit-field |
| 124 | +/// cast target replaces the source's metadata outright rather than merging |
| 125 | +/// into it, so a key only the right input's field carries is dropped rather |
| 126 | +/// than surviving into the result. A column read from the left is already |
| 127 | +/// exactly the left input's own field, so it needs no such treatment. This is |
| 128 | +/// robust: a `Field`'s own metadata travels with it through any later |
| 129 | +/// rewrite, since a rebuilt schema is still assembled from the same `Field`s. |
| 130 | +/// |
| 131 | +/// At the schema level: [`build_join_schema`](datafusion::logical_expr::build_join_schema) |
| 132 | +/// chains the two inputs' schema metadata maps (right's entries, then |
| 133 | +/// left's), so a conflicting key resolves to the left's value but a key |
| 134 | +/// present only on the right's schema survives, and [`projection_schema`] |
| 135 | +/// copies that merged map verbatim onto a projection built directly on top |
| 136 | +/// of the join. The projection this function returns is therefore built with |
| 137 | +/// an explicit schema instead, whose metadata is the left input's own schema |
| 138 | +/// metadata, unmodified - correct for the plan as returned here, and for a |
| 139 | +/// caller that reads its schema without executing it. |
| 140 | +/// |
| 141 | +/// That schema-level fix is not preserved through arbitrary later rewrites, |
| 142 | +/// unlike the field-level one. `LogicalPlan::recompute_schema` rebuilds a |
| 143 | +/// `Join`'s or a `Projection`'s schema-level metadata from its (possibly |
| 144 | +/// rewritten) children's *current* schema-level metadata, discarding |
| 145 | +/// whatever this function set. DataFusion's optimizer calls it whenever it |
| 146 | +/// judges anything in the plan changed, including something unrelated |
| 147 | +/// elsewhere in the same query, so by the time this plan reaches physical |
| 148 | +/// planning its schema-level metadata may again be the join's merged one |
| 149 | +/// rather than the left input's. This holds even for the simplest two-input, |
| 150 | +/// non-correlated case, with no join filter or `INTERSECTION_PRIMARY` union |
| 151 | +/// involved. There is currently no supported way from this crate to opt a |
| 152 | +/// join's or a projection's schema-level metadata out of that |
| 153 | +/// recomputation. |
125 | 154 | /// |
126 | 155 | /// [Set Operation rules]: https://substrait.io/relations/logical_relations/#set-operation |
127 | 156 | fn intersect_rel( |
@@ -193,15 +222,57 @@ fn intersect_rel( |
193 | 222 | }) |
194 | 223 | .collect::<Vec<_>>(); |
195 | 224 |
|
196 | | - left.join_detailed( |
197 | | - right, |
198 | | - JoinType::Inner, |
199 | | - (left_columns, right_columns), |
200 | | - None, |
201 | | - NullEquality::NullEqualsNull, |
202 | | - )? |
203 | | - .project(exprs)? |
204 | | - .build() |
| 225 | + // Captured before `left` is consumed by `join_detailed` below: this is |
| 226 | + // the metadata the projection's schema must end up with, not the join's. |
| 227 | + let left_schema_metadata = left.schema().metadata().clone(); |
| 228 | + |
| 229 | + let joined = left |
| 230 | + .join_detailed( |
| 231 | + right, |
| 232 | + JoinType::Inner, |
| 233 | + (left_columns, right_columns), |
| 234 | + None, |
| 235 | + NullEquality::NullEqualsNull, |
| 236 | + )? |
| 237 | + .build()?; |
| 238 | + |
| 239 | + // `projection_schema` would copy the join's own schema metadata, which |
| 240 | + // (per `build_join_schema`) can carry a key that only the right side's |
| 241 | + // schema has. Build the fields the normal way, then swap in the left |
| 242 | + // input's schema metadata so the projection's schema, like its fields, |
| 243 | + // describes the left input exactly. |
| 244 | + // |
| 245 | + // This is correct for the plan `intersect_rel` returns, but it is not |
| 246 | + // preserved through arbitrary later rewrites: `LogicalPlan::Join` and |
| 247 | + // `LogicalPlan::Projection` both recompute their schema's metadata from |
| 248 | + // their (possibly rewritten) children whenever `recompute_schema` runs on |
| 249 | + // them (see its match arms in `datafusion/expr/src/logical_plan/plan.rs`), |
| 250 | + // which happens whenever the surrounding optimizer decides anything in |
| 251 | + // the plan changed - even something unrelated elsewhere in the same |
| 252 | + // query. There is no supported way from this crate to make a join's or a |
| 253 | + // projection's schema-level metadata (as opposed to per-field metadata, |
| 254 | + // which is carried on the `Field`s themselves and does survive) opt out |
| 255 | + // of that recomputation. See the discussion on this PR for confirmation |
| 256 | + // this is a general limitation, not specific to this construction. |
| 257 | + let projected_schema = projection_schema(&joined, &exprs)?; |
| 258 | + let schema = Arc::new( |
| 259 | + DFSchema::new_with_metadata( |
| 260 | + projected_schema |
| 261 | + .iter() |
| 262 | + .map(|(qualifier, field)| (qualifier.cloned(), Arc::clone(field))) |
| 263 | + .collect(), |
| 264 | + left_schema_metadata, |
| 265 | + )? |
| 266 | + .with_functional_dependencies( |
| 267 | + projected_schema.functional_dependencies().clone(), |
| 268 | + )?, |
| 269 | + ); |
| 270 | + |
| 271 | + Ok(LogicalPlan::Projection(Projection::try_new_with_schema( |
| 272 | + exprs, |
| 273 | + Arc::new(joined), |
| 274 | + schema, |
| 275 | + )?)) |
205 | 276 | } |
206 | 277 |
|
207 | 278 | async fn except_rels( |
|
0 commit comments