Skip to content

Commit 2a8d8f5

Browse files
Preserve the left field's metadata exactly for the intersection narrowing
alias_qualified_with_metadata merges the alias metadata into the right column's own, since Expr::Alias's field derivation extends rather than replaces. A key only the right input carried could therefore survive into the result alongside the left field's metadata. Route the right column through a Cast to an explicit target field instead: both the logical (cast_output_field) and physical (CastExpr::new_with_target_field) field derivation for an explicit target use its metadata exactly, with nothing merged in from the source. Extend the intersect_nullability regression test to compare the complete metadata map, not just one key, for the tagged case.
1 parent 650a5de commit 2a8d8f5

2 files changed

Lines changed: 44 additions & 18 deletions

File tree

‎datafusion/substrait/src/logical_plan/consumer/rel/set_rel.rs‎

Lines changed: 33 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,12 @@
1616
// under the License.
1717

1818
use crate::logical_plan::consumer::SubstraitConsumer;
19-
use datafusion::common::metadata::FieldMetadata;
19+
use datafusion::arrow::datatypes::Field;
2020
use datafusion::common::{JoinType, NullEquality, not_impl_err, substrait_err};
2121
use datafusion::logical_expr::{
22-
Expr, LogicalPlan, LogicalPlanBuilder, requalify_sides_if_needed,
22+
Cast, Expr, LogicalPlan, LogicalPlanBuilder, requalify_sides_if_needed,
2323
};
24+
use std::sync::Arc;
2425
use substrait::proto::set_rel::SetOp;
2526
use substrait::proto::{Rel, SetRel};
2627

@@ -112,13 +113,15 @@ async fn intersect_rels(
112113
/// semi join does.
113114
///
114115
/// Differing metadata does not change which path is taken, as it is no part of
115-
/// nullability. The result should describe the left input, as
116-
/// [`LogicalPlanBuilder::intersect`] does, so on conflicting keys the left's
117-
/// metadata wins: a column read from the right is aliased with the left field's
118-
/// metadata, and an inner join's schema already lets the left input's schema
119-
/// metadata win, in the logical and in the physical plan alike. Keys only the
120-
/// right input carries are merged in, and the physical plan and the batches
121-
/// carry the same metadata as the logical plan.
116+
/// nullability. The result should describe the left input, exactly as
117+
/// [`LogicalPlanBuilder::intersect`] does: a column read from the right is cast
118+
/// to an explicit target field carrying the left field's metadata, and an
119+
/// explicit-field cast target replaces the source's metadata outright rather
120+
/// than merging into it, so a key only the right input carries is dropped
121+
/// rather than surviving into the result. A column read from the left is
122+
/// already exactly the left input's own field, in the logical and in the
123+
/// physical plan alike, so it needs no such treatment. The physical plan and
124+
/// the batches carry the same metadata as the logical plan.
122125
///
123126
/// [Set Operation rules]: https://substrait.io/relations/logical_relations/#set-operation
124127
fn intersect_rel(
@@ -130,7 +133,7 @@ fn intersect_rel(
130133
let right_fields = right.schema().fields();
131134
// A field is read from the right side when the left leaves it nullable and
132135
// the right requires it. Its metadata does not matter here: it is read from
133-
// the right with the left field's metadata layered over it.
136+
// the right with the left field's metadata substituted for its own.
134137
let from_right: Vec<bool> = left_fields
135138
.iter()
136139
.zip(right_fields.iter())
@@ -164,11 +167,26 @@ fn intersect_rel(
164167
.zip(&from_right)
165168
.map(|(((field, left), right), from_right)| {
166169
if *from_right {
167-
Expr::Column(right.clone()).alias_qualified_with_metadata(
168-
left.relation.clone(),
169-
&left.name,
170-
Some(FieldMetadata::from(field.metadata().clone())),
171-
)
170+
// `alias_qualified_with_metadata` would not do: `Expr::Alias`'s
171+
// field derivation extends the aliased expression's own
172+
// metadata with the alias's, so a key only the right column
173+
// carries would survive alongside the left field's metadata.
174+
// An explicit-field `Cast` target's metadata is instead used
175+
// exactly as given, in both the logical and the physical
176+
// plan, so casting to the left field's type (already proven
177+
// equal to the right's) and metadata drops the right's own
178+
// metadata outright. The qualifier and name still need
179+
// `alias_qualified` on top, since a `Cast`'s own field is not
180+
// renamed to its target field's name.
181+
let target_field = Arc::new(
182+
Field::new(&left.name, field.data_type().clone(), false)
183+
.with_metadata(field.metadata().clone()),
184+
);
185+
Expr::Cast(Cast::new_from_field(
186+
Box::new(Expr::Column(right.clone())),
187+
target_field,
188+
))
189+
.alias_qualified(left.relation.clone(), &left.name)
172190
} else {
173191
Expr::Column(left.clone())
174192
}

‎datafusion/substrait/tests/cases/logical_plans.rs‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -393,16 +393,24 @@ mod tests {
393393

394394
if tagged {
395395
// Where the inputs disagree, the primary table's metadata wins,
396-
// also on the columns read from a secondary input
396+
// also on the columns read from a secondary input. Comparing
397+
// the complete map, not just the "column" key, catches a
398+
// secondary-only key ("only_in_secondary") leaking through:
399+
// the primary table never carries that key, so its presence
400+
// would fail this exact-equality check.
397401
assert_eq!(
398402
plan.schema().metadata().get("table").map(String::as_str),
399403
Some("data"),
400404
"schema metadata of {file}"
401405
);
402406
for field in plan.schema().fields() {
407+
let expected_metadata = HashMap::from([(
408+
"column".to_string(),
409+
format!("data.{}", field.name()),
410+
)]);
403411
assert_eq!(
404-
field.metadata().get("column"),
405-
Some(&format!("data.{}", field.name())),
412+
field.metadata(),
413+
&expected_metadata,
406414
"metadata of column {} of {file}",
407415
field.name()
408416
);

0 commit comments

Comments
 (0)