Skip to content

Commit 650a5de

Browse files
Keep the intersection nullability fix when input metadata differs
The narrowing path fell back to `LogicalPlanBuilder::intersect` when the inputs' schema metadata differed, and skipped a field whose metadata differed, so the nullable left-side schema came back for inputs that `ensure_schema_compatibility` accepts. Take the narrowing path regardless of metadata. A column read from the right is aliased with the left field's metadata, and the inner join already lets the left input's schema metadata win, so the left's metadata wins wherever the inputs disagree. `intersect_nullability` now also runs every plan over tables with differing schema and field metadata, and checks that the logical, the physical and the collected batch schemas agree. Co-Authored-By: Claude Sonnet 5 <[email protected]>
1 parent 1a42032 commit 650a5de

2 files changed

Lines changed: 93 additions & 17 deletions

File tree

‎datafusion/substrait/src/logical_plan/consumer/rel/set_rel.rs‎

Lines changed: 25 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
// under the License.
1717

1818
use crate::logical_plan::consumer::SubstraitConsumer;
19+
use datafusion::common::metadata::FieldMetadata;
1920
use datafusion::common::{JoinType, NullEquality, not_impl_err, substrait_err};
2021
use datafusion::logical_expr::{
2122
Expr, LogicalPlan, LogicalPlanBuilder, requalify_sides_if_needed,
@@ -110,6 +111,15 @@ async fn intersect_rels(
110111
/// Joining against distinct right rows keeps each left row at most once, as the
111112
/// semi join does.
112113
///
114+
/// Differing metadata does not change which path is taken, as it is no part of
115+
/// nullability. The result should describe the left input, as
116+
/// [`LogicalPlanBuilder::intersect`] does, so on conflicting keys the left's
117+
/// metadata wins: a column read from the right is aliased with the left field's
118+
/// metadata, and an inner join's schema already lets the left input's schema
119+
/// metadata win, in the logical and in the physical plan alike. Keys only the
120+
/// right input carries are merged in, and the physical plan and the batches
121+
/// carry the same metadata as the logical plan.
122+
///
113123
/// [Set Operation rules]: https://substrait.io/relations/logical_relations/#set-operation
114124
fn intersect_rel(
115125
left: LogicalPlan,
@@ -118,25 +128,21 @@ fn intersect_rel(
118128
) -> datafusion::common::Result<LogicalPlan> {
119129
let left_fields = left.schema().fields();
120130
let right_fields = right.schema().fields();
121-
// Only a field that differs from its right counterpart in nullability alone
122-
// is read from the right side, so every other attribute stays the left's.
131+
// A field is read from the right side when the left leaves it nullable and
132+
// the right requires it. Its metadata does not matter here: it is read from
133+
// the right with the left field's metadata layered over it.
123134
let from_right: Vec<bool> = left_fields
124135
.iter()
125136
.zip(right_fields.iter())
126137
.map(|(left, right)| {
127138
left.is_nullable()
128139
&& !right.is_nullable()
129140
&& left.data_type() == right.data_type()
130-
&& left.metadata() == right.metadata()
131141
})
132142
.collect();
133143

134-
// `intersect` also reports inputs of different widths. The join would merge
135-
// the right input's schema metadata into the result, so that must match too.
136-
if left_fields.len() != right_fields.len()
137-
|| left.schema().metadata() != right.schema().metadata()
138-
|| !from_right.contains(&true)
139-
{
144+
// `intersect` also reports inputs of different widths.
145+
if left_fields.len() != right_fields.len() || !from_right.contains(&true) {
140146
return LogicalPlanBuilder::intersect(left, right, is_all);
141147
}
142148

@@ -149,14 +155,20 @@ fn intersect_rel(
149155

150156
let left_columns = left.schema().columns();
151157
let right_columns = right.schema().columns();
152-
let exprs = left_columns
158+
let exprs = left
159+
.schema()
160+
.fields()
153161
.iter()
162+
.zip(&left_columns)
154163
.zip(&right_columns)
155164
.zip(&from_right)
156-
.map(|((left, right), from_right)| {
165+
.map(|(((field, left), right), from_right)| {
157166
if *from_right {
158-
Expr::Column(right.clone())
159-
.alias_qualified(left.relation.clone(), &left.name)
167+
Expr::Column(right.clone()).alias_qualified_with_metadata(
168+
left.relation.clone(),
169+
&left.name,
170+
Some(FieldMetadata::from(field.metadata().clone())),
171+
)
160172
} else {
161173
Expr::Column(left.clone())
162174
}

‎datafusion/substrait/tests/cases/logical_plans.rs‎

Lines changed: 68 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,11 @@ mod tests {
2222
use crate::cases::roundtrip_logical_plan::higher_order_function_ctx;
2323
use crate::utils::test::{add_plan_schemas_to_ctx, read_json};
2424
use datafusion::arrow::array::{ArrayRef, Int64Array, RecordBatch};
25+
use datafusion::arrow::datatypes::{Field, Schema, SchemaRef};
2526
use datafusion::assert_batches_sorted_eq;
2627
use datafusion::common::test_util::format_batches;
2728
use datafusion::datasource::MemTable;
28-
use std::collections::HashSet;
29+
use std::collections::{HashMap, HashSet};
2930
use std::sync::Arc;
3031

3132
use datafusion::common::Result;
@@ -270,7 +271,39 @@ mod tests {
270271
),
271272
];
272273

273-
for (file, expected_nullability, expected_rows) in [
274+
// Schema and field metadata are no part of an input's nullability, so the
275+
// result must not depend on whether the tables carry any. With metadata,
276+
// every table describes itself and the secondary tables add keys the
277+
// primary one lacks: the result has to keep the primary table's metadata
278+
// where they disagree, including on the columns read from a secondary
279+
// input, and the physical plan and the batches have to agree with it. The
280+
// secondary tables share their metadata, as `INTERSECTION_PRIMARY` unions
281+
// them, and a union of inputs with differing field metadata reports
282+
// different metadata in its logical and in its physical schema.
283+
let with_metadata = |schema: &SchemaRef, table: &str| -> SchemaRef {
284+
let role = if table == "data" { "data" } else { "secondary" };
285+
let tag = |mut metadata: HashMap<String, String>| {
286+
if role == "secondary" {
287+
metadata.insert("only_in_secondary".to_string(), "yes".to_string());
288+
}
289+
metadata
290+
};
291+
let fields: Vec<Field> = schema
292+
.fields()
293+
.iter()
294+
.map(|field| {
295+
let metadata = HashMap::from([(
296+
"column".to_string(),
297+
format!("{role}.{}", field.name()),
298+
)]);
299+
field.as_ref().clone().with_metadata(tag(metadata))
300+
})
301+
.collect();
302+
let metadata = HashMap::from([("table".to_string(), role.to_string())]);
303+
Arc::new(Schema::new_with_metadata(fields, tag(metadata)))
304+
};
305+
306+
let cases = [
274307
// Nullable in the primary input and in at least one secondary input.
275308
(
276309
"intersect_primary_mixed_nullability",
@@ -308,13 +341,23 @@ mod tests {
308341
"+---+---+---+---+---+---+",
309342
][..],
310343
),
311-
] {
344+
];
345+
346+
for ((file, expected_nullability, expected_rows), tagged) in cases
347+
.into_iter()
348+
.flat_map(|case| [(case, false), (case, true)])
349+
{
312350
let proto_plan =
313351
read_json(&format!("tests/testdata/test_plans/{file}.substrait.json"));
314352
let ctx = add_plan_schemas_to_ctx(SessionContext::new(), &proto_plan)?;
315353
// Give each table rows, so the batch schemas below come from real batches
316354
for (table, rows) in rows {
317355
let schema = ctx.table_provider(table).await?.schema();
356+
let schema = if tagged {
357+
with_metadata(&schema, table)
358+
} else {
359+
schema
360+
};
318361
let columns = (0..schema.fields().len())
319362
.map(|i| {
320363
Arc::new(rows.iter().map(|row| row[i]).collect::<Int64Array>())
@@ -343,7 +386,28 @@ mod tests {
343386
})
344387
.collect::<Vec<_>>()
345388
.join(", ");
346-
assert_eq!(nullability, expected_nullability, "nullability of {file}");
389+
assert_eq!(
390+
nullability, expected_nullability,
391+
"nullability of {file} (tagged: {tagged})"
392+
);
393+
394+
if tagged {
395+
// Where the inputs disagree, the primary table's metadata wins,
396+
// also on the columns read from a secondary input
397+
assert_eq!(
398+
plan.schema().metadata().get("table").map(String::as_str),
399+
Some("data"),
400+
"schema metadata of {file}"
401+
);
402+
for field in plan.schema().fields() {
403+
assert_eq!(
404+
field.metadata().get("column"),
405+
Some(&format!("data.{}", field.name())),
406+
"metadata of column {} of {file}",
407+
field.name()
408+
);
409+
}
410+
}
347411

348412
// The physical plan and the batches it produces must carry the same
349413
// schema as the logical plan, not the left input's nullability

0 commit comments

Comments
 (0)