Skip to content

Commit d5bd10d

Browse files
dariocurralamb
andauthored
physical-plan: coerce UNION/INTERLEAVE schema mismatches at plan time (apache#24094)
## Which issue does this PR close? Follow-up to apache#23861 (issue apache#15394). Not closing a new issue. ## Rationale for this change apache#23861 fixed `UNION ALL` batches carrying the wrong nullability when one leg is `NOT NULL` and another isn't, by re-stamping each batch's schema inside `UnionExec`/`InterleaveExec`'s own `execute()`. In review, @alamb noted: > ideally we could coerce the schema at plan time but I don't know how to > coerce nullability This PR does that: the coercion becomes an explicit node in the plan tree, inserted when the plan is built, instead of invisible logic inside `execute()`. ## What changes are included in this PR? - Adds `CoerceSchemaExec`, a single-child passthrough `ExecutionPlan` node. `UnionExec::try_new`/`InterleaveExec::try_new` insert it above any child whose own output schema disagrees with the computed union schema (in practice, only nullability differs -- `UnionExec::try_new` already rejects real data-type mismatches via `calculate_union`). - The actual batch re-stamping logic (`SchemaConformingStream`) is unchanged; it just lives under `CoerceSchemaExec::execute()` now instead of being called directly from `UnionExec`/`InterleaveExec::execute()`. - Because it's a real plan node, `CoerceSchemaExec` implements the full `ExecutionPlan` surface a pure 1:1 passthrough needs to stay transparent to the optimizer: statistics passthrough, filter/limit pushdown, `benefits_from_input_partitioning() -> false` (so it doesn't trigger a spurious repartition), and proto (de)serialization -- the node erases itself on encode and is reconstructed by `try_new` on decode, so no protobuf schema change was needed. - A genuine data-type mismatch (as opposed to nullability-only) is now rejected eagerly at plan-build time (via `EquivalenceProperties:: with_new_schema`) rather than lazily at `execute()`. ## Are these changes tested? - New unit test `test_union_partition_statistics_with_mismatched_nullability` in `union.rs`, proving statistics aren't poisoned to `Absent` through the new node. - Existing `union_nullable`/`union_nullable_spill` regression tests from apache#23861 continue to pass unchanged. - Updated the `sqllogictest` golden file (`union.slt`) where `EXPLAIN` output now shows the new node for pre-existing nullability-mismatched `UNION ALL` cases. - Benchmarked against the previous (inline) approach: no measurable performance difference in either the coerced or matched-schema case (differences were within run-to-run noise). ## Are there any user-facing changes? `EXPLAIN` output for a `UNION ALL`/interleaved plan with a nullability mismatch across legs will now show a `CoerceSchemaExec` node that wasn't there before. No behavioral or correctness change. --------- Co-authored-by: Andrew Lamb <[email protected]>
1 parent a251b94 commit d5bd10d

6 files changed

Lines changed: 338 additions & 88 deletions

File tree

‎datafusion/core/tests/dataframe/mod.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7075,7 +7075,8 @@ async fn test_copy_to_preserves_order() -> Result<()> {
70757075
DataSinkExec: sink=CsvSink(file_groups=[])
70767076
SortExec: expr=[column1@0 DESC], preserve_partitioning=[false]
70777077
DataSourceExec: partitions=1, partition_sizes=[1]
7078-
DataSourceExec: partitions=1, partition_sizes=[1]
7078+
ProjectionExec: expr=[CAST(column1@0 AS UInt64) as count]
7079+
DataSourceExec: partitions=1, partition_sizes=[1]
70797080
"
70807081
);
70817082
Ok(())

‎datafusion/physical-expr/src/projection.rs‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -848,6 +848,22 @@ fn project_column_statistics_through_expr(
848848
let inner_stats =
849849
project_column_statistics_through_expr(cast_expr.expr.as_ref(), column_stats);
850850
let target_type = cast_expr.cast_type();
851+
852+
// A cast whose source values are already of the target `DataType` never
853+
// changes any value -- see `cast_array_by_name`'s same-type fast path in
854+
// `ColumnarValue::cast_to`. In that case every statistic, not just
855+
// min/max, carries over unchanged (this is what a cast that only
856+
// re-stamps a column's nullability, as `UnionExec`/`InterleaveExec`
857+
// insert, looks like here).
858+
let already_target_type = matches!(
859+
(inner_stats.min_value.get_value(), inner_stats.max_value.get_value()),
860+
(Some(min), Some(max))
861+
if min.data_type() == *target_type && max.data_type() == *target_type
862+
);
863+
if already_target_type {
864+
return inner_stats;
865+
}
866+
851867
ColumnStatistics {
852868
min_value: inner_stats
853869
.min_value
@@ -2934,6 +2950,35 @@ pub(crate) mod tests {
29342950
Ok(())
29352951
}
29362952

2953+
#[test]
2954+
fn test_project_statistics_with_same_type_cast_is_exact_passthrough() -> Result<()> {
2955+
// A cast to the column's own `DataType` (e.g. one that only re-stamps
2956+
// nullability via `CastExpr::new_with_target_field`, as `UnionExec`/
2957+
// `InterleaveExec` insert) never changes any value, so every
2958+
// statistic -- not just min/max -- should carry over unchanged.
2959+
let input_stats = get_stats();
2960+
let col0_stats = input_stats.column_statistics[0].clone();
2961+
let input_schema = get_schema();
2962+
2963+
let projection = ProjectionExprs::new(vec![ProjectionExpr {
2964+
expr: Arc::new(CastExpr::new(
2965+
Arc::new(Column::new("col0", 0)),
2966+
DataType::Int64,
2967+
None,
2968+
)),
2969+
alias: "casted".to_string(),
2970+
}]);
2971+
2972+
let output_stats = projection.project_statistics(
2973+
input_stats,
2974+
&projection.project_schema(&input_schema)?,
2975+
)?;
2976+
2977+
assert_eq!(output_stats.column_statistics[0], col0_stats);
2978+
2979+
Ok(())
2980+
}
2981+
29372982
#[test]
29382983
fn test_project_statistics_with_cast() -> Result<()> {
29392984
let input_stats = get_stats();

‎datafusion/physical-plan/src/union.rs‎

Lines changed: 179 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -42,104 +42,92 @@ use crate::filter_pushdown::{
4242
FilterPushdownPropagation, PushedDown,
4343
};
4444
use crate::metrics::BaselineMetrics;
45-
use crate::projection::{ProjectionExec, make_with_child};
45+
use crate::projection::{ProjectionExec, ProjectionExpr, make_with_child};
4646
use crate::statistics::{ChildStats, StatisticsArgs};
4747
use crate::stream::ObservedStream;
4848

49-
use arrow::array::RecordBatchOptions;
5049
use arrow::datatypes::{Field, Schema, SchemaRef};
5150
use arrow::record_batch::RecordBatch;
5251
use datafusion_common::config::ConfigOptions;
5352
use datafusion_common::stats::NdvFallback;
5453
use datafusion_common::tree_node::TreeNodeRecursion;
5554
use datafusion_common::{
56-
Result, assert_or_internal_err, exec_err, internal_datafusion_err,
55+
Result, assert_or_internal_err, exec_err, internal_datafusion_err, plan_err,
5756
};
5857
use datafusion_execution::TaskContext;
58+
use datafusion_physical_expr::expressions::{CastExpr, Column};
5959
use datafusion_physical_expr::{
6060
EquivalenceProperties, PhysicalExpr, calculate_union, conjunction,
6161
};
6262

63-
use futures::{Stream, StreamExt};
63+
use futures::Stream;
6464
use itertools::Itertools;
6565
use log::{debug, trace, warn};
6666
use tokio::macros::support::thread_rng_n;
6767

68-
/// Wraps a child stream so that every batch it yields is re-stamped with
69-
/// `schema` instead of the child's own schema.
68+
/// Coerces `input`'s output schema to exactly `schema` via a `ProjectionExec`
69+
/// that re-stamps each column with the union's merged field (same
70+
/// `DataType`, but the union's merged nullability/name/metadata), or returns
71+
/// `input` unchanged if its schema already matches. [`UnionExec::try_new`]
72+
/// and [`InterleaveExec::try_new`] call this on every child, so the coercion
73+
/// is visible in the plan tree (e.g. in `EXPLAIN`) instead of happening
74+
/// invisibly inside the union operator's own `execute()`.
7075
///
71-
/// This is used by both [`UnionExec`] and [`InterleaveExec`] when a child's
72-
/// output schema disagrees with the operator's declared output schema --
73-
/// in practice this only happens for nullability (the declared schema is
74-
/// nullable wherever *any* input's field is, but casts are only inserted
75-
/// between inputs when the *type* differs, not when only nullability
76-
/// does). For [`UnionExec`], [`UnionExec::try_new`] guarantees this: it
77-
/// calls `calculate_union`, which rejects any input whose field data types
78-
/// don't match the computed union schema. [`InterleaveExec::try_new`] does
79-
/// not repeat that check -- its inputs are only ever produced by the
80-
/// optimizer rewriting an already-validated `UnionExec`, whose children's
81-
/// types are therefore already known to agree -- but if this wrapper ever
82-
/// did see a genuine data type mismatch (e.g. from a hand-built
83-
/// `InterleaveExec`), `RecordBatch::try_new_with_options` below reports it
84-
/// as an error rather than silently yielding a corrupt batch.
85-
struct SchemaConformingStream {
86-
schema: SchemaRef,
87-
inner: SendableRecordBatchStream,
88-
}
89-
90-
impl SchemaConformingStream {
91-
fn new(schema: SchemaRef, inner: SendableRecordBatchStream) -> Self {
92-
Self { schema, inner }
93-
}
94-
}
95-
96-
impl RecordBatchStream for SchemaConformingStream {
97-
fn schema(&self) -> SchemaRef {
98-
Arc::clone(&self.schema)
99-
}
100-
}
101-
102-
impl Stream for SchemaConformingStream {
103-
type Item = Result<RecordBatch>;
104-
105-
fn poll_next(
106-
mut self: Pin<&mut Self>,
107-
cx: &mut Context<'_>,
108-
) -> Poll<Option<Self::Item>> {
109-
self.inner.poll_next_unpin(cx).map(|opt| {
110-
opt.map(|batch_result| {
111-
batch_result.and_then(|batch| {
112-
let options =
113-
RecordBatchOptions::new().with_row_count(Some(batch.num_rows()));
114-
RecordBatch::try_new_with_options(
115-
Arc::clone(&self.schema),
116-
batch.columns().to_vec(),
117-
&options,
118-
)
119-
.map_err(Into::into)
120-
})
76+
/// A column whose `DataType` doesn't already match the union's is a genuine
77+
/// data type mismatch (as opposed to a nullability/name/metadata-only one),
78+
/// and is rejected eagerly here rather than silently cast or deferred to a
79+
/// runtime failure -- this only ever changes a column's declared schema,
80+
/// never its values.
81+
///
82+
/// Casting a column to its own `DataType` (only the `Field`'s nullability,
83+
/// name, or metadata changes) is a zero-copy relabeling: the cast kernel's
84+
/// same-type fast path (`cast_array_by_name`) just clones the `Arc<dyn
85+
/// Array>`, so this carries no runtime overhead over the schema it replaces.
86+
///
87+
/// See <https://github.com/apache/datafusion/issues/15394>.
88+
fn coerce_schema(
89+
input: Arc<dyn ExecutionPlan>,
90+
schema: &SchemaRef,
91+
) -> Result<Arc<dyn ExecutionPlan>> {
92+
let input_schema = input.schema();
93+
if &input_schema == schema {
94+
return Ok(input);
95+
}
96+
97+
let exprs = input_schema
98+
.fields()
99+
.iter()
100+
.zip(schema.fields())
101+
.enumerate()
102+
.map(|(i, (input_field, target_field))| {
103+
if input_field.data_type() != target_field.data_type() {
104+
return plan_err!(
105+
"UnionExec/InterleaveExec requires all inputs to have the same \
106+
data type per column; column {i} has type {} in one input, but \
107+
the union schema expects {}",
108+
input_field.data_type(),
109+
target_field.data_type()
110+
);
111+
}
112+
let column: Arc<dyn PhysicalExpr> =
113+
Arc::new(Column::new(input_field.name(), i));
114+
let expr = if input_field == target_field {
115+
column
116+
} else {
117+
Arc::new(CastExpr::new_with_target_field(
118+
column,
119+
Arc::clone(target_field),
120+
None,
121+
)) as Arc<dyn PhysicalExpr>
122+
};
123+
Ok(ProjectionExpr {
124+
expr,
125+
alias: target_field.name().clone(),
121126
})
122127
})
123-
}
128+
.collect::<Result<Vec<_>>>()?;
124129

125-
fn size_hint(&self) -> (usize, Option<usize>) {
126-
self.inner.size_hint()
127-
}
128-
}
129-
130-
/// Wraps `stream` in a [`SchemaConformingStream`] if its schema disagrees
131-
/// with `schema`, otherwise returns it unchanged. See
132-
/// [`SchemaConformingStream`] and
133-
/// <https://github.com/apache/datafusion/issues/15394>.
134-
fn conform_stream_schema(
135-
schema: SchemaRef,
136-
stream: SendableRecordBatchStream,
137-
) -> SendableRecordBatchStream {
138-
if stream.schema() == schema {
139-
stream
140-
} else {
141-
Box::pin(SchemaConformingStream::new(schema, stream))
142-
}
130+
Ok(Arc::new(ProjectionExec::try_new(exprs, input)?))
143131
}
144132

145133
/// `UnionExec`: `UNION ALL` execution plan.
@@ -209,6 +197,10 @@ impl UnionExec {
209197
// The schema of the inputs and the union schema is consistent when:
210198
// - They have the same number of fields, and
211199
// - Their fields have same types at the same indices.
200+
let inputs = inputs
201+
.into_iter()
202+
.map(|input| coerce_schema(input, &schema))
203+
.collect::<Result<Vec<_>>>()?;
212204
let cache = Self::compute_properties(&inputs, schema)?;
213205
Ok(Arc::new(UnionExec {
214206
inputs,
@@ -380,7 +372,6 @@ impl ExecutionPlan for UnionExec {
380372
if partition < input.output_partitioning().partition_count() {
381373
let stream = input.execute(partition, context)?;
382374
debug!("Found a Union partition to execute");
383-
let stream = conform_stream_schema(self.schema(), stream);
384375
return Ok(Box::pin(ObservedStream::new(
385376
stream,
386377
baseline_metrics,
@@ -650,7 +641,12 @@ impl InterleaveExec {
650641
can_interleave(inputs.iter()),
651642
"Not all InterleaveExec children have a consistent hash or range partitioning"
652643
);
653-
let cache = Self::compute_properties(&inputs)?;
644+
let schema = union_schema(&inputs)?;
645+
let inputs = inputs
646+
.into_iter()
647+
.map(|input| coerce_schema(input, &schema))
648+
.collect::<Result<Vec<_>>>()?;
649+
let cache = Self::compute_properties(&inputs, schema)?;
654650
Ok(InterleaveExec {
655651
inputs,
656652
metrics: ExecutionPlanMetricsSet::new(),
@@ -664,8 +660,10 @@ impl InterleaveExec {
664660
}
665661

666662
/// This function creates the cache object that stores the plan properties such as schema, equivalence properties, ordering, partitioning, etc.
667-
fn compute_properties(inputs: &[Arc<dyn ExecutionPlan>]) -> Result<PlanProperties> {
668-
let schema = union_schema(inputs)?;
663+
fn compute_properties(
664+
inputs: &[Arc<dyn ExecutionPlan>],
665+
schema: SchemaRef,
666+
) -> Result<PlanProperties> {
669667
let eq_properties = EquivalenceProperties::new(schema);
670668
// Get output partitioning:
671669
let output_partitioning = inputs[0].output_partitioning().clone();
@@ -763,7 +761,7 @@ impl ExecutionPlan for InterleaveExec {
763761
for input in self.inputs.iter() {
764762
if partition < input.output_partitioning().partition_count() {
765763
let stream = input.execute(partition, Arc::clone(&context))?;
766-
input_stream_vec.push(conform_stream_schema(self.schema(), stream));
764+
input_stream_vec.push(stream);
767765
} else {
768766
// Do not find a partition to execute
769767
break;
@@ -1277,6 +1275,107 @@ mod tests {
12771275
Ok(())
12781276
}
12791277

1278+
#[test]
1279+
fn test_union_partition_statistics_with_mismatched_nullability() -> Result<()> {
1280+
// Regression test for the `ProjectionExec` wrapper `UnionExec::try_new`
1281+
// inserts above the non-nullable leg here (via `coerce_schema`):
1282+
// exact column statistics (min/max/null/distinct/sum/byte_size) must
1283+
// still make it through the wrapper's same-type `CastExpr`, not get
1284+
// poisoned into `Absent` the way a generic (type-changing) cast's
1285+
// statistics would be.
1286+
let (_, left, right, expected) = stats_merge_inputs();
1287+
1288+
// `total_byte_size` differs from the plain-merge fixture (52): the
1289+
// wrapper is a `ProjectionExec`, whose `statistics_from_inputs`
1290+
// recomputes `total_byte_size` from the (unchanged) schema's row
1291+
// width times row count, rather than trusting the wrapped leg's own
1292+
// self-reported total -- still `Exact`, just derived differently.
1293+
// left: 5 rows * 4 bytes (UInt32) = 20 (was 23); right is untouched
1294+
// (already nullable, so `coerce_schema` doesn't wrap it): 20 + 29 = 49.
1295+
let expected = expected.with_total_byte_size(Precision::Exact(49));
1296+
1297+
let non_nullable_schema =
1298+
Schema::new(vec![Field::new("a", DataType::UInt32, false)]);
1299+
let nullable_schema = Schema::new(vec![Field::new("a", DataType::UInt32, true)]);
1300+
1301+
let left: Arc<dyn ExecutionPlan> =
1302+
Arc::new(StatisticsExec::new(left, non_nullable_schema));
1303+
let right: Arc<dyn ExecutionPlan> =
1304+
Arc::new(StatisticsExec::new(right, nullable_schema));
1305+
1306+
let union = UnionExec::try_new(vec![left, right])?;
1307+
let stats =
1308+
StatisticsContext::new().compute(union.as_ref(), &StatisticsArgs::new())?;
1309+
1310+
assert_eq!(stats.as_ref(), &expected);
1311+
Ok(())
1312+
}
1313+
1314+
#[tokio::test]
1315+
async fn test_coerce_schema_no_op_when_already_matching() -> Result<()> {
1316+
let schema_not_null =
1317+
Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
1318+
let input: Arc<dyn ExecutionPlan> =
1319+
TestMemoryExec::try_new_exec(&[vec![]], Arc::clone(&schema_not_null), None)?;
1320+
1321+
let coerced = coerce_schema(Arc::clone(&input), &schema_not_null)?;
1322+
assert!(Arc::ptr_eq(&coerced, &input));
1323+
1324+
Ok(())
1325+
}
1326+
1327+
#[tokio::test]
1328+
async fn test_coerce_schema_casts_only_nullability() -> Result<()> {
1329+
// Mismatched nullability: the input gets wrapped in a `ProjectionExec`
1330+
// whose `CastExpr` re-stamps the column with the target's `Field`
1331+
// (same `DataType`, so this is a zero-copy relabeling, not a real cast).
1332+
let schema_not_null =
1333+
Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
1334+
let batch_not_null = RecordBatch::try_new(
1335+
Arc::clone(&schema_not_null),
1336+
vec![Arc::new(arrow::array::Int32Array::from(vec![1, 2]))],
1337+
)?;
1338+
let input: Arc<dyn ExecutionPlan> = TestMemoryExec::try_new_exec(
1339+
&[vec![batch_not_null]],
1340+
Arc::clone(&schema_not_null),
1341+
None,
1342+
)?;
1343+
1344+
let nullable_schema =
1345+
Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));
1346+
let coerced = coerce_schema(Arc::clone(&input), &nullable_schema)?;
1347+
assert_eq!(&coerced.schema(), &nullable_schema);
1348+
let plan_str = crate::displayable(coerced.as_ref())
1349+
.indent(true)
1350+
.to_string();
1351+
assert!(
1352+
plan_str.contains("CAST"),
1353+
"expected a CAST in the coerced plan:\n{plan_str}"
1354+
);
1355+
1356+
let task_ctx = Arc::new(TaskContext::default());
1357+
let batches = collect(coerced, task_ctx).await?;
1358+
assert_eq!(batches.len(), 1);
1359+
assert_eq!(batches[0].schema(), nullable_schema);
1360+
1361+
Ok(())
1362+
}
1363+
1364+
#[test]
1365+
fn test_coerce_schema_rejects_genuine_type_mismatch() -> Result<()> {
1366+
let schema_int =
1367+
Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
1368+
let input: Arc<dyn ExecutionPlan> =
1369+
TestMemoryExec::try_new_exec(&[vec![]], Arc::clone(&schema_int), None)?;
1370+
1371+
let schema_utf8 =
1372+
Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, false)]));
1373+
let err = coerce_schema(input, &schema_utf8).unwrap_err();
1374+
assert!(err.to_string().contains("same data type per column"));
1375+
1376+
Ok(())
1377+
}
1378+
12801379
#[test]
12811380
fn test_interleave_partition_statistics_uses_shared_statistics_merge() -> Result<()> {
12821381
let (schema, left, right, expected) = stats_merge_inputs();

0 commit comments

Comments
 (0)