Skip to content

Commit bed030d

Browse files
committed
fix: UnionExec now conforms each batch to the union's declared schema
UNION ALL between an input with a NOT NULL column and one where the same column is nullable produced a valid, correctly-typed logical plan (the analyzer already OR's nullability across legs in coerce_union_schema), but UnionExec::execute() handed out each child's RecordBatches completely unchanged. The leg whose column was already NOT NULL (and so needed no CAST) kept emitting batches with a NOT NULL field, contradicting the union's own declared (nullable) schema. Any consumer that checks schema equality across batches from the same stream -- e.g. pyarrow.Table.from_batches via the Arrow C Stream FFI used by the datafusion Python bindings -- then rejects the stream with `ArrowInvalid: Schema at index N was different`, even though every individual SELECT runs fine on its own and DataFusion's own execution never errors. UnionExec::execute() now re-stamps each child's batches with the union's own schema whenever they disagree. This is always safe: the union's schema can only be more permissive than any single input's (nullability is OR'd, never narrowed), and only the Field::nullable flag changes -- the underlying array data and data type are untouched. Closes apache#15394.
1 parent 3f81613 commit bed030d

3 files changed

Lines changed: 292 additions & 1 deletion

File tree

‎datafusion/core/tests/sql/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ mod runtime_config;
7171
pub mod select;
7272
mod sql_api;
7373
mod union_comparison;
74+
mod union_nullable;
7475
mod unparser;
7576

7677
async fn register_aggregate_csv_by_sql(ctx: &SessionContext) {
Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
//! Regression tests for `UNION ALL` between inputs where the same column is
19+
//! nullable on one side and `NOT NULL` on the other.
20+
//!
21+
//! DataFusion's analyzer computes the union's output schema by OR-ing the
22+
//! nullability of each input's field (`coerce_union_schema`), so the
23+
//! *declared* output field is nullable whenever any leg is. But before this
24+
//! fix, `UnionExec::execute` handed out each child's `RecordBatch`es
25+
//! completely unchanged, so a leg whose column was already `NOT NULL` (and
26+
//! therefore needed no `CAST`) kept producing batches with a `NOT NULL`
27+
//! field, contradicting the union's own declared (nullable) schema. Any
28+
//! consumer that checks schema equality across batches from the same
29+
//! stream -- e.g. `pyarrow.Table.from_batches` via the Arrow C Stream FFI
30+
//! used by the `datafusion` Python bindings -- then rejects the stream with
31+
//! `ArrowInvalid: Schema at index N was different`, even though every
32+
//! individual `SELECT` runs fine on its own. See
33+
//! <https://github.com/apache/datafusion/issues/15394>.
34+
35+
use std::sync::Arc;
36+
37+
use arrow::array::{Int64Array, RecordBatch, StringArray};
38+
use arrow::datatypes::{DataType, Field, Schema};
39+
use datafusion::prelude::*;
40+
use datafusion_common::Result;
41+
42+
/// Builds two single-partition tables that agree on `id`/`status` types but
43+
/// disagree on whether `status` is nullable, then runs `UNION ALL` over them.
44+
async fn union_all_mismatched_nullable(
45+
left_nullable: bool,
46+
right_nullable: bool,
47+
) -> Result<DataFrame> {
48+
let ctx = SessionContext::new();
49+
50+
let schema_a = Arc::new(Schema::new(vec![
51+
Field::new("id", DataType::Int64, false),
52+
Field::new("status", DataType::Utf8, left_nullable),
53+
]));
54+
let batch_a = RecordBatch::try_new(
55+
Arc::clone(&schema_a),
56+
vec![
57+
Arc::new(Int64Array::from(vec![1, 2])),
58+
Arc::new(StringArray::from(vec!["ok", "ok"])),
59+
],
60+
)?;
61+
ctx.register_batch("table_a", batch_a)?;
62+
63+
let schema_b = Arc::new(Schema::new(vec![
64+
Field::new("id", DataType::Int64, false),
65+
Field::new("status", DataType::Utf8, right_nullable),
66+
]));
67+
let status_values: Vec<Option<&str>> = if right_nullable {
68+
vec![Some("done"), None]
69+
} else {
70+
vec![Some("done"), Some("also-done")]
71+
};
72+
let batch_b = RecordBatch::try_new(
73+
Arc::clone(&schema_b),
74+
vec![
75+
Arc::new(Int64Array::from(vec![3, 4])),
76+
Arc::new(StringArray::from(status_values)),
77+
],
78+
)?;
79+
ctx.register_batch("table_b", batch_b)?;
80+
81+
ctx.sql(
82+
"SELECT id, status FROM table_a \
83+
UNION ALL \
84+
SELECT id, status FROM table_b",
85+
)
86+
.await
87+
}
88+
89+
/// The schema DataFusion actually commits to for a query: the logical plan
90+
/// after the `Analyzer` (which includes the `UNION` nullability/type
91+
/// coercion this test targets) and `Optimizer` have run. `DataFrame::schema`
92+
/// alone is not enough here -- it reflects the raw, pre-`Analyzer` plan (see
93+
/// `SessionState::create_logical_plan`), which for a `UNION` still has the
94+
/// first leg's un-coerced type.
95+
async fn analyzed_schema(df: &DataFrame) -> Result<Schema> {
96+
Ok(df
97+
.clone()
98+
.into_optimized_plan()?
99+
.schema()
100+
.as_arrow()
101+
.clone())
102+
}
103+
104+
/// Every `RecordBatch` actually produced by a `UNION ALL` must match the
105+
/// query's analyzed output schema field-for-field -- including
106+
/// nullability -- no matter which leg it came from.
107+
async fn assert_every_batch_matches_declared_schema(df: DataFrame) -> Result<()> {
108+
let declared_schema = analyzed_schema(&df).await?;
109+
110+
let batches = df.collect().await?;
111+
assert!(!batches.is_empty());
112+
for batch in &batches {
113+
assert_eq!(
114+
batch.schema().as_ref(),
115+
&declared_schema,
116+
"a UNION ALL leg produced a RecordBatch whose schema disagrees \
117+
with the union's declared output schema (commonly a dropped \
118+
nullable flag) -- this is what downstream consumers that check \
119+
schema equality across batches (e.g. pyarrow) reject with \
120+
`ArrowInvalid: Schema at index N was different`"
121+
);
122+
}
123+
Ok(())
124+
}
125+
126+
#[tokio::test]
127+
async fn union_all_same_type_left_not_null_right_nullable() -> Result<()> {
128+
let df = union_all_mismatched_nullable(false, true).await?;
129+
assert!(
130+
analyzed_schema(&df)
131+
.await?
132+
.field_with_name("status")?
133+
.is_nullable()
134+
);
135+
assert_every_batch_matches_declared_schema(df).await
136+
}
137+
138+
#[tokio::test]
139+
async fn union_all_same_type_left_nullable_right_not_null() -> Result<()> {
140+
let df = union_all_mismatched_nullable(true, false).await?;
141+
assert!(
142+
analyzed_schema(&df)
143+
.await?
144+
.field_with_name("status")?
145+
.is_nullable()
146+
);
147+
assert_every_batch_matches_declared_schema(df).await
148+
}
149+
150+
#[tokio::test]
151+
async fn union_all_same_type_both_not_null_stays_not_null() -> Result<()> {
152+
let df = union_all_mismatched_nullable(false, false).await?;
153+
let declared_schema = analyzed_schema(&df).await?;
154+
assert!(
155+
!declared_schema.field_with_name("status")?.is_nullable(),
156+
"status should remain NOT NULL when neither leg is nullable"
157+
);
158+
assert_every_batch_matches_declared_schema(df).await
159+
}
160+
161+
/// Same bug, but the coercion also has to widen the *type* (Int32 -> Int64)
162+
/// on one leg. The leg that already matched the target type still needed
163+
/// its nullability reconciled at execution time, independent of whichever
164+
/// legs needed a `CAST`.
165+
#[tokio::test]
166+
async fn union_all_widening_cast_also_fixes_nullable() -> Result<()> {
167+
use arrow::array::Int32Array;
168+
169+
let ctx = SessionContext::new();
170+
171+
let schema_a = Arc::new(Schema::new(vec![
172+
Field::new("id", DataType::Int64, false),
173+
Field::new("val", DataType::Int32, false),
174+
]));
175+
let batch_a = RecordBatch::try_new(
176+
Arc::clone(&schema_a),
177+
vec![
178+
Arc::new(Int64Array::from(vec![1, 2])),
179+
Arc::new(Int32Array::from(vec![10, 20])),
180+
],
181+
)?;
182+
ctx.register_batch("table_a", batch_a)?;
183+
184+
let schema_b = Arc::new(Schema::new(vec![
185+
Field::new("id", DataType::Int64, false),
186+
Field::new("val", DataType::Int64, true),
187+
]));
188+
let batch_b = RecordBatch::try_new(
189+
Arc::clone(&schema_b),
190+
vec![
191+
Arc::new(Int64Array::from(vec![3, 4])),
192+
Arc::new(Int64Array::from(vec![Some(30), None])),
193+
],
194+
)?;
195+
ctx.register_batch("table_b", batch_b)?;
196+
197+
let df = ctx
198+
.sql(
199+
"SELECT id, val FROM table_a \
200+
UNION ALL \
201+
SELECT id, val FROM table_b",
202+
)
203+
.await?;
204+
205+
let declared_schema = analyzed_schema(&df).await?;
206+
assert_eq!(
207+
declared_schema.field_with_name("val")?.data_type(),
208+
&DataType::Int64
209+
);
210+
assert!(declared_schema.field_with_name("val")?.is_nullable());
211+
212+
let batches = df.collect().await?;
213+
for batch in &batches {
214+
assert_eq!(batch.schema().as_ref(), &declared_schema);
215+
}
216+
Ok(())
217+
}

‎datafusion/physical-plan/src/union.rs‎

Lines changed: 74 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ use crate::projection::{ProjectionExec, make_with_child};
4646
use crate::statistics::{ChildStats, StatisticsArgs};
4747
use crate::stream::ObservedStream;
4848

49+
use arrow::array::RecordBatchOptions;
4950
use arrow::datatypes::{Field, Schema, SchemaRef};
5051
use arrow::record_batch::RecordBatch;
5152
use datafusion_common::config::ConfigOptions;
@@ -58,11 +59,68 @@ use datafusion_physical_expr::{
5859
EquivalenceProperties, PhysicalExpr, calculate_union, conjunction,
5960
};
6061

61-
use futures::Stream;
62+
use futures::{Stream, StreamExt};
6263
use itertools::Itertools;
6364
use log::{debug, trace, warn};
6465
use tokio::macros::support::thread_rng_n;
6566

67+
/// Wraps a child stream so that every batch it yields is re-stamped with
68+
/// `schema` instead of the child's own schema.
69+
///
70+
/// This is used by [`UnionExec`] when a child's output schema disagrees
71+
/// with the union's declared output schema -- in practice this only
72+
/// happens for nullability (the union's schema is nullable wherever *any*
73+
/// input's field is, but casts are only inserted between inputs when the
74+
/// *type* differs, not when only nullability does). Re-stamping is safe:
75+
/// the union's schema can only be *more* permissive than any single
76+
/// input's (nullability is combined with logical OR, never narrowed), and
77+
/// the underlying data type is unaffected -- only the `Field::nullable`
78+
/// flag (and equivalent metadata) changes.
79+
struct SchemaConformingStream {
80+
schema: SchemaRef,
81+
inner: SendableRecordBatchStream,
82+
}
83+
84+
impl SchemaConformingStream {
85+
fn new(schema: SchemaRef, inner: SendableRecordBatchStream) -> Self {
86+
Self { schema, inner }
87+
}
88+
}
89+
90+
impl RecordBatchStream for SchemaConformingStream {
91+
fn schema(&self) -> SchemaRef {
92+
Arc::clone(&self.schema)
93+
}
94+
}
95+
96+
impl Stream for SchemaConformingStream {
97+
type Item = Result<RecordBatch>;
98+
99+
fn poll_next(
100+
mut self: Pin<&mut Self>,
101+
cx: &mut Context<'_>,
102+
) -> Poll<Option<Self::Item>> {
103+
self.inner.poll_next_unpin(cx).map(|opt| {
104+
opt.map(|batch_result| {
105+
batch_result.and_then(|batch| {
106+
let options =
107+
RecordBatchOptions::new().with_row_count(Some(batch.num_rows()));
108+
RecordBatch::try_new_with_options(
109+
Arc::clone(&self.schema),
110+
batch.columns().to_vec(),
111+
&options,
112+
)
113+
.map_err(Into::into)
114+
})
115+
})
116+
})
117+
}
118+
119+
fn size_hint(&self) -> (usize, Option<usize>) {
120+
self.inner.size_hint()
121+
}
122+
}
123+
66124
/// `UnionExec`: `UNION ALL` execution plan.
67125
///
68126
/// `UnionExec` combines multiple inputs with the same schema by
@@ -294,6 +352,21 @@ impl ExecutionPlan for UnionExec {
294352
if partition < input.output_partitioning().partition_count() {
295353
let stream = input.execute(partition, context)?;
296354
debug!("Found a Union partition to execute");
355+
// An input's schema can disagree with `UnionExec`'s own
356+
// (declared) schema -- most commonly a field that is
357+
// nullable in the union's schema (because another input's
358+
// field is nullable) but not on this particular input's
359+
// batches. Re-stamp each batch with the union's schema so
360+
// every partition -- regardless of which input it came from
361+
// -- reports the same schema, which downstream consumers
362+
// that check schema equality across batches (e.g. Arrow's
363+
// C Stream FFI / pyarrow) require. See
364+
// <https://github.com/apache/datafusion/issues/15394>.
365+
let stream = if stream.schema() == self.schema() {
366+
stream
367+
} else {
368+
Box::pin(SchemaConformingStream::new(self.schema(), stream))
369+
};
297370
return Ok(Box::pin(ObservedStream::new(
298371
stream,
299372
baseline_metrics,

0 commit comments

Comments
 (0)