Skip to content

Commit 131796d

Browse files
committed
fix: also conform InterleaveExec child batches to the declared schema
UnionExec::execute() already re-stamped each child's batches with the union's declared schema whenever nullability disagreed, but the same mismatch was still reachable through InterleaveExec: ensure_distribution can rewrite a UnionExec into an InterleaveExec when the children are interleavable, and InterleaveExec::execute() handed CombinedRecordBatchStream the child streams unchanged. Extract the existing UnionExec fix into a small conform_stream_schema helper and apply it to each InterleaveExec child stream before combining. Also trim the union_nullable.rs module doc-comment down to the invariant and issue link, per review feedback. Addresses review comments from kosiew on PR #23861.
1 parent bed030d commit 131796d

2 files changed

Lines changed: 65 additions & 31 deletions

File tree

‎datafusion/core/tests/sql/union_nullable.rs‎

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -15,21 +15,9 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
//! Regression tests for `UNION ALL` between inputs where the same column is
19-
//! nullable on one side and `NOT NULL` on the other.
20-
//!
21-
//! DataFusion's analyzer computes the union's output schema by OR-ing the
22-
//! nullability of each input's field (`coerce_union_schema`), so the
23-
//! *declared* output field is nullable whenever any leg is. But before this
24-
//! fix, `UnionExec::execute` handed out each child's `RecordBatch`es
25-
//! completely unchanged, so a leg whose column was already `NOT NULL` (and
26-
//! therefore needed no `CAST`) kept producing batches with a `NOT NULL`
27-
//! field, contradicting the union's own declared (nullable) schema. Any
28-
//! consumer that checks schema equality across batches from the same
29-
//! stream -- e.g. `pyarrow.Table.from_batches` via the Arrow C Stream FFI
30-
//! used by the `datafusion` Python bindings -- then rejects the stream with
31-
//! `ArrowInvalid: Schema at index N was different`, even though every
32-
//! individual `SELECT` runs fine on its own. See
18+
//! Regression tests asserting that every batch yielded by a `UNION ALL`
19+
//! reports the union's own declared schema, even when the same column is
20+
//! `NOT NULL` on one leg and nullable on another. See
3321
//! <https://github.com/apache/datafusion/issues/15394>.
3422
3523
use std::sync::Arc;

‎datafusion/physical-plan/src/union.rs‎

Lines changed: 62 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,21 @@ impl Stream for SchemaConformingStream {
121121
}
122122
}
123123

124+
/// Wraps `stream` in a [`SchemaConformingStream`] if its schema disagrees
125+
/// with `schema`, otherwise returns it unchanged. See
126+
/// [`SchemaConformingStream`] and
127+
/// <https://github.com/apache/datafusion/issues/15394>.
128+
fn conform_stream_schema(
129+
schema: SchemaRef,
130+
stream: SendableRecordBatchStream,
131+
) -> SendableRecordBatchStream {
132+
if stream.schema() == schema {
133+
stream
134+
} else {
135+
Box::pin(SchemaConformingStream::new(schema, stream))
136+
}
137+
}
138+
124139
/// `UnionExec`: `UNION ALL` execution plan.
125140
///
126141
/// `UnionExec` combines multiple inputs with the same schema by
@@ -352,21 +367,7 @@ impl ExecutionPlan for UnionExec {
352367
if partition < input.output_partitioning().partition_count() {
353368
let stream = input.execute(partition, context)?;
354369
debug!("Found a Union partition to execute");
355-
// An input's schema can disagree with `UnionExec`'s own
356-
// (declared) schema -- most commonly a field that is
357-
// nullable in the union's schema (because another input's
358-
// field is nullable) but not on this particular input's
359-
// batches. Re-stamp each batch with the union's schema so
360-
// every partition -- regardless of which input it came from
361-
// -- reports the same schema, which downstream consumers
362-
// that check schema equality across batches (e.g. Arrow's
363-
// C Stream FFI / pyarrow) require. See
364-
// <https://github.com/apache/datafusion/issues/15394>.
365-
let stream = if stream.schema() == self.schema() {
366-
stream
367-
} else {
368-
Box::pin(SchemaConformingStream::new(self.schema(), stream))
369-
};
370+
let stream = conform_stream_schema(self.schema(), stream);
370371
return Ok(Box::pin(ObservedStream::new(
371372
stream,
372373
baseline_metrics,
@@ -741,7 +742,8 @@ impl ExecutionPlan for InterleaveExec {
741742
let mut input_stream_vec = vec![];
742743
for input in self.inputs.iter() {
743744
if partition < input.output_partitioning().partition_count() {
744-
input_stream_vec.push(input.execute(partition, Arc::clone(&context))?);
745+
let stream = input.execute(partition, Arc::clone(&context))?;
746+
input_stream_vec.push(conform_stream_schema(self.schema(), stream));
745747
} else {
746748
// Do not find a partition to execute
747749
break;
@@ -1055,6 +1057,50 @@ mod tests {
10551057
Ok(())
10561058
}
10571059

1060+
#[tokio::test]
1061+
async fn test_interleave_conforms_batch_schema() -> Result<()> {
1062+
// Two inputs agree on the column's type but disagree on nullability;
1063+
// InterleaveExec's declared schema ORs nullability across inputs, so
1064+
// every yielded batch must be re-stamped with that schema. See
1065+
// <https://github.com/apache/datafusion/issues/15394>.
1066+
let task_ctx = Arc::new(TaskContext::default());
1067+
1068+
let schema_not_null =
1069+
Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
1070+
let batch_not_null = RecordBatch::try_new(
1071+
Arc::clone(&schema_not_null),
1072+
vec![Arc::new(arrow::array::Int32Array::from(vec![1, 2]))],
1073+
)?;
1074+
1075+
let schema_nullable =
1076+
Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));
1077+
let batch_nullable = RecordBatch::try_new(
1078+
Arc::clone(&schema_nullable),
1079+
vec![Arc::new(arrow::array::Int32Array::from(vec![3, 4]))],
1080+
)?;
1081+
1082+
let hash_expr = vec![col("a", schema_not_null.as_ref())?];
1083+
let left: Arc<dyn ExecutionPlan> = Arc::new(RepartitionExec::try_new(
1084+
TestMemoryExec::try_new_exec(&[vec![batch_not_null]], schema_not_null, None)?,
1085+
Partitioning::Hash(hash_expr.clone(), 1),
1086+
)?);
1087+
let right: Arc<dyn ExecutionPlan> = Arc::new(RepartitionExec::try_new(
1088+
TestMemoryExec::try_new_exec(&[vec![batch_nullable]], schema_nullable, None)?,
1089+
Partitioning::Hash(hash_expr, 1),
1090+
)?);
1091+
1092+
let interleave: Arc<dyn ExecutionPlan> =
1093+
Arc::new(InterleaveExec::try_new(vec![left, right])?);
1094+
let interleave_schema = interleave.schema();
1095+
assert!(interleave_schema.field(0).is_nullable());
1096+
1097+
for batch in collect(interleave, task_ctx).await? {
1098+
assert_eq!(batch.schema(), interleave_schema);
1099+
}
1100+
1101+
Ok(())
1102+
}
1103+
10581104
fn stats_merge_inputs() -> (SchemaRef, Statistics, Statistics, Statistics) {
10591105
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::UInt32, true)]));
10601106

0 commit comments

Comments
 (0)