@@ -121,6 +121,21 @@ impl Stream for SchemaConformingStream {
121121 }
122122}
123123
124+ /// Wraps `stream` in a [`SchemaConformingStream`] if its schema disagrees
125+ /// with `schema`, otherwise returns it unchanged. See
126+ /// [`SchemaConformingStream`] and
127+ /// <https://github.com/apache/datafusion/issues/15394>.
128+ fn conform_stream_schema (
129+ schema : SchemaRef ,
130+ stream : SendableRecordBatchStream ,
131+ ) -> SendableRecordBatchStream {
132+ if stream. schema ( ) == schema {
133+ stream
134+ } else {
135+ Box :: pin ( SchemaConformingStream :: new ( schema, stream) )
136+ }
137+ }
138+
124139/// `UnionExec`: `UNION ALL` execution plan.
125140///
126141/// `UnionExec` combines multiple inputs with the same schema by
@@ -352,21 +367,7 @@ impl ExecutionPlan for UnionExec {
352367 if partition < input. output_partitioning ( ) . partition_count ( ) {
353368 let stream = input. execute ( partition, context) ?;
354369 debug ! ( "Found a Union partition to execute" ) ;
355- // An input's schema can disagree with `UnionExec`'s own
356- // (declared) schema -- most commonly a field that is
357- // nullable in the union's schema (because another input's
358- // field is nullable) but not on this particular input's
359- // batches. Re-stamp each batch with the union's schema so
360- // every partition -- regardless of which input it came from
361- // -- reports the same schema, which downstream consumers
362- // that check schema equality across batches (e.g. Arrow's
363- // C Stream FFI / pyarrow) require. See
364- // <https://github.com/apache/datafusion/issues/15394>.
365- let stream = if stream. schema ( ) == self . schema ( ) {
366- stream
367- } else {
368- Box :: pin ( SchemaConformingStream :: new ( self . schema ( ) , stream) )
369- } ;
370+ let stream = conform_stream_schema ( self . schema ( ) , stream) ;
370371 return Ok ( Box :: pin ( ObservedStream :: new (
371372 stream,
372373 baseline_metrics,
@@ -741,7 +742,8 @@ impl ExecutionPlan for InterleaveExec {
741742 let mut input_stream_vec = vec ! [ ] ;
742743 for input in self . inputs . iter ( ) {
743744 if partition < input. output_partitioning ( ) . partition_count ( ) {
744- input_stream_vec. push ( input. execute ( partition, Arc :: clone ( & context) ) ?) ;
745+ let stream = input. execute ( partition, Arc :: clone ( & context) ) ?;
746+ input_stream_vec. push ( conform_stream_schema ( self . schema ( ) , stream) ) ;
745747 } else {
746748 // Do not find a partition to execute
747749 break ;
@@ -1055,6 +1057,50 @@ mod tests {
10551057 Ok ( ( ) )
10561058 }
10571059
1060+ #[ tokio:: test]
1061+ async fn test_interleave_conforms_batch_schema ( ) -> Result < ( ) > {
1062+ // Two inputs agree on the column's type but disagree on nullability;
1063+ // InterleaveExec's declared schema ORs nullability across inputs, so
1064+ // every yielded batch must be re-stamped with that schema. See
1065+ // <https://github.com/apache/datafusion/issues/15394>.
1066+ let task_ctx = Arc :: new ( TaskContext :: default ( ) ) ;
1067+
1068+ let schema_not_null =
1069+ Arc :: new ( Schema :: new ( vec ! [ Field :: new( "a" , DataType :: Int32 , false ) ] ) ) ;
1070+ let batch_not_null = RecordBatch :: try_new (
1071+ Arc :: clone ( & schema_not_null) ,
1072+ vec ! [ Arc :: new( arrow:: array:: Int32Array :: from( vec![ 1 , 2 ] ) ) ] ,
1073+ ) ?;
1074+
1075+ let schema_nullable =
1076+ Arc :: new ( Schema :: new ( vec ! [ Field :: new( "a" , DataType :: Int32 , true ) ] ) ) ;
1077+ let batch_nullable = RecordBatch :: try_new (
1078+ Arc :: clone ( & schema_nullable) ,
1079+ vec ! [ Arc :: new( arrow:: array:: Int32Array :: from( vec![ 3 , 4 ] ) ) ] ,
1080+ ) ?;
1081+
1082+ let hash_expr = vec ! [ col( "a" , schema_not_null. as_ref( ) ) ?] ;
1083+ let left: Arc < dyn ExecutionPlan > = Arc :: new ( RepartitionExec :: try_new (
1084+ TestMemoryExec :: try_new_exec ( & [ vec ! [ batch_not_null] ] , schema_not_null, None ) ?,
1085+ Partitioning :: Hash ( hash_expr. clone ( ) , 1 ) ,
1086+ ) ?) ;
1087+ let right: Arc < dyn ExecutionPlan > = Arc :: new ( RepartitionExec :: try_new (
1088+ TestMemoryExec :: try_new_exec ( & [ vec ! [ batch_nullable] ] , schema_nullable, None ) ?,
1089+ Partitioning :: Hash ( hash_expr, 1 ) ,
1090+ ) ?) ;
1091+
1092+ let interleave: Arc < dyn ExecutionPlan > =
1093+ Arc :: new ( InterleaveExec :: try_new ( vec ! [ left, right] ) ?) ;
1094+ let interleave_schema = interleave. schema ( ) ;
1095+ assert ! ( interleave_schema. field( 0 ) . is_nullable( ) ) ;
1096+
1097+ for batch in collect ( interleave, task_ctx) . await ? {
1098+ assert_eq ! ( batch. schema( ) , interleave_schema) ;
1099+ }
1100+
1101+ Ok ( ( ) )
1102+ }
1103+
10581104 fn stats_merge_inputs ( ) -> ( SchemaRef , Statistics , Statistics , Statistics ) {
10591105 let schema = Arc :: new ( Schema :: new ( vec ! [ Field :: new( "a" , DataType :: UInt32 , true ) ] ) ) ;
10601106
0 commit comments