@@ -42,104 +42,92 @@ use crate::filter_pushdown::{
4242 FilterPushdownPropagation , PushedDown ,
4343} ;
4444use crate :: metrics:: BaselineMetrics ;
45- use crate :: projection:: { ProjectionExec , make_with_child} ;
45+ use crate :: projection:: { ProjectionExec , ProjectionExpr , make_with_child} ;
4646use crate :: statistics:: { ChildStats , StatisticsArgs } ;
4747use crate :: stream:: ObservedStream ;
4848
49- use arrow:: array:: RecordBatchOptions ;
5049use arrow:: datatypes:: { Field , Schema , SchemaRef } ;
5150use arrow:: record_batch:: RecordBatch ;
5251use datafusion_common:: config:: ConfigOptions ;
5352use datafusion_common:: stats:: NdvFallback ;
5453use datafusion_common:: tree_node:: TreeNodeRecursion ;
5554use datafusion_common:: {
56- Result , assert_or_internal_err, exec_err, internal_datafusion_err,
55+ Result , assert_or_internal_err, exec_err, internal_datafusion_err, plan_err ,
5756} ;
5857use datafusion_execution:: TaskContext ;
58+ use datafusion_physical_expr:: expressions:: { CastExpr , Column } ;
5959use datafusion_physical_expr:: {
6060 EquivalenceProperties , PhysicalExpr , calculate_union, conjunction,
6161} ;
6262
63- use futures:: { Stream , StreamExt } ;
63+ use futures:: Stream ;
6464use itertools:: Itertools ;
6565use log:: { debug, trace, warn} ;
6666use tokio:: macros:: support:: thread_rng_n;
6767
68- /// Wraps a child stream so that every batch it yields is re-stamped with
69- /// `schema` instead of the child's own schema.
68+ /// Coerces `input`'s output schema to exactly `schema` via a `ProjectionExec`
69+ /// that re-stamps each column with the union's merged field (same
70+ /// `DataType`, but the union's merged nullability/name/metadata), or returns
71+ /// `input` unchanged if its schema already matches. [`UnionExec::try_new`]
72+ /// and [`InterleaveExec::try_new`] call this on every child, so the coercion
73+ /// is visible in the plan tree (e.g. in `EXPLAIN`) instead of happening
74+ /// invisibly inside the union operator's own `execute()`.
7075///
71- /// This is used by both [`UnionExec`] and [`InterleaveExec`] when a child's
72- /// output schema disagrees with the operator's declared output schema --
73- /// in practice this only happens for nullability (the declared schema is
74- /// nullable wherever *any* input's field is, but casts are only inserted
75- /// between inputs when the *type* differs, not when only nullability
76- /// does). For [`UnionExec`], [`UnionExec::try_new`] guarantees this: it
77- /// calls `calculate_union`, which rejects any input whose field data types
78- /// don't match the computed union schema. [`InterleaveExec::try_new`] does
79- /// not repeat that check -- its inputs are only ever produced by the
80- /// optimizer rewriting an already-validated `UnionExec`, whose children's
81- /// types are therefore already known to agree -- but if this wrapper ever
82- /// did see a genuine data type mismatch (e.g. from a hand-built
83- /// `InterleaveExec`), `RecordBatch::try_new_with_options` below reports it
84- /// as an error rather than silently yielding a corrupt batch.
85- struct SchemaConformingStream {
86- schema : SchemaRef ,
87- inner : SendableRecordBatchStream ,
88- }
89-
90- impl SchemaConformingStream {
91- fn new ( schema : SchemaRef , inner : SendableRecordBatchStream ) -> Self {
92- Self { schema , inner }
93- }
94- }
95-
96- impl RecordBatchStream for SchemaConformingStream {
97- fn schema ( & self ) -> SchemaRef {
98- Arc :: clone ( & self . schema )
99- }
100- }
101-
102- impl Stream for SchemaConformingStream {
103- type Item = Result < RecordBatch > ;
104-
105- fn poll_next (
106- mut self : Pin < & mut Self > ,
107- cx : & mut Context < ' _ > ,
108- ) -> Poll < Option < Self :: Item > > {
109- self . inner . poll_next_unpin ( cx ) . map ( |opt| {
110- opt . map ( |batch_result| {
111- batch_result . and_then ( |batch| {
112- let options =
113- RecordBatchOptions :: new ( ) . with_row_count ( Some ( batch . num_rows ( ) ) ) ;
114- RecordBatch :: try_new_with_options (
115- Arc :: clone ( & self . schema ) ,
116- batch . columns ( ) . to_vec ( ) ,
117- & options ,
118- )
119- . map_err ( Into :: into )
120- } )
76+ /// A column whose `DataType` doesn't already match the union's is a genuine
77+ /// data type mismatch (as opposed to a nullability/name/metadata-only one),
78+ /// and is rejected eagerly here rather than silently cast or deferred to a
79+ /// runtime failure -- this only ever changes a column's declared schema,
80+ /// never its values.
81+ ///
82+ /// Casting a column to its own `DataType` (only the `Field`'s nullability,
83+ /// name, or metadata changes) is a zero-copy relabeling: the cast kernel's
84+ /// same-type fast path (`cast_array_by_name`) just clones the `Arc<dyn
85+ /// Array>`, so this carries no runtime overhead over the schema it replaces.
86+ ///
87+ /// See <https://github.com/apache/datafusion/issues/15394>.
88+ fn coerce_schema (
89+ input : Arc < dyn ExecutionPlan > ,
90+ schema : & SchemaRef ,
91+ ) -> Result < Arc < dyn ExecutionPlan > > {
92+ let input_schema = input . schema ( ) ;
93+ if & input_schema == schema {
94+ return Ok ( input ) ;
95+ }
96+
97+ let exprs = input_schema
98+ . fields ( )
99+ . iter ( )
100+ . zip ( schema . fields ( ) )
101+ . enumerate ( )
102+ . map ( | ( i , ( input_field , target_field ) ) | {
103+ if input_field . data_type ( ) != target_field . data_type ( ) {
104+ return plan_err ! (
105+ "UnionExec/InterleaveExec requires all inputs to have the same \
106+ data type per column; column {i} has type {} in one input, but \
107+ the union schema expects {}" ,
108+ input_field . data_type ( ) ,
109+ target_field . data_type ( )
110+ ) ;
111+ }
112+ let column : Arc < dyn PhysicalExpr > =
113+ Arc :: new ( Column :: new ( input_field . name ( ) , i ) ) ;
114+ let expr = if input_field == target_field {
115+ column
116+ } else {
117+ Arc :: new ( CastExpr :: new_with_target_field (
118+ column ,
119+ Arc :: clone ( target_field ) ,
120+ None ,
121+ ) ) as Arc < dyn PhysicalExpr >
122+ } ;
123+ Ok ( ProjectionExpr {
124+ expr ,
125+ alias : target_field . name ( ) . clone ( ) ,
121126 } )
122127 } )
123- }
128+ . collect :: < Result < Vec < _ > > > ( ) ? ;
124129
125- fn size_hint ( & self ) -> ( usize , Option < usize > ) {
126- self . inner . size_hint ( )
127- }
128- }
129-
130- /// Wraps `stream` in a [`SchemaConformingStream`] if its schema disagrees
131- /// with `schema`, otherwise returns it unchanged. See
132- /// [`SchemaConformingStream`] and
133- /// <https://github.com/apache/datafusion/issues/15394>.
134- fn conform_stream_schema (
135- schema : SchemaRef ,
136- stream : SendableRecordBatchStream ,
137- ) -> SendableRecordBatchStream {
138- if stream. schema ( ) == schema {
139- stream
140- } else {
141- Box :: pin ( SchemaConformingStream :: new ( schema, stream) )
142- }
130+ Ok ( Arc :: new ( ProjectionExec :: try_new ( exprs, input) ?) )
143131}
144132
145133/// `UnionExec`: `UNION ALL` execution plan.
@@ -209,6 +197,10 @@ impl UnionExec {
209197 // The schema of the inputs and the union schema is consistent when:
210198 // - They have the same number of fields, and
211199 // - Their fields have same types at the same indices.
200+ let inputs = inputs
201+ . into_iter ( )
202+ . map ( |input| coerce_schema ( input, & schema) )
203+ . collect :: < Result < Vec < _ > > > ( ) ?;
212204 let cache = Self :: compute_properties ( & inputs, schema) ?;
213205 Ok ( Arc :: new ( UnionExec {
214206 inputs,
@@ -380,7 +372,6 @@ impl ExecutionPlan for UnionExec {
380372 if partition < input. output_partitioning ( ) . partition_count ( ) {
381373 let stream = input. execute ( partition, context) ?;
382374 debug ! ( "Found a Union partition to execute" ) ;
383- let stream = conform_stream_schema ( self . schema ( ) , stream) ;
384375 return Ok ( Box :: pin ( ObservedStream :: new (
385376 stream,
386377 baseline_metrics,
@@ -650,7 +641,12 @@ impl InterleaveExec {
650641 can_interleave( inputs. iter( ) ) ,
651642 "Not all InterleaveExec children have a consistent hash or range partitioning"
652643 ) ;
653- let cache = Self :: compute_properties ( & inputs) ?;
644+ let schema = union_schema ( & inputs) ?;
645+ let inputs = inputs
646+ . into_iter ( )
647+ . map ( |input| coerce_schema ( input, & schema) )
648+ . collect :: < Result < Vec < _ > > > ( ) ?;
649+ let cache = Self :: compute_properties ( & inputs, schema) ?;
654650 Ok ( InterleaveExec {
655651 inputs,
656652 metrics : ExecutionPlanMetricsSet :: new ( ) ,
@@ -664,8 +660,10 @@ impl InterleaveExec {
664660 }
665661
666662 /// This function creates the cache object that stores the plan properties such as schema, equivalence properties, ordering, partitioning, etc.
667- fn compute_properties ( inputs : & [ Arc < dyn ExecutionPlan > ] ) -> Result < PlanProperties > {
668- let schema = union_schema ( inputs) ?;
663+ fn compute_properties (
664+ inputs : & [ Arc < dyn ExecutionPlan > ] ,
665+ schema : SchemaRef ,
666+ ) -> Result < PlanProperties > {
669667 let eq_properties = EquivalenceProperties :: new ( schema) ;
670668 // Get output partitioning:
671669 let output_partitioning = inputs[ 0 ] . output_partitioning ( ) . clone ( ) ;
@@ -763,7 +761,7 @@ impl ExecutionPlan for InterleaveExec {
763761 for input in self . inputs . iter ( ) {
764762 if partition < input. output_partitioning ( ) . partition_count ( ) {
765763 let stream = input. execute ( partition, Arc :: clone ( & context) ) ?;
766- input_stream_vec. push ( conform_stream_schema ( self . schema ( ) , stream) ) ;
764+ input_stream_vec. push ( stream) ;
767765 } else {
768766 // Do not find a partition to execute
769767 break ;
@@ -1277,6 +1275,107 @@ mod tests {
12771275 Ok ( ( ) )
12781276 }
12791277
1278+ #[ test]
1279+ fn test_union_partition_statistics_with_mismatched_nullability ( ) -> Result < ( ) > {
1280+ // Regression test for the `ProjectionExec` wrapper `UnionExec::try_new`
1281+ // inserts above the non-nullable leg here (via `coerce_schema`):
1282+ // exact column statistics (min/max/null/distinct/sum/byte_size) must
1283+ // still make it through the wrapper's same-type `CastExpr`, not get
1284+ // poisoned into `Absent` the way a generic (type-changing) cast's
1285+ // statistics would be.
1286+ let ( _, left, right, expected) = stats_merge_inputs ( ) ;
1287+
1288+ // `total_byte_size` differs from the plain-merge fixture (52): the
1289+ // wrapper is a `ProjectionExec`, whose `statistics_from_inputs`
1290+ // recomputes `total_byte_size` from the (unchanged) schema's row
1291+ // width times row count, rather than trusting the wrapped leg's own
1292+ // self-reported total -- still `Exact`, just derived differently.
1293+ // left: 5 rows * 4 bytes (UInt32) = 20 (was 23); right is untouched
1294+ // (already nullable, so `coerce_schema` doesn't wrap it): 20 + 29 = 49.
1295+ let expected = expected. with_total_byte_size ( Precision :: Exact ( 49 ) ) ;
1296+
1297+ let non_nullable_schema =
1298+ Schema :: new ( vec ! [ Field :: new( "a" , DataType :: UInt32 , false ) ] ) ;
1299+ let nullable_schema = Schema :: new ( vec ! [ Field :: new( "a" , DataType :: UInt32 , true ) ] ) ;
1300+
1301+ let left: Arc < dyn ExecutionPlan > =
1302+ Arc :: new ( StatisticsExec :: new ( left, non_nullable_schema) ) ;
1303+ let right: Arc < dyn ExecutionPlan > =
1304+ Arc :: new ( StatisticsExec :: new ( right, nullable_schema) ) ;
1305+
1306+ let union = UnionExec :: try_new ( vec ! [ left, right] ) ?;
1307+ let stats =
1308+ StatisticsContext :: new ( ) . compute ( union. as_ref ( ) , & StatisticsArgs :: new ( ) ) ?;
1309+
1310+ assert_eq ! ( stats. as_ref( ) , & expected) ;
1311+ Ok ( ( ) )
1312+ }
1313+
1314+ #[ tokio:: test]
1315+ async fn test_coerce_schema_no_op_when_already_matching ( ) -> Result < ( ) > {
1316+ let schema_not_null =
1317+ Arc :: new ( Schema :: new ( vec ! [ Field :: new( "a" , DataType :: Int32 , false ) ] ) ) ;
1318+ let input: Arc < dyn ExecutionPlan > =
1319+ TestMemoryExec :: try_new_exec ( & [ vec ! [ ] ] , Arc :: clone ( & schema_not_null) , None ) ?;
1320+
1321+ let coerced = coerce_schema ( Arc :: clone ( & input) , & schema_not_null) ?;
1322+ assert ! ( Arc :: ptr_eq( & coerced, & input) ) ;
1323+
1324+ Ok ( ( ) )
1325+ }
1326+
1327+ #[ tokio:: test]
1328+ async fn test_coerce_schema_casts_only_nullability ( ) -> Result < ( ) > {
1329+ // Mismatched nullability: the input gets wrapped in a `ProjectionExec`
1330+ // whose `CastExpr` re-stamps the column with the target's `Field`
1331+ // (same `DataType`, so this is a zero-copy relabeling, not a real cast).
1332+ let schema_not_null =
1333+ Arc :: new ( Schema :: new ( vec ! [ Field :: new( "a" , DataType :: Int32 , false ) ] ) ) ;
1334+ let batch_not_null = RecordBatch :: try_new (
1335+ Arc :: clone ( & schema_not_null) ,
1336+ vec ! [ Arc :: new( arrow:: array:: Int32Array :: from( vec![ 1 , 2 ] ) ) ] ,
1337+ ) ?;
1338+ let input: Arc < dyn ExecutionPlan > = TestMemoryExec :: try_new_exec (
1339+ & [ vec ! [ batch_not_null] ] ,
1340+ Arc :: clone ( & schema_not_null) ,
1341+ None ,
1342+ ) ?;
1343+
1344+ let nullable_schema =
1345+ Arc :: new ( Schema :: new ( vec ! [ Field :: new( "a" , DataType :: Int32 , true ) ] ) ) ;
1346+ let coerced = coerce_schema ( Arc :: clone ( & input) , & nullable_schema) ?;
1347+ assert_eq ! ( & coerced. schema( ) , & nullable_schema) ;
1348+ let plan_str = crate :: displayable ( coerced. as_ref ( ) )
1349+ . indent ( true )
1350+ . to_string ( ) ;
1351+ assert ! (
1352+ plan_str. contains( "CAST" ) ,
1353+ "expected a CAST in the coerced plan:\n {plan_str}"
1354+ ) ;
1355+
1356+ let task_ctx = Arc :: new ( TaskContext :: default ( ) ) ;
1357+ let batches = collect ( coerced, task_ctx) . await ?;
1358+ assert_eq ! ( batches. len( ) , 1 ) ;
1359+ assert_eq ! ( batches[ 0 ] . schema( ) , nullable_schema) ;
1360+
1361+ Ok ( ( ) )
1362+ }
1363+
1364+ #[ test]
1365+ fn test_coerce_schema_rejects_genuine_type_mismatch ( ) -> Result < ( ) > {
1366+ let schema_int =
1367+ Arc :: new ( Schema :: new ( vec ! [ Field :: new( "a" , DataType :: Int32 , false ) ] ) ) ;
1368+ let input: Arc < dyn ExecutionPlan > =
1369+ TestMemoryExec :: try_new_exec ( & [ vec ! [ ] ] , Arc :: clone ( & schema_int) , None ) ?;
1370+
1371+ let schema_utf8 =
1372+ Arc :: new ( Schema :: new ( vec ! [ Field :: new( "a" , DataType :: Utf8 , false ) ] ) ) ;
1373+ let err = coerce_schema ( input, & schema_utf8) . unwrap_err ( ) ;
1374+ assert ! ( err. to_string( ) . contains( "same data type per column" ) ) ;
1375+
1376+ Ok ( ( ) )
1377+ }
1378+
12801379 #[ test]
12811380 fn test_interleave_partition_statistics_uses_shared_statistics_merge ( ) -> Result < ( ) > {
12821381 let ( schema, left, right, expected) = stats_merge_inputs ( ) ;
0 commit comments