@@ -49,7 +49,9 @@ use crate::execution::{
4949} ;
5050use crate :: jvm_bridge:: { jni_call, JVMClasses , ShufflePartitionPusher } ;
5151use arrow:: compute:: CastOptions ;
52- use arrow:: datatypes:: { DataType , Field , FieldRef , Schema , TimeUnit , DECIMAL128_MAX_PRECISION } ;
52+ use arrow:: datatypes:: {
53+ DataType , Field , FieldRef , Fields , Schema , TimeUnit , DECIMAL128_MAX_PRECISION ,
54+ } ;
5355use arrow:: ffi_stream:: FFI_ArrowArrayStream ;
5456use datafusion:: functions_aggregate:: bit_and_or_xor:: { bit_and_udaf, bit_or_udaf, bit_xor_udaf} ;
5557use datafusion:: functions_aggregate:: count:: count_udaf;
@@ -218,6 +220,39 @@ fn make_all_fields_nullable(data_type: &DataType) -> DataType {
218220 }
219221}
220222
223+ /// Return a copy of a `Map` type with only the outer entries `value` field marked nullable, keeping
224+ /// the key field non-nullable and every nested key/value type byte-for-byte unchanged. Any non-`Map`
225+ /// type is returned unchanged.
226+ ///
227+ /// This is the shallow counterpart of `make_all_fields_nullable`, used for `map_entries`.
228+ /// `map_entries` reuses the input map's entries array as its output list values but declares that
229+ /// element's `value` field nullable (`Struct(key non-null, value nullable)`), deriving both nested
230+ /// types verbatim from the input. Arrow's `GenericListArray::try_new` compares the declared element
231+ /// field's type against the reused values array's type in full, so the ONLY field that can mismatch
232+ /// is the entries `value` field's own `nullable` flag; widening just that field is sufficient.
233+ /// Recursing into nested types (as `make_all_fields_nullable` does) would additionally flip, say, a
234+ /// nested `Map`'s `valueContainsNull`, which then diverges from the Spark-serialized `return_type`
235+ /// and makes a downstream `make_array` see unequal map types and panic.
236+ fn widen_map_entry_value_nullable ( data_type : & DataType ) -> DataType {
237+ match data_type {
238+ DataType :: Map ( entries, sorted) => match entries. data_type ( ) {
239+ DataType :: Struct ( kv) if kv. len ( ) == 2 && !kv[ 1 ] . is_nullable ( ) => {
240+ let new_value = Arc :: new ( kv[ 1 ] . as_ref ( ) . clone ( ) . with_nullable ( true ) ) ;
241+ let new_kv: Fields = vec ! [ Arc :: clone( & kv[ 0 ] ) , new_value] . into ( ) ;
242+ let new_entries = Arc :: new (
243+ entries
244+ . as_ref ( )
245+ . clone ( )
246+ . with_data_type ( DataType :: Struct ( new_kv) ) ,
247+ ) ;
248+ DataType :: Map ( new_entries, * sorted)
249+ }
250+ _ => data_type. clone ( ) ,
251+ } ,
252+ other => other. clone ( ) ,
253+ }
254+ }
255+
221256/// If `expr` evaluates to `Timestamp(_, Some(_))` against `schema`, wrap it in a
222257/// metadata-only cast to `Timestamp(_, None)`. This is required because
223258/// DataFusion's `SortMergeJoinExec` comparator only supports timezone-less
@@ -3490,6 +3525,17 @@ impl PhysicalPlanner {
34903525 . collect :: < Result < Vec < _ > , _ > > ( ) ?;
34913526
34923527 let fun_name = & expr. func ;
3528+ // `map_entries` needs its argument's entry `value` field widened to nullable first (only
3529+ // that outer field). See `widen_map_entry_value_nullable`.
3530+ let args = if fun_name == "map_entries" {
3531+ args. into_iter ( )
3532+ . map ( |arg| {
3533+ Self :: coerce_child_to ( arg, & input_schema, widen_map_entry_value_nullable)
3534+ } )
3535+ . collect :: < Result < Vec < _ > , ExecutionError > > ( ) ?
3536+ } else {
3537+ args
3538+ } ;
34933539 let input_expr_types = args
34943540 . iter ( )
34953541 . map ( |x| x. data_type ( input_schema. as_ref ( ) ) )
@@ -3653,6 +3699,26 @@ impl PhysicalPlanner {
36533699 Ok ( Arc :: new ( CastExpr :: new ( child, nullable_type, None ) ) )
36543700 }
36553701
3702+ /// Casts `child` so its type matches `widen(child_type)`, wrapping it in a `CastExpr` only when
3703+ /// the widened type differs. Used for the `map_entries` argument widening (with
3704+ /// `widen_map_entry_value_nullable`). Unlike `coerce_collect_child_nullability`, which casts
3705+ /// unconditionally as a normalization barrier for aggregate accumulators, this casts only when
3706+ /// the type actually changes. That is sufficient for the scalar `map_entries`, whose reused
3707+ /// entries array only needs its declared element type to line up.
3708+ fn coerce_child_to (
3709+ child : Arc < dyn PhysicalExpr > ,
3710+ schema : & SchemaRef ,
3711+ widen : impl Fn ( & DataType ) -> DataType ,
3712+ ) -> Result < Arc < dyn PhysicalExpr > , ExecutionError > {
3713+ let child_type = child. data_type ( schema. as_ref ( ) ) ?;
3714+ let widened = widen ( & child_type) ;
3715+ if child_type. equals_datatype ( & widened) {
3716+ Ok ( child)
3717+ } else {
3718+ Ok ( Arc :: new ( CastExpr :: new ( child, widened, None ) ) )
3719+ }
3720+ }
3721+
36563722 fn create_aggr_func_expr (
36573723 name : & str ,
36583724 schema : SchemaRef ,
0 commit comments