@@ -53,6 +53,8 @@ use arrow::{
5353 } ,
5454} ;
5555use arrow_array:: { ArrowNativeTypeOp , Scalar } ;
56+ use arrow_buffer:: Buffer ;
57+ use arrow_schema:: { UnionFields , UnionMode } ;
5658
5759pub use struct_builder:: ScalarStructBuilder ;
5860
@@ -275,6 +277,11 @@ pub enum ScalarValue {
275277 DurationMicrosecond ( Option < i64 > ) ,
276278 /// Duration in nanoseconds
277279 DurationNanosecond ( Option < i64 > ) ,
280+ /// A nested datatype that can represent slots of differing types. Components:
281+ /// `.0`: a tuple of union `type_id` and the single value held by this Scalar
282+ /// `.1`: the list of fields, zero-to-one of which will by set in `.0`
283+ /// `.2`: the physical storage of the source/destination UnionArray from which this Scalar came
284+ Union ( Option < ( i8 , Box < ScalarValue > ) > , UnionFields , UnionMode ) ,
278285 /// Dictionary type: index type and value
279286 Dictionary ( Box < DataType > , Box < ScalarValue > ) ,
280287}
@@ -375,6 +382,10 @@ impl PartialEq for ScalarValue {
375382 ( IntervalDayTime ( _) , _) => false ,
376383 ( IntervalMonthDayNano ( v1) , IntervalMonthDayNano ( v2) ) => v1. eq ( v2) ,
377384 ( IntervalMonthDayNano ( _) , _) => false ,
385+ ( Union ( val1, fields1, mode1) , Union ( val2, fields2, mode2) ) => {
386+ val1. eq ( val2) && fields1. eq ( fields2) && mode1. eq ( mode2)
387+ }
388+ ( Union ( _, _, _) , _) => false ,
378389 ( Dictionary ( k1, v1) , Dictionary ( k2, v2) ) => k1. eq ( k2) && v1. eq ( v2) ,
379390 ( Dictionary ( _, _) , _) => false ,
380391 ( Null , Null ) => true ,
@@ -500,6 +511,14 @@ impl PartialOrd for ScalarValue {
500511 ( DurationMicrosecond ( _) , _) => None ,
501512 ( DurationNanosecond ( v1) , DurationNanosecond ( v2) ) => v1. partial_cmp ( v2) ,
502513 ( DurationNanosecond ( _) , _) => None ,
514+ ( Union ( v1, t1, m1) , Union ( v2, t2, m2) ) => {
515+ if t1. eq ( t2) && m1. eq ( m2) {
516+ v1. partial_cmp ( v2)
517+ } else {
518+ None
519+ }
520+ }
521+ ( Union ( _, _, _) , _) => None ,
503522 ( Dictionary ( k1, v1) , Dictionary ( k2, v2) ) => {
504523 // Don't compare if the key types don't match (it is effectively a different datatype)
505524 if k1 == k2 {
@@ -663,6 +682,11 @@ impl std::hash::Hash for ScalarValue {
663682 IntervalYearMonth ( v) => v. hash ( state) ,
664683 IntervalDayTime ( v) => v. hash ( state) ,
665684 IntervalMonthDayNano ( v) => v. hash ( state) ,
685+ Union ( v, t, m) => {
686+ v. hash ( state) ;
687+ t. hash ( state) ;
688+ m. hash ( state) ;
689+ }
666690 Dictionary ( k, v) => {
667691 k. hash ( state) ;
668692 v. hash ( state) ;
@@ -1093,6 +1117,7 @@ impl ScalarValue {
10931117 ScalarValue :: DurationNanosecond ( _) => {
10941118 DataType :: Duration ( TimeUnit :: Nanosecond )
10951119 }
1120+ ScalarValue :: Union ( _, fields, mode) => DataType :: Union ( fields. clone ( ) , * mode) ,
10961121 ScalarValue :: Dictionary ( k, v) => {
10971122 DataType :: Dictionary ( k. clone ( ) , Box :: new ( v. data_type ( ) ) )
10981123 }
@@ -1292,6 +1317,7 @@ impl ScalarValue {
12921317 ScalarValue :: DurationMillisecond ( v) => v. is_none ( ) ,
12931318 ScalarValue :: DurationMicrosecond ( v) => v. is_none ( ) ,
12941319 ScalarValue :: DurationNanosecond ( v) => v. is_none ( ) ,
1320+ ScalarValue :: Union ( v, _, _) => v. is_none ( ) ,
12951321 ScalarValue :: Dictionary ( _, v) => v. is_null ( ) ,
12961322 }
12971323 }
@@ -2087,6 +2113,39 @@ impl ScalarValue {
20872113 e,
20882114 size
20892115 ) ,
2116+ ScalarValue :: Union ( value, fields, _mode) => match value {
2117+ Some ( ( v_id, value) ) => {
2118+ let mut field_type_ids = Vec :: < i8 > :: with_capacity ( fields. len ( ) ) ;
2119+ let mut child_arrays =
2120+ Vec :: < ( Field , ArrayRef ) > :: with_capacity ( fields. len ( ) ) ;
2121+ for ( f_id, field) in fields. iter ( ) {
2122+ let ar = if f_id == * v_id {
2123+ value. to_array_of_size ( size) ?
2124+ } else {
2125+ let dt = field. data_type ( ) ;
2126+ new_null_array ( dt, size)
2127+ } ;
2128+ let field = ( * * field) . clone ( ) ;
2129+ child_arrays. push ( ( field, ar) ) ;
2130+ field_type_ids. push ( f_id) ;
2131+ }
2132+ let type_ids = repeat ( * v_id) . take ( size) . collect :: < Vec < _ > > ( ) ;
2133+ let type_ids = Buffer :: from_slice_ref ( type_ids) ;
2134+ let value_offsets: Option < Buffer > = None ;
2135+ let ar = UnionArray :: try_new (
2136+ field_type_ids. as_slice ( ) ,
2137+ type_ids,
2138+ value_offsets,
2139+ child_arrays,
2140+ )
2141+ . map_err ( |e| DataFusionError :: ArrowError ( e, None ) ) ?;
2142+ Arc :: new ( ar)
2143+ }
2144+ None => {
2145+ let dt = self . data_type ( ) ;
2146+ new_null_array ( & dt, size)
2147+ }
2148+ } ,
20902149 ScalarValue :: Dictionary ( key_type, v) => {
20912150 // values array is one element long (the value)
20922151 match key_type. as_ref ( ) {
@@ -2626,6 +2685,9 @@ impl ScalarValue {
26262685 ScalarValue :: DurationNanosecond ( val) => {
26272686 eq_array_primitive ! ( array, index, DurationNanosecondArray , val) ?
26282687 }
2688+ ScalarValue :: Union ( _, _, _) => {
2689+ return _not_impl_err ! ( "Union is not supported yet" )
2690+ }
26292691 ScalarValue :: Dictionary ( key_type, v) => {
26302692 let ( values_array, values_index) = match key_type. as_ref ( ) {
26312693 DataType :: Int8 => get_dict_value :: < Int8Type > ( array, index) ?,
@@ -2703,6 +2765,15 @@ impl ScalarValue {
27032765 ScalarValue :: LargeList ( arr) => arr. get_array_memory_size ( ) ,
27042766 ScalarValue :: FixedSizeList ( arr) => arr. get_array_memory_size ( ) ,
27052767 ScalarValue :: Struct ( arr) => arr. get_array_memory_size ( ) ,
2768+ ScalarValue :: Union ( vals, fields, _mode) => {
2769+ vals. as_ref ( )
2770+ . map ( |( _id, sv) | sv. size ( ) - std:: mem:: size_of_val ( sv) )
2771+ . unwrap_or_default ( )
2772+ // `fields` is boxed, so it is NOT already included in `self`
2773+ + std:: mem:: size_of_val ( fields)
2774+ + ( std:: mem:: size_of :: < Field > ( ) * fields. len ( ) )
2775+ + fields. iter ( ) . map ( |( _idx, field) | field. size ( ) - std:: mem:: size_of_val ( field) ) . sum :: < usize > ( )
2776+ }
27062777 ScalarValue :: Dictionary ( dt, sv) => {
27072778 // `dt` and `sv` are boxed, so they are NOT already included in `self`
27082779 dt. size ( ) + sv. size ( )
@@ -3048,6 +3119,9 @@ impl TryFrom<&DataType> for ScalarValue {
30483119 . to_owned ( )
30493120 . into ( ) ,
30503121 ) ,
3122+ DataType :: Union ( fields, mode) => {
3123+ ScalarValue :: Union ( None , fields. clone ( ) , * mode)
3124+ }
30513125 DataType :: Null => ScalarValue :: Null ,
30523126 _ => {
30533127 return _not_impl_err ! (
@@ -3164,6 +3238,10 @@ impl fmt::Display for ScalarValue {
31643238 . join( "," )
31653239 ) ?
31663240 }
3241+ ScalarValue :: Union ( val, _fields, _mode) => match val {
3242+ Some ( ( id, val) ) => write ! ( f, "{}:{}" , id, val) ?,
3243+ None => write ! ( f, "NULL" ) ?,
3244+ } ,
31673245 ScalarValue :: Dictionary ( _k, v) => write ! ( f, "{v}" ) ?,
31683246 ScalarValue :: Null => write ! ( f, "NULL" ) ?,
31693247 } ;
@@ -3279,6 +3357,10 @@ impl fmt::Debug for ScalarValue {
32793357 ScalarValue :: DurationNanosecond ( _) => {
32803358 write ! ( f, "DurationNanosecond(\" {self}\" )" )
32813359 }
3360+ ScalarValue :: Union ( val, _fields, _mode) => match val {
3361+ Some ( ( id, val) ) => write ! ( f, "Union {}:{}" , id, val) ,
3362+ None => write ! ( f, "Union(NULL)" ) ,
3363+ } ,
32823364 ScalarValue :: Dictionary ( k, v) => write ! ( f, "Dictionary({k:?}, {v:?})" ) ,
32833365 ScalarValue :: Null => write ! ( f, "NULL" ) ,
32843366 }
0 commit comments