Skip to content

Commit 8074ca1

Browse files
Support Union types in ScalarValue (apache#9683)
Support Union types in `ScalarValue` (apache#9683)
1 parent 0974759 commit 8074ca1

9 files changed

Lines changed: 496 additions & 3 deletions

File tree

‎datafusion/common/src/error.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ pub enum DataFusionError {
6363
IoError(io::Error),
6464
/// Error when SQL is syntactically incorrect.
6565
///
66-
/// 2nd argument is for optional backtrace
66+
/// 2nd argument is for optional backtrace
6767
SQL(ParserError, Option<String>),
6868
/// Error when a feature is not yet implemented.
6969
///
@@ -101,7 +101,7 @@ pub enum DataFusionError {
101101
/// This error can be returned in cases such as when schema inference is not
102102
/// possible and when column names are not unique.
103103
///
104-
/// 2nd argument is for optional backtrace
104+
/// 2nd argument is for optional backtrace
105105
/// Boxing the optional backtrace to prevent <https://rust-lang.github.io/rust-clippy/master/index.html#/result_large_err>
106106
SchemaError(SchemaError, Box<Option<String>>),
107107
/// Error during execution of the query.

‎datafusion/common/src/scalar/mod.rs‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ use arrow::{
5353
},
5454
};
5555
use arrow_array::{ArrowNativeTypeOp, Scalar};
56+
use arrow_buffer::Buffer;
57+
use arrow_schema::{UnionFields, UnionMode};
5658

5759
pub use struct_builder::ScalarStructBuilder;
5860

@@ -275,6 +277,11 @@ pub enum ScalarValue {
275277
DurationMicrosecond(Option<i64>),
276278
/// Duration in nanoseconds
277279
DurationNanosecond(Option<i64>),
280+
/// A nested datatype that can represent slots of differing types. Components:
281+
/// `.0`: a tuple of union `type_id` and the single value held by this Scalar
282+
/// `.1`: the list of fields, zero-to-one of which will by set in `.0`
283+
/// `.2`: the physical storage of the source/destination UnionArray from which this Scalar came
284+
Union(Option<(i8, Box<ScalarValue>)>, UnionFields, UnionMode),
278285
/// Dictionary type: index type and value
279286
Dictionary(Box<DataType>, Box<ScalarValue>),
280287
}
@@ -375,6 +382,10 @@ impl PartialEq for ScalarValue {
375382
(IntervalDayTime(_), _) => false,
376383
(IntervalMonthDayNano(v1), IntervalMonthDayNano(v2)) => v1.eq(v2),
377384
(IntervalMonthDayNano(_), _) => false,
385+
(Union(val1, fields1, mode1), Union(val2, fields2, mode2)) => {
386+
val1.eq(val2) && fields1.eq(fields2) && mode1.eq(mode2)
387+
}
388+
(Union(_, _, _), _) => false,
378389
(Dictionary(k1, v1), Dictionary(k2, v2)) => k1.eq(k2) && v1.eq(v2),
379390
(Dictionary(_, _), _) => false,
380391
(Null, Null) => true,
@@ -500,6 +511,14 @@ impl PartialOrd for ScalarValue {
500511
(DurationMicrosecond(_), _) => None,
501512
(DurationNanosecond(v1), DurationNanosecond(v2)) => v1.partial_cmp(v2),
502513
(DurationNanosecond(_), _) => None,
514+
(Union(v1, t1, m1), Union(v2, t2, m2)) => {
515+
if t1.eq(t2) && m1.eq(m2) {
516+
v1.partial_cmp(v2)
517+
} else {
518+
None
519+
}
520+
}
521+
(Union(_, _, _), _) => None,
503522
(Dictionary(k1, v1), Dictionary(k2, v2)) => {
504523
// Don't compare if the key types don't match (it is effectively a different datatype)
505524
if k1 == k2 {
@@ -663,6 +682,11 @@ impl std::hash::Hash for ScalarValue {
663682
IntervalYearMonth(v) => v.hash(state),
664683
IntervalDayTime(v) => v.hash(state),
665684
IntervalMonthDayNano(v) => v.hash(state),
685+
Union(v, t, m) => {
686+
v.hash(state);
687+
t.hash(state);
688+
m.hash(state);
689+
}
666690
Dictionary(k, v) => {
667691
k.hash(state);
668692
v.hash(state);
@@ -1093,6 +1117,7 @@ impl ScalarValue {
10931117
ScalarValue::DurationNanosecond(_) => {
10941118
DataType::Duration(TimeUnit::Nanosecond)
10951119
}
1120+
ScalarValue::Union(_, fields, mode) => DataType::Union(fields.clone(), *mode),
10961121
ScalarValue::Dictionary(k, v) => {
10971122
DataType::Dictionary(k.clone(), Box::new(v.data_type()))
10981123
}
@@ -1292,6 +1317,7 @@ impl ScalarValue {
12921317
ScalarValue::DurationMillisecond(v) => v.is_none(),
12931318
ScalarValue::DurationMicrosecond(v) => v.is_none(),
12941319
ScalarValue::DurationNanosecond(v) => v.is_none(),
1320+
ScalarValue::Union(v, _, _) => v.is_none(),
12951321
ScalarValue::Dictionary(_, v) => v.is_null(),
12961322
}
12971323
}
@@ -2087,6 +2113,39 @@ impl ScalarValue {
20872113
e,
20882114
size
20892115
),
2116+
ScalarValue::Union(value, fields, _mode) => match value {
2117+
Some((v_id, value)) => {
2118+
let mut field_type_ids = Vec::<i8>::with_capacity(fields.len());
2119+
let mut child_arrays =
2120+
Vec::<(Field, ArrayRef)>::with_capacity(fields.len());
2121+
for (f_id, field) in fields.iter() {
2122+
let ar = if f_id == *v_id {
2123+
value.to_array_of_size(size)?
2124+
} else {
2125+
let dt = field.data_type();
2126+
new_null_array(dt, size)
2127+
};
2128+
let field = (**field).clone();
2129+
child_arrays.push((field, ar));
2130+
field_type_ids.push(f_id);
2131+
}
2132+
let type_ids = repeat(*v_id).take(size).collect::<Vec<_>>();
2133+
let type_ids = Buffer::from_slice_ref(type_ids);
2134+
let value_offsets: Option<Buffer> = None;
2135+
let ar = UnionArray::try_new(
2136+
field_type_ids.as_slice(),
2137+
type_ids,
2138+
value_offsets,
2139+
child_arrays,
2140+
)
2141+
.map_err(|e| DataFusionError::ArrowError(e, None))?;
2142+
Arc::new(ar)
2143+
}
2144+
None => {
2145+
let dt = self.data_type();
2146+
new_null_array(&dt, size)
2147+
}
2148+
},
20902149
ScalarValue::Dictionary(key_type, v) => {
20912150
// values array is one element long (the value)
20922151
match key_type.as_ref() {
@@ -2626,6 +2685,9 @@ impl ScalarValue {
26262685
ScalarValue::DurationNanosecond(val) => {
26272686
eq_array_primitive!(array, index, DurationNanosecondArray, val)?
26282687
}
2688+
ScalarValue::Union(_, _, _) => {
2689+
return _not_impl_err!("Union is not supported yet")
2690+
}
26292691
ScalarValue::Dictionary(key_type, v) => {
26302692
let (values_array, values_index) = match key_type.as_ref() {
26312693
DataType::Int8 => get_dict_value::<Int8Type>(array, index)?,
@@ -2703,6 +2765,15 @@ impl ScalarValue {
27032765
ScalarValue::LargeList(arr) => arr.get_array_memory_size(),
27042766
ScalarValue::FixedSizeList(arr) => arr.get_array_memory_size(),
27052767
ScalarValue::Struct(arr) => arr.get_array_memory_size(),
2768+
ScalarValue::Union(vals, fields, _mode) => {
2769+
vals.as_ref()
2770+
.map(|(_id, sv)| sv.size() - std::mem::size_of_val(sv))
2771+
.unwrap_or_default()
2772+
// `fields` is boxed, so it is NOT already included in `self`
2773+
+ std::mem::size_of_val(fields)
2774+
+ (std::mem::size_of::<Field>() * fields.len())
2775+
+ fields.iter().map(|(_idx, field)| field.size() - std::mem::size_of_val(field)).sum::<usize>()
2776+
}
27062777
ScalarValue::Dictionary(dt, sv) => {
27072778
// `dt` and `sv` are boxed, so they are NOT already included in `self`
27082779
dt.size() + sv.size()
@@ -3048,6 +3119,9 @@ impl TryFrom<&DataType> for ScalarValue {
30483119
.to_owned()
30493120
.into(),
30503121
),
3122+
DataType::Union(fields, mode) => {
3123+
ScalarValue::Union(None, fields.clone(), *mode)
3124+
}
30513125
DataType::Null => ScalarValue::Null,
30523126
_ => {
30533127
return _not_impl_err!(
@@ -3164,6 +3238,10 @@ impl fmt::Display for ScalarValue {
31643238
.join(",")
31653239
)?
31663240
}
3241+
ScalarValue::Union(val, _fields, _mode) => match val {
3242+
Some((id, val)) => write!(f, "{}:{}", id, val)?,
3243+
None => write!(f, "NULL")?,
3244+
},
31673245
ScalarValue::Dictionary(_k, v) => write!(f, "{v}")?,
31683246
ScalarValue::Null => write!(f, "NULL")?,
31693247
};
@@ -3279,6 +3357,10 @@ impl fmt::Debug for ScalarValue {
32793357
ScalarValue::DurationNanosecond(_) => {
32803358
write!(f, "DurationNanosecond(\"{self}\")")
32813359
}
3360+
ScalarValue::Union(val, _fields, _mode) => match val {
3361+
Some((id, val)) => write!(f, "Union {}:{}", id, val),
3362+
None => write!(f, "Union(NULL)"),
3363+
},
32823364
ScalarValue::Dictionary(k, v) => write!(f, "Dictionary({k:?}, {v:?})"),
32833365
ScalarValue::Null => write!(f, "NULL"),
32843366
}

‎datafusion/physical-plan/src/filter.rs‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -441,7 +441,9 @@ mod tests {
441441
use crate::test::exec::StatisticsExec;
442442
use crate::ExecutionPlan;
443443

444+
use crate::empty::EmptyExec;
444445
use arrow::datatypes::{DataType, Field, Schema};
446+
use arrow_schema::{UnionFields, UnionMode};
445447
use datafusion_common::{ColumnStatistics, ScalarValue};
446448
use datafusion_expr::Operator;
447449

@@ -1090,4 +1092,37 @@ mod tests {
10901092
assert_eq!(statistics.total_byte_size, Precision::Inexact(1600));
10911093
Ok(())
10921094
}
1095+
1096+
#[test]
1097+
fn test_equivalence_properties_union_type() -> Result<()> {
1098+
let union_type = DataType::Union(
1099+
UnionFields::new(
1100+
vec![0, 1],
1101+
vec![
1102+
Field::new("f1", DataType::Int32, true),
1103+
Field::new("f2", DataType::Utf8, true),
1104+
],
1105+
),
1106+
UnionMode::Sparse,
1107+
);
1108+
1109+
let schema = Arc::new(Schema::new(vec![
1110+
Field::new("c1", DataType::Int32, true),
1111+
Field::new("c2", union_type, true),
1112+
]));
1113+
1114+
let exec = FilterExec::try_new(
1115+
binary(
1116+
binary(col("c1", &schema)?, Operator::GtEq, lit(1i32), &schema)?,
1117+
Operator::And,
1118+
binary(col("c1", &schema)?, Operator::LtEq, lit(4i32), &schema)?,
1119+
&schema,
1120+
)?,
1121+
Arc::new(EmptyExec::new(schema.clone())),
1122+
)?;
1123+
1124+
exec.statistics().unwrap();
1125+
1126+
Ok(())
1127+
}
10931128
}

‎datafusion/proto/proto/datafusion.proto‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -988,6 +988,20 @@ message IntervalMonthDayNanoValue {
988988
int64 nanos = 3;
989989
}
990990

991+
message UnionField {
992+
int32 field_id = 1;
993+
Field field = 2;
994+
}
995+
996+
message UnionValue {
997+
// Note that a null union value must have one or more fields, so we
998+
// encode a null UnionValue as one with value_id == 128
999+
int32 value_id = 1;
1000+
ScalarValue value = 2;
1001+
repeated UnionField fields = 3;
1002+
UnionMode mode = 4;
1003+
}
1004+
9911005
message ScalarFixedSizeBinary{
9921006
bytes values = 1;
9931007
int32 length = 2;
@@ -1042,6 +1056,7 @@ message ScalarValue{
10421056
ScalarTime64Value time64_value = 30;
10431057
IntervalMonthDayNanoValue interval_month_day_nano = 31;
10441058
ScalarFixedSizeBinary fixed_size_binary_value = 34;
1059+
UnionValue union_value = 42;
10451060
}
10461061
}
10471062

0 commit comments

Comments
 (0)