Skip to content

Commit 766f129

Browse files
authored
feat: Support Decimal type in approx_distinct (apache#23190)
## Which issue does this PR close? - Relates to apache#22989 but does not close it. More types are coming. ## Rationale for this change - Support the SQL `Decimal` type for `approx_distinct` - The Arrow types `Decimal32`, `Decimal64`, `Decimal128` and `Decimal256` can be directly supported for `NumericHLLAccumulator` and `HllGroupsAccumulator` ## What changes are included in this PR? - Enable `NumericHLLAccumulator` and `HllGroupsAccumulator` to support `Decimal32`, `Decimal64`, `Decimal128` and `Decimal256`. - Tests for the non-grouped and grouped path as part of `approx_distinct.rst` and `aggregate.slt` for `Decimal128` and `Decimal256` - Benchmarks ## Are these changes tested? Yes ## Are there any user-facing changes? Yes, `approx_distinct` supports now `Decimal` but no breaking changes.
1 parent a59c4f5 commit 766f129

3 files changed

Lines changed: 331 additions & 8 deletions

File tree

‎datafusion/functions-aggregate/benches/approx_distinct.rs‎

Lines changed: 149 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,11 @@ use std::hint::black_box;
1919
use std::sync::Arc;
2020

2121
use arrow::array::{
22-
ArrayRef, Int8Array, Int16Array, Int64Array, StringArray, StringViewArray,
23-
UInt8Array, UInt16Array,
22+
ArrayRef, Decimal32Array, Decimal64Array, Decimal128Array, Decimal256Array,
23+
Int8Array, Int16Array, Int64Array, StringArray, StringViewArray, UInt8Array,
24+
UInt16Array,
2425
};
25-
use arrow::datatypes::{DataType, Field, Schema};
26+
use arrow::datatypes::{DataType, Field, Schema, i256};
2627
use criterion::{Criterion, criterion_group, criterion_main};
2728
use datafusion_expr::function::AccumulatorArgs;
2829
use datafusion_expr::{
@@ -44,6 +45,12 @@ const N_GROUPS: usize = 50_000;
4445
const AVG_ROWS_PER_GROUP: usize = 8;
4546
const STRING_POOL_SIZE: usize = 100_000;
4647

48+
const DECIMAL32_PRECISION: u8 = 9;
49+
const DECIMAL64_PRECISION: u8 = 18;
50+
const DECIMAL128_PRECISION: u8 = 10;
51+
const DECIMAL256_PRECISION: u8 = 40;
52+
const DECIMAL_SCALE: i8 = 2;
53+
4754
fn prepare_accumulator(data_type: DataType) -> Box<dyn Accumulator> {
4855
let schema = Arc::new(Schema::new(vec![Field::new("f", data_type, true)]));
4956
let expr = col("f", &schema).unwrap();
@@ -61,6 +68,52 @@ fn prepare_accumulator(data_type: DataType) -> Box<dyn Accumulator> {
6168
ApproxDistinct::new().accumulator(accumulator_args).unwrap()
6269
}
6370

71+
/// Creates a `Decimal32Array` from a pool of `n_distinct` values.
72+
fn create_decimal32_array(n_distinct: usize) -> Decimal32Array {
73+
let mut rng = StdRng::seed_from_u64(42);
74+
let pool: Vec<i32> = (0..n_distinct).map(|i| i as i32 * 50).collect();
75+
(0..BATCH_SIZE)
76+
.map(|_| Some(pool[rng.random_range(0..pool.len())]))
77+
.collect::<Decimal32Array>()
78+
.with_precision_and_scale(DECIMAL32_PRECISION, DECIMAL_SCALE)
79+
.unwrap()
80+
}
81+
82+
/// Creates a `Decimal64Array` from a pool of `n_distinct` values.
83+
fn create_decimal64_array(n_distinct: usize) -> Decimal64Array {
84+
let mut rng = StdRng::seed_from_u64(42);
85+
let pool: Vec<i64> = (0..n_distinct).map(|i| i as i64 * 50).collect();
86+
(0..BATCH_SIZE)
87+
.map(|_| Some(pool[rng.random_range(0..pool.len())]))
88+
.collect::<Decimal64Array>()
89+
.with_precision_and_scale(DECIMAL64_PRECISION, DECIMAL_SCALE)
90+
.unwrap()
91+
}
92+
93+
/// Creates a `Decimal128Array` from a pool of `n_distinct` values.
94+
fn create_decimal128_array(n_distinct: usize) -> Decimal128Array {
95+
let mut rng = StdRng::seed_from_u64(42);
96+
let pool: Vec<i128> = (0..n_distinct).map(|i| i as i128 * 50).collect();
97+
(0..BATCH_SIZE)
98+
.map(|_| Some(pool[rng.random_range(0..pool.len())]))
99+
.collect::<Decimal128Array>()
100+
.with_precision_and_scale(DECIMAL128_PRECISION, DECIMAL_SCALE)
101+
.unwrap()
102+
}
103+
104+
/// Creates a `Decimal256Array` from a pool of `n_distinct` values.
105+
fn create_decimal256_array(n_distinct: usize) -> Decimal256Array {
106+
let mut rng = StdRng::seed_from_u64(42);
107+
let pool: Vec<i256> = (0..n_distinct)
108+
.map(|i| i256::from_i128(i as i128 * 50))
109+
.collect();
110+
(0..BATCH_SIZE)
111+
.map(|_| Some(pool[rng.random_range(0..pool.len())]))
112+
.collect::<Decimal256Array>()
113+
.with_precision_and_scale(DECIMAL256_PRECISION, DECIMAL_SCALE)
114+
.unwrap()
115+
}
116+
64117
/// Creates an Int64Array where values are drawn from `0..n_distinct`.
65118
fn create_i64_array(n_distinct: usize) -> Int64Array {
66119
let mut rng = StdRng::seed_from_u64(42);
@@ -224,6 +277,62 @@ fn approx_distinct_benchmark(c: &mut Criterion) {
224277
.unwrap()
225278
})
226279
});
280+
281+
// Decimal32
282+
let values = Arc::new(create_decimal32_array(200)) as ArrayRef;
283+
c.bench_function("approx_distinct decimal32", |b| {
284+
b.iter(|| {
285+
let mut accumulator = prepare_accumulator(DataType::Decimal32(
286+
DECIMAL32_PRECISION,
287+
DECIMAL_SCALE,
288+
));
289+
accumulator
290+
.update_batch(std::slice::from_ref(&values))
291+
.unwrap()
292+
})
293+
});
294+
295+
// Decimal64
296+
let values = Arc::new(create_decimal64_array(200)) as ArrayRef;
297+
c.bench_function("approx_distinct decimal64", |b| {
298+
b.iter(|| {
299+
let mut accumulator = prepare_accumulator(DataType::Decimal64(
300+
DECIMAL64_PRECISION,
301+
DECIMAL_SCALE,
302+
));
303+
accumulator
304+
.update_batch(std::slice::from_ref(&values))
305+
.unwrap()
306+
})
307+
});
308+
309+
// Decimal128
310+
let values = Arc::new(create_decimal128_array(200)) as ArrayRef;
311+
c.bench_function("approx_distinct decimal128", |b| {
312+
b.iter(|| {
313+
let mut accumulator = prepare_accumulator(DataType::Decimal128(
314+
DECIMAL128_PRECISION,
315+
DECIMAL_SCALE,
316+
));
317+
accumulator
318+
.update_batch(std::slice::from_ref(&values))
319+
.unwrap()
320+
})
321+
});
322+
323+
// Decimal256
324+
let values = Arc::new(create_decimal256_array(200)) as ArrayRef;
325+
c.bench_function("approx_distinct decimal256", |b| {
326+
b.iter(|| {
327+
let mut accumulator = prepare_accumulator(DataType::Decimal256(
328+
DECIMAL256_PRECISION,
329+
DECIMAL_SCALE,
330+
));
331+
accumulator
332+
.update_batch(std::slice::from_ref(&values))
333+
.unwrap()
334+
})
335+
});
227336
}
228337

229338
/// Build a `GroupsAccumulator` the same way the aggregate operator does: use the
@@ -287,6 +396,34 @@ fn build_grouped_batches(data_type: &DataType) -> Vec<(ArrayRef, Vec<usize>)> {
287396
.map(|_| Some(pool[rng.random_range(0..pool.len())].as_str()))
288397
.collect::<StringViewArray>(),
289398
),
399+
DataType::Decimal32(p, s) => Arc::new(
400+
(0..BATCH_SIZE)
401+
.map(|_| Some(rng.random::<i32>()))
402+
.collect::<Decimal32Array>()
403+
.with_precision_and_scale(*p, *s)
404+
.unwrap(),
405+
),
406+
DataType::Decimal64(p, s) => Arc::new(
407+
(0..BATCH_SIZE)
408+
.map(|_| Some(rng.random::<i64>()))
409+
.collect::<Decimal64Array>()
410+
.with_precision_and_scale(*p, *s)
411+
.unwrap(),
412+
),
413+
DataType::Decimal128(p, s) => Arc::new(
414+
(0..BATCH_SIZE)
415+
.map(|_| Some(rng.random::<i64>() as i128))
416+
.collect::<Decimal128Array>()
417+
.with_precision_and_scale(*p, *s)
418+
.unwrap(),
419+
),
420+
DataType::Decimal256(p, s) => Arc::new(
421+
(0..BATCH_SIZE)
422+
.map(|_| Some(i256::from_i128(rng.random::<i64>() as i128)))
423+
.collect::<Decimal256Array>()
424+
.with_precision_and_scale(*p, *s)
425+
.unwrap(),
426+
),
290427
other => panic!("unsupported grouped bench type: {other}"),
291428
};
292429
(values, group_indices)
@@ -300,7 +437,15 @@ fn approx_distinct_grouped_benchmark(c: &mut Criterion) {
300437
let mut group = c.benchmark_group("approx_distinct_grouped");
301438
group.sample_size(10);
302439

303-
for data_type in [DataType::Int64, DataType::Utf8, DataType::Utf8View] {
440+
for data_type in [
441+
DataType::Int64,
442+
DataType::Utf8,
443+
DataType::Utf8View,
444+
DataType::Decimal32(DECIMAL32_PRECISION, DECIMAL_SCALE),
445+
DataType::Decimal64(DECIMAL64_PRECISION, DECIMAL_SCALE),
446+
DataType::Decimal128(DECIMAL128_PRECISION, DECIMAL_SCALE),
447+
DataType::Decimal256(DECIMAL256_PRECISION, DECIMAL_SCALE),
448+
] {
304449
let batches = build_grouped_batches(&data_type);
305450
let label = format!("{data_type:?} {N_GROUPS} groups");
306451
group.bench_function(&label, |b| {

‎datafusion/functions-aggregate/src/approx_distinct.rs‎

Lines changed: 153 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,10 @@ use arrow::array::{
2424
};
2525
use arrow::buffer::NullBuffer;
2626
use arrow::datatypes::{
27-
ArrowPrimitiveType, DataType, Date32Type, Date64Type, Field, FieldRef, Int32Type,
28-
Int64Type, Time32MillisecondType, Time32SecondType, Time64MicrosecondType,
29-
Time64NanosecondType, TimeUnit, TimestampMicrosecondType, TimestampMillisecondType,
27+
ArrowPrimitiveType, DataType, Date32Type, Date64Type, Decimal32Type, Decimal64Type,
28+
Decimal128Type, Decimal256Type, Field, FieldRef, Int32Type, Int64Type,
29+
Time32MillisecondType, Time32SecondType, Time64MicrosecondType, Time64NanosecondType,
30+
TimeUnit, TimestampMicrosecondType, TimestampMillisecondType,
3031
TimestampNanosecondType, TimestampSecondType, UInt32Type, UInt64Type,
3132
};
3233
use datafusion_common::ScalarValue;
@@ -758,6 +759,18 @@ impl AggregateUDFImpl for ApproxDistinct {
758759
DataType::Timestamp(TimeUnit::Nanosecond, _) => {
759760
Box::new(NumericHLLAccumulator::<TimestampNanosecondType>::new())
760761
}
762+
DataType::Decimal32(_, _) => {
763+
Box::new(NumericHLLAccumulator::<Decimal32Type>::new())
764+
}
765+
DataType::Decimal64(_, _) => {
766+
Box::new(NumericHLLAccumulator::<Decimal64Type>::new())
767+
}
768+
DataType::Decimal128(_, _) => {
769+
Box::new(NumericHLLAccumulator::<Decimal128Type>::new())
770+
}
771+
DataType::Decimal256(_, _) => {
772+
Box::new(NumericHLLAccumulator::<Decimal256Type>::new())
773+
}
761774
DataType::Utf8
762775
| DataType::LargeUtf8
763776
| DataType::Utf8View
@@ -818,6 +831,10 @@ fn is_hll_groups_type(data_type: &DataType) -> bool {
818831
| DataType::Timestamp(TimeUnit::Millisecond, _)
819832
| DataType::Timestamp(TimeUnit::Microsecond, _)
820833
| DataType::Timestamp(TimeUnit::Nanosecond, _)
834+
| DataType::Decimal32(_, _)
835+
| DataType::Decimal64(_, _)
836+
| DataType::Decimal128(_, _)
837+
| DataType::Decimal256(_, _)
821838
| DataType::Utf8
822839
| DataType::LargeUtf8
823840
| DataType::Utf8View
@@ -834,7 +851,11 @@ mod tests {
834851
#[cfg(not(feature = "force_hash_collisions"))]
835852
mod real_hash_test {
836853
use super::*;
837-
use arrow::array::{AsArray, Int64Array, StringViewArray};
854+
use arrow::array::{
855+
AsArray, Decimal32Array, Decimal64Array, Decimal128Array, Decimal256Array,
856+
Int64Array, StringViewArray,
857+
};
858+
use arrow::datatypes::i256;
838859
use std::sync::Arc;
839860
// A string longer than the 12-byte inline limit
840861
const LONG: &str = "this string is definitely longer than twelve bytes";
@@ -846,6 +867,134 @@ mod tests {
846867
}
847868
}
848869

870+
fn assert_count_numerical_acc_and_group_acc<T>(array: ArrayRef, expected: u64)
871+
where
872+
T: ArrowPrimitiveType + Debug,
873+
T::Native: Hash,
874+
{
875+
assert!(
876+
is_hll_groups_type(array.data_type()),
877+
"{} should be groups-capable",
878+
array.data_type()
879+
);
880+
881+
let mut acc = NumericHLLAccumulator::<T>::new();
882+
acc.update_batch(&[Arc::clone(&array)]).unwrap();
883+
let per_group_count = match acc.evaluate().unwrap() {
884+
ScalarValue::UInt64(Some(v)) => v,
885+
other => panic!("unexpected evaluate result: {other:?}"),
886+
};
887+
888+
let group_indices = vec![0usize; array.len()];
889+
let mut acc = HllGroupsAccumulator::new();
890+
acc.update_batch(std::slice::from_ref(&array), &group_indices, None, 1)
891+
.unwrap();
892+
let groups_count = acc
893+
.evaluate(EmitTo::All)
894+
.unwrap()
895+
.as_any()
896+
.downcast_ref::<UInt64Array>()
897+
.unwrap()
898+
.value(0);
899+
900+
assert_eq!(
901+
per_group_count,
902+
groups_count,
903+
"paths disagree for {}",
904+
array.data_type()
905+
);
906+
assert_eq!(
907+
per_group_count,
908+
expected,
909+
"wrong count for {}",
910+
array.data_type()
911+
);
912+
}
913+
914+
#[test]
915+
fn decimal_support_numerical_acc_and_group_acc() {
916+
let decimal_32: ArrayRef = Arc::new(
917+
Decimal32Array::from(vec![
918+
1i32,
919+
2,
920+
2,
921+
3,
922+
3,
923+
3,
924+
0,
925+
0,
926+
123_456_789,
927+
999_999_999,
928+
999_999_999,
929+
])
930+
.with_precision_and_scale(9, 2)
931+
.unwrap(),
932+
);
933+
assert_count_numerical_acc_and_group_acc::<Decimal32Type>(decimal_32, 6);
934+
935+
let decimal_64: ArrayRef = Arc::new(
936+
Decimal64Array::from(vec![
937+
1i64,
938+
2,
939+
2,
940+
3,
941+
3,
942+
3,
943+
0,
944+
0,
945+
1_234_567_890_123,
946+
9_999_999_999_999,
947+
9_999_999_999_999,
948+
])
949+
.with_precision_and_scale(18, 2)
950+
.unwrap(),
951+
);
952+
assert_count_numerical_acc_and_group_acc::<Decimal64Type>(decimal_64, 6);
953+
954+
let decimal_128: ArrayRef = Arc::new(
955+
Decimal128Array::from(vec![
956+
1i128,
957+
2,
958+
2,
959+
3,
960+
3,
961+
3,
962+
0,
963+
0,
964+
1_234_567_890,
965+
9_999_999_999,
966+
9_999_999_999,
967+
])
968+
.with_precision_and_scale(38, 2)
969+
.unwrap(),
970+
);
971+
assert_count_numerical_acc_and_group_acc::<Decimal128Type>(decimal_128, 6);
972+
973+
let big_256_a =
974+
i256::from_string("123456789012345678901234567890123456").unwrap();
975+
let big_256_b =
976+
i256::from_string("987654321098765432109876543210987654").unwrap();
977+
978+
let decimal_256: ArrayRef = Arc::new(
979+
Decimal256Array::from(vec![
980+
i256::from_i128(1),
981+
i256::from_i128(2),
982+
i256::from_i128(2),
983+
i256::from_i128(3),
984+
i256::from_i128(3),
985+
i256::from_i128(3),
986+
i256::from_i128(0),
987+
i256::from_i128(0),
988+
big_256_a,
989+
big_256_b,
990+
big_256_b,
991+
])
992+
.with_precision_and_scale(40, 2)
993+
.unwrap(),
994+
);
995+
assert_count_numerical_acc_and_group_acc::<Decimal256Type>(decimal_256, 6);
996+
}
997+
849998
/// `approx_distinct(v) FILTER (WHERE nullable_bool)` — a NULL filter row
850999
/// must not be counted (null filter is treated the same as false).
8511000
#[test]

0 commit comments

Comments
 (0)