Skip to content

Commit ebf0625

Browse files
authored
feat: Support duration type in approx_distinct (apache#23291)
## Which issue does this PR close? - Relates to apache#22989 but does not close it. More types are coming. ## Rationale for this change - Support the `Duration` type for `approx_distinct` - The Arrow type `Duration` can be directly supported for `NumericHLLAccumulator` and `HllGroupsAccumulator` ## What changes are included in this PR? - Enable `NumericHLLAccumulator` and `HllGroupsAccumulator` to support `Duration` - Tests for the non-grouped and grouped path as part of `aggregate.slt` ## Are these changes tested? Yes ## Are there any user-facing changes? Yes, `approx_distinct` supports now `Duration` but no breaking changes.
1 parent 6a0e76e commit ebf0625

2 files changed

Lines changed: 42 additions & 1 deletion

File tree

‎datafusion/functions-aggregate/src/approx_distinct.rs‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@ use arrow::array::{
2525
use arrow::buffer::NullBuffer;
2626
use arrow::datatypes::{
2727
ArrowPrimitiveType, DataType, Date32Type, Date64Type, Decimal32Type, Decimal64Type,
28-
Decimal128Type, Decimal256Type, Field, FieldRef, Int32Type, Int64Type,
28+
Decimal128Type, Decimal256Type, DurationMicrosecondType, DurationMillisecondType,
29+
DurationNanosecondType, DurationSecondType, Field, FieldRef, Int32Type, Int64Type,
2930
IntervalDayTimeType, IntervalMonthDayNanoType, IntervalUnit, IntervalYearMonthType,
3031
Time32MillisecondType, Time32SecondType, Time64MicrosecondType, Time64NanosecondType,
3132
TimeUnit, TimestampMicrosecondType, TimestampMillisecondType,
@@ -781,6 +782,18 @@ impl AggregateUDFImpl for ApproxDistinct {
781782
DataType::Decimal256(_, _) => {
782783
Box::new(NumericHLLAccumulator::<Decimal256Type>::new())
783784
}
785+
DataType::Duration(TimeUnit::Second) => {
786+
Box::new(NumericHLLAccumulator::<DurationSecondType>::new())
787+
}
788+
DataType::Duration(TimeUnit::Millisecond) => {
789+
Box::new(NumericHLLAccumulator::<DurationMillisecondType>::new())
790+
}
791+
DataType::Duration(TimeUnit::Microsecond) => {
792+
Box::new(NumericHLLAccumulator::<DurationMicrosecondType>::new())
793+
}
794+
DataType::Duration(TimeUnit::Nanosecond) => {
795+
Box::new(NumericHLLAccumulator::<DurationNanosecondType>::new())
796+
}
784797
DataType::Utf8
785798
| DataType::LargeUtf8
786799
| DataType::Utf8View
@@ -848,6 +861,7 @@ fn is_hll_groups_type(data_type: &DataType) -> bool {
848861
| DataType::Decimal64(_, _)
849862
| DataType::Decimal128(_, _)
850863
| DataType::Decimal256(_, _)
864+
| DataType::Duration(_)
851865
| DataType::Utf8
852866
| DataType::LargeUtf8
853867
| DataType::Utf8View

‎datafusion/sqllogictest/test_files/aggregate.slt‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2005,6 +2005,33 @@ FROM approx_distinct_decimal_test GROUP BY g ORDER BY g;
20052005
statement ok
20062006
DROP TABLE approx_distinct_decimal_test;
20072007

2008+
# This test runs approx_distinct over all four Duration units for the scalar and the grouped path.
2009+
statement ok
2010+
CREATE TABLE approx_distinct_duration_test AS VALUES
2011+
(1, arrow_cast(1, 'Duration(Second)'), arrow_cast(1, 'Duration(Millisecond)'), arrow_cast(1, 'Duration(Microsecond)'), arrow_cast(1, 'Duration(Nanosecond)')),
2012+
(1, arrow_cast(2, 'Duration(Second)'), arrow_cast(2, 'Duration(Millisecond)'), arrow_cast(2, 'Duration(Microsecond)'), arrow_cast(2, 'Duration(Nanosecond)')),
2013+
(1, arrow_cast(2, 'Duration(Second)'), arrow_cast(2, 'Duration(Millisecond)'), arrow_cast(2, 'Duration(Microsecond)'), arrow_cast(2, 'Duration(Nanosecond)')),
2014+
(2, arrow_cast(3, 'Duration(Second)'), arrow_cast(3, 'Duration(Millisecond)'), arrow_cast(3, 'Duration(Microsecond)'), arrow_cast(3, 'Duration(Nanosecond)')),
2015+
(2, arrow_cast(0, 'Duration(Second)'), arrow_cast(0, 'Duration(Millisecond)'), arrow_cast(0, 'Duration(Microsecond)'), arrow_cast(0, 'Duration(Nanosecond)')),
2016+
(2, arrow_cast(0, 'Duration(Second)'), arrow_cast(0, 'Duration(Millisecond)'), arrow_cast(0, 'Duration(Microsecond)'), arrow_cast(0, 'Duration(Nanosecond)'));
2017+
2018+
# Scalar path
2019+
query IIII
2020+
SELECT approx_distinct(column2), approx_distinct(column3), approx_distinct(column4), approx_distinct(column5) FROM approx_distinct_duration_test;
2021+
----
2022+
4 4 4 4
2023+
2024+
# Grouped path
2025+
query IIIII
2026+
SELECT column1, approx_distinct(column2), approx_distinct(column3), approx_distinct(column4), approx_distinct(column5)
2027+
FROM approx_distinct_duration_test GROUP BY column1 ORDER BY column1;
2028+
----
2029+
1 2 2 2 2
2030+
2 2 2 2 2
2031+
2032+
statement ok
2033+
DROP TABLE approx_distinct_duration_test;
2034+
20082035

20092036
# This test runs approx_distinct over the intervals YearMonth,
20102037
# DayTime, MonthDayNano for the scalar and the grouped path.

0 commit comments

Comments
 (0)