Skip to content

Commit 98e76bf

Browse files
mkleengrevend
authored andcommitted
feat: Support RunEndEncoded type in approx_distinct (apache#25951)
## Which issue does this PR close? - Closes apache#22989 ## Rationale for this change - Support the Arrow type `RunEndEncoded` type for `approx_distinct` ## What changes are included in this PR? - Enable `HLLAccumulator` and `HllGroupsAccumulator` to support `RunEndEncoded` - Tests for the non-grouped and grouped path as part of `aggregate.slt` ## Are these changes tested? Yes. ## Are there any user-facing changes? Yes, `approx_distinct` supports now `RunEndEncoded` but no breaking changes. (cherry picked from commit a93c9b8)
1 parent b08862c commit 98e76bf

2 files changed

Lines changed: 64 additions & 7 deletions

File tree

‎datafusion/functions-aggregate/src/approx_distinct.rs‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -835,7 +835,9 @@ impl AggregateUDFImpl for ApproxDistinct {
835835
| DataType::Struct(_)
836836
| DataType::Union(_, _)
837837
| DataType::LargeBinary => Box::new(HLLAccumulator::new()),
838-
DataType::Dictionary(_, _) if is_supported_type(data_type) => {
838+
DataType::Dictionary(_, _) | DataType::RunEndEncoded(_, _)
839+
if is_supported_type(data_type) =>
840+
{
839841
Box::new(HLLAccumulator::new())
840842
}
841843
DataType::Null => {
@@ -885,25 +887,31 @@ fn is_fixed_domain_type(data_type: &DataType) -> bool {
885887
}
886888

887889
fn is_supported_type(data_type: &DataType) -> bool {
888-
let value_type = dictionary_value_type(data_type);
890+
let value_type = encoded_value_type(data_type);
889891
matches!(value_type, DataType::Null)
890892
|| is_fixed_domain_type(value_type)
891893
|| is_hll_groups_type(value_type)
892894
}
893895

894-
fn dictionary_value_type(data_type: &DataType) -> &DataType {
896+
fn encoded_value_type(data_type: &DataType) -> &DataType {
895897
let mut value_type = data_type;
896-
while let DataType::Dictionary(_, inner) = value_type {
897-
value_type = inner;
898+
loop {
899+
match value_type {
900+
DataType::Dictionary(_, inner) => value_type = inner,
901+
DataType::RunEndEncoded(_, values) => value_type = values.data_type(),
902+
_ => return value_type,
903+
}
898904
}
899-
value_type
900905
}
901906

902907
/// Returns true for the data types backed by the HyperLogLog
903908
/// [`HllGroupsAccumulator`]. The fixed-domain types (booleans / small ints) and
904909
/// `Null` fall back to the per-group [`Accumulator`] path.
905910
fn is_hll_groups_type(data_type: &DataType) -> bool {
906-
if matches!(data_type, DataType::Dictionary(_, _)) {
911+
if matches!(
912+
data_type,
913+
DataType::Dictionary(_, _) | DataType::RunEndEncoded(_, _)
914+
) {
907915
return is_supported_type(data_type);
908916
}
909917

‎datafusion/sqllogictest/test_files/aggregate.slt‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1811,6 +1811,55 @@ SELECT approx_distinct(arrow_cast(i, 'Dictionary(Int32, Float64)')) FROM approx_
18111811
statement error DataFusion error: This feature is not implemented: Support for 'approx_distinct' for data type Dictionary\(Int32, Float64\) is not implemented
18121812
SELECT g, approx_distinct(arrow_cast(i, 'Dictionary(Int32, Float64)')) FROM approx_distinct_group_test GROUP BY g;
18131813

1814+
# RunEndEncoded: run-end encoded values must hash identically to the plain
1815+
# values, so the counts below match the Utf8 / Int32 cases above.
1816+
query I
1817+
SELECT approx_distinct(arrow_cast(s, 'RunEndEncoded("run_ends": non-null Int32, "values": Utf8)')) FROM approx_distinct_group_test WHERE g = 2;
1818+
----
1819+
2
1820+
1821+
query II
1822+
SELECT g, approx_distinct(arrow_cast(s, 'RunEndEncoded("run_ends": non-null Int32, "values": Utf8)')) FROM approx_distinct_group_test GROUP BY g ORDER BY g;
1823+
----
1824+
1 2
1825+
2 2
1826+
3 0
1827+
4 1
1828+
1829+
query II
1830+
SELECT g, approx_distinct(arrow_cast(i, 'RunEndEncoded("run_ends": non-null Int64, "values": Int32)')) FROM approx_distinct_group_test GROUP BY g ORDER BY g;
1831+
----
1832+
1 2
1833+
2 2
1834+
3 0
1835+
4 1
1836+
1837+
# RunEndEncoded over a fixed-domain value type (Int8) goes through the
1838+
# HyperLogLog accumulators and must match the Int32 counts above.
1839+
query II
1840+
SELECT g, approx_distinct(arrow_cast(arrow_cast(i, 'Int8'), 'RunEndEncoded("run_ends": non-null Int32, "values": Int8)')) FROM approx_distinct_group_test GROUP BY g ORDER BY g;
1841+
----
1842+
1 2
1843+
2 2
1844+
3 0
1845+
4 1
1846+
1847+
# RunEndEncoded over a dictionary
1848+
query II
1849+
SELECT g, approx_distinct(arrow_cast(s, 'RunEndEncoded("run_ends": non-null Int32, "values": Dictionary(Int32, Utf8))')) FROM approx_distinct_group_test GROUP BY g ORDER BY g;
1850+
----
1851+
1 2
1852+
2 2
1853+
3 0
1854+
4 1
1855+
1856+
# Unsupported value types are rejected, as for a bare Float64
1857+
statement error DataFusion error: This feature is not implemented: Support for 'approx_distinct' for data type RunEndEncoded
1858+
SELECT approx_distinct(arrow_cast(i, 'RunEndEncoded("run_ends": non-null Int32, "values": Float64)')) FROM approx_distinct_group_test;
1859+
1860+
statement error DataFusion error: This feature is not implemented: Support for 'approx_distinct' for data type RunEndEncoded
1861+
SELECT g, approx_distinct(arrow_cast(i, 'RunEndEncoded("run_ends": non-null Int32, "values": Float64)')) FROM approx_distinct_group_test GROUP BY g;
1862+
18141863

18151864
# List
18161865
statement ok

0 commit comments

Comments
 (0)