Skip to content

Commit 1bbae17

Browse files
mkleenkosiew
authored andcommitted
feat: Support Struct type in approx_distinct (apache#23663)
## Which issue does this PR close? - Relates to apache#22989 but does not close it. More types are coming. ## Rationale for this change - Support the Arrow type `Struct` type for `approx_distinct` ## What changes are included in this PR? - Enable `HLLAccumulator` and `HllGroupsAccumulator` to support `Struct` - Tests for the non-grouped and grouped path as part of `aggregate.slt` ## Are these changes tested? Yes and results are compared to DuckDB: ```sql D CREATE TABLE approx_distinct_struct_test AS SELECT * FROM (VALUES (1, {'a': 1, 'b': 2}), (1, {'a': 1, 'b': 2}), (1, {'a': 3, 'b': 3}), (2, {'a': 4, 'b': 4}), (2, NULL), (3, NULL), (3, NULL), (4, {'a': 5, 'b': 5}) ) AS t(g, s); D SELECT approx_count_distinct(s) FROM approx_distinct_struct_test WHERE g = 1; ┌──────────────────────────┐ │ approx_count_distinct(s) │ │ int64 │ ├──────────────────────────┤ │ 2 │ └──────────────────────────┘ D SELECT g, approx_count_distinct(s) FROM approx_distinct_struct_test GROUP BY g ORDER BY g; ┌───────┬──────────────────────────┐ │ g │ approx_count_distinct(s) │ │ int32 │ int64 │ ├───────┼──────────────────────────┤ │ 1 │ 2 │ │ 2 │ 1 │ │ 3 │ 0 │ │ 4 │ 1 │ └───────┴──────────────────────────┘ D SELECT approx_count_distinct(s) FROM approx_distinct_struct_test; ┌──────────────────────────┐ │ approx_count_distinct(s) │ │ int64 │ ├──────────────────────────┤ │ 4 │ └──────────────────────────┘ ``` ## Are there any user-facing changes? Yes, `approx_distinct` supports now `Struct` but no breaking changes.
1 parent b1fb639 commit 1bbae17

2 files changed

Lines changed: 40 additions & 0 deletions

File tree

‎datafusion/functions-aggregate/src/approx_distinct.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -843,6 +843,7 @@ impl AggregateUDFImpl for ApproxDistinct {
843843
| DataType::ListView(_)
844844
| DataType::LargeListView(_)
845845
| DataType::Map(_, _)
846+
| DataType::Struct(_)
846847
| DataType::LargeBinary => Box::new(HLLAccumulator::new()),
847848
DataType::Null => {
848849
Box::new(NoopAccumulator::new(ScalarValue::UInt64(Some(0))))
@@ -920,6 +921,7 @@ fn is_hll_groups_type(data_type: &DataType) -> bool {
920921
| DataType::ListView(_)
921922
| DataType::LargeListView(_)
922923
| DataType::Map(_, _)
924+
| DataType::Struct(_)
923925
)
924926
}
925927

‎datafusion/sqllogictest/test_files/aggregate.slt‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2133,6 +2133,44 @@ SELECT approx_distinct(m) FROM approx_distinct_map_test;
21332133
statement ok
21342134
DROP TABLE approx_distinct_map_test;
21352135

2136+
# Struct
2137+
statement ok
2138+
CREATE TABLE approx_distinct_struct_test AS SELECT * FROM (VALUES
2139+
(1, named_struct('a', 1, 'b', 2)), (1, named_struct('a', 1, 'b', 2)), (1, named_struct('a', 3, 'b', 3)),
2140+
(2, named_struct('a', 4, 'b', 4)), (2, NULL),
2141+
(3, NULL), (3, NULL),
2142+
(4, named_struct('a', 5, 'b', 5))
2143+
) AS t(g, s);
2144+
2145+
# Struct non-grouped
2146+
query I
2147+
SELECT approx_distinct(s) FROM approx_distinct_struct_test WHERE g = 1;
2148+
----
2149+
2
2150+
2151+
# Struct grouped
2152+
# Group 1 -> {{a:1,b:2},{a:3,b:3}}=2,
2153+
# Group 2 -> {{a:4,b:4}}=1 (NULL excluded),
2154+
# Group 3 -> all null=0,
2155+
# Group 4 -> {{a:5,b:5}}=1
2156+
query II
2157+
SELECT g, approx_distinct(s) FROM approx_distinct_struct_test GROUP BY g ORDER BY g;
2158+
----
2159+
1 2
2160+
2 1
2161+
3 0
2162+
4 1
2163+
2164+
# The non-group path must agree with the grouped path on
2165+
# the same data, and distinct structs across groups are still counted overall.
2166+
query I
2167+
SELECT approx_distinct(s) FROM approx_distinct_struct_test;
2168+
----
2169+
4
2170+
2171+
statement ok
2172+
DROP TABLE approx_distinct_struct_test;
2173+
21362174
# Integers (Int32): group 1 -> {10,20}=2, group 2 -> {30,40}=2, group 3 -> 0, group 4 -> {50}=1
21372175
query II
21382176
SELECT g, approx_distinct(i) FROM approx_distinct_group_test GROUP BY g ORDER BY g;

0 commit comments

Comments
 (0)