Skip to content

Commit 84eaef9

Browse files
mkleennuno-faria
andauthored
feat: Support Map type in approx_distinct (apache#23526)
## Which issue does this PR close? - Relates to apache#22989 but does not close it. More types are coming. ## Rationale for this change - Support the Arrow types `Map` type for `approx_distinct` ## What changes are included in this PR? - Enable `HLLAccumulator` and `HllGroupsAccumulator` to support `Map` - Tests for the non-grouped and grouped path as part of `aggregate.slt` ## Are these changes tested? Yes and results are compared to DuckDB: ``` D CREATE TABLE approx_distinct_map_test AS SELECT * FROM (VALUES (1, MAP {'a': 1, 'b': 2}), (1, MAP {'a': 1, 'b': 2}), (1, MAP {'c': 3}), (2, MAP {'d': 4}), (2, NULL), (3, NULL), (3, NULL), (4, MAP {'e': 5}) ) AS t(g, m); D SELECT approx_count_distinct(m) FROM approx_distinct_map_test WHERE g = 1; ┌──────────────────────────┐ │ approx_count_distinct(m) │ │ int64 │ ├──────────────────────────┤ │ 2 │ └──────────────────────────┘ D SELECT g, approx_count_distinct(m) FROM approx_distinct_map_test GROUP BY g ORDER BY g; ┌───────┬──────────────────────────┐ │ g │ approx_count_distinct(m) │ │ int32 │ int64 │ ├───────┼──────────────────────────┤ │ 1 │ 2 │ │ 2 │ 1 │ │ 3 │ 0 │ │ 4 │ 1 │ └───────┴──────────────────────────┘ D SELECT approx_count_distinct(m) FROM approx_distinct_map_test; ┌──────────────────────────┐ │ approx_count_distinct(m) │ │ int64 │ ├──────────────────────────┤ │ 4 │ └──────────────────────────┘ ``` ## Are there any user-facing changes? Yes, `approx_distinct` supports now `Map` but no breaking changes. Co-authored-by: Nuno Faria <[email protected]>
1 parent 95cda37 commit 84eaef9

2 files changed

Lines changed: 40 additions & 0 deletions

File tree

‎datafusion/functions-aggregate/src/approx_distinct.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -842,6 +842,7 @@ impl AggregateUDFImpl for ApproxDistinct {
842842
| DataType::FixedSizeList(_, _)
843843
| DataType::ListView(_)
844844
| DataType::LargeListView(_)
845+
| DataType::Map(_, _)
845846
| DataType::LargeBinary => Box::new(HLLAccumulator::new()),
846847
DataType::Null => {
847848
Box::new(NoopAccumulator::new(ScalarValue::UInt64(Some(0))))
@@ -918,6 +919,7 @@ fn is_hll_groups_type(data_type: &DataType) -> bool {
918919
| DataType::FixedSizeList(_, _)
919920
| DataType::ListView(_)
920921
| DataType::LargeListView(_)
922+
| DataType::Map(_, _)
921923
)
922924
}
923925

‎datafusion/sqllogictest/test_files/aggregate.slt‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2095,6 +2095,44 @@ SELECT approx_distinct(arrow_cast(l, 'FixedSizeList(2, Int32)')) FROM approx_dis
20952095
statement ok
20962096
DROP TABLE approx_distinct_list_test;
20972097

2098+
# Map
2099+
statement ok
2100+
CREATE TABLE approx_distinct_map_test AS SELECT * FROM (VALUES
2101+
(1, MAP {'a': 1, 'b': 2}), (1, MAP {'a': 1, 'b': 2}), (1, MAP {'c': 3}),
2102+
(2, MAP {'d': 4}), (2, NULL),
2103+
(3, NULL), (3, NULL),
2104+
(4, MAP {'e': 5})
2105+
) AS t(g, m);
2106+
2107+
# Map non-grouped
2108+
query I
2109+
SELECT approx_distinct(m) FROM approx_distinct_map_test WHERE g = 1;
2110+
----
2111+
2
2112+
2113+
# Map grouped
2114+
# Group 1 -> {{a:1,b:2},{c:3}}=2,
2115+
# Group 2 -> {{d:4}}=1 (NULL excluded),
2116+
# Group 3 -> all null=0,
2117+
# Group 4 -> {{e:5}}=1
2118+
query II
2119+
SELECT g, approx_distinct(m) FROM approx_distinct_map_test GROUP BY g ORDER BY g;
2120+
----
2121+
1 2
2122+
2 1
2123+
3 0
2124+
4 1
2125+
2126+
# The non-group path must agree with the grouped path on
2127+
# the same data, and distinct maps across groups are still counted overall.
2128+
query I
2129+
SELECT approx_distinct(m) FROM approx_distinct_map_test;
2130+
----
2131+
4
2132+
2133+
statement ok
2134+
DROP TABLE approx_distinct_map_test;
2135+
20982136
# Integers (Int32): group 1 -> {10,20}=2, group 2 -> {30,40}=2, group 3 -> 0, group 4 -> {50}=1
20992137
query II
21002138
SELECT g, approx_distinct(i) FROM approx_distinct_group_test GROUP BY g ORDER BY g;

0 commit comments

Comments
 (0)