Skip to content

Commit 04f336f

Browse files
stdpainHappenLee
authored andcommitted
[1/2] support multi-distinct (apache#29)
1 parent b13a4a9 commit 04f336f

7 files changed

Lines changed: 315 additions & 47 deletions

File tree

‎be/src/vec/CMakeLists.txt‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,12 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/vec")
2020
set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/vec")
2121

2222
set(VEC_FILES
23+
aggregate_functions/aggregate_function_avg.cpp
2324
aggregate_functions/aggregate_function_count.cpp
24-
aggregate_functions/aggregate_function_null.cpp
2525
aggregate_functions/aggregate_function_sum.cpp
2626
aggregate_functions/aggregate_function_min_max.cpp
27-
aggregate_functions/aggregate_function_avg.cpp
27+
aggregate_functions/aggregate_function_null.cpp
28+
aggregate_functions/aggregate_function_uniq.cpp
2829
columns/collator.cpp
2930
columns/column.cpp
3031
columns/column_const.cpp

‎be/src/vec/aggregate_functions/aggregate_function_simple_factory.h‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@ namespace doris::vectorized {
3131

3232
class AggregateFunctionSimpleFactory;
3333
void registerAggregateFunctionSum(AggregateFunctionSimpleFactory& factory);
34-
void registerAggregateFunctionCount(AggregateFunctionSimpleFactory& factory);
3534
void registerAggregateFunctionCombinatorNull(AggregateFunctionSimpleFactory& factory);
3635
void registerAggregateFunctionMinMax(AggregateFunctionSimpleFactory& factory);
3736
void registerAggregateFunctionAvg(AggregateFunctionSimpleFactory& factory);
@@ -52,6 +51,12 @@ class AggregateFunctionSimpleFactory {
5251
AggregateFunctions nullable_aggregate_functions;
5352

5453
public:
54+
void registerNullableFunctionCombinator(Creator creator) {
55+
for (auto entity : aggregate_functions) {
56+
nullable_aggregate_functions[entity.first] = creator;
57+
}
58+
}
59+
5560
void registerFunction(const std::string& name, Creator creator, bool nullable = false) {
5661
if (nullable) {
5762
nullable_aggregate_functions[name] = creator;
@@ -69,11 +74,13 @@ class AggregateFunctionSimpleFactory {
6974
}
7075
}
7176
if (nullable) {
72-
return nullable_aggregate_functions[name] == nullptr ? nullptr :
73-
nullable_aggregate_functions[name](name, argument_types, parameters);
77+
return nullable_aggregate_functions[name] == nullptr
78+
? nullptr
79+
: nullable_aggregate_functions[name](name, argument_types, parameters);
7480
} else {
75-
return aggregate_functions[name] == nullptr ? nullptr :
76-
aggregate_functions[name](name, argument_types, parameters);
81+
return aggregate_functions[name] == nullptr
82+
? nullptr
83+
: aggregate_functions[name](name, argument_types, parameters);
7784
}
7885
}
7986

@@ -83,7 +90,6 @@ class AggregateFunctionSimpleFactory {
8390
static AggregateFunctionSimpleFactory instance;
8491
std::call_once(oc, [&]() {
8592
registerAggregateFunctionSum(instance);
86-
registerAggregateFunctionCount(instance);
8793
registerAggregateFunctionMinMax(instance);
8894
registerAggregateFunctionAvg(instance);
8995
registerAggregateFunctionCombinatorNull(instance);
Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#include <vec/aggregate_functions/aggregate_function_simple_factory.h>
2+
#include <vec/aggregate_functions/aggregate_function_uniq.h>
3+
#include <vec/aggregate_functions/factory_helpers.h>
4+
#include <vec/aggregate_functions/helpers.h>
5+
#include <vec/data_types/data_type_string.h>
6+
7+
namespace doris::vectorized {
8+
9+
namespace ErrorCodes {
10+
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
11+
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
12+
} // namespace ErrorCodes
13+
14+
template <template <typename> class Data, typename DataForVariadic>
15+
AggregateFunctionPtr createAggregateFunctionUniq(const std::string& name,
16+
const DataTypes& argument_types,
17+
const Array& params) {
18+
assertNoParameters(name, params);
19+
20+
if (argument_types.empty())
21+
throw Exception("Incorrect number of arguments for aggregate function " + name,
22+
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
23+
24+
if (argument_types.size() == 1) {
25+
const IDataType& argument_type = *argument_types[0];
26+
27+
AggregateFunctionPtr res(createWithNumericType<AggregateFunctionUniq, Data>(
28+
*argument_types[0], argument_types));
29+
30+
WhichDataType which(argument_type);
31+
// TODO: DateType
32+
if (res)
33+
return res;
34+
else if (which.isStringOrFixedString())
35+
return std::make_shared<AggregateFunctionUniq<String, Data<String>>>(argument_types);
36+
}
37+
38+
return nullptr;
39+
}
40+
41+
void registerAggregateFunctionsUniq(AggregateFunctionSimpleFactory& factory) {
42+
AggregateFunctionCreator creator =
43+
createAggregateFunctionUniq<AggregateFunctionUniqExactData,
44+
AggregateFunctionUniqExactData<String>>;
45+
factory.registerFunction("uniqExact", creator);
46+
}
47+
48+
} // namespace doris::vectorized
Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
#pragma once
2+
3+
#include <city.h>
4+
5+
#include <type_traits>
6+
7+
#include "vec/common/bit_cast.h"
8+
9+
// #include <IO/WriteHelpers.h>
10+
// #include <IO/ReadHelpers.h>
11+
12+
// #include <DataTypes/DataTypesNumber.h>
13+
#include <vec/common/hash_table/hash_set.h>
14+
15+
#include "vec/common/aggregation_common.h"
16+
#include "vec/data_types/data_types_number.h"
17+
// #include <Common/HyperLogLogWithSmallSetOptimization.h>
18+
// #include <Common/CombinedCardinalityEstimator.h>
19+
#include <vec/common/assert_cast.h>
20+
#include <vec/common/typeid_cast.h>
21+
22+
// #include <AggregateFunctions/UniquesHashSet.h>
23+
#include "vec/aggregate_functions/aggregate_function.h"
24+
// #include <AggregateFunctions/UniqVariadicHash.h>
25+
26+
namespace doris::vectorized {
27+
28+
/// uniqExact
29+
30+
template <typename T>
31+
struct AggregateFunctionUniqExactData {
32+
using Key = T;
33+
34+
/// When creating, the hash table must be small.
35+
using Set = HashSet<Key, HashCRC32<Key>, HashTableGrower<4>,
36+
HashTableAllocatorWithStackMemory<sizeof(Key) * (1 << 4)>>;
37+
38+
Set set;
39+
40+
static String getName() { return "uniqExact"; }
41+
};
42+
43+
/// For rows, we put the SipHash values (128 bits) into the hash table.
44+
template <>
45+
struct AggregateFunctionUniqExactData<String> {
46+
using Key = UInt128;
47+
48+
/// When creating, the hash table must be small.
49+
using Set = HashSet<Key, UInt128TrivialHash, HashTableGrower<3>,
50+
HashTableAllocatorWithStackMemory<sizeof(Key) * (1 << 3)>>;
51+
52+
Set set;
53+
54+
static String getName() { return "uniqExact"; }
55+
};
56+
57+
namespace detail {
58+
59+
/** The structure for the delegation work to add one element to the `uniq` aggregate functions.
60+
* Used for partial specialization to add strings.
61+
*/
62+
template <typename T, typename Data>
63+
struct OneAdder {
64+
static void ALWAYS_INLINE add(Data& data, const IColumn& column, size_t row_num) {
65+
if constexpr (std::is_same_v<Data, AggregateFunctionUniqExactData<T>>) {
66+
if constexpr (!std::is_same_v<T, String>) {
67+
data.set.insert(assert_cast<const ColumnVector<T>&>(column).getData()[row_num]);
68+
} else {
69+
StringRef value = column.getDataAt(row_num);
70+
71+
UInt128 key;
72+
SipHash hash;
73+
hash.update(value.data, value.size);
74+
hash.get128(key.low, key.high);
75+
76+
data.set.insert(key);
77+
}
78+
}
79+
}
80+
};
81+
82+
} // namespace detail
83+
84+
/// Calculates the number of different values approximately or exactly.
85+
template <typename T, typename Data>
86+
class AggregateFunctionUniq final
87+
: public IAggregateFunctionDataHelper<Data, AggregateFunctionUniq<T, Data>> {
88+
public:
89+
AggregateFunctionUniq(const DataTypes& argument_types_)
90+
: IAggregateFunctionDataHelper<Data, AggregateFunctionUniq<T, Data>>(argument_types_,
91+
{}) {}
92+
93+
String getName() const override { return Data::getName(); }
94+
95+
DataTypePtr getReturnType() const override { return std::make_shared<DataTypeUInt64>(); }
96+
97+
void add(AggregateDataPtr place, const IColumn** columns, size_t row_num,
98+
Arena*) const override {
99+
detail::OneAdder<T, Data>::add(this->data(place), *columns[0], row_num);
100+
}
101+
102+
void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena*) const override {
103+
this->data(place).set.merge(this->data(rhs).set);
104+
}
105+
106+
void serialize(ConstAggregateDataPtr place, std::ostream& buf) const override {
107+
// this->data(place).set.write(buf);
108+
}
109+
110+
void deserialize(AggregateDataPtr place, std::istream& buf, Arena*) const override {
111+
// this->data(place).set.read(buf);
112+
}
113+
114+
void insertResultInto(ConstAggregateDataPtr place, IColumn& to) const override {
115+
assert_cast<ColumnUInt64&>(to).getData().push_back(this->data(place).set.size());
116+
}
117+
118+
const char* getHeaderFilePath() const override { return __FILE__; }
119+
};
120+
121+
} // namespace doris::vectorized
Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
#pragma once
2+
3+
#include <vec/common/hash_table/hash.h>
4+
#include <vec/common/hash_table/hash_table.h>
5+
#include <vec/common/hash_table/hash_table_allocator.h>
6+
7+
// #include <IO/WriteBuffer.h>
8+
// #include <IO/WriteHelpers.h>
9+
// #include <IO/ReadBuffer.h>
10+
// #include <IO/ReadHelpers.h>
11+
// #include <IO/VarInt.h>
12+
13+
/** NOTE HashSet could only be used for memmoveable (position independent) types.
14+
* Example: std::string is not position independent in libstdc++ with C++11 ABI or in libc++.
15+
* Also, key must be of type, that zero bytes is compared equals to zero key.
16+
*/
17+
18+
template <typename Key, typename TCell, typename Hash = DefaultHash<Key>,
19+
typename Grower = HashTableGrower<>, typename Allocator = HashTableAllocator>
20+
class HashSetTable : public HashTable<Key, TCell, Hash, Grower, Allocator> {
21+
public:
22+
using Self = HashSetTable;
23+
using Cell = TCell;
24+
25+
using Base = HashTable<Key, TCell, Hash, Grower, Allocator>;
26+
using typename Base::LookupResult;
27+
28+
void merge(const Self& rhs) {
29+
if (!this->hasZero() && rhs.hasZero()) {
30+
this->setHasZero();
31+
++this->m_size;
32+
}
33+
34+
for (size_t i = 0; i < rhs.grower.bufSize(); ++i)
35+
if (!rhs.buf[i].isZero(*this)) this->insert(Cell::getKey(rhs.buf[i].getValue()));
36+
}
37+
38+
// void readAndMerge(DB::ReadBuffer & rb)
39+
// {
40+
// Cell::State::read(rb);
41+
42+
// size_t new_size = 0;
43+
// DB::readVarUInt(new_size, rb);
44+
45+
// this->resize(new_size);
46+
47+
// for (size_t i = 0; i < new_size; ++i)
48+
// {
49+
// Cell x;
50+
// x.read(rb);
51+
// this->insert(Cell::getKey(x.getValue()));
52+
// }
53+
// }
54+
};
55+
56+
template <typename Key, typename Hash, typename TState = HashTableNoState>
57+
struct HashSetCellWithSavedHash : public HashTableCell<Key, Hash, TState> {
58+
using Base = HashTableCell<Key, Hash, TState>;
59+
60+
size_t saved_hash;
61+
62+
HashSetCellWithSavedHash() : Base() {}
63+
HashSetCellWithSavedHash(const Key& key_, const typename Base::State& state)
64+
: Base(key_, state) {}
65+
66+
bool keyEquals(const Key& key_) const { return this->key == key_; }
67+
bool keyEquals(const Key& key_, size_t hash_) const {
68+
return saved_hash == hash_ && this->key == key_;
69+
}
70+
bool keyEquals(const Key& key_, size_t hash_, const typename Base::State&) const {
71+
return keyEquals(key_, hash_);
72+
}
73+
74+
void setHash(size_t hash_value) { saved_hash = hash_value; }
75+
size_t getHash(const Hash& /*hash_function*/) const { return saved_hash; }
76+
};
77+
78+
template <typename Key, typename Hash, typename State>
79+
ALWAYS_INLINE inline auto lookupResultGetKey(HashSetCellWithSavedHash<Key, Hash, State>* cell) {
80+
return &cell->key;
81+
}
82+
83+
template <typename Key, typename Hash, typename State>
84+
ALWAYS_INLINE inline void* lookupResultGetMapped(HashSetCellWithSavedHash<Key, Hash, State>*) {
85+
return nullptr;
86+
}
87+
88+
template <typename Key, typename Hash = DefaultHash<Key>, typename Grower = HashTableGrower<>,
89+
typename Allocator = HashTableAllocator>
90+
using HashSet = HashSetTable<Key, HashTableCell<Key, Hash>, Hash, Grower, Allocator>;
91+
92+
template <typename Key, typename Hash = DefaultHash<Key>, typename Grower = HashTableGrower<>,
93+
typename Allocator = HashTableAllocator>
94+
using HashSetWithSavedHash =
95+
HashSetTable<Key, HashSetCellWithSavedHash<Key, Hash>, Hash, Grower, Allocator>;

0 commit comments

Comments
 (0)