Skip to content

Commit 19d0370

Browse files
stdpainHappenLee
authored andcommitted
[AGG] support multi-tablet no group AGG (apache#14)
* support mutl-tablet no group agg
1 parent d4ba082 commit 19d0370

2 files changed

Lines changed: 89 additions & 5 deletions

File tree

‎be/src/vec/exec/aggregation_node.cpp‎

Lines changed: 86 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
#include "runtime/mem_pool.h"
2222
#include "runtime/row_batch.h"
2323
#include "vec/core/block.h"
24+
#include "vec/data_types/data_type_nullable.h"
2425
#include "vec/exprs/vexpr.h"
2526
#include "vec/exprs/vexpr_context.h"
2627
#include "vec/exprs/vslot_ref.h"
@@ -36,6 +37,7 @@ AggregationNode::AggregationNode(ObjectPool* pool, const TPlanNode& tnode,
3637
_output_tuple_id(tnode.agg_node.output_tuple_id),
3738
_output_tuple_desc(NULL),
3839
_needs_finalize(tnode.agg_node.need_finalize),
40+
_is_merge(false),
3941
_agg_data() {}
4042

4143
AggregationNode::~AggregationNode() {}
@@ -54,6 +56,10 @@ Status AggregationNode::init(const TPlanNode& tnode, RuntimeState* state) {
5456
AggFnEvaluator::create(_pool, tnode.agg_node.aggregate_functions[i], &evaluator));
5557
_aggregate_evaluators.push_back(evaluator);
5658
}
59+
60+
auto agg_functions = tnode.agg_node.aggregate_functions;
61+
_is_merge = std::any_of(agg_functions.begin(), agg_functions.end(),
62+
[](const auto& e) { return e.nodes[0].agg_expr.is_merge_agg; });
5763
return Status::OK();
5864
}
5965

@@ -111,11 +117,23 @@ Status AggregationNode::prepare(RuntimeState* state) {
111117

112118
_create_agg_status(_agg_data.without_key);
113119

114-
_executor.execute = std::bind<Status>(&AggregationNode::_execute_without_key, this,
115-
std::placeholders::_1);
116-
_executor.get_result = std::bind<Status>(&AggregationNode::_get_without_key_result, this,
117-
std::placeholders::_1, std::placeholders::_2,
118-
std::placeholders::_3);
120+
if (_is_merge) {
121+
_executor.execute = std::bind<Status>(&AggregationNode::_merge_without_key, this,
122+
std::placeholders::_1);
123+
} else {
124+
_executor.execute = std::bind<Status>(&AggregationNode::_execute_without_key, this,
125+
std::placeholders::_1);
126+
}
127+
128+
if (_needs_finalize) {
129+
_executor.get_result = std::bind<Status>(&AggregationNode::_get_without_key_result,
130+
this, std::placeholders::_1,
131+
std::placeholders::_2, std::placeholders::_3);
132+
} else {
133+
_executor.get_result = std::bind<Status>(&AggregationNode::_serialize_without_key, this,
134+
std::placeholders::_1, std::placeholders::_2,
135+
std::placeholders::_3);
136+
}
119137

120138
} else {
121139
_agg_data.init(AggregatedDataVariants::Type::serialized);
@@ -206,6 +224,7 @@ Status AggregationNode::_get_without_key_result(RuntimeState* state, Block* bloc
206224
const auto column_type = block_schema[i].type;
207225
if (!column_type->equals(*data_types[i])) {
208226
DCHECK(column_type->isNullable());
227+
DCHECK(((DataTypeNullable*)column_type.get())->getNestedType()->equals(*data_types[i]));
209228
DCHECK(!data_types[i]->isNullable());
210229
ColumnPtr ptr = std::move(columns[i]);
211230
ptr = makeNullable(ptr);
@@ -218,6 +237,41 @@ Status AggregationNode::_get_without_key_result(RuntimeState* state, Block* bloc
218237
return Status::OK();
219238
}
220239

240+
Status AggregationNode::_serialize_without_key(RuntimeState* state, Block* block, bool* eos) {
241+
DCHECK(_agg_data.without_key != nullptr);
242+
int agg_size = _aggregate_evaluators.size();
243+
244+
MutableColumns value_columns(agg_size);
245+
std::vector<DataTypePtr> data_types(agg_size);
246+
247+
// will serialize data to string column
248+
for (int i = 0; i < _aggregate_evaluators.size(); ++i) {
249+
data_types[i] = makeNullable(std::make_shared<DataTypeString>());
250+
value_columns[i] = data_types[i]->createColumn();
251+
}
252+
253+
// TODO: we could use pod char as buffer instead of ostream ?
254+
std::ostringstream buf;
255+
for (int i = 0; i < _aggregate_evaluators.size(); ++i) {
256+
_aggregate_evaluators[i]->function()->serialize(
257+
_agg_data.without_key + _offsets_of_aggregate_states[i], buf);
258+
value_columns[i]->insertData(buf.str().c_str(), buf.str().length());
259+
buf.str().clear();
260+
}
261+
{
262+
ColumnsWithTypeAndName data_with_schema;
263+
for (int i = 0; i < _aggregate_evaluators.size(); ++i) {
264+
ColumnWithTypeAndName column_with_schema = {nullptr, data_types[i], ""};
265+
data_with_schema.push_back(std::move(column_with_schema));
266+
}
267+
*block = Block(data_with_schema);
268+
}
269+
270+
block->setColumns(std::move(value_columns));
271+
*eos = true;
272+
return Status::OK();
273+
}
274+
221275
Status AggregationNode::_execute_without_key(Block* block) {
222276
DCHECK(_agg_data.without_key != nullptr);
223277
for (int i = 0; i < _aggregate_evaluators.size(); ++i) {
@@ -227,6 +281,33 @@ Status AggregationNode::_execute_without_key(Block* block) {
227281
return Status::OK();
228282
}
229283

284+
Status AggregationNode::_merge_without_key(Block* block) {
285+
DCHECK(_agg_data.without_key != nullptr);
286+
std::unique_ptr<char[]> deserialize_buffer(new char[_total_size_of_aggregate_states]);
287+
int rows = block->rows();
288+
for (int i = 0; i < _aggregate_evaluators.size(); ++i) {
289+
auto column = block->getByPosition(i).column;
290+
if (column->isNullable()) {
291+
column = ((ColumnNullable*)column.get())->getNestedColumnPtr();
292+
}
293+
for (int j = 0; j < rows; ++j) {
294+
std::string data_buffer;
295+
StringRef ref = column->getDataAt(j);
296+
data_buffer.assign(ref.data, ref.size);
297+
std::istringstream buf(data_buffer);
298+
299+
_aggregate_evaluators[i]->function()->deserialize(
300+
deserialize_buffer.get() + _offsets_of_aggregate_states[i], buf,
301+
&_agg_arena_pool);
302+
303+
_aggregate_evaluators[i]->function()->merge(
304+
_agg_data.without_key + _offsets_of_aggregate_states[i],
305+
deserialize_buffer.get() + _offsets_of_aggregate_states[i], &_agg_arena_pool);
306+
}
307+
}
308+
return Status::OK();
309+
}
310+
230311
Status AggregationNode::_execute_with_serialized_key(Block* block) {
231312
DCHECK(!_probe_expr_ctxs.empty());
232313
// now we only support serialized key

‎be/src/vec/exec/aggregation_node.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@ class AggregationNode : public ::doris::ExecNode {
122122
TupleDescriptor* _output_tuple_desc;
123123

124124
bool _needs_finalize;
125+
bool _is_merge;
125126
std::unique_ptr<MemPool> _mem_pool;
126127

127128
size_t _align_aggregate_states = 1;
@@ -138,7 +139,9 @@ class AggregationNode : public ::doris::ExecNode {
138139
private:
139140
Status _create_agg_status(AggregateDataPtr data);
140141
Status _get_without_key_result(RuntimeState* state, Block* block, bool* eos);
142+
Status _serialize_without_key(RuntimeState* state, Block* block, bool* eos);
141143
Status _execute_without_key(Block* block);
144+
Status _merge_without_key(Block* block);
142145

143146
Status _execute_with_serialized_key(Block* block);
144147
Status _get_with_serialized_key_result(RuntimeState* state, Block* block, bool* eos);

0 commit comments

Comments
 (0)