Skip to content

Commit 7138f6b

Browse files
stdpainHappenLee
authored andcommitted
support multi-tablet group by agg (apache#15)
1 parent 2a00fdf commit 7138f6b

6 files changed

Lines changed: 257 additions & 16 deletions

File tree

‎be/src/vec/core/block.cpp‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -696,4 +696,26 @@ void Block::serialize(PBlock* pblock) const {
696696
}
697697
}
698698

699+
int MutableBlock::rows() {
700+
for (const auto& column : _columns)
701+
if (column) return column->size();
702+
703+
return 0;
704+
}
705+
706+
void MutableBlock::add_row(const Block* block, int row) {
707+
auto& src_columns_with_schema = block->getColumnsWithTypeAndName();
708+
for (int i = 0; i < _columns.size(); ++i) {
709+
_columns[i]->insertFrom(*src_columns_with_schema[i].column.get(), row);
710+
}
711+
}
712+
713+
Block MutableBlock::to_block() {
714+
ColumnsWithTypeAndName columns_with_schema;
715+
for (int i = 0; i < _columns.size(); ++i) {
716+
columns_with_schema.emplace_back(std::move(_columns[i]), _data_types[i], "");
717+
}
718+
return columns_with_schema;
719+
}
720+
699721
} // namespace doris::vectorized

‎be/src/vec/core/block.h‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ class Block {
158158
std::string dumpData() const;
159159

160160
static void filter_block(Block* block, int filter_conlumn_id, int column_to_keep);
161-
// serialize block to PRowBatch
161+
// serialize block to PRowBatch
162162
void serialize(PBlock* pblock) const;
163163

164164
private:
@@ -180,4 +180,37 @@ using BlocksPtrs = std::shared_ptr<std::vector<BlocksPtr>>;
180180
/// Calculate difference in structure of blocks and write description into output strings. NOTE It doesn't compare values of constant columns.
181181
// void getBlocksDifference(const Block & lhs, const Block & rhs, std::string & out_lhs_diff, std::string & out_rhs_diff);
182182

183+
class MutableBlock {
184+
private:
185+
MutableColumns _columns;
186+
DataTypes _data_types;
187+
188+
public:
189+
MutableBlock() = default;
190+
~MutableBlock() = default;
191+
192+
MutableBlock(MutableColumns&& columns, DataTypes&& data_types)
193+
: _columns(std::move(columns)), _data_types(std::move(data_types)) {}
194+
195+
int rows();
196+
197+
bool empty() { return rows() == 0; }
198+
199+
MutableColumns& mutable_columns() { return _columns; }
200+
201+
DataTypes& data_types() { return _data_types; }
202+
203+
Block to_block();
204+
205+
void add_row(const Block* block, int row);
206+
207+
void clear() {
208+
_columns.clear();
209+
_data_types.clear();
210+
}
211+
212+
// TODO: use add_rows instead of this
213+
// add_rows(Block* block,PODArray<Int32>& group, int group_num);
214+
};
215+
183216
} // namespace doris::vectorized

‎be/src/vec/exec/vaggregation_node.cpp‎

Lines changed: 142 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -137,12 +137,23 @@ Status AggregationNode::prepare(RuntimeState* state) {
137137

138138
} else {
139139
_agg_data.init(AggregatedDataVariants::Type::serialized);
140+
if (_is_merge) {
141+
_executor.execute = std::bind<Status>(&AggregationNode::_merge_with_serialized_key,
142+
this, std::placeholders::_1);
143+
} else {
144+
_executor.execute = std::bind<Status>(&AggregationNode::_execute_with_serialized_key,
145+
this, std::placeholders::_1);
146+
}
140147

141-
_executor.execute = std::bind<Status>(&AggregationNode::_execute_with_serialized_key, this,
142-
std::placeholders::_1);
143-
_executor.get_result = std::bind<Status>(&AggregationNode::_get_with_serialized_key_result,
144-
this, std::placeholders::_1, std::placeholders::_2,
145-
std::placeholders::_3);
148+
if (_needs_finalize) {
149+
_executor.get_result = std::bind<Status>(
150+
&AggregationNode::_get_with_serialized_key_result, this, std::placeholders::_1,
151+
std::placeholders::_2, std::placeholders::_3);
152+
} else {
153+
_executor.get_result = std::bind<Status>(
154+
&AggregationNode::_serialize_with_serialized_key_result, this,
155+
std::placeholders::_1, std::placeholders::_2, std::placeholders::_3);
156+
}
146157
}
147158

148159
return Status::OK();
@@ -410,4 +421,130 @@ Status AggregationNode::_get_with_serialized_key_result(RuntimeState* state, Blo
410421
return Status::OK();
411422
}
412423

424+
Status AggregationNode::_serialize_with_serialized_key_result(RuntimeState* state, Block* block,
425+
bool* eos) {
426+
DCHECK(_agg_data.serialized != nullptr);
427+
using Method = AggregationMethodSerialized<AggregatedDataWithStringKey>;
428+
using AggState = Method::State;
429+
430+
auto& method = *_agg_data.serialized;
431+
auto& data = _agg_data.serialized->data;
432+
433+
int key_size = _probe_expr_ctxs.size();
434+
int agg_size = _aggregate_evaluators.size();
435+
MutableColumns value_columns(agg_size);
436+
DataTypes value_data_types(agg_size);
437+
438+
MutableColumns key_columns;
439+
for (int i = 0; i < key_size; ++i) {
440+
key_columns.emplace_back(_probe_expr_ctxs[i]->root()->data_type()->createColumn());
441+
}
442+
443+
// will serialize data to string column
444+
for (int i = 0; i < _aggregate_evaluators.size(); ++i) {
445+
value_data_types[i] = makeNullable(std::make_shared<DataTypeString>());
446+
value_columns[i] = value_data_types[i]->createColumn();
447+
}
448+
449+
data.forEachValue([&](const auto& key, auto& mapped) {
450+
// insert keys
451+
method.insertKeyIntoColumns(key, key_columns, {});
452+
453+
std::ostringstream buf;
454+
// serialize values
455+
for (size_t i = 0; i < _aggregate_evaluators.size(); ++i) {
456+
_aggregate_evaluators[i]->function()->serialize(
457+
mapped + _offsets_of_aggregate_states[i], buf);
458+
value_columns[i]->insertData(buf.str().c_str(), buf.str().length());
459+
buf.str().clear();
460+
}
461+
});
462+
463+
ColumnsWithTypeAndName columns_with_schema;
464+
for (int i = 0; i < key_size; ++i) {
465+
columns_with_schema.emplace_back(std::move(key_columns[i]),
466+
_probe_expr_ctxs[i]->root()->data_type(), "");
467+
}
468+
469+
for (int i = 0; i < agg_size; ++i) {
470+
columns_with_schema.emplace_back(std::move(value_columns[i]), value_data_types[i], "");
471+
}
472+
473+
*block = Block(columns_with_schema);
474+
*eos = true;
475+
return Status::OK();
476+
}
477+
478+
Status AggregationNode::_merge_with_serialized_key(Block* block) {
479+
DCHECK(!_probe_expr_ctxs.empty());
480+
// now we only support serialized key
481+
// TODO:
482+
DCHECK(_agg_data.serialized != nullptr);
483+
484+
using Method = AggregationMethodSerialized<AggregatedDataWithStringKey>;
485+
using AggState = Method::State;
486+
487+
auto& method = *_agg_data.serialized;
488+
489+
size_t key_size = _probe_expr_ctxs.size();
490+
ColumnRawPtrs key_columns(key_size);
491+
492+
for (size_t i = 0; i < key_size; ++i) {
493+
int result_column_id = -1;
494+
RETURN_IF_ERROR(_probe_expr_ctxs[i]->execute(block, &result_column_id));
495+
key_columns[i] = block->getByPosition(result_column_id).column.get();
496+
}
497+
498+
int rows = block->rows();
499+
PODArray<AggregateDataPtr> places(rows);
500+
501+
AggState state(key_columns, {}, nullptr);
502+
503+
/// For all rows.
504+
for (size_t i = 0; i < rows; ++i) {
505+
AggregateDataPtr aggregate_data = nullptr;
506+
507+
auto emplace_result = state.emplaceKey(method.data, i, _agg_arena_pool);
508+
509+
/// If a new key is inserted, initialize the states of the aggregate functions, and possibly something related to the key.
510+
if (emplace_result.isInserted()) {
511+
/// exception-safety - if you can not allocate memory or create states, then destructors will not be called.
512+
emplace_result.setMapped(nullptr);
513+
514+
aggregate_data = _agg_arena_pool.alignedAlloc(_total_size_of_aggregate_states,
515+
_align_aggregate_states);
516+
_create_agg_status(aggregate_data);
517+
518+
emplace_result.setMapped(aggregate_data);
519+
} else
520+
aggregate_data = emplace_result.getMapped();
521+
522+
places[i] = aggregate_data;
523+
assert(places[i] != nullptr);
524+
}
525+
526+
std::unique_ptr<char[]> deserialize_buffer(new char[_total_size_of_aggregate_states]);
527+
for (int i = 0; i < _aggregate_evaluators.size(); ++i) {
528+
auto column = block->getByPosition(i + key_size).column;
529+
if (column->isNullable()) {
530+
column = ((ColumnNullable*)column.get())->getNestedColumnPtr();
531+
}
532+
for (int j = 0; j < rows; ++j) {
533+
std::string data_buffer;
534+
StringRef ref = column->getDataAt(j);
535+
data_buffer.assign(ref.data, ref.size);
536+
std::istringstream buf(data_buffer);
537+
538+
_aggregate_evaluators[i]->function()->deserialize(
539+
deserialize_buffer.get() + _offsets_of_aggregate_states[i], buf,
540+
&_agg_arena_pool);
541+
542+
_aggregate_evaluators[i]->function()->merge(
543+
places.data()[j] + _offsets_of_aggregate_states[i],
544+
deserialize_buffer.get() + _offsets_of_aggregate_states[i], &_agg_arena_pool);
545+
}
546+
}
547+
return Status::OK();
548+
}
549+
413550
} // namespace doris::vectorized

‎be/src/vec/exec/vaggregation_node.h‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,13 +139,16 @@ class AggregationNode : public ::doris::ExecNode {
139139

140140
private:
141141
Status _create_agg_status(AggregateDataPtr data);
142+
142143
Status _get_without_key_result(RuntimeState* state, Block* block, bool* eos);
143144
Status _serialize_without_key(RuntimeState* state, Block* block, bool* eos);
144145
Status _execute_without_key(Block* block);
145146
Status _merge_without_key(Block* block);
146147

147-
Status _execute_with_serialized_key(Block* block);
148148
Status _get_with_serialized_key_result(RuntimeState* state, Block* block, bool* eos);
149+
Status _serialize_with_serialized_key_result(RuntimeState* state, Block* block, bool* eos);
150+
Status _execute_with_serialized_key(Block* block);
151+
Status _merge_with_serialized_key(Block* block);
149152

150153
using vectorized_execute = std::function<Status(Block* block)>;
151154
using vectorized_get_result =

‎be/src/vec/sink/vdata_stream_sender.cpp‎

Lines changed: 51 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
#include "runtime/exec_env.h"
99
#include "runtime/mem_tracker.h"
1010
#include "runtime/runtime_state.h"
11+
#include "vec/common/sip_hash.h"
1112
#include "vec/runtime/vpartition_info.h"
1213

1314
namespace doris::vectorized {
@@ -17,7 +18,7 @@ Status VDataStreamSender::Channel::init(RuntimeState* state) {
1718

1819
_capacity = std::max(1, _buffer_size / std::max(_row_desc.get_row_size(), 1));
1920
//_batch.reset(new RowBatch(_row_desc, capacity, _parent->_mem_tracker.get()));
20-
_block.reset(new Block());
21+
_mutable_block.reset(new MutableBlock());
2122

2223
if (_brpc_dest_addr.hostname.empty()) {
2324
LOG(WARNING) << "there is no brpc destination address's hostname"
@@ -46,13 +47,14 @@ Status VDataStreamSender::Channel::init(RuntimeState* state) {
4647
Status VDataStreamSender::Channel::send_current_block(bool eos) {
4748
{
4849
SCOPED_TIMER(_parent->_serialize_batch_timer);
49-
_block->serialize(&_pb_block);
50+
_pb_block.Clear();
51+
_mutable_block->to_block().serialize(&_pb_block);
52+
_mutable_block->clear();
5053
int uncompressed_bytes = _pb_block.ByteSize();
5154
int bytes = uncompressed_bytes;
5255
COUNTER_UPDATE(_parent->_bytes_sent_counter, bytes);
5356
COUNTER_UPDATE(_parent->_uncompressed_bytes_counter, uncompressed_bytes);
5457
}
55-
_block->clear();
5658
RETURN_IF_ERROR(send_block(&_pb_block, eos));
5759
return Status::OK();
5860
}
@@ -87,6 +89,23 @@ Status VDataStreamSender::Channel::send_block(PBlock* block, bool eos) {
8789
return Status::OK();
8890
}
8991

92+
Status VDataStreamSender::Channel::add_row(Block* block, int row) {
93+
if (_fragment_instance_id.lo == -1) {
94+
return Status::OK();
95+
}
96+
int batch_size = _parent->state()->batch_size();
97+
if (block->rows() == batch_size) {
98+
RETURN_IF_ERROR(send_current_block());
99+
}
100+
if (_mutable_block->rows() == 0) {
101+
auto empty_block = block->cloneEmpty();
102+
_mutable_block.reset(
103+
new MutableBlock(empty_block.mutateColumns(), empty_block.getDataTypes()));
104+
}
105+
_mutable_block->add_row(block, row);
106+
return Status::OK();
107+
}
108+
90109
Status VDataStreamSender::Channel::close_wait(RuntimeState* state) {
91110
if (_need_close) {
92111
Status st = _wait_last_brpc();
@@ -96,7 +115,7 @@ Status VDataStreamSender::Channel::close_wait(RuntimeState* state) {
96115
_need_close = false;
97116
return st;
98117
}
99-
_block.reset();
118+
_mutable_block.reset();
100119
return Status::OK();
101120
}
102121

@@ -106,8 +125,8 @@ Status VDataStreamSender::Channel::close_internal() {
106125
}
107126
VLOG_RPC << "Channel::close() instance_id=" << _fragment_instance_id
108127
<< " dest_node=" << _dest_node_id
109-
<< " #rows= " << ((_block == nullptr) ? 0 : _block->rows());
110-
if (_block != nullptr && _block->rows() > 0) {
128+
<< " #rows= " << ((_mutable_block == nullptr) ? 0 : _mutable_block->rows());
129+
if (_mutable_block != nullptr && _mutable_block->rows() > 0) {
111130
RETURN_IF_ERROR(send_current_block(true));
112131
} else {
113132
RETURN_IF_ERROR(send_block(nullptr, true));
@@ -281,8 +300,32 @@ Status VDataStreamSender::send(RuntimeState* state, Block* block) {
281300
// 3. send batch
282301
// 4. switch proto
283302
} else if (_part_type == TPartitionType::HASH_PARTITIONED) {
284-
// 1. caculate hash
285-
// 2. dispatch rows to channel
303+
// TODO: use vectorized hash caculate
304+
int num_channels = _channels.size();
305+
// will only copy schema
306+
// we don't want send temp columns
307+
auto send_block = *block;
308+
309+
std::vector<int> result(_partition_expr_ctxs.size());
310+
int counter = 0;
311+
312+
for (auto ctx : _partition_expr_ctxs) {
313+
RETURN_IF_ERROR(ctx->execute(block, &result[counter++]));
314+
}
315+
316+
// caculate hash
317+
int rows = block->rows();
318+
for (int i = 0; i < rows; ++i) {
319+
SipHash siphash;
320+
for (int j = 0; j < result.size(); ++j) {
321+
auto column = block->getByPosition(result[j]).column;
322+
column->updateHashWithValue(i, siphash);
323+
}
324+
auto target_channel_id = siphash.get64() % num_channels;
325+
RETURN_IF_ERROR(_channels[target_channel_id]->add_row(&send_block, i));
326+
}
327+
return Status::OK();
328+
286329
} else if (_part_type == TPartitionType::BUCKET_SHFFULE_HASH_PARTITIONED) {
287330
// 1. caculate hash
288331
// 2. dispatch rows to channel

‎be/src/vec/sink/vdata_stream_sender.h‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,8 @@ class VDataStreamSender::Channel {
155155
// if batch is nullptr, send the eof packet
156156
Status send_block(PBlock* block, bool eos = false);
157157

158+
Status add_row(Block* block, int row);
159+
158160
Status send_current_block(bool eos = false);
159161

160162
// Flush buffered rows and close channel. This function don't wait the response
@@ -212,7 +214,8 @@ class VDataStreamSender::Channel {
212214

213215
// we're accumulating rows into this batch
214216
//boost::scoped_ptr<RowBatch> _batch;
215-
std::unique_ptr<Block> _block;
217+
// std::unique_ptr<Block> _block;
218+
std::unique_ptr<MutableBlock> _mutable_block;
216219

217220
bool _need_close;
218221
int _be_number;

0 commit comments

Comments
 (0)