Skip to content

Commit 087cc7f

Browse files
authored
Merge pull request #4 from liutang123/pip-profile
[Exec][Pipeline] Add profile
2 parents af481e5 + dcd88f1 commit 087cc7f

23 files changed

Lines changed: 340 additions & 79 deletions

‎be/src/exec/exec_node.cpp‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -88,10 +88,6 @@ namespace doris {
8888

8989
const std::string ExecNode::ROW_THROUGHPUT_COUNTER = "RowsReturnedRate";
9090

91-
int ExecNode::get_node_id_from_profile(RuntimeProfile* p) {
92-
return p->metadata();
93-
}
94-
9591
ExecNode::RowBatchQueue::RowBatchQueue(int max_batches) : BlockingQueue<RowBatch*>(max_batches) {}
9692

9793
ExecNode::RowBatchQueue::~RowBatchQueue() {

‎be/src/exec/exec_node.h‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -213,9 +213,6 @@ class ExecNode {
213213

214214
virtual std::string get_name();
215215

216-
// Extract node id from p->name().
217-
static int get_node_id_from_profile(RuntimeProfile* p);
218-
219216
// Names of counters shared by all exec nodes
220217
static const std::string ROW_THROUGHPUT_COUNTER;
221218

‎be/src/pipeline/exec/aggregation_sink_operator.cpp‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -65,16 +65,16 @@ bool AggSinkOperator::can_write() {
6565
}
6666

6767
Status AggSinkOperator::sink(RuntimeState* state, vectorized::Block* in_block, bool eos) {
68-
if ((!in_block || in_block->rows() == 0) && !eos) {
69-
return Status::OK();
70-
}
68+
SCOPED_TIMER(_runtime_profile->total_time_counter());
7169
if (!_agg_node->is_streaming_preagg()) {
70+
if (!in_block || in_block->rows() == 0) {
71+
return Status::OK();
72+
}
7273
RETURN_IF_ERROR(_agg_node->_executor.execute(in_block));
74+
_num_rows_returned += in_block->rows();
75+
COUNTER_SET(_rows_returned_counter, _num_rows_returned);
7376
} else {
74-
// TODO pipeline 不能这里是返回block
75-
7677
Status ret = Status::OK();
77-
7878
if (in_block && in_block->rows() > 0) {
7979
auto* bock_from_ctx = _agg_context->get_free_block();
8080
ret = _agg_node->_executor.pre_agg(in_block, bock_from_ctx);
@@ -89,6 +89,7 @@ Status AggSinkOperator::sink(RuntimeState* state, vectorized::Block* in_block, b
8989
_agg_node->_make_nullable_output_key(bock_from_ctx);
9090
_agg_context->push_block(bock_from_ctx);
9191
COUNTER_SET(_agg_node->_rows_returned_counter, _num_rows_returned);
92+
COUNTER_SET(_rows_returned_counter, _num_rows_returned);
9293
}
9394
}
9495

@@ -109,6 +110,7 @@ Status AggSinkOperator::sink(RuntimeState* state, vectorized::Block* in_block, b
109110
_agg_node->_make_nullable_output_key(bock_from_ctx);
110111
_agg_context->push_block(bock_from_ctx);
111112
COUNTER_SET(_agg_node->_rows_returned_counter, _num_rows_returned);
113+
COUNTER_SET(_rows_returned_counter, _num_rows_returned);
112114
}
113115
}
114116
// 尽快出发下游结束,这里不在finalize中做。

‎be/src/pipeline/exec/exchange_sink_operator.cpp‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -105,14 +105,9 @@ Status ExchangeSinkOperator::prepare(RuntimeState* state) {
105105
for (const auto& channel : _sink->_channels) {
106106
instances.emplace_back(channel->get_fragment_instance_id_str());
107107
}
108-
// std::string title = fmt::format("ExchangeSinkOperator (dst_id={}, dst_fragments=[{}])",
109-
// _sink->_dest_node_id, instances);
110-
std::string title = "ExchangeSinkOperator dst_id, dst_fragments TODO";
111-
// sink的pool用的是context的
112-
_profile = _pool->add(new RuntimeProfile(title));
113-
SCOPED_TIMER(_profile->total_time_counter());
114108
_mem_tracker = std::make_unique<MemTracker>(
115-
"ExchangeSinkOperator:" + print_id(state->fragment_instance_id()), _profile);
109+
"ExchangeSinkOperator:" + print_id(state->fragment_instance_id()),
110+
_runtime_profile.get());
116111
SCOPED_CONSUME_MEM_TRACKER(_mem_tracker.get());
117112

118113
if (_part_type == TPartitionType::UNPARTITIONED || _part_type == TPartitionType::RANDOM) {
@@ -174,7 +169,7 @@ Status ExchangeSinkOperator::finalize(RuntimeState* state) {
174169
}
175170

176171
Status ExchangeSinkOperator::sink(RuntimeState* state, vectorized::Block* block, bool eos) {
177-
SCOPED_TIMER(_profile->total_time_counter());
172+
SCOPED_TIMER(_runtime_profile->total_time_counter());
178173
SCOPED_CONSUME_MEM_TRACKER(_mem_tracker.get());
179174
if (_part_type == TPartitionType::UNPARTITIONED || _channels.size() == 1) {
180175
// 1. serialize depends on it is not local exchange
@@ -270,6 +265,10 @@ Status ExchangeSinkOperator::sink(RuntimeState* state, vectorized::Block* block,
270265
// 1. calculate range
271266
// 2. dispatch rows to channel
272267
}
268+
if(block) {
269+
_num_rows_returned += block->rows();
270+
COUNTER_SET(_rows_returned_counter, _num_rows_returned);
271+
}
273272
return Status::OK();
274273
}
275274

‎be/src/pipeline/exec/exchange_sink_operator.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,6 @@ class ExchangeSinkOperator : public Operator {
5959
std::vector<vectorized::VPartitionInfo*> _partition_infos;
6060
std::unique_ptr<SinkBuffer> _sink_buffer;
6161
vectorized::VDataStreamSender* _sink;
62-
RuntimeProfile* _profile = nullptr; // Allocated from _pool
6362
RuntimeState* _state = nullptr;
6463

6564
std::vector<Channel*> _channels;

‎be/src/pipeline/exec/exchange_source_operator.cpp‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -41,17 +41,13 @@ Status ExchangeSourceOperator::prepare(RuntimeState* state) {
4141
_exchange_node->_id, _exchange_node->_num_senders, config::exchg_node_buffer_size_bytes,
4242
_exchange_node->_runtime_profile.get(), _exchange_node->_is_merging,
4343
_exchange_node->_sub_plan_query_statistics_recvr);
44-
_exchange_node->_rows_returned_counter =
45-
ADD_COUNTER(_runtime_profile, "RowsReturned", TUnit::UNIT);
44+
_exchange_node->_rows_returned_counter = _rows_returned_counter;
4645

4746
// TODO pipeline 支持merge读
48-
if (_exchange_node->_is_merging) {
49-
return Status::NotSupported("Now pipeline shuffle not support merging.");
50-
}
5147
if (_exchange_node->_is_merging) {
5248
return Status::NotSupported("Not Implemented merging exchange source operator");
5349
}
54-
return Status::OK();
50+
return Operator::prepare(state);
5551
}
5652

5753
Status ExchangeSourceOperator::open(RuntimeState* state) {
@@ -72,7 +68,12 @@ bool ExchangeSourceOperator::can_read() {
7268
}
7369

7470
Status ExchangeSourceOperator::get_block(RuntimeState* state, vectorized::Block* block, bool* eos) {
75-
return _exchange_node->get_next(state, block, eos);
71+
SCOPED_TIMER(_runtime_profile->total_time_counter());
72+
auto st = _exchange_node->get_next(state, block, eos);
73+
if (block) {
74+
_num_rows_returned += block->rows();
75+
}
76+
return st;
7677
// TODO pipeline 支持merge读
7778
// VDataStreamRecvr::get_next
7879
// VSortedRunMerger::get_next

‎be/src/pipeline/exec/final_aggregation_source_operator.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ bool FinalAggSourceOperator::can_read() {
5050
}
5151

5252
Status FinalAggSourceOperator::get_block(RuntimeState* state, vectorized::Block* block, bool* eos) {
53+
SCOPED_TIMER(_runtime_profile->total_time_counter());
5354
RETURN_IF_ERROR(_agg_node->_executor.get_result(state, block, eos));
5455
_agg_node->_make_nullable_output_key(block);
5556
// dispose the having clause, should not be execute in prestreaming agg

‎be/src/pipeline/exec/operator.cpp‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,10 @@
2020
namespace doris::pipeline {
2121

2222
Operator::Operator(OperatorTemplate* operator_template)
23-
: _operator_template(operator_template), _limit(-1), _is_closed(false) {}
23+
: _operator_template(operator_template),
24+
_num_rows_returned(0),
25+
_limit(-1),
26+
_is_closed(false) {}
2427

2528
bool Operator::is_sink() const {
2629
return _operator_template->is_sink();
@@ -31,14 +34,30 @@ bool Operator::is_source() const {
3134
}
3235

3336
Status Operator::init(const ExecNode* exec_node, RuntimeState* state) {
34-
_runtime_profile.reset(new RuntimeProfile("Operator"));
35-
_runtime_profile->set_metadata(_operator_template->id());
37+
_runtime_profile.reset(new RuntimeProfile(_operator_template->get_name()));
38+
_rows_returned_counter = ADD_COUNTER(_runtime_profile, "RowsReturned", TUnit::UNIT);
39+
_rows_returned_rate = runtime_profile()->add_derived_counter(
40+
ExecNode::ROW_THROUGHPUT_COUNTER, TUnit::UNIT_PER_SECOND,
41+
std::bind<int64_t>(&RuntimeProfile::units_per_second, _rows_returned_counter,
42+
runtime_profile()->total_time_counter()),
43+
"");
3644
if (exec_node && exec_node->limit() >= 0) {
3745
_limit = exec_node->limit();
3846
}
3947
return Status::OK();
4048
}
4149

50+
Status Operator::link_profile(RuntimeProfile* parent) {
51+
if (!_runtime_profile) {
52+
return Status::InternalError("link profile error");
53+
}
54+
if (_child) {
55+
RETURN_IF_ERROR(_child->link_profile(_runtime_profile.get()));
56+
}
57+
parent->add_child(_runtime_profile.get(), true, nullptr);
58+
return Status::OK();
59+
}
60+
4261
Status Operator::prepare(RuntimeState* state) {
4362
_mem_tracker = std::make_unique<MemTracker>("Operator:" + _runtime_profile->name(),
4463
_runtime_profile.get());
@@ -66,6 +85,9 @@ Status Operator::close(RuntimeState* state) {
6685
if (_operator_template->exec_node()) {
6786
_operator_template->exec_node()->close_self(state);
6887
}
88+
if (_rows_returned_counter != nullptr) {
89+
COUNTER_SET(_rows_returned_counter, _num_rows_returned);
90+
}
6991
return Status::OK();
7092
}
7193

‎be/src/pipeline/exec/operator.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,9 @@ class Operator {
153153

154154
int64_t rows_returned() const { return _num_rows_returned; }
155155

156+
Status link_profile(RuntimeProfile* parent);
157+
RuntimeProfile* runtime_profile() { return _runtime_profile.get(); }
158+
156159
protected:
157160
std::unique_ptr<MemTracker> _mem_tracker;
158161

@@ -163,6 +166,11 @@ class Operator {
163166

164167
std::unique_ptr<RuntimeProfile> _runtime_profile;
165168
int64_t _num_rows_returned;
169+
RuntimeProfile::Counter* _rows_returned_counter = nullptr;
170+
RuntimeProfile::Counter* _rows_returned_rate = nullptr;
171+
// TODO pipeline Account for peak memory used by this operator
172+
RuntimeProfile::Counter* _memory_used_counter = nullptr;
173+
166174
int64_t _limit;
167175

168176
private:

‎be/src/pipeline/exec/pre_aggregation_source_operator.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ bool PreAggSourceOperator::can_read() {
4343
}
4444

4545
Status PreAggSourceOperator::get_block(RuntimeState* state, vectorized::Block* block, bool* eos) {
46+
SCOPED_TIMER(_runtime_profile->total_time_counter());
4647
vectorized::Block* agg_block = nullptr;
4748
RETURN_IF_ERROR(_agg_context->get_block(&agg_block, eos));
4849

@@ -53,6 +54,8 @@ Status PreAggSourceOperator::get_block(RuntimeState* state, vectorized::Block* b
5354
DCHECK(agg_block != nullptr);
5455

5556
block->swap(*agg_block);
57+
_num_rows_returned += block->rows();
58+
COUNTER_SET(_rows_returned_counter, _num_rows_returned);
5659
_agg_context->return_free_block(agg_block);
5760

5861
return Status::OK();

0 commit comments

Comments
 (0)