Skip to content

Commit 5d17948

Browse files
authored
Merge pull request apache#17 from HappenLee/pipeline
Refactor the code of Agg Source Opeartor / Sink Operator
2 parents 3be25d6 + b84f813 commit 5d17948

8 files changed

Lines changed: 53 additions & 55 deletions

‎be/src/pipeline/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ set(PIPELINE_FILES
3434
exec/sink_buffer.cpp
3535
exec/result_sink_operator.cpp
3636
exec/aggregation_sink_operator.cpp
37-
exec/final_aggregation_source_operator.cpp
37+
exec/aggregation_source_operator.cpp
3838
exec/pre_aggregation_source_operator.cpp
3939
exec/agg_context.cpp
4040
exec/sort_source_operator.cpp

‎be/src/pipeline/exec/aggregation_sink_operator.cpp‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -56,12 +56,7 @@ bool AggSinkOperator::can_write() {
5656
Status AggSinkOperator::sink(RuntimeState* state, vectorized::Block* in_block, bool eos) {
5757
SCOPED_TIMER(_runtime_profile->total_time_counter());
5858
if (!_agg_node->is_streaming_preagg()) {
59-
if (!in_block || in_block->rows() == 0) {
60-
return Status::OK();
61-
}
62-
RETURN_IF_ERROR(_agg_node->_executor.execute(in_block));
63-
_num_rows_returned += in_block->rows();
64-
COUNTER_SET(_rows_returned_counter, _num_rows_returned);
59+
RETURN_IF_ERROR(_agg_node->sink(state, in_block, eos));
6560
} else {
6661
Status ret = Status::OK();
6762
if (in_block && in_block->rows() > 0) {
@@ -106,6 +101,7 @@ Status AggSinkOperator::sink(RuntimeState* state, vectorized::Block* in_block, b
106101
_agg_context->set_finish();
107102
}
108103
}
104+
// TODO: remove it after we split the stream agg and normal agg
109105
_agg_node->_executor.update_memusage();
110106
return Status::OK();
111107
}

‎be/src/pipeline/exec/aggregation_sink_operator.h‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,17 +40,14 @@ class AggSinkOperator : public Operator {
4040

4141
Status sink(RuntimeState* state, vectorized::Block* block, bool eos) override;
4242

43-
Status finalize(RuntimeState* state) override {
44-
_is_finalized = true;
45-
return Status::OK();
46-
}
4743
bool can_write() override;
4844

4945
Status close(RuntimeState* state) override;
5046

47+
Status finalize(doris::RuntimeState* state) override { return Status::OK(); }
48+
5149
private:
5250
vectorized::AggregationNode* _agg_node;
53-
bool _is_finalized;
5451
vectorized::Block _preagg_block = vectorized::Block();
5552

5653
int64_t get_child_return_rows() { return _child->rows_returned(); }

be/src/pipeline/exec/final_aggregation_source_operator.cpp renamed to be/src/pipeline/exec/aggregation_source_operator.cpp

Lines changed: 17 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -15,63 +15,58 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
#include "final_aggregation_source_operator.h"
18+
#include "aggregation_source_operator.h"
1919

2020
#include "vec/exec/vaggregation_node.h"
2121

2222
namespace doris {
2323
namespace pipeline {
2424

25-
FinalAggSourceOperator::FinalAggSourceOperator(OperatorTemplate* templ,
26-
vectorized::AggregationNode* node)
25+
AggregationSourceOperator::AggregationSourceOperator(OperatorTemplate* templ,
26+
vectorized::AggregationNode* node)
2727
: Operator(templ), _agg_node(node) {}
28-
Status FinalAggSourceOperator::init(ExecNode* exec_node, RuntimeState* state) {
28+
29+
Status AggregationSourceOperator::init(ExecNode* exec_node, RuntimeState* state) {
2930
RETURN_IF_ERROR(Operator::init(exec_node, state));
3031
return Status::OK();
3132
}
3233

3334
// for poc
34-
Status FinalAggSourceOperator::prepare(RuntimeState* state) {
35+
Status AggregationSourceOperator::prepare(RuntimeState* state) {
3536
_mem_tracker = std::make_unique<MemTracker>(
36-
"FinalAggSourceOperator:" + _runtime_profile->name(), _runtime_profile.get());
37+
"AggregationSourceOperator:" + _runtime_profile->name(), _runtime_profile.get());
3738
return Status::OK();
3839
}
3940

4041
// for poc
41-
Status FinalAggSourceOperator::open(RuntimeState* state) {
42+
Status AggregationSourceOperator::open(RuntimeState* state) {
4243
return Status::OK();
4344
}
4445

45-
bool FinalAggSourceOperator::can_read() {
46-
// TODO: Rethink use can_read() to replace the dependency
47-
return true;
46+
bool AggregationSourceOperator::can_read() {
47+
return _agg_node->can_read();
4848
}
4949

50-
Status FinalAggSourceOperator::get_block(RuntimeState* state, vectorized::Block* block, bool* eos) {
51-
SCOPED_TIMER(_runtime_profile->total_time_counter());
52-
RETURN_IF_ERROR(_agg_node->_executor.get_result(state, block, eos));
53-
_agg_node->_make_nullable_output_key(block);
54-
// dispose the having clause, should not be execute in prestreaming agg
55-
RETURN_IF_ERROR(vectorized::VExprContext::filter_block(_agg_node->_vconjunct_ctx_ptr, block,
56-
block->columns()));
57-
reached_limit(block, eos);
50+
Status AggregationSourceOperator::get_block(RuntimeState* state, vectorized::Block* block,
51+
bool* eos) {
52+
RETURN_IF_ERROR(_agg_node->pull(state, block, eos));
5853
_agg_node->_executor.update_memusage();
5954
return Status::OK();
6055
}
6156

62-
Status FinalAggSourceOperator::close(RuntimeState* state) {
57+
Status AggregationSourceOperator::close(RuntimeState* state) {
6358
_agg_node->release_resource(state);
6459
return Status::OK();
6560
}
6661

6762
/////////////////////////////// operator template ////////////////////////////////
6863

69-
FinalAggSourceOperatorTemplate::FinalAggSourceOperatorTemplate(
64+
AggregationSourceOperatorTemplate::AggregationSourceOperatorTemplate(
7065
int32_t id, const std::string& name, vectorized::AggregationNode* exec_node)
7166
: OperatorTemplate(id, name, exec_node) {}
7267

73-
OperatorPtr FinalAggSourceOperatorTemplate::build_operator() {
74-
return std::make_shared<FinalAggSourceOperator>(
68+
OperatorPtr AggregationSourceOperatorTemplate::build_operator() {
69+
return std::make_shared<AggregationSourceOperator>(
7570
this, assert_cast<vectorized::AggregationNode*>(_related_exec_node));
7671
}
7772

be/src/pipeline/exec/final_aggregation_source_operator.h renamed to be/src/pipeline/exec/aggregation_source_operator.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,9 @@ class AggregationNode;
2626
namespace pipeline {
2727

2828
// For read none streaming agg sink operator's data
29-
class FinalAggSourceOperator : public Operator {
29+
class AggregationSourceOperator : public Operator {
3030
public:
31-
FinalAggSourceOperator(OperatorTemplate*, vectorized::AggregationNode*);
31+
AggregationSourceOperator(OperatorTemplate*, vectorized::AggregationNode*);
3232
Status init(ExecNode* exec_node, RuntimeState* state = nullptr) override;
3333
Status prepare(RuntimeState* state) override;
3434
Status open(RuntimeState* state) override;
@@ -40,9 +40,9 @@ class FinalAggSourceOperator : public Operator {
4040
vectorized::AggregationNode* _agg_node;
4141
};
4242

43-
class FinalAggSourceOperatorTemplate : public OperatorTemplate {
43+
class AggregationSourceOperatorTemplate : public OperatorTemplate {
4444
public:
45-
FinalAggSourceOperatorTemplate(int32_t, const std::string&, vectorized::AggregationNode*);
45+
AggregationSourceOperatorTemplate(int32_t, const std::string&, vectorized::AggregationNode*);
4646

4747
bool is_source() const override { return true; }
4848

‎be/src/pipeline/pipeline_fragment_context.cpp‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,10 @@
1919

2020
#include "exec/agg_context.h"
2121
#include "exec/aggregation_sink_operator.h"
22+
#include "exec/aggregation_source_operator.h"
2223
#include "exec/data_sink.h"
2324
#include "exec/exchange_sink_operator.h"
2425
#include "exec/exchange_source_operator.h"
25-
#include "exec/final_aggregation_source_operator.h"
2626
#include "exec/olap_scan_operator.h"
2727
#include "exec/pre_aggregation_source_operator.h"
2828
#include "exec/result_sink_operator.h"
@@ -300,10 +300,8 @@ Status PipelineFragmentContext::_build_pipelines(ExecNode* node, PipelinePtr cur
300300
next_operator_template_id(), "PAggSourceOperator", agg_node, agg_ctx);
301301
RETURN_IF_ERROR(cur_pipe->add_operator(agg_source));
302302
} else {
303-
// TODO: Use can read to replace dependency
304-
cur_pipe->add_dependency(new_pipe);
305-
OperatorTemplatePtr agg_source = std::make_shared<FinalAggSourceOperatorTemplate>(
306-
next_operator_template_id(), "FinalAggSourceOperator", agg_node);
303+
OperatorTemplatePtr agg_source = std::make_shared<AggregationSourceOperatorTemplate>(
304+
next_operator_template_id(), "AggregationSourceOperator", agg_node);
307305
RETURN_IF_ERROR(cur_pipe->add_operator(agg_source));
308306
}
309307
break;

‎be/src/vec/exec/vaggregation_node.cpp‎

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -486,10 +486,7 @@ Status AggregationNode::open(RuntimeState* state) {
486486
release_block_memory(block);
487487
RETURN_IF_ERROR_AND_CHECK_SPAN(_children[0]->get_next_after_projects(state, &block, &eos),
488488
_children[0]->get_next_span(), eos);
489-
if (block.rows() == 0) {
490-
continue;
491-
}
492-
RETURN_IF_ERROR(_executor.execute(&block));
489+
RETURN_IF_ERROR(sink(state, &block, eos));
493490
_executor.update_memusage();
494491
}
495492
_children[0]->close(state);
@@ -527,17 +524,30 @@ Status AggregationNode::get_next(RuntimeState* state, Block* block, bool* eos) {
527524
_make_nullable_output_key(block);
528525
COUNTER_SET(_rows_returned_counter, _num_rows_returned);
529526
} else {
530-
RETURN_IF_ERROR(_executor.get_result(state, block, eos));
531-
_make_nullable_output_key(block);
532-
// dispose the having clause, should not be execute in prestreaming agg
533-
RETURN_IF_ERROR(VExprContext::filter_block(_vconjunct_ctx_ptr, block, block->columns()));
534-
reached_limit(block, eos);
527+
RETURN_IF_ERROR(pull(state, block, eos));
535528
}
536-
537529
_executor.update_memusage();
538530
return Status::OK();
539531
}
540532

533+
Status AggregationNode::pull(doris::RuntimeState* state, vectorized::Block* block, bool* eos) {
534+
RETURN_IF_ERROR(_executor.get_result(state, block, eos));
535+
_make_nullable_output_key(block);
536+
// dispose the having clause, should not be execute in prestreaming agg
537+
RETURN_IF_ERROR(VExprContext::filter_block(_vconjunct_ctx_ptr, block, block->columns()));
538+
reached_limit(block, eos);
539+
540+
return Status::OK();
541+
}
542+
543+
Status AggregationNode::sink(doris::RuntimeState* state, vectorized::Block* in_block, bool eos) {
544+
if (in_block->rows() > 0) {
545+
RETURN_IF_ERROR(_executor.execute(in_block));
546+
}
547+
if (eos) _can_read = true;
548+
return Status::OK();
549+
}
550+
541551
void AggregationNode::release_resource(RuntimeState* state) {
542552
for (auto* aggregate_evaluator : _aggregate_evaluators) aggregate_evaluator->close(state);
543553
VExpr::close(_probe_expr_ctxs, state);

‎be/src/vec/exec/vaggregation_node.h‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ class MemPool;
3636

3737
namespace pipeline {
3838
class AggSinkOperator;
39-
class FinalAggSourceOperator;
39+
class AggregationSourceOperator;
4040
class PreAggSourceOperator;
4141
} // namespace pipeline
4242

@@ -764,11 +764,13 @@ class AggregationNode final : public ::doris::ExecNode {
764764
virtual Status get_next(RuntimeState* state, Block* block, bool* eos) override;
765765
virtual Status close(RuntimeState* state) override;
766766
virtual void release_resource(RuntimeState* state) override;
767+
Status pull(doris::RuntimeState* state, vectorized::Block* output_block, bool* eos) override;
768+
Status sink(doris::RuntimeState* state, vectorized::Block* input_block, bool eos) override;
767769
bool is_streaming_preagg() { return _is_streaming_preagg; }
768770

769771
private:
770772
friend class pipeline::AggSinkOperator;
771-
friend class pipeline::FinalAggSourceOperator;
773+
friend class pipeline::AggregationSourceOperator;
772774
friend class pipeline::PreAggSourceOperator;
773775
// group by k1,k2
774776
std::vector<VExprContext*> _probe_expr_ctxs;

0 commit comments

Comments
 (0)