1717
1818#pragma once
1919
20+ #include " concurrentqueue.h"
2021#include " runtime/descriptors.h"
2122#include " scanner_context.h"
2223
@@ -32,9 +33,11 @@ class PipScannerContext : public vectorized::ScannerContext {
3233 const TupleDescriptor* input_tuple_desc,
3334 const TupleDescriptor* output_tuple_desc,
3435 const std::list<vectorized::VScannerSPtr>& scanners, int64_t limit,
35- int64_t max_bytes_in_blocks_queue, const std::vector<int >& col_distribute_ids)
36+ int64_t max_bytes_in_blocks_queue, const std::vector<int >& col_distribute_ids,
37+ const int num_parallel_instances)
3638 : vectorized::ScannerContext(state, parent, input_tuple_desc, output_tuple_desc,
37- scanners, limit, max_bytes_in_blocks_queue),
39+ scanners, limit, max_bytes_in_blocks_queue,
40+ num_parallel_instances),
3841 _col_distribute_ids (col_distribute_ids),
3942 _need_colocate_distribute(!_col_distribute_ids.empty()) {}
4043
@@ -52,11 +55,7 @@ class PipScannerContext : public vectorized::ScannerContext {
5255 }
5356
5457 {
55- std::unique_lock<std::mutex> l (*_queue_mutexs[id]);
56- if (!_blocks_queues[id].empty ()) {
57- *block = std::move (_blocks_queues[id].front ());
58- _blocks_queues[id].pop_front ();
59- } else {
58+ if (!_blocks_queues[id].try_dequeue (*block)) {
6059 *eos = _is_finished || _should_stop;
6160 return Status::OK ();
6261 }
@@ -69,7 +68,7 @@ class PipScannerContext : public vectorized::ScannerContext {
6968 bool done () override { return _is_finished || _should_stop || _status_error; }
7069
7170 void append_blocks_to_queue (std::vector<vectorized::BlockUPtr>& blocks) override {
72- const int queue_size = _queue_mutexs .size ();
71+ const int queue_size = _blocks_queues .size ();
7372 const int block_size = blocks.size ();
7473 int64_t local_bytes = 0 ;
7574
@@ -78,7 +77,7 @@ class PipScannerContext : public vectorized::ScannerContext {
7877 for (const auto & block : blocks) {
7978 // vectorized calculate hash
8079 int rows = block->rows ();
81- const auto element_size = _max_queue_size ;
80+ const auto element_size = _num_parallel_instances ;
8281 hash_vals.resize (rows);
8382 std::fill (hash_vals.begin (), hash_vals.end (), 0 );
8483 auto * __restrict hashes = hash_vals.data ();
@@ -113,9 +112,8 @@ class PipScannerContext : public vectorized::ScannerContext {
113112 for (int i = 0 ; i < queue_size && i < block_size; ++i) {
114113 int queue = _next_queue_to_feed;
115114 {
116- std::lock_guard<std::mutex> l (*_queue_mutexs[queue]);
117115 for (int j = i; j < block_size; j += queue_size) {
118- _blocks_queues[queue].emplace_back (std::move (blocks[j]));
116+ _blocks_queues[queue].enqueue (std::move (blocks[j]));
119117 }
120118 }
121119 _next_queue_to_feed = queue + 1 < queue_size ? queue + 1 : 0 ;
@@ -124,65 +122,117 @@ class PipScannerContext : public vectorized::ScannerContext {
124122 _current_used_bytes += local_bytes;
125123 }
126124
127- bool empty_in_queue (int id) override {
128- std::unique_lock<std::mutex> l (*_queue_mutexs[id]);
129- return _blocks_queues[id].empty ();
130- }
125+ bool empty_in_queue (int id) override { return _blocks_queues[id].size_approx () == 0 ; }
131126
132- void set_max_queue_size (const int max_queue_size) override {
133- _max_queue_size = max_queue_size;
134- for (int i = 0 ; i < max_queue_size; ++i) {
135- _queue_mutexs.emplace_back (new std::mutex);
136- _blocks_queues.emplace_back (std::list<vectorized::BlockUPtr>());
127+ Status init () override {
128+ for (int i = 0 ; i < _num_parallel_instances; ++i) {
129+ _blocks_queues.emplace_back (moodycamel::ConcurrentQueue<vectorized::BlockUPtr>());
137130 }
131+ RETURN_IF_ERROR (ScannerContext::init ());
138132 if (_need_colocate_distribute) {
139- int real_block_size =
140- limit == -1 ? _batch_size : std::min (static_cast <int64_t >(_batch_size), limit);
141- int64_t free_blocks_memory_usage = 0 ;
142- for (int i = 0 ; i < _max_queue_size; ++i) {
143- auto block = vectorized::Block::create_unique (_output_tuple_desc->slots (),
144- real_block_size,
145- true /* ignore invalid slots*/ );
146- free_blocks_memory_usage += block->allocated_bytes ();
147- _colocate_mutable_blocks.emplace_back (
148- vectorized::MutableBlock::create_unique (block.get ()));
149- _colocate_blocks.emplace_back (std::move (block));
150- _colocate_block_mutexs.emplace_back (new std::mutex);
151- }
152- _free_blocks_memory_usage->add (free_blocks_memory_usage);
133+ _init_colocate_block ();
134+ }
135+ return Status::OK ();
136+ }
137+
138+ void _init_colocate_block () {
139+ int real_block_size =
140+ limit == -1 ? _batch_size : std::min (static_cast <int64_t >(_batch_size), limit);
141+ int64_t free_blocks_memory_usage = 0 ;
142+ for (int i = 0 ; i < _num_parallel_instances; ++i) {
143+ auto block = vectorized::Block::create_unique (
144+ _output_tuple_desc->slots (), real_block_size, true /* ignore invalid slots*/ );
145+ free_blocks_memory_usage += block->allocated_bytes ();
146+ _colocate_mutable_blocks.emplace_back (
147+ vectorized::MutableBlock::create_unique (block.get ()));
148+ _colocate_blocks.emplace_back (std::move (block));
149+ _colocate_block_mutexs.emplace_back (new std::mutex);
153150 }
151+ _free_blocks_memory_usage->add (free_blocks_memory_usage);
154152 }
155153
156154 bool has_enough_space_in_blocks_queue () const override {
157- return _current_used_bytes < _max_bytes_in_queue / 2 * _max_queue_size ;
155+ return _current_used_bytes < _max_bytes_in_queue / 2 * _num_parallel_instances ;
158156 }
159157
160158 void _dispose_coloate_blocks_not_in_queue () override {
161159 if (_need_colocate_distribute) {
162- for (int i = 0 ; i < _max_queue_size; ++i) {
163- std::scoped_lock s (*_colocate_block_mutexs[i], *_queue_mutexs[i]);
160+ for (int i = 0 ; i < _num_parallel_instances; ++i) {
164161 if (_colocate_blocks[i] && !_colocate_blocks[i]->empty ()) {
165162 _current_used_bytes += _colocate_blocks[i]->allocated_bytes ();
166- _blocks_queues[i].emplace_back (std::move (_colocate_blocks[i]));
163+ _blocks_queues[i].enqueue (std::move (_colocate_blocks[i]));
167164 _colocate_mutable_blocks[i]->clear ();
168165 }
169166 }
170167 }
171168 }
172169
170+ vectorized::BlockUPtr get_free_block (bool * has_free_block,
171+ bool get_not_empty_block = false ) override {
172+ {
173+ vectorized::BlockUPtr block;
174+ if (_free_blocks_queues.try_dequeue (block)) {
175+ if (!get_not_empty_block || block->mem_reuse ()) {
176+ _total_free_block_num--;
177+ _free_blocks_memory_usage->add (-block->allocated_bytes ());
178+ return block;
179+ }
180+ }
181+ }
182+ *has_free_block = false ;
183+
184+ COUNTER_UPDATE (_newly_create_free_blocks_num, 1 );
185+ return vectorized::Block::create_unique (_real_tuple_desc->slots (), _batch_size,
186+ true /* ignore invalid slots*/ );
187+ }
188+
189+ void return_free_block (std::unique_ptr<vectorized::Block> block) override {
190+ block->clear_column_data ();
191+ _free_blocks_memory_usage->add (block->allocated_bytes ());
192+ _free_blocks_queues.enqueue (std::move (block));
193+ _total_free_block_num++;
194+ }
195+
196+ void _init_free_block (int pre_alloc_block_count, int real_block_size) override {
197+ // The free blocks is used for final output block of scanners.
198+ // So use _output_tuple_desc;
199+ int64_t free_blocks_memory_usage = 0 ;
200+ auto block = vectorized::Block::create_unique (_output_tuple_desc->slots (), real_block_size,
201+ true /* ignore invalid slots*/ );
202+ free_blocks_memory_usage += block->allocated_bytes ();
203+ _free_blocks_queues.enqueue (std::move (block));
204+ _total_free_block_num = pre_alloc_block_count;
205+ _free_blocks_memory_usage->add (free_blocks_memory_usage);
206+ }
207+
208+ int cal_thread_slot_num_by_free_block_num () override {
209+ // For pipeline engine, we don't promise `thread_slot_num` is exact (e.g. scanners to
210+ // schedule may need more free blocks than available free blocks).
211+ // This is because we don't want a heavy lock for free block queues.
212+ int local_val = _total_free_block_num;
213+ int thread_slot_num = local_val / _block_per_scanner;
214+ thread_slot_num += (local_val % _block_per_scanner != 0 );
215+ thread_slot_num = std::min (thread_slot_num, _max_thread_num - _num_running_scanners);
216+ if (thread_slot_num <= 0 ) {
217+ thread_slot_num = 1 ;
218+ }
219+ return thread_slot_num;
220+ }
221+
173222private:
174- int _max_queue_size = 1 ;
175223 int _next_queue_to_feed = 0 ;
176- std::vector<std::unique_ptr<std::mutex>> _queue_mutexs;
177- std::vector<std::list<vectorized::BlockUPtr>> _blocks_queues;
224+ std::vector<moodycamel::ConcurrentQueue<vectorized::BlockUPtr>> _blocks_queues;
178225 std::atomic_int64_t _current_used_bytes = 0 ;
226+ std::atomic_int32_t _total_free_block_num = 0 ;
179227
180228 const std::vector<int >& _col_distribute_ids;
181229 const bool _need_colocate_distribute;
182230 std::vector<vectorized::BlockUPtr> _colocate_blocks;
183231 std::vector<std::unique_ptr<vectorized::MutableBlock>> _colocate_mutable_blocks;
184232 std::vector<std::unique_ptr<std::mutex>> _colocate_block_mutexs;
185233
234+ moodycamel::ConcurrentQueue<vectorized::BlockUPtr> _free_blocks_queues;
235+
186236 void _add_rows_colocate_blocks (vectorized::Block* block, int loc,
187237 const std::vector<int >& rows) {
188238 int row_wait_add = rows.size ();
@@ -205,10 +255,7 @@ class PipScannerContext : public vectorized::ScannerContext {
205255
206256 if (row_add == max_add) {
207257 _current_used_bytes += _colocate_blocks[loc]->allocated_bytes ();
208- {
209- std::lock_guard<std::mutex> queue_l (*_queue_mutexs[loc]);
210- _blocks_queues[loc].emplace_back (std::move (_colocate_blocks[loc]));
211- }
258+ { _blocks_queues[loc].enqueue (std::move (_colocate_blocks[loc])); }
212259 bool get_block_not_empty = true ;
213260 _colocate_blocks[loc] = get_free_block (&get_block_not_empty, get_block_not_empty);
214261 _colocate_mutable_blocks[loc]->set_muatable_columns (
0 commit comments