Skip to content

Commit 6deeab8

Browse files
authored
[fix](core)Provide add_blocks interface for VDataStreamRecvr (#57888)
We previously had a crash. The cause is that we should not access the request after calling add_block(...) because add_block may enqueue a closure that runs on another thread and frees the request ``` ==730145==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x7be1efd803a0 at pc 0x556b38d9d625 bp 0x7b16bf0193f0 sp 0x7b16bf0193e8 READ of size 4 at 0x7be1efd803a0 thread T1559 #0 0x556b38d9d624 in google::protobuf::internal::RepeatedPtrFieldBase::size() const /home/zcp/repo_center/doris_master/doris/thirdparty/installed/include/google/protobuf/repeated_ptr_field.h:185:29 #1 0x556b408ab062 in google::protobuf::RepeatedPtrField<doris::PBlock>::size() const /home/zcp/repo_center/doris_master/doris/thirdparty/installed/include/google/protobuf/repeated_ptr_field.h:1248:32 #2 0x556b408aaff4 in doris::PTransmitDataParams::_internal_blocks_size() const /home/zcp/repo_center/doris_master/doris/be/../gensrc/build/gen_cpp/internal_service.pb.h:32149:25 #3 0x556b4089731c in doris::PTransmitDataParams::blocks_size() const /home/zcp/repo_center/doris_master/doris/be/../gensrc/build/gen_cpp/internal_service.pb.h:32152:10 #4 0x556b60a83c17 in doris::vectorized::VDataStreamMgr::transmit_block(doris::PTransmitDataParams const*, google::protobuf::Closure**, long) /home/zcp/repo_center/doris_master/doris/be/src/vec/runtime/vdata_stream_mgr.cpp:150:38 #5 0x556b407f7408 in doris::PInternalService::_transmit_block(google::protobuf::RpcController*, doris::PTransmitDataParams const*, doris::PTransmitDataResult*, google::protobuf::Closure*, doris::Status const&, long) /home/zcp/repo_center/doris_master/doris/be/src/service/internal_service.cpp:1673:40 #6 0x556b407f52bb in doris::PInternalService::transmit_block(google::protobuf::RpcController*, doris::PTransmitDataParams const*, doris::PTransmitDataResult*, google::protobuf::Closure*) /home/zcp/repo_center/doris_master/doris/be/src/service/internal_service.cpp:1610:9 #7 0x556b43fceba2 in doris::PBackendService::CallMethod(google::protobuf::MethodDescriptor const*, google::protobuf::RpcController*, google::protobuf::Message const*, google::protobuf::Message*, google::protobuf::Closure*) /home/zcp/repo_center/doris_master/doris/gensrc/build/gen_cpp/internal_service.pb.cc:49452:7 #8 0x556b6736273e in brpc::policy::ProcessRpcRequest(brpc::InputMessageBase*) (/mnt/hdd01/selectdb-cloud-chaos/cluster0/be/lib/doris_be+0x770bd73e) #9 0x556b67357426 in brpc::ProcessInputMessage(void*) (/mnt/hdd01/selectdb-cloud-chaos/cluster0/be/lib/doris_be+0x770b2426) #10 0x556b67357f20 in brpc::InputMessenger::InputMessageClosure::~InputMessageClosure() (/mnt/hdd01/selectdb-cloud-chaos/cluster0/be/lib/doris_be+0x770b2f20) #11 0x556b673588dd in brpc::InputMessenger::OnNewMessages(brpc::Socket*) (/mnt/hdd01/selectdb-cloud-chaos/cluster0/be/lib/doris_be+0x770b38dd) #12 0x556b674a0adc in brpc::Socket::ProcessEvent(void*) (/mnt/hdd01/selectdb-cloud-chaos/cluster0/be/lib/doris_be+0x771fbadc) #13 0x556b672e0f76 in bthread::TaskGroup::task_runner(long) (/mnt/hdd01/selectdb-cloud-chaos/cluster0/be/lib/doris_be+0x7703bf76) #14 0x556b672cbbe0 in bthread_make_fcontext (/mnt/hdd01/selectdb-cloud-chaos/cluster0/be/lib/doris_be+0x77026be0) ``` ``` Note: The done pointer will be saved in add_block and may be called in another thread via done->Run(). For example, when blocks_size == 1, the process is as follows: transmit_block (i=0) └─> recvr->add_block(..., done, ...) // Pass done └─> SenderQueue::add_block └─> _pending_closures.push(done) // done is saved get_batch() [another thread] └─> closure_pair.first->Run() // ⚠️ done->Run() is called └─> brpc releases request and response transmit_block (i=1) [original thread continues] └─> request->blocks_size() // ⚠️ request has already been released! At this point, a use-after-free issue occurs. TODO: We should consider refactoring this part because add_block may release the request. We should not access the request after calling add_block. ``` #50113
1 parent bcaaf60 commit 6deeab8

4 files changed

Lines changed: 140 additions & 131 deletions

File tree

‎be/src/pipeline/exec/exchange_sink_buffer.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,8 @@ ExchangeSinkBuffer::ExchangeSinkBuffer(PUniqueId query_id, PlanNodeId dest_node_
101101
_state(state),
102102
_context(state->get_query_ctx()),
103103
_exchange_sink_num(sender_ins_ids.size()),
104-
_send_multi_blocks(false) {
104+
_send_multi_blocks(state->query_options().__isset.exchange_multi_blocks_byte_size &&
105+
state->query_options().exchange_multi_blocks_byte_size > 0) {
105106
if (_send_multi_blocks) {
106107
_send_multi_blocks_byte_size = state->query_options().exchange_multi_blocks_byte_size;
107108
}

‎be/src/vec/runtime/vdata_stream_mgr.cpp‎

Lines changed: 13 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -146,50 +146,25 @@ Status VDataStreamMgr::transmit_block(const PTransmitDataParams* request,
146146
}
147147

148148
bool eos = request->eos();
149-
if (!request->blocks().empty()) {
150-
for (int i = 0; i < request->blocks_size(); i++) {
151-
// Previously there was a const_cast here, but in our internal tests this occasionally caused a hard-to-reproduce core dump.
152-
// We suspect it was caused by the const_cast, so we switched to making a copy here.
153-
// In fact, for PBlock, most of the data resides in the PColumnMeta column_metas field, so the copy overhead is small.
154-
// To make the intent explicit, we do not use
155-
// std::unique_ptr<PBlock> pblock_ptr = std::make_unique<PBlock>(request->blocks(i));
156-
std::unique_ptr<PBlock> pblock_ptr = std::make_unique<PBlock>();
157-
pblock_ptr->CopyFrom(request->blocks(i));
158-
auto pass_done = [&]() -> ::google::protobuf::Closure** {
159-
// If it is eos, no callback is needed, done can be nullptr
160-
if (eos) {
161-
return nullptr;
162-
}
163-
// If it is the last block, a callback is needed, pass done
164-
if (i == request->blocks_size() - 1) {
165-
return done;
166-
} else {
167-
// If it is not the last block, the blocks in the request currently belong to the same queue,
168-
// and the callback is handled by the done of the last block
169-
return nullptr;
170-
}
171-
};
172-
RETURN_IF_ERROR(recvr->add_block(
173-
std::move(pblock_ptr), request->sender_id(), request->be_number(),
174-
request->packet_seq() - request->blocks_size() + i, pass_done(),
175-
wait_for_worker, cpu_time_stop_watch.elapsed_time()));
176-
}
177-
}
149+
Status exec_status =
150+
request->has_exec_status() ? Status::create(request->exec_status()) : Status::OK();
178151

179-
// old logic, for compatibility
180-
if (request->has_block()) {
152+
auto sender_id = request->sender_id();
153+
auto be_number = request->be_number();
154+
if (!request->blocks().empty()) {
155+
RETURN_IF_ERROR(recvr->add_blocks(request, done, wait_for_worker,
156+
cpu_time_stop_watch.elapsed_time()));
157+
} else if (request->has_block()) {
158+
// old logic, for compatibility
181159
std::unique_ptr<PBlock> pblock_ptr = std::make_unique<PBlock>();
182160
pblock_ptr->CopyFrom(request->block());
183-
RETURN_IF_ERROR(recvr->add_block(std::move(pblock_ptr), request->sender_id(),
184-
request->be_number(), request->packet_seq(),
185-
eos ? nullptr : done, wait_for_worker,
186-
cpu_time_stop_watch.elapsed_time()));
161+
RETURN_IF_ERROR(recvr->add_block(std::move(pblock_ptr), sender_id, be_number,
162+
request->packet_seq(), eos ? nullptr : done,
163+
wait_for_worker, cpu_time_stop_watch.elapsed_time()));
187164
}
188165

189166
if (eos) {
190-
Status exec_status =
191-
request->has_exec_status() ? Status::create(request->exec_status()) : Status::OK();
192-
recvr->remove_sender(request->sender_id(), request->be_number(), exec_status);
167+
recvr->remove_sender(sender_id, be_number, exec_status);
193168
}
194169
return Status::OK();
195170
}

‎be/src/vec/runtime/vdata_stream_recvr.cpp‎

Lines changed: 96 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -55,27 +55,19 @@ VDataStreamRecvr::SenderQueue::SenderQueue(
5555
}
5656

5757
VDataStreamRecvr::SenderQueue::~SenderQueue() {
58-
// Check pending closures, if it is not empty, should clear it here. but it should not happen.
59-
// closure will delete itself during run method. If it is not called, brpc will memory leak.
60-
DCHECK(_pending_closures.empty());
61-
for (auto closure_pair : _pending_closures) {
62-
closure_pair.first->Run();
63-
int64_t elapse_time = closure_pair.second.elapsed_time();
64-
if (_recvr->_max_wait_to_process_time->value() < elapse_time) {
65-
_recvr->_max_wait_to_process_time->set(elapse_time);
66-
}
58+
for (auto& block_item : _block_queue) {
59+
block_item.call_done(_recvr);
6760
}
68-
_pending_closures.clear();
61+
_block_queue.clear();
6962
}
7063

7164
Status VDataStreamRecvr::SenderQueue::get_batch(Block* block, bool* eos) {
7265
#ifndef NDEBUG
7366
if (!_is_cancelled && _block_queue.empty() && _num_remaining_senders > 0) {
7467
throw doris::Exception(ErrorCode::INTERNAL_ERROR,
7568
"_is_cancelled: {}, _block_queue_empty: {}, "
76-
"_num_remaining_senders: {}, _debug_string_info: {}",
77-
_is_cancelled, _block_queue.empty(), _num_remaining_senders,
78-
_debug_string_info());
69+
"_num_remaining_senders: {}",
70+
_is_cancelled, _block_queue.empty(), _num_remaining_senders);
7971
}
8072
#endif
8173
BlockItem block_item;
@@ -111,25 +103,14 @@ Status VDataStreamRecvr::SenderQueue::get_batch(Block* block, bool* eos) {
111103
_recvr->_memory_used_counter->update(-(int64_t)block_byte_size);
112104
INJECT_MOCK_SLEEP(std::lock_guard<std::mutex> l(_lock));
113105
sub_blocks_memory_usage(block_byte_size);
114-
_record_debug_info();
115106
if (_block_queue.empty() && _source_dependency) {
116107
if (!_is_cancelled && _num_remaining_senders > 0) {
117108
_source_dependency->block();
118109
}
119110
}
120111

121-
if (!_pending_closures.empty()) {
122-
auto closure_pair = _pending_closures.front();
123-
closure_pair.first->Run();
124-
int64_t elapse_time = closure_pair.second.elapsed_time();
125-
if (_recvr->_max_wait_to_process_time->value() < elapse_time) {
126-
_recvr->_max_wait_to_process_time->set(elapse_time);
127-
}
128-
_pending_closures.pop_front();
112+
block_item.call_done(_recvr);
129113

130-
closure_pair.second.stop();
131-
_recvr->_buffer_full_total_timer->update(closure_pair.second.elapsed_time());
132-
}
133114
DCHECK(block->empty());
134115
block->swap(*next_block);
135116
*eos = false;
@@ -187,7 +168,7 @@ Status VDataStreamRecvr::SenderQueue::add_block(std::unique_ptr<PBlock> pblock,
187168

188169
DCHECK(_num_remaining_senders >= 0);
189170
if (_num_remaining_senders == 0) {
190-
DCHECK(_sender_eos_set.end() != _sender_eos_set.find(be_number));
171+
DCHECK(_sender_eos_set.contains(be_number));
191172
return Status::OK();
192173
}
193174
}
@@ -209,22 +190,88 @@ Status VDataStreamRecvr::SenderQueue::add_block(std::unique_ptr<PBlock> pblock,
209190

210191
_block_queue.emplace_back(std::move(pblock), block_byte_size);
211192
COUNTER_UPDATE(_recvr->_remote_bytes_received_counter, block_byte_size);
212-
_record_debug_info();
213193
set_source_ready(l);
214194

215195
// if done is nullptr, this function can't delay this response
216196
if (done != nullptr && _recvr->exceeds_limit(block_byte_size)) {
217-
MonotonicStopWatch monotonicStopWatch;
218-
monotonicStopWatch.start();
219-
DCHECK(*done != nullptr);
220-
_pending_closures.emplace_back(*done, monotonicStopWatch);
197+
_block_queue.back().set_done(*done);
221198
*done = nullptr;
222199
}
223200
_recvr->_memory_used_counter->update(block_byte_size);
224201
add_blocks_memory_usage(block_byte_size);
225202
return Status::OK();
226203
}
227204

205+
Status VDataStreamRecvr::SenderQueue::add_blocks(const PTransmitDataParams* request,
206+
::google::protobuf::Closure** done,
207+
const int64_t wait_for_worker,
208+
const uint64_t time_to_find_recvr) {
209+
{
210+
INJECT_MOCK_SLEEP(std::lock_guard<std::mutex> l(_lock));
211+
if (_is_cancelled) {
212+
return Status::OK();
213+
}
214+
const int be_number = request->be_number();
215+
// In the request, the packet_seq for blocks is [request->packet_seq() - blocks_size(), request->packet_seq())
216+
// Note this is a left-closed, right-open interval; the packet_seq of the last block is request->packet_seq() - 1
217+
// We store the packet_seq of the last block in _packet_seq_map so we can compare it with the packet_seq of the next received packet
218+
const int64_t packet_seq = request->packet_seq() - 1;
219+
auto iter = _packet_seq_map.find(be_number);
220+
if (iter != _packet_seq_map.end()) {
221+
if (iter->second > (packet_seq - request->blocks_size())) {
222+
return Status::InternalError(
223+
"packet already exist [cur_packet_id= {} receive_packet_id={}]",
224+
iter->second, packet_seq);
225+
}
226+
iter->second = packet_seq;
227+
} else {
228+
_packet_seq_map.emplace(be_number, packet_seq);
229+
}
230+
231+
DCHECK(_num_remaining_senders >= 0);
232+
if (_num_remaining_senders == 0) {
233+
DCHECK(_sender_eos_set.end() != _sender_eos_set.find(be_number));
234+
return Status::OK();
235+
}
236+
}
237+
238+
INJECT_MOCK_SLEEP(std::lock_guard<std::mutex> l(_lock));
239+
if (_is_cancelled) {
240+
return Status::OK();
241+
}
242+
243+
int64_t total_block_byte_size = 0;
244+
for (int i = 0; i < request->blocks_size(); i++) {
245+
std::unique_ptr<PBlock> pblock = std::make_unique<PBlock>();
246+
pblock->CopyFrom(request->blocks(i));
247+
248+
const auto block_byte_size = pblock->ByteSizeLong();
249+
COUNTER_UPDATE(_recvr->_blocks_produced_counter, 1);
250+
if (_recvr->_max_wait_worker_time->value() < wait_for_worker) {
251+
_recvr->_max_wait_worker_time->set(wait_for_worker);
252+
}
253+
254+
if (_recvr->_max_find_recvr_time->value() < time_to_find_recvr) {
255+
_recvr->_max_find_recvr_time->set((int64_t)time_to_find_recvr);
256+
}
257+
258+
_block_queue.emplace_back(std::move(pblock), block_byte_size);
259+
COUNTER_UPDATE(_recvr->_remote_bytes_received_counter, block_byte_size);
260+
total_block_byte_size += block_byte_size;
261+
}
262+
263+
set_source_ready(l);
264+
265+
// if done is nullptr, this function can't delay this response
266+
if (done != nullptr && _recvr->exceeds_limit(total_block_byte_size)) {
267+
_block_queue.back().set_done(*done);
268+
*done = nullptr;
269+
}
270+
_recvr->_memory_used_counter->update(total_block_byte_size);
271+
add_blocks_memory_usage(total_block_byte_size);
272+
return Status::OK();
273+
}
274+
228275
void VDataStreamRecvr::SenderQueue::add_block(Block* block, bool use_move) {
229276
if (block->rows() == 0) {
230277
return;
@@ -260,7 +307,6 @@ void VDataStreamRecvr::SenderQueue::add_block(Block* block, bool use_move) {
260307
return;
261308
}
262309
_block_queue.emplace_back(std::move(nblock), block_mem_size);
263-
_record_debug_info();
264310
set_source_ready(l);
265311
COUNTER_UPDATE(_recvr->_local_bytes_received_counter, block_mem_size);
266312
_recvr->_memory_used_counter->update(block_mem_size);
@@ -276,7 +322,6 @@ void VDataStreamRecvr::SenderQueue::decrement_senders(int be_number) {
276322
_sender_eos_set.insert(be_number);
277323
DCHECK_GT(_num_remaining_senders, 0);
278324
_num_remaining_senders--;
279-
_record_debug_info();
280325
VLOG_FILE << "decremented senders: fragment_instance_id="
281326
<< print_id(_recvr->fragment_instance_id()) << " node_id=" << _recvr->dest_node_id()
282327
<< " #senders=" << _num_remaining_senders;
@@ -300,14 +345,10 @@ void VDataStreamRecvr::SenderQueue::cancel(Status cancel_status) {
300345
}
301346
{
302347
INJECT_MOCK_SLEEP(std::lock_guard<std::mutex> l(_lock));
303-
for (auto closure_pair : _pending_closures) {
304-
closure_pair.first->Run();
305-
int64_t elapse_time = closure_pair.second.elapsed_time();
306-
if (_recvr->_max_wait_to_process_time->value() < elapse_time) {
307-
_recvr->_max_wait_to_process_time->set(elapse_time);
308-
}
348+
for (auto& block_item : _block_queue) {
349+
block_item.call_done(_recvr);
309350
}
310-
_pending_closures.clear();
351+
_block_queue.clear();
311352
}
312353
}
313354

@@ -319,14 +360,9 @@ void VDataStreamRecvr::SenderQueue::close() {
319360
_is_cancelled = true;
320361
set_source_ready(l);
321362

322-
for (auto closure_pair : _pending_closures) {
323-
closure_pair.first->Run();
324-
int64_t elapse_time = closure_pair.second.elapsed_time();
325-
if (_recvr->_max_wait_to_process_time->value() < elapse_time) {
326-
_recvr->_max_wait_to_process_time->set(elapse_time);
327-
}
363+
for (auto& block_item : _block_queue) {
364+
block_item.call_done(_recvr);
328365
}
329-
_pending_closures.clear();
330366
// Delete any batches queued in _block_queue
331367
_block_queue.clear();
332368
}
@@ -423,6 +459,19 @@ Status VDataStreamRecvr::add_block(std::unique_ptr<PBlock> pblock, int sender_id
423459
wait_for_worker, time_to_find_recvr);
424460
}
425461

462+
Status VDataStreamRecvr::add_blocks(const PTransmitDataParams* request,
463+
::google::protobuf::Closure** done,
464+
const int64_t wait_for_worker,
465+
const uint64_t time_to_find_recvr) {
466+
SCOPED_ATTACH_TASK(_resource_ctx);
467+
if (_query_context->low_memory_mode()) {
468+
set_low_memory_mode();
469+
}
470+
int use_sender_id = _is_merging ? request->sender_id() : 0;
471+
return _sender_queues[use_sender_id]->add_blocks(request, done, wait_for_worker,
472+
time_to_find_recvr);
473+
}
474+
426475
void VDataStreamRecvr::add_block(Block* block, int sender_id, bool use_move) {
427476
if (_query_context->low_memory_mode()) {
428477
set_low_memory_mode();

0 commit comments

Comments
 (0)