Repository navigation
Commit 6deeab8
authored
[fix](core)Provide add_blocks interface for VDataStreamRecvr (#57888)
We previously had a crash. The cause is that we should not access the
request after calling add_block(...) because add_block may enqueue a
closure that runs on another thread and frees the request
```
==730145==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x7be1efd803a0 at pc 0x556b38d9d625 bp 0x7b16bf0193f0 sp 0x7b16bf0193e8
READ of size 4 at 0x7be1efd803a0 thread T1559
#0 0x556b38d9d624 in google::protobuf::internal::RepeatedPtrFieldBase::size() const /home/zcp/repo_center/doris_master/doris/thirdparty/installed/include/google/protobuf/repeated_ptr_field.h:185:29
#1 0x556b408ab062 in google::protobuf::RepeatedPtrField<doris::PBlock>::size() const /home/zcp/repo_center/doris_master/doris/thirdparty/installed/include/google/protobuf/repeated_ptr_field.h:1248:32
#2 0x556b408aaff4 in doris::PTransmitDataParams::_internal_blocks_size() const /home/zcp/repo_center/doris_master/doris/be/../gensrc/build/gen_cpp/internal_service.pb.h:32149:25
#3 0x556b4089731c in doris::PTransmitDataParams::blocks_size() const /home/zcp/repo_center/doris_master/doris/be/../gensrc/build/gen_cpp/internal_service.pb.h:32152:10
#4 0x556b60a83c17 in doris::vectorized::VDataStreamMgr::transmit_block(doris::PTransmitDataParams const*, google::protobuf::Closure**, long) /home/zcp/repo_center/doris_master/doris/be/src/vec/runtime/vdata_stream_mgr.cpp:150:38
#5 0x556b407f7408 in doris::PInternalService::_transmit_block(google::protobuf::RpcController*, doris::PTransmitDataParams const*, doris::PTransmitDataResult*, google::protobuf::Closure*, doris::Status const&, long) /home/zcp/repo_center/doris_master/doris/be/src/service/internal_service.cpp:1673:40
#6 0x556b407f52bb in doris::PInternalService::transmit_block(google::protobuf::RpcController*, doris::PTransmitDataParams const*, doris::PTransmitDataResult*, google::protobuf::Closure*) /home/zcp/repo_center/doris_master/doris/be/src/service/internal_service.cpp:1610:9
#7 0x556b43fceba2 in doris::PBackendService::CallMethod(google::protobuf::MethodDescriptor const*, google::protobuf::RpcController*, google::protobuf::Message const*, google::protobuf::Message*, google::protobuf::Closure*) /home/zcp/repo_center/doris_master/doris/gensrc/build/gen_cpp/internal_service.pb.cc:49452:7
#8 0x556b6736273e in brpc::policy::ProcessRpcRequest(brpc::InputMessageBase*) (/mnt/hdd01/selectdb-cloud-chaos/cluster0/be/lib/doris_be+0x770bd73e)
#9 0x556b67357426 in brpc::ProcessInputMessage(void*) (/mnt/hdd01/selectdb-cloud-chaos/cluster0/be/lib/doris_be+0x770b2426)
#10 0x556b67357f20 in brpc::InputMessenger::InputMessageClosure::~InputMessageClosure() (/mnt/hdd01/selectdb-cloud-chaos/cluster0/be/lib/doris_be+0x770b2f20)
#11 0x556b673588dd in brpc::InputMessenger::OnNewMessages(brpc::Socket*) (/mnt/hdd01/selectdb-cloud-chaos/cluster0/be/lib/doris_be+0x770b38dd)
#12 0x556b674a0adc in brpc::Socket::ProcessEvent(void*) (/mnt/hdd01/selectdb-cloud-chaos/cluster0/be/lib/doris_be+0x771fbadc)
#13 0x556b672e0f76 in bthread::TaskGroup::task_runner(long) (/mnt/hdd01/selectdb-cloud-chaos/cluster0/be/lib/doris_be+0x7703bf76)
#14 0x556b672cbbe0 in bthread_make_fcontext (/mnt/hdd01/selectdb-cloud-chaos/cluster0/be/lib/doris_be+0x77026be0)
```
```
Note: The done pointer will be saved in add_block and may be called in another thread via done->Run().
For example, when blocks_size == 1, the process is as follows:
transmit_block (i=0)
└─> recvr->add_block(..., done, ...) // Pass done
└─> SenderQueue::add_block
└─> _pending_closures.push(done) // done is saved
get_batch() [another thread]
└─> closure_pair.first->Run() // 1 parent bcaaf60 commit 6deeab8
4 files changed
Lines changed: 140 additions & 131 deletions
File tree
- be/src
- pipeline/exec
- vec/runtime
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
| |||
101 | 101 | | |
102 | 102 | | |
103 | 103 | | |
104 | | - | |
| 104 | + | |
| 105 | + | |
105 | 106 | | |
106 | 107 | | |
107 | 108 | | |
| |||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
| |||
146 | 146 | | |
147 | 147 | | |
148 | 148 | | |
149 | | - | |
150 | | - | |
151 | | - | |
152 | | - | |
153 | | - | |
154 | | - | |
155 | | - | |
156 | | - | |
157 | | - | |
158 | | - | |
159 | | - | |
160 | | - | |
161 | | - | |
162 | | - | |
163 | | - | |
164 | | - | |
165 | | - | |
166 | | - | |
167 | | - | |
168 | | - | |
169 | | - | |
170 | | - | |
171 | | - | |
172 | | - | |
173 | | - | |
174 | | - | |
175 | | - | |
176 | | - | |
177 | | - | |
| 149 | + | |
| 150 | + | |
178 | 151 | | |
179 | | - | |
180 | | - | |
| 152 | + | |
| 153 | + | |
| 154 | + | |
| 155 | + | |
| 156 | + | |
| 157 | + | |
| 158 | + | |
181 | 159 | | |
182 | 160 | | |
183 | | - | |
184 | | - | |
185 | | - | |
186 | | - | |
| 161 | + | |
| 162 | + | |
| 163 | + | |
187 | 164 | | |
188 | 165 | | |
189 | 166 | | |
190 | | - | |
191 | | - | |
192 | | - | |
| 167 | + | |
193 | 168 | | |
194 | 169 | | |
195 | 170 | | |
| |||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
| |||
55 | 55 | | |
56 | 56 | | |
57 | 57 | | |
58 | | - | |
59 | | - | |
60 | | - | |
61 | | - | |
62 | | - | |
63 | | - | |
64 | | - | |
65 | | - | |
66 | | - | |
| 58 | + | |
| 59 | + | |
67 | 60 | | |
68 | | - | |
| 61 | + | |
69 | 62 | | |
70 | 63 | | |
71 | 64 | | |
72 | 65 | | |
73 | 66 | | |
74 | 67 | | |
75 | 68 | | |
76 | | - | |
77 | | - | |
78 | | - | |
| 69 | + | |
| 70 | + | |
79 | 71 | | |
80 | 72 | | |
81 | 73 | | |
| |||
111 | 103 | | |
112 | 104 | | |
113 | 105 | | |
114 | | - | |
115 | 106 | | |
116 | 107 | | |
117 | 108 | | |
118 | 109 | | |
119 | 110 | | |
120 | 111 | | |
121 | | - | |
122 | | - | |
123 | | - | |
124 | | - | |
125 | | - | |
126 | | - | |
127 | | - | |
128 | | - | |
| 112 | + | |
129 | 113 | | |
130 | | - | |
131 | | - | |
132 | | - | |
133 | 114 | | |
134 | 115 | | |
135 | 116 | | |
| |||
187 | 168 | | |
188 | 169 | | |
189 | 170 | | |
190 | | - | |
| 171 | + | |
191 | 172 | | |
192 | 173 | | |
193 | 174 | | |
| |||
209 | 190 | | |
210 | 191 | | |
211 | 192 | | |
212 | | - | |
213 | 193 | | |
214 | 194 | | |
215 | 195 | | |
216 | 196 | | |
217 | | - | |
218 | | - | |
219 | | - | |
220 | | - | |
| 197 | + | |
221 | 198 | | |
222 | 199 | | |
223 | 200 | | |
224 | 201 | | |
225 | 202 | | |
226 | 203 | | |
227 | 204 | | |
| 205 | + | |
| 206 | + | |
| 207 | + | |
| 208 | + | |
| 209 | + | |
| 210 | + | |
| 211 | + | |
| 212 | + | |
| 213 | + | |
| 214 | + | |
| 215 | + | |
| 216 | + | |
| 217 | + | |
| 218 | + | |
| 219 | + | |
| 220 | + | |
| 221 | + | |
| 222 | + | |
| 223 | + | |
| 224 | + | |
| 225 | + | |
| 226 | + | |
| 227 | + | |
| 228 | + | |
| 229 | + | |
| 230 | + | |
| 231 | + | |
| 232 | + | |
| 233 | + | |
| 234 | + | |
| 235 | + | |
| 236 | + | |
| 237 | + | |
| 238 | + | |
| 239 | + | |
| 240 | + | |
| 241 | + | |
| 242 | + | |
| 243 | + | |
| 244 | + | |
| 245 | + | |
| 246 | + | |
| 247 | + | |
| 248 | + | |
| 249 | + | |
| 250 | + | |
| 251 | + | |
| 252 | + | |
| 253 | + | |
| 254 | + | |
| 255 | + | |
| 256 | + | |
| 257 | + | |
| 258 | + | |
| 259 | + | |
| 260 | + | |
| 261 | + | |
| 262 | + | |
| 263 | + | |
| 264 | + | |
| 265 | + | |
| 266 | + | |
| 267 | + | |
| 268 | + | |
| 269 | + | |
| 270 | + | |
| 271 | + | |
| 272 | + | |
| 273 | + | |
| 274 | + | |
228 | 275 | | |
229 | 276 | | |
230 | 277 | | |
| |||
260 | 307 | | |
261 | 308 | | |
262 | 309 | | |
263 | | - | |
264 | 310 | | |
265 | 311 | | |
266 | 312 | | |
| |||
276 | 322 | | |
277 | 323 | | |
278 | 324 | | |
279 | | - | |
280 | 325 | | |
281 | 326 | | |
282 | 327 | | |
| |||
300 | 345 | | |
301 | 346 | | |
302 | 347 | | |
303 | | - | |
304 | | - | |
305 | | - | |
306 | | - | |
307 | | - | |
308 | | - | |
| 348 | + | |
| 349 | + | |
309 | 350 | | |
310 | | - | |
| 351 | + | |
311 | 352 | | |
312 | 353 | | |
313 | 354 | | |
| |||
319 | 360 | | |
320 | 361 | | |
321 | 362 | | |
322 | | - | |
323 | | - | |
324 | | - | |
325 | | - | |
326 | | - | |
327 | | - | |
| 363 | + | |
| 364 | + | |
328 | 365 | | |
329 | | - | |
330 | 366 | | |
331 | 367 | | |
332 | 368 | | |
| |||
423 | 459 | | |
424 | 460 | | |
425 | 461 | | |
| 462 | + | |
| 463 | + | |
| 464 | + | |
| 465 | + | |
| 466 | + | |
| 467 | + | |
| 468 | + | |
| 469 | + | |
| 470 | + | |
| 471 | + | |
| 472 | + | |
| 473 | + | |
| 474 | + | |
426 | 475 | | |
427 | 476 | | |
428 | 477 | | |
| |||
0 commit comments