Skip to content

Commit e2d3d01

Browse files
authored
dd a method to get doris current memory usage (#6979)
Add all memory usage check when TryConsume memory
1 parent ad0d2b8 commit e2d3d01

18 files changed

Lines changed: 147 additions & 85 deletions

‎be/src/exec/hash_table.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,9 @@ void HashTable::resize_buckets(int64_t num_buckets) {
181181

182182
int64_t old_num_buckets = _num_buckets;
183183
int64_t delta_bytes = (num_buckets - old_num_buckets) * sizeof(Bucket);
184-
if (!_mem_tracker->TryConsume(delta_bytes)) {
184+
Status st = _mem_tracker->TryConsume(delta_bytes);
185+
WARN_IF_ERROR(st, "resize bucket failed");
186+
if (!st) {
185187
mem_limit_exceeded(delta_bytes);
186188
return;
187189
}

‎be/src/exec/partitioned_hash_table.cc‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -310,7 +310,9 @@ Status PartitionedHashTableCtx::ExprValuesCache::Init(RuntimeState* state,
310310
MAX_EXPR_VALUES_ARRAY_SIZE / expr_values_bytes_per_row_));
311311

312312
int mem_usage = MemUsage(capacity_, expr_values_bytes_per_row_, num_exprs_);
313-
if (UNLIKELY(!tracker->TryConsume(mem_usage))) {
313+
Status st = tracker->TryConsume(mem_usage);
314+
WARN_IF_ERROR(st, "PartitionedHashTableCtx::ExprValuesCache failed");
315+
if (UNLIKELY(!st)) {
314316
capacity_ = 0;
315317
string details = Substitute(
316318
"PartitionedHashTableCtx::ExprValuesCache failed to allocate $0 bytes.", mem_usage);

‎be/src/runtime/buffered_block_mgr2.cc‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -324,8 +324,9 @@ bool BufferedBlockMgr2::consume_memory(Client* client, int64_t size) {
324324
}
325325
int buffers_needed = BitUtil::ceil(size, max_block_size());
326326
unique_lock<mutex> lock(_lock);
327-
328-
if (size < max_block_size() && _mem_tracker->TryConsume(size)) {
327+
Status st = _mem_tracker->TryConsume(size);
328+
WARN_IF_ERROR(st, "consume failed");
329+
if (size < max_block_size() && st) {
329330
// For small allocations (less than a block size), just let the allocation through.
330331
client->_tracker->ConsumeLocal(size, client->_query_tracker.get());
331332
// client->_tracker->Consume(size);
@@ -335,8 +336,9 @@ bool BufferedBlockMgr2::consume_memory(Client* client, int64_t size) {
335336
if (available_buffers(client) + client->_num_tmp_reserved_buffers < buffers_needed) {
336337
return false;
337338
}
338-
339-
if (_mem_tracker->TryConsume(size)) {
339+
st = _mem_tracker->TryConsume(size);
340+
WARN_IF_ERROR(st, "consume failed");
341+
if (st) {
340342
// There was still unallocated memory, don't need to recycle allocated blocks.
341343
client->_tracker->ConsumeLocal(size, client->_query_tracker.get());
342344
// client->_tracker->Consume(size);
@@ -393,7 +395,9 @@ bool BufferedBlockMgr2::consume_memory(Client* client, int64_t size) {
393395

394396
DCHECK_GE(buffers_acquired * max_block_size(), size);
395397
_mem_tracker->Release(buffers_acquired * max_block_size());
396-
if (!_mem_tracker->TryConsume(size)) {
398+
st = _mem_tracker->TryConsume(size);
399+
WARN_IF_ERROR(st, "consume failed");
400+
if (!st) {
397401
return false;
398402
}
399403
client->_tracker->ConsumeLocal(size, client->_query_tracker.get());
@@ -465,7 +469,9 @@ Status BufferedBlockMgr2::get_new_block(Client* client, Block* unpin_block, Bloc
465469

466470
if (len > 0 && len < _max_block_size) {
467471
DCHECK(unpin_block == nullptr);
468-
if (client->_tracker->TryConsume(len)) {
472+
Status st = client->_tracker->TryConsume(len);
473+
WARN_IF_ERROR(st, "get_new_block failed");
474+
if (st) {
469475
// TODO: Have a cache of unused blocks of size 'len' (0, _max_block_size)
470476
uint8_t* buffer = new uint8_t[len];
471477
// Descriptors for non-I/O sized buffers are deleted when the block is deleted.
@@ -1088,9 +1094,10 @@ Status BufferedBlockMgr2::find_buffer_for_block(Block* block, bool* in_mem) {
10881094
Status BufferedBlockMgr2::find_buffer(unique_lock<mutex>& lock, BufferDescriptor** buffer_desc) {
10891095
*buffer_desc = nullptr;
10901096

1097+
Status st = _mem_tracker->TryConsume(_max_block_size);
1098+
WARN_IF_ERROR(st, "try to allocate a new buffer failed");
10911099
// First, try to allocate a new buffer.
1092-
if (_free_io_buffers.size() < _block_write_threshold &&
1093-
_mem_tracker->TryConsume(_max_block_size)) {
1100+
if (_free_io_buffers.size() < _block_write_threshold && st) {
10941101
uint8_t* new_buffer = new uint8_t[_max_block_size];
10951102
*buffer_desc = _obj_pool.add(new BufferDescriptor(new_buffer, _max_block_size));
10961103
(*buffer_desc)->all_buffers_it =

‎be/src/runtime/bufferpool/reservation_tracker.cc‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,9 @@ bool ReservationTracker::TryConsumeFromMemTracker(int64_t reservation_increase)
187187
if (GetParentMemTracker() == nullptr) {
188188
// At the topmost link, which may be a MemTracker with a limit, we need to use
189189
// TryConsume() to check the limit.
190-
return mem_tracker_->TryConsume(reservation_increase);
190+
Status st = mem_tracker_->TryConsume(reservation_increase);
191+
WARN_IF_ERROR(st, "TryConsumeFromMemTracker failed");
192+
return st.ok();
191193
} else {
192194
// For lower links, there shouldn't be a limit to enforce, so we just need to
193195
// update the consumption of the linked MemTracker since the reservation is

‎be/src/runtime/exec_env_init.cpp‎

Lines changed: 40 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -70,8 +70,8 @@ DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(scanner_thread_pool_queue_size, MetricUnit::N
7070
DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(etl_thread_pool_queue_size, MetricUnit::NOUNIT);
7171
DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(send_batch_thread_pool_thread_num, MetricUnit::NOUNIT);
7272
DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(send_batch_thread_pool_queue_size, MetricUnit::NOUNIT);
73-
DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG(query_mem_consumption, MetricUnit::BYTES, "",
74-
mem_consumption, Labels({{"type", "query"}}));
73+
DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG(query_mem_consumption, MetricUnit::BYTES, "", mem_consumption,
74+
Labels({{"type", "query"}}));
7575

7676
Status ExecEnv::init(ExecEnv* env, const std::vector<StorePath>& store_paths) {
7777
return env->_init(store_paths);
@@ -95,20 +95,20 @@ Status ExecEnv::_init(const std::vector<StorePath>& store_paths) {
9595
_pool_mem_trackers = new PoolMemTrackerRegistry();
9696
_thread_mgr = new ThreadResourceMgr();
9797
_scan_thread_pool = new PriorityThreadPool(config::doris_scanner_thread_pool_thread_num,
98-
config::doris_scanner_thread_pool_queue_size);
98+
config::doris_scanner_thread_pool_queue_size);
9999

100100
ThreadPoolBuilder("LimitedScanThreadPool")
101-
.set_min_threads(1)
101+
.set_min_threads(1)
102102
.set_max_threads(config::doris_scanner_thread_pool_thread_num)
103103
.set_max_queue_size(config::doris_scanner_thread_pool_queue_size)
104-
.build(&_limited_scan_thread_pool);
104+
.build(&_limited_scan_thread_pool);
105105

106106
ThreadPoolBuilder("SendBatchThreadPool")
107107
.set_min_threads(1)
108108
.set_max_threads(config::send_batch_thread_pool_thread_num)
109109
.set_max_queue_size(config::send_batch_thread_pool_queue_size)
110110
.build(&_send_batch_thread_pool);
111-
111+
112112
_etl_thread_pool = new PriorityThreadPool(config::etl_thread_pool_size,
113113
config::etl_thread_pool_queue_size);
114114
_cgroups_mgr = new CgroupsMgr(this, config::doris_cgroups);
@@ -158,34 +158,38 @@ Status ExecEnv::_init_mem_tracker() {
158158
int64_t global_memory_limit_bytes = 0;
159159
bool is_percent = false;
160160
std::stringstream ss;
161-
global_memory_limit_bytes = ParseUtil::parse_mem_spec(config::mem_limit, -1, &is_percent);
161+
global_memory_limit_bytes =
162+
ParseUtil::parse_mem_spec(config::mem_limit, -1, MemInfo::physical_mem(), &is_percent);
162163
if (global_memory_limit_bytes <= 0) {
163164
ss << "Failed to parse mem limit from '" + config::mem_limit + "'.";
164165
return Status::InternalError(ss.str());
165166
}
166167

167168
if (global_memory_limit_bytes > MemInfo::physical_mem()) {
168-
LOG(WARNING) << "Memory limit " << PrettyPrinter::print(global_memory_limit_bytes, TUnit::BYTES)
169+
LOG(WARNING) << "Memory limit "
170+
<< PrettyPrinter::print(global_memory_limit_bytes, TUnit::BYTES)
169171
<< " exceeds physical memory of "
170172
<< PrettyPrinter::print(MemInfo::physical_mem(), TUnit::BYTES)
171173
<< ". Using physical memory instead";
172174
global_memory_limit_bytes = MemInfo::physical_mem();
173175
}
174-
_mem_tracker = MemTracker::CreateTracker(global_memory_limit_bytes, "Process", MemTracker::GetRootTracker(),
175-
false, false, MemTrackerLevel::OVERVIEW);
176-
REGISTER_HOOK_METRIC(query_mem_consumption, [this]() {
177-
return _mem_tracker->consumption();
178-
});
179-
LOG(INFO) << "Using global memory limit: " << PrettyPrinter::print(global_memory_limit_bytes, TUnit::BYTES)
180-
<< ", origin config value: " << config::mem_limit;
176+
_mem_tracker = MemTracker::CreateTracker(global_memory_limit_bytes, "Process",
177+
MemTracker::GetRootTracker(), false, false,
178+
MemTrackerLevel::OVERVIEW);
179+
REGISTER_HOOK_METRIC(query_mem_consumption, [this]() { return _mem_tracker->consumption(); });
180+
LOG(INFO) << "Using global memory limit: "
181+
<< PrettyPrinter::print(global_memory_limit_bytes, TUnit::BYTES)
182+
<< ", origin config value: " << config::mem_limit;
181183

182184
// 2. init buffer pool
183185
if (!BitUtil::IsPowerOf2(config::min_buffer_size)) {
184186
ss << "Config min_buffer_size must be a power-of-two: " << config::min_buffer_size;
185187
return Status::InternalError(ss.str());
186188
}
187189

188-
int64_t buffer_pool_limit = ParseUtil::parse_mem_spec(config::buffer_pool_limit, global_memory_limit_bytes, &is_percent);
190+
int64_t buffer_pool_limit =
191+
ParseUtil::parse_mem_spec(config::buffer_pool_limit, global_memory_limit_bytes,
192+
MemInfo::physical_mem(), &is_percent);
189193
if (buffer_pool_limit <= 0) {
190194
ss << "Invalid config buffer_pool_limit value, must be a percentage or "
191195
"positive bytes value or percentage: "
@@ -201,7 +205,8 @@ Status ExecEnv::_init_mem_tracker() {
201205
}
202206

203207
int64_t clean_pages_limit =
204-
ParseUtil::parse_mem_spec(config::buffer_pool_clean_pages_limit, buffer_pool_limit, &is_percent);
208+
ParseUtil::parse_mem_spec(config::buffer_pool_clean_pages_limit, buffer_pool_limit,
209+
MemInfo::physical_mem(), &is_percent);
205210
if (clean_pages_limit <= 0) {
206211
ss << "Invalid buffer_pool_clean_pages_limit value, must be a percentage or "
207212
"positive bytes value or percentage: "
@@ -213,22 +218,25 @@ Status ExecEnv::_init_mem_tracker() {
213218
clean_pages_limit = clean_pages_limit / 2;
214219
}
215220
_init_buffer_pool(config::min_buffer_size, buffer_pool_limit, clean_pages_limit);
216-
LOG(INFO) << "Buffer pool memory limit: " << PrettyPrinter::print(buffer_pool_limit, TUnit::BYTES)
217-
<< ", origin config value: " << config::buffer_pool_limit
218-
<< ". clean pages limit: " << PrettyPrinter::print(clean_pages_limit, TUnit::BYTES)
219-
<< ", origin config value: " << config::buffer_pool_clean_pages_limit;
221+
LOG(INFO) << "Buffer pool memory limit: "
222+
<< PrettyPrinter::print(buffer_pool_limit, TUnit::BYTES)
223+
<< ", origin config value: " << config::buffer_pool_limit
224+
<< ". clean pages limit: " << PrettyPrinter::print(clean_pages_limit, TUnit::BYTES)
225+
<< ", origin config value: " << config::buffer_pool_clean_pages_limit;
220226

221227
// 3. init storage page cache
222228
int64_t storage_cache_limit =
223-
ParseUtil::parse_mem_spec(config::storage_page_cache_limit, global_memory_limit_bytes, &is_percent);
229+
ParseUtil::parse_mem_spec(config::storage_page_cache_limit, global_memory_limit_bytes,
230+
MemInfo::physical_mem(), &is_percent);
224231
while (!is_percent && storage_cache_limit > global_memory_limit_bytes / 2) {
225232
// Reason same as buffer_pool_limit
226233
storage_cache_limit = storage_cache_limit / 2;
227234
}
228235
int32_t index_page_cache_percentage = config::index_page_cache_percentage;
229236
StoragePageCache::create_global_cache(storage_cache_limit, index_page_cache_percentage);
230-
LOG(INFO) << "Storage page cache memory limit: " << PrettyPrinter::print(storage_cache_limit, TUnit::BYTES)
231-
<< ", origin config value: " << config::storage_page_cache_limit;
237+
LOG(INFO) << "Storage page cache memory limit: "
238+
<< PrettyPrinter::print(storage_cache_limit, TUnit::BYTES)
239+
<< ", origin config value: " << config::storage_page_cache_limit;
232240

233241
SegmentLoader::create_global_instance(config::segment_cache_capacity);
234242

@@ -250,21 +258,17 @@ void ExecEnv::_init_buffer_pool(int64_t min_page_size, int64_t capacity,
250258
}
251259

252260
void ExecEnv::_register_metrics() {
253-
REGISTER_HOOK_METRIC(scanner_thread_pool_queue_size, [this]() {
254-
return _scan_thread_pool->get_queue_size();
255-
});
261+
REGISTER_HOOK_METRIC(scanner_thread_pool_queue_size,
262+
[this]() { return _scan_thread_pool->get_queue_size(); });
256263

257-
REGISTER_HOOK_METRIC(etl_thread_pool_queue_size, [this]() {
258-
return _etl_thread_pool->get_queue_size();
259-
});
264+
REGISTER_HOOK_METRIC(etl_thread_pool_queue_size,
265+
[this]() { return _etl_thread_pool->get_queue_size(); });
260266

261-
REGISTER_HOOK_METRIC(send_batch_thread_pool_thread_num, [this]() {
262-
return _send_batch_thread_pool->num_threads();
263-
});
267+
REGISTER_HOOK_METRIC(send_batch_thread_pool_thread_num,
268+
[this]() { return _send_batch_thread_pool->num_threads(); });
264269

265-
REGISTER_HOOK_METRIC(send_batch_thread_pool_queue_size, [this]() {
266-
return _send_batch_thread_pool->get_queue_size();
267-
});
270+
REGISTER_HOOK_METRIC(send_batch_thread_pool_queue_size,
271+
[this]() { return _send_batch_thread_pool->get_queue_size(); });
268272
}
269273

270274
void ExecEnv::_deregister_metrics() {

‎be/src/runtime/mem_pool.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,9 @@ bool MemPool::find_chunk(size_t min_size, bool check_limits) {
116116

117117
chunk_size = BitUtil::RoundUpToPowerOfTwo(chunk_size);
118118
if (check_limits) {
119-
if (!mem_tracker_->TryConsume(chunk_size)) return false;
119+
Status st = mem_tracker_->TryConsume(chunk_size);
120+
WARN_IF_ERROR(st, "try to allocate a new buffer failed");
121+
if (!st) return false;
120122
} else {
121123
mem_tracker_->Consume(chunk_size);
122124
}

‎be/src/runtime/mem_tracker.h‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030

3131
#include "common/status.h"
3232
#include "gen_cpp/Types_types.h" // for TUniqueId
33+
#include "util/mem_info.h"
3334
#include "util/metrics.h"
3435
#include "util/runtime_profile.h"
3536
#include "util/spinlock.h"
@@ -166,11 +167,16 @@ class MemTracker : public std::enable_shared_from_this<MemTracker> {
166167
/// other callers that may not tolerate allocation failures have a better chance
167168
/// of success. Returns true if the consumption was successfully updated.
168169
WARN_UNUSED_RESULT
169-
bool TryConsume(int64_t bytes, MemLimit mode = MemLimit::HARD) {
170+
Status TryConsume(int64_t bytes, MemLimit mode = MemLimit::HARD) {
170171
// DCHECK_GE(bytes, 0);
171172
if (bytes <= 0) {
172173
Release(-bytes);
173-
return true;
174+
return Status::OK();
175+
}
176+
if (MemInfo::current_mem() + bytes >= MemInfo::mem_limit()) {
177+
return Status::MemoryLimitExceeded(fmt::format(
178+
"{}: TryConsume failed, bytes={} process whole consumption={} mem limit={}",
179+
label_, bytes, MemInfo::current_mem(), MemInfo::mem_limit()));
174180
}
175181
// if (UNLIKELY(bytes == 0)) return true;
176182
// if (UNLIKELY(bytes < 0)) return false; // needed in RELEASE, hits DCHECK in DEBUG
@@ -189,26 +195,27 @@ class MemTracker : public std::enable_shared_from_this<MemTracker> {
189195
while (true) {
190196
if (LIKELY(tracker->consumption_->try_add(bytes, limit))) break;
191197

192-
VLOG_RPC << "TryConsume failed, bytes=" << bytes
193-
<< " consumption=" << tracker->consumption_->current_value()
194-
<< " limit=" << limit << " attempting to GC";
195198
if (UNLIKELY(tracker->GcMemory(limit - bytes))) {
196199
DCHECK_GE(i, 0);
197200
// Failed for this mem tracker. Roll back the ones that succeeded.
198201
for (int j = all_trackers_.size() - 1; j > i; --j) {
199202
all_trackers_[j]->consumption_->add(-bytes);
200203
}
201-
return false;
204+
return Status::MemoryLimitExceeded(fmt::format(
205+
"{}: TryConsume failed, bytes={} consumption={} imit={} "
206+
"attempting to GC",
207+
tracker->label(), bytes, tracker->consumption_->current_value(),
208+
limit));
202209
}
203-
VLOG_RPC << "GC succeeded, TryConsume bytes=" << bytes
204-
<< " consumption=" << tracker->consumption_->current_value()
205-
<< " limit=" << limit;
210+
VLOG_NOTICE << "GC succeeded, TryConsume bytes=" << bytes
211+
<< " consumption=" << tracker->consumption_->current_value()
212+
<< " limit=" << limit;
206213
}
207214
}
208215
}
209216
// Everyone succeeded, return.
210217
DCHECK_EQ(i, -1);
211-
return true;
218+
return Status::OK();
212219
}
213220

214221
/// Decreases consumption of this tracker and its ancestors by 'bytes'.

‎be/src/service/doris_main.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,10 @@ int main(int argc, char** argv) {
275275
#if defined(LEAK_SANITIZER)
276276
__lsan_do_leak_check();
277277
#endif
278+
279+
#if !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && !defined(THREAD_SANITIZER)
280+
doris::MemInfo::refresh_current_mem();
281+
#endif
278282
sleep(10);
279283
}
280284

‎be/src/util/mem_info.cpp‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,15 +25,19 @@
2525
#include <iostream>
2626
#include <sstream>
2727

28+
#include "common/config.h"
2829
#include "gutil/strings/split.h"
2930
#include "util/cgroup_util.h"
31+
#include "util/parse_util.h"
3032
#include "util/pretty_printer.h"
3133
#include "util/string_parser.hpp"
3234

3335
namespace doris {
3436

3537
bool MemInfo::_s_initialized = false;
3638
int64_t MemInfo::_s_physical_mem = -1;
39+
int64_t MemInfo::_s_mem_limit = -1;
40+
size_t MemInfo::_s_current_mem = 0;
3741

3842
void MemInfo::init() {
3943
// Read from /proc/meminfo
@@ -79,16 +83,21 @@ void MemInfo::init() {
7983
LOG(WARNING) << "Could not determine amount of physical memory on this machine.";
8084
}
8185

82-
LOG(INFO) << "Physical Memory: " << PrettyPrinter::print(_s_physical_mem, TUnit::BYTES);
86+
bool is_percent = true;
87+
_s_mem_limit = ParseUtil::parse_mem_spec(config::mem_limit, -1, _s_physical_mem, &is_percent);
8388

89+
LOG(INFO) << "Physical Memory: " << PrettyPrinter::print(_s_physical_mem, TUnit::BYTES);
8490
_s_initialized = true;
8591
}
8692

8793
std::string MemInfo::debug_string() {
8894
DCHECK(_s_initialized);
8995
CGroupUtil util;
9096
std::stringstream stream;
91-
stream << "Mem Info: " << PrettyPrinter::print(_s_physical_mem, TUnit::BYTES) << std::endl;
97+
stream << "Physical Memory: " << PrettyPrinter::print(_s_physical_mem, TUnit::BYTES)
98+
<< std::endl;
99+
stream << "Memory Limt: " << PrettyPrinter::print(_s_mem_limit, TUnit::BYTES) << std::endl;
100+
stream << "Current Usage: " << PrettyPrinter::print(_s_current_mem, TUnit::BYTES) << std::endl;
92101
stream << "CGroup Info: " << util.debug_string() << std::endl;
93102
return stream.str();
94103
}

0 commit comments

Comments
 (0)