@@ -70,8 +70,8 @@ DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(scanner_thread_pool_queue_size, MetricUnit::N
7070DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG (etl_thread_pool_queue_size, MetricUnit::NOUNIT );
7171DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG (send_batch_thread_pool_thread_num, MetricUnit::NOUNIT );
7272DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG (send_batch_thread_pool_queue_size, MetricUnit::NOUNIT );
73- DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG (query_mem_consumption, MetricUnit::BYTES , " " ,
74- mem_consumption, Labels({{" type" , " query" }}));
73+ DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG (query_mem_consumption, MetricUnit::BYTES , " " , mem_consumption,
74+ Labels ({{" type" , " query" }}));
7575
7676Status ExecEnv::init (ExecEnv* env, const std::vector<StorePath>& store_paths) {
7777 return env->_init (store_paths);
@@ -95,20 +95,20 @@ Status ExecEnv::_init(const std::vector<StorePath>& store_paths) {
9595 _pool_mem_trackers = new PoolMemTrackerRegistry ();
9696 _thread_mgr = new ThreadResourceMgr ();
9797 _scan_thread_pool = new PriorityThreadPool (config::doris_scanner_thread_pool_thread_num,
98- config::doris_scanner_thread_pool_queue_size);
98+ config::doris_scanner_thread_pool_queue_size);
9999
100100 ThreadPoolBuilder (" LimitedScanThreadPool" )
101- .set_min_threads (1 )
101+ .set_min_threads (1 )
102102 .set_max_threads (config::doris_scanner_thread_pool_thread_num)
103103 .set_max_queue_size (config::doris_scanner_thread_pool_queue_size)
104- .build (&_limited_scan_thread_pool);
104+ .build (&_limited_scan_thread_pool);
105105
106106 ThreadPoolBuilder (" SendBatchThreadPool" )
107107 .set_min_threads (1 )
108108 .set_max_threads (config::send_batch_thread_pool_thread_num)
109109 .set_max_queue_size (config::send_batch_thread_pool_queue_size)
110110 .build (&_send_batch_thread_pool);
111-
111+
112112 _etl_thread_pool = new PriorityThreadPool (config::etl_thread_pool_size,
113113 config::etl_thread_pool_queue_size);
114114 _cgroups_mgr = new CgroupsMgr (this , config::doris_cgroups);
@@ -158,34 +158,38 @@ Status ExecEnv::_init_mem_tracker() {
158158 int64_t global_memory_limit_bytes = 0 ;
159159 bool is_percent = false ;
160160 std::stringstream ss;
161- global_memory_limit_bytes = ParseUtil::parse_mem_spec (config::mem_limit, -1 , &is_percent);
161+ global_memory_limit_bytes =
162+ ParseUtil::parse_mem_spec (config::mem_limit, -1 , MemInfo::physical_mem (), &is_percent);
162163 if (global_memory_limit_bytes <= 0 ) {
163164 ss << " Failed to parse mem limit from '" + config::mem_limit + " '." ;
164165 return Status::InternalError (ss.str ());
165166 }
166167
167168 if (global_memory_limit_bytes > MemInfo::physical_mem ()) {
168- LOG (WARNING ) << " Memory limit " << PrettyPrinter::print (global_memory_limit_bytes, TUnit::BYTES )
169+ LOG (WARNING ) << " Memory limit "
170+ << PrettyPrinter::print (global_memory_limit_bytes, TUnit::BYTES )
169171 << " exceeds physical memory of "
170172 << PrettyPrinter::print (MemInfo::physical_mem (), TUnit::BYTES )
171173 << " . Using physical memory instead" ;
172174 global_memory_limit_bytes = MemInfo::physical_mem ();
173175 }
174- _mem_tracker = MemTracker::CreateTracker (global_memory_limit_bytes, " Process" , MemTracker::GetRootTracker (),
175- false , false , MemTrackerLevel:: OVERVIEW );
176- REGISTER_HOOK_METRIC (query_mem_consumption, [ this ]() {
177- return _mem_tracker->consumption ();
178- });
179- LOG ( INFO ) << " Using global memory limit: " << PrettyPrinter::print (global_memory_limit_bytes, TUnit::BYTES )
180- << " , origin config value: " << config::mem_limit;
176+ _mem_tracker = MemTracker::CreateTracker (global_memory_limit_bytes, " Process" ,
177+ MemTracker::GetRootTracker (), false , false ,
178+ MemTrackerLevel:: OVERVIEW );
179+ REGISTER_HOOK_METRIC (query_mem_consumption, [ this ]() { return _mem_tracker->consumption (); } );
180+ LOG ( INFO ) << " Using global memory limit: "
181+ << PrettyPrinter::print (global_memory_limit_bytes, TUnit::BYTES )
182+ << " , origin config value: " << config::mem_limit;
181183
182184 // 2. init buffer pool
183185 if (!BitUtil::IsPowerOf2 (config::min_buffer_size)) {
184186 ss << " Config min_buffer_size must be a power-of-two: " << config::min_buffer_size;
185187 return Status::InternalError (ss.str ());
186188 }
187189
188- int64_t buffer_pool_limit = ParseUtil::parse_mem_spec (config::buffer_pool_limit, global_memory_limit_bytes, &is_percent);
190+ int64_t buffer_pool_limit =
191+ ParseUtil::parse_mem_spec (config::buffer_pool_limit, global_memory_limit_bytes,
192+ MemInfo::physical_mem (), &is_percent);
189193 if (buffer_pool_limit <= 0 ) {
190194 ss << " Invalid config buffer_pool_limit value, must be a percentage or "
191195 " positive bytes value or percentage: "
@@ -201,7 +205,8 @@ Status ExecEnv::_init_mem_tracker() {
201205 }
202206
203207 int64_t clean_pages_limit =
204- ParseUtil::parse_mem_spec (config::buffer_pool_clean_pages_limit, buffer_pool_limit, &is_percent);
208+ ParseUtil::parse_mem_spec (config::buffer_pool_clean_pages_limit, buffer_pool_limit,
209+ MemInfo::physical_mem (), &is_percent);
205210 if (clean_pages_limit <= 0 ) {
206211 ss << " Invalid buffer_pool_clean_pages_limit value, must be a percentage or "
207212 " positive bytes value or percentage: "
@@ -213,22 +218,25 @@ Status ExecEnv::_init_mem_tracker() {
213218 clean_pages_limit = clean_pages_limit / 2 ;
214219 }
215220 _init_buffer_pool (config::min_buffer_size, buffer_pool_limit, clean_pages_limit);
216- LOG (INFO ) << " Buffer pool memory limit: " << PrettyPrinter::print (buffer_pool_limit, TUnit::BYTES )
217- << " , origin config value: " << config::buffer_pool_limit
218- << " . clean pages limit: " << PrettyPrinter::print (clean_pages_limit, TUnit::BYTES )
219- << " , origin config value: " << config::buffer_pool_clean_pages_limit;
221+ LOG (INFO ) << " Buffer pool memory limit: "
222+ << PrettyPrinter::print (buffer_pool_limit, TUnit::BYTES )
223+ << " , origin config value: " << config::buffer_pool_limit
224+ << " . clean pages limit: " << PrettyPrinter::print (clean_pages_limit, TUnit::BYTES )
225+ << " , origin config value: " << config::buffer_pool_clean_pages_limit;
220226
221227 // 3. init storage page cache
222228 int64_t storage_cache_limit =
223- ParseUtil::parse_mem_spec (config::storage_page_cache_limit, global_memory_limit_bytes, &is_percent);
229+ ParseUtil::parse_mem_spec (config::storage_page_cache_limit, global_memory_limit_bytes,
230+ MemInfo::physical_mem (), &is_percent);
224231 while (!is_percent && storage_cache_limit > global_memory_limit_bytes / 2 ) {
225232 // Reason same as buffer_pool_limit
226233 storage_cache_limit = storage_cache_limit / 2 ;
227234 }
228235 int32_t index_page_cache_percentage = config::index_page_cache_percentage;
229236 StoragePageCache::create_global_cache (storage_cache_limit, index_page_cache_percentage);
230- LOG (INFO ) << " Storage page cache memory limit: " << PrettyPrinter::print (storage_cache_limit, TUnit::BYTES )
231- << " , origin config value: " << config::storage_page_cache_limit;
237+ LOG (INFO ) << " Storage page cache memory limit: "
238+ << PrettyPrinter::print (storage_cache_limit, TUnit::BYTES )
239+ << " , origin config value: " << config::storage_page_cache_limit;
232240
233241 SegmentLoader::create_global_instance (config::segment_cache_capacity);
234242
@@ -250,21 +258,17 @@ void ExecEnv::_init_buffer_pool(int64_t min_page_size, int64_t capacity,
250258}
251259
252260void ExecEnv::_register_metrics () {
253- REGISTER_HOOK_METRIC (scanner_thread_pool_queue_size, [this ]() {
254- return _scan_thread_pool->get_queue_size ();
255- });
261+ REGISTER_HOOK_METRIC (scanner_thread_pool_queue_size,
262+ [this ]() { return _scan_thread_pool->get_queue_size (); });
256263
257- REGISTER_HOOK_METRIC (etl_thread_pool_queue_size, [this ]() {
258- return _etl_thread_pool->get_queue_size ();
259- });
264+ REGISTER_HOOK_METRIC (etl_thread_pool_queue_size,
265+ [this ]() { return _etl_thread_pool->get_queue_size (); });
260266
261- REGISTER_HOOK_METRIC (send_batch_thread_pool_thread_num, [this ]() {
262- return _send_batch_thread_pool->num_threads ();
263- });
267+ REGISTER_HOOK_METRIC (send_batch_thread_pool_thread_num,
268+ [this ]() { return _send_batch_thread_pool->num_threads (); });
264269
265- REGISTER_HOOK_METRIC (send_batch_thread_pool_queue_size, [this ]() {
266- return _send_batch_thread_pool->get_queue_size ();
267- });
270+ REGISTER_HOOK_METRIC (send_batch_thread_pool_queue_size,
271+ [this ]() { return _send_batch_thread_pool->get_queue_size (); });
268272}
269273
270274void ExecEnv::_deregister_metrics () {
0 commit comments