Skip to content

Commit 934cc05

Browse files
authored
Merge pull request apache#1 from apache/master
Update latest code
2 parents ada39dd + bc923b8 commit 934cc05

437 files changed

Lines changed: 12566 additions & 5809 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitignore‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ thirdparty/installed
2020
core.*
2121
extension/**/.classpath
2222
extension/**/target
23+
nohup.out
2324

2425
fe/log
2526
custom_env.sh
@@ -34,8 +35,14 @@ fe_plugins/**/.classpath
3435
fe_plugins/**/.factorypath
3536
samples/**/.classpath
3637
fe/fe-core/src/main/resources/static/
37-
nohup.out
38-
38+
fs_brokers/apache_hdfs_broker/src/main/resources/
39+
fs_brokers/apache_hdfs_broker/src/main/thrift/
40+
samples/doris-demo/remote-udf-python-demo/*_pb2.py
41+
samples/doris-demo/remote-udf-python-demo/*_pb2_grpc.py
42+
samples/doris-demo/remote-udf-cpp-demo/*.o
43+
samples/doris-demo/remote-udf-cpp-demo/*.pb.h
44+
samples/doris-demo/remote-udf-cpp-demo/*.pb.cc
45+
samples/doris-demo/remote-udf-cpp-demo/function_server_demo
3946
regression-test/framework/target
4047
regression-test/framework/dependency-reduced-pom.xml
4148

‎NOTICE.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
Apache Doris (incubating)
2-
Copyright 2018-2021 The Apache Software Foundation
2+
Copyright 2018-2022 The Apache Software Foundation
33

44
This product includes software developed at
55
The Apache Software Foundation (http://www.apache.org/).

‎be/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ cmake_minimum_required(VERSION 3.19.2)
2020
project(doris CXX C)
2121

2222
# Write compile_commands.json
23-
set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
23+
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
2424
# set platforms
2525

2626
if (CMAKE_SYSTEM_PROCESSOR MATCHES "amd64|x86_64")

‎be/src/agent/cgroups_mgr.cpp‎

Lines changed: 49 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -62,14 +62,14 @@ CgroupsMgr::CgroupsMgr(ExecEnv* exec_env, const string& root_cgroups_path)
6262

6363
CgroupsMgr::~CgroupsMgr() {}
6464

65-
AgentStatus CgroupsMgr::update_local_cgroups(const TFetchResourceResult& new_fetched_resource) {
65+
Status CgroupsMgr::update_local_cgroups(const TFetchResourceResult& new_fetched_resource) {
6666
std::lock_guard<std::mutex> lck(_update_cgroups_mtx);
6767
if (!_is_cgroups_init_success) {
68-
return AgentStatus::DORIS_ERROR;
68+
return Status::InternalError("Cgroup not inited");
6969
}
7070

7171
if (_cur_version >= new_fetched_resource.resourceVersion) {
72-
return AgentStatus::DORIS_SUCCESS;
72+
return Status::OK();
7373
}
7474

7575
const std::map<std::string, TUserResource>& new_user_resource =
@@ -110,7 +110,7 @@ AgentStatus CgroupsMgr::update_local_cgroups(const TFetchResourceResult& new_fet
110110

111111
// Using resource version, not subscribe version
112112
_cur_version = new_fetched_resource.resourceVersion;
113-
return AgentStatus::DORIS_SUCCESS;
113+
return Status::OK();
114114
}
115115

116116
void CgroupsMgr::_config_user_disk_throttle(
@@ -157,11 +157,11 @@ int64_t CgroupsMgr::_get_resource_value(
157157
return resource_value;
158158
}
159159

160-
AgentStatus CgroupsMgr::_config_disk_throttle(std::string user_name, std::string level,
161-
int64_t hdd_read_iops, int64_t hdd_write_iops,
162-
int64_t hdd_read_mbps, int64_t hdd_write_mbps,
163-
int64_t ssd_read_iops, int64_t ssd_write_iops,
164-
int64_t ssd_read_mbps, int64_t ssd_write_mbps) {
160+
Status CgroupsMgr::_config_disk_throttle(std::string user_name, std::string level,
161+
int64_t hdd_read_iops, int64_t hdd_write_iops,
162+
int64_t hdd_read_mbps, int64_t hdd_write_mbps,
163+
int64_t ssd_read_iops, int64_t ssd_write_iops,
164+
int64_t ssd_read_mbps, int64_t ssd_write_mbps) {
165165
string cgroups_path = this->_root_cgroups_path + "/" + user_name + "/" + level;
166166
string read_bps_path = cgroups_path + "/blkio.throttle.read_bps_device";
167167
string write_bps_path = cgroups_path + "/blkio.throttle.write_bps_device";
@@ -171,7 +171,7 @@ AgentStatus CgroupsMgr::_config_disk_throttle(std::string user_name, std::string
171171
if (!is_file_exist(cgroups_path.c_str())) {
172172
if (!std::filesystem::create_directory(cgroups_path)) {
173173
LOG(ERROR) << "Create cgroups: " << cgroups_path << " failed";
174-
return AgentStatus::DORIS_ERROR;
174+
return Status::InternalError("Create cgroups " + cgroups_path + " failed");
175175
}
176176
}
177177

@@ -226,18 +226,18 @@ AgentStatus CgroupsMgr::_config_disk_throttle(std::string user_name, std::string
226226
ctrl_cmd.str(std::string());
227227
}
228228
}
229-
return AgentStatus::DORIS_SUCCESS;
229+
return Status::OK();
230230
}
231231

232-
AgentStatus CgroupsMgr::modify_user_cgroups(const string& user_name,
233-
const map<string, int32_t>& user_share,
234-
const map<string, int32_t>& level_share) {
232+
Status CgroupsMgr::modify_user_cgroups(const string& user_name,
233+
const map<string, int32_t>& user_share,
234+
const map<string, int32_t>& level_share) {
235235
// Check if the user's cgroups exists, if not create it
236236
string user_cgroups_path = this->_root_cgroups_path + "/" + user_name;
237237
if (!is_file_exist(user_cgroups_path.c_str())) {
238238
if (!std::filesystem::create_directory(user_cgroups_path)) {
239239
LOG(ERROR) << "Create cgroups for user " << user_name << " failed";
240-
return AgentStatus::DORIS_ERROR;
240+
return Status::InternalError("Create cgroups for user " + user_name + " failed");
241241
}
242242
}
243243

@@ -250,7 +250,7 @@ AgentStatus CgroupsMgr::modify_user_cgroups(const string& user_name,
250250
string user_resource_path = user_cgroups_path + "/" + resource_file_name;
251251
std::ofstream user_cgroups(user_resource_path.c_str(), std::ios::out | std::ios::app);
252252
if (!user_cgroups.is_open()) {
253-
return AgentStatus::DORIS_ERROR;
253+
return Status::InternalError("User cgroup is not open");
254254
}
255255
user_cgroups << user_share_weight << std::endl;
256256
user_cgroups.close();
@@ -264,26 +264,26 @@ AgentStatus CgroupsMgr::modify_user_cgroups(const string& user_name,
264264
string level_cgroups_path = user_cgroups_path + "/" + level_name;
265265
if (!is_file_exist(level_cgroups_path.c_str())) {
266266
if (!std::filesystem::create_directory(level_cgroups_path)) {
267-
return AgentStatus::DORIS_ERROR;
267+
return Status::InternalError("User level cgroups not exist");
268268
}
269269
}
270270

271271
// Append the share_weight value to the file
272272
string level_resource_path = level_cgroups_path + "/" + resource_file_name;
273273
std::ofstream level_cgroups(level_resource_path.c_str(), std::ios::out | std::ios::app);
274274
if (!level_cgroups.is_open()) {
275-
return AgentStatus::DORIS_ERROR;
275+
return Status::InternalError("User level cgroup is not open");
276276
}
277277
level_cgroups << level_share_weight << std::endl;
278278
level_cgroups.close();
279279

280280
LOG(INFO) << "Append " << level_share_weight << " to " << level_resource_path;
281281
}
282282
}
283-
return AgentStatus::DORIS_SUCCESS;
283+
return Status::OK();
284284
}
285285

286-
AgentStatus CgroupsMgr::init_cgroups() {
286+
Status CgroupsMgr::init_cgroups() {
287287
std::string root_cgroups_tasks_path = this->_root_cgroups_path + "/tasks";
288288
// Check if the root cgroups exists
289289
if (is_directory(this->_root_cgroups_path.c_str()) &&
@@ -295,37 +295,40 @@ AgentStatus CgroupsMgr::init_cgroups() {
295295
if (fs_type.f_type != CGROUP_SUPER_MAGIC) {
296296
LOG(ERROR) << _root_cgroups_path << " is not a cgroups file system.";
297297
_is_cgroups_init_success = false;
298-
return AgentStatus::DORIS_ERROR;
298+
return Status::InternalError(_root_cgroups_path + " is not a cgroups file system");
299+
;
299300
}
300301
#endif
301302
// Check if current user have write permission to cgroup folder
302303
if (access(_root_cgroups_path.c_str(), W_OK) != 0) {
303304
LOG(ERROR) << "Doris does not have write permission to " << _root_cgroups_path;
304305
_is_cgroups_init_success = false;
305-
return AgentStatus::DORIS_ERROR;
306+
return Status::InternalError("Doris does not have write permission to " +
307+
_root_cgroups_path);
306308
}
307309
// If root folder exists, then delete all subfolders under it
308310
std::filesystem::directory_iterator item_begin(this->_root_cgroups_path);
309311
std::filesystem::directory_iterator item_end;
310312
for (; item_begin != item_end; item_begin++) {
311313
if (is_directory(item_begin->path().string().c_str())) {
312314
// Delete the sub folder
313-
if (delete_user_cgroups(item_begin->path().filename().string()) !=
314-
AgentStatus::DORIS_SUCCESS) {
315+
if (!delete_user_cgroups(item_begin->path().filename().string()).ok()) {
315316
LOG(ERROR) << "Could not clean subfolder " << item_begin->path().string();
316317
_is_cgroups_init_success = false;
317-
return AgentStatus::DORIS_ERROR;
318+
return Status::InternalError("Could not clean subfolder " +
319+
item_begin->path().string());
318320
}
319321
}
320322
}
321323
LOG(INFO) << "Initialize doris cgroups successfully under folder " << _root_cgroups_path;
322324
_is_cgroups_init_success = true;
323-
return AgentStatus::DORIS_SUCCESS;
325+
return Status::OK();
324326
} else {
325327
VLOG_NOTICE << "Could not find a valid cgroups path for resource isolation,"
326328
<< "current value is " << _root_cgroups_path << ". ignore it.";
327329
_is_cgroups_init_success = false;
328-
return AgentStatus::DORIS_ERROR;
330+
return Status::InternalError("Could not find a valid cgroups path for resource isolation");
331+
;
329332
}
330333
}
331334

@@ -337,18 +340,18 @@ void CgroupsMgr::apply_cgroup(const string& user_name, const string& level) {
337340
s_global_cg_mgr->assign_to_cgroups(user_name, level);
338341
}
339342

340-
AgentStatus CgroupsMgr::assign_to_cgroups(const string& user_name, const string& level) {
343+
Status CgroupsMgr::assign_to_cgroups(const string& user_name, const string& level) {
341344
if (!_is_cgroups_init_success) {
342-
return AgentStatus::DORIS_ERROR;
345+
return Status::InternalError("Cgroups not inited");
343346
}
344347
int64_t tid = gettid();
345348
return assign_thread_to_cgroups(tid, user_name, level);
346349
}
347350

348-
AgentStatus CgroupsMgr::assign_thread_to_cgroups(int64_t thread_id, const string& user_name,
349-
const string& level) {
351+
Status CgroupsMgr::assign_thread_to_cgroups(int64_t thread_id, const string& user_name,
352+
const string& level) {
350353
if (!_is_cgroups_init_success) {
351-
return AgentStatus::DORIS_ERROR;
354+
return Status::InternalError("Cgroups not inited");
352355
}
353356
string tasks_path = _root_cgroups_path + "/" + user_name + "/" + level + "/tasks";
354357
if (!is_file_exist(_root_cgroups_path + "/" + user_name)) {
@@ -359,22 +362,22 @@ AgentStatus CgroupsMgr::assign_thread_to_cgroups(int64_t thread_id, const string
359362
}
360363
if (!is_file_exist(tasks_path.c_str())) {
361364
LOG(ERROR) << "Cgroups path " << tasks_path << " not exist!";
362-
return AgentStatus::DORIS_ERROR;
365+
return Status::InternalError("Cgroups path not exist");
363366
}
364367
std::ofstream tasks(tasks_path.c_str(), std::ios::out | std::ios::app);
365368
if (!tasks.is_open()) {
366369
// This means doris could not open this file. May be it does not have access to it
367370
LOG(ERROR) << "Echo thread: " << thread_id << " to " << tasks_path << " failed!";
368-
return AgentStatus::DORIS_ERROR;
371+
return Status::InternalError("Echo thread to cgroup failed");
369372
}
370373
// Append thread id to the tasks file directly
371374
tasks << thread_id << std::endl;
372375
tasks.close();
373376

374-
return AgentStatus::DORIS_SUCCESS;
377+
return Status::OK();
375378
}
376379

377-
AgentStatus CgroupsMgr::delete_user_cgroups(const string& user_name) {
380+
Status CgroupsMgr::delete_user_cgroups(const string& user_name) {
378381
string user_cgroups_path = this->_root_cgroups_path + "/" + user_name;
379382
if (is_file_exist(user_cgroups_path.c_str())) {
380383
// Delete sub cgroups --> level cgroups
@@ -383,20 +386,16 @@ AgentStatus CgroupsMgr::delete_user_cgroups(const string& user_name) {
383386
for (; item_begin != item_end; item_begin++) {
384387
if (is_directory(item_begin->path().string().c_str())) {
385388
string cur_cgroups_path = item_begin->path().string();
386-
if (this->drop_cgroups(cur_cgroups_path) < 0) {
387-
return AgentStatus::DORIS_ERROR;
388-
}
389+
RETURN_IF_ERROR(drop_cgroups(cur_cgroups_path));
389390
}
390391
}
391392
// Delete user cgroups
392-
if (this->drop_cgroups(user_cgroups_path) < 0) {
393-
return AgentStatus::DORIS_ERROR;
394-
}
393+
RETURN_IF_ERROR(drop_cgroups(user_cgroups_path));
395394
}
396-
return AgentStatus::DORIS_SUCCESS;
395+
return Status::OK();
397396
}
398397

399-
AgentStatus CgroupsMgr::drop_cgroups(const string& deleted_cgroups_path) {
398+
Status CgroupsMgr::drop_cgroups(const string& deleted_cgroups_path) {
400399
// Try to delete the cgroups folder
401400
// If failed then there maybe exist active tasks under it and try to relocate them
402401
// Currently, try 10 times to relocate and delete the cgroups.
@@ -410,22 +409,22 @@ AgentStatus CgroupsMgr::drop_cgroups(const string& deleted_cgroups_path) {
410409
#endif
411410
if (i == this->_drop_retry_times) {
412411
LOG(ERROR) << "drop cgroups under path: " << deleted_cgroups_path << " failed.";
413-
return AgentStatus::DORIS_ERROR;
412+
return Status::InternalError("Drop cgroup failed");
414413
}
415414
}
416-
return AgentStatus::DORIS_SUCCESS;
415+
return Status::OK();
417416
}
418417

419-
AgentStatus CgroupsMgr::relocate_tasks(const string& src_cgroups, const string& dest_cgroups) {
418+
Status CgroupsMgr::relocate_tasks(const string& src_cgroups, const string& dest_cgroups) {
420419
string src_tasks_path = src_cgroups + "/tasks";
421420
string dest_tasks_path = dest_cgroups + "/tasks";
422421
std::ifstream src_tasks(src_tasks_path.c_str());
423422
if (!src_tasks) {
424-
return AgentStatus::DORIS_ERROR;
423+
return Status::InternalError("Src tasks is null");
425424
}
426425
std::ofstream dest_tasks(dest_tasks_path.c_str(), std::ios::out | std::ios::app);
427426
if (!dest_tasks) {
428-
return AgentStatus::DORIS_ERROR;
427+
return Status::InternalError("Desk task is null");
429428
}
430429
int64_t taskid;
431430
while (src_tasks >> taskid) {
@@ -436,7 +435,7 @@ AgentStatus CgroupsMgr::relocate_tasks(const string& src_cgroups, const string&
436435
}
437436
src_tasks.close();
438437
dest_tasks.close();
439-
return AgentStatus::DORIS_SUCCESS;
438+
return Status::OK();
440439
}
441440

442441
void CgroupsMgr::_echo_cmd_to_cgroup(stringstream& ctrl_cmd, string& cgroups_path) {

‎be/src/agent/cgroups_mgr.h‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
#include <mutex>
2424
#include <string>
2525

26-
#include "agent/status.h"
26+
#include "common/status.h"
2727
#include "gen_cpp/MasterService_types.h"
2828

2929
namespace doris {
@@ -40,10 +40,10 @@ class CgroupsMgr {
4040

4141
// Compare the old user resource and new user resource to find deleted user
4242
// then delete nonexisting cgroups, create new user cgroups, update all user cgroups
43-
AgentStatus update_local_cgroups(const TFetchResourceResult& new_fetched_resource);
43+
Status update_local_cgroups(const TFetchResourceResult& new_fetched_resource);
4444

4545
// Delete all existing cgroups under root path
46-
AgentStatus init_cgroups();
46+
Status init_cgroups();
4747

4848
// Modify cgroup resource shares under cgroups_root_path.
4949
// Create related cgroups if it not exist.
@@ -56,7 +56,7 @@ class CgroupsMgr {
5656
//
5757
// level_share: a mapping for shares for different levels under the user.
5858
// mapping key is level name; value is level's share. Currently, different resource using the same share.
59-
AgentStatus modify_user_cgroups(const std::string& user_name,
59+
Status modify_user_cgroups(const std::string& user_name,
6060
const std::map<std::string, int32_t>& user_share,
6161
const std::map<std::string, int32_t>& level_share);
6262

@@ -68,34 +68,34 @@ class CgroupsMgr {
6868
//
6969
// Input parameters:
7070
// user_name&level: the user name and level used to find the cgroup
71-
AgentStatus assign_to_cgroups(const std::string& user_name, const std::string& level);
71+
Status assign_to_cgroups(const std::string& user_name, const std::string& level);
7272

7373
// Assign the thread identified by thread id to the cgroup identified by user name and level
7474
//
7575
// Input parameters:
7676
// thread_id: the unique id for the thread
7777
// user_name&level: the user name and level used to find the cgroup
78-
AgentStatus assign_thread_to_cgroups(int64_t thread_id, const std::string& user_name,
78+
Status assign_thread_to_cgroups(int64_t thread_id, const std::string& user_name,
7979
const std::string& level);
8080

8181
// Delete the user's cgroups and its sub level cgroups using DropCgroups
8282
// Input parameters:
8383
// user name: user name to be deleted
84-
AgentStatus delete_user_cgroups(const std::string& user_name);
84+
Status delete_user_cgroups(const std::string& user_name);
8585
// Delete a cgroup
8686
// If there are active tasks in this cgroups, they will be relocated
8787
// to root cgroups.
8888
// If there are sub cgroups, it will return error.
8989
// Input parameters:
9090
// deleted_cgroups_path: the absolute cgroups path to be deleted
91-
AgentStatus drop_cgroups(const std::string& deleted_cgroups_path);
91+
Status drop_cgroups(const std::string& deleted_cgroups_path);
9292

9393
// Relocate all threads or processes in src cgroups to dest cgroups
9494
// Ignore errors when echo to dest cgroups
9595
// Input parameters:
9696
// src_cgroups: absolute path for src cgroups folder
9797
// dest_cgroups: absolute path for dest cgroups folder
98-
AgentStatus relocate_tasks(const std::string& src_cgroups, const std::string& dest_cgroups);
98+
Status relocate_tasks(const std::string& src_cgroups, const std::string& dest_cgroups);
9999

100100
int64_t get_cgroups_version() { return _cur_version; }
101101

@@ -123,7 +123,7 @@ class CgroupsMgr {
123123
// ssd_write_iops: write iops number for ssd disk.
124124
// ssd_read_mbps: read bps number for ssd disk, using mb not byte or kb.
125125
// ssd_write_mbps: write bps number for ssd disk, using mb not byte or kb.
126-
AgentStatus _config_disk_throttle(std::string user_name, std::string level,
126+
Status _config_disk_throttle(std::string user_name, std::string level,
127127
int64_t hdd_read_iops, int64_t hdd_write_iops,
128128
int64_t hdd_read_mbps, int64_t hdd_write_mbps,
129129
int64_t ssd_read_iops, int64_t ssd_write_iops,

0 commit comments

Comments
 (0)