Skip to content

Commit 5c28273

Browse files
authored
Merge pull request apache#5 from dataroaring/upstream_master
merge apache master
2 parents 1841b25 + d7af9f7 commit 5c28273

306 files changed

Lines changed: 4476 additions & 3585 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/CMakeLists.txt‎

Lines changed: 43 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -272,9 +272,25 @@ set_target_properties(aws-sdk-core PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR
272272
add_library(aws-sdk-s3 STATIC IMPORTED)
273273
set_target_properties(aws-sdk-s3 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-cpp-sdk-s3.a)
274274

275+
add_library(aws-sdk-transfer STATIC IMPORTED)
276+
set_target_properties(aws-sdk-transfer PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-cpp-sdk-transfer.a)
277+
278+
add_library(aws-sdk-s3-crt STATIC IMPORTED)
279+
set_target_properties(aws-sdk-s3-crt PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-cpp-sdk-s3-crt.a)
280+
281+
282+
add_library(aws-crt-cpp STATIC IMPORTED)
283+
set_target_properties(aws-crt-cpp PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-crt-cpp.a)
284+
275285
add_library(aws-c-cal STATIC IMPORTED)
276286
set_target_properties(aws-c-cal PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-c-cal.a)
277287

288+
add_library(aws-c-auth STATIC IMPORTED)
289+
set_target_properties(aws-c-auth PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-c-auth.a)
290+
291+
add_library(aws-c-compression STATIC IMPORTED)
292+
set_target_properties(aws-c-compression PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-c-compression.a)
293+
278294
add_library(aws-c-common STATIC IMPORTED)
279295
set_target_properties(aws-c-common PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-c-common.a)
280296

@@ -284,9 +300,18 @@ set_target_properties(aws-c-event-stream PROPERTIES IMPORTED_LOCATION ${THIRDPAR
284300
add_library(aws-c-io STATIC IMPORTED)
285301
set_target_properties(aws-c-io PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-c-io.a)
286302

303+
add_library(aws-c-http STATIC IMPORTED)
304+
set_target_properties(aws-c-http PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-c-http.a)
305+
306+
add_library(aws-c-mqtt STATIC IMPORTED)
307+
set_target_properties(aws-c-mqtt PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-c-mqtt.a)
308+
287309
add_library(aws-checksums STATIC IMPORTED)
288310
set_target_properties(aws-checksums PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-checksums.a)
289311

312+
add_library(aws-c-s3 STATIC IMPORTED)
313+
set_target_properties(aws-c-s3 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-c-s3.a)
314+
290315
add_library(aws-s2n STATIC IMPORTED)
291316
set_target_properties(aws-s2n PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libs2n.a)
292317

@@ -352,11 +377,11 @@ if (COMPILER_GCC)
352377
# https://stackoverflow.com/questions/67584073/gcc-11-false-array-subscript-is-partly-outside-array-bounds-warning
353378
# https://stackoverflow.com/questions/69426070/gcc-11-order-of-arguments-triggers-false-positive-wstringop-overflow-is-this-bu
354379
set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-array-bounds -Wno-stringop-overread")
355-
add_compile_options(-Wno-stringop-overflow)
380+
add_compile_options(-Wno-stringop-overflow -fdiagnostics-color=always)
356381
endif ()
357382

358383
if (COMPILER_CLANG)
359-
384+
add_compile_options (-fcolor-diagnostics)
360385
if(MAKE_TEST STREQUAL "OFF")
361386
add_compile_options(-Qunused-arguments)
362387
endif()
@@ -432,12 +457,6 @@ SET(CMAKE_CXX_FLAGS "${CXX_COMMON_FLAGS} ${CMAKE_CXX_FLAGS}")
432457

433458
message(STATUS "Compiler Flags: ${CMAKE_CXX_FLAGS}")
434459

435-
if (CMAKE_GENERATOR STREQUAL "Ninja" AND NOT DISABLE_COLORED_BUILD)
436-
# Turn on colored output. https://github.com/ninja-build/ninja/wiki/FAQ
437-
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=always")
438-
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-color=always")
439-
endif ()
440-
441460
# Thrift requires these two definitions for some types that we use
442461
add_definitions(-DHAVE_INTTYPES_H -DHAVE_NETINET_IN_H)
443462

@@ -452,15 +471,28 @@ include_directories(
452471
${GENSRC_DIR}/
453472
${THIRDPARTY_DIR}/include
454473
${GPERFTOOLS_HOME}/include
455-
${THIRDPARTY_DIR}/include/thrift/
456-
${THIRDPARTY_DIR}/include/event/
457474
${THIRDPARTY_DIR}/include/breakpad/
458475
)
459476

460477
set(WL_START_GROUP "-Wl,--start-group")
461478
set(WL_END_GROUP "-Wl,--end-group")
462479

463-
set(AWS_LIBS aws-sdk-s3 aws-sdk-core aws-checksums aws-c-io aws-c-event-stream aws-c-common aws-c-cal aws-s2n)
480+
set(AWS_LIBS
481+
aws-sdk-s3
482+
aws-sdk-core
483+
aws-checksums
484+
aws-c-io
485+
aws-c-event-stream
486+
aws-c-common
487+
aws-c-cal
488+
aws-s2n
489+
aws-c-s3
490+
aws-c-auth
491+
aws-crt-cpp
492+
aws-c-compression
493+
aws-c-http
494+
aws-c-mqtt
495+
aws-sdk-s3-crt)
464496

465497
# Set Doris libraries
466498
set(DORIS_LINK_LIBS

‎be/src/agent/task_worker_pool.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -517,7 +517,7 @@ void TaskWorkerPool::_alter_tablet(const TAgentTaskRequest& agent_task_req, int6
517517
string process_name;
518518
switch (task_type) {
519519
case TTaskType::ALTER:
520-
process_name = "alter";
520+
process_name = "AlterTablet";
521521
break;
522522
default:
523523
std::string task_name;
@@ -831,7 +831,7 @@ void TaskWorkerPool::_update_tablet_meta_worker_thread_callback() {
831831
<< " schema_hash=" << tablet_meta_info.schema_hash;
832832
continue;
833833
}
834-
WriteLock wrlock(tablet->get_header_lock_ptr());
834+
WriteLock wrlock(tablet->get_header_lock());
835835
// update tablet meta
836836
if (!tablet_meta_info.__isset.meta_type) {
837837
tablet->set_partition_id(tablet_meta_info.partition_id);

‎be/src/agent/topic_subscriber.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@
1616
// under the License.
1717

1818
#include "agent/topic_subscriber.h"
19-
2019
#include "common/logging.h"
20+
#include "util/mutex.h"
2121

2222
namespace doris {
2323

@@ -38,13 +38,13 @@ TopicSubscriber::~TopicSubscriber() {
3838

3939
void TopicSubscriber::register_listener(TTopicType::type topic_type, TopicListener* listener) {
4040
// Unique lock here to prevent access to listeners
41-
std::unique_lock<std::shared_mutex> lock(_listener_mtx);
41+
WriteLock lock(_listener_mtx);
4242
this->_registered_listeners[topic_type].push_back(listener);
4343
}
4444

4545
void TopicSubscriber::handle_updates(const TAgentPublishRequest& agent_publish_request) {
4646
// Shared lock here in order to avoid updates in listeners' map
47-
std::shared_lock<std::shared_mutex> lock(_listener_mtx);
47+
ReadLock lock(_listener_mtx);
4848
// Currently, not deal with protocol version, the listener should deal with protocol version
4949
const std::vector<TTopicUpdate>& topic_updates = agent_publish_request.updates;
5050
std::vector<TTopicUpdate>::const_iterator topic_update_it = topic_updates.begin();

‎be/src/common/config.h‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -169,8 +169,10 @@ CONF_mInt32(doris_scan_range_row_count, "524288");
169169
CONF_mInt32(doris_scan_range_max_mb, "0");
170170
// size of scanner queue between scanner thread and compute thread
171171
CONF_mInt32(doris_scanner_queue_size, "1024");
172-
// single read execute fragment row size
172+
// single read execute fragment row number
173173
CONF_mInt32(doris_scanner_row_num, "16384");
174+
// single read execute fragment row bytes
175+
CONF_mInt32(doris_scanner_row_bytes, "10485760");
174176
// number of max scan keys
175177
CONF_mInt32(doris_max_scan_key_num, "1024");
176178
// the max number of push down values of a single column.
@@ -604,10 +606,23 @@ CONF_mInt32(remote_storage_read_buffer_mb, "16");
604606

605607
// Default level of MemTracker to show in web page
606608
// now MemTracker support two level:
607-
// RELEASE: 0
608-
// DEBUG: 1
609+
// OVERVIEW: 0
610+
// TASK: 1
611+
// INSTANCE: 2
612+
// VERBOSE: 3
609613
// the level equal or lower than mem_tracker_level will show in web page
610-
CONF_Int16(mem_tracker_level, "0");
614+
CONF_mInt16(mem_tracker_level, "0");
615+
616+
// The minimum length when TCMalloc Hook consumes/releases MemTracker, consume size
617+
// smaller than this value will continue to accumulate. specified as number of bytes.
618+
// Decreasing this value will increase the frequency of consume/release.
619+
// Increasing this value will cause MemTracker statistics to be inaccurate.
620+
CONF_mInt32(mem_tracker_consume_min_size_bytes, "2097152");
621+
622+
// When MemTracker is a negative value, it is considered that a memory leak has occurred,
623+
// but the actual MemTracker records inaccurately will also cause a negative value,
624+
// so this feature is in the experimental stage.
625+
CONF_mBool(memory_leak_detection, "false");
611626

612627
// The version information of the tablet will be stored in the memory
613628
// in an adjacency graph data structure.

‎be/src/common/daemon.cpp‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -95,17 +95,6 @@ void Daemon::memory_maintenance_thread() {
9595
if (env != nullptr) {
9696
BufferPool* buffer_pool = env->buffer_pool();
9797
if (buffer_pool != nullptr) buffer_pool->Maintenance();
98-
99-
// The process limit as measured by our trackers may get out of sync with the
100-
// process usage if memory is allocated or freed without updating a MemTracker.
101-
// The metric is refreshed whenever memory is consumed or released via a MemTracker,
102-
// so on a system with queries executing it will be refreshed frequently. However
103-
// if the system is idle, we need to refresh the tracker occasionally since
104-
// untracked memory may be allocated or freed, e.g. by background threads.
105-
if (env->process_mem_tracker() != nullptr &&
106-
!env->process_mem_tracker()->is_consumption_metric_null()) {
107-
env->process_mem_tracker()->RefreshConsumptionFromMetric();
108-
}
10998
}
11099
}
111100
}

‎be/src/exec/analytic_eval_node.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,7 @@ Status AnalyticEvalNode::open(RuntimeState* state) {
201201
"Failed to acquire initial read buffer for analytic function "
202202
"evaluation. Reducing query concurrency or increasing the memory limit may "
203203
"help this query to complete successfully.");
204-
return mem_tracker()->MemLimitExceeded(state, msg, -1);
204+
RETURN_LIMIT_EXCEEDED(mem_tracker(), state, msg);
205205
}
206206

207207
DCHECK_EQ(_evaluators.size(), _fn_ctxs.size());

‎be/src/exec/base_scanner.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ BaseScanner::BaseScanner(RuntimeState* state, RuntimeProfile* profile,
4242
#if BE_TEST
4343
_mem_tracker(new MemTracker()),
4444
#else
45-
_mem_tracker(MemTracker::CreateTracker(
45+
_mem_tracker(MemTracker::create_tracker(
4646
-1, "BaseScanner:" + std::to_string(state->load_job_id()),
4747
state->instance_mem_tracker())),
4848
#endif

‎be/src/exec/broker_scan_node.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -318,7 +318,7 @@ Status BrokerScanNode::scanner_scan(const TBrokerScanRange& scan_range,
318318
// 1. too many batches in queue, or
319319
// 2. at least one batch in queue and memory exceed limit.
320320
(_batch_queue.size() >= _max_buffered_batches ||
321-
(mem_tracker()->AnyLimitExceeded(MemLimit::HARD) && !_batch_queue.empty()))) {
321+
(mem_tracker()->any_limit_exceeded() && !_batch_queue.empty()))) {
322322
_queue_writer_cond.wait_for(l, std::chrono::seconds(1));
323323
}
324324
// Process already set failed, so we just return OK

‎be/src/exec/broker_scanner.cpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@
3535
#include "exprs/expr.h"
3636
#include "runtime/descriptors.h"
3737
#include "runtime/exec_env.h"
38-
#include "runtime/mem_tracker.h"
3938
#include "runtime/raw_value.h"
4039
#include "runtime/stream_load/load_stream_mgr.h"
4140
#include "runtime/stream_load/stream_load_pipe.h"

‎be/src/exec/broker_scanner.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,6 @@ class ExprContext;
4646
class TupleDescriptor;
4747
class TupleRow;
4848
class RowDescriptor;
49-
class MemTracker;
5049
class RuntimeProfile;
5150
class StreamLoadPipe;
5251

0 commit comments

Comments
 (0)