Skip to content

Data-race in DatabaseReplicated #67270

Closed
#67262
Closed
Data-race in DatabaseReplicated#67270
#67262
Assignees
alexey-milovidov
Labels
testingSpecial issue with list of bugs found by CI
@yakov-olkhovskiy

Description

@yakov-olkhovskiy
yakov-olkhovskiy
opened on Jul 27, 2024

CI: https://s3.amazonaws.com/clickhouse-test-reports/67236/0f4274ac9659efda64e9f362fc74383c0853ba24/stress_test__tsan_.html

WARNING: ThreadSanitizer: data race (pid=77086)
  Write of size 8 at 0x726400101680 by main thread:
    #0 std::__1::unique_ptr<DB::DatabaseReplicatedDDLWorker, std::__1::default_delete<DB::DatabaseReplicatedDDLWorker>>::reset[abi:v15007](DB::DatabaseReplicatedDDLWorker*) build_docker/./contrib/llvm-project/libcxx/include/__memory/unique_ptr.h:303:20 (clickhouse+0x16daf0ec) (BuildId: c1b94c9c0ad0ab77bf54d6a0d0696bdfa7d8b450)
    #1 std::__1::unique_ptr<DB::DatabaseReplicatedDDLWorker, std::__1::default_delete<DB::DatabaseReplicatedDDLWorker>>::operator=[abi:v15007](std::nullptr_t) build_docker/./contrib/llvm-project/libcxx/include/__memory/unique_ptr.h:263:5 (clickhouse+0x16daf0ec)
    #2 DB::DatabaseReplicated::shutdown() build_docker/./src/Databases/DatabaseReplicated.cpp:1534:16 (clickhouse+0x16daf0ec)
    #3 DB::DatabaseCatalog::shutdownImpl() build_docker/./src/Interpreters/DatabaseCatalog.cpp:273:26 (clickhouse+0x1723419e) (BuildId: c1b94c9c0ad0ab77bf54d6a0d0696bdfa7d8b450)
    #4 DB::DatabaseCatalog::shutdown() build_docker/./src/Interpreters/DatabaseCatalog.cpp:873:27 (clickhouse+0x1723d2eb) (BuildId: c1b94c9c0ad0ab77bf54d6a0d0696bdfa7d8b450)
    #5 DB::ContextSharedPart::shutdown() build_docker/./src/Interpreters/Context.cpp:620:9 (clickhouse+0x17183b9c) (BuildId: c1b94c9c0ad0ab77bf54d6a0d0696bdfa7d8b450)
    #6 DB::Context::shutdown() build_docker/./src/Interpreters/Context.cpp:4705:13 (clickhouse+0x17171b41) (BuildId: c1b94c9c0ad0ab77bf54d6a0d0696bdfa7d8b450)
    #7 DB::Server::main(std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>>> const&)::$_2::operator()() const build_docker/./programs/server/Server.cpp:923:5 (clickhouse+0xfa3a685) (BuildId: c1b94c9c0ad0ab77bf54d6a0d0696bdfa7d8b450)
    #8 BasicScopeGuard<DB::Server::main(std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>>> const&)::$_2>::invoke() build_docker/./base/base/../base/scope_guard.h:101:9 (clickhouse+0xfa3a685)
    #9 BasicScopeGuard<DB::Server::main(std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>>> const&)::$_2>::~BasicScopeGuard() build_docker/./base/base/../base/scope_guard.h:50:26 (clickhouse+0xfa3a685)
    #10 DB::Server::main(std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>>> const&) build_docker/./programs/server/Server.cpp:2310:1 (clickhouse+0xfa2a475) (BuildId: c1b94c9c0ad0ab77bf54d6a0d0696bdfa7d8b450)
    #11 Poco::Util::Application::run() build_docker/./base/poco/Util/src/Application.cpp:315:8 (clickhouse+0x1e0ccb9e) (BuildId: c1b94c9c0ad0ab77bf54d6a0d0696bdfa7d8b450)
    #12 DB::Server::run() build_docker/./programs/server/Server.cpp:394:25 (clickhouse+0xfa10185) (BuildId: c1b94c9c0ad0ab77bf54d6a0d0696bdfa7d8b450)
    #13 Poco::Util::ServerApplication::run(int, char**) build_docker/./base/poco/Util/src/ServerApplication.cpp:131:9 (clickhouse+0x1e0eb3c0) (BuildId: c1b94c9c0ad0ab77bf54d6a0d0696bdfa7d8b450)
    #14 mainEntryClickHouseServer(int, char**) build_docker/./programs/server/Server.cpp:200:20 (clickhouse+0xfa0cfb5) (BuildId: c1b94c9c0ad0ab77bf54d6a0d0696bdfa7d8b450)
    #15 main build_docker/./programs/main.cpp:248:21 (clickhouse+0x73f38e8) (BuildId: c1b94c9c0ad0ab77bf54d6a0d0696bdfa7d8b450)

  Previous read of size 8 at 0x726400101680 by thread T78 (mutexes: write M0, write M1):
    #0 std::__1::unique_ptr<DB::DatabaseReplicatedDDLWorker, std::__1::default_delete<DB::DatabaseReplicatedDDLWorker>>::operator->[abi:v15007]() const build_docker/./contrib/llvm-project/libcxx/include/__memory/unique_ptr.h:274:19 (clickhouse+0x16db23f9) (BuildId: c1b94c9c0ad0ab77bf54d6a0d0696bdfa7d8b450)
    #1 DB::DatabaseReplicated::canExecuteReplicatedMetadataAlter() const build_docker/./src/Databases/DatabaseReplicated.cpp:1682:38 (clickhouse+0x16db23f9)
    #2 DB::ReplicatedMergeTreeQueue::shouldExecuteLogEntry(DB::ReplicatedMergeTreeLogEntry const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&, DB::MergeTreeDataMergerMutator&, DB::MergeTreeData&, std::__1::unique_lock<std::__1::mutex>&) const build_docker/./src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp:1565:24 (clickhouse+0x1985fece) (BuildId: c1b94c9c0ad0ab77bf54d6a0d0696bdfa7d8b450)
    #3 DB::ReplicatedMergeTreeQueue::selectEntryToProcess(DB::MergeTreeDataMergerMutator&, DB::MergeTreeData&) build_docker/./src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp:1785:13 (clickhouse+0x1986567f) (BuildId: c1b94c9c0ad0ab77bf54d6a0d0696bdfa7d8b450)
    #4 DB::StorageReplicatedMergeTree::selectQueueEntry() build_docker/./src/Storages/StorageReplicatedMergeTree.cpp:3626:26 (clickhouse+0x18e9549e) (BuildId: c1b94c9c0ad0ab77bf54d6a0d0696bdfa7d8b450)
    #5 DB::StorageReplicatedMergeTree::scheduleDataProcessingJob(DB::BackgroundJobsAssignee&) build_docker/./src/Storages/StorageReplicatedMergeTree.cpp:3690:65 (clickhouse+0x18e9579c) (BuildId: c1b94c9c0ad0ab77bf54d6a0d0696bdfa7d8b450)
    #6 DB::BackgroundJobsAssignee::threadFunc() build_docker/./src/Storages/MergeTree/BackgroundJobsAssignee.cpp:131:28 (clickhouse+0x193147b6) (BuildId: c1b94c9c0ad0ab77bf54d6a0d0696bdfa7d8b450)
    #7 DB::BackgroundJobsAssignee::start()::$_0::operator()() const build_docker/./src/Storages/MergeTree/BackgroundJobsAssignee.cpp:102:113 (clickhouse+0x19314a01) (BuildId: c1b94c9c0ad0ab77bf54d6a0d0696bdfa7d8b450)
    #8 decltype(std::declval<DB::BackgroundJobsAssignee::start()::$_0&>()()) std::__1::__invoke[abi:v15007]<DB::BackgroundJobsAssignee::start()::$_0&>(DB::BackgroundJobsAssignee::start()::$_0&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:394:23 (clickhouse+0x19314a01)
    #9 void std::__1::__invoke_void_return_wrapper<void, true>::__call<DB::BackgroundJobsAssignee::start()::$_0&>(DB::BackgroundJobsAssignee::start()::$_0&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:479:9 (clickhouse+0x19314a01)
    #10 std::__1::__function::__default_alloc_func<DB::BackgroundJobsAssignee::start()::$_0, void ()>::operator()[abi:v15007]() build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:235:12 (clickhouse+0x19314a01)
    #11 void std::__1::__function::__policy_invoker<void ()>::__call_impl<std::__1::__function::__default_alloc_func<DB::BackgroundJobsAssignee::start()::$_0, void ()>>(std::__1::__function::__policy_storage const*) build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:716:16 (clickhouse+0x19314a01)
    #12 std::__1::__function::__policy_func<void ()>::operator()[abi:v15007]() const build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:848:16 (clickhouse+0x166d5311) (BuildId: c1b94c9c0ad0ab77bf54d6a0d0696bdfa7d8b450)
    #13 std::__1::function<void ()>::operator()() const build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:1197:12 (clickhouse+0x166d5311)
    #14 DB::BackgroundSchedulePoolTaskInfo::execute() build_docker/./src/Core/BackgroundSchedulePool.cpp:106:9 (clickhouse+0x166d5311)
    #15 DB::BackgroundSchedulePool::threadFunction() build_docker/./src/Core/BackgroundSchedulePool.cpp:304:19 (clickhouse+0x166d962d) (BuildId: c1b94c9c0ad0ab77bf54d6a0d0696bdfa7d8b450)
    #16 DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0::operator()() const build_docker/./src/Core/BackgroundSchedulePool.cpp:170:79 (clickhouse+0x166d9fd2) (BuildId: c1b94c9c0ad0ab77bf54d6a0d0696bdfa7d8b450)
    #17 decltype(std::declval<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&>()()) std::__1::__invoke[abi:v15007]<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:394:23 (clickhouse+0x166d9fd2)
    #18 decltype(auto) std::__1::__apply_tuple_impl[abi:v15007]<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&, std::__1::tuple<>&>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&, std::__1::tuple<>&, std::__1::__tuple_indices<...>) build_docker/./contrib/llvm-project/libcxx/include/tuple:1789:1 (clickhouse+0x166d9fd2)
    #19 decltype(auto) std::__1::apply[abi:v15007]<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&, std::__1::tuple<>&>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&, std::__1::tuple<>&) build_docker/./contrib/llvm-project/libcxx/include/tuple:1798:1 (clickhouse+0x166d9fd2)
    #20 ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'()::operator()() build_docker/./src/Common/ThreadPool.h:251:13 (clickhouse+0x166d9fd2)
    #21 decltype(std::declval<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>()()) std::__1::__invoke[abi:v15007]<ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'()&>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:394:23 (clickhouse+0x166d9fd2)
    #22 void std::__1::__invoke_void_return_wrapper<void, true>::__call<ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'()&>(ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'()&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:479:9 (clickhouse+0x166d9fd2)
    #23 std::__1::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'(), void ()>::operator()[abi:v15007]() build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:235:12 (clickhouse+0x166d9fd2)
    #24 void std::__1::__function::__policy_invoker<void ()>::__call_impl<std::__1::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'(), void ()>>(std::__1::__function::__policy_storage const*) build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:716:16 (clickhouse+0x166d9fd2)
    #25 std::__1::__function::__policy_func<void ()>::operator()[abi:v15007]() const build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:848:16 (clickhouse+0xf8f7263) (BuildId: c1b94c9c0ad0ab77bf54d6a0d0696bdfa7d8b450)
    #26 std::__1::function<void ()>::operator()() const build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:1197:12 (clickhouse+0xf8f7263)
    #27 ThreadPoolImpl<std::__1::thread>::worker(std::__1::__list_iterator<std::__1::thread, void*>) build_docker/./src/Common/ThreadPool.cpp:462:13 (clickhouse+0xf8f7263)
    #28 void ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, Priority, std::__1::optional<unsigned long>, bool)::'lambda0'()::operator()() const build_docker/./src/Common/ThreadPool.cpp:219:73 (clickhouse+0xf8fd931) (BuildId: c1b94c9c0ad0ab77bf54d6a0d0696bdfa7d8b450)
    #29 decltype(std::declval<void>()()) std::__1::__invoke[abi:v15007]<void ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, Priority, std::__1::optional<unsigned long>, bool)::'lambda0'()>(void&&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:394:23 (clickhouse+0xf8fd931)
    #30 void std::__1::__thread_execute[abi:v15007]<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct>>, void ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, Priority, std::__1::optional<unsigned long>, bool)::'lambda0'()>(std::__1::tuple<void, void ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, Priority, std::__1::optional<unsigned long>, bool)::'lambda0'()>&, std::__1::__tuple_indices<...>) build_docker/./contrib/llvm-project/libcxx/include/thread:284:5 (clickhouse+0xf8fd931)
    #31 void* std::__1::__thread_proxy[abi:v15007]<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct>>, void ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, Priority, std::__1::optional<unsigned long>, bool)::'lambda0'()>>(void*) build_docker/./contrib/llvm-project/libcxx/include/thread:295:5 (clickhouse+0xf8fd931)

Metadata

Metadata

Assignees

Labels

testingSpecial issue with list of bugs found by CI

Type

No type

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions