Skip to content

Deadlock in ProcessList #37794

@tavplubix

Description

@tavplubix

https://s3.amazonaws.com/clickhouse-test-reports/37740/002f95741a1363373ee91e3dca28aff3c83c8794/stress_test__memory__actions_.html

Looks like this (however, I cannot find the root cause in the stacktraces):


Thread 1 (Thread 0x7f026f965d40 (LWP 665)):
#0  0x00007f026fcbe110 in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x00007f026fcb60a3 in pthread_mutex_lock () from /lib/x86_64-linux-gnu/libpthread.so.0
#2  0x000000000b5cd285 in pthread_mutex_lock ()
#3  0x0000000058e40339 in std::__1::__libcpp_mutex_lock (__m=0x723000002380) at ../contrib/libcxx/include/__threading_support:303
#4  std::__1::mutex::lock (this=0x723000002380) at ../contrib/libcxx/src/mutex.cpp:33
#5  0x000000003f028849 in std::__1::lock_guard<std::__1::mutex>::lock_guard (this=0x7ffd03ff7650, __m=...) at ../contrib/libcxx/include/__mutex_base:91
#6  DB::ProcessList::killAllQueries (this=0x723000002380) at ../src/Interpreters/ProcessList.cpp:443
#7  0x000000000b666987 in DB::Server::main(std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&)::$_10::operator()() const (this=<optimized out>) at ../programs/server/Server.cpp:1728
#8  basic_scope_guard<DB::Server::main(std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&)::$_10>::invoke() (this=<optimized out>) at ../base/base/../base/scope_guard.h:99
#9  basic_scope_guard<DB::Server::main(std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&)::$_10>::~basic_scope_guard() (this=this@entry=0x7ffd03ff8e50) at ../base/base/../base/scope_guard.h:48
#10 0x000000000b652b04 in DB::Server::main (this=<optimized out>) at ../programs/server/Server.cpp:1786
#11 0x00000000531a1eb6 in Poco::Util::Application::run (this=<optimized out>) at ../contrib/poco/Util/src/Application.cpp:334
#12 0x000000000b618622 in DB::Server::run (this=<optimized out>) at ../programs/server/Server.cpp:463
#13 0x00000000531fc2de in Poco::Util::ServerApplication::run (this=0x7ffd03ff9250, argc=<optimized out>, argv=0x703000014d90) at ../contrib/poco/Util/src/ServerApplication.cpp:611
#14 0x000000000b610a8f in mainEntryClickHouseServer (argc=<optimized out>, argv=0x703000014d90) at ../programs/server/Server.cpp:187
#15 0x000000000b6096fe in main (argc_=<optimized out>, argv_=0xfffffffffffffef0) at ../programs/main.cpp:439


Thread 1390 (Thread 0x7efc59bdf700 (LWP 5395)):
#0  0x00007f026fcbe110 in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x00007f026fcb60a3 in pthread_mutex_lock () from /lib/x86_64-linux-gnu/libpthread.so.0
#2  0x000000000b5cd285 in pthread_mutex_lock ()
#3  0x0000000058e40339 in std::__1::__libcpp_mutex_lock (__m=0x723000002380) at ../contrib/libcxx/include/__threading_support:303
#4  std::__1::mutex::lock (this=0x723000002380) at ../contrib/libcxx/src/mutex.cpp:33
#5  0x000000003f0141c8 in std::__1::unique_lock<std::__1::mutex>::unique_lock (this=0x7efc59bd3358, __m=...) at ../contrib/libcxx/include/__mutex_base:119
#6  DB::ProcessList::insert (this=<optimized out>, query_=..., ast=<optimized out>, query_context=...) at ../src/Interpreters/ProcessList.cpp:84
#7  0x000000003f9cc467 in DB::executeQueryImpl (begin=<optimized out>, end=<optimized out>, context=..., internal=<optimized out>, stage=<optimized out>, istr=<optimized out>) at ../src/Interpreters/executeQuery.cpp:580
#8  0x000000003f9c5ed7 in DB::executeQuery (query=..., context=..., internal=<optimized out>, stage=<optimized out>) at ../src/Interpreters/executeQuery.cpp:1069
#9  0x00000000428ac31c in DB::TCPHandler::runImpl (this=<optimized out>) at ../src/Server/TCPHandler.cpp:332
#10 0x00000000428e7e4c in DB::TCPHandler::run (this=0x71a0031d2e00) at ../src/Server/TCPHandler.cpp:1781
#11 0x000000005315aa1c in Poco::Net::TCPServerConnection::start (this=0x723000002380) at ../contrib/poco/Net/src/TCPServerConnection.cpp:43
#12 0x000000005315be66 in Poco::Net::TCPServerDispatcher::run (this=<optimized out>) at ../contrib/poco/Net/src/TCPServerDispatcher.cpp:115
#13 0x0000000053878492 in Poco::PooledThread::run (this=0x7150003d65a0) at ../contrib/poco/Foundation/src/ThreadPool.cpp:199
#14 0x00000000538738c0 in Poco::(anonymous namespace)::RunnableHolder::run (this=<optimized out>) at ../contrib/poco/Foundation/src/Thread.cpp:55
#15 0x000000005386f756 in Poco::ThreadImpl::runnableEntry (pThread=<optimized out>) at ../contrib/poco/Foundation/src/Thread_POSIX.cpp:345
#16 0x00007f026fcb3609 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#17 0x00007f026fbd8163 in clone () from /lib/x86_64-linux-gnu/libc.so.6

Thread 863 (Thread 0x7efe6a0b2700 (LWP 3014)):
#0  0x00007f026fcbe110 in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x00007f026fcb60a3 in pthread_mutex_lock () from /lib/x86_64-linux-gnu/libpthread.so.0
#2  0x000000000b5cd285 in pthread_mutex_lock ()
#3  0x0000000058e40339 in std::__1::__libcpp_mutex_lock (__m=0x723000002380) at ../contrib/libcxx/include/__threading_support:303
#4  std::__1::mutex::lock (this=0x723000002380) at ../contrib/libcxx/src/mutex.cpp:33
#5  0x000000003f022ca5 in std::__1::lock_guard<std::__1::mutex>::lock_guard (this=0x7efe6a0a83b0, __m=...) at ../contrib/libcxx/include/__mutex_base:91
#6  DB::ProcessListEntry::~ProcessListEntry (this=0x703001816428) at ../src/Interpreters/ProcessList.cpp:269
#7  0x000000003f03592d in std::__1::__destroy_at<DB::ProcessListEntry, 0> (__loc=0x723000002380) at ../contrib/libcxx/include/__memory/construct_at.h:56
#8  std::__1::destroy_at<DB::ProcessListEntry, 0> (__loc=0x723000002380) at ../contrib/libcxx/include/__memory/construct_at.h:81
#9  std::__1::allocator_traits<std::__1::allocator<DB::ProcessListEntry> >::destroy<DB::ProcessListEntry, void, void> (__p=0x723000002380) at ../contrib/libcxx/include/__memory/allocator_traits.h:317
#10 std::__1::__shared_ptr_emplace<DB::ProcessListEntry, std::__1::allocator<DB::ProcessListEntry> >::__on_zero_shared (this=<optimized out>) at ../contrib/libcxx/include/__memory/shared_ptr.h:310
#11 0x000000003babde63 in std::__1::__shared_count::__release_shared (this=0x703001816410) at ../contrib/libcxx/include/__memory/shared_ptr.h:174
#12 std::__1::__shared_weak_count::__release_shared (this=0x703001816410) at ../contrib/libcxx/include/__memory/shared_ptr.h:216
#13 std::__1::shared_ptr<DB::ProcessListEntry>::~shared_ptr (this=0x7efe6a0a84b0) at ../contrib/libcxx/include/__memory/shared_ptr.h:703
#14 std::__1::shared_ptr<DB::ProcessListEntry>::reset (this=0x71a0026402c0) at ../contrib/libcxx/include/__memory/shared_ptr.h:769
#15 DB::BlockIO::reset (this=0x71a0026402c0) at ../src/QueryPipeline/BlockIO.cpp:21
#16 DB::BlockIO::operator= (this=0x71a0026402c0, rhs=...) at ../src/QueryPipeline/BlockIO.cpp:32
#17 0x00000000428f0fa1 in DB::QueryState::operator= (this=<optimized out>, this@entry=0x71a0026401e0) at ../src/Server/TCPHandler.h:46
#18 0x00000000428acac0 in DB::QueryState::reset (this=0x71a0026401e0) at ../src/Server/TCPHandler.h:110
#19 DB::TCPHandler::runImpl (this=<optimized out>) at ../src/Server/TCPHandler.cpp:393
#20 0x00000000428e7e4c in DB::TCPHandler::run (this=0x71a002640000) at ../src/Server/TCPHandler.cpp:1781
#21 0x000000005315aa1c in Poco::Net::TCPServerConnection::start (this=0x723000002380) at ../contrib/poco/Net/src/TCPServerConnection.cpp:43
#22 0x000000005315be66 in Poco::Net::TCPServerDispatcher::run (this=<optimized out>) at ../contrib/poco/Net/src/TCPServerDispatcher.cpp:115
#23 0x0000000053878492 in Poco::PooledThread::run (this=0x7150002b4d20) at ../contrib/poco/Foundation/src/ThreadPool.cpp:199
#24 0x00000000538738c0 in Poco::(anonymous namespace)::RunnableHolder::run (this=<optimized out>) at ../contrib/poco/Foundation/src/Thread.cpp:55
#25 0x000000005386f756 in Poco::ThreadImpl::runnableEntry (pThread=<optimized out>) at ../contrib/poco/Foundation/src/Thread_POSIX.cpp:345
#26 0x00007f026fcb3609 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#27 0x00007f026fbd8163 in clone () from /lib/x86_64-linux-gnu/libc.so.6

Metadata

Metadata

Assignees

Labels

fuzzProblem found by one of the fuzzers

Type

No type

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions