markus makela added a comment - 2022-02-03 04:49 Looks like it's a deadlock between the MainWorker and the libmicrohttpd daemon thread: Thread 10 (Thread 0x7f2e7c654640 (LWP 901045) "MHD-single"): #0 0x00007f2e88ee0a8a in __futex_abstimed_wait_common64 () from /lib64/libpthread.so.0 #1 0x00007f2e88edcc98 in __new_sem_wait_slow64.constprop.0 () from /lib64/libpthread.so.0 #2 0x00007f2e897a5a55 in maxbase::Semaphore::wait (this=0x7f2e7c64f240, signal_approach=maxbase::Semaphore::IGNORE_SIGNALS) at /home/markusjm/MaxScale/maxutils/maxbase/include/maxbase/semaphore.hh:144 #3 0x00007f2e89964939 in maxbase::Worker::call(std::function<void ()>, maxbase::Worker::execute_mode_t) (this=0x7ffe7c8fa640, func=..., mode=maxbase::Worker::EXECUTE_AUTO) at /home/markusjm/MaxScale/maxutils/maxbase/src/worker.cc:535 #4 0x00007f2e898107a6 in resource_handle_request (request=...) at /home/markusjm/MaxScale/server/core/resource.cc:1547 #5 0x00007f2e895d46b6 in Client::process (this=0x60d00004d9c0, url="/v1/servers", method="GET", upload_data=0x0, upload_size=0x7f2e7c650078) at /home/markusjm/MaxScale/server/core/admin.cc:721 #6 0x00007f2e895d3ab1 in Client::handle (this=0x60d00004d9c0, url="/v1/servers", method="GET", upload_data=0x0, upload_data_size=0x7f2e7c650078) at /home/markusjm/MaxScale/server/core/admin.cc:660 #7 0x00007f2e895cda85 in (anonymous namespace)::handle_client (cls=0x0, connection=0x614000090840, url=0x62d000064404 "/v1/servers", method=0x62d000064400 "GET", version=0x62d000064410 "HTTP/1.1", upload_data=0x0, upload_data_size=0x7f2e7c650078, con_cls=0x614000090898) at /home/markusjm/MaxScale/server/core/admin.cc:168 #8 0x00007f2e89a32caf in call_connection_handler (connection=connection@entry=0x614000090840) at connection.c:2222 #9 0x00007f2e89a347f8 in MHD_connection_handle_idle (connection=connection@entry=0x614000090840) at connection.c:3517 #10 0x00007f2e89a367ae in call_handlers (con=con@entry=0x614000090840, read_ready=<optimized out>, write_ready=<optimized out>, force_close=<optimized out>) at daemon.c:1207 #11 0x00007f2e89a3b480 in MHD_epoll (daemon=0x617000001900, may_block=<optimized out>) at daemon.c:4581 #12 0x00007f2e89a3c225 in MHD_polling_thread (cls=0x617000001900) at daemon.c:4746 #13 0x00007f2e88ed4299 in start_thread () from /lib64/libpthread.so.0 #14 0x00007f2e8850d353 in clone () from /lib64/libc.so.6     Thread 1 (Thread 0x7f2e879287c0 (LWP 901036) "maxscale"): #0 0x00007f2e88ee0a8a in __futex_abstimed_wait_common64 () from /lib64/libpthread.so.0 #1 0x00007f2e88ed5793 in __pthread_clockjoin_ex () from /lib64/libpthread.so.0 #2 0x00007f2e89a3c996 in MHD_stop_daemon (daemon=0x617000001900) at daemon.c:6880 #3 0x00007f2e895d8fc7 in mxs_admin_shutdown () at /home/markusjm/MaxScale/server/core/admin.cc:1011 #4 0x00007f2e89767d2e in operator() (__closure=0x60400001edd8) at /home/markusjm/MaxScale/server/core/mainworker.cc:308 #5 0x00007f2e89769c42 in std::__invoke_impl<void, maxscale::MainWorker::start_shutdown()::<lambda()>&>(std::__invoke_other, struct {...} &) (__f=...) at /usr/include/c++/11/bits/invoke.h:61 #6 0x00007f2e897693ff in std::__invoke_r<void, maxscale::MainWorker::start_shutdown()::<lambda()>&>(struct {...} &) (__fn=...) at /usr/include/c++/11/bits/invoke.h:154 #7 0x00007f2e89768c16 in std::_Function_handler<void(), maxscale::MainWorker::start_shutdown()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/11/bits/std_function.h:291 #8 0x00007f2e8984904e in std::function<void ()>::operator()() const (this=0x60400001edd8) at /usr/include/c++/11/bits/std_function.h:560 #9 0x00007f2e89964398 in maxbase::Worker::CustomTask::execute (this=0x60400001edd0, worker=...) at /home/markusjm/MaxScale/maxutils/maxbase/src/worker.cc:504 #10 0x00007f2e89965d3f in maxbase::Worker::handle_message (this=0x7ffe7c8fa640, queue=..., msg=...) at /home/markusjm/MaxScale/maxutils/maxbase/src/worker.cc:666 #11 0x00007f2e89974a26 in maxbase::MessageQueue::handle_poll_events (this=0x604000001790, pWorker=0x7ffe7c8fa640, events=1) at /home/markusjm/MaxScale/maxutils/maxbase/src/messagequeue.cc:307 #12 0x00007f2e89974d0f in maxbase::MessageQueue::poll_handler (pData=0x604000001790, pWorker=0x7ffe7c8fa640, events=1) at /home/markusjm/MaxScale/maxutils/maxbase/src/messagequeue.cc:342 #13 0x00007f2e89966d3b in maxbase::Worker::poll_waitevents (this=0x7ffe7c8fa640) at /home/markusjm/MaxScale/maxutils/maxbase/src/worker.cc:877 #14 0x00007f2e89964e26 in maxbase::Worker::run (this=0x7ffe7c8fa640, pSem=0x0) at /home/markusjm/MaxScale/maxutils/maxbase/src/worker.cc:570 #15 0x0000000000423bb5 in maxbase::Worker::run (this=0x7ffe7c8fa640) at /home/markusjm/MaxScale/maxutils/maxbase/include/maxbase/worker.hh:449 #16 0x000000000041c9dc in main (argc=4, argv=0x7ffe7c8fc1c8) at /home/markusjm/MaxScale/server/core/gateway.cc:2168