Race condition in service destruction

=================================================================

==12860==ERROR: AddressSanitizer: heap-use-after-free on address 0x61900001d4b0 at pc 0x7f6431cd1d80 bp 0x7f641febab60 sp 0x7f641febab50

READ of size 8 at 0x61900001d4b0 thread T9

==12860==AddressSanitizer: while reporting a bug found another one. Ignoring.

==12860==AddressSanitizer: while reporting a bug found another one. Ignoring.

==12860==AddressSanitizer: while reporting a bug found another one. Ignoring.

    #0 0x7f6431cd1d7f in maxscale::WorkerLocal<std::unique_ptr<maxscale::UserAccountCache, std::default_delete<maxscale::UserAccountCache> >, maxscale::DefaultConstructor<std::unique_ptr<maxscale::UserAccountCache, std::default_delete<maxscale::UserAccountCache> > > >::get_local_value() const /home/vagrant/MaxScale/include/maxscale/workerlocal.hh:129

    #1 0x7f6431cc9659 in maxscale::WorkerLocal<std::unique_ptr<maxscale::UserAccountCache, std::default_delete<maxscale::UserAccountCache> >, maxscale::DefaultConstructor<std::unique_ptr<maxscale::UserAccountCache, std::default_delete<maxscale::UserAccountCache> > > >::operator*() (/usr/lib64/maxscale/libmaxscale-common.so.1.0.0+0x784659)

    #2 0x7f6431cb81f7 in operator() /home/vagrant/MaxScale/server/core/service.cc:1951

    #3 0x7f6431cbc4fd in _M_invoke /opt/rh/devtoolset-7/root/usr/include/c++/7/bits/std_function.h:316

    #4 0x7f6431c4eb6d in std::function<void ()>::operator()() const /opt/rh/devtoolset-7/root/usr/include/c++/7/bits/std_function.h:706

    #5 0x7f6431e67b33 in execute /home/vagrant/MaxScale/maxutils/maxbase/src/worker.cc:492

    #6 0x7f6431e692d3 in maxbase::Worker::handle_message(maxbase::MessageQueue&, maxbase::MessageQueueMessage const&) /home/vagrant/MaxScale/maxutils/maxbase/src/worker.cc:654

    #7 0x7f6431e7852f in maxbase::MessageQueue::handle_poll_events(maxbase::Worker*, unsigned int) /home/vagrant/MaxScale/maxutils/maxbase/src/messagequeue.cc:307

    #8 0x7f6431e787ea in maxbase::MessageQueue::poll_handler(MXB_POLL_DATA*, MXB_WORKER*, unsigned int) /home/vagrant/MaxScale/maxutils/maxbase/src/messagequeue.cc:342

    #9 0x7f6431e6a5e3 in maxbase::Worker::poll_waitevents() /home/vagrant/MaxScale/maxutils/maxbase/src/worker.cc:863

    #10 0x7f6431e68366 in maxbase::Worker::run(maxbase::Semaphore*) /home/vagrant/MaxScale/maxutils/maxbase/src/worker.cc:558

    #11 0x7f6431e693e6 in maxbase::Worker::thread_main(maxbase::Worker*, maxbase::Semaphore*) /home/vagrant/MaxScale/maxutils/maxbase/src/worker.cc:684

    #12 0x7f6431e6ff3c in void std::__invoke_impl<void, void (*)(maxbase::Worker*, maxbase::Semaphore*), maxbase::Worker*, maxbase::Semaphore*>(std::__invoke_other, void (*&&)(maxbase::Worker*, maxbase::Semaphore*), maxbase::Worker*&&, maxbase::Semaphore*&&) /opt/rh/devtoolset-7/root/usr/include/c++/7/bits/invoke.h:60

    #13 0x7f6431e6e6c2 in std::__invoke_result<void (*)(maxbase::Worker*, maxbase::Semaphore*), maxbase::Worker*, maxbase::Semaphore*>::type std::__invoke<void (*)(maxbase::Worker*, maxbase::Semaphore*), maxbase::Worker*, maxbase::Semaphore*>(void (*&&)(maxbase::Worker*, maxbase::Semaphore*), maxbase::Worker*&&, maxbase::Semaphore*&&) /opt/rh/devtoolset-7/root/usr/include/c++/7/bits/invoke.h:95

    #14 0x7f6431e764ca in decltype (__invoke((_S_declval<0ul>)(), (_S_declval<1ul>)(), (_S_declval<2ul>)())) std::thread::_Invoker<std::tuple<void (*)(maxbase::Worker*, maxbase::Semaphore*), maxbase::Worker*, maxbase::Semaphore*> >::_M_invoke<0ul, 1ul, 2ul>(std::_Index_tuple<0ul, 1ul, 2ul>) /opt/rh/devtoolset-7/root/usr/include/c++/7/thread:234

    #15 0x7f6431e7641a in std::thread::_Invoker<std::tuple<void (*)(maxbase::Worker*, maxbase::Semaphore*), maxbase::Worker*, maxbase::Semaphore*> >::operator()() /opt/rh/devtoolset-7/root/usr/include/c++/7/thread:243

    #16 0x7f6431e76243 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (*)(maxbase::Worker*, maxbase::Semaphore*), maxbase::Worker*, maxbase::Semaphore*> > >::_M_run() /opt/rh/devtoolset-7/root/usr/include/c++/7/thread:186

    #17 0x7f6431f3bdae in execute_native_thread_routine (/usr/lib64/maxscale/libmaxscale-common.so.1.0.0+0x9f6dae)

    #18 0x7f642f665ea4 in start_thread (/lib64/libpthread.so.0+0x7ea4)

    #19 0x7f642ea6a9fc in __clone (/lib64/libc.so.6+0xfe9fc)

​

0x61900001d4b0 is located 816 bytes inside of 976-byte region [0x61900001d180,0x61900001d550)

freed by thread T0 here:

    #0 0x7f643255b728 in operator delete(void*, unsigned long) (/lib64/libasan.so.4+0xe1728)

    #1 0x7f6431ca3760 in Service::~Service() /home/vagrant/MaxScale/server/core/service.cc:532

    #2 0x7f6431cb7523 in operator() /home/vagrant/MaxScale/server/core/service.cc:1891

    #3 0x7f6431cbbed7 in _M_invoke /opt/rh/devtoolset-7/root/usr/include/c++/7/bits/std_function.h:316

    #4 0x7f6431c4eb6d in std::function<void ()>::operator()() const /opt/rh/devtoolset-7/root/usr/include/c++/7/bits/std_function.h:706

    #5 0x7f6431e67b33 in execute /home/vagrant/MaxScale/maxutils/maxbase/src/worker.cc:492

    #6 0x7f6431e67914 in maxbase::Worker::execute(maxbase::WorkerTask*, maxbase::Semaphore*, maxbase::Worker::execute_mode_t) /home/vagrant/MaxScale/maxutils/maxbase/src/worker.cc:423

    #7 0x7f6431e67c0c in maxbase::Worker::execute(std::function<void ()> const&, maxbase::Semaphore*, maxbase::Worker::execute_mode_t) /home/vagrant/MaxScale/maxutils/maxbase/src/worker.cc:504

    #8 0x41f84c in maxbase::Worker::execute(std::function<void ()>, maxbase::Worker::execute_mode_t) (/usr/bin/maxscale+0x41f84c)

    #9 0x7f6431cb765c in Service::decref() /home/vagrant/MaxScale/server/core/service.cc:1889

    #10 0x7f6431ca39a4 in Service::destroy(Service*) /home/vagrant/MaxScale/server/core/service.cc:540

    #11 0x7f6431aa7b7e in runtime_destroy_service(Service*, bool) /home/vagrant/MaxScale/server/core/config_runtime.cc:1846

    #12 0x7f6431bf84d9 in cb_delete_service /home/vagrant/MaxScale/server/core/resource.cc:653

    #13 0x7f6431bf159a in Resource::call(HttpRequest const&) const /home/vagrant/MaxScale/server/core/resource.cc:151

    #14 0x7f6431c04c05 in process_request /home/vagrant/MaxScale/server/core/resource.cc:1541

    #15 0x7f6431c0805a in handle_request /home/vagrant/MaxScale/server/core/resource.cc:1736

    #16 0x7f6431c08dce in operator() /home/vagrant/MaxScale/server/core/resource.cc:1803

    #17 0x7f6431c099ee in _M_invoke /opt/rh/devtoolset-7/root/usr/include/c++/7/bits/std_function.h:316

    #18 0x7f6431c4eb6d in std::function<void ()>::operator()() const /opt/rh/devtoolset-7/root/usr/include/c++/7/bits/std_function.h:706

    #19 0x7f6431e67b33 in execute /home/vagrant/MaxScale/maxutils/maxbase/src/worker.cc:492

    #20 0x7f6431e692d3 in maxbase::Worker::handle_message(maxbase::MessageQueue&, maxbase::MessageQueueMessage const&) /home/vagrant/MaxScale/maxutils/maxbase/src/worker.cc:654

    #21 0x7f6431e7852f in maxbase::MessageQueue::handle_poll_events(maxbase::Worker*, unsigned int) /home/vagrant/MaxScale/maxutils/maxbase/src/messagequeue.cc:307

    #22 0x7f6431e787ea in maxbase::MessageQueue::poll_handler(MXB_POLL_DATA*, MXB_WORKER*, unsigned int) /home/vagrant/MaxScale/maxutils/maxbase/src/messagequeue.cc:342

    #23 0x7f6431e6a5e3 in maxbase::Worker::poll_waitevents() /home/vagrant/MaxScale/maxutils/maxbase/src/worker.cc:863

    #24 0x7f6431e68366 in maxbase::Worker::run(maxbase::Semaphore*) /home/vagrant/MaxScale/maxutils/maxbase/src/worker.cc:558

    #25 0x41f81c in maxbase::Worker::run() (/usr/bin/maxscale+0x41f81c)

    #26 0x419027 in main /home/vagrant/MaxScale/server/core/gateway.cc:2212

    #27 0x7f642e98e554 in __libc_start_main (/lib64/libc.so.6+0x22554)

​

previously allocated by thread T0 here:

    #0 0x7f643255a1a8 in operator new(unsigned long) (/lib64/libasan.so.4+0xe01a8)

    #1 0x7f6431cc1639 in Service* Service::create<json_t*, std::set<std::string, std::less<std::string>, std::allocator<std::string> > >(std::string const&, json_t*, std::set<std::string, std::less<std::string>, std::allocator<std::string> >) /home/vagrant/MaxScale/server/core/service.cc:309

    #2 0x7f6431ca0ae7 in Service::create(char const*, json_t*) /home/vagrant/MaxScale/server/core/service.cc:405

    #3 0x7f6431aaadb9 in runtime_create_service_from_json(json_t*) /home/vagrant/MaxScale/server/core/config_runtime.cc:2130

    #4 0x7f6431bf47bd in cb_create_service /home/vagrant/MaxScale/server/core/resource.cc:419

    #5 0x7f6431bf159a in Resource::call(HttpRequest const&) const /home/vagrant/MaxScale/server/core/resource.cc:151

    #6 0x7f6431c04c05 in process_request /home/vagrant/MaxScale/server/core/resource.cc:1541

    #7 0x7f6431c0805a in handle_request /home/vagrant/MaxScale/server/core/resource.cc:1736

    #8 0x7f6431c08dce in operator() /home/vagrant/MaxScale/server/core/resource.cc:1803

    #9 0x7f6431c099ee in _M_invoke /opt/rh/devtoolset-7/root/usr/include/c++/7/bits/std_function.h:316

    #10 0x7f6431c4eb6d in std::function<void ()>::operator()() const /opt/rh/devtoolset-7/root/usr/include/c++/7/bits/std_function.h:706

    #11 0x7f6431e67b33 in execute /home/vagrant/MaxScale/maxutils/maxbase/src/worker.cc:492

    #12 0x7f6431e692d3 in maxbase::Worker::handle_message(maxbase::MessageQueue&, maxbase::MessageQueueMessage const&) /home/vagrant/MaxScale/maxutils/maxbase/src/worker.cc:654

    #13 0x7f6431e7852f in maxbase::MessageQueue::handle_poll_events(maxbase::Worker*, unsigned int) /home/vagrant/MaxScale/maxutils/maxbase/src/messagequeue.cc:307

    #14 0x7f6431e787ea in maxbase::MessageQueue::poll_handler(MXB_POLL_DATA*, MXB_WORKER*, unsigned int) /home/vagrant/MaxScale/maxutils/maxbase/src/messagequeue.cc:342

    #15 0x7f6431e6a5e3 in maxbase::Worker::poll_waitevents() /home/vagrant/MaxScale/maxutils/maxbase/src/worker.cc:863

    #16 0x7f6431e68366 in maxbase::Worker::run(maxbase::Semaphore*) /home/vagrant/MaxScale/maxutils/maxbase/src/worker.cc:558

    #17 0x41f81c in maxbase::Worker::run() (/usr/bin/maxscale+0x41f81c)

    #18 0x419027 in main /home/vagrant/MaxScale/server/core/gateway.cc:2212

    #19 0x7f642e98e554 in __libc_start_main (/lib64/libc.so.6+0x22554)

​

Thread T9 created by T0 here:

    #0 0x7f64324b1a7f in pthread_create (/lib64/libasan.so.4+0x37a7f)

    #1 0x7f6431f3be44 in std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) (/usr/lib64/maxscale/libmaxscale-common.so.1.0.0+0x9f6e44)

    #2 0x7f6431e689ee in maxbase::Worker::start() /home/vagrant/MaxScale/maxutils/maxbase/src/worker.cc:585

    #3 0x7f6431c3c7a0 in maxscale::RoutingWorker::start_workers() /home/vagrant/MaxScale/server/core/routingworker.cc:374

    #4 0x418eb4 in main /home/vagrant/MaxScale/server/core/gateway.cc:2203

    #5 0x7f642e98e554 in __libc_start_main (/lib64/libc.so.6+0x22554)