Details
Minor Description
We have seen regularly several times that MariaDB crashed with a signal 6 (system abort) during the McAfee database vulnerability scan. We think it might be caused by the aborted connections during the scan.
In a recent scan, we enabled core dump and collected the core files. The backtrace seems to be pointing to `gcs_gcomm.cpp`. See partial backtrace below, and the full-text files are attached.
The system abort crash started from the below records in the system log file.
...
|
Nov 5 11:53:11 hostname mariadbd[1766]: terminate called after throwing an instance of 'boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<std::system_error> >'
|
Nov 5 11:53:11 hostname mariadbd[1766]: what(): remote_endpoint: Transport endpoint is not connected
|
Nov 5 11:53:11 hostname mariadbd[1766]: 211105 11:53:11 [ERROR] mysqld got signal 6 ;
|
...
|
We noticed that the system abort triggered a segment fault. The segment fault is a known old issue, and we are mostly interested in the system abort. If you need more information, just let me know.
Partial backtrace:
...
|
#5 0x00007fcfc9d73a78 in __GI_abort () at abort.c:90
|
save_stage = 2
|
act = {__sigaction_handler = {sa_handler = 0x0, sa_sigaction = 0x0}, sa_mask = {__val = {0 <repeats 15 times>, 140530425020864}}, sa_flags = -1475798056, sa_restorer = 0x7fcfca104868 <stderr>}
|
sigs = {__val = {32, 0 <repeats 15 times>}}
|
#6 0x00007fcfca46ca95 in __gnu_cxx::__verbose_terminate_handler () at ../../../../libstdc++-v3/libsupc++/vterminate.cc:95
|
terminating = true
|
t = <optimized out>
|
#7 0x00007fcfca46aa06 in __cxxabiv1::__terminate (handler=<optimized out>) at ../../../../libstdc++-v3/libsupc++/eh_terminate.cc:38
|
No locals.
|
#8 0x00007fcfca46aa33 in std::terminate () at ../../../../libstdc++-v3/libsupc++/eh_terminate.cc:48
|
No locals.
|
#9 0x00007fcfca46ac53 in __cxxabiv1::__cxa_throw (obj=0x7fcfa8065ea0, tinfo=0x7fcfc6fc3e00 <typeinfo for boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<std::system_error> >>, dest=0x7fcfc6d3f980 <boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<std::system_error> >::~clone_impl()>) at ../../../../libstdc++-v3/libsupc++/eh_throw.cc:87
|
globals = <optimized out>
|
#10 0x00007fcfc6d41ab4 in boost::throw_exception<std::system_error> (e=...) at /usr/include/boost/throw_exception.hpp:67
|
No locals.
|
#11 0x00007fcfc6d41b5c in asio::detail::do_throw_error (err=..., location=location@entry=0x7fcfc6d7c666 "remote_endpoint") at /home/buildbot/buildbot/build/asio/asio/detail/impl/throw_error.ipp:49
|
e = {<std::runtime_error> = {<std::exception> = {_vptr.exception = 0x7fcfca6fbaf0 <vtable for std::system_error+16>}, _M_msg = "remote_endpoint: Transport endpoint is not connected"}, _M_code = {_M_value = 107, _M_cat = 0x7fcfc6fc8d38 <asio::system_category()::instance>}}
|
#12 0x00007fcfc6d4dcbf in throw_error (location=<optimized out>, err=...) at /home/buildbot/buildbot/build/asio/asio/detail/throw_error.hpp:41
|
No locals.
|
#13 remote_endpoint (this=0x7fcfa808f488) at /home/buildbot/buildbot/build/asio/asio/basic_socket.hpp:1425
|
ec = {_M_value = 107, _M_cat = 0x7fcfc6fc8d38 <asio::system_category()::instance>}
|
#14 gu::AsioStreamReact::assign_addresses (this=0x7fcfa808f468) at /home/buildbot/buildbot/build/galerautils/src/gu_asio_stream_react.cpp:552
|
No locals.
|
#15 0x00007fcfc6d502e2 in gu::AsioAcceptorReact::accept_handler (this=0x56335462aaa8, socket=std::shared_ptr (count 1, weak 1) 0x7fcfa808f468, handler=std::shared_ptr (count 2, weak 1) 0x56335462abd8, ec=...) at /home/buildbot/buildbot/build/galerautils/src/gu_asio_stream_react.cpp:879
|
result = <optimized out>
|
__FUNCTION__ = "accept_handler"
|
#16 0x00007fcfc6d54533 in call<std::shared_ptr<gu::AsioAcceptorReact>, std::shared_ptr<gu::AsioStreamReact> const, std::shared_ptr<gu::AsioAcceptorHandler> const, std::error_code const> (b3=..., b2=std::shared_ptr (count 2, weak 1) 0x56335462abd8, b1=std::shared_ptr (count 1, weak 1) 0x7fcfa808f468, u=std::shared_ptr (count 2, weak 1) 0x56335462aaa8, this=0x7fcfc4aa38f0) at /usr/include/boost/bind/mem_fn_template.hpp:384
|
No locals.
|
#17 operator()<std::shared_ptr<gu::AsioAcceptorReact> > (a3=..., a2=std::shared_ptr (count 2, weak 1) 0x56335462abd8, a1=std::shared_ptr (count 1, weak 1) 0x7fcfa808f468, u=std::shared_ptr (count 2, weak 1) 0x56335462aaa8, this=0x7fcfc4aa38f0) at /usr/include/boost/bind/mem_fn_template.hpp:399
|
p = 0x0
|
#18 operator()<boost::_mfi::mf3<void, gu::AsioAcceptorReact, const std::shared_ptr<gu::AsioStreamReact>&, const std::shared_ptr<gu::AsioAcceptorHandler>&, const std::error_code&>, boost::_bi::list1<const std::error_code&> > (a=<synthetic pointer>, f=..., this=0x7fcfc4aa3900) at /usr/include/boost/bind/bind.hpp:457
|
No locals.
|
#19 operator()<std::error_code> (a1=..., this=0x7fcfc4aa38f0) at /usr/include/boost/bind/bind_template.hpp:47
|
No locals.
|
#20 operator() (this=0x7fcfc4aa38f0) at /home/buildbot/buildbot/build/asio/asio/detail/bind_handler.hpp:46
|
No locals.
|
#21 asio_handler_invoke<asio::detail::binder1<boost::_bi::bind_t<void, boost::_mfi::mf3<void, gu::AsioAcceptorReact, std::shared_ptr<gu::AsioStreamReact> const&, std::shared_ptr<gu::AsioAcceptorHandler> const&, std::error_code const&>, boost::_bi::list4<boost::_bi::value<std::shared_ptr<gu::AsioAcceptorReact> >, boost::_bi::value<std::shared_ptr<gu::AsioStreamReact> >, boost::_bi::value<std::shared_ptr<gu::AsioAcceptorHandler> >, boost::arg<1> (*)()> >, std::error_code> > (function=...) at /home/buildbot/buildbot/build/asio/asio/handler_invoke_hook.hpp:68
|
No locals.
|
#22 invoke<asio::detail::binder1<boost::_bi::bind_t<void, boost::_mfi::mf3<void, gu::AsioAcceptorReact, std::shared_ptr<gu::AsioStreamReact> const&, std::shared_ptr<gu::AsioAcceptorHandler> const&, std::error_code const&>, boost::_bi::list4<boost::_bi::value<std::shared_ptr<gu::AsioAcceptorReact> >, boost::_bi::value<std::shared_ptr<gu::AsioStreamReact> >, boost::_bi::value<std::shared_ptr<gu::AsioAcceptorHandler> >, boost::arg<1> (*)()> >, std::error_code>, boost::_bi::bind_t<void, boost::_mfi::mf3<void, gu::AsioAcceptorReact, std::shared_ptr<gu::AsioStreamReact> const&, std::shared_ptr<gu::AsioAcceptorHandler> const&, std::error_code const&>, boost::_bi::list4<boost::_bi::value<std::shared_ptr<gu::AsioAcceptorReact> >, boost::_bi::value<std::shared_ptr<gu::AsioStreamReact> >, boost::_bi::value<std::shared_ptr<gu::AsioAcceptorHandler> >, boost::arg<1> (*)()> > > (context=..., function=...) at /home/buildbot/buildbot/build/asio/asio/detail/handler_invoke_helpers.hpp:37
|
No locals.
|
#23 asio::detail::reactive_socket_accept_op<asio::basic_socket<asio::ip::tcp, asio::stream_socket_service<asio::ip::tcp> >, asio::ip::tcp, boost::_bi::bind_t<void, boost::_mfi::mf3<void, gu::AsioAcceptorReact, std::shared_ptr<gu::AsioStreamReact> const&, std::shared_ptr<gu::AsioAcceptorHandler> const&, std::error_code const&>, boost::_bi::list4<boost::_bi::value<std::shared_ptr<gu::AsioAcceptorReact> >, boost::_bi::value<std::shared_ptr<gu::AsioStreamReact> >, boost::_bi::value<std::shared_ptr<gu::AsioAcceptorHandler> >, boost::arg<1> (*)()> > >::do_complete (owner=0x563354613140, base=0x7fcfa804c4c0) at /home/buildbot/buildbot/build/asio/asio/detail/reactive_socket_accept_op.hpp:122
|
o = 0x7fcfa804c4c0
|
p = {h = 0x7fcfc4aa38f0, v = 0x0, p = 0x0}
|
handler = {handler_ = {f_ = {f_ = (void (gu::AsioAcceptorReact::*)(gu::AsioAcceptorReact * const, const std::shared_ptr<gu::AsioStreamReact> &, const std::shared_ptr<gu::AsioAcceptorHandler> &, const std::error_code &)) 0x7fcfc6d50220 <gu::AsioAcceptorReact::accept_handler(std::shared_ptr<gu::AsioStreamReact> const&, std::shared_ptr<gu::AsioAcceptorHandler> const&, std::error_code const&)>}, l_ = {<boost::_bi::storage4<boost::_bi::value<std::shared_ptr<gu::AsioAcceptorReact> >, boost::_bi::value<std::shared_ptr<gu::AsioStreamReact> >, boost::_bi::value<std::shared_ptr<gu::AsioAcceptorHandler> >, boost::arg<1> (*)()>> = {<boost::_bi::storage3<boost::_bi::value<std::shared_ptr<gu::AsioAcceptorReact> >, boost::_bi::value<std::shared_ptr<gu::AsioStreamReact> >, boost::_bi::value<std::shared_ptr<gu::AsioAcceptorHandler> > >> = {<boost::_bi::storage2<boost::_bi::value<std::shared_ptr<gu::AsioAcceptorReact> >, boost::_bi::value<std::shared_ptr<gu::AsioStreamReact> > >> = {<boost::_bi::storage1<boost::_bi::value<std::shared_ptr<gu::AsioAcceptorReact> > >> = {a1_ = {t_ = std::shared_ptr (count 2, weak 1) 0x56335462aaa8}}, a2_ = {t_ = std::shared_ptr (count 1, weak 1) 0x7fcfa808f468}}, a3_ = {t_ = std::shared_ptr (count 2, weak 1) 0x56335462abd8}}, <No data fields>}, <No data fields>}}, arg1_ = {_M_value = 0, _M_cat = 0x7fcfca7133b8 <(anonymous namespace)::system_category_instance>}}
|
#24 0x00007fcfc6d40eeb in complete (bytes_transferred=<optimized out>, ec=..., owner=..., this=<optimized out>) at /home/buildbot/buildbot/build/asio/asio/detail/task_io_service_operation.hpp:37
|
No locals.
|
#25 asio::detail::task_io_service::do_run_one (this=this@entry=0x563354613140, lock=..., this_thread=..., ec=...) at /home/buildbot/buildbot/build/asio/asio/detail/impl/task_io_service.ipp:371
|
task_result = <optimized out>
|
on_exit = {task_io_service_ = 0x563354613140, lock_ = 0x7fcfc4aa39f0, this_thread_ = 0x7fcfc4aa3a20}
|
o = <optimized out>
|
more_handlers = true
|
#26 0x00007fcfc6d3cd81 in run (ec=..., this=0x563354613140) at /home/buildbot/buildbot/build/asio/asio/detail/impl/task_io_service.ipp:148
|
this_thread = {<asio::detail::thread_info_base> = {<asio::detail::noncopyable> = {<No data fields>}, reusable_memory_ = 0x7fcfa804c4c0}, private_op_queue = {<asio::detail::noncopyable> = {<No data fields>}, front_ = 0x0, back_ = 0x0}, private_outstanding_work = 0}
|
ctx = {<asio::detail::noncopyable> = {<No data fields>}, key_ = 0x563354613140, value_ = 0x7fcfc4aa3a20, next_ = 0x0}
|
lock = {<asio::detail::noncopyable> = {<No data fields>}, mutex_ = @0x563354613170, locked_ = false}
|
n = <optimized out>
|
#27 run (this=<optimized out>) at /home/buildbot/buildbot/build/asio/asio/impl/io_service.ipp:58
|
ec = {_M_value = 0, _M_cat = 0x7fcfca7133b8 <(anonymous namespace)::system_category_instance>}
|
#28 gu::AsioIoService::run (this=this@entry=0x563354612e00) at /home/buildbot/buildbot/build/galerautils/src/gu_asio.cpp:701
|
No locals.
|
#29 0x00007fcfc6c69599 in gcomm::AsioProtonet::event_loop (this=0x563354612d60, period=...) at /home/buildbot/buildbot/build/gcomm/src/asio_protonet.cpp:103
|
No locals.
|
#30 0x00007fcfc6c505d7 in GCommConn::run (this=0x563354616700) at /home/buildbot/buildbot/build/gcs/src/gcs_gcomm.cpp:435
|
__FUNCTION__ = "run"
|
#31 0x00007fcfc6c50989 in run_fn (arg=<optimized out>) at /home/buildbot/buildbot/build/gcs/src/gcs_gcomm.cpp:304
|
No locals.
|
#32 0x00007fcfca91fea5 in start_thread (arg=0x7fcfc4aa4700) at pthread_create.c:307
|
__res = <optimized out>
|
pd = 0x7fcfc4aa4700
|
now = <optimized out>
|
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140530334451456, 2370313829753134567, 0, 8392704, 0, 140530334451456, -2343417123144884761, -2343385845144511001}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
|
not_first_call = <optimized out>
|
pagesize_m1 = <optimized out>
|
sp = <optimized out>
|
freesize = <optimized out>
|
#33 0x00007fcfc9e3a9fd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
No locals.
|
The versions of MariaDB components:
[root@hostname username]# yum list installed | grep -i "mariadb"
|
Repodata is over 2 weeks old. Install yum-cron? Or run: yum makecache fast
|
MariaDB-backup.x86_64 10.5.12-1.el7.centos @mariadb
|
MariaDB-client.x86_64 10.5.12-1.el7.centos @mariadb
|
MariaDB-common.x86_64 10.5.12-1.el7.centos @mariadb
|
MariaDB-compat.x86_64 10.5.12-1.el7.centos @mariadb
|
MariaDB-server.x86_64 10.5.12-1.el7.centos @mariadb
|
MariaDB-server-debuginfo.x86_64 10.5.12-1.el7.centos @mariadb
|
galera-4.x86_64 26.4.9-1.el7.centos @mariadb
|
[root@hostname username]#
|
Attachments
Issue Links
- relates to
-
MDEV-25068 Node crashes with Transport endpoint is not connected mysqld got signal 6 ;
-
- Closed
-