Details
-
Bug
-
Status: Closed (View Workflow)
-
Major
-
Resolution: Fixed
-
10.2.17
-
None
Description
Happening at a customer of ours.
This issue repeats itself every saturday at approx the same time - looks like something in the workload is triggering it.
The server locks up and does not process any more data (connections are accepted but not handled). After the timeout set in innodb_fatal_semaphore_wait_threshold the InnoDB kernel commits suicide.
I was able to capture a stack trace right before the fatal wait was reached.
367 __lll_lock_wait,__GI___pthread_mutex_lock,do_handle_one_connection,handle_one_connection,start_thread,clone
|
347 poll,vio_io_wait,vio_socket_io_wait,vio_read,::??,my_net_read_packet_reallen,my_net_read_packet,do_command,do_handle_one_connection,handle_one_connection,start_thr
|
ead,clone
|
251 pthread_cond_wait,::??,::??,::??,::??,::??,handler::ha_write_row,write_record,mysql_insert,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_ha
|
ndle_one_connection,handle_one_connection,start_thread,clone
|
157 nanosleep,::??,::??,::??,handler::ha_index_read_map,::??,sub_select,JOIN::exec_inner,JOIN::exec,mysql_select,handle_select,::??,mysql_execute_command,mysql_parse,:
|
:??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
|
123 nanosleep,::??,::??,::??,handler::index_read_idx_map,handler::ha_index_read_idx_map,::??,::??,::??,JOIN::optimize_inner,JOIN::optimize,mysql_select,handle_select,:
|
:??,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
|
67 pthread_cond_wait,::??,::??,::??,::??,::??,handler::ha_index_read_map,handler::read_range_first,handler::multi_range_read_next,Mrr_simple_index_reader::get_next,Ds
|
Mrr_impl::dsmrr_next,QUICK_RANGE_SELECT::get_next,::??,mysql_update,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_
|
connection,start_thread,clone
|
52 pthread_cond_wait,::??,::??,::??,::??,::??,::??,handler::ha_open,open_table_from_share,open_table,open_tables,open_and_lock_tables,::??,mysql_execute_command,mysql
|
_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
|
30 nanosleep,::??,::??,::??,handler::ha_index_read_map,handler::read_range_first,handler::multi_range_read_next,Mrr_simple_index_reader::get_next,DsMrr_impl::dsmrr_ne
|
xt,QUICK_RANGE_SELECT::get_next,QUICK_ROR_UNION_SELECT::reset,filesort,create_sort_index,st_join_table::sort_table,join_init_read_record,sub_select,JOIN::exec_inner,JOIN::
|
exec,mysql_select,handle_select,::??,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
|
18 nanosleep,::??,::??,::??,handler::ha_write_row,write_record,mysql_insert,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connectio
|
n,handle_one_connection,start_thread,clone
|
17 __lll_lock_wait,__GI___pthread_mutex_lock,unlink_thd,one_thread_per_connection_end,do_handle_one_connection,handle_one_connection,start_thread,clone
|
15 pthread_cond_wait,::??,::??,::??,::??,::??,::??,handler::ha_open,open_table_from_share,open_table,open_tables,open_and_lock_tables,::??,mysql_execute_command,sp_in
|
str_stmt::exec_core,sp_lex_keeper::reset_lex_and_exec_core,sp_instr_stmt::execute,sp_head::execute,sp_head::execute_procedure,::??,mysql_execute_command,mysql_parse,::??,d
|
ispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
|
13 pthread_cond_wait,::??,::??,::??,::??,::??,::??,handler::ha_open,open_table_from_share,open_table,open_tables,mysql_update,mysql_execute_command,mysql_parse,::??,d
|
ispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
|
10 libaio::??,::??,::??,::??,::??,::??,start_thread,clone
|
9 pthread_cond_wait,::??,::??,::??,::??,::??,::??,handler::index_read_idx_map,handler::ha_index_read_idx_map,::??,::??,::??,JOIN::optimize_inner,JOIN::optimize,mysql
|
_select,handle_select,::??,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
|
9 nanosleep,::??,::??,::??,::??,handler::ha_rnd_next,rr_sequential,sub_select,JOIN::exec_inner,JOIN::exec,mysql_select,handle_select,::??,mysql_execute_command,mysql
|
_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
|
8 pthread_cond_wait,::??,::??,start_thread,clone
|
7 pthread_cond_wait,::??,::??,::??,::??,::??,::??,handler::ha_open,open_table_from_share,open_table,open_tables,open_and_lock_tables,mysql_insert,mysql_execute_comma
|
nd,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
|
6 pthread_cond_wait,::??,::??,::??,::??,::??,::??,handler::ha_index_read_map,::??,sub_select,JOIN::exec_inner,JOIN::exec,mysql_select,handle_select,::??,mysql_execut
|
e_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
|
4 pthread_cond_wait,fifo_lock_get,gu_fifo_get_head,gcs_recv,galera::Gcs::recv,galera::GcsActionSource::process,galera::ReplicatorSMM::async_recv,galera_recv,::??,sta
|
rt_wsrep_THD,start_thread,clone
|
4 nanosleep,::??,::??,::??,::??,handler::ha_rnd_next,filesort,create_sort_index,st_join_table::sort_table,join_init_read_record,sub_select,JOIN::exec_inner,JOIN::exe
|
c,mysql_select,handle_select,::??,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
|
3 pthread_cond_wait,::??,::??,::??,::??,open_tables,open_and_lock_tables,::??,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connec
|
tion,handle_one_connection,start_thread,clone
|
3 pthread_cond_wait,::??,::??,::??,::??,::??,handler::ha_update_row,mysql_update,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_con
|
nection,handle_one_connection,start_thread,clone
|
2 pthread_cond_wait,::??,::??,::??,start_thread,clone
|
2 pthread_cond_timedwait,::??,::??,::??,start_thread,clone
|
2 nanosleep,::??,::??,start_thread,clone
|
2 nanosleep,::??,::??,::??,handler::ha_index_read_map,handler::read_range_first,handler::multi_range_read_next,Mrr_simple_index_reader::get_next,DsMrr_impl::dsmrr_ne
|
xt,QUICK_RANGE_SELECT::get_next,::??,mysql_update,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_t
|
hread,clone
|
2 nanosleep,::??,::??,::??,handler::ha_index_read_map,handler::read_range_first,handler::multi_range_read_next,Mrr_simple_index_reader::get_next,DsMrr_impl::dsmrr_ne
|
xt,QUICK_RANGE_SELECT::get_next,filesort,create_sort_index,st_join_table::sort_table,join_init_read_record,sub_select,JOIN::exec_inner,JOIN::exec,mysql_select,handle_selec
|
t,::??,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
|
1 pthread_cond_wait,::??,start_wsrep_THD,start_thread,clone
|
1 pthread_cond_wait,::??,start_thread,clone
|
1 pthread_cond_wait,::??,::??,::??,::??,::??,start_thread,clone
|
1 pthread_cond_wait,::??,::??,::??,::??,::??,plugin_foreach_with_mask,ha_kill_query,THD::awake,kill_one_thread,::??,mysql_execute_command,mysql_parse,::??,dispatch_c
|
ommand,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
|
1 pthread_cond_wait,::??,::??,::??,::??,::??,::??,ha_show_status,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_o
|
ne_connection,start_thread,clone
|
1 pthread_cond_wait,handle_slave_background,start_thread,clone
|
1 pthread_cond_wait,::??,::??,::??,::??,::??,::??,::??,handler::ha_rnd_next,filesort,create_sort_index,st_join_table::sort_table,join_init_read_record,sub_select,JOI
|
N::exec_inner,JOIN::exec,mysql_select,handle_select,::??,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,
|
start_thread,clone
|
1 pthread_cond_wait,gu::Lock::wait,RecvBuf::front,gcomm_recv,core_msg_recv,gcs_core_recv,gcs_recv_thread,start_thread,clone
|
1 pthread_cond_wait,gu::Lock::wait,galera::ServiceThd::thd_func,start_thread,clone
|
1 pthread_cond_wait,::??,::??,::??,::??,::??,fill_status,get_schema_tables_result,JOIN::exec_inner,JOIN::exec,mysql_select,handle_select,::??,mysql_execute_command,m
|
ysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
|
1 pthread_cond_wait,binlog_background_thread,start_thread,clone
|
1 pthread_cond_timedwait,::??,start_thread,clone
|
1 pthread_cond_timedwait,::??,::??,start_thread,clone
|
1 pthread_cond_timedwait,::??,::??,::??,::??,start_thread,clone
|
1 nanosleep,::??,::??,::??,start_thread,clone
|
1 nanosleep,::??,::??,::??,handler::ha_index_next_same,::??,sub_select,::??,sub_select,JOIN::exec_inner,JOIN::exec,mysql_select,handle_select,::??,mysql_execute_comm
|
and,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
|
1 nanosleep,::??,::??,::??,::??,handler::ha_index_first,::??,sub_select,JOIN::exec_inner,JOIN::exec,mysql_select,handle_select,::??,mysql_execute_command,mysql_parse
|
,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
|
1 __lll_lock_wait,__GI___pthread_mutex_lock,::??,THD::~THD,THD::~THD,CONNECT::close_with_error,handle_connections_sockets,mysqld_main,__libc_start_main,_start
|
1 __lll_lock_wait,__GI___pthread_mutex_lock,::??,thd_query_safe,::??,::??,::??,::??,get_schema_tables_result,JOIN::exec_inner,JOIN::exec,mysql_select,handle_select,:
|
:??,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
|
1 __lll_lock_wait,__GI___pthread_mutex_lock,one_thread_per_connection_end,do_handle_one_connection,handle_one_connection,start_thread,clone
|
1 __lll_lock_wait,__GI___pthread_mutex_lock,::??,mysqld_list_processes,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,ha
|
ndle_one_connection,start_thread,clone
|
1 __lll_lock_wait,__GI___pthread_mutex_lock,::??,fill_status,get_schema_tables_result,JOIN::exec_inner,JOIN::exec,st_select_lex_unit::exec,mysql_union,handle_select,
|
::??,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
|
1 __lll_lock_wait,__GI___pthread_mutex_lock,::??,fill_status,get_schema_tables_result,JOIN::exec_inner,JOIN::exec,mysql_select,handle_select,::??,mysql_execute_comma
|
nd,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
|
1 __lll_lock_wait,__GI___pthread_mutex_lock,::??,calc_sum_of_all_status,fill_status,get_schema_tables_result,JOIN::exec_inner,JOIN::exec,mysql_select,handle_select,:
|
:??,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
|
1 epoll_wait,asio::detail::epoll_reactor::run(epoll_reactor.ipp:391),asio::detail::task_io_service::do_run_one(task_io_service.ipp:355),asio::detail::task_io_service
|
::run(task_io_service.ipp:355),asio::io_service::run(task_io_service.ipp:355),gcomm::AsioProtonet::event_loop(task_io_service.ipp:355),GCommConn::run,GCommConn::run_fn,sta
|
rt_thread,clone
|
1 do_sigwait,__sigwait,signal_hand,start_thread,clone
|
|
Attachments
Issue Links
- relates to
-
MDEV-17190 MariaDB Server deadlock
- Closed
-
MDEV-18464 Port kill_one_trx fixes from 10.4 to 10.1
- Closed