Uploaded image for project: 'MariaDB Server'
  1. MariaDB Server
  2. MDEV-17063

server locks up and is killed after innodb_fatal_semaphore_wait_threshold is reached

    XMLWordPrintable

Details

    Description

      Happening at a customer of ours.
      This issue repeats itself every saturday at approx the same time - looks like something in the workload is triggering it.
      The server locks up and does not process any more data (connections are accepted but not handled). After the timeout set in innodb_fatal_semaphore_wait_threshold the InnoDB kernel commits suicide.

      I was able to capture a stack trace right before the fatal wait was reached.

          367 __lll_lock_wait,__GI___pthread_mutex_lock,do_handle_one_connection,handle_one_connection,start_thread,clone
          347 poll,vio_io_wait,vio_socket_io_wait,vio_read,::??,my_net_read_packet_reallen,my_net_read_packet,do_command,do_handle_one_connection,handle_one_connection,start_thr
      ead,clone
          251 pthread_cond_wait,::??,::??,::??,::??,::??,handler::ha_write_row,write_record,mysql_insert,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_ha
      ndle_one_connection,handle_one_connection,start_thread,clone
          157 nanosleep,::??,::??,::??,handler::ha_index_read_map,::??,sub_select,JOIN::exec_inner,JOIN::exec,mysql_select,handle_select,::??,mysql_execute_command,mysql_parse,:
      :??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
          123 nanosleep,::??,::??,::??,handler::index_read_idx_map,handler::ha_index_read_idx_map,::??,::??,::??,JOIN::optimize_inner,JOIN::optimize,mysql_select,handle_select,:
      :??,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
           67 pthread_cond_wait,::??,::??,::??,::??,::??,handler::ha_index_read_map,handler::read_range_first,handler::multi_range_read_next,Mrr_simple_index_reader::get_next,Ds
      Mrr_impl::dsmrr_next,QUICK_RANGE_SELECT::get_next,::??,mysql_update,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_
      connection,start_thread,clone
           52 pthread_cond_wait,::??,::??,::??,::??,::??,::??,handler::ha_open,open_table_from_share,open_table,open_tables,open_and_lock_tables,::??,mysql_execute_command,mysql
      _parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
           30 nanosleep,::??,::??,::??,handler::ha_index_read_map,handler::read_range_first,handler::multi_range_read_next,Mrr_simple_index_reader::get_next,DsMrr_impl::dsmrr_ne
      xt,QUICK_RANGE_SELECT::get_next,QUICK_ROR_UNION_SELECT::reset,filesort,create_sort_index,st_join_table::sort_table,join_init_read_record,sub_select,JOIN::exec_inner,JOIN::
      exec,mysql_select,handle_select,::??,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
           18 nanosleep,::??,::??,::??,handler::ha_write_row,write_record,mysql_insert,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connectio
      n,handle_one_connection,start_thread,clone
           17 __lll_lock_wait,__GI___pthread_mutex_lock,unlink_thd,one_thread_per_connection_end,do_handle_one_connection,handle_one_connection,start_thread,clone
           15 pthread_cond_wait,::??,::??,::??,::??,::??,::??,handler::ha_open,open_table_from_share,open_table,open_tables,open_and_lock_tables,::??,mysql_execute_command,sp_in
      str_stmt::exec_core,sp_lex_keeper::reset_lex_and_exec_core,sp_instr_stmt::execute,sp_head::execute,sp_head::execute_procedure,::??,mysql_execute_command,mysql_parse,::??,d
      ispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
           13 pthread_cond_wait,::??,::??,::??,::??,::??,::??,handler::ha_open,open_table_from_share,open_table,open_tables,mysql_update,mysql_execute_command,mysql_parse,::??,d
      ispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
           10 libaio::??,::??,::??,::??,::??,::??,start_thread,clone
            9 pthread_cond_wait,::??,::??,::??,::??,::??,::??,handler::index_read_idx_map,handler::ha_index_read_idx_map,::??,::??,::??,JOIN::optimize_inner,JOIN::optimize,mysql
      _select,handle_select,::??,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
            9 nanosleep,::??,::??,::??,::??,handler::ha_rnd_next,rr_sequential,sub_select,JOIN::exec_inner,JOIN::exec,mysql_select,handle_select,::??,mysql_execute_command,mysql
      _parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
            8 pthread_cond_wait,::??,::??,start_thread,clone
            7 pthread_cond_wait,::??,::??,::??,::??,::??,::??,handler::ha_open,open_table_from_share,open_table,open_tables,open_and_lock_tables,mysql_insert,mysql_execute_comma
      nd,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
            6 pthread_cond_wait,::??,::??,::??,::??,::??,::??,handler::ha_index_read_map,::??,sub_select,JOIN::exec_inner,JOIN::exec,mysql_select,handle_select,::??,mysql_execut
      e_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
            4 pthread_cond_wait,fifo_lock_get,gu_fifo_get_head,gcs_recv,galera::Gcs::recv,galera::GcsActionSource::process,galera::ReplicatorSMM::async_recv,galera_recv,::??,sta
      rt_wsrep_THD,start_thread,clone
            4 nanosleep,::??,::??,::??,::??,handler::ha_rnd_next,filesort,create_sort_index,st_join_table::sort_table,join_init_read_record,sub_select,JOIN::exec_inner,JOIN::exe
      c,mysql_select,handle_select,::??,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
            3 pthread_cond_wait,::??,::??,::??,::??,open_tables,open_and_lock_tables,::??,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connec
      tion,handle_one_connection,start_thread,clone
            3 pthread_cond_wait,::??,::??,::??,::??,::??,handler::ha_update_row,mysql_update,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_con
      nection,handle_one_connection,start_thread,clone
            2 pthread_cond_wait,::??,::??,::??,start_thread,clone
            2 pthread_cond_timedwait,::??,::??,::??,start_thread,clone
            2 nanosleep,::??,::??,start_thread,clone
            2 nanosleep,::??,::??,::??,handler::ha_index_read_map,handler::read_range_first,handler::multi_range_read_next,Mrr_simple_index_reader::get_next,DsMrr_impl::dsmrr_ne
      xt,QUICK_RANGE_SELECT::get_next,::??,mysql_update,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_t
      hread,clone
            2 nanosleep,::??,::??,::??,handler::ha_index_read_map,handler::read_range_first,handler::multi_range_read_next,Mrr_simple_index_reader::get_next,DsMrr_impl::dsmrr_ne
      xt,QUICK_RANGE_SELECT::get_next,filesort,create_sort_index,st_join_table::sort_table,join_init_read_record,sub_select,JOIN::exec_inner,JOIN::exec,mysql_select,handle_selec
      t,::??,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
            1 pthread_cond_wait,::??,start_wsrep_THD,start_thread,clone
            1 pthread_cond_wait,::??,start_thread,clone
            1 pthread_cond_wait,::??,::??,::??,::??,::??,start_thread,clone
            1 pthread_cond_wait,::??,::??,::??,::??,::??,plugin_foreach_with_mask,ha_kill_query,THD::awake,kill_one_thread,::??,mysql_execute_command,mysql_parse,::??,dispatch_c
      ommand,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
            1 pthread_cond_wait,::??,::??,::??,::??,::??,::??,ha_show_status,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_o
      ne_connection,start_thread,clone
            1 pthread_cond_wait,handle_slave_background,start_thread,clone
            1 pthread_cond_wait,::??,::??,::??,::??,::??,::??,::??,handler::ha_rnd_next,filesort,create_sort_index,st_join_table::sort_table,join_init_read_record,sub_select,JOI
      N::exec_inner,JOIN::exec,mysql_select,handle_select,::??,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,
      start_thread,clone
            1 pthread_cond_wait,gu::Lock::wait,RecvBuf::front,gcomm_recv,core_msg_recv,gcs_core_recv,gcs_recv_thread,start_thread,clone
            1 pthread_cond_wait,gu::Lock::wait,galera::ServiceThd::thd_func,start_thread,clone
            1 pthread_cond_wait,::??,::??,::??,::??,::??,fill_status,get_schema_tables_result,JOIN::exec_inner,JOIN::exec,mysql_select,handle_select,::??,mysql_execute_command,m
      ysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
            1 pthread_cond_wait,binlog_background_thread,start_thread,clone
            1 pthread_cond_timedwait,::??,start_thread,clone
            1 pthread_cond_timedwait,::??,::??,start_thread,clone
            1 pthread_cond_timedwait,::??,::??,::??,::??,start_thread,clone
            1 nanosleep,::??,::??,::??,start_thread,clone
            1 nanosleep,::??,::??,::??,handler::ha_index_next_same,::??,sub_select,::??,sub_select,JOIN::exec_inner,JOIN::exec,mysql_select,handle_select,::??,mysql_execute_comm
      and,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
            1 nanosleep,::??,::??,::??,::??,handler::ha_index_first,::??,sub_select,JOIN::exec_inner,JOIN::exec,mysql_select,handle_select,::??,mysql_execute_command,mysql_parse
      ,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
            1 __lll_lock_wait,__GI___pthread_mutex_lock,::??,THD::~THD,THD::~THD,CONNECT::close_with_error,handle_connections_sockets,mysqld_main,__libc_start_main,_start
            1 __lll_lock_wait,__GI___pthread_mutex_lock,::??,thd_query_safe,::??,::??,::??,::??,get_schema_tables_result,JOIN::exec_inner,JOIN::exec,mysql_select,handle_select,:
      :??,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
            1 __lll_lock_wait,__GI___pthread_mutex_lock,one_thread_per_connection_end,do_handle_one_connection,handle_one_connection,start_thread,clone
            1 __lll_lock_wait,__GI___pthread_mutex_lock,::??,mysqld_list_processes,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,ha
      ndle_one_connection,start_thread,clone
            1 __lll_lock_wait,__GI___pthread_mutex_lock,::??,fill_status,get_schema_tables_result,JOIN::exec_inner,JOIN::exec,st_select_lex_unit::exec,mysql_union,handle_select,
      ::??,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
            1 __lll_lock_wait,__GI___pthread_mutex_lock,::??,fill_status,get_schema_tables_result,JOIN::exec_inner,JOIN::exec,mysql_select,handle_select,::??,mysql_execute_comma
      nd,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
            1 __lll_lock_wait,__GI___pthread_mutex_lock,::??,calc_sum_of_all_status,fill_status,get_schema_tables_result,JOIN::exec_inner,JOIN::exec,mysql_select,handle_select,:
      :??,mysql_execute_command,mysql_parse,::??,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,start_thread,clone
            1 epoll_wait,asio::detail::epoll_reactor::run(epoll_reactor.ipp:391),asio::detail::task_io_service::do_run_one(task_io_service.ipp:355),asio::detail::task_io_service
      ::run(task_io_service.ipp:355),asio::io_service::run(task_io_service.ipp:355),gcomm::AsioProtonet::event_loop(task_io_service.ipp:355),GCommConn::run,GCommConn::run_fn,sta
      rt_thread,clone
            1 do_sigwait,__sigwait,signal_hand,start_thread,clone
      
      

      Attachments

        1. crash.log
          407 kB
        2. sema.log.gz
          22 kB

        Issue Links

          Activity

            People

              sysprg Julius Goryavsky
              rpizzi Rick Pizzi
              Votes:
              2 Vote for this issue
              Watchers:
              10 Start watching this issue

              Dates

                Created:
                Updated:
                Resolved:

                Git Integration

                  Error rendering 'com.xiplink.jira.git.jira_git_plugin:git-issue-webpanel'. Please contact your Jira administrators.