Uploaded image for project: 'MariaDB Server'
  1. MariaDB Server
  2. MDEV-4712

Server crashes upon a race condition between SHUTDOWN and SHOW STATUS, main.shutdown fails sporadically in buildbot

    XMLWordPrintable

    Details

    • Type: Bug
    • Status: Closed (View Workflow)
    • Priority: Major
    • Resolution: Fixed
    • Affects Version/s: None
    • Fix Version/s: 10.0.4
    • Component/s: None
    • Labels:

      Description

      The new test main.shutdown fails sporadically with a crash on shutdown, see for example http://buildbot.askmonty.org/buildbot/builders/kvm-deb-debian6-x86/builds/2146/steps/test_3/logs/stdio (server error log: http://buildbot.askmonty.org/buildbot/builders/kvm-deb-debian6-x86/builds/2146/steps/test_3/logs/mysqld.1.err.1)

      Below is a slightly modified version of main.shutdown, which seems to increase the probability of hitting the failure, although it's far from deterministic.

      All threads stack trace:

      Thread 5 (Thread 0x7f34c4874700 (LWP 9507)):
      #0  0x00007f34c802a023 in select () at ../sysdeps/unix/syscall-template.S:82
      #1  0x0000000000d23405 in my_sleep (m_seconds=20000) at 10.0-base/mysys/my_sleep.c:29
      #2  0x0000000000563b8d in close_connections () at 10.0-base/sql/mysqld.cc:1510
      #3  0x000000000056423b in kill_server (sig_ptr=0x0) at 10.0-base/sql/mysqld.cc:1700
      #4  0x0000000000564271 in kill_server_thread (arg=0x7f34c514fe48) at 10.0-base/sql/mysqld.cc:1728
      #5  0x00000000009a4b74 in pfs_spawn_thread (arg=0x7f34b4001118) at 10.0-base/storage/perfschema/pfs.cc:1015
      #6  0x00007f34c8d3ce9a in start_thread (arg=0x7f34c4874700) at pthread_create.c:308
      #7  0x00007f34c8030cbd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
      #8  0x0000000000000000 in ?? ()
       
      Thread 4 (Thread 0x7f34c977c740 (LWP 9491)):
      #0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
      #1  0x0000000000d32488 in safe_cond_wait (cond=0x1553540, mp=0x15523c0, file=0xd84ab0 "10.0-base/sql/mysqld.cc", line=5222) at 10.0-base/mysys/thr_mutex.c:493
      #2  0x00000000005621d8 in inline_mysql_cond_wait (that=0x1553540, mutex=0x15523c0, src_file=0xd84ab0 "10.0-base/sql/mysqld.cc", src_line=5222) at 10.0-base/include/mysql/psi/mysql_thread.h:984
      #3  0x000000000056a3c0 in mysqld_main (argc=95, argv=0x2c4b518) at 10.0-base/sql/mysqld.cc:5222
      #4  0x0000000000561974 in main (argc=6, argv=0x7fff81da30c8) at 10.0-base/sql/main.cc:25
       
      Thread 3 (Thread 0x7f34c7d26700 (LWP 9493)):
      #0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:215
      #1  0x0000000000d32792 in safe_cond_timedwait (cond=0x1d89a00, mp=0x1d89940, abstime=0x7f34c7d25db0, file=0xec84b0 "10.0-base/storage/maria/ma_servicethread.c", line=119) at 10.0-base/mysys/thr_mutex.c:547
      #2  0x0000000000b6425a in inline_mysql_cond_timedwait (that=0x1d89a00, mutex=0x1d89940, abstime=0x7f34c7d25db0, src_file=0xec84b0 "10.0-base/storage/maria/ma_servicethread.c", src_line=119) at 10.0-base/include/mysql/psi/mysql_thread.h:1017
      #3  0x0000000000b6475a in my_service_thread_sleep (control=0x1419e90, sleep_time=29000000000) at 10.0-base/storage/maria/ma_servicethread.c:118
      #4  0x0000000000b579c1 in ma_checkpoint_background (arg=0x1e) at 10.0-base/storage/maria/ma_checkpoint.c:705
      #5  0x00000000009a4b74 in pfs_spawn_thread (arg=0x2cfc488) at 10.0-base/storage/perfschema/pfs.cc:1015
      #6  0x00007f34c8d3ce9a in start_thread (arg=0x7f34c7d26700) at pthread_create.c:308
      #7  0x00007f34c8030cbd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
      #8  0x0000000000000000 in ?? ()
       
      Thread 2 (Thread 0x7f34c5150700 (LWP 9494)):
      #0  do_sigwait (set=<optimized out>, sig=0x7f34c514fe48) at ../nptl/sysdeps/unix/sysv/linux/../../../../../sysdeps/unix/sysv/linux/sigwait.c:65
      #1  0x00007f34c8d44a79 in __sigwait (set=<optimized out>, sig=<optimized out>) at ../nptl/sysdeps/unix/sysv/linux/../../../../../sysdeps/unix/sysv/linux/sigwait.c:100
      #2  0x0000000000566ca8 in signal_hand (arg=0x0) at 10.0-base/sql/mysqld.cc:3142
      #3  0x00000000009a4b74 in pfs_spawn_thread (arg=0x2cca8f8) at 10.0-base/storage/perfschema/pfs.cc:1015
      #4  0x00007f34c8d3ce9a in start_thread (arg=0x7f34c5150700) at pthread_create.c:308
      #5  0x00007f34c8030cbd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
      #6  0x0000000000000000 in ?? ()
       
      Thread 1 (Thread 0x7f34c4906700 (LWP 9503)):
      #0  __pthread_kill (threadid=<optimized out>, signo=<optimized out>) at ../nptl/sysdeps/unix/sysv/linux/pthread_kill.c:63
      #1  0x0000000000d287ec in my_write_core (sig=11) at 10.0-base/mysys/stacktrace.c:457
      #2  0x00000000007fc15e in handle_fatal_signal (sig=11) at 10.0-base/sql/signal_handler.cc:262
      #3  <signal handler called>
      #4  0x0000000000cf6632 in my_hash_first (hash=0x3a0, key=0x7f34c4902cf0 "", length=0, current_record=0x7f34c4902c7c) at 10.0-base/mysys/hash.c:254
      #5  0x0000000000cf659f in my_hash_search (hash=0x3a0, key=0x7f34c4902cf0 "", length=0) at 10.0-base/mysys/hash.c:223
      #6  0x000000000072f12c in Master_info_index::get_master_info (this=0x0, connection_name=0x2e7c338, warning=MYSQL_ERROR::WARN_LEVEL_NOTE) at 10.0-base/sql/rpl_mi.cc:1062
      #7  0x000000000056b9ef in show_heartbeat_period (thd=0x2e7b988, var=0x7f34c4902e90, buff=0x7f34c4902f20 "0") at 10.0-base/sql/mysqld.cc:6819
      #8  0x000000000069d4f6 in show_status_array (thd=0x2e7b988, wild=0x0, variables=0x2c8d480, value_type=OPT_SESSION, status_var=0x7f34c49043d0, prefix=0xdb14cb "", table=0x7f34ac01df00, ucase_names=false, cond=0x0) at 10.0-base/sql/sql_show.cc:2876
      #9  0x00000000006adef1 in fill_status (thd=0x2e7b988, tables=0x7f34ac046e30, cond=0x0) at 10.0-base/sql/sql_show.cc:7170
      #10 0x00000000006b01ad in do_fill_table (thd=0x2e7b988, table_list=0x7f34ac046e30, join_table=0x7f34ac022a20) at 10.0-base/sql/sql_show.cc:7920
      #11 0x00000000006b05bf in get_schema_tables_result (join=0x7f34ac0219e0, executed_place=PROCESSED_BY_JOIN_EXEC) at 10.0-base/sql/sql_show.cc:8027
      #12 0x00000000006579f4 in JOIN::exec_inner (this=0x7f34ac0219e0) at 10.0-base/sql/sql_select.cc:2453
      #13 0x0000000000657051 in JOIN::exec (this=0x7f34ac0219e0) at 10.0-base/sql/sql_select.cc:2296
      #14 0x000000000065a633 in mysql_select (thd=0x2e7b988, rref_pointer_array=0x2e7f898, tables=0x7f34ac046e30, wild_num=0, fields=..., conds=0x0, og_num=0, order=0x0, group=0x0, having=0x0, proc_param=0x0, select_options=2684619520, result=0x7f34ac0ef180, unit=0x2e7eef8, select_lex=0x2e7f5e8) at 10.0-base/sql/sql_select.cc:3225
      #15 0x0000000000650dca in handle_select (thd=0x2e7b988, lex=0x2e7ee40, result=0x7f34ac0ef180, setup_tables_done_option=0) at 10.0-base/sql/sql_select.cc:376
      #16 0x000000000062491d in execute_sqlcom_select (thd=0x2e7b988, all_tables=0x7f34ac046e30) at 10.0-base/sql/sql_parse.cc:4837
      #17 0x0000000000624a8a in execute_show_status (thd=0x2e7b988, all_tables=0x7f34ac046e30) at 10.0-base/sql/sql_parse.cc:4857
      #18 0x000000000061cc71 in mysql_execute_command (thd=0x2e7b988) at 10.0-base/sql/sql_parse.cc:2201
      #19 0x0000000000627187 in mysql_parse (thd=0x2e7b988, rawbuf=0x7f34ac12f8d0 "show status", length=11, parser_state=0x7f34c4905510) at 10.0-base/sql/sql_parse.cc:5960
      #20 0x000000000061a238 in dispatch_command (command=COM_QUERY, thd=0x2e7b988, packet=0x2e806d9 "show status", packet_length=11) at 10.0-base/sql/sql_parse.cc:1091
      #21 0x0000000000619767 in do_command (thd=0x2e7b988) at 10.0-base/sql/sql_parse.cc:810
      #22 0x0000000000737df7 in do_handle_one_connection (thd_arg=0x2e7b988) at 10.0-base/sql/sql_connect.cc:1266
      #23 0x0000000000737b65 in handle_one_connection (arg=0x2e7b988) at 10.0-base/sql/sql_connect.cc:1181
      #24 0x00000000009a4b74 in pfs_spawn_thread (arg=0x2e5f0d8) at 10.0-base/storage/perfschema/pfs.cc:1015
      #25 0x00007f34c8d3ce9a in start_thread (arg=0x7f34c4906700) at pthread_create.c:308
      #26 0x00007f34c8030cbd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
      #27 0x0000000000000000 in ?? ()

      bzr version-info

      revision-id: sergii@pisem.net-20130624185655-3ysky07m0gvet6gl
      revno: 3669
      branch-nick: 10.0-base

      Test case:

      --source include/not_embedded.inc
      #
      # SHUTDOWN statement
      #
      create user user1@localhost;
       
      connect (c1,localhost,user1,,);
       
      --error ER_SPECIFIC_ACCESS_DENIED_ERROR
      shutdown;
       
      connection default;
      disconnect c1;
      --connect (c1,localhost,root,,)
       
      --let $_server_id= `SELECT @@server_id`
      --let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$_server_id.expect
      --exec echo "wait" > $_expect_file_name
       
      --send shutdown
      --connection default
      --source include/wait_until_disconnected.inc
       
      --exec echo "restart" > $_expect_file_name
      --enable_reconnect
      --source include/wait_until_connected_again.inc
       
      drop user user1@localhost;

        Attachments

          Activity

            People

            Assignee:
            wlad Vladislav Vaintroub
            Reporter:
            elenst Elena Stepanova
            Votes:
            0 Vote for this issue
            Watchers:
            2 Start watching this issue

              Dates

              Created:
              Updated:
              Resolved: