[MDEV-4712] Server crashes upon a race condition between SHUTDOWN and SHOW STATUS, main.shutdown fails sporadically in buildbot Created: 2013-06-26  Updated: 2013-07-31  Resolved: 2013-07-31

Status: Closed
Project: MariaDB Server
Component/s: None
Affects Version/s: None
Fix Version/s: 10.0.4

Type: Bug Priority: Major
Reporter: Elena Stepanova Assignee: Vladislav Vaintroub
Resolution: Fixed Votes: 0
Labels: buildbot


 Description   

The new test main.shutdown fails sporadically with a crash on shutdown, see for example http://buildbot.askmonty.org/buildbot/builders/kvm-deb-debian6-x86/builds/2146/steps/test_3/logs/stdio (server error log: http://buildbot.askmonty.org/buildbot/builders/kvm-deb-debian6-x86/builds/2146/steps/test_3/logs/mysqld.1.err.1)

Below is a slightly modified version of main.shutdown, which seems to increase the probability of hitting the failure, although it's far from deterministic.

All threads stack trace:

Thread 5 (Thread 0x7f34c4874700 (LWP 9507)):
 
#0  0x00007f34c802a023 in select () at ../sysdeps/unix/syscall-template.S:82
 
#1  0x0000000000d23405 in my_sleep (m_seconds=20000) at 10.0-base/mysys/my_sleep.c:29
 
#2  0x0000000000563b8d in close_connections () at 10.0-base/sql/mysqld.cc:1510
 
#3  0x000000000056423b in kill_server (sig_ptr=0x0) at 10.0-base/sql/mysqld.cc:1700
 
#4  0x0000000000564271 in kill_server_thread (arg=0x7f34c514fe48) at 10.0-base/sql/mysqld.cc:1728
 
#5  0x00000000009a4b74 in pfs_spawn_thread (arg=0x7f34b4001118) at 10.0-base/storage/perfschema/pfs.cc:1015
 
#6  0x00007f34c8d3ce9a in start_thread (arg=0x7f34c4874700) at pthread_create.c:308
 
#7  0x00007f34c8030cbd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
 
#8  0x0000000000000000 in ?? ()
 
 
 
Thread 4 (Thread 0x7f34c977c740 (LWP 9491)):
 
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
 
#1  0x0000000000d32488 in safe_cond_wait (cond=0x1553540, mp=0x15523c0, file=0xd84ab0 "10.0-base/sql/mysqld.cc", line=5222) at 10.0-base/mysys/thr_mutex.c:493
 
#2  0x00000000005621d8 in inline_mysql_cond_wait (that=0x1553540, mutex=0x15523c0, src_file=0xd84ab0 "10.0-base/sql/mysqld.cc", src_line=5222) at 10.0-base/include/mysql/psi/mysql_thread.h:984
 
#3  0x000000000056a3c0 in mysqld_main (argc=95, argv=0x2c4b518) at 10.0-base/sql/mysqld.cc:5222
 
#4  0x0000000000561974 in main (argc=6, argv=0x7fff81da30c8) at 10.0-base/sql/main.cc:25
 
 
 
Thread 3 (Thread 0x7f34c7d26700 (LWP 9493)):
 
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:215
 
#1  0x0000000000d32792 in safe_cond_timedwait (cond=0x1d89a00, mp=0x1d89940, abstime=0x7f34c7d25db0, file=0xec84b0 "10.0-base/storage/maria/ma_servicethread.c", line=119) at 10.0-base/mysys/thr_mutex.c:547
 
#2  0x0000000000b6425a in inline_mysql_cond_timedwait (that=0x1d89a00, mutex=0x1d89940, abstime=0x7f34c7d25db0, src_file=0xec84b0 "10.0-base/storage/maria/ma_servicethread.c", src_line=119) at 10.0-base/include/mysql/psi/mysql_thread.h:1017
 
#3  0x0000000000b6475a in my_service_thread_sleep (control=0x1419e90, sleep_time=29000000000) at 10.0-base/storage/maria/ma_servicethread.c:118
 
#4  0x0000000000b579c1 in ma_checkpoint_background (arg=0x1e) at 10.0-base/storage/maria/ma_checkpoint.c:705
 
#5  0x00000000009a4b74 in pfs_spawn_thread (arg=0x2cfc488) at 10.0-base/storage/perfschema/pfs.cc:1015
 
#6  0x00007f34c8d3ce9a in start_thread (arg=0x7f34c7d26700) at pthread_create.c:308
 
#7  0x00007f34c8030cbd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
 
#8  0x0000000000000000 in ?? ()
 
 
 
Thread 2 (Thread 0x7f34c5150700 (LWP 9494)):
 
#0  do_sigwait (set=<optimized out>, sig=0x7f34c514fe48) at ../nptl/sysdeps/unix/sysv/linux/../../../../../sysdeps/unix/sysv/linux/sigwait.c:65
 
#1  0x00007f34c8d44a79 in __sigwait (set=<optimized out>, sig=<optimized out>) at ../nptl/sysdeps/unix/sysv/linux/../../../../../sysdeps/unix/sysv/linux/sigwait.c:100
 
#2  0x0000000000566ca8 in signal_hand (arg=0x0) at 10.0-base/sql/mysqld.cc:3142
 
#3  0x00000000009a4b74 in pfs_spawn_thread (arg=0x2cca8f8) at 10.0-base/storage/perfschema/pfs.cc:1015
 
#4  0x00007f34c8d3ce9a in start_thread (arg=0x7f34c5150700) at pthread_create.c:308
 
#5  0x00007f34c8030cbd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
 
#6  0x0000000000000000 in ?? ()
 
 
 
Thread 1 (Thread 0x7f34c4906700 (LWP 9503)):
 
#0  __pthread_kill (threadid=<optimized out>, signo=<optimized out>) at ../nptl/sysdeps/unix/sysv/linux/pthread_kill.c:63
 
#1  0x0000000000d287ec in my_write_core (sig=11) at 10.0-base/mysys/stacktrace.c:457
 
#2  0x00000000007fc15e in handle_fatal_signal (sig=11) at 10.0-base/sql/signal_handler.cc:262
 
#3  <signal handler called>
 
#4  0x0000000000cf6632 in my_hash_first (hash=0x3a0, key=0x7f34c4902cf0 "", length=0, current_record=0x7f34c4902c7c) at 10.0-base/mysys/hash.c:254
 
#5  0x0000000000cf659f in my_hash_search (hash=0x3a0, key=0x7f34c4902cf0 "", length=0) at 10.0-base/mysys/hash.c:223
 
#6  0x000000000072f12c in Master_info_index::get_master_info (this=0x0, connection_name=0x2e7c338, warning=MYSQL_ERROR::WARN_LEVEL_NOTE) at 10.0-base/sql/rpl_mi.cc:1062
 
#7  0x000000000056b9ef in show_heartbeat_period (thd=0x2e7b988, var=0x7f34c4902e90, buff=0x7f34c4902f20 "0") at 10.0-base/sql/mysqld.cc:6819
 
#8  0x000000000069d4f6 in show_status_array (thd=0x2e7b988, wild=0x0, variables=0x2c8d480, value_type=OPT_SESSION, status_var=0x7f34c49043d0, prefix=0xdb14cb "", table=0x7f34ac01df00, ucase_names=false, cond=0x0) at 10.0-base/sql/sql_show.cc:2876
 
#9  0x00000000006adef1 in fill_status (thd=0x2e7b988, tables=0x7f34ac046e30, cond=0x0) at 10.0-base/sql/sql_show.cc:7170
 
#10 0x00000000006b01ad in do_fill_table (thd=0x2e7b988, table_list=0x7f34ac046e30, join_table=0x7f34ac022a20) at 10.0-base/sql/sql_show.cc:7920
 
#11 0x00000000006b05bf in get_schema_tables_result (join=0x7f34ac0219e0, executed_place=PROCESSED_BY_JOIN_EXEC) at 10.0-base/sql/sql_show.cc:8027
 
#12 0x00000000006579f4 in JOIN::exec_inner (this=0x7f34ac0219e0) at 10.0-base/sql/sql_select.cc:2453
 
#13 0x0000000000657051 in JOIN::exec (this=0x7f34ac0219e0) at 10.0-base/sql/sql_select.cc:2296
 
#14 0x000000000065a633 in mysql_select (thd=0x2e7b988, rref_pointer_array=0x2e7f898, tables=0x7f34ac046e30, wild_num=0, fields=..., conds=0x0, og_num=0, order=0x0, group=0x0, having=0x0, proc_param=0x0, select_options=2684619520, result=0x7f34ac0ef180, unit=0x2e7eef8, select_lex=0x2e7f5e8) at 10.0-base/sql/sql_select.cc:3225
 
#15 0x0000000000650dca in handle_select (thd=0x2e7b988, lex=0x2e7ee40, result=0x7f34ac0ef180, setup_tables_done_option=0) at 10.0-base/sql/sql_select.cc:376
 
#16 0x000000000062491d in execute_sqlcom_select (thd=0x2e7b988, all_tables=0x7f34ac046e30) at 10.0-base/sql/sql_parse.cc:4837
 
#17 0x0000000000624a8a in execute_show_status (thd=0x2e7b988, all_tables=0x7f34ac046e30) at 10.0-base/sql/sql_parse.cc:4857
 
#18 0x000000000061cc71 in mysql_execute_command (thd=0x2e7b988) at 10.0-base/sql/sql_parse.cc:2201
 
#19 0x0000000000627187 in mysql_parse (thd=0x2e7b988, rawbuf=0x7f34ac12f8d0 "show status", length=11, parser_state=0x7f34c4905510) at 10.0-base/sql/sql_parse.cc:5960
 
#20 0x000000000061a238 in dispatch_command (command=COM_QUERY, thd=0x2e7b988, packet=0x2e806d9 "show status", packet_length=11) at 10.0-base/sql/sql_parse.cc:1091
 
#21 0x0000000000619767 in do_command (thd=0x2e7b988) at 10.0-base/sql/sql_parse.cc:810
 
#22 0x0000000000737df7 in do_handle_one_connection (thd_arg=0x2e7b988) at 10.0-base/sql/sql_connect.cc:1266
 
#23 0x0000000000737b65 in handle_one_connection (arg=0x2e7b988) at 10.0-base/sql/sql_connect.cc:1181
 
#24 0x00000000009a4b74 in pfs_spawn_thread (arg=0x2e5f0d8) at 10.0-base/storage/perfschema/pfs.cc:1015
 
#25 0x00007f34c8d3ce9a in start_thread (arg=0x7f34c4906700) at pthread_create.c:308
 
#26 0x00007f34c8030cbd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
 
#27 0x0000000000000000 in ?? ()

bzr version-info

revision-id: sergii@pisem.net-20130624185655-3ysky07m0gvet6gl
 
revno: 3669
 
branch-nick: 10.0-base

Test case:

--source include/not_embedded.inc
 
#
 
# SHUTDOWN statement
 
#
 
create user user1@localhost;
 
 
 
connect (c1,localhost,user1,,);
 
 
 
--error ER_SPECIFIC_ACCESS_DENIED_ERROR
 
shutdown;
 
 
 
connection default;
 
disconnect c1;
 
--connect (c1,localhost,root,,)
 
 
 
--let $_server_id= `SELECT @@server_id`
 
--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$_server_id.expect
 
--exec echo "wait" > $_expect_file_name
 
 
 
--send shutdown
 
--connection default
 
--source include/wait_until_disconnected.inc
 
 
 
--exec echo "restart" > $_expect_file_name
 
--enable_reconnect
 
--source include/wait_until_connected_again.inc
 
 
 
drop user user1@localhost;


Generated at Thu Feb 08 06:58:34 UTC 2024 using Jira 8.20.16#820016-sha1:9d11dbea5f4be3d4cc21f03a88dd11d8c8687422.