Details
-
Bug
-
Status: Closed (View Workflow)
-
Major
-
Resolution: Fixed
-
2.2.9
-
None
-
MXS-SPRINT-65, MXS-SPRINT-66
Description
Maxscale instance hang during failover , having multiple instance running on different node with
passive=0 and passive=1 value .
Maxscale Instance hang was configured with passive=1.
Stacktrace from the Hang Maxscale Instance:-
|
|
[Thread debugging using libthread_db enabled]
|
0x00007f06e60e1334 in __lll_lock_wait () from /lib64/libpthread.so.0
|
|
|
Thread 9 (Thread 0x7f06e4a68700 (LWP 6038)):
|
#0 0x00007f06e65c3a6e in thr_filewriter_fun (data=0x927e00) at /home/ec2-user/MaxScale/server/core/log_manager.cc:2353
|
#1 0x00007f06e60daaa1 in start_thread () from /lib64/libpthread.so.0
|
#2 0x00007f06e517cbbd in clone () from /lib64/libc.so.6
|
|
|
Thread 8 (Thread 0x7f06e2748700 (LWP 6039)):
|
#0 0x00007f06e60dd8a9 in pthread_mutex_unlock () from /lib64/libpthread.so.0
|
#1 0x00007f06e65e6d0a in simple_mutex_unlock (sm=0x927928) at /home/ec2-user/MaxScale/server/core/skygw_utils.cc:496
|
#2 0x00007f06e65c2cbf in blockbuf_get_writepos (priority=5, modname=<value optimized out>, file=<value optimized out>, line=<value optimized out>, function=<value optimized out>, format=<value optimized out>) at /home/ec2-user/MaxScale/server/core/log_manager.cc:1019
|
#3 logmanager_write_log (priority=5, modname=<value optimized out>, file=<value optimized out>, line=<value optimized out>, function=<value optimized out>, format=<value optimized out>) at /home/ec2-user/MaxScale/server/core/log_manager.cc:838
|
#4 log_write (priority=5, modname=<value optimized out>, file=<value optimized out>, line=<value optimized out>, function=<value optimized out>, format=<value optimized out>) at /home/ec2-user/MaxScale/server/core/log_manager.cc:1309
|
#5 mxs_log_message (priority=5, modname=<value optimized out>, file=<value optimized out>, line=<value optimized out>, function=<value optimized out>, format=<value optimized out>) at /home/ec2-user/MaxScale/server/core/log_manager.cc:3039
|
#6 0x00007f06e65c93ff in mon_log_state_change (ptr=0x93d560) at /home/ec2-user/MaxScale/server/core/monitor.cc:1520
|
#7 0x00007f06e65cc89d in mon_process_state_changes (monitor=0x93c110, script=0x0, events=4194303) at /home/ec2-user/MaxScale/server/core/monitor.cc:1765
|
#8 0x00007f06e361d453 in monitorMain (arg=0x93e300) at /home/ec2-user/MaxScale/server/modules/monitor/mariadbmon/mariadbmon.cc:2510
|
#9 0x00007f06e60daaa1 in start_thread () from /lib64/libpthread.so.0
|
#10 0x00007f06e517cbbd in clone () from /lib64/libc.so.6
|
|
|
Thread 7 (Thread 0x7f06e1d47700 (LWP 6040)):
|
#0 0x00007f06e60e200d in nanosleep () from /lib64/libpthread.so.0
|
#1 0x00000000004061bf in log_flush_cb (arg=<value optimized out>) at /home/ec2-user/MaxScale/server/core/gateway.cc:2388
|
#2 0x00007f06e60daaa1 in start_thread () from /lib64/libpthread.so.0
|
#3 0x00007f06e517cbbd in clone () from /lib64/libc.so.6
|
|
|
Thread 6 (Thread 0x7f06e1346700 (LWP 6041)):
|
#0 0x00007f06e60e200d in nanosleep () from /lib64/libpthread.so.0
|
#1 0x00007f06e65e7ffe in thread_millisleep (ms=<value optimized out>) at /home/ec2-user/MaxScale/server/core/thread.cc:70
|
#2 0x00007f06e65ba275 in hkthread (data=<value optimized out>) at /home/ec2-user/MaxScale/server/core/housekeeper.cc:242
|
#3 0x00007f06e60daaa1 in start_thread () from /lib64/libpthread.so.0
|
#4 0x00007f06e517cbbd in clone () from /lib64/libc.so.6
|
|
|
Thread 5 (Thread 0x7f06d3fff700 (LWP 6042)):
|
#0 0x00007f06e60e14c5 in __lll_unlock_wake () from /lib64/libpthread.so.0
|
#1 0x00007f06e60dd947 in _L_unlock_657 () from /lib64/libpthread.so.0
|
#2 0x00007f06e60dd8af in pthread_mutex_unlock () from /lib64/libpthread.so.0
|
#3 0x00007f06e65e6d0a in simple_mutex_unlock (sm=0x927928) at /home/ec2-user/MaxScale/server/core/skygw_utils.cc:496
|
#4 0x00007f06e65c2cbf in blockbuf_get_writepos (priority=3, modname=<value optimized out>, file=<value optimized out>, line=<value optimized out>, function=<value optimized out>, format=<value optimized out>) at /home/ec2-user/MaxScale/server/core/log_manager.cc:1019
|
#5 logmanager_write_log (priority=3, modname=<value optimized out>, file=<value optimized out>, line=<value optimized out>, function=<value optimized out>, format=<value optimized out>) at /home/ec2-user/MaxScale/server/core/log_manager.cc:838
|
#6 log_write (priority=3, modname=<value optimized out>, file=<value optimized out>, line=<value optimized out>, function=<value optimized out>, format=<value optimized out>) at /home/ec2-user/MaxScale/server/core/log_manager.cc:1309
|
#7 mxs_log_message (priority=3, modname=<value optimized out>, file=<value optimized out>, line=<value optimized out>, function=<value optimized out>, format=<value optimized out>) at /home/ec2-user/MaxScale/server/core/log_manager.cc:3039
|
#8 0x00007f06e3a48734 in select_connect_backend_servers (router_nservers=<value optimized out>, max_nslaves=2, session=0x7f06ccf17470, config=<value optimized out>, backends=std::list = {...}, current_master=std::tr1::shared_ptr (empty) 0x0, sescmd_list=0x0, expected_responses=0x0, type=ALL) at /home/ec2-user/MaxScale/server/modules/routing/readwritesplit/rwsplit_select_backends.cc:306
|
#9 0x00007f06e3a41f31 in RWSplitSession::create (router=0x9589a0, session=0x7f06ccf17470) at /home/ec2-user/MaxScale/server/modules/routing/readwritesplit/readwritesplit.cc:797
|
#10 0x00007f06e3a42149 in newSession (router_inst=<value optimized out>, session=<value optimized out>) at /home/ec2-user/MaxScale/server/modules/routing/readwritesplit/readwritesplit.cc:869
|
#11 0x00007f06e65e421e in session_alloc_body (service=0x92d0f0, client_dcb=0x7f06cdd059c0, session=0x7f06ccf17470) at /home/ec2-user/MaxScale/server/core/session.cc:167
|
#12 0x00007f06e29f521a in gw_read_do_authentication (dcb=0x7f06cdd059c0) at /home/ec2-user/MaxScale/server/modules/protocol/MySQL/mariadbclient/mysql_client.cc:742
|
#13 gw_read_client_event (dcb=0x7f06cdd059c0) at /home/ec2-user/MaxScale/server/modules/protocol/MySQL/mariadbclient/mysql_client.cc:503
|
#14 0x00007f06e65b51bc in dcb_process_poll_events (dcb=0x7f06cdd059c0, events=5) at /home/ec2-user/MaxScale/server/core/dcb.cc:3098
|
#15 0x00007f06e65b5388 in dcb_handler (dcb=0x7f06cdd059c0, events=5) at /home/ec2-user/MaxScale/server/core/dcb.cc:3183
|
#16 0x00007f06e65edb6b in maxscale::Worker::poll_waitevents (this=0x933c10) at /home/ec2-user/MaxScale/server/core/worker.cc:1205
|
#17 0x00007f06e65ede41 in maxscale::Worker::run (this=0x933c10) at /home/ec2-user/MaxScale/server/core/worker.cc:885
|
#18 0x00007f06e60daaa1 in start_thread () from /lib64/libpthread.so.0
|
#19 0x00007f06e517cbbd in clone () from /lib64/libc.so.6
|
|
|
Thread 4 (Thread 0x7f06d35fe700 (LWP 6043)):
|
#0 0x00007f06e60e1334 in __lll_lock_wait () from /lib64/libpthread.so.0
|
#1 0x00007f06e60dc5d8 in _L_lock_854 () from /lib64/libpthread.so.0
|
#2 0x00007f06e60dc4a7 in pthread_mutex_lock () from /lib64/libpthread.so.0
|
#3 0x00007f06e65e718f in simple_mutex_lock (sm=0x927928, block=<value optimized out>) at /home/ec2-user/MaxScale/server/core/skygw_utils.cc:463
|
#4 0x00007f06e65c2d1a in blockbuf_get_writepos (priority=3, modname=<value optimized out>, file=<value optimized out>, line=<value optimized out>, function=<value optimized out>, format=<value optimized out>) at /home/ec2-user/MaxScale/server/core/log_manager.cc:1044
|
#5 logmanager_write_log (priority=3, modname=<value optimized out>, file=<value optimized out>, line=<value optimized out>, function=<value optimized out>, format=<value optimized out>) at /home/ec2-user/MaxScale/server/core/log_manager.cc:838
|
#6 log_write (priority=3, modname=<value optimized out>, file=<value optimized out>, line=<value optimized out>, function=<value optimized out>, format=<value optimized out>) at /home/ec2-user/MaxScale/server/core/log_manager.cc:1309
|
#7 mxs_log_message (priority=3, modname=<value optimized out>, file=<value optimized out>, line=<value optimized out>, function=<value optimized out>, format=<value optimized out>) at /home/ec2-user/MaxScale/server/core/log_manager.cc:3039
|
#8 0x00007f06e3a48734 in select_connect_backend_servers (router_nservers=<value optimized out>, max_nslaves=2, session=0x7f06c4ddb920, config=<value optimized out>, backends=std::list = {...}, current_master=std::tr1::shared_ptr (empty) 0x0, sescmd_list=0x0, expected_responses=0x0, type=ALL) at /home/ec2-user/MaxScale/server/modules/routing/readwritesplit/rwsplit_select_backends.cc:306
|
#9 0x00007f06e3a41f31 in RWSplitSession::create (router=0x9589a0, session=0x7f06c4ddb920) at /home/ec2-user/MaxScale/server/modules/routing/readwritesplit/readwritesplit.cc:797
|
#10 0x00007f06e3a42149 in newSession (router_inst=<value optimized out>, session=<value optimized out>) at /home/ec2-user/MaxScale/server/modules/routing/readwritesplit/readwritesplit.cc:869
|
#11 0x00007f06e65e421e in session_alloc_body (service=0x92d0f0, client_dcb=0x7f06c55a5450, session=0x7f06c4ddb920) at /home/ec2-user/MaxScale/server/core/session.cc:167
|
#12 0x00007f06e29f521a in gw_read_do_authentication (dcb=0x7f06c55a5450) at /home/ec2-user/MaxScale/server/modules/protocol/MySQL/mariadbclient/mysql_client.cc:742
|
#13 gw_read_client_event (dcb=0x7f06c55a5450) at /home/ec2-user/MaxScale/server/modules/protocol/MySQL/mariadbclient/mysql_client.cc:503
|
#14 0x00007f06e65b51bc in dcb_process_poll_events (dcb=0x7f06c55a5450, events=5) at /home/ec2-user/MaxScale/server/core/dcb.cc:3098
|
#15 0x00007f06e65b5388 in dcb_handler (dcb=0x7f06c55a5450, events=5) at /home/ec2-user/MaxScale/server/core/dcb.cc:3183
|
#16 0x00007f06e65edb6b in maxscale::Worker::poll_waitevents (this=0x92a770) at /home/ec2-user/MaxScale/server/core/worker.cc:1205
|
#17 0x00007f06e65ede41 in maxscale::Worker::run (this=0x92a770) at /home/ec2-user/MaxScale/server/core/worker.cc:885
|
#18 0x00007f06e60daaa1 in start_thread () from /lib64/libpthread.so.0
|
#19 0x00007f06e517cbbd in clone () from /lib64/libc.so.6
|
|
|
Thread 3 (Thread 0x7f06d2bfd700 (LWP 6044)):
|
#0 0x00007f06e60e1334 in __lll_lock_wait () from /lib64/libpthread.so.0
|
#1 0x00007f06e60dc5d8 in _L_lock_854 () from /lib64/libpthread.so.0
|
#2 0x00007f06e60dc4a7 in pthread_mutex_lock () from /lib64/libpthread.so.0
|
#3 0x00007f06e65e718f in simple_mutex_lock (sm=0x7f06c93e4358, block=<value optimized out>) at /home/ec2-user/MaxScale/server/core/skygw_utils.cc:463
|
#4 0x00007f06e65c2cd5 in blockbuf_get_writepos (priority=3, modname=<value optimized out>, file=<value optimized out>, line=<value optimized out>, function=<value optimized out>, format=<value optimized out>) at /home/ec2-user/MaxScale/server/core/log_manager.cc:1025
|
#5 logmanager_write_log (priority=3, modname=<value optimized out>, file=<value optimized out>, line=<value optimized out>, function=<value optimized out>, format=<value optimized out>) at /home/ec2-user/MaxScale/server/core/log_manager.cc:838
|
#6 log_write (priority=3, modname=<value optimized out>, file=<value optimized out>, line=<value optimized out>, function=<value optimized out>, format=<value optimized out>) at /home/ec2-user/MaxScale/server/core/log_manager.cc:1309
|
#7 mxs_log_message (priority=3, modname=<value optimized out>, file=<value optimized out>, line=<value optimized out>, function=<value optimized out>, format=<value optimized out>) at /home/ec2-user/MaxScale/server/core/log_manager.cc:3039
|
#8 0x00007f06e65e45ec in session_alloc_body (service=0x92d0f0, client_dcb=0x7f06c8d91d90, session=0x7f06c915bde0) at /home/ec2-user/MaxScale/server/core/session.cc:172
|
#9 0x00007f06e29f521a in gw_read_do_authentication (dcb=0x7f06c8d91d90) at /home/ec2-user/MaxScale/server/modules/protocol/MySQL/mariadbclient/mysql_client.cc:742
|
#10 gw_read_client_event (dcb=0x7f06c8d91d90) at /home/ec2-user/MaxScale/server/modules/protocol/MySQL/mariadbclient/mysql_client.cc:503
|
#11 0x00007f06e65b51bc in dcb_process_poll_events (dcb=0x7f06c8d91d90, events=5) at /home/ec2-user/MaxScale/server/core/dcb.cc:3098
|
#12 0x00007f06e65b5388 in dcb_handler (dcb=0x7f06c8d91d90, events=5) at /home/ec2-user/MaxScale/server/core/dcb.cc:3183
|
#13 0x00007f06e65edb6b in maxscale::Worker::poll_waitevents (this=0x92e210) at /home/ec2-user/MaxScale/server/core/worker.cc:1205
|
#14 0x00007f06e65ede41 in maxscale::Worker::run (this=0x92e210) at /home/ec2-user/MaxScale/server/core/worker.cc:885
|
#15 0x00007f06e60daaa1 in start_thread () from /lib64/libpthread.so.0
|
#16 0x00007f06e517cbbd in clone () from /lib64/libc.so.6
|
|
|
Thread 2 (Thread 0x7f06d21fc700 (LWP 6045)):
|
#0 0x00007f06e517d1b3 in epoll_wait () from /lib64/libc.so.6
|
#1 0x00007f06e669b181 in MHD_epoll (daemon=0x990d60, may_block=<value optimized out>) at daemon.c:4267
|
#2 0x00007f06e669bf15 in MHD_select_thread (cls=0x990d60) at daemon.c:4544
|
#3 0x00007f06e60daaa1 in start_thread () from /lib64/libpthread.so.0
|
#4 0x00007f06e517cbbd in clone () from /lib64/libc.so.6
|
|
|
Thread 1 (Thread 0x7f06e6b08c00 (LWP 6037)):
|
#0 0x00007f06e60e1334 in __lll_lock_wait () from /lib64/libpthread.so.0
|
#1 0x00007f06e60dc5d8 in _L_lock_854 () from /lib64/libpthread.so.0
|
#2 0x00007f06e60dc4a7 in pthread_mutex_lock () from /lib64/libpthread.so.0
|
#3 0x00007f06e65e718f in simple_mutex_lock (sm=0x7f06cd294768, block=<value optimized out>) at /home/ec2-user/MaxScale/server/core/skygw_utils.cc:463
|
#4 0x00007f06e65c2cd5 in blockbuf_get_writepos (priority=3, modname=<value optimized out>, file=<value optimized out>, line=<value optimized out>, function=<value optimized out>, format=<value optimized out>) at /home/ec2-user/MaxScale/server/core/log_manager.cc:1025
|
#5 logmanager_write_log (priority=3, modname=<value optimized out>, file=<value optimized out>, line=<value optimized out>, function=<value optimized out>, format=<value optimized out>) at /home/ec2-user/MaxScale/server/core/log_manager.cc:838
|
#6 log_write (priority=3, modname=<value optimized out>, file=<value optimized out>, line=<value optimized out>, function=<value optimized out>, format=<value optimized out>) at /home/ec2-user/MaxScale/server/core/log_manager.cc:1309
|
#7 mxs_log_message (priority=3, modname=<value optimized out>, file=<value optimized out>, line=<value optimized out>, function=<value optimized out>, format=<value optimized out>) at /home/ec2-user/MaxScale/server/core/log_manager.cc:3039
|
#8 0x00007f06e65e45ec in session_alloc_body (service=0x92d0f0, client_dcb=0x14206a0, session=0x14df240) at /home/ec2-user/MaxScale/server/core/session.cc:172
|
#9 0x00007f06e29f521a in gw_read_do_authentication (dcb=0x14206a0) at /home/ec2-user/MaxScale/server/modules/protocol/MySQL/mariadbclient/mysql_client.cc:742
|
#10 gw_read_client_event (dcb=0x14206a0) at /home/ec2-user/MaxScale/server/modules/protocol/MySQL/mariadbclient/mysql_client.cc:503
|
#11 0x00007f06e65b51bc in dcb_process_poll_events (dcb=0x14206a0, events=5) at /home/ec2-user/MaxScale/server/core/dcb.cc:3098
|
#12 0x00007f06e65b5388 in dcb_handler (dcb=0x14206a0, events=5) at /home/ec2-user/MaxScale/server/core/dcb.cc:3183
|
#13 0x00007f06e65edb6b in maxscale::Worker::poll_waitevents (this=0x938160) at /home/ec2-user/MaxScale/server/core/worker.cc:1205
|
#14 0x00007f06e65ede41 in maxscale::Worker::run (this=0x938160) at /home/ec2-user/MaxScale/server/core/worker.cc:885
|
#15 0x0000000000408a72 in main (argc=<value optimized out>, argv=<value optimized out>) at /home/ec2-user/MaxScale/server/core/gateway.cc:2251
|
|