galera.MW-328A MTR failed: "Semaphore wait has lasted > 600 seconds" and do not release port 16002

galera.MW-328A 'innodb'                  w2 [ fail ]  timeout after 900 seconds

        Test ended at 2020-05-20 06:37:02

Test case timeout after 900 seconds

== /dev/shm/var/2/log/MW-328A.log == 

connection node_2;

connection node_1;

CREATE TABLE t1 (f1 INTEGER AUTO_INCREMENT PRIMARY KEY, f2 CHAR(20) DEFAULT 'abc') ENGINE=InnoDB;

INSERT INTO t1 (f1) VALUES (1);

CREATE TABLE t2 (f1 CHAR(20)) ENGINE=InnoDB;

CREATE PROCEDURE proc_update ()

BEGIN

DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END;

SET SESSION wsrep_sync_wait = 0;

WHILE 1 DO

UPDATE t1 SET f2 = LEFT(MD5(RAND()), 4);

END WHILE;

END|

connect node_1X, 127.0.0.1, root, , test, $NODE_MYPORT_1;

connection node_1X;

CALL proc_update();;

connection node_2;

SET SESSION wsrep_retry_autocommit = 0;

connection node_1;

 == /dev/shm/var/2/tmp/analyze-timeout-mysqld.1.err ==

mysqltest: Could not open connection 'default' after 500 attempts: 2002 Can't connect to local MySQL server through socket '/dev/shm/var/tmp/2/mysqld.1.sock' (111)

 == /dev/shm/var/2/tmp/analyze-timeout-mysqld.2.err ==

mysqltest: Could not open connection 'default' after 500 attempts: 2002 Can't connect to local MySQL server through socket '/dev/shm/var/tmp/2/mysqld.2.sock' (111)

 - found 'core.9074' (0/0)

Trying 'dbx' to get a backtrace

Trying 'gdb' to get a backtrace from coredump /dev/shm/var/2/log/galera.MW-328A-innodb/mysqld.2/data/core.9074

Core generated by '/usr/sbin/mysqld'

Output from gdb follows. The first stack trace is from the failing thread.

The following stack traces are from all threads (so the failing one is

duplicated).

--------------------------

[New Thread 9074]

[New Thread 9076]

[New Thread 9260]

[New Thread 9265]

[New Thread 9250]

[New Thread 9077]

[New Thread 9252]

[New Thread 9272]

[New Thread 9080]

[New Thread 9266]

[New Thread 9078]

[New Thread 9251]

[New Thread 9267]

[New Thread 9253]

[New Thread 9268]

[New Thread 9079]

[New Thread 9269]

[New Thread 9254]

[New Thread 9081]

[New Thread 9270]

[New Thread 9255]

[New Thread 9274]

[New Thread 9275]

[New Thread 9257]

[New Thread 9262]

[New Thread 9271]

[New Thread 9273]

[New Thread 9278]

[New Thread 9256]

[New Thread 9264]

[New Thread 9261]

[New Thread 9263]

[Thread debugging using libthread_db enabled]

Core was generated by `/usr/sbin/mysqld --defaults-group-suffix=.2 --defaults-file=/dev/shm/var/2/my.c'.

Program terminated with signal 6, Aborted.

#0  0x00007fbba146e63c in pthread_kill () from /lib64/libpthread.so.0

#0  0x00007fbba146e63c in pthread_kill () from /lib64/libpthread.so.0

#1  0x00007fbba21b5d0e in handle_fatal_signal () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/sql/signal_handler.cc:343

#2  <signal handler called>

#3  0x00007fbb9fb79613 in poll () from /lib64/libc.so.6

#4  0x00007fbba1ef20fa in handle_connections_sockets() () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/sql/mysqld.cc:6481

#5  0x00007fbba1ef332d in mysqld_main(int, char**) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/sql/mysqld.cc:5917

#6  0x00007fbb9fabbcdd in __libc_start_main () from /lib64/libc.so.6

#7  0x00007fbba1ee6ae1 in _start ()

Thread 32 (Thread 0x7fbb561fc700 (LWP 9263)):

#0  0x00007fbba146d75b in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0

#1  0x00007fbba23d6353 in os_event::timed_wait(timespec const*) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0event.cc:275

#2  0x00007fbba23d6561 in os_event::wait_time_low(unsigned long, long) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0event.cc:385

#3  0x00007fbba25428c7 in dict_stats_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/dict/dict0stats_bg.cc:476

#4  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#5  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 31 (Thread 0x7fbb56bfd700 (LWP 9261)):

#0  0x00007fbba146d75b in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0

#1  0x00007fbba23d6353 in os_event::timed_wait(timespec const*) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0event.cc:275

#2  0x00007fbba23d6561 in os_event::wait_time_low(unsigned long, long) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0event.cc:385

#3  0x00007fbba245e5cc in srv_error_monitor_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/srv/srv0srv.cc:1853

#4  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#5  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 30 (Thread 0x7fbb557fb700 (LWP 9264)):

#0  0x00007fbba146d75b in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0

#1  0x00007fbba23d6353 in os_event::timed_wait(timespec const*) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0event.cc:275

#2  0x00007fbba23d6561 in os_event::wait_time_low(unsigned long, long) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0event.cc:385

#3  0x00007fbba24a776e in ib_wqueue_timedwait(ib_wqueue_t*, unsigned long) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/ut/ut0wqueue.cc:156

#4  0x00007fbba258b312 in fts_optimize_thread(void*) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/fts/fts0opt.cc:2834

#5  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#6  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 29 (Thread 0x7fbb717fb700 (LWP 9256)):

#0  0x00007fbba1059614 in ?? () from /lib64/libaio.so.1

#1  0x00007fbba23cd28a in LinuxAIOHandler::collect() () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0file.cc:1819

#2  0x00007fbba23cd822 in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0file.cc:1980

#3  0x00007fbba23d5329 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0file.cc:2034

#4  0x00007fbba254ffa1 in fil_aio_wait(unsigned long) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/fil/fil0fil.cc:4350

#5  0x00007fbba2461588 in io_handler_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/srv/srv0start.cc:324

#6  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#7  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 28 (Thread 0x7fbb88081700 (LWP 9278)):

#0  0x00007fbba146d75b in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0

#1  0x00007fbba1eeb2dc in one_thread_per_connection_end(THD*, bool) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/include/mysql/psi/mysql_thread.h:1211

#2  0x00007fbba209711b in do_handle_one_connection(CONNECT*) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/sql/sql_connect.cc:1423

#3  0x00007fbba209726d in handle_one_connection () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/sql/sql_connect.cc:1316

#4  0x00007fbba26ced6d in pfs_spawn_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/perfschema/pfs.cc:1869

#5  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#6  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 27 (Thread 0x7fbb335fe700 (LWP 9273)):

#0  0x00007fbba146d3dc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0

#1  0x00007fbba2353793 in thd_destructor_proxy () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/include/mysql/psi/mysql_thread.h:1185

#2  0x00007fbba26ced6d in pfs_spawn_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/perfschema/pfs.cc:1869

#3  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#4  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 26 (Thread 0x7fbb4cdfa700 (LWP 9271)):

#0  0x00007fbba1470ccd in nanosleep () from /lib64/libpthread.so.0

#1  0x00007fbba23d6aa0 in os_thread_sleep(unsigned long) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0thread.cc:231

#2  0x00007fbba24ded5a in btr_defragment_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/btr/btr0defragment.cc:717

#3  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#4  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 25 (Thread 0x7fbb57fff700 (LWP 9262)):

#0  0x00007fbba146d75b in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0

#1  0x00007fbba23d6353 in os_event::timed_wait(timespec const*) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0event.cc:275

#2  0x00007fbba23d6561 in os_event::wait_time_low(unsigned long, long) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0event.cc:385

#3  0x00007fbba245dec0 in srv_monitor_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/srv/srv0srv.cc:1689

#4  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#5  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 24 (Thread 0x7fbb70dfa700 (LWP 9257)):

#0  0x00007fbba146d75b in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0

#1  0x00007fbba23d6353 in os_event::timed_wait(timespec const*) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0event.cc:275

#2  0x00007fbba23d6561 in os_event::wait_time_low(unsigned long, long) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0event.cc:385

#3  0x00007fbba24fb96d in buf_flush_page_cleaner_coordinator () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/buf/buf0flu.cc:2640

#4  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#5  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 23 (Thread 0x7fbb880cb700 (LWP 9275)):

#0  0x00007fbba146d3dc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0

#1  0x00007fbba1f06328 in handle_slave_background () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/include/mysql/psi/mysql_thread.h:1174

#2  0x00007fbba26ced6d in pfs_spawn_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/perfschema/pfs.cc:1869

#3  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#4  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 22 (Thread 0x7fbb88115700 (LWP 9274)):

#0  0x00007fbb9fad0ab7 in sigwaitinfo () from /lib64/libc.so.6

#1  0x00007fbba1ee9353 in signal_hand () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/include/my_pthread.h:196

#2  0x00007fbba26ced6d in pfs_spawn_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/perfschema/pfs.cc:1869

#3  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#4  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 21 (Thread 0x7fbb721fc700 (LWP 9255)):

#0  0x00007fbba1059614 in ?? () from /lib64/libaio.so.1

#1  0x00007fbba23cd28a in LinuxAIOHandler::collect() () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0file.cc:1819

#2  0x00007fbba23cd822 in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0file.cc:1980

#3  0x00007fbba23d5329 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0file.cc:2034

#4  0x00007fbba254ffa1 in fil_aio_wait(unsigned long) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/fil/fil0fil.cc:4350

#5  0x00007fbba2461588 in io_handler_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/srv/srv0start.cc:324

#6  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#7  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 20 (Thread 0x7fbb4d7fb700 (LWP 9270)):

#0  0x00007fbba146d3dc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0

#1  0x00007fbba23d6861 in os_event_wait_low(os_event*, long) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0event.cc:158

#2  0x00007fbba24f5dd7 in buf_dump_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/buf/buf0dump.cc:832

#3  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#4  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 19 (Thread 0x7fbb980e9700 (LWP 9081)):

#0  0x00007fbba146d3dc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0

#1  0x00007fbb99482a4e in fifo_lock_get (q=0x7fbb99437010, err=0x7fbb980e8630) at galerautils/src/gu_fifo.c:262

#2  gu_fifo_get_head (q=0x7fbb99437010, err=0x7fbb980e8630) at galerautils/src/gu_fifo.c:330

#3  0x00007fbb9959eed7 in gcs_recv (conn=0x7fbba491dcd0, action=0x7fbb980e86b0) at gcs/src/gcs.cpp:2024

#4  0x00007fbb995d7cd3 in galera::GcsActionSource::process (this=0x7fbba491d3b0, recv_ctx=0x7fbb980e8c70, exit_loop=@0x7fbb980e8870) at galera/src/gcs_action_source.cpp:162

#5  0x00007fbb995fbe83 in galera::ReplicatorSMM::async_recv (this=0x7fbba4915830, recv_ctx=0x7fbb980e8c70) at galera/src/replicator_smm.cpp:390

#6  0x00007fbb996156a3 in galera_recv (gh=Unhandled dwarf expression opcode 0xf3

) at galera/src/wsrep_provider.cpp:236

#7  0x00007fbba27badee in wsrep::wsrep_provider_v26::run_applier(wsrep::high_priority_service*) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/wsrep-lib/src/wsrep_provider_v26.cpp:731

#8  0x00007fbba212a8f2 in wsrep_replication_process(THD*, void*) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/sql/wsrep_thd.cc:58

#9  0x00007fbba211ee5d in start_wsrep_THD(void*) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/sql/wsrep_mysqld.cc:2789

#10 0x00007fbba26ced6d in pfs_spawn_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/perfschema/pfs.cc:1869

#11 0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#12 0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 18 (Thread 0x7fbb72bfd700 (LWP 9254)):

#0  0x00007fbba1059614 in ?? () from /lib64/libaio.so.1

#1  0x00007fbba23cd28a in LinuxAIOHandler::collect() () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0file.cc:1819

#2  0x00007fbba23cd822 in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0file.cc:1980

#3  0x00007fbba23d5329 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0file.cc:2034

#4  0x00007fbba254ffa1 in fil_aio_wait(unsigned long) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/fil/fil0fil.cc:4350

#5  0x00007fbba2461588 in io_handler_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/srv/srv0start.cc:324

#6  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#7  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 17 (Thread 0x7fbb4e1fc700 (LWP 9269)):

#0  0x00007fbba146d3dc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0

#1  0x00007fbba23d6861 in os_event_wait_low(os_event*, long) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0event.cc:158

#2  0x00007fbba2459e4b in srv_resume_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/srv/srv0srv.cc:911

#3  0x00007fbba245f9a7 in srv_worker_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/srv/srv0srv.cc:2512

#4  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#5  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 16 (Thread 0x7fbb825fb700 (LWP 9079)):

#0  0x00007fbba146d3dc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0

#1  0x00007fbb995aa373 in wait (backend=Unhandled dwarf expression opcode 0xf3

) at galerautils/src/gu_lock.hpp:45

#2  front (backend=Unhandled dwarf expression opcode 0xf3

) at gcs/src/gcs_gcomm.cpp:114

#3  gcomm_recv (backend=Unhandled dwarf expression opcode 0xf3

) at gcs/src/gcs_gcomm.cpp:608

#4  0x00007fbb9959bbe7 in core_msg_recv (conn=0x7fbba491df50, recv_act=0x7fbb825fad20, timeout=9223372035999999999) at gcs/src/gcs_core.cpp:480

#5  gcs_core_recv (conn=0x7fbba491df50, recv_act=0x7fbb825fad20, timeout=9223372035999999999) at gcs/src/gcs_core.cpp:1184

#6  0x00007fbb995a724d in gcs_recv_thread (arg=0x7fbba491dcd0) at gcs/src/gcs.cpp:1398

#7  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#8  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 15 (Thread 0x7fbb4ebfd700 (LWP 9268)):

#0  0x00007fbba146d3dc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0

#1  0x00007fbba23d6861 in os_event_wait_low(os_event*, long) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0event.cc:158

#2  0x00007fbba2459e4b in srv_resume_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/srv/srv0srv.cc:911

#3  0x00007fbba245f9a7 in srv_worker_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/srv/srv0srv.cc:2512

#4  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#5  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 14 (Thread 0x7fbb811f9700 (LWP 9253)):

#0  0x00007fbba1059614 in ?? () from /lib64/libaio.so.1

#1  0x00007fbba23cd28a in LinuxAIOHandler::collect() () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0file.cc:1819

#2  0x00007fbba23cd822 in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0file.cc:1980

#3  0x00007fbba23d5329 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0file.cc:2034

#4  0x00007fbba254ffa1 in fil_aio_wait(unsigned long) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/fil/fil0fil.cc:4350

#5  0x00007fbba2461588 in io_handler_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/srv/srv0start.cc:324

#6  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#7  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 13 (Thread 0x7fbb54dfa700 (LWP 9267)):

#0  0x00007fbba146d3dc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0

#1  0x00007fbba23d6861 in os_event_wait_low(os_event*, long) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0event.cc:158

#2  0x00007fbba2459e4b in srv_resume_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/srv/srv0srv.cc:911

#3  0x00007fbba245f9a7 in srv_worker_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/srv/srv0srv.cc:2512

#4  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#5  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 12 (Thread 0x7fbb835fe700 (LWP 9251)):

#0  0x00007fbba1059614 in ?? () from /lib64/libaio.so.1

#1  0x00007fbba23cd28a in LinuxAIOHandler::collect() () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0file.cc:1819

#2  0x00007fbba23cd822 in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0file.cc:1980

#3  0x00007fbba23d5329 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0file.cc:2034

#4  0x00007fbba254ffa1 in fil_aio_wait(unsigned long) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/fil/fil0fil.cc:4350

#5  0x00007fbba2461588 in io_handler_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/srv/srv0start.cc:324

#6  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#7  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 11 (Thread 0x7fbb8affc700 (LWP 9078)):

#0  0x00007fbb9fb832c3 in epoll_wait () from /lib64/libc.so.6

#1  0x00007fbb995693de in asio::detail::epoll_reactor::run (this=0x7fbba494dc40, block=Unhandled dwarf expression opcode 0xf3

) at asio/asio/detail/impl/epoll_reactor.ipp:391

#2  0x00007fbb99569d6b in do_run_one (this=0x7fbba494ce90, ec=...) at asio/asio/detail/impl/task_io_service.ipp:355

#3  asio::detail::task_io_service::run (this=0x7fbba494ce90, ec=...) at asio/asio/detail/impl/task_io_service.ipp:148

#4  0x00007fbb99583727 in run (this=0x7fbba494cd60, period=Unhandled dwarf expression opcode 0xf3

) at asio/asio/impl/io_service.ipp:58

#5  gcomm::AsioProtonet::event_loop (this=0x7fbba494cd60, period=Unhandled dwarf expression opcode 0xf3

) at gcomm/src/asio_protonet.cpp:116

#6  0x00007fbb995ac46b in GCommConn::run (this=0x7fbba494d6d0) at gcs/src/gcs_gcomm.cpp:433

#7  0x00007fbb995af089 in GCommConn::run_fn (arg=Unhandled dwarf expression opcode 0xf3

) at gcs/src/gcs_gcomm.cpp:174

#8  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#9  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 10 (Thread 0x7fbb4f5fe700 (LWP 9266)):

#0  0x00007fbba146d3dc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0

#1  0x00007fbba23d6861 in os_event_wait_low(os_event*, long) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0event.cc:158

#2  0x00007fbba246068a in srv_purge_coordinator_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/srv/srv0srv.cc:911

#3  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#4  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 9 (Thread 0x7fbb98133700 (LWP 9080)):

#0  0x00007fbba146d3dc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0

#1  0x00007fbba212a25b in wsrep_rollback_process(THD*, void*) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/include/mysql/psi/mysql_thread.h:1185

#2  0x00007fbba211ee5d in start_wsrep_THD(void*) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/sql/wsrep_mysqld.cc:2789

#3  0x00007fbba26ced6d in pfs_spawn_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/perfschema/pfs.cc:1869

#4  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#5  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 8 (Thread 0x7fbb33fff700 (LWP 9272)):

#0  0x00007fbba146d3dc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0

#1  0x00007fbba23d6861 in os_event_wait_low(os_event*, long) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0event.cc:158

#2  0x00007fbba24ee132 in buf_resize_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/buf/buf0buf.cc:3156

#3  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#4  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 7 (Thread 0x7fbb81bfa700 (LWP 9252)):

#0  0x00007fbba1059614 in ?? () from /lib64/libaio.so.1

#1  0x00007fbba23cd28a in LinuxAIOHandler::collect() () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0file.cc:1819

#2  0x00007fbba23cd822 in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0file.cc:1980

#3  0x00007fbba23d5329 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0file.cc:2034

#4  0x00007fbba254ffa1 in fil_aio_wait(unsigned long) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/fil/fil0fil.cc:4350

#5  0x00007fbba2461588 in io_handler_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/srv/srv0start.cc:324

#6  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#7  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 6 (Thread 0x7fbb99335700 (LWP 9077)):

#0  0x00007fbba146d3dc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0

#1  0x00007fbb995d2501 in wait (arg=0x7fbba4915ec8) at galerautils/src/gu_lock.hpp:45

#2  galera::ServiceThd::thd_func (arg=0x7fbba4915ec8) at galera/src/galera_service_thd.cpp:30

#3  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#4  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 5 (Thread 0x7fbb89117700 (LWP 9250)):

#0  0x00007fbba146d75b in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0

#1  0x00007fbba25fbc7f in my_service_thread_sleep ()

#2  0x00007fbba25f34ce in ma_checkpoint_background ()

#3  0x00007fbba26ced6d in pfs_spawn_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/perfschema/pfs.cc:1869

#4  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#5  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 4 (Thread 0x7fbb4ffff700 (LWP 9265)):

#0  0x00007fbba1470ccd in nanosleep () from /lib64/libpthread.so.0

#1  0x00007fbba23d6aa0 in os_thread_sleep(unsigned long) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0thread.cc:231

#2  0x00007fbba245eff9 in srv_master_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/srv/srv0srv.cc:2336

#3  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#4  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 3 (Thread 0x7fbb575fe700 (LWP 9260)):

#0  0x00007fbba146d75b in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0

#1  0x00007fbba23d6353 in os_event::timed_wait(timespec const*) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0event.cc:275

#2  0x00007fbba23d6561 in os_event::wait_time_low(unsigned long, long) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/os/os0event.cc:385

#3  0x00007fbba23ac7ef in lock_wait_timeout_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/innobase/lock/lock0wait.cc:522

#4  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#5  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 2 (Thread 0x7fbb9a2c3700 (LWP 9076)):

#0  0x00007fbba146d75b in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0

#1  0x00007fbba27232ba in timer_handler ()

#2  0x00007fbba26ced6d in pfs_spawn_thread () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/storage/perfschema/pfs.cc:1869

#3  0x00007fbba14697f1 in start_thread () from /lib64/libpthread.so.0

#4  0x00007fbb9fb82ccd in clone () from /lib64/libc.so.6

Thread 1 (Thread 0x7fbba188b7e0 (LWP 9074)):

#0  0x00007fbba146e63c in pthread_kill () from /lib64/libpthread.so.0

#1  0x00007fbba21b5d0e in handle_fatal_signal () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/sql/signal_handler.cc:343

#2  <signal handler called>

#3  0x00007fbb9fb79613 in poll () from /lib64/libc.so.6

#4  0x00007fbba1ef20fa in handle_connections_sockets() () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/sql/mysqld.cc:6481

#5  0x00007fbba1ef332d in mysqld_main(int, char**) () at /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.14/sql/mysqld.cc:5917

#6  0x00007fbb9fabbcdd in __libc_start_main () from /lib64/libc.so.6

#7  0x00007fbba1ee6ae1 in _start ()

Compressed file /dev/shm/var/2/log/galera.MW-328A-innodb/mysqld.2/data/core.9074

 - found 'core.9019' (1/0)