+++ readlink -f /home/timofey_turenko_mariadb_com/MariaDBEnterprise/scripts/es-test.sh ++ dirname /home/timofey_turenko_mariadb_com/MariaDBEnterprise/scripts/es-test.sh + export script_dir=/home/timofey_turenko_mariadb_com/MariaDBEnterprise/scripts + script_dir=/home/timofey_turenko_mariadb_com/MariaDBEnterprise/scripts + MYSQL_USER=mysql + MYSQL_GROUP=mysql + MYSQL_VARDIR=/var/tmp/mtr + MYSQL_DATADIR=/var/lib/mysql + MTR_DEFAULT_ARGS='--max-save-core=0 --max-save-datadir=1 --force --vardir=/var/tmp/mtr ' + MTR_RUN_ARGS='--max-save-core=0 --max-save-datadir=1 --force --vardir=/var/tmp/mtr ' + PACKAGE=MariaDB-ES + unset WSREP_EXPORT + MTR_JUNIT_ARGS=' --xml-report=mtr.xml ' ++ getent passwd mysql + [[ ! -n '' ]] + case ${platform} in + sudo groupadd -r mysql + sudo useradd -M -r --home /var/lib/mysql --shell /sbin/nologin --comment 'MySQL server' --gid mysql mysql + true ++ sudo find /usr -type f -name 'libgalera*smm.so' + export WSREP_PROVIDER=/usr/lib64/galera/libgalera_smm.so + WSREP_PROVIDER=/usr/lib64/galera/libgalera_smm.so + export 'WSREP_EXPORT=export WSREP_PROVIDER=/usr/lib64/galera/libgalera_smm.so; ' + WSREP_EXPORT='export WSREP_PROVIDER=/usr/lib64/galera/libgalera_smm.so; ' ++ dirname /home/timofey_turenko_mariadb_com/MariaDBEnterprise/scripts/es-test.sh + cd /home/timofey_turenko_mariadb_com/MariaDBEnterprise/scripts/../mysql-test/ ++ pwd + RUNDIR=/home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test + sudo chown -R mysql:mysql /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test + sudo chmod a+rx . + sudo chmod a+rx /home -R + sudo mkdir -p /var/tmp/mtr + sudo mkdir -p /var/lib/mysql + sudo chown mysql:mysql /var/tmp/mtr + sudo chown mysql:mysql /var/lib/mysql + cat lib/mtr_report.pm + grep xml_report if ($::opt_xml_report) { my $xml_report = ""; $xml_report = "\n"; $xml_report .= qq(\n); $xml_report .= "\t\n"; $xml_report .= qq(\t\n); $xml_report .= qq(\t\t\n\t\t\t\n\n\t\t\t\n\t\t\n); $xml_report .= qq(>\n\t\t\t\n\t\t\n); $xml_report .= qq(>\n\t\t\t\n\t\t\n); $xml_report .= " />\n"; $xml_report .= "\t\n\n"; my $xml_file = $::opt_xml_report; print $XML_UFILE $xml_report; + '[' 0 == 0 ']' + echo 'Trying to create JUnit report' Trying to create JUnit report + unset MSAN_ADDS + [[ '' == \Y ]] + sudo su - mysql -s /bin/bash -c ' export WSREP_PROVIDER=/usr/lib64/galera/libgalera_smm.so; cd /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test && perl mysql-test-run.pl --suite=galera,wsrep,galera_3nodes --max-test-fail=0 --testcase-timeout=20 --big-test --retry=3 --parallel=1 --max-save-core=0 --max-save-datadir=1 --force --vardir=/var/tmp/mtr --xml-report=mtr.xml ' su: warning: cannot change directory to /home/mysql: No such file or directory Logging: mysql-test-run.pl --suite=galera,wsrep,galera_3nodes --max-test-fail=0 --testcase-timeout=20 --big-test --retry=3 --parallel=1 --max-save-core=0 --max-save-datadir=1 --force --vardir=/var/tmp/mtr --xml-report=mtr.xml vardir: /var/tmp/mtr Removing old var directory... Creating var directory '/var/tmp/mtr'... Checking supported features... MariaDB Version 10.3.27-10-MariaDB-enterprise-debug - SSL connections supported - binaries are debug compiled - binaries built with wsrep patch Using suites: galera,wsrep,galera_3nodes Collecting tests... Installing system database... ============================================================================== TEST RESULT TIME (ms) or COMMENT -------------------------------------------------------------------------- worker[1] Using MTR_BUILD_THREAD 300, with reserved ports 16000..16019 galera.MW-286 [ disabled ] MDEV-18464 Killing thread can cause mutex deadlock if done concurrently with Galera/replication victim kill galera.MW-328A [ disabled ] MDEV-22666 galera.MW-328A MTR failed: "Semaphore wait has lasted > 600 seconds" and do not release port 16002 galera.MW-328B [ disabled ] MDEV-22666 galera.MW-328A MTR failed: "Semaphore wait has lasted > 600 seconds" and do not release port 16002 galera.MW-329 [ disabled ] MDEV-19962 Galera test failure on MW-329 galera.galera_as_slave_replication_bundle [ disabled ] MDEV-15785 OPTION_GTID_BEGIN is set in Gtid_log_event::do_apply_event() galera.galera_binlog_stmt_autoinc [ disabled ] MDEV-19959 Galera test failure on galera_binlog_stmt_autoinc galera.galera_gcache_recover_manytrx [ disabled ] MDEV-18834 Galera test failure galera.galera_parallel_simple [ disabled ] MDEV-20318 galera.galera_parallel_simple fails galera.galera_shutdown_nonprim [ disabled ] MDEV-21493 galera.galera_shutdown_nonprim galera.galera_ssl_upgrade [ disabled ] MDEV-19950 Galera test failure on galera_ssl_upgrade galera.galera_sst_mariabackup_encrypt_with_key [ disabled ] MDEV-21484 galera_sst_mariabackup_encrypt_with_key galera.galera_var_node_address [ disabled ] MDEV-20485 Galera test failure galera.galera_wan [ disabled ] MDEV-17259 Test failure on galera.galera_wan galera.partition [ disabled ] MDEV-19958 Galera test failure on galera.partition galera.query_cache [ disabled ] MDEV-15805 Test failure on galera.query_cache galera.sql_log_bin [ disabled ] MDEV-21491 galera.sql_log_bin galera_3nodes.galera_gtid_2_cluster [ disabled ] MDEV-23775 Galera test failure on galera_3nodes.galera_gtid_2_cluster galera_3nodes.galera_ist_gcache_rollover [ disabled ] MDEV-23578 WSREP: exception caused by message: {v=0,t=1,ut=255,o=4,s=0,sr=0,as=1,f=6,src=50524cfe,srcvid=view_id(REG,50524cfe,4),insvid=view_id(UNKNOWN,00000000,0),ru=00000000,r=[-1,-1],fs=75,nl=(} galera_3nodes.galera_slave_options_do [ disabled ] MDEV-8798 galera_3nodes.galera_slave_options_ignore [ disabled ] MDEV-8798 galera_3nodes.galera_garbd 'innodb' [ skipped ] Need garbd galera.MW-86-wait1 'innodb' [ skipped ] Test requires Galera debug library with debug_sync functionality galera.galera_log_output_csv 'innodb' [ pass ] 2094 galera.galera_forced_binlog_format 'innodb' [ pass ] 2123 galera.MDEV-22021 'binlogon,innodb' [ pass ] 2070 galera.MW-44 'innodb' [ pass ] 2063 galera.galera_wsrep_new_cluster 'innodb' [ pass ] 2669 galera.galera_var_notify_cmd 'innodb' [ pass ] 2024 galera.GAL-419 'innodb' [ pass ] 10361 galera.galera_bf_lock_wait 'innodb' [ fail ] Test ended at 2021-01-16 07:39:01 CURRENT_TEST: galera.galera_bf_lock_wait Server [mysqld.2 - pid: 11265, winpid: 11265, exit: 256] failed during test run Server log from this test: ----------SERVER LOG START----------- 2021-01-16 7:38:58 0 [Note] WSREP: (7ae2955e, 'tcp://0.0.0.0:16005') turning message relay requesting off mysqld: /home/timofey_turenko_mariadb_com/server/storage/innobase/include/sync0policy.ic:64: void MutexDebug::enter(const Mutex*, const char*, unsigned int) [with Mutex = TTASEventMutex]: Assertion `!is_owned()' failed. 210116 7:39:00 [ERROR] mysqld got signal 6 ; This could be because you hit a bug. It is also possible that this binary or one of the libraries it was linked against is corrupt, improperly built, or misconfigured. This error can also be caused by malfunctioning hardware. To report this bug, see https://mariadb.com/kb/en/reporting-bugs We will try our best to scrape up some info that will hopefully help diagnose the problem, but since we have already crashed, something is definitely wrong and this may fail. Server version: 10.3.27-10-MariaDB-enterprise-debug-log key_buffer_size=1048576 read_buffer_size=131072 max_used_connections=3 max_threads=153 thread_count=11 It is possible that mysqld could use up to key_buffer_size + (read_buffer_size + sort_buffer_size)*max_threads = 63311 K bytes of memory Hope that's ok; if not, decrease some variables in the equation. Thread pointer: 0x7fed54000d50 Attempting backtrace. You can use the following information to find out where mysqld died. If you see no messages after this, something went terribly wrong... stack_bottom = 0x7fed78395cb8 thread_stack 0x49000 mysys/stacktrace.c:174(my_print_stacktrace)[0x564bab76c044] sql/signal_handler.cc:206(handle_fatal_signal)[0x564baafb7d4e] /lib64/libpthread.so.0(+0x132d0)[0x7fed857b82d0] /lib64/libc.so.6(gsignal+0x110)[0x7fed83497520] /lib64/libc.so.6(abort+0x151)[0x7fed83498b01] /lib64/libc.so.6(+0x31b1a)[0x7fed8348fb1a] /lib64/libc.so.6(+0x31b92)[0x7fed8348fb92] include/sync0policy.h:62(MutexDebug >::Context::Context(latch_id_t))[0x564bab1ea185] include/sync0policy.h:345(GenericPolicy >::enter(TTASEventMutex const&, char const*, unsigned int))[0x564bab1ea31c] handler/ha_innodb.cc:5099(innobase_kill_query(handlerton*, THD*, thd_kill_levels))[0x564bab1dcb1e] sql/handler.cc:849(kill_handlerton(THD*, st_plugin_int**, void*))[0x564baafb8a5a] sql/sql_plugin.cc:2436(plugin_foreach_with_mask(THD*, char (*)(THD*, st_plugin_int**, void*), int, unsigned int, void*))[0x564baad42940] sql/handler.cc:855(ha_kill_query(THD*, thd_kill_levels))[0x564baafbb253] sql/sql_class.cc:1851(THD::awake_no_mutex(killed_state))[0x564baacd774d] psi/mysql_thread.h:778(wsrep_thd_awake)[0x564baaf035ee] handler/ha_innodb.cc:18931(wsrep_innobase_kill_one_trx(void*, trx_t const*, trx_t*, unsigned long))[0x564bab1daea1] lock/lock0lock.cc:1126(wsrep_kill_victim(trx_t const*, ib_lock_t const*))[0x564bab2416fc] lock/lock0lock.cc:1168(lock_rec_other_has_conflicting(unsigned long, buf_block_t const*, unsigned long, trx_t const*))[0x564bab24c0d6] lock/lock0lock.cc:1935(lock_rec_lock(bool, unsigned long, buf_block_t const*, unsigned long, dict_index_t*, que_thr_t*))[0x564bab2509ef] lock/lock0lock.cc:5810(lock_clust_rec_read_check_and_lock(unsigned long, buf_block_t const*, unsigned char const*, dict_index_t*, unsigned short const*, lock_mode, unsigned long, que_thr_t*))[0x564bab2519b2] row/row0sel.cc:1250(sel_set_rec_lock(btr_pcur_t*, unsigned char const*, dict_index_t*, unsigned short const*, unsigned long, unsigned long, que_thr_t*, mtr_t*))[0x564bab3737b7] row/row0sel.cc:5094(row_search_mvcc(unsigned char*, page_cur_mode_t, row_prebuilt_t*, unsigned long, unsigned long))[0x564bab3793f1] handler/ha_innodb.cc:9281(ha_innobase::index_read(unsigned char*, unsigned char const*, unsigned int, ha_rkey_function))[0x564bab1d9070] handler/ha_innodb.cc:9783(ha_innobase::rnd_pos(unsigned char*, unsigned char*))[0x564bab1bcd60] sql/handler.cc:2952(handler::ha_rnd_pos(unsigned char*, unsigned char*))[0x564baafc026b] sql/handler.h:3632(handler::rnd_pos_by_record(unsigned char*))[0x564baafcdcb0] sql/sql_class.h:6456(handler::ha_rnd_pos_by_record(unsigned char*))[0x564bab10bb93] sql/log_event.cc:14641(Update_rows_log_event::do_exec_row(rpl_group_info*))[0x564bab10cef1] sql/log_event.cc:11611(Rows_log_event::do_apply_event(rpl_group_info*))[0x564bab0fa10b] sql/log_event.h:1484(Log_event::apply_event(rpl_group_info*))[0x564baaf0d34f] src/trx_handle.cpp:312(galera::TrxHandle::apply(void*, wsrep_cb_status (*)(void*, void const*, unsigned long, unsigned int, wsrep_trx_meta const*), wsrep_trx_meta const&) const)[0x7fed79e6cf28] src/replicator_smm.cpp:92(apply_trx_ws(void*, wsrep_cb_status (*)(void*, void const*, unsigned long, unsigned int, wsrep_trx_meta const*), wsrep_cb_status (*)(void*, unsigned int, wsrep_trx_meta const*, bool*, bool), galera::TrxHandle const&, wsrep_trx_meta const&))[0x7fed79eba66f] src/replicator_smm.cpp:458(galera::ReplicatorSMM::apply_trx(void*, galera::TrxHandle*))[0x7fed79ebde15] src/replicator_smm.cpp:1258(galera::ReplicatorSMM::process_trx(void*, galera::TrxHandle*))[0x7fed79ec08d6] src/gcs_action_source.cpp:116(galera::GcsActionSource::dispatch(void*, gcs_action const&, bool&))[0x7fed79e98408] src/gcs_action_source.cpp:28(Release::~Release())[0x7fed79e99548] src/replicator_smm.cpp:362(galera::ReplicatorSMM::async_recv(void*))[0x7fed79ec108d] src/wsrep_provider.cpp:271(galera_recv)[0x7fed79edb628] sql/wsrep_thd.cc:470(wsrep_replication_process(THD*))[0x564baaf0e1bc] sql/wsrep_mysqld.cc:2160(start_wsrep_THD)[0x564baaefed05] perfschema/pfs.cc:1872(pfs_spawn_thread)[0x564bab711797] /lib64/libpthread.so.0(+0x84f9)[0x7fed857ad4f9] /lib64/libc.so.6(clone+0x3f)[0x7fed83559fbf] Trying to get some variables. Some pointers may be invalid and cause the dump to abort. Query (0x7fed6c005c6b): update t1 set b=connection_id() where a=1 Connection ID (thread ID): 2 Status: NOT_KILLED Optimizer switch: index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_merge_sort_intersection=off,engine_condition_pushdown=off,index_condition_pushdown=on,derived_merge=on,derived_with_keys=on,firstmatch=on,loosescan=on,materialization=on,in_to_exists=on,semijoin=on,partial_match_rowid_merge=on,partial_match_table_scan=on,subquery_cache=on,mrr=off,mrr_cost_based=off,mrr_sort_keys=off,outer_join_with_cache=on,semijoin_with_cache=on,join_cache_incremental=on,join_cache_hashed=on,join_cache_bka=on,optimize_join_buffer_size=off,table_elimination=on,extended_keys=on,exists_to_in=on,orderby_uses_equalities=on,condition_pushdown_for_derived=on,split_materialized=on The manual page at https://mariadb.com/kb/en/how-to-produce-a-full-stack-trace-for-mysqld/ contains information that should help you find out what is causing the crash. Writing a core file... Working directory at /var/tmp/mtr/mysqld.2/data Resource Limits: Limit Soft Limit Hard Limit Units Max cpu time unlimited unlimited seconds Max file size unlimited unlimited bytes Max data size unlimited unlimited bytes Max stack size 8388608 unlimited bytes Max core file size unlimited unlimited bytes Max resident set unlimited unlimited bytes Max processes 31642 31642 processes Max open files 1024 1024 files Max locked memory 65536 65536 bytes Max address space unlimited unlimited bytes Max file locks unlimited unlimited locks Max pending signals 31642 31642 signals Max msgqueue size 819200 819200 bytes Max nice priority 0 0 Max realtime priority 0 0 Max realtime timeout unlimited unlimited us Core pattern: |/bin/false ----------SERVER LOG END------------- mysqltest failed but provided no output The result from queries just before the failure was: < snip > update t1 set b=connection_id() where a=1; commit; SET current_num = current_num + 1; END WHILE; END| connection node_2; connect node_1_p1, 127.0.0.1, root, , test, $NODE_MYPORT_1; SET SESSION wsrep_sync_wait=0; call p1(1000); connect node_1_p2, 127.0.0.1, root, , test, $NODE_MYPORT_1; SET SESSION wsrep_sync_wait=0; call p1(1000); connect node_2_p1, 127.0.0.1, root, , test, $NODE_MYPORT_2; SET SESSION wsrep_sync_wait=0; call p1(1000); connect node_2_p2, 127.0.0.1, root, , test, $NODE_MYPORT_2; SET SESSION wsrep_sync_wait=0; call p1(1000); connection node_1; checking error log for 'BF lock wait long' message for 10 times every 10 seconds ... - saving '/var/tmp/mtr/log/galera.galera_bf_lock_wait-innodb/' to '/var/tmp/mtr/log/galera.galera_bf_lock_wait-innodb/' Retrying test galera.galera_bf_lock_wait, attempt(2/3)... ***Warnings generated in error logs during shutdown after running tests: galera.galera_bf_lock_wait galera.GAL-419 mysqld: /home/timofey_turenko_mariadb_com/server/storage/innobase/include/sync0policy.ic:64: void MutexDebug::enter(const Mutex*, const char*, unsigned int) [with Mutex = TTASEventMutex]: Assertion `!is_owned()' failed. Attempting backtrace. You can use the following information to find out galera.galera_bf_lock_wait 'innodb' [ retry-pass ] 102151 Retrying test galera.galera_bf_lock_wait, attempt(3/3)... galera.galera_fulltext 'innodb' [ pass ] 7626 galera.galera_bf_lock_wait 'innodb' [ retry-pass ] 102130 galera.galera_kill_largechanges 'innodb' [ pass ] 128533 galera.galera_many_columns 'innodb' [ pass ] 2524 galera.galera_many_tables_nopk 'innodb' [ pass ] 42306 galera.galera_many_tables_pk 'innodb' [ pass ] 6350 galera.galera_parallel_autoinc_largetrx 'innodb' [ pass ] 6918 galera.galera_parallel_autoinc_manytrx 'innodb' [ pass ] 20302 galera.galera_rsu_add_pk 'innodb' [ pass ] 32827 galera.galera_rsu_drop_pk 'innodb' [ pass ] 62464 galera.galera_toi_ddl_error 'innodb' [ pass ] 10150 galera.galera_toi_ddl_fk_insert 'innodb' [ pass ] 7428 galera.galera_toi_ddl_fk_update 'innodb' [ pass ] 7184 galera.galera_var_load_data_splitting 'innodb' [ pass ] 11137 galera.galera_wsrep_desync_wsrep_on 'innodb' [ pass ] 27705 galera.GAL-382 'innodb' [ pass ] 2095 galera.GAL-401 'innodb' [ pass ] 17264 galera.GAL-480 'innodb' [ pass ] 2689 galera.MDEV-20225 'innodb' [ skipped ] Test requires Galera debug library with debug_sync functionality galera.MDEV-22021 'binlogoff,innodb' [ pass ] 2066 galera.MDEV-22707 'innodb' [ skipped ] Test requires Galera debug library with debug_sync functionality galera.MW-252 'innodb' [ pass ] 2799 galera.MW-258 'innodb' [ pass ] 10075 galera.MW-259 'innodb' [ pass ] 4155 galera.MW-285 'innodb' [ pass ] 2130 galera.MW-292 'innodb' [ skipped ] Test requires Galera debug library with debug_sync functionality galera.MW-309 'innodb' [ pass ] 2075 galera.MW-328D 'innodb' [ pass ] 4075 galera.MW-328E 'innodb' [ pass ] 4080 galera.MW-357 'innodb' [ pass ] 2047 galera.MW-369 'innodb' [ skipped ] Test requires Galera debug library with debug_sync functionality galera.MW-388 'innodb' [ skipped ] Test requires Galera debug library with debug_sync functionality galera.MW-402 'innodb' [ skipped ] Test requires Galera debug library with debug_sync functionality galera.MW-416 'innodb' [ pass ] 2045 galera.basic 'innodb' [ pass ] 2073 galera.binlog_checksum 'innodb' [ pass ] 2075 galera.create 'innodb' [ pass ] 2271 galera.enforce_storage_engine 'innodb' [ pass ] 2063 galera.ev51914 'innodb' [ pass ] 2566 galera.fk 'innodb' [ pass ] 2257 galera.galera#500 'innodb' [ skipped ] Test requires Galera debug library with debug_sync functionality galera.galera#505 'innodb' [ pass ] 2017 galera.galera_FK_duplicate_client_insert 'innodb' [ pass ] 3022 galera.galera_admin 'innodb' [ pass ] 3731 galera.galera_alter_engine_innodb 'innodb' [ pass ] 2091 galera.galera_alter_engine_myisam 'innodb' [ pass ] 2075 galera.galera_alter_table_force 'innodb' [ pass ] 2076 galera.galera_applier_ftwrl_table 'innodb' [ pass ] 2077 galera.galera_bf_abort 'innodb' [ pass ] 2054 galera.galera_bf_abort_flush_for_export 'innodb' [ pass ] 2072 galera.galera_bf_abort_for_update 'innodb' [ pass ] 2062 galera.galera_bf_abort_ftwrl 'innodb' [ pass ] 2055 galera.galera_bf_abort_get_lock 'innodb' [ pass ] 3056 galera.galera_bf_abort_lock_table 'innodb' [ pass ] 2057 galera.galera_bf_abort_sleep 'innodb' [ pass ] 2051 galera.galera_binlog_cache_size 'innodb' [ pass ] 3119 galera.galera_binlog_row_image 'innodb' [ pass ] 2238 galera.galera_concurrent_ctas 'innodb' [ pass ] 6652 galera.galera_create_function 'innodb' [ pass ] 2072 galera.galera_create_procedure 'innodb' [ pass ] 2101 galera.galera_create_table_like 'innodb' [ pass ] 2174 galera.galera_create_trigger 'innodb' [ pass ] 2230 galera.galera_ddl_multiline 'innodb' [ pass ] 2251 galera.galera_defaults 'innodb' [ pass ] 2028 galera.galera_delete_limit 'innodb' [ pass ] 2159 galera.galera_desync_overlapped 'innodb' [ pass ] 3290 galera.galera_drop_database 'innodb' [ pass ] 10120 galera.galera_drop_multi 'innodb' [ pass ] 2107 galera.galera_enum 'innodb' [ pass ] 2123 galera.galera_events 'innodb' [ pass ] 3062 galera.galera_events2 'innodb' [ pass ] 17468 galera.galera_fk_cascade_delete 'innodb' [ pass ] 2234 galera.galera_fk_cascade_delete_debug 'innodb' [ pass ] 2191 galera.galera_fk_cascade_update 'innodb' [ pass ] 2125 galera.galera_fk_conflict 'innodb' [ pass ] 2087 galera.galera_fk_mismatch 'innodb' [ pass ] 2167 galera.galera_fk_multicolumn 'innodb' [ pass ] 2124 galera.galera_fk_multitable 'innodb' [ pass ] 2097 galera.galera_fk_no_pk 'innodb' [ pass ] 2095 galera.galera_fk_selfreferential 'innodb' [ pass ] 2056 galera.galera_fk_setnull 'innodb' [ pass ] 2096 galera.galera_flush_local 'innodb' [ skipped ] Test requires: 'have_wsrep_replicate_myisam' galera.galera_ftwrl 'innodb' [ pass ] 4059 galera.galera_ftwrl_drain 'innodb' [ skipped ] Test requires Galera debug library with debug_sync functionality galera.galera_gcs_fc_limit 'innodb' [ pass ] 3096 galera.galera_gra_log 'innodb' [ pass ] 2064 galera.galera_insert_ignore 'innodb' [ pass ] 2165 galera.galera_insert_multi 'innodb' [ pass ] 2221 galera.galera_kill_applier 'innodb' [ pass ] 2057 galera.galera_kill_ddl 'innodb' [ pass ] 21450 galera.galera_kill_nochanges 'innodb' [ pass ] 17393 galera.galera_kill_smallchanges 'innodb' [ pass ] 19461 galera.galera_lock_table 'innodb' [ pass ] 4094 galera.galera_many_indexes 'innodb' [ pass ] 3338 galera.galera_mdev_10812 'innodb' [ pass ] 6057 galera.galera_mdl_race 'innodb' [ pass ] 2094 galera.galera_multi_database 'innodb' [ pass ] 2110 galera.galera_myisam_autocommit 'innodb' [ pass ] 2065 galera.galera_myisam_transactions 'innodb' [ pass ] 2081 galera.galera_nopk_bit 'innodb' [ pass ] 2096 galera.galera_nopk_blob 'innodb' [ pass ] 2095 galera.galera_nopk_large_varchar 'innodb' [ pass ] 2094 galera.galera_nopk_unicode 'innodb' [ pass ] 2074 galera.galera_parallel_apply_lock_table 'innodb' [ pass ] 4099 galera.galera_pc_ignore_sb 'innodb' [ pass ] 21356 galera.galera_performance_schema 'innodb' [ pass ] 2112 galera.galera_pk_bigint_signed 'innodb' [ pass ] 2060 galera.galera_pk_bigint_unsigned 'innodb' [ pass ] 2057 galera.galera_prepared_statement 'innodb' [ pass ] 2224 galera.galera_read_only 'innodb' [ pass ] 2067 galera.galera_repl_key_format_flat16 'innodb' [ pass ] 2085 galera.galera_repl_max_ws_size 'innodb' [ pass ] 2049 galera.galera_restart_nochanges 'innodb' [ pass ] 8479 galera.galera_restart_on_unknown_option 'innodb' [ pass ] 80414 galera.galera_roles 'innodb' [ pass ] 5335 galera.galera_rsu_error 'innodb' [ pass ] 2079 galera.galera_rsu_simple 'innodb' [ pass ] 2062 galera.galera_rsu_wsrep_desync 'innodb' [ pass ] 4207 galera.galera_sbr 'innodb' [ pass ] 2058 galera.galera_schema_dirty_reads 'innodb' [ pass ] 2014 galera.galera_serializable 'innodb' [ pass ] 6079 galera.galera_server 'innodb' [ pass ] 2040 galera.galera_sp_bf_abort 'innodb' [ skipped ] Test requires Galera debug library with debug_sync functionality galera.galera_split_brain 'innodb' [ pass ] 31446 galera.galera_sql_log_bin_zero 'innodb' [ pass ] 2053 galera.galera_status_cluster 'innodb' [ pass ] 2016 galera.galera_status_local_index 'innodb' [ pass ] 2056 galera.galera_status_local_state 'innodb' [ pass ] 2018 galera.galera_suspend_slave 'innodb' [ pass ] 26470 galera.galera_sync_wait_show 'innodb' [ pass ] 2148 galera.galera_toi_alter_auto_increment 'innodb' [ pass ] 4202 galera.galera_toi_ddl_locking 'innodb' [ pass ] 5091 galera.galera_toi_ddl_nonconflicting 'innodb' [ pass ] 2083 galera.galera_toi_ddl_sequential 'innodb' [ pass ] 2099 galera.galera_toi_drop_database 'innodb' [ pass ] 52576 galera.galera_toi_ftwrl 'innodb' [ pass ] 2073 galera.galera_toi_lock_exclusive 'innodb' [ pass ] 3076 galera.galera_toi_lock_shared 'innodb' [ pass ] 2067 galera.galera_toi_truncate 'innodb' [ pass ] 2112 galera.galera_transaction_read_only 'innodb' [ pass ] 2047 galera.galera_transaction_replay 'innodb' [ skipped ] Test requires Galera debug library with debug_sync functionality galera.galera_trigger 'innodb' [ pass ] 2077 galera.galera_truncate 'innodb' [ pass ] 2258 galera.galera_truncate_temporary 'innodb' [ pass ] 2178 galera.galera_unicode_identifiers 'innodb' [ pass ] 2181 galera.galera_unicode_pk 'innodb' [ pass ] 2061 galera.galera_update_limit 'innodb' [ pass ] 2125 galera.galera_var_OSU_method 'innodb' [ pass ] 2160 galera.galera_var_OSU_method2 'innodb' [ pass ] 2165 galera.galera_var_auto_inc_control_off 'innodb' [ pass ] 4063 galera.galera_var_certify_nonPK_off 'innodb' [ pass ] 2089 galera.galera_var_cluster_address 'innodb' [ pass ] 10170 galera.galera_var_desync_on 'innodb' [ pass ] 3146 galera.galera_var_dirty_reads 'innodb' [ pass ] 8186 galera.galera_var_fkchecks 'innodb' [ pass ] 2130 galera.galera_var_innodb_disallow_writes 'innodb' [ pass ] 12088 galera.galera_var_max_ws_rows 'innodb' [ pass ] 7793 galera.galera_var_max_ws_size 'innodb' [ pass ] 2046 galera.galera_var_mysql_replication_bundle 'innodb' [ pass ] 2051 galera.galera_var_reject_queries 'innodb' [ pass ] 4056 galera.galera_var_replicate_myisam_off 'innodb' [ pass ] 2045 galera.galera_var_replicate_myisam_on 'innodb' [ pass ] 2282 galera.galera_var_retry_autocommit 'innodb' [ skipped ] Test requires Galera debug library with debug_sync functionality galera.galera_var_slave_threads 'innodb' [ pass ] 2588 galera.galera_var_sync_wait 'innodb' [ pass ] 2119 galera.galera_var_wsrep_on_off 'innodb' [ pass ] 2059 galera.galera_wsrep_provider_options_syntax 'innodb' [ pass ] 2023 galera.galera_wsrep_provider_unset_set 'innodb' [ pass ] 11213 galera.galera_zero_length_column 'innodb' [ pass ] 2113 galera.grant 'innodb' [ pass ] 2036 galera.lp1276424 'innodb' [ pass ] 2054 galera.lp1347768 'innodb' [ pass ] 2055 galera.lp1376747 'innodb' [ pass ] 4113 galera.lp1376747-2 'innodb' [ pass ] 2097 galera.lp1376747-3 'innodb' [ pass ] 2100 galera.lp1376747-4 'innodb' [ pass ] 3199 galera.lp1438990 'innodb' [ pass ] 2150 galera.lp959512 'innodb' [ pass ] 2095 galera.mdev-22543 'innodb' [ pass ] 20009 galera.mdev_9290 'innodb' [ pass ] 2045 galera.mysql-wsrep#110 'innodb' [ pass ] 2144 galera.mysql-wsrep#198 'innodb' [ pass ] 2210 galera.mysql-wsrep#237 'innodb' [ pass ] 3080 galera.mysql-wsrep#247 'innodb' [ pass ] 3039 galera.mysql-wsrep#31 'innodb' [ pass ] 10690 galera.mysql-wsrep#332 'innodb' [ skipped ] Test requires Galera debug library with debug_sync functionality galera.mysql-wsrep#90 'innodb' [ pass ] 2327 galera.pxc-421 'innodb' [ pass ] 11253 galera.rename 'innodb' [ pass ] 2095 galera.unique_key 'innodb' [ pass ] 2143 galera.view 'innodb' [ pass ] 2171 galera.MDEV-15443 'innodb' [ pass ] 13078 galera.MW-284 'innodb' [ pass ] 10508 galera.MW-313 'innodb' [ pass ] 2086 galera.MW-336 'innodb' [ pass ] 2339 galera.MW-86-wait8 'innodb' [ skipped ] Test requires Galera debug library with debug_sync functionality galera.enforce_storage_engine2 'innodb' [ pass ] 2062 galera.galera#414 'innodb' [ pass ] 8445 galera.galera_applier_ftwrl_table_alter 'innodb' [ pass ] 8083 galera.galera_as_master 'innodb' [ pass ] 2582 galera.galera_as_master_large 'innodb' [ pass ] 2097 galera.galera_as_slave_autoinc 'innodb' [ pass ] 2535 galera.galera_as_slave_ctas 'innodb' [ pass ] 2528 galera.galera_as_slave_gtid 'innodb' [ pass ] 3162 galera.galera_as_slave_gtid_myisam 'innodb' [ pass ] 3527 galera.galera_as_slave_nonprim 'innodb' [ pass ] 16615 galera.galera_autoinc_sst_mariabackup 'innodb' [ pass ] 27498 galera.galera_bf_background_statistics 'innodb' [ pass ] 4569 galera.galera_binlog_checksum 'innodb' [ pass ] 2084 galera.galera_binlog_event_max_size_max 'innodb' [ pass ] 22347 galera.galera_binlog_event_max_size_min 'innodb' [ pass ] 2046 galera.galera_encrypt_tmp_files 'innodb' [ pass ] 2227 galera.galera_gcache_recover 'innodb' [ pass ] 18890 galera.galera_gcache_recover_full_gcache 'innodb' [ pass ] 24707 galera.galera_gcs_fragment 'innodb' [ skipped ] Test requires Galera debug library with debug_sync functionality galera.galera_gcs_max_packet_size 'innodb' [ pass ] 3315 galera.galera_gtid 'innodb' [ pass ] 2054 galera.galera_gtid_slave 'innodb' [ pass ] 2554 galera.galera_gtid_slave_sst_rsync 'innodb' [ pass ] 17260 galera.galera_ist_mariabackup 'debug,innodb' [ pass ] 57959 galera.galera_ist_mariabackup_innodb_flush_logs 'debug,innodb' [ pass ] 39720 galera.galera_ist_mysqldump 'debug,innodb' [ pass ] 131289 galera.galera_ist_progress 'innodb' [ pass ] 15648 galera.galera_ist_recv_bind 'innodb' [ pass ] 12398 galera.galera_ist_restart_joiner 'innodb' [ skipped ] Test requires Galera debug library with debug_sync functionality galera.galera_ist_rsync 'debug,innodb' [ pass ] 46300 galera.galera_load_data 'innodb' [ pass ] 2205 galera.galera_log_bin 'innodb' [ pass ] 2598 galera.galera_many_rows 'innodb' [ pass ] 76424 galera.galera_mdev_13787 'innodb' [ pass ] 2054 galera.galera_mdev_15611 'innodb' [ pass ] 3000 galera.galera_partition 'innodb' [ pass ] 11083 galera.galera_query_cache 'innodb' [ pass ] 2108 galera.galera_query_cache_sync_wait 'innodb' [ skipped ] Test requires Galera debug library with debug_sync functionality galera.galera_sbr_binlog 'innodb' [ pass ] 2050 galera.galera_slave_replay 'innodb' [ skipped ] Test requires Galera debug library with debug_sync functionality galera.galera_ssl 'innodb' [ pass ] 2053 galera.galera_ssl_compression 'innodb' [ pass ] 2066 galera.galera_sst_mariabackup 'debug,innodb' [ pass ] 93793 galera.galera_sst_mariabackup_data_dir 'debug,innodb' [ pass ] 93445 galera.galera_sst_mariabackup_lost_found 'innodb' [ pass ] 22993 galera.galera_sst_mariabackup_table_options '16k,clear,innodb' [ pass ] 23309 galera.galera_sst_mariabackup_table_options '16k,crypt,innodb' [ pass ] 23305 galera.galera_sst_mariabackup_table_options '4k,clear,innodb' [ pass ] 23166 galera.galera_sst_mariabackup_table_options '4k,crypt,innodb' [ pass ] 23396 galera.galera_sst_mariabackup_table_options '8k,clear,innodb' [ pass ] 23314 galera.galera_sst_mariabackup_table_options '8k,crypt,innodb' [ pass ] 23234 galera.galera_sst_mysqldump 'debug,innodb' [ pass ] 210889 galera.galera_sst_mysqldump_with_key 'debug,innodb' [ pass ] 130518 galera.galera_sst_rsync 'debug,innodb' [ pass ] 47884 galera.galera_sst_rsync2 'debug,innodb' [ pass ] 46851 galera.galera_sst_rsync_data_dir 'debug,innodb' [ pass ] 47913 galera.galera_var_auto_inc_control_on 'innodb' [ pass ] 2071 galera.galera_var_gtid_domain_id 'innodb' [ pass ] 2633 galera.galera_var_log_bin 'innodb' [ pass ] 2054 galera.galera_var_sst_auth 'innodb' [ pass ] 2012 galera.galera_wan_restart_ist 'innodb' [ pass ] 31091 galera.galera_wan_restart_sst 'innodb' [ pass ] 60668 galera.galera_wsrep_log_conficts 'innodb' [ pass ] 2091 galera.mdev_10518 'innodb' [ pass ] 2736 galera.mysql-wsrep#201 'innodb' [ pass ] 4683 galera.mysql-wsrep#33 'innodb' [ pass ] 50013 galera.rpl_row_annotate 'innodb' [ pass ] 2065 galera.versioning_trx_id 'innodb' [ pass ] 2073 galera_3nodes.galera_certification_ccc 'innodb' [ pass ] 20077 galera_3nodes.galera_certification_double_failure 'innodb' [ pass ] 3128 galera_3nodes.galera_evs_suspect_timeout 'innodb' [ pass ] 17827 galera_3nodes.galera_join_with_cc_A 'innodb' [ skipped ] Test requires Galera debug library with debug_sync functionality galera_3nodes.galera_join_with_cc_B 'innodb' [ skipped ] Test requires Galera debug library with debug_sync functionality galera_3nodes.galera_join_with_cc_C 'innodb' [ skipped ] Test requires Galera debug library with debug_sync functionality galera_3nodes.galera_parallel_apply_3nodes 'innodb' [ pass ] 3059 galera_3nodes.galera_var_dirty_reads2 'innodb' [ pass ] 20583 galera_3nodes.GAL-501 'innodb' [ pass ] 14495 galera_3nodes.galera_ipv6_mariabackup 'innodb' [ pass ] 13687 galera_3nodes.galera_ipv6_mariabackup_section 'innodb' [ pass ] 13383 galera_3nodes.galera_ipv6_mysqldump 'innodb' [ pass ] 10640 galera_3nodes.galera_ipv6_rsync 'innodb' [ pass ] 13418 galera_3nodes.galera_ipv6_rsync_section 'innodb' [ pass ] 14457 galera_3nodes.galera_load_data_ist 'innodb' [ pass ] 26209 galera_3nodes.galera_pc_bootstrap 'innodb' [ pass ] 38791 galera_3nodes.galera_pc_weight 'innodb' [ pass ] 18383 galera_3nodes.galera_safe_to_bootstrap 'innodb' [ pass ] 53387 wsrep.pool_of_threads 'innodb' [ pass ] 8 wsrep.foreign_key 'innodb' [ pass ] 85 wsrep.binlog_format [ pass ] 30 wsrep.mdev_6832 [ pass ] 4035 wsrep.mdev_7798 [ pass ] 3432 wsrep.mysql_tzinfo_to_sql_symlink 'innodb' [ pass ] 41 wsrep.mysql_tzinfo_to_sql_symlink_skip 'innodb' [ pass ] 37 wsrep.trans 'innodb' [ pass ] 3 wsrep.plugin [ pass ] 4 wsrep.alter_table_innodb 'innodb' [ pass ] 3741 wsrep.mdev_10186 [ pass ] 2 wsrep.variables 'innodb' [ skipped ] Test requires Galera debug library with debug_sync functionality wsrep.wsrep-recover-v25 'binlogon,innodb' [ pass ] 12243 wsrep.wsrep-recover-v25 'binlogoff,innodb' [ pass ] 12094 wsrep.wsrep_rpl 'innodb,mix' [ pass ] 1001 wsrep.wsrep_rpl 'innodb,row' [ pass ] 1111 wsrep.wsrep_rpl 'innodb,stmt' [ pass ] 1042 -------------------------------------------------------------------------- The servers were restarted 112 times Spent 3319.030 of 4938 seconds executing testcases Completed: Failed 1/284 tests, 99.65% were successful. Failing test(s): galera.galera_bf_lock_wait The log files in var/log may give you some hint of what went wrong. If you want to report this error, please read first the documentation at http://dev.mysql.com/doc/mysql/en/mysql-test-suite.html Errors/warnings were found in logfiles during server shutdown after running the following sequence(s) of tests: galera.galera_bf_lock_wait galera.GAL-419 24 tests were skipped, 23 by the test itself. mysql-test-run: *** ERROR: there were failing test cases + MTR_RETCODE=1 + [[ 1 -ne 0 ]] + tar czf logs.tar.gz /var/tmp/mtr tar: Removing leading `/' from member names + sudo ls -la /tmp/ total 12 drwxrwxrwt 10 root root 236 Jan 16 08:57 . drwxr-xr-x 21 root root 274 Jan 16 07:35 .. drwxrwxrwt 2 root root 6 Jan 16 07:28 .ICE-unix drwxrwxrwt 2 root root 6 Jan 16 07:28 .Test-unix drwxrwxrwt 2 root root 6 Jan 16 07:28 .X11-unix drwxrwxrwt 2 root root 6 Jan 16 07:28 .XIM-unix drwxrwxrwt 2 root root 6 Jan 16 07:28 .font-unix -rw-r--r-- 1 root root 0 Jan 16 07:36 chef-solo drwxrwxrwx 2 mysql users 17 Jan 16 07:36 mysql-unique-ids drwxrwx--x 2 mysql users 8192 Jan 16 08:53 sst_log_archive drwx------ 3 root root 17 Jan 16 07:35 systemd-private-556716a808ab455d80939121f56b4df0-chronyd.service-KenWGE + sudo mv -vf logs.tar.gz /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/../ renamed 'logs.tar.gz' -> '/home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/../logs.tar.gz' + sudo mv -vf /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/mtr.xml /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/../ renamed '/home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/mtr.xml' -> '/home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/../mtr.xml' + sudo chmod 777 /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/../COPYING /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/../CREDITS /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/../EXCEPTIONS-CLIENT /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/../INSTALL-BINARY /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/../README.md /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/../THIRDPARTY /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/../VERSION /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/../bin /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/../data /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/../etc /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/../include /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/../lib /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/../logs.tar.gz /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/../man /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/../mtr.xml /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/../mysql-test /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/../scripts /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/../share /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/../sql-bench /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mysql-test/../support-files + exit 1 -rwxrwxrwx 1 mysql users 43897 Jan 16 08:59 /home/timofey_turenko_mariadb_com/MariaDBEnterprise/mtr.xml