The customer reports that an attempt to join crashed node to the cluster through IST ends with another crash after wsrep_recover. Wsrep thread did not wait that recover thread to
finish the job, but initiate IST procedure and crashes again - backtraces are below.
And full error log is in the case. Platform is FreeBSD.
2022-05-15 1:55:21 96 [Note] WSREP: Starting applier thread 96
|
2022-05-15 1:55:21 97 [Note] WSREP: Starting applier thread 97
|
2022-05-15 1:55:21 98 [Note] WSREP: Starting applier thread 98
|
2022-05-15 1:55:21 101 [Note] WSREP: Starting applier thread 101
|
2022-05-15 1:55:21 99 [Note] WSREP: Starting applier thread 99
|
2022-05-15 1:55:21 0 [Note] /usr/local/libexec/mariadbd: ready for connections.
|
Version: '10.6.8-4-MariaDB-enterprise-log' socket: '/tmp/mysql.sock' port: 3306 MariaDB Enterprise Server
|
2022-05-15 1:55:21 103 [Note] WSREP: Starting applier thread 103
|
0x13ef71e <my_print_stacktrace+0x2e> at /usr/local/libexec/mariadbd
|
mysys/my_addr_resolve.c:299(my_addr_resolve)[0xce6460]
|
0x801935580 <pthread_sigmask+0x540> at /lib/libthr.so.3
|
thread/thr_sig.c:0(handle_signal)[0x801934b3f]
|
0x7ffffffff2d3 <__gxx_personality_v0+0x7ffffeb6ae03> at ???
|
0xd9ec24 <thd_get_thread_id+0x4> at /usr/local/libexec/mariadbd
|
mysys/my_addr_resolve.c:299(my_addr_resolve)[0x12b9590]
|
0x12b8f9b <_Z9lock_waitP9que_thr_t+0x5b> at /usr/local/libexec/mariadbd
|
0x132f9b1 <_Z23row_mysql_handle_errorsP7dberr_tP5trx_tP9que_thr_tP12trx_savept_t+0x61> at /usr/local/libexec/mariadbd
|
sql/mysqld.cc:1848(unireg_abort)[0x13497a6]
|
maria/ma_check.c:3055(writekeys)[0x11a7891]
|
maria/ma_check.c:3159(_ma_flush_table_files_before_swap)[0xbc80e2]
|
0xbd3d29 <_ZN7handler17rnd_pos_by_recordEPh+0x59> at /usr/local/libexec/mariadbd
|
perfschema/pfs_stat.h:76(PFS_single_stat::aggregate(PFS_single_stat const*))[0xc97b40]
|
0xc98b70 <_ZN21Update_rows_log_event11do_exec_rowEP14rpl_group_info+0x190> at /usr/local/libexec/mariadbd
|
perfschema/table_events_statements.cc:239(table_events_statements_common::make_row_part_1(PFS_events_statements*, sql_digest_storage*))[0xc92f88]
|
0xd351b8 <_ZN9Log_event11apply_eventEP14rpl_group_info+0x68> at /usr/local/libexec/mariadbd
|
0x1189d12 <_Z18wsrep_apply_eventsP3THDP14Relay_log_infoPKvm+0x392> at /usr/local/libexec/mariadbd
|
sql/sys_vars.inl:508(Sys_var_charptr_base)[0x116ba87]
|
0x147df0c <_ZN5wsrep12server_state8on_applyERNS_21high_priority_serviceERKNS_9ws_handleERKNS_7ws_metaERKNS_12const_bufferE+0x3cc> at /usr/local/libexec/mariadbd
|
0x1489eef <_ZN12_GLOBAL__N_18apply_cbEPvPK15wsrep_ws_handlejPK9wsrep_bufPK14wsrep_trx_metaPb+0xaf> at /usr/local/libexec/mariadbd
|
0x80391b690 <wsrep_ps_free_node_stat+0x9510> at /rw_part/usr-local/lib/mysql/libgalera_enterprise_smm.so
|
src/trx_handle.cpp:392(galera::TrxHandleSlave::apply(void*, wsrep_cb_status (*)(void*, wsrep_ws_handle const*, unsigned int, wsrep_buf const*, wsrep_trx_meta const*, bool*), wsrep_trx_meta const&, bool&))[0x80392143a]
|
0x803969b00 <wsrep_ps_free_node_stat+0x57980> at /rw_part/usr-local/lib/mysql/libgalera_enterprise_smm.so
|
0x803969441 <wsrep_ps_free_node_stat+0x572c1> at /rw_part/usr-local/lib/mysql/libgalera_enterprise_smm.so
|
0x803920847 <wsrep_ps_free_node_stat+0xe6c7> at /rw_part/usr-local/lib/mysql/libgalera_enterprise_smm.so
|
src/replicator_smm.cpp:538(galera::ReplicatorSMM::apply_trx(void*, galera::TrxHandleSlave&))[0x80390b001]
|
0x148a63b <_ZN5wsrep18wsrep_provider_v2611run_applierEPNS_21high_priority_serviceE+0xb> at /usr/local/libexec/mariadbd
|
mysys/my_addr_resolve.c:299(my_addr_resolve)[0x118a20d]
|
0x117a083 <_Z15start_wsrep_THDPv+0x2e3> at /usr/local/libexec/mariadbd
|
0x110c177 <pfs_spawn_thread+0xd7> at /usr/local/libexec/mariadbd
|
|
Trying to get some variables.
|
Some pointers may be invalid and cause the dump to abort.
|
Query (0x1d02204e5b): UPDATE nl_game_providers.game_bettings SET ext_bet_status = 'Won' WHERE provider_id = '2' AND recno = '13076845542'
|
|
Connection ID (thread ID): 37
|
Status: NOT_KILLED
|
|
Optimizer switch: index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_merge_sort_intersection=off,engine_condition_pushdown=off,index_condition_pushdown=on,derived_merge=on,derived_with_keys=on,firstmatch=on,loosescan=on,materialization=on,in_to_exists=on,semijoin=on,partial_match_rowid_merge=on,partial_match_table_scan=on,subquery_cache=on,mrr=off,mrr_cost_based=off,mrr_sort_keys=off,outer_join_with_cache=on,semijoin_with_cache=on,join_cache_incremental=on,join_cache_hashed=on,join_cache_bka=on,optimize_join_buffer_size=on,table_elimination=on,extended_keys=on,exists_to_in=on,orderby_uses_equalities=on,condition_pushdown_for_derived=on,split_materialized=on,condition_pushdown_for_subquery=on,rowid_filter=on,condition_pushdown_from_having=on,not_null_range_scan=off
|
|
The manual page at https://mariadb.com/kb/en/how-to-produce-a-full-stack-trace-for-mysqld/ contains
|
information that should help you find out what is causing the crash.
|
Core pattern: %N.core
|
There are no comments yet on this issue.
{"report":{"fcp":1543.5,"ttfb":682.8999996185303,"pageVisibility":"visible","entityId":111078,"key":"jira.project.issue.view-issue","isInitial":true,"threshold":1000,"elementTimings":{},"userDeviceMemory":8,"userDeviceProcessors":64,"apdex":0.5,"journeyId":"703e1b55-9063-4643-b37e-ebeecddd502f","navigationType":0,"readyForUser":1625.1999998092651,"redirectCount":0,"resourceLoadedEnd":2037.5999994277954,"resourceLoadedStart":689.3999996185303,"resourceTiming":[{"duration":336.1000003814697,"initiatorType":"link","name":"https://jira.mariadb.org/s/2c21342762a6a02add1c328bed317ffd-CDN/lu2cib/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/css/_super/batch.css","startTime":689.3999996185303,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":689.3999996185303,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1025.5,"responseStart":0,"secureConnectionStart":0},{"duration":336.30000019073486,"initiatorType":"link","name":"https://jira.mariadb.org/s/7ebd35e77e471bc30ff0eba799ebc151-CDN/lu2cib/820016/12ta74/494e4c556ecbb29f90a3d3b4f09cb99c/_/download/contextbatch/css/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true&whisper-enabled=true","startTime":689.6999998092651,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":689.6999998092651,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1026,"responseStart":0,"secureConnectionStart":0},{"duration":392,"initiatorType":"script","name":"https://jira.mariadb.org/s/0917945aaa57108d00c5076fea35e069-CDN/lu2cib/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/js/_super/batch.js?locale=en","startTime":689.8999996185303,"connectEnd":689.8999996185303,"connectStart":689.8999996185303,"domainLookupEnd":689.8999996185303,"domainLookupStart":689.8999996185303,"fetchStart":689.8999996185303,"redirectEnd":0,"redirectStart":0,"requestStart":689.8999996185303,"responseEnd":1081.8999996185303,"responseStart":1081.8999996185303,"secureConnectionStart":689.8999996185303},{"duration":454.70000076293945,"initiatorType":"script","name":"https://jira.mariadb.org/s/2d8175ec2fa4c816e8023260bd8c1786-CDN/lu2cib/820016/12ta74/494e4c556ecbb29f90a3d3b4f09cb99c/_/download/contextbatch/js/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true&whisper-enabled=true","startTime":690.0999994277954,"connectEnd":690.0999994277954,"connectStart":690.0999994277954,"domainLookupEnd":690.0999994277954,"domainLookupStart":690.0999994277954,"fetchStart":690.0999994277954,"redirectEnd":0,"redirectStart":0,"requestStart":690.0999994277954,"responseEnd":1144.8000001907349,"responseStart":1144.8000001907349,"secureConnectionStart":690.0999994277954},{"duration":458.19999980926514,"initiatorType":"script","name":"https://jira.mariadb.org/s/a9324d6758d385eb45c462685ad88f1d-CDN/lu2cib/820016/12ta74/c92c0caa9a024ae85b0ebdbed7fb4bd7/_/download/contextbatch/js/atl.global,-_super/batch.js?locale=en","startTime":690.3999996185303,"connectEnd":690.3999996185303,"connectStart":690.3999996185303,"domainLookupEnd":690.3999996185303,"domainLookupStart":690.3999996185303,"fetchStart":690.3999996185303,"redirectEnd":0,"redirectStart":0,"requestStart":690.3999996185303,"responseEnd":1148.5999994277954,"responseStart":1148.5999994277954,"secureConnectionStart":690.3999996185303},{"duration":459,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-en/jira.webresources:calendar-en.js","startTime":690.5999994277954,"connectEnd":690.5999994277954,"connectStart":690.5999994277954,"domainLookupEnd":690.5999994277954,"domainLookupStart":690.5999994277954,"fetchStart":690.5999994277954,"redirectEnd":0,"redirectStart":0,"requestStart":690.5999994277954,"responseEnd":1149.5999994277954,"responseStart":1149.5999994277954,"secureConnectionStart":690.5999994277954},{"duration":459.80000019073486,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-localisation-moment/jira.webresources:calendar-localisation-moment.js","startTime":690.6999998092651,"connectEnd":690.6999998092651,"connectStart":690.6999998092651,"domainLookupEnd":690.6999998092651,"domainLookupStart":690.6999998092651,"fetchStart":690.6999998092651,"redirectEnd":0,"redirectStart":0,"requestStart":690.6999998092651,"responseEnd":1150.5,"responseStart":1150.5,"secureConnectionStart":690.6999998092651},{"duration":534.3000001907349,"initiatorType":"link","name":"https://jira.mariadb.org/s/b04b06a02d1959df322d9cded3aeecc1-CDN/lu2cib/820016/12ta74/a2ff6aa845ffc9a1d22fe23d9ee791fc/_/download/contextbatch/css/jira.global.look-and-feel,-_super/batch.css","startTime":690.8999996185303,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":690.8999996185303,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1225.1999998092651,"responseStart":0,"secureConnectionStart":0},{"duration":460.30000019073486,"initiatorType":"script","name":"https://jira.mariadb.org/rest/api/1.0/shortcuts/820016/47140b6e0a9bc2e4913da06536125810/shortcuts.js?context=issuenavigation&context=issueaction","startTime":691,"connectEnd":691,"connectStart":691,"domainLookupEnd":691,"domainLookupStart":691,"fetchStart":691,"redirectEnd":0,"redirectStart":0,"requestStart":691,"responseEnd":1151.3000001907349,"responseStart":1151.3000001907349,"secureConnectionStart":691},{"duration":534.1000003814697,"initiatorType":"link","name":"https://jira.mariadb.org/s/3ac36323ba5e4eb0af2aa7ac7211b4bb-CDN/lu2cib/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/css/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.css?jira.create.linked.issue=true","startTime":691.1999998092651,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":691.1999998092651,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1225.3000001907349,"responseStart":0,"secureConnectionStart":0},{"duration":460.69999980926514,"initiatorType":"script","name":"https://jira.mariadb.org/s/5d5e8fe91fbc506585e83ea3b62ccc4b-CDN/lu2cib/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/js/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.js?jira.create.linked.issue=true&locale=en","startTime":691.3000001907349,"connectEnd":691.3000001907349,"connectStart":691.3000001907349,"domainLookupEnd":691.3000001907349,"domainLookupStart":691.3000001907349,"fetchStart":691.3000001907349,"redirectEnd":0,"redirectStart":0,"requestStart":691.3000001907349,"responseEnd":1152,"responseStart":1152,"secureConnectionStart":691.3000001907349},{"duration":827.7000007629395,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-js/jira.webresources:bigpipe-js.js","startTime":692.0999994277954,"connectEnd":692.0999994277954,"connectStart":692.0999994277954,"domainLookupEnd":692.0999994277954,"domainLookupStart":692.0999994277954,"fetchStart":692.0999994277954,"redirectEnd":0,"redirectStart":0,"requestStart":692.0999994277954,"responseEnd":1519.8000001907349,"responseStart":1519.8000001907349,"secureConnectionStart":692.0999994277954},{"duration":1345.5,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-init/jira.webresources:bigpipe-init.js","startTime":692.0999994277954,"connectEnd":692.0999994277954,"connectStart":692.0999994277954,"domainLookupEnd":692.0999994277954,"domainLookupStart":692.0999994277954,"fetchStart":692.0999994277954,"redirectEnd":0,"redirectStart":0,"requestStart":692.0999994277954,"responseEnd":2037.5999994277954,"responseStart":2037.5999994277954,"secureConnectionStart":692.0999994277954},{"duration":288.1000003814697,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":1236.8999996185303,"connectEnd":1236.8999996185303,"connectStart":1236.8999996185303,"domainLookupEnd":1236.8999996185303,"domainLookupStart":1236.8999996185303,"fetchStart":1236.8999996185303,"redirectEnd":0,"redirectStart":0,"requestStart":1236.8999996185303,"responseEnd":1525,"responseStart":1524.8999996185303,"secureConnectionStart":1236.8999996185303},{"duration":601.8999996185303,"initiatorType":"script","name":"https://www.google-analytics.com/analytics.js","startTime":1533.6999998092651,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":1533.6999998092651,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":2135.5999994277954,"responseStart":0,"secureConnectionStart":0},{"duration":539.1000003814697,"initiatorType":"link","name":"https://jira.mariadb.org/s/d5715adaadd168a9002b108b2b039b50-CDN/lu2cib/820016/12ta74/be4b45e9cec53099498fa61c8b7acba4/_/download/contextbatch/css/jira.project.sidebar,-_super,-project.issue.navigator,-jira.general,-jira.browse.project,-jira.view.issue,-jira.global,-atl.general,-com.atlassian.jira.projects.sidebar.init/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true&whisper-enabled=true","startTime":1556.8999996185303,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":1556.8999996185303,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":2096,"responseStart":0,"secureConnectionStart":0},{"duration":517.3000001907349,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/e65b778d185daf5aee24936755b43da6/_/download/contextbatch/js/browser-metrics-plugin.contrib,-_super,-project.issue.navigator,-jira.view.issue,-atl.general/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true&whisper-enabled=true","startTime":1557.8999996185303,"connectEnd":1557.8999996185303,"connectStart":1557.8999996185303,"domainLookupEnd":1557.8999996185303,"domainLookupStart":1557.8999996185303,"fetchStart":1557.8999996185303,"redirectEnd":0,"redirectStart":0,"requestStart":1557.8999996185303,"responseEnd":2075.199999809265,"responseStart":2075.199999809265,"secureConnectionStart":1557.8999996185303},{"duration":551.3000001907349,"initiatorType":"script","name":"https://jira.mariadb.org/s/097ae97cb8fbec7d6ea4bbb1f26955b9-CDN/lu2cib/820016/12ta74/be4b45e9cec53099498fa61c8b7acba4/_/download/contextbatch/js/jira.project.sidebar,-_super,-project.issue.navigator,-jira.general,-jira.browse.project,-jira.view.issue,-jira.global,-atl.general,-com.atlassian.jira.projects.sidebar.init/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true&whisper-enabled=true","startTime":1558.3999996185303,"connectEnd":1558.3999996185303,"connectStart":1558.3999996185303,"domainLookupEnd":1558.3999996185303,"domainLookupStart":1558.3999996185303,"fetchStart":1558.3999996185303,"redirectEnd":0,"redirectStart":0,"requestStart":1558.3999996185303,"responseEnd":2109.699999809265,"responseStart":2109.5999994277954,"secureConnectionStart":1558.3999996185303}],"fetchStart":0,"domainLookupStart":0,"domainLookupEnd":0,"connectStart":0,"connectEnd":0,"requestStart":520,"responseStart":683,"responseEnd":686,"domLoading":687,"domInteractive":2055,"domContentLoadedEventStart":2055,"domContentLoadedEventEnd":2096,"domComplete":2363,"loadEventStart":2363,"loadEventEnd":2363,"userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","marks":[{"name":"bigPipe.sidebar-id.start","time":2039.5},{"name":"bigPipe.sidebar-id.end","time":2040.3999996185303},{"name":"bigPipe.activity-panel-pipe-id.start","time":2040.5},{"name":"bigPipe.activity-panel-pipe-id.end","time":2041.0999994277954},{"name":"activityTabFullyLoaded","time":2140}],"measures":[],"correlationId":"cca84ce3d78c82","effectiveType":"4g","downlink":10,"rtt":0,"serverDuration":98,"dbReadsTimeInMs":13,"dbConnsTimeInMs":22,"applicationHash":"9d11dbea5f4be3d4cc21f03a88dd11d8c8687422","experiments":[]}}