The customer reports that an attempt to join crashed node to the cluster through IST ends with another crash after wsrep_recover. Wsrep thread did not wait that recover thread to
finish the job, but initiate IST procedure and crashes again - backtraces are below.
And full error log is in the case. Platform is FreeBSD.
2022-05-15 1:55:21 96 [Note] WSREP: Starting applier thread 96
|
2022-05-15 1:55:21 97 [Note] WSREP: Starting applier thread 97
|
2022-05-15 1:55:21 98 [Note] WSREP: Starting applier thread 98
|
2022-05-15 1:55:21 101 [Note] WSREP: Starting applier thread 101
|
2022-05-15 1:55:21 99 [Note] WSREP: Starting applier thread 99
|
2022-05-15 1:55:21 0 [Note] /usr/local/libexec/mariadbd: ready for connections.
|
Version: '10.6.8-4-MariaDB-enterprise-log' socket: '/tmp/mysql.sock' port: 3306 MariaDB Enterprise Server
|
2022-05-15 1:55:21 103 [Note] WSREP: Starting applier thread 103
|
0x13ef71e <my_print_stacktrace+0x2e> at /usr/local/libexec/mariadbd
|
mysys/my_addr_resolve.c:299(my_addr_resolve)[0xce6460]
|
0x801935580 <pthread_sigmask+0x540> at /lib/libthr.so.3
|
thread/thr_sig.c:0(handle_signal)[0x801934b3f]
|
0x7ffffffff2d3 <__gxx_personality_v0+0x7ffffeb6ae03> at ???
|
0xd9ec24 <thd_get_thread_id+0x4> at /usr/local/libexec/mariadbd
|
mysys/my_addr_resolve.c:299(my_addr_resolve)[0x12b9590]
|
0x12b8f9b <_Z9lock_waitP9que_thr_t+0x5b> at /usr/local/libexec/mariadbd
|
0x132f9b1 <_Z23row_mysql_handle_errorsP7dberr_tP5trx_tP9que_thr_tP12trx_savept_t+0x61> at /usr/local/libexec/mariadbd
|
sql/mysqld.cc:1848(unireg_abort)[0x13497a6]
|
maria/ma_check.c:3055(writekeys)[0x11a7891]
|
maria/ma_check.c:3159(_ma_flush_table_files_before_swap)[0xbc80e2]
|
0xbd3d29 <_ZN7handler17rnd_pos_by_recordEPh+0x59> at /usr/local/libexec/mariadbd
|
perfschema/pfs_stat.h:76(PFS_single_stat::aggregate(PFS_single_stat const*))[0xc97b40]
|
0xc98b70 <_ZN21Update_rows_log_event11do_exec_rowEP14rpl_group_info+0x190> at /usr/local/libexec/mariadbd
|
perfschema/table_events_statements.cc:239(table_events_statements_common::make_row_part_1(PFS_events_statements*, sql_digest_storage*))[0xc92f88]
|
0xd351b8 <_ZN9Log_event11apply_eventEP14rpl_group_info+0x68> at /usr/local/libexec/mariadbd
|
0x1189d12 <_Z18wsrep_apply_eventsP3THDP14Relay_log_infoPKvm+0x392> at /usr/local/libexec/mariadbd
|
sql/sys_vars.inl:508(Sys_var_charptr_base)[0x116ba87]
|
0x147df0c <_ZN5wsrep12server_state8on_applyERNS_21high_priority_serviceERKNS_9ws_handleERKNS_7ws_metaERKNS_12const_bufferE+0x3cc> at /usr/local/libexec/mariadbd
|
0x1489eef <_ZN12_GLOBAL__N_18apply_cbEPvPK15wsrep_ws_handlejPK9wsrep_bufPK14wsrep_trx_metaPb+0xaf> at /usr/local/libexec/mariadbd
|
0x80391b690 <wsrep_ps_free_node_stat+0x9510> at /rw_part/usr-local/lib/mysql/libgalera_enterprise_smm.so
|
src/trx_handle.cpp:392(galera::TrxHandleSlave::apply(void*, wsrep_cb_status (*)(void*, wsrep_ws_handle const*, unsigned int, wsrep_buf const*, wsrep_trx_meta const*, bool*), wsrep_trx_meta const&, bool&))[0x80392143a]
|
0x803969b00 <wsrep_ps_free_node_stat+0x57980> at /rw_part/usr-local/lib/mysql/libgalera_enterprise_smm.so
|
0x803969441 <wsrep_ps_free_node_stat+0x572c1> at /rw_part/usr-local/lib/mysql/libgalera_enterprise_smm.so
|
0x803920847 <wsrep_ps_free_node_stat+0xe6c7> at /rw_part/usr-local/lib/mysql/libgalera_enterprise_smm.so
|
src/replicator_smm.cpp:538(galera::ReplicatorSMM::apply_trx(void*, galera::TrxHandleSlave&))[0x80390b001]
|
0x148a63b <_ZN5wsrep18wsrep_provider_v2611run_applierEPNS_21high_priority_serviceE+0xb> at /usr/local/libexec/mariadbd
|
mysys/my_addr_resolve.c:299(my_addr_resolve)[0x118a20d]
|
0x117a083 <_Z15start_wsrep_THDPv+0x2e3> at /usr/local/libexec/mariadbd
|
0x110c177 <pfs_spawn_thread+0xd7> at /usr/local/libexec/mariadbd
|
|
Trying to get some variables.
|
Some pointers may be invalid and cause the dump to abort.
|
Query (0x1d02204e5b): UPDATE nl_game_providers.game_bettings SET ext_bet_status = 'Won' WHERE provider_id = '2' AND recno = '13076845542'
|
|
Connection ID (thread ID): 37
|
Status: NOT_KILLED
|
|
Optimizer switch: index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_merge_sort_intersection=off,engine_condition_pushdown=off,index_condition_pushdown=on,derived_merge=on,derived_with_keys=on,firstmatch=on,loosescan=on,materialization=on,in_to_exists=on,semijoin=on,partial_match_rowid_merge=on,partial_match_table_scan=on,subquery_cache=on,mrr=off,mrr_cost_based=off,mrr_sort_keys=off,outer_join_with_cache=on,semijoin_with_cache=on,join_cache_incremental=on,join_cache_hashed=on,join_cache_bka=on,optimize_join_buffer_size=on,table_elimination=on,extended_keys=on,exists_to_in=on,orderby_uses_equalities=on,condition_pushdown_for_derived=on,split_materialized=on,condition_pushdown_for_subquery=on,rowid_filter=on,condition_pushdown_from_having=on,not_null_range_scan=off
|
|
The manual page at https://mariadb.com/kb/en/how-to-produce-a-full-stack-trace-for-mysqld/ contains
|
information that should help you find out what is causing the crash.
|
Core pattern: %N.core
|
{"report":{"fcp":1353.5,"ttfb":430,"pageVisibility":"visible","entityId":111078,"key":"jira.project.issue.view-issue","isInitial":true,"threshold":1000,"elementTimings":{},"userDeviceMemory":8,"userDeviceProcessors":64,"apdex":0.5,"journeyId":"4f97a394-a216-4022-bfb4-3f2948de3324","navigationType":0,"readyForUser":1444.1000003814697,"redirectCount":0,"resourceLoadedEnd":2722.6000003814697,"resourceLoadedStart":435.5,"resourceTiming":[{"duration":378.19999980926514,"initiatorType":"link","name":"https://jira.mariadb.org/s/2c21342762a6a02add1c328bed317ffd-CDN/lu2cib/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/css/_super/batch.css","startTime":435.5,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":435.5,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":813.6999998092651,"responseStart":0,"secureConnectionStart":0},{"duration":378.30000019073486,"initiatorType":"link","name":"https://jira.mariadb.org/s/7ebd35e77e471bc30ff0eba799ebc151-CDN/lu2cib/820016/12ta74/494e4c556ecbb29f90a3d3b4f09cb99c/_/download/contextbatch/css/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true&whisper-enabled=true","startTime":435.69999980926514,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":435.69999980926514,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":814,"responseStart":0,"secureConnectionStart":0},{"duration":387.3999996185303,"initiatorType":"script","name":"https://jira.mariadb.org/s/0917945aaa57108d00c5076fea35e069-CDN/lu2cib/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/js/_super/batch.js?locale=en","startTime":435.9000005722046,"connectEnd":435.9000005722046,"connectStart":435.9000005722046,"domainLookupEnd":435.9000005722046,"domainLookupStart":435.9000005722046,"fetchStart":435.9000005722046,"redirectEnd":0,"redirectStart":0,"requestStart":435.9000005722046,"responseEnd":823.3000001907349,"responseStart":823.1999998092651,"secureConnectionStart":435.9000005722046},{"duration":460.69999980926514,"initiatorType":"script","name":"https://jira.mariadb.org/s/2d8175ec2fa4c816e8023260bd8c1786-CDN/lu2cib/820016/12ta74/494e4c556ecbb29f90a3d3b4f09cb99c/_/download/contextbatch/js/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true&whisper-enabled=true","startTime":436.1000003814697,"connectEnd":436.1000003814697,"connectStart":436.1000003814697,"domainLookupEnd":436.1000003814697,"domainLookupStart":436.1000003814697,"fetchStart":436.1000003814697,"redirectEnd":0,"redirectStart":0,"requestStart":436.1000003814697,"responseEnd":896.8000001907349,"responseStart":896.8000001907349,"secureConnectionStart":436.1000003814697},{"duration":464.5,"initiatorType":"script","name":"https://jira.mariadb.org/s/a9324d6758d385eb45c462685ad88f1d-CDN/lu2cib/820016/12ta74/c92c0caa9a024ae85b0ebdbed7fb4bd7/_/download/contextbatch/js/atl.global,-_super/batch.js?locale=en","startTime":436.30000019073486,"connectEnd":436.30000019073486,"connectStart":436.30000019073486,"domainLookupEnd":436.30000019073486,"domainLookupStart":436.30000019073486,"fetchStart":436.30000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":436.30000019073486,"responseEnd":900.8000001907349,"responseStart":900.8000001907349,"secureConnectionStart":436.30000019073486},{"duration":464.5999994277954,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-en/jira.webresources:calendar-en.js","startTime":436.6000003814697,"connectEnd":436.6000003814697,"connectStart":436.6000003814697,"domainLookupEnd":436.6000003814697,"domainLookupStart":436.6000003814697,"fetchStart":436.6000003814697,"redirectEnd":0,"redirectStart":0,"requestStart":436.6000003814697,"responseEnd":901.1999998092651,"responseStart":901.1999998092651,"secureConnectionStart":436.6000003814697},{"duration":464.8999996185303,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-localisation-moment/jira.webresources:calendar-localisation-moment.js","startTime":436.80000019073486,"connectEnd":436.80000019073486,"connectStart":436.80000019073486,"domainLookupEnd":436.80000019073486,"domainLookupStart":436.80000019073486,"fetchStart":436.80000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":436.80000019073486,"responseEnd":901.6999998092651,"responseStart":901.6999998092651,"secureConnectionStart":436.80000019073486},{"duration":542.8000001907349,"initiatorType":"link","name":"https://jira.mariadb.org/s/b04b06a02d1959df322d9cded3aeecc1-CDN/lu2cib/820016/12ta74/a2ff6aa845ffc9a1d22fe23d9ee791fc/_/download/contextbatch/css/jira.global.look-and-feel,-_super/batch.css","startTime":437,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":437,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":979.8000001907349,"responseStart":0,"secureConnectionStart":0},{"duration":464.9000005722046,"initiatorType":"script","name":"https://jira.mariadb.org/rest/api/1.0/shortcuts/820016/47140b6e0a9bc2e4913da06536125810/shortcuts.js?context=issuenavigation&context=issueaction","startTime":437.19999980926514,"connectEnd":437.19999980926514,"connectStart":437.19999980926514,"domainLookupEnd":437.19999980926514,"domainLookupStart":437.19999980926514,"fetchStart":437.19999980926514,"redirectEnd":0,"redirectStart":0,"requestStart":437.19999980926514,"responseEnd":902.1000003814697,"responseStart":902.1000003814697,"secureConnectionStart":437.19999980926514},{"duration":542.8000001907349,"initiatorType":"link","name":"https://jira.mariadb.org/s/3ac36323ba5e4eb0af2aa7ac7211b4bb-CDN/lu2cib/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/css/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.css?jira.create.linked.issue=true","startTime":437.30000019073486,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":437.30000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":980.1000003814697,"responseStart":0,"secureConnectionStart":0},{"duration":465.30000019073486,"initiatorType":"script","name":"https://jira.mariadb.org/s/5d5e8fe91fbc506585e83ea3b62ccc4b-CDN/lu2cib/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/js/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.js?jira.create.linked.issue=true&locale=en","startTime":437.5,"connectEnd":437.5,"connectStart":437.5,"domainLookupEnd":437.5,"domainLookupStart":437.5,"fetchStart":437.5,"redirectEnd":0,"redirectStart":0,"requestStart":437.5,"responseEnd":902.8000001907349,"responseStart":902.8000001907349,"secureConnectionStart":437.5},{"duration":652.2999992370605,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-js/jira.webresources:bigpipe-js.js","startTime":442.9000005722046,"connectEnd":442.9000005722046,"connectStart":442.9000005722046,"domainLookupEnd":442.9000005722046,"domainLookupStart":442.9000005722046,"fetchStart":442.9000005722046,"redirectEnd":0,"redirectStart":0,"requestStart":442.9000005722046,"responseEnd":1095.1999998092651,"responseStart":1095.1999998092651,"secureConnectionStart":442.9000005722046},{"duration":2258.0999994277954,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-init/jira.webresources:bigpipe-init.js","startTime":443.1000003814697,"connectEnd":443.1000003814697,"connectStart":443.1000003814697,"domainLookupEnd":443.1000003814697,"domainLookupStart":443.1000003814697,"fetchStart":443.1000003814697,"redirectEnd":0,"redirectStart":0,"requestStart":443.1000003814697,"responseEnd":2701.199999809265,"responseStart":2701.199999809265,"secureConnectionStart":443.1000003814697},{"duration":104.19999980926514,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":992.4000005722046,"connectEnd":992.4000005722046,"connectStart":992.4000005722046,"domainLookupEnd":992.4000005722046,"domainLookupStart":992.4000005722046,"fetchStart":992.4000005722046,"redirectEnd":0,"redirectStart":0,"requestStart":992.4000005722046,"responseEnd":1096.6000003814697,"responseStart":1096.6000003814697,"secureConnectionStart":992.4000005722046},{"duration":1437.6999998092651,"initiatorType":"link","name":"https://jira.mariadb.org/s/d5715adaadd168a9002b108b2b039b50-CDN/lu2cib/820016/12ta74/be4b45e9cec53099498fa61c8b7acba4/_/download/contextbatch/css/jira.project.sidebar,-_super,-project.issue.navigator,-jira.general,-jira.browse.project,-jira.view.issue,-jira.global,-atl.general,-com.atlassian.jira.projects.sidebar.init/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true&whisper-enabled=true","startTime":1277.1000003814697,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":1277.1000003814697,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":2714.800000190735,"responseStart":0,"secureConnectionStart":0},{"duration":1437.3000001907349,"initiatorType":"link","name":"https://jira.mariadb.org/s/50bc9be5bfead1a25e72c1a9338c94f6-CDN/lu2cib/820016/12ta74/e108c7645258ccb43280ed3404e3e949/_/download/contextbatch/css/com.atlassian.jira.plugins.jira-development-integration-plugin:0,-_super,-jira.view.issue,-jira.global,-jira.general,-jira.browse.project,-project.issue.navigator,-atl.general/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true&whisper-enabled=true","startTime":1277.6999998092651,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":1277.6999998092651,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":2715,"responseStart":0,"secureConnectionStart":0},{"duration":1436.6999998092651,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/e65b778d185daf5aee24936755b43da6/_/download/contextbatch/js/browser-metrics-plugin.contrib,-_super,-project.issue.navigator,-jira.view.issue,-atl.general/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true&whisper-enabled=true","startTime":1278.6000003814697,"connectEnd":1278.6000003814697,"connectStart":1278.6000003814697,"domainLookupEnd":1278.6000003814697,"domainLookupStart":1278.6000003814697,"fetchStart":1278.6000003814697,"redirectEnd":0,"redirectStart":0,"requestStart":1278.6000003814697,"responseEnd":2715.300000190735,"responseStart":2715.300000190735,"secureConnectionStart":1278.6000003814697},{"duration":1421.3999996185303,"initiatorType":"script","name":"https://www.google-analytics.com/analytics.js","startTime":1347.1000003814697,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":1347.1000003814697,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":2768.5,"responseStart":0,"secureConnectionStart":0},{"duration":1442.1999998092651,"initiatorType":"script","name":"https://jira.mariadb.org/s/097ae97cb8fbec7d6ea4bbb1f26955b9-CDN/lu2cib/820016/12ta74/be4b45e9cec53099498fa61c8b7acba4/_/download/contextbatch/js/jira.project.sidebar,-_super,-project.issue.navigator,-jira.general,-jira.browse.project,-jira.view.issue,-jira.global,-atl.general,-com.atlassian.jira.projects.sidebar.init/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true&whisper-enabled=true","startTime":1279,"connectEnd":1279,"connectStart":1279,"domainLookupEnd":1279,"domainLookupStart":1279,"fetchStart":1279,"redirectEnd":0,"redirectStart":0,"requestStart":1279,"responseEnd":2721.199999809265,"responseStart":2721.199999809265,"secureConnectionStart":1279},{"duration":1443.1999998092651,"initiatorType":"script","name":"https://jira.mariadb.org/s/e0bf5781d46ea69fb123572974cf39de-CDN/lu2cib/820016/12ta74/e108c7645258ccb43280ed3404e3e949/_/download/contextbatch/js/com.atlassian.jira.plugins.jira-development-integration-plugin:0,-_super,-jira.view.issue,-jira.global,-jira.general,-jira.browse.project,-project.issue.navigator,-atl.general/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true&whisper-enabled=true","startTime":1279.4000005722046,"connectEnd":1279.4000005722046,"connectStart":1279.4000005722046,"domainLookupEnd":1279.4000005722046,"domainLookupStart":1279.4000005722046,"fetchStart":1279.4000005722046,"redirectEnd":0,"redirectStart":0,"requestStart":1279.4000005722046,"responseEnd":2722.6000003814697,"responseStart":2722.6000003814697,"secureConnectionStart":1279.4000005722046}],"fetchStart":0,"domainLookupStart":0,"domainLookupEnd":0,"connectStart":0,"connectEnd":0,"requestStart":207,"responseStart":430,"responseEnd":439,"domLoading":433,"domInteractive":2723,"domContentLoadedEventStart":2723,"domContentLoadedEventEnd":2764,"domComplete":4550,"loadEventStart":4550,"loadEventEnd":4552,"userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","marks":[{"name":"bigPipe.sidebar-id.start","time":2703},{"name":"bigPipe.sidebar-id.end","time":2703.800000190735},{"name":"bigPipe.activity-panel-pipe-id.start","time":2704},{"name":"bigPipe.activity-panel-pipe-id.end","time":2706.6000003814697},{"name":"activityTabFullyLoaded","time":2772.5}],"measures":[],"correlationId":"b556d8ee5c9945","effectiveType":"4g","downlink":10,"rtt":0,"serverDuration":160,"dbReadsTimeInMs":9,"dbConnsTimeInMs":18,"applicationHash":"9d11dbea5f4be3d4cc21f03a88dd11d8c8687422","experiments":[]}}