Hosts in this cluster are occasionally crashing, with this message each time.
Here's what got written to our mysql-error.log ...
mysqld: /home/buildbot/buildbot/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/mariadb-10.4.11/wsrep-lib/src/client_state.cpp:121: int wsrep::client_state::before_command(): Assertion `server_state_.rollback_mode() == wsrep::server_state::rm_async' failed.
200211 21:41:48 [ERROR] mysqld got signal 6 ;
This could be because you hit a bug. It is also possible that this binary
or one of the libraries it was linked against is corrupt, improperly built,
or misconfigured. This error can also be caused by malfunctioning hardware.
To report this bug, see https://mariadb.com/kb/en/reporting-bugs
We will try our best to scrape up some info that will hopefully help
diagnose the problem, but since we have already crashed,
something is definitely wrong and this may fail.
Server version: 10.4.11-MariaDB
key_buffer_size=33554432
read_buffer_size=131072
max_used_connections=278
max_threads=65541
thread_count=238
It is possible that mysqld could use up to
key_buffer_size + (read_buffer_size + sort_buffer_size)*max_threads = 142750971 K bytes of memory
Hope that's ok; if not, decrease some variables in the equation.
Thread pointer: 0x7f113c0c73b8
Attempting backtrace. You can use the following information to find out
where mysqld died. If you see no messages after this, something went
terribly wrong...
stack_bottom = 0x7f12fb93ec98 thread_stack 0x49000
-
-
- stack smashing detected ***: /usr/sbin/mysqld terminated
======= Backtrace: =========
/lib64/libc.so.6(__fortify_fail+0x37)[0x7f1513812877]
/lib64/libc.so.6(__fortify_fail+0x0)[0x7f1513812840]
/usr/sbin/mysqld(+0xe3b93c)[0x562eb83c893c]
/usr/sbin/mysqld(my_print_stacktrace+0x1c6)[0x562eb83b17a6]
/usr/sbin/mysqld(handle_fatal_signal+0x4b7)[0x562eb7e80557]
/lib64/libpthread.so.0(+0x3ff8c0f7e0)[0x7f15150e87e0]
/lib64/libc.so.6(gsignal+0x35)[0x7f15137424f5]
/lib64/libc.so.6(abort+0x175)[0x7f1513743cd5]
/lib64/libc.so.6(+0x3ff882b66e)[0x7f151373b66e]
/lib64/libc.so.6(__assert_perror_fail+0x0)[0x7f151373b730]
/usr/sbin/mysqld(_ZN5wsrep12client_state14before_commandEv+0x320)[0x562eb8431150]
/usr/sbin/mysqld(_Z10do_commandP3THD+0x1a4)[0x562eb7c9d1d4]
/usr/sbin/mysqld(_Z11tp_callbackP13TP_connection+0x58)[0x562eb7e55568]
/usr/sbin/mysqld(+0xa56dd8)[0x562eb7fe3dd8]
/usr/sbin/mysqld(+0xa856fd)[0x562eb80126fd]
/lib64/libpthread.so.0(+0x3ff8c07aa1)[0x7f15150e0aa1]
/lib64/libc.so.6(clone+0x6d)[0x7f15137f8c4d]
{"report":{"fcp":1759.3999996185303,"ttfb":507.5,"pageVisibility":"visible","entityId":83197,"key":"jira.project.issue.view-issue","isInitial":true,"threshold":1000,"elementTimings":{},"userDeviceMemory":8,"userDeviceProcessors":64,"apdex":0.5,"journeyId":"7c804836-22a3-48ac-8d25-26917ea5569c","navigationType":0,"readyForUser":1880.7999992370605,"redirectCount":0,"resourceLoadedEnd":2208.199999809265,"resourceLoadedStart":512.8999996185303,"resourceTiming":[{"duration":662.8000001907349,"initiatorType":"link","name":"https://jira.mariadb.org/s/2c21342762a6a02add1c328bed317ffd-CDN/lu2cib/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/css/_super/batch.css","startTime":512.8999996185303,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":512.8999996185303,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1175.6999998092651,"responseStart":0,"secureConnectionStart":0},{"duration":662.8000001907349,"initiatorType":"link","name":"https://jira.mariadb.org/s/7ebd35e77e471bc30ff0eba799ebc151-CDN/lu2cib/820016/12ta74/494e4c556ecbb29f90a3d3b4f09cb99c/_/download/contextbatch/css/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true&whisper-enabled=true","startTime":513.1999998092651,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":513.1999998092651,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1176,"responseStart":0,"secureConnectionStart":0},{"duration":718.8999996185303,"initiatorType":"script","name":"https://jira.mariadb.org/s/0917945aaa57108d00c5076fea35e069-CDN/lu2cib/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/js/_super/batch.js?locale=en","startTime":513.5,"connectEnd":513.5,"connectStart":513.5,"domainLookupEnd":513.5,"domainLookupStart":513.5,"fetchStart":513.5,"redirectEnd":0,"redirectStart":0,"requestStart":513.5,"responseEnd":1232.3999996185303,"responseStart":1232.3999996185303,"secureConnectionStart":513.5},{"duration":781.3000001907349,"initiatorType":"script","name":"https://jira.mariadb.org/s/2d8175ec2fa4c816e8023260bd8c1786-CDN/lu2cib/820016/12ta74/494e4c556ecbb29f90a3d3b4f09cb99c/_/download/contextbatch/js/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true&whisper-enabled=true","startTime":513.5999994277954,"connectEnd":513.5999994277954,"connectStart":513.5999994277954,"domainLookupEnd":513.5999994277954,"domainLookupStart":513.5999994277954,"fetchStart":513.5999994277954,"redirectEnd":0,"redirectStart":0,"requestStart":513.5999994277954,"responseEnd":1294.8999996185303,"responseStart":1294.8999996185303,"secureConnectionStart":513.5999994277954},{"duration":785.2000007629395,"initiatorType":"script","name":"https://jira.mariadb.org/s/a9324d6758d385eb45c462685ad88f1d-CDN/lu2cib/820016/12ta74/c92c0caa9a024ae85b0ebdbed7fb4bd7/_/download/contextbatch/js/atl.global,-_super/batch.js?locale=en","startTime":513.7999992370605,"connectEnd":513.7999992370605,"connectStart":513.7999992370605,"domainLookupEnd":513.7999992370605,"domainLookupStart":513.7999992370605,"fetchStart":513.7999992370605,"redirectEnd":0,"redirectStart":0,"requestStart":513.7999992370605,"responseEnd":1299,"responseStart":1299,"secureConnectionStart":513.7999992370605},{"duration":785.5,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-en/jira.webresources:calendar-en.js","startTime":514,"connectEnd":514,"connectStart":514,"domainLookupEnd":514,"domainLookupStart":514,"fetchStart":514,"redirectEnd":0,"redirectStart":0,"requestStart":514,"responseEnd":1299.5,"responseStart":1299.5,"secureConnectionStart":514},{"duration":785.8000001907349,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-localisation-moment/jira.webresources:calendar-localisation-moment.js","startTime":514.1999998092651,"connectEnd":514.1999998092651,"connectStart":514.1999998092651,"domainLookupEnd":514.1999998092651,"domainLookupStart":514.1999998092651,"fetchStart":514.1999998092651,"redirectEnd":0,"redirectStart":0,"requestStart":514.1999998092651,"responseEnd":1300,"responseStart":1300,"secureConnectionStart":514.1999998092651},{"duration":885.6000003814697,"initiatorType":"link","name":"https://jira.mariadb.org/s/b04b06a02d1959df322d9cded3aeecc1-CDN/lu2cib/820016/12ta74/a2ff6aa845ffc9a1d22fe23d9ee791fc/_/download/contextbatch/css/jira.global.look-and-feel,-_super/batch.css","startTime":514.3999996185303,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":514.3999996185303,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1400,"responseStart":0,"secureConnectionStart":0},{"duration":785.9000005722046,"initiatorType":"script","name":"https://jira.mariadb.org/rest/api/1.0/shortcuts/820016/47140b6e0a9bc2e4913da06536125810/shortcuts.js?context=issuenavigation&context=issueaction","startTime":514.5999994277954,"connectEnd":514.5999994277954,"connectStart":514.5999994277954,"domainLookupEnd":514.5999994277954,"domainLookupStart":514.5999994277954,"fetchStart":514.5999994277954,"redirectEnd":0,"redirectStart":0,"requestStart":514.5999994277954,"responseEnd":1300.5,"responseStart":1300.5,"secureConnectionStart":514.5999994277954},{"duration":885.4000005722046,"initiatorType":"link","name":"https://jira.mariadb.org/s/3ac36323ba5e4eb0af2aa7ac7211b4bb-CDN/lu2cib/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/css/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.css?jira.create.linked.issue=true","startTime":514.7999992370605,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":514.7999992370605,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1400.1999998092651,"responseStart":0,"secureConnectionStart":0},{"duration":786.3000001907349,"initiatorType":"script","name":"https://jira.mariadb.org/s/5d5e8fe91fbc506585e83ea3b62ccc4b-CDN/lu2cib/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/js/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.js?jira.create.linked.issue=true&locale=en","startTime":514.8999996185303,"connectEnd":514.8999996185303,"connectStart":514.8999996185303,"domainLookupEnd":514.8999996185303,"domainLookupStart":514.8999996185303,"fetchStart":514.8999996185303,"redirectEnd":0,"redirectStart":0,"requestStart":514.8999996185303,"responseEnd":1301.1999998092651,"responseStart":1301.0999994277954,"secureConnectionStart":514.8999996185303},{"duration":1274.6999998092651,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-js/jira.webresources:bigpipe-js.js","startTime":515.6999998092651,"connectEnd":515.6999998092651,"connectStart":515.6999998092651,"domainLookupEnd":515.6999998092651,"domainLookupStart":515.6999998092651,"fetchStart":515.6999998092651,"redirectEnd":0,"redirectStart":0,"requestStart":515.6999998092651,"responseEnd":1790.3999996185303,"responseStart":1790.3999996185303,"secureConnectionStart":515.6999998092651},{"duration":1692.4000005722046,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-init/jira.webresources:bigpipe-init.js","startTime":515.7999992370605,"connectEnd":515.7999992370605,"connectStart":515.7999992370605,"domainLookupEnd":515.7999992370605,"domainLookupStart":515.7999992370605,"fetchStart":515.7999992370605,"redirectEnd":0,"redirectStart":0,"requestStart":515.7999992370605,"responseEnd":2208.199999809265,"responseStart":2208.199999809265,"secureConnectionStart":515.7999992370605},{"duration":382.80000019073486,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":1411.6999998092651,"connectEnd":1411.6999998092651,"connectStart":1411.6999998092651,"domainLookupEnd":1411.6999998092651,"domainLookupStart":1411.6999998092651,"fetchStart":1411.6999998092651,"redirectEnd":0,"redirectStart":0,"requestStart":1411.6999998092651,"responseEnd":1794.5,"responseStart":1794.5,"secureConnectionStart":1411.6999998092651},{"duration":577.7000007629395,"initiatorType":"script","name":"https://www.google-analytics.com/analytics.js","startTime":1751.2999992370605,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":1751.2999992370605,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":2329,"responseStart":0,"secureConnectionStart":0}],"fetchStart":0,"domainLookupStart":0,"domainLookupEnd":0,"connectStart":0,"connectEnd":0,"requestStart":346,"responseStart":508,"responseEnd":509,"domLoading":511,"domInteractive":2253,"domContentLoadedEventStart":2253,"domContentLoadedEventEnd":2326,"domComplete":3642,"loadEventStart":3642,"loadEventEnd":3643,"userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","marks":[{"name":"bigPipe.sidebar-id.start","time":2216.5},{"name":"bigPipe.sidebar-id.end","time":2217.5},{"name":"bigPipe.activity-panel-pipe-id.start","time":2217.5999994277954},{"name":"bigPipe.activity-panel-pipe-id.end","time":2220.0999994277954},{"name":"activityTabFullyLoaded","time":2332.699999809265}],"measures":[],"correlationId":"29d188f9cb2834","effectiveType":"4g","downlink":10,"rtt":0,"serverDuration":106,"dbReadsTimeInMs":12,"dbConnsTimeInMs":21,"applicationHash":"9d11dbea5f4be3d4cc21f03a88dd11d8c8687422","experiments":[]}}