While evaluating MariaDB 10.1.9, I found that creating a user or creating a stored procedure caused all three of our galera nodes to crash (the second trace below). Other operations may cause trigger the issue, but these are the only ones I tested. After the crash I had one node back up, I created a procedure which also caused a crash (the first trace below). While testing the issue on a VM running 10.1.10, I was able to narrow down the cause to having wsrep_forced_binlog_format to ROW with binlogging enabled. When wsrep_forced_binlog_format is not set, but binlogging is enabled, the crash doesn't happen. When binlogging is disabled but wsrep_forced_binlog_format is set, the crash doesn't happen.
I was able to start the server with binlogging enabled and wsrep_forced_binlog_format not set and was able to create a procedure. Then I set wsrep_forced_binlog_format to "ROW" and tried creating a procedure again and the server crashed. The crash also happens even if wsrep_on is not set and the only wsrep_* variable set is wsrep_forced_binlog_format.
You should be able to replicate this issue on a new 10.1.10 by enabling binlogging and setting wsrep_forced_binlog_format to "ROW" and creating a procedure or a user.
Based on the stack trace, this may be related to MDEV-9296.
Creating a procedure:
/usr/sbin/mysqld(my_print_stacktrace+0x2b) [0x7ff4b222b49b]
|
/usr/sbin/mysqld(handle_fatal_signal+0x475) [0x7ff4b1d8c7b5]
|
/lib64/libpthread.so.0(+0xf710) [0x7ff4b13a3710]
|
/usr/sbin/mysqld(bitmap_bits_set+0x1) [0x7ff4b2220d71]
|
/usr/sbin/mysqld(pack_row(TABLE*, st_bitmap const*, unsigned char*, unsigned char const*)+0x2f) [0x7ff4b1e5dabf]
|
/usr/sbin/mysqld(THD::binlog_write_row(TABLE*, bool, unsigned char const*)+0x83) [0x7ff4b1bf6a83]
|
/usr/sbin/mysqld(handler::ha_write_row(unsigned char*)+0x207) [0x7ff4b1d960b7]
|
/usr/sbin/mysqld(sp_create_routine(THD*, stored_procedure_type, sp_head*)+0x8f2) [0x7ff4b1e86472]
|
/usr/sbin/mysqld(mysql_execute_command(THD*)+0x5b8d) [0x7ff4b1c2037d]
|
/usr/sbin/mysqld(mysql_parse(THD*, char*, unsigned int, Parser_state*)+0x22d) [0x7ff4b1c2390d]
|
/usr/sbin/mysqld(+0x450a0c) [0x7ff4b1c23a0c]
|
/usr/sbin/mysqld(dispatch_command(enum_server_command, THD*, char*, unsigned int)+0x1c63) [0x7ff4b1c25d93]
|
/usr/sbin/mysqld(do_command(THD*)+0x16b) [0x7ff4b1c2695b]
|
/usr/sbin/mysqld(do_handle_one_connection(THD*)+0x17f) [0x7ff4b1ce1c6f]
|
/usr/sbin/mysqld(handle_one_connection+0x47) [0x7ff4b1ce1dc7]
|
/usr/sbin/mysqld(+0x94525d) [0x7ff4b211825d]
|
/lib64/libpthread.so.0(+0x79d1) [0x7ff4b139b9d1]
|
/lib64/libc.so.6(clone+0x6d) [0x7ff4af8818fd]
|
creating a user:
/usr/sbin/mysqld(my_print_stacktrace+0x2b) [0x7f4de6e5049b]
|
/usr/sbin/mysqld(handle_fatal_signal+0x475) [0x7f4de69b17b5]
|
/lib64/libpthread.so.0(+0xf710) [0x7f4de5fc8710]
|
/usr/sbin/mysqld(bitmap_bits_set+0x1) [0x7f4de6e45d71]
|
/usr/sbin/mysqld(pack_row(TABLE*, st_bitmap const*, unsigned char*, unsigned char const*)+0x2f) [0x7f4de6a82abf]
|
/usr/sbin/mysqld(THD::binlog_write_row(TABLE*, bool, unsigned char const*)+0x83) [0x7f4de681ba83]
|
/usr/sbin/mysqld(handler::ha_write_row(unsigned char*)+0x207) [0x7f4de69bb0b7]
|
/usr/sbin/mysqld(+0x3e913f) [0x7f4de67e113f]
|
/usr/sbin/mysqld(mysql_create_user(THD*, List<st_lex_user>&, bool)+0x5e7) [0x7f4de67ea0a7]
|
/usr/sbin/mysqld(mysql_execute_command(THD*)+0x2447) [0x7f4de6841c37]
|
/usr/sbin/mysqld(mysql_parse(THD*, char*, unsigned int, Parser_state*)+0x22d) [0x7f4de684890d]
|
/usr/sbin/mysqld(Query_log_event::do_apply_event(rpl_group_info*, char const*, unsigned int)+0x1184) [0x7f4de6a78794]
|
/usr/sbin/mysqld(wsrep_apply_cb(void*, void const*, unsigned long, unsigned int, wsrep_trx_meta const*)+0x542) [0x7f4de6960462]
|
/usr/lib64/galera/libgalera_smm.so(galera::TrxHandle::apply(void*, wsrep_cb_status (*)(void*, void const*, unsigned long, unsigned int, wsrep_trx_meta const*), wsrep_trx_meta const&) const+0xd3) [0x7f4dac5119f3]
|
/usr/lib64/galera/libgalera_smm.so(+0x21576e) [0x7f4dac54376e]
|
/usr/lib64/galera/libgalera_smm.so(galera::ReplicatorSMM::apply_trx(void*, galera::TrxHandle*)+0xa4) [0x7f4dac545414]
|
/usr/lib64/galera/libgalera_smm.so(galera::ReplicatorSMM::process_trx(void*, galera::TrxHandle*)+0x40) [0x7f4dac546810]
|
/usr/lib64/galera/libgalera_smm.so(galera::GcsActionSource::dispatch(void*, gcs_action const&, bool&)+0x185) [0x7f4dac529f15]
|
/usr/lib64/galera/libgalera_smm.so(galera::GcsActionSource::process(void*, bool&)+0x63) [0x7f4dac52a273]
|
/usr/lib64/galera/libgalera_smm.so(galera::ReplicatorSMM::async_recv(void*)+0x6d) [0x7f4dac5481fd]
|
/usr/lib64/galera/libgalera_smm.so(galera_recv+0x23) [0x7f4dac559083]
|
/usr/sbin/mysqld(+0x569202) [0x7f4de6961202]
|
/usr/sbin/mysqld(start_wsrep_THD+0x3c1) [0x7f4de69528c1]
|
/lib64/libpthread.so.0(+0x79d1) [0x7f4de5fc09d1]
|
/lib64/libc.so.6(clone+0x6d) [0x7f4de44a68fd]
|
{"report":{"fcp":966,"ttfb":247.19999980926514,"pageVisibility":"visible","entityId":55221,"key":"jira.project.issue.view-issue","isInitial":true,"threshold":1000,"elementTimings":{},"userDeviceMemory":8,"userDeviceProcessors":32,"apdex":0.5,"journeyId":"50a14ead-695c-4225-ad7d-c9e1620c0bb8","navigationType":0,"readyForUser":1104.0999999046326,"redirectCount":0,"resourceLoadedEnd":1034.1999998092651,"resourceLoadedStart":255.90000009536743,"resourceTiming":[{"duration":28.59999990463257,"initiatorType":"link","name":"https://jira.mariadb.org/s/2c21342762a6a02add1c328bed317ffd-CDN/lu2cib/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/css/_super/batch.css","startTime":255.90000009536743,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":255.90000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":284.5,"responseStart":0,"secureConnectionStart":0},{"duration":28.90000009536743,"initiatorType":"link","name":"https://jira.mariadb.org/s/7ebd35e77e471bc30ff0eba799ebc151-CDN/lu2cib/820016/12ta74/2bf333562ca6724060a9d5f1535471f6/_/download/contextbatch/css/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true","startTime":256.09999990463257,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":256.09999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":285,"responseStart":0,"secureConnectionStart":0},{"duration":92.09999990463257,"initiatorType":"script","name":"https://jira.mariadb.org/s/0917945aaa57108d00c5076fea35e069-CDN/lu2cib/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/js/_super/batch.js?locale=en","startTime":256.40000009536743,"connectEnd":256.40000009536743,"connectStart":256.40000009536743,"domainLookupEnd":256.40000009536743,"domainLookupStart":256.40000009536743,"fetchStart":256.40000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":256.40000009536743,"responseEnd":348.5,"responseStart":348.5,"secureConnectionStart":256.40000009536743},{"duration":199.90000009536743,"initiatorType":"script","name":"https://jira.mariadb.org/s/2d8175ec2fa4c816e8023260bd8c1786-CDN/lu2cib/820016/12ta74/2bf333562ca6724060a9d5f1535471f6/_/download/contextbatch/js/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true","startTime":256.90000009536743,"connectEnd":256.90000009536743,"connectStart":256.90000009536743,"domainLookupEnd":256.90000009536743,"domainLookupStart":256.90000009536743,"fetchStart":256.90000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":256.90000009536743,"responseEnd":456.80000019073486,"responseStart":456.80000019073486,"secureConnectionStart":256.90000009536743},{"duration":210.90000009536743,"initiatorType":"script","name":"https://jira.mariadb.org/s/a9324d6758d385eb45c462685ad88f1d-CDN/lu2cib/820016/12ta74/c92c0caa9a024ae85b0ebdbed7fb4bd7/_/download/contextbatch/js/atl.global,-_super/batch.js?locale=en","startTime":257.09999990463257,"connectEnd":257.09999990463257,"connectStart":257.09999990463257,"domainLookupEnd":257.09999990463257,"domainLookupStart":257.09999990463257,"fetchStart":257.09999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":257.09999990463257,"responseEnd":468,"responseStart":468,"secureConnectionStart":257.09999990463257},{"duration":212,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-en/jira.webresources:calendar-en.js","startTime":257.19999980926514,"connectEnd":257.19999980926514,"connectStart":257.19999980926514,"domainLookupEnd":257.19999980926514,"domainLookupStart":257.19999980926514,"fetchStart":257.19999980926514,"redirectEnd":0,"redirectStart":0,"requestStart":257.19999980926514,"responseEnd":469.19999980926514,"responseStart":469.19999980926514,"secureConnectionStart":257.19999980926514},{"duration":212.7999997138977,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-localisation-moment/jira.webresources:calendar-localisation-moment.js","startTime":257.40000009536743,"connectEnd":257.40000009536743,"connectStart":257.40000009536743,"domainLookupEnd":257.40000009536743,"domainLookupStart":257.40000009536743,"fetchStart":257.40000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":257.40000009536743,"responseEnd":470.19999980926514,"responseStart":470.19999980926514,"secureConnectionStart":257.40000009536743},{"duration":288.59999990463257,"initiatorType":"link","name":"https://jira.mariadb.org/s/b04b06a02d1959df322d9cded3aeecc1-CDN/lu2cib/820016/12ta74/a2ff6aa845ffc9a1d22fe23d9ee791fc/_/download/contextbatch/css/jira.global.look-and-feel,-_super/batch.css","startTime":257.5,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":257.5,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":546.0999999046326,"responseStart":0,"secureConnectionStart":0},{"duration":221.5,"initiatorType":"script","name":"https://jira.mariadb.org/rest/api/1.0/shortcuts/820016/47140b6e0a9bc2e4913da06536125810/shortcuts.js?context=issuenavigation&context=issueaction","startTime":257.69999980926514,"connectEnd":257.69999980926514,"connectStart":257.69999980926514,"domainLookupEnd":257.69999980926514,"domainLookupStart":257.69999980926514,"fetchStart":257.69999980926514,"redirectEnd":0,"redirectStart":0,"requestStart":257.69999980926514,"responseEnd":479.19999980926514,"responseStart":479.19999980926514,"secureConnectionStart":257.69999980926514},{"duration":288.5,"initiatorType":"link","name":"https://jira.mariadb.org/s/3ac36323ba5e4eb0af2aa7ac7211b4bb-CDN/lu2cib/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/css/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.css?jira.create.linked.issue=true","startTime":257.80000019073486,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":257.80000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":546.3000001907349,"responseStart":0,"secureConnectionStart":0},{"duration":227.5,"initiatorType":"script","name":"https://jira.mariadb.org/s/5d5e8fe91fbc506585e83ea3b62ccc4b-CDN/lu2cib/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/js/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.js?jira.create.linked.issue=true&locale=en","startTime":258,"connectEnd":258,"connectStart":258,"domainLookupEnd":258,"domainLookupStart":258,"fetchStart":258,"redirectEnd":0,"redirectStart":0,"requestStart":258,"responseEnd":485.5,"responseStart":485.5,"secureConnectionStart":258},{"duration":439.19999980926514,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-js/jira.webresources:bigpipe-js.js","startTime":285.40000009536743,"connectEnd":285.40000009536743,"connectStart":285.40000009536743,"domainLookupEnd":285.40000009536743,"domainLookupStart":285.40000009536743,"fetchStart":285.40000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":285.40000009536743,"responseEnd":724.5999999046326,"responseStart":724.5999999046326,"secureConnectionStart":285.40000009536743},{"duration":578.8999996185303,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-init/jira.webresources:bigpipe-init.js","startTime":291.30000019073486,"connectEnd":291.30000019073486,"connectStart":291.30000019073486,"domainLookupEnd":291.30000019073486,"domainLookupStart":291.30000019073486,"fetchStart":291.30000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":291.30000019073486,"responseEnd":870.1999998092651,"responseStart":870.1999998092651,"secureConnectionStart":291.30000019073486},{"duration":151.80000019073486,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":574.0999999046326,"connectEnd":574.0999999046326,"connectStart":574.0999999046326,"domainLookupEnd":574.0999999046326,"domainLookupStart":574.0999999046326,"fetchStart":574.0999999046326,"redirectEnd":0,"redirectStart":0,"requestStart":574.0999999046326,"responseEnd":725.9000000953674,"responseStart":725.9000000953674,"secureConnectionStart":574.0999999046326},{"duration":41,"initiatorType":"link","name":"https://jira.mariadb.org/s/d5715adaadd168a9002b108b2b039b50-CDN/lu2cib/820016/12ta74/be4b45e9cec53099498fa61c8b7acba4/_/download/contextbatch/css/jira.project.sidebar,-_super,-project.issue.navigator,-jira.general,-jira.browse.project,-jira.view.issue,-jira.global,-atl.general,-com.atlassian.jira.projects.sidebar.init/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true","startTime":868.9000000953674,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":868.9000000953674,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":909.9000000953674,"responseStart":0,"secureConnectionStart":0},{"duration":141.69999980926514,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/e65b778d185daf5aee24936755b43da6/_/download/contextbatch/js/browser-metrics-plugin.contrib,-_super,-project.issue.navigator,-jira.view.issue,-atl.general/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true","startTime":870,"connectEnd":870,"connectStart":870,"domainLookupEnd":870,"domainLookupStart":870,"fetchStart":870,"redirectEnd":0,"redirectStart":0,"requestStart":870,"responseEnd":1011.6999998092651,"responseStart":1011.6999998092651,"secureConnectionStart":870},{"duration":163.69999980926514,"initiatorType":"script","name":"https://jira.mariadb.org/s/097ae97cb8fbec7d6ea4bbb1f26955b9-CDN/lu2cib/820016/12ta74/be4b45e9cec53099498fa61c8b7acba4/_/download/contextbatch/js/jira.project.sidebar,-_super,-project.issue.navigator,-jira.general,-jira.browse.project,-jira.view.issue,-jira.global,-atl.general,-com.atlassian.jira.projects.sidebar.init/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true","startTime":870.5,"connectEnd":870.5,"connectStart":870.5,"domainLookupEnd":870.5,"domainLookupStart":870.5,"fetchStart":870.5,"redirectEnd":0,"redirectStart":0,"requestStart":870.5,"responseEnd":1034.1999998092651,"responseStart":1034.1999998092651,"secureConnectionStart":870.5},{"duration":245.90000009536743,"initiatorType":"script","name":"https://www.google-analytics.com/analytics.js","startTime":952.6999998092651,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":952.6999998092651,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1198.5999999046326,"responseStart":0,"secureConnectionStart":0},{"duration":164.30000019073486,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":976.5999999046326,"connectEnd":976.5999999046326,"connectStart":976.5999999046326,"domainLookupEnd":976.5999999046326,"domainLookupStart":976.5999999046326,"fetchStart":976.5999999046326,"redirectEnd":0,"redirectStart":0,"requestStart":976.5999999046326,"responseEnd":1140.9000000953674,"responseStart":1140.9000000953674,"secureConnectionStart":976.5999999046326}],"fetchStart":0,"domainLookupStart":0,"domainLookupEnd":0,"connectStart":0,"connectEnd":0,"requestStart":82,"responseStart":247,"responseEnd":294,"domLoading":251,"domInteractive":1239,"domContentLoadedEventStart":1239,"domContentLoadedEventEnd":1314,"domComplete":1792,"loadEventStart":1792,"loadEventEnd":1792,"userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","marks":[{"name":"bigPipe.sidebar-id.start","time":1201.1999998092651},{"name":"bigPipe.sidebar-id.end","time":1202.0999999046326},{"name":"bigPipe.activity-panel-pipe-id.start","time":1202.3000001907349},{"name":"bigPipe.activity-panel-pipe-id.end","time":1203.0999999046326},{"name":"activityTabFullyLoaded","time":1336.4000000953674}],"measures":[],"correlationId":"27da96bdc9e058","effectiveType":"4g","downlink":9.4,"rtt":0,"serverDuration":103,"dbReadsTimeInMs":15,"dbConnsTimeInMs":23,"applicationHash":"9d11dbea5f4be3d4cc21f03a88dd11d8c8687422","experiments":[]}}
Bug
Major
I have since updated my servers to 10.1.11 and the issue is still happening.