mariadb.service entered failed mode after power down/up event
galera cluster was powered down /up after power supply problem
mysql process is running but cannot login to mysql shell and it's found that
mariadb.service was entered failed mode
note: cluster was in synced state before the power down
# ps aux | grep -v grep | grep mysql
|
mysql 965 0.0 2.2 629204 46616 ? Ssl 02:08 0:40 /usr/sbin/mysqld --wsrep_start_position=dff6e041-1005-11e8-85c9-965f304f37bc:131668
|
|
# systemctl status mariadb.service
|
● mariadb.service - MariaDB 10.3.5 database server
|
Loaded: loaded (/usr/lib/systemd/system/mariadb.service; enabled; vendor preset: disabled)
|
Drop-In: /etc/systemd/system/mariadb.service.d
|
└─migrated-from-my.cnf-settings.conf
|
Active: failed (Result: timeout) since Tue 2018-03-27 02:11:48 EEST; 15h ago
|
Docs: man:mysqld(8)
|
https://mariadb.com/kb/en/library/systemd/
|
Process: 867 ExecStartPre=/bin/sh -c [ ! -e /usr/bin/galera_recovery ] && VAR= || VAR=`/usr/bin/galera_recovery`; [ $? -eq 0 ] && systemctl set-environment _WSREP_START_POSITION=$VAR || exit 1 (code=exited, status=0/SUCCESS)
|
Process: 860 ExecStartPre=/bin/sh -c systemctl unset-environment _WSREP_START_POSITION (code=exited, status=0/SUCCESS)
|
Main PID: 965
|
CGroup: /system.slice/mariadb.service
|
└─965 /usr/sbin/mysqld --wsrep_start_position=dff6e041-1005-11e8-85c9-965f304f37bc:131668
|
|
Mar 27 02:08:44 localhost.localdomain systemd[1]: Starting MariaDB 10.3.5 database server...
|
Mar 27 02:08:48 localhost.localdomain sh[867]: WSREP: Recovered position dff6e041-1005-11e8-85c9-965f304f37bc:131668
|
Mar 27 02:08:48 localhost.localdomain mysqld[965]: 2018-03-27 2:08:48 0 [Note] /usr/sbin/mysqld (mysqld 10.3.5-MariaDB) starting as process 965 ...
|
Mar 27 02:10:18 t4w5.xentio.lan systemd[1]: mariadb.service start operation timed out. Terminating.
|
Mar 27 02:11:48 t4w5.xentio.lan systemd[1]: mariadb.service stop-final-sigterm timed out. Skipping SIGKILL. Entering failed mode.
|
Mar 27 02:11:48 t4w5.xentio.lan systemd[1]: Failed to start MariaDB 10.3.5 database server.
|
Mar 27 02:11:48 t4w5.xentio.lan systemd[1]: Unit mariadb.service entered failed state.
|
Mar 27 02:11:48 t4w5.xentio.lan systemd[1]: mariadb.service failed.
|
|
{"report":{"fcp":925.2999999523163,"ttfb":251.90000009536743,"pageVisibility":"visible","entityId":66546,"key":"jira.project.issue.view-issue","isInitial":true,"threshold":1000,"elementTimings":{},"userDeviceMemory":8,"userDeviceProcessors":64,"apdex":0.5,"journeyId":"710e6278-b7aa-417c-8ea8-88bf41157069","navigationType":0,"readyForUser":1045.0999999046326,"redirectCount":0,"resourceLoadedEnd":972.5999999046326,"resourceLoadedStart":257,"resourceTiming":[{"duration":153,"initiatorType":"link","name":"https://jira.mariadb.org/s/2c21342762a6a02add1c328bed317ffd-CDN/lu2cib/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/css/_super/batch.css","startTime":257,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":257,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":410,"responseStart":0,"secureConnectionStart":0},{"duration":153,"initiatorType":"link","name":"https://jira.mariadb.org/s/7ebd35e77e471bc30ff0eba799ebc151-CDN/lu2cib/820016/12ta74/494e4c556ecbb29f90a3d3b4f09cb99c/_/download/contextbatch/css/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true&whisper-enabled=true","startTime":257.2999999523163,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":257.2999999523163,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":410.2999999523163,"responseStart":0,"secureConnectionStart":0},{"duration":197,"initiatorType":"script","name":"https://jira.mariadb.org/s/0917945aaa57108d00c5076fea35e069-CDN/lu2cib/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/js/_super/batch.js?locale=en","startTime":257.5,"connectEnd":257.5,"connectStart":257.5,"domainLookupEnd":257.5,"domainLookupStart":257.5,"fetchStart":257.5,"redirectEnd":0,"redirectStart":0,"requestStart":257.5,"responseEnd":454.5,"responseStart":454.5,"secureConnectionStart":257.5},{"duration":303.59999990463257,"initiatorType":"script","name":"https://jira.mariadb.org/s/2d8175ec2fa4c816e8023260bd8c1786-CDN/lu2cib/820016/12ta74/494e4c556ecbb29f90a3d3b4f09cb99c/_/download/contextbatch/js/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true&whisper-enabled=true","startTime":257.7000000476837,"connectEnd":257.7000000476837,"connectStart":257.7000000476837,"domainLookupEnd":257.7000000476837,"domainLookupStart":257.7000000476837,"fetchStart":257.7000000476837,"redirectEnd":0,"redirectStart":0,"requestStart":257.7000000476837,"responseEnd":561.2999999523163,"responseStart":561.2999999523163,"secureConnectionStart":257.7000000476837},{"duration":198.60000014305115,"initiatorType":"script","name":"https://jira.mariadb.org/s/a9324d6758d385eb45c462685ad88f1d-CDN/lu2cib/820016/12ta74/c92c0caa9a024ae85b0ebdbed7fb4bd7/_/download/contextbatch/js/atl.global,-_super/batch.js?locale=en","startTime":257.7999999523163,"connectEnd":257.7999999523163,"connectStart":257.7999999523163,"domainLookupEnd":257.7999999523163,"domainLookupStart":257.7999999523163,"fetchStart":257.7999999523163,"redirectEnd":0,"redirectStart":0,"requestStart":257.7999999523163,"responseEnd":456.40000009536743,"responseStart":456.40000009536743,"secureConnectionStart":257.7999999523163},{"duration":198.90000009536743,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-en/jira.webresources:calendar-en.js","startTime":258,"connectEnd":258,"connectStart":258,"domainLookupEnd":258,"domainLookupStart":258,"fetchStart":258,"redirectEnd":0,"redirectStart":0,"requestStart":258,"responseEnd":456.90000009536743,"responseStart":456.90000009536743,"secureConnectionStart":258},{"duration":199.20000004768372,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-localisation-moment/jira.webresources:calendar-localisation-moment.js","startTime":258.2000000476837,"connectEnd":258.2000000476837,"connectStart":258.2000000476837,"domainLookupEnd":258.2000000476837,"domainLookupStart":258.2000000476837,"fetchStart":258.2000000476837,"redirectEnd":0,"redirectStart":0,"requestStart":258.2000000476837,"responseEnd":457.40000009536743,"responseStart":457.40000009536743,"secureConnectionStart":258.2000000476837},{"duration":198.89999985694885,"initiatorType":"link","name":"https://jira.mariadb.org/s/b04b06a02d1959df322d9cded3aeecc1-CDN/lu2cib/820016/12ta74/a2ff6aa845ffc9a1d22fe23d9ee791fc/_/download/contextbatch/css/jira.global.look-and-feel,-_super/batch.css","startTime":258.40000009536743,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":258.40000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":457.2999999523163,"responseStart":0,"secureConnectionStart":0},{"duration":199.40000009536743,"initiatorType":"script","name":"https://jira.mariadb.org/rest/api/1.0/shortcuts/820016/47140b6e0a9bc2e4913da06536125810/shortcuts.js?context=issuenavigation&context=issueaction","startTime":258.5,"connectEnd":258.5,"connectStart":258.5,"domainLookupEnd":258.5,"domainLookupStart":258.5,"fetchStart":258.5,"redirectEnd":0,"redirectStart":0,"requestStart":258.5,"responseEnd":457.90000009536743,"responseStart":457.90000009536743,"secureConnectionStart":258.5},{"duration":199.39999985694885,"initiatorType":"link","name":"https://jira.mariadb.org/s/3ac36323ba5e4eb0af2aa7ac7211b4bb-CDN/lu2cib/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/css/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.css?jira.create.linked.issue=true","startTime":258.7000000476837,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":258.7000000476837,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":458.09999990463257,"responseStart":0,"secureConnectionStart":0},{"duration":306.5,"initiatorType":"script","name":"https://jira.mariadb.org/s/5d5e8fe91fbc506585e83ea3b62ccc4b-CDN/lu2cib/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/js/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.js?jira.create.linked.issue=true&locale=en","startTime":258.7999999523163,"connectEnd":258.7999999523163,"connectStart":258.7999999523163,"domainLookupEnd":258.7999999523163,"domainLookupStart":258.7999999523163,"fetchStart":258.7999999523163,"redirectEnd":0,"redirectStart":0,"requestStart":258.7999999523163,"responseEnd":565.2999999523163,"responseStart":565.2999999523163,"secureConnectionStart":258.7999999523163},{"duration":384.40000009536743,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-js/jira.webresources:bigpipe-js.js","startTime":269.59999990463257,"connectEnd":269.59999990463257,"connectStart":269.59999990463257,"domainLookupEnd":269.59999990463257,"domainLookupStart":269.59999990463257,"fetchStart":269.59999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":269.59999990463257,"responseEnd":654,"responseStart":654,"secureConnectionStart":269.59999990463257},{"duration":454.90000009536743,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-init/jira.webresources:bigpipe-init.js","startTime":270.5,"connectEnd":270.5,"connectStart":270.5,"domainLookupEnd":270.5,"domainLookupStart":270.5,"fetchStart":270.5,"redirectEnd":0,"redirectStart":0,"requestStart":270.5,"responseEnd":725.4000000953674,"responseStart":725.4000000953674,"secureConnectionStart":270.5},{"duration":44,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":611.2000000476837,"connectEnd":611.2000000476837,"connectStart":611.2000000476837,"domainLookupEnd":611.2000000476837,"domainLookupStart":611.2000000476837,"fetchStart":611.2000000476837,"redirectEnd":0,"redirectStart":0,"requestStart":611.2000000476837,"responseEnd":655.2000000476837,"responseStart":655.2000000476837,"secureConnectionStart":611.2000000476837},{"duration":203.29999995231628,"initiatorType":"link","name":"https://jira.mariadb.org/s/d5715adaadd168a9002b108b2b039b50-CDN/lu2cib/820016/12ta74/be4b45e9cec53099498fa61c8b7acba4/_/download/contextbatch/css/jira.project.sidebar,-_super,-project.issue.navigator,-jira.general,-jira.browse.project,-jira.view.issue,-jira.global,-atl.general,-com.atlassian.jira.projects.sidebar.init/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true&whisper-enabled=true","startTime":677,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":677,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":880.2999999523163,"responseStart":0,"secureConnectionStart":0},{"duration":202.70000004768372,"initiatorType":"link","name":"https://jira.mariadb.org/s/50bc9be5bfead1a25e72c1a9338c94f6-CDN/lu2cib/820016/12ta74/e108c7645258ccb43280ed3404e3e949/_/download/contextbatch/css/com.atlassian.jira.plugins.jira-development-integration-plugin:0,-_super,-jira.view.issue,-jira.global,-jira.general,-jira.browse.project,-project.issue.navigator,-atl.general/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true&whisper-enabled=true","startTime":677.7999999523163,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":677.7999999523163,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":880.5,"responseStart":0,"secureConnectionStart":0},{"duration":287.89999985694885,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/e65b778d185daf5aee24936755b43da6/_/download/contextbatch/js/browser-metrics-plugin.contrib,-_super,-project.issue.navigator,-jira.view.issue,-atl.general/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true&whisper-enabled=true","startTime":678.7000000476837,"connectEnd":678.7000000476837,"connectStart":678.7000000476837,"domainLookupEnd":678.7000000476837,"domainLookupStart":678.7000000476837,"fetchStart":678.7000000476837,"redirectEnd":0,"redirectStart":0,"requestStart":678.7000000476837,"responseEnd":966.5999999046326,"responseStart":966.5999999046326,"secureConnectionStart":678.7000000476837},{"duration":292.2000000476837,"initiatorType":"script","name":"https://jira.mariadb.org/s/097ae97cb8fbec7d6ea4bbb1f26955b9-CDN/lu2cib/820016/12ta74/be4b45e9cec53099498fa61c8b7acba4/_/download/contextbatch/js/jira.project.sidebar,-_super,-project.issue.navigator,-jira.general,-jira.browse.project,-jira.view.issue,-jira.global,-atl.general,-com.atlassian.jira.projects.sidebar.init/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true&whisper-enabled=true","startTime":679.0999999046326,"connectEnd":679.0999999046326,"connectStart":679.0999999046326,"domainLookupEnd":679.0999999046326,"domainLookupStart":679.0999999046326,"fetchStart":679.0999999046326,"redirectEnd":0,"redirectStart":0,"requestStart":679.0999999046326,"responseEnd":971.2999999523163,"responseStart":971.2999999523163,"secureConnectionStart":679.0999999046326},{"duration":293.19999980926514,"initiatorType":"script","name":"https://jira.mariadb.org/s/e0bf5781d46ea69fb123572974cf39de-CDN/lu2cib/820016/12ta74/e108c7645258ccb43280ed3404e3e949/_/download/contextbatch/js/com.atlassian.jira.plugins.jira-development-integration-plugin:0,-_super,-jira.view.issue,-jira.global,-jira.general,-jira.browse.project,-project.issue.navigator,-atl.general/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true&whisper-enabled=true","startTime":679.4000000953674,"connectEnd":679.4000000953674,"connectStart":679.4000000953674,"domainLookupEnd":679.4000000953674,"domainLookupStart":679.4000000953674,"fetchStart":679.4000000953674,"redirectEnd":0,"redirectStart":0,"requestStart":679.4000000953674,"responseEnd":972.5999999046326,"responseStart":972.5999999046326,"secureConnectionStart":679.4000000953674},{"duration":162.30000019073486,"initiatorType":"script","name":"https://www.google-analytics.com/analytics.js","startTime":919.0999999046326,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":919.0999999046326,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1081.4000000953674,"responseStart":0,"secureConnectionStart":0},{"duration":168,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":926.9000000953674,"connectEnd":926.9000000953674,"connectStart":926.9000000953674,"domainLookupEnd":926.9000000953674,"domainLookupStart":926.9000000953674,"fetchStart":926.9000000953674,"redirectEnd":0,"redirectStart":0,"requestStart":926.9000000953674,"responseEnd":1094.9000000953674,"responseStart":1094.9000000953674,"secureConnectionStart":926.9000000953674}],"fetchStart":0,"domainLookupStart":38,"domainLookupEnd":48,"connectStart":48,"connectEnd":69,"secureConnectionStart":57,"requestStart":69,"responseStart":251,"responseEnd":270,"domLoading":255,"domInteractive":1124,"domContentLoadedEventStart":1124,"domContentLoadedEventEnd":1169,"domComplete":1445,"loadEventStart":1445,"loadEventEnd":1447,"userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","marks":[{"name":"bigPipe.sidebar-id.start","time":1083.0999999046326},{"name":"bigPipe.sidebar-id.end","time":1083.9000000953674},{"name":"bigPipe.activity-panel-pipe-id.start","time":1084.2000000476837},{"name":"bigPipe.activity-panel-pipe-id.end","time":1086.7000000476837},{"name":"activityTabFullyLoaded","time":1183.4000000953674}],"measures":[],"correlationId":"e8e43ededc2619","effectiveType":"4g","downlink":10,"rtt":0,"serverDuration":103,"dbReadsTimeInMs":9,"dbConnsTimeInMs":18,"applicationHash":"9d11dbea5f4be3d4cc21f03a88dd11d8c8687422","experiments":[]}}