We have three nodes MariaDB-Galera-Cluster 10.0.30
Today one of nodes (Node Number 2) disconnected from group.
After reviewing error log, I saw an error happened:
This was error:
170418 18:11:48 [ERROR] WSREP: Local state seqno (3472060319) is greater than group seqno (3472057835): states diverged. Aborting to avoid potential data loss. Remove '/home/mysql//grastate.dat' file and restart if you wish to continue. (FATAL).
So after review all three servers log, I think in the interval 18:06:58 to 18:11:50 The internet connection of node 2 have been disorder, and in the meantime (When it was Non-primary) several query could be executed and commited.
I've attached my error logs to see.
Node 1 IP : ..*.35
Node 2 IP : ..*.60 (Impacted)
Node 3 IP : ..*.206
Sorry for my bad English.
{"report":{"fcp":908.9000000953674,"ttfb":204.7000002861023,"pageVisibility":"visible","entityId":60964,"key":"jira.project.issue.view-issue","isInitial":true,"threshold":1000,"elementTimings":{},"userDeviceMemory":8,"userDeviceProcessors":32,"apdex":1,"journeyId":"d1c99a9d-cb15-42b8-8c98-419ce5bd30ef","navigationType":0,"readyForUser":983.4000000953674,"redirectCount":0,"resourceLoadedEnd":995.4000000953674,"resourceLoadedStart":218.30000019073486,"resourceTiming":[{"duration":115.5,"initiatorType":"link","name":"https://jira.mariadb.org/s/2c21342762a6a02add1c328bed317ffd-CDN/lu2cib/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/css/_super/batch.css","startTime":218.30000019073486,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":218.30000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":333.80000019073486,"responseStart":0,"secureConnectionStart":0},{"duration":116.5,"initiatorType":"link","name":"https://jira.mariadb.org/s/7ebd35e77e471bc30ff0eba799ebc151-CDN/lu2cib/820016/12ta74/494e4c556ecbb29f90a3d3b4f09cb99c/_/download/contextbatch/css/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true&whisper-enabled=true","startTime":218.7000002861023,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":218.7000002861023,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":335.2000002861023,"responseStart":0,"secureConnectionStart":0},{"duration":185.19999980926514,"initiatorType":"script","name":"https://jira.mariadb.org/s/0917945aaa57108d00c5076fea35e069-CDN/lu2cib/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/js/_super/batch.js?locale=en","startTime":218.80000019073486,"connectEnd":218.80000019073486,"connectStart":218.80000019073486,"domainLookupEnd":218.80000019073486,"domainLookupStart":218.80000019073486,"fetchStart":218.80000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":218.80000019073486,"responseEnd":404,"responseStart":404,"secureConnectionStart":218.80000019073486},{"duration":241.5,"initiatorType":"script","name":"https://jira.mariadb.org/s/2d8175ec2fa4c816e8023260bd8c1786-CDN/lu2cib/820016/12ta74/494e4c556ecbb29f90a3d3b4f09cb99c/_/download/contextbatch/js/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true&whisper-enabled=true","startTime":219,"connectEnd":219,"connectStart":219,"domainLookupEnd":219,"domainLookupStart":219,"fetchStart":219,"redirectEnd":0,"redirectStart":0,"requestStart":219,"responseEnd":460.5,"responseStart":460.5,"secureConnectionStart":219},{"duration":244.89999961853027,"initiatorType":"script","name":"https://jira.mariadb.org/s/a9324d6758d385eb45c462685ad88f1d-CDN/lu2cib/820016/12ta74/c92c0caa9a024ae85b0ebdbed7fb4bd7/_/download/contextbatch/js/atl.global,-_super/batch.js?locale=en","startTime":219.2000002861023,"connectEnd":219.2000002861023,"connectStart":219.2000002861023,"domainLookupEnd":219.2000002861023,"domainLookupStart":219.2000002861023,"fetchStart":219.2000002861023,"redirectEnd":0,"redirectStart":0,"requestStart":219.2000002861023,"responseEnd":464.09999990463257,"responseStart":464.09999990463257,"secureConnectionStart":219.2000002861023},{"duration":245.30000019073486,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-en/jira.webresources:calendar-en.js","startTime":219.40000009536743,"connectEnd":219.40000009536743,"connectStart":219.40000009536743,"domainLookupEnd":219.40000009536743,"domainLookupStart":219.40000009536743,"fetchStart":219.40000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":219.40000009536743,"responseEnd":464.7000002861023,"responseStart":464.7000002861023,"secureConnectionStart":219.40000009536743},{"duration":245.59999990463257,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-localisation-moment/jira.webresources:calendar-localisation-moment.js","startTime":219.5,"connectEnd":219.5,"connectStart":219.5,"domainLookupEnd":219.5,"domainLookupStart":219.5,"fetchStart":219.5,"redirectEnd":0,"redirectStart":0,"requestStart":219.5,"responseEnd":465.09999990463257,"responseStart":465,"secureConnectionStart":219.5},{"duration":326.69999980926514,"initiatorType":"link","name":"https://jira.mariadb.org/s/b04b06a02d1959df322d9cded3aeecc1-CDN/lu2cib/820016/12ta74/a2ff6aa845ffc9a1d22fe23d9ee791fc/_/download/contextbatch/css/jira.global.look-and-feel,-_super/batch.css","startTime":219.7000002861023,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":219.7000002861023,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":546.4000000953674,"responseStart":0,"secureConnectionStart":0},{"duration":245.59999990463257,"initiatorType":"script","name":"https://jira.mariadb.org/rest/api/1.0/shortcuts/820016/47140b6e0a9bc2e4913da06536125810/shortcuts.js?context=issuenavigation&context=issueaction","startTime":219.90000009536743,"connectEnd":219.90000009536743,"connectStart":219.90000009536743,"domainLookupEnd":219.90000009536743,"domainLookupStart":219.90000009536743,"fetchStart":219.90000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":219.90000009536743,"responseEnd":465.5,"responseStart":465.5,"secureConnectionStart":219.90000009536743},{"duration":326.40000009536743,"initiatorType":"link","name":"https://jira.mariadb.org/s/3ac36323ba5e4eb0af2aa7ac7211b4bb-CDN/lu2cib/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/css/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.css?jira.create.linked.issue=true","startTime":220.09999990463257,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":220.09999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":546.5,"responseStart":0,"secureConnectionStart":0},{"duration":245.89999961853027,"initiatorType":"script","name":"https://jira.mariadb.org/s/5d5e8fe91fbc506585e83ea3b62ccc4b-CDN/lu2cib/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/js/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.js?jira.create.linked.issue=true&locale=en","startTime":220.2000002861023,"connectEnd":220.2000002861023,"connectStart":220.2000002861023,"domainLookupEnd":220.2000002861023,"domainLookupStart":220.2000002861023,"fetchStart":220.2000002861023,"redirectEnd":0,"redirectStart":0,"requestStart":220.2000002861023,"responseEnd":466.09999990463257,"responseStart":466.09999990463257,"secureConnectionStart":220.2000002861023},{"duration":722.8000001907349,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-js/jira.webresources:bigpipe-js.js","startTime":221.09999990463257,"connectEnd":221.09999990463257,"connectStart":221.09999990463257,"domainLookupEnd":221.09999990463257,"domainLookupStart":221.09999990463257,"fetchStart":221.09999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":221.09999990463257,"responseEnd":943.9000000953674,"responseStart":943.9000000953674,"secureConnectionStart":221.09999990463257},{"duration":718.3000001907349,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-init/jira.webresources:bigpipe-init.js","startTime":226.5,"connectEnd":226.5,"connectStart":226.5,"domainLookupEnd":226.5,"domainLookupStart":226.5,"fetchStart":226.5,"redirectEnd":0,"redirectStart":0,"requestStart":226.5,"responseEnd":944.8000001907349,"responseStart":944.8000001907349,"secureConnectionStart":226.5},{"duration":95.2999997138977,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":558.2000002861023,"connectEnd":558.2000002861023,"connectStart":558.2000002861023,"domainLookupEnd":558.2000002861023,"domainLookupStart":558.2000002861023,"fetchStart":558.2000002861023,"redirectEnd":0,"redirectStart":0,"requestStart":558.2000002861023,"responseEnd":653.5,"responseStart":653.5,"secureConnectionStart":558.2000002861023},{"duration":180.30000019073486,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/e65b778d185daf5aee24936755b43da6/_/download/contextbatch/js/browser-metrics-plugin.contrib,-_super,-atl.general/batch.js?agile_global_admin_condition=true&jag=true&slack-enabled=true&whisper-enabled=true","startTime":815.0999999046326,"connectEnd":815.0999999046326,"connectStart":815.0999999046326,"domainLookupEnd":815.0999999046326,"domainLookupStart":815.0999999046326,"fetchStart":815.0999999046326,"redirectEnd":0,"redirectStart":0,"requestStart":815.0999999046326,"responseEnd":995.4000000953674,"responseStart":995.3000001907349,"secureConnectionStart":815.0999999046326},{"duration":112.30000019073486,"initiatorType":"script","name":"https://www.google-analytics.com/analytics.js","startTime":902.4000000953674,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":902.4000000953674,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1014.7000002861023,"responseStart":0,"secureConnectionStart":0}],"fetchStart":1,"domainLookupStart":1,"domainLookupEnd":1,"connectStart":1,"connectEnd":1,"requestStart":55,"responseStart":205,"responseEnd":219,"domLoading":216,"domInteractive":1044,"domContentLoadedEventStart":1044,"domContentLoadedEventEnd":1089,"domComplete":1250,"loadEventStart":1250,"loadEventEnd":1251,"userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","marks":[{"name":"bigPipe.sidebar-id.start","time":1021.7000002861023},{"name":"bigPipe.sidebar-id.end","time":1022.5},{"name":"bigPipe.activity-panel-pipe-id.start","time":1022.5},{"name":"bigPipe.activity-panel-pipe-id.end","time":1023.5999999046326},{"name":"activityTabFullyLoaded","time":1107.0999999046326}],"measures":[],"correlationId":"44eb9d71fe41fd","effectiveType":"4g","downlink":10,"rtt":0,"serverDuration":98,"dbReadsTimeInMs":13,"dbConnsTimeInMs":23,"applicationHash":"9d11dbea5f4be3d4cc21f03a88dd11d8c8687422","experiments":[]}}
Bug
Major
I experience the same on MariaDB 10.3 and I have seen it before on other versions.
I think this has to do with network instability and frequent IST on clusters that are actively used. Somehow, if the write node gets disconnected a lot, it ends up in this state thinking it is inconsistent. Each time I ran into it I tried to find out if it really was inconsistent, but the few transactions that I checked to verify consistency were correct: The modified row was the same in the entire cluster, and the same row was present on all cluster nodes.
My guess is that IST succeeded but this did not get stored as succeeded in grastate.dat and/or inoodb transactional galera status.