Our environment is 3 nodes Galera cluster - 2 DB nodes + 1 arbitrator. HAProxy redirected traffic to one 1 DB node.
We encountered index corruption and, then, node inconsistent on standby node.
We needed to restart DB node manually and let it triggered implicitly SST to get resolve.
Kindly advise what we can check to troubleshot this case.
2024-01-03 14:52:10 7 [ERROR] InnoDB: We detected index corruption in an InnoDB type table. You have to dump + drop + reimport the table or, in a case of widespread corruption, dump all InnoDB tables and recreate the whole tablespace. If the mariadbd server crashes after the startup or when you dump the tables. Please refer to https://mariadb.com/kb/en/library/innodb-recovery-modes/ for information about forcing recovery.
|
2024-01-03 14:52:10 12 [ERROR] mariadbd: Index for table 'pa_pos_psdo' is corrupt; try to repair it
|
2024-01-03 14:52:10 7 [ERROR] mariadbd: Index for table 'pa_pos_psdo' is corrupt; try to repair it
|
2024-01-03 14:52:10 7 [ERROR] Slave SQL: Could not execute Write_rows_v1 event on table tes-public.pa_pos_psdo; Index for table 'pa_pos_psdo' is corrupt; try to repair it, Error_code: 1034; handler error HA_ERR_CRASHED; the event's master log FIRST, end_log_pos 9325, Internal MariaDB error code: 1034
|
2024-01-03 14:52:10 7 [Warning] WSREP: Event 6 Write_rows_v1 apply failed: 126, seqno 27074073
|
2024-01-03 14:52:10 12 [ERROR] Slave SQL: Could not execute Write_rows_v1 event on table tes-public.pa_pos_psdo; Table tes@002dpublic/pa_pos_psdo in tablespace 94889212653016 corrupted., Error_code: 126; Index for table 'pa_pos_psdo' is corrupt; try to repair it, Error_code: 1034; handler error HA_ERR_CRASHED; the event's master log FIRST, end_log_pos 9335, Internal MariaDB error code: 1034
|
2024-01-03 14:52:10 12 [Warning] WSREP: Event 6 Write_rows_v1 apply failed: 126, seqno 27074074
|
2024-01-03 14:52:10 6 [ERROR] mariadbd: Index for table 'pa_pos_psdo' is corrupt; try to repair it
|
2024-01-03 14:52:10 6 [ERROR] Slave SQL: Could not execute Write_rows_v1 event on table tes-public.pa_pos_psdo; Table tes@002dpublic/pa_pos_psdo in tablespace 94889212653016 corrupted., Error_code: 126; Index for table 'pa_pos_psdo' is corrupt; try to repair it, Error_code: 1034; handler error HA_ERR_CRASHED; the event's master log FIRST, end_log_pos 9325, Internal MariaDB error code: 1034
|
2024-01-03 14:52:10 6 [Warning] WSREP: Event 6 Write_rows_v1 apply failed: 126, seqno 27074076
|
2024-01-03 14:52:10 0 [Note] WSREP: Member 1(t2vdbs-hpftes-tesdbppd-2-01) initiates vote on 9b0bd445-e8dc-11ed-aff1-6ec3700a3571:27074073,9359a244aa1e2038: Index for table 'pa_pos_psdo' is corrupt; try to repair it, Error_code: 1034;
|
|
xxx
|
|
2024-01-03 14:52:10 7 [ERROR] WSREP: Inconsistency detected: Inconsistent by consensus on 9b0bd445-e8dc-11ed-aff1-6ec3700a3571:27074073
|
at /home/buildbot/buildbot/build/galera/src/replicator_smm.cpp:process_apply_error():1357
|
{"report":{"fcp":755.5,"ttfb":162.29999995231628,"pageVisibility":"visible","entityId":127207,"key":"jira.project.issue.view-issue","isInitial":true,"threshold":1000,"elementTimings":{},"userDeviceMemory":8,"userDeviceProcessors":64,"apdex":1,"journeyId":"9cf676cb-ea60-49e3-8159-701bb89a2ff2","navigationType":0,"readyForUser":833.8999998569489,"redirectCount":0,"resourceLoadedEnd":680.7000000476837,"resourceLoadedStart":168.29999995231628,"resourceTiming":[{"duration":48.700000047683716,"initiatorType":"link","name":"https://jira.mariadb.org/s/2c21342762a6a02add1c328bed317ffd-CDN/lu2bu7/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/css/_super/batch.css","startTime":168.29999995231628,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":168.29999995231628,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":217,"responseStart":0,"secureConnectionStart":0},{"duration":49.200000047683716,"initiatorType":"link","name":"https://jira.mariadb.org/s/7ebd35e77e471bc30ff0eba799ebc151-CDN/lu2bu7/820016/12ta74/8679b4946efa1a0bb029a3a22206fb5d/_/download/contextbatch/css/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true","startTime":168.5,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":168.5,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":217.70000004768372,"responseStart":0,"secureConnectionStart":0},{"duration":107.5,"initiatorType":"script","name":"https://jira.mariadb.org/s/fbf975c0cce4b1abf04784eeae9ba1f4-CDN/lu2bu7/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/js/_super/batch.js?locale=en","startTime":168.70000004768372,"connectEnd":168.70000004768372,"connectStart":168.70000004768372,"domainLookupEnd":168.70000004768372,"domainLookupStart":168.70000004768372,"fetchStart":168.70000004768372,"redirectEnd":0,"redirectStart":0,"requestStart":168.70000004768372,"responseEnd":276.2000000476837,"responseStart":276.2000000476837,"secureConnectionStart":168.70000004768372},{"duration":212.59999990463257,"initiatorType":"script","name":"https://jira.mariadb.org/s/099b33461394b8015fc36c0a4b96e19f-CDN/lu2bu7/820016/12ta74/8679b4946efa1a0bb029a3a22206fb5d/_/download/contextbatch/js/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true","startTime":168.79999995231628,"connectEnd":168.79999995231628,"connectStart":168.79999995231628,"domainLookupEnd":168.79999995231628,"domainLookupStart":168.79999995231628,"fetchStart":168.79999995231628,"redirectEnd":0,"redirectStart":0,"requestStart":168.79999995231628,"responseEnd":381.39999985694885,"responseStart":381.39999985694885,"secureConnectionStart":168.79999995231628},{"duration":216.40000009536743,"initiatorType":"script","name":"https://jira.mariadb.org/s/94c15bff32baef80f4096a08aceae8bc-CDN/lu2bu7/820016/12ta74/c92c0caa9a024ae85b0ebdbed7fb4bd7/_/download/contextbatch/js/atl.global,-_super/batch.js?locale=en","startTime":168.89999985694885,"connectEnd":168.89999985694885,"connectStart":168.89999985694885,"domainLookupEnd":168.89999985694885,"domainLookupStart":168.89999985694885,"fetchStart":168.89999985694885,"redirectEnd":0,"redirectStart":0,"requestStart":168.89999985694885,"responseEnd":385.2999999523163,"responseStart":385.2999999523163,"secureConnectionStart":168.89999985694885},{"duration":216.70000004768372,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bu7/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-en/jira.webresources:calendar-en.js","startTime":169.09999990463257,"connectEnd":169.09999990463257,"connectStart":169.09999990463257,"domainLookupEnd":169.09999990463257,"domainLookupStart":169.09999990463257,"fetchStart":169.09999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":169.09999990463257,"responseEnd":385.7999999523163,"responseStart":385.7999999523163,"secureConnectionStart":169.09999990463257},{"duration":217.29999995231628,"initiatorType":"link","name":"https://jira.mariadb.org/s/b04b06a02d1959df322d9cded3aeecc1-CDN/lu2bu7/820016/12ta74/a2ff6aa845ffc9a1d22fe23d9ee791fc/_/download/contextbatch/css/jira.global.look-and-feel,-_super/batch.css","startTime":169.20000004768372,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":169.20000004768372,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":386.5,"responseStart":0,"secureConnectionStart":0},{"duration":216.89999985694885,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bu7/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-localisation-moment/jira.webresources:calendar-localisation-moment.js","startTime":169.20000004768372,"connectEnd":169.20000004768372,"connectStart":169.20000004768372,"domainLookupEnd":169.20000004768372,"domainLookupStart":169.20000004768372,"fetchStart":169.20000004768372,"redirectEnd":0,"redirectStart":0,"requestStart":169.20000004768372,"responseEnd":386.09999990463257,"responseStart":386.09999990463257,"secureConnectionStart":169.20000004768372},{"duration":217.20000004768372,"initiatorType":"script","name":"https://jira.mariadb.org/rest/api/1.0/shortcuts/820016/47140b6e0a9bc2e4913da06536125810/shortcuts.js?context=issuenavigation&context=issueaction","startTime":169.29999995231628,"connectEnd":169.29999995231628,"connectStart":169.29999995231628,"domainLookupEnd":169.29999995231628,"domainLookupStart":169.29999995231628,"fetchStart":169.29999995231628,"redirectEnd":0,"redirectStart":0,"requestStart":169.29999995231628,"responseEnd":386.5,"responseStart":386.5,"secureConnectionStart":169.29999995231628},{"duration":217.39999985694885,"initiatorType":"link","name":"https://jira.mariadb.org/s/3ac36323ba5e4eb0af2aa7ac7211b4bb-CDN/lu2bu7/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/css/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.css?jira.create.linked.issue=true","startTime":169.5,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":169.5,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":386.89999985694885,"responseStart":0,"secureConnectionStart":0},{"duration":217.60000014305115,"initiatorType":"script","name":"https://jira.mariadb.org/s/3339d87fa2538a859872f2df449bf8d0-CDN/lu2bu7/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/js/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.js?jira.create.linked.issue=true&locale=en","startTime":169.59999990463257,"connectEnd":169.59999990463257,"connectStart":169.59999990463257,"domainLookupEnd":169.59999990463257,"domainLookupStart":169.59999990463257,"fetchStart":169.59999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":169.59999990463257,"responseEnd":387.2000000476837,"responseStart":387.09999990463257,"secureConnectionStart":169.59999990463257},{"duration":465.2000000476837,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bu7/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-js/jira.webresources:bigpipe-js.js","startTime":170.39999985694885,"connectEnd":170.39999985694885,"connectStart":170.39999985694885,"domainLookupEnd":170.39999985694885,"domainLookupStart":170.39999985694885,"fetchStart":170.39999985694885,"redirectEnd":0,"redirectStart":0,"requestStart":170.39999985694885,"responseEnd":635.5999999046326,"responseStart":635.5999999046326,"secureConnectionStart":170.39999985694885},{"duration":510.30000019073486,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bu7/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-init/jira.webresources:bigpipe-init.js","startTime":170.39999985694885,"connectEnd":170.39999985694885,"connectStart":170.39999985694885,"domainLookupEnd":170.39999985694885,"domainLookupStart":170.39999985694885,"fetchStart":170.39999985694885,"redirectEnd":0,"redirectStart":0,"requestStart":170.39999985694885,"responseEnd":680.7000000476837,"responseStart":680.5999999046326,"secureConnectionStart":170.39999985694885},{"duration":202.60000014305115,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":433.39999985694885,"connectEnd":433.39999985694885,"connectStart":433.39999985694885,"domainLookupEnd":433.39999985694885,"domainLookupStart":433.39999985694885,"fetchStart":433.39999985694885,"redirectEnd":0,"redirectStart":0,"requestStart":433.39999985694885,"responseEnd":636,"responseStart":636,"secureConnectionStart":433.39999985694885}],"fetchStart":0,"domainLookupStart":0,"domainLookupEnd":0,"connectStart":0,"connectEnd":0,"requestStart":31,"responseStart":162,"responseEnd":165,"domLoading":165,"domInteractive":895,"domContentLoadedEventStart":895,"domContentLoadedEventEnd":946,"domComplete":1209,"loadEventStart":1209,"loadEventEnd":1209,"userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","marks":[{"name":"bigPipe.sidebar-id.start","time":867.7999999523163},{"name":"bigPipe.sidebar-id.end","time":868.8999998569489},{"name":"bigPipe.activity-panel-pipe-id.start","time":869},{"name":"bigPipe.activity-panel-pipe-id.end","time":871.0999999046326},{"name":"activityTabFullyLoaded","time":967.7999999523163}],"measures":[],"correlationId":"4bb5fbc9234d28","effectiveType":"4g","downlink":9.2,"rtt":0,"serverDuration":79,"dbReadsTimeInMs":13,"dbConnsTimeInMs":20,"applicationHash":"9d11dbea5f4be3d4cc21f03a88dd11d8c8687422","experiments":[]}}
Bug
Major
I think that this could be a duplicate of
MDEV-31767orMDEV-30531.