The MCOL-4692 change eliminates a global mutex symbol that protects access to Columnstore.xml file crit section in OAMCache. This mutex allows to fetch the updated Columnstore.xml from disk on every operation. The current cluster configuration has two important parts: mariadb runtime and MCS runtime that in its turn includes a number of processes. Whilst CMAPI restarts MCS runtime processes every time when the cluster topology changes, e.g. failover case there is nobody to restart mariadb so the plugin code that resides in mariadb runtime doesn't fetch the updated Columnstore.xml so it doesn't know where DBRM controller or ExeMgr1 now are so clientrotator code fails to connect to ExeMgr1 until mariadb is restarted.
The suggested solution is to either:
- return the global mutex but use it in the plugin code only
- introduce service thread to watch and read Columnstore.xml
{"report":{"fcp":814.0999999046326,"ttfb":175.09999990463257,"pageVisibility":"visible","entityId":104843,"key":"jira.project.issue.view-issue","isInitial":true,"threshold":1000,"elementTimings":{},"userDeviceMemory":8,"userDeviceProcessors":64,"apdex":1,"journeyId":"a1db1583-a992-4e1c-9135-a86d6b3dff07","navigationType":0,"readyForUser":883.1999998092651,"redirectCount":0,"resourceLoadedEnd":552.5,"resourceLoadedStart":183.59999990463257,"resourceTiming":[{"duration":9.5,"initiatorType":"link","name":"https://jira.mariadb.org/s/2c21342762a6a02add1c328bed317ffd-CDN/lu2bsh/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/css/_super/batch.css","startTime":183.59999990463257,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":183.59999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":193.09999990463257,"responseStart":0,"secureConnectionStart":0},{"duration":10.699999809265137,"initiatorType":"link","name":"https://jira.mariadb.org/s/7ebd35e77e471bc30ff0eba799ebc151-CDN/lu2bsh/820016/12ta74/eb142f92e4bd16bd1ef8b08c1b9d5d56/_/download/contextbatch/css/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true","startTime":183.80000019073486,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":183.80000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":194.5,"responseStart":0,"secureConnectionStart":0},{"duration":210.90000009536743,"initiatorType":"script","name":"https://jira.mariadb.org/s/c54b129276d75dc2a3460e1d78f37913-CDN/lu2bsh/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/js/_super/batch.js?locale=en","startTime":184,"connectEnd":194.69999980926514,"connectStart":194.69999980926514,"domainLookupEnd":194.69999980926514,"domainLookupStart":194.69999980926514,"fetchStart":184,"redirectEnd":0,"redirectStart":0,"requestStart":194.80000019073486,"responseEnd":394.90000009536743,"responseStart":206.30000019073486,"secureConnectionStart":194.69999980926514},{"duration":33.200000286102295,"initiatorType":"script","name":"https://jira.mariadb.org/s/ffdb17665775c5b4d6f097f3974ee359-CDN/lu2bsh/820016/12ta74/c92c0caa9a024ae85b0ebdbed7fb4bd7/_/download/contextbatch/js/atl.global,-_super/batch.js?locale=en","startTime":184.69999980926514,"connectEnd":184.69999980926514,"connectStart":184.69999980926514,"domainLookupEnd":184.69999980926514,"domainLookupStart":184.69999980926514,"fetchStart":184.69999980926514,"redirectEnd":0,"redirectStart":0,"requestStart":197.69999980926514,"responseEnd":217.90000009536743,"responseStart":216.59999990463257,"secureConnectionStart":184.69999980926514},{"duration":367.80000019073486,"initiatorType":"script","name":"https://jira.mariadb.org/s/f867843cd2fdb209d4d1d4f760f86346-CDN/lu2bsh/820016/12ta74/eb142f92e4bd16bd1ef8b08c1b9d5d56/_/download/contextbatch/js/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true","startTime":184.69999980926514,"connectEnd":184.69999980926514,"connectStart":184.69999980926514,"domainLookupEnd":184.69999980926514,"domainLookupStart":184.69999980926514,"fetchStart":184.69999980926514,"redirectEnd":0,"redirectStart":0,"requestStart":197.09999990463257,"responseEnd":552.5,"responseStart":211.40000009536743,"secureConnectionStart":184.69999980926514},{"duration":37.69999980926514,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bsh/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-en/jira.webresources:calendar-en.js","startTime":184.90000009536743,"connectEnd":184.90000009536743,"connectStart":184.90000009536743,"domainLookupEnd":184.90000009536743,"domainLookupStart":184.90000009536743,"fetchStart":184.90000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":201.40000009536743,"responseEnd":222.59999990463257,"responseStart":220.40000009536743,"secureConnectionStart":184.90000009536743},{"duration":38.19999980926514,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bsh/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-localisation-moment/jira.webresources:calendar-localisation-moment.js","startTime":184.90000009536743,"connectEnd":184.90000009536743,"connectStart":184.90000009536743,"domainLookupEnd":184.90000009536743,"domainLookupStart":184.90000009536743,"fetchStart":184.90000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":202.09999990463257,"responseEnd":223.09999990463257,"responseStart":221.19999980926514,"secureConnectionStart":184.90000009536743},{"duration":14.400000095367432,"initiatorType":"link","name":"https://jira.mariadb.org/s/b04b06a02d1959df322d9cded3aeecc1-CDN/lu2bsh/820016/12ta74/a2ff6aa845ffc9a1d22fe23d9ee791fc/_/download/contextbatch/css/jira.global.look-and-feel,-_super/batch.css","startTime":185.09999990463257,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":185.09999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":199.5,"responseStart":0,"secureConnectionStart":0},{"duration":38.700000286102295,"initiatorType":"script","name":"https://jira.mariadb.org/rest/api/1.0/shortcuts/820016/47140b6e0a9bc2e4913da06536125810/shortcuts.js?context=issuenavigation&context=issueaction","startTime":185.09999990463257,"connectEnd":185.09999990463257,"connectStart":185.09999990463257,"domainLookupEnd":185.09999990463257,"domainLookupStart":185.09999990463257,"fetchStart":185.09999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":202.90000009536743,"responseEnd":223.80000019073486,"responseStart":221.69999980926514,"secureConnectionStart":185.09999990463257},{"duration":15.699999809265137,"initiatorType":"link","name":"https://jira.mariadb.org/s/3ac36323ba5e4eb0af2aa7ac7211b4bb-CDN/lu2bsh/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/css/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.css?jira.create.linked.issue=true","startTime":185.30000019073486,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":185.30000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":201,"responseStart":0,"secureConnectionStart":0},{"duration":42.59999990463257,"initiatorType":"script","name":"https://jira.mariadb.org/s/81b5d7c27af3ebc078cc4a36383678ba-CDN/lu2bsh/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/js/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.js?jira.create.linked.issue=true&locale=en","startTime":185.40000009536743,"connectEnd":185.40000009536743,"connectStart":185.40000009536743,"domainLookupEnd":185.40000009536743,"domainLookupStart":185.40000009536743,"fetchStart":185.40000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":203.40000009536743,"responseEnd":228,"responseStart":225.59999990463257,"secureConnectionStart":185.40000009536743},{"duration":295.5,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bsh/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-js/jira.webresources:bigpipe-js.js","startTime":188.19999980926514,"connectEnd":188.19999980926514,"connectStart":188.19999980926514,"domainLookupEnd":188.19999980926514,"domainLookupStart":188.19999980926514,"fetchStart":188.19999980926514,"redirectEnd":0,"redirectStart":0,"requestStart":227.40000009536743,"responseEnd":483.69999980926514,"responseStart":480.80000019073486,"secureConnectionStart":188.19999980926514},{"duration":302.90000009536743,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bsh/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-init/jira.webresources:bigpipe-init.js","startTime":188.69999980926514,"connectEnd":188.69999980926514,"connectStart":188.69999980926514,"domainLookupEnd":188.69999980926514,"domainLookupStart":188.69999980926514,"fetchStart":188.69999980926514,"redirectEnd":0,"redirectStart":0,"requestStart":245.90000009536743,"responseEnd":491.59999990463257,"responseStart":485.59999990463257,"secureConnectionStart":188.69999980926514},{"duration":96.09999990463257,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":538.3000001907349,"connectEnd":538.3000001907349,"connectStart":538.3000001907349,"domainLookupEnd":538.3000001907349,"domainLookupStart":538.3000001907349,"fetchStart":538.3000001907349,"redirectEnd":0,"redirectStart":0,"requestStart":607,"responseEnd":634.4000000953674,"responseStart":633.9000000953674,"secureConnectionStart":538.3000001907349},{"duration":240.5,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":770.0999999046326,"connectEnd":770.0999999046326,"connectStart":770.0999999046326,"domainLookupEnd":770.0999999046326,"domainLookupStart":770.0999999046326,"fetchStart":770.0999999046326,"redirectEnd":0,"redirectStart":0,"requestStart":976.9000000953674,"responseEnd":1010.5999999046326,"responseStart":1009.8000001907349,"secureConnectionStart":770.0999999046326},{"duration":171,"initiatorType":"script","name":"https://www.google-analytics.com/analytics.js","startTime":807.4000000953674,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":807.4000000953674,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":978.4000000953674,"responseStart":0,"secureConnectionStart":0}],"fetchStart":0,"domainLookupStart":0,"domainLookupEnd":0,"connectStart":0,"connectEnd":0,"requestStart":49,"responseStart":175,"responseEnd":188,"domLoading":178,"domInteractive":975,"domContentLoadedEventStart":975,"domContentLoadedEventEnd":1038,"domComplete":1815,"loadEventStart":1815,"loadEventEnd":1815,"userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","marks":[{"name":"bigPipe.sidebar-id.start","time":949.6999998092651},{"name":"bigPipe.sidebar-id.end","time":950.4000000953674},{"name":"bigPipe.activity-panel-pipe-id.start","time":950.5999999046326},{"name":"bigPipe.activity-panel-pipe-id.end","time":951.4000000953674},{"name":"activityTabFullyLoaded","time":1050.5}],"measures":[],"correlationId":"45fe1b8e40184","effectiveType":"4g","downlink":10,"rtt":0,"serverDuration":71,"dbReadsTimeInMs":10,"dbConnsTimeInMs":17,"applicationHash":"9d11dbea5f4be3d4cc21f03a88dd11d8c8687422","experiments":[]}}
We originally thought the problem was in cmapi, but it turns out the real problem is in engine here.