I'm running 10.2.14, with roughly 300GB data (1000K +/- tables). 95% tables are innodb. I have 64GB RAM, with INNODB buffer pool size set to 46GB (full my.cnf attached). The OS is Ubuntu 16.04.4. This is a dedicated MariaDB server.
I have swappiness set to 0.
I'm suffering from MariaDB 10.2.14 ripping threw allocated memory and overflowing into swap. innodb_numa_interleave=1 did not solve the problem. Actually, the problem has become even more aggressive recently. Ever since enabling full text search on thousands of tables, MariaDB has become way more aggressive in over consuming memory. I'm faced with either hitting OOM Killer, feeding it by creating more swap, or restarting MariaDB every other day. Of course, all three scenarios are unsustainable.
Not sure if this is related to to indicated issues, or where to take this issue from here.
As you can see in
on Mar 7, I grab the following numa maps summary to see how unbalanced it might be when swap was eaten up:
N0 : 7861090 ( 29.99 GB)
|
N1 : 5629622 ( 21.48 GB)
|
active : 10546055 ( 40.23 GB)
|
anon : 13441950 ( 51.28 GB)
|
dirty : 13176616 ( 50.26 GB)
|
kernelpagesize_kB: 3628 ( 0.01 GB)
|
mapmax : 244 ( 0.00 GB)
|
mapped : 66876 ( 0.26 GB)
|
swapcache : 265334 ( 1.01 GB)
|
|
Thanks,
Mike
{"report":{"fcp":930.7000000476837,"ttfb":128.89999985694885,"pageVisibility":"visible","entityId":73056,"key":"jira.project.issue.view-issue","isInitial":true,"threshold":1000,"elementTimings":{},"userDeviceMemory":8,"userDeviceProcessors":64,"apdex":1,"journeyId":"8ad3c65a-4067-4936-851d-37157a1d2984","navigationType":0,"readyForUser":996.2000000476837,"redirectCount":0,"resourceLoadedEnd":1078.5,"resourceLoadedStart":134.29999995231628,"resourceTiming":[{"duration":368.40000009536743,"initiatorType":"link","name":"https://jira.mariadb.org/s/2c21342762a6a02add1c328bed317ffd-CDN/lu2bu7/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/css/_super/batch.css","startTime":134.29999995231628,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":134.29999995231628,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":502.7000000476837,"responseStart":0,"secureConnectionStart":0},{"duration":368.10000014305115,"initiatorType":"link","name":"https://jira.mariadb.org/s/7ebd35e77e471bc30ff0eba799ebc151-CDN/lu2bu7/820016/12ta74/8679b4946efa1a0bb029a3a22206fb5d/_/download/contextbatch/css/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true","startTime":134.59999990463257,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":134.59999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":502.7000000476837,"responseStart":0,"secureConnectionStart":0},{"duration":377.5,"initiatorType":"script","name":"https://jira.mariadb.org/s/fbf975c0cce4b1abf04784eeae9ba1f4-CDN/lu2bu7/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/js/_super/batch.js?locale=en","startTime":134.59999990463257,"connectEnd":134.59999990463257,"connectStart":134.59999990463257,"domainLookupEnd":134.59999990463257,"domainLookupStart":134.59999990463257,"fetchStart":134.59999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":134.59999990463257,"responseEnd":512.0999999046326,"responseStart":512.0999999046326,"secureConnectionStart":134.59999990463257},{"duration":401.7999999523163,"initiatorType":"script","name":"https://jira.mariadb.org/s/099b33461394b8015fc36c0a4b96e19f-CDN/lu2bu7/820016/12ta74/8679b4946efa1a0bb029a3a22206fb5d/_/download/contextbatch/js/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true","startTime":134.70000004768372,"connectEnd":134.70000004768372,"connectStart":134.70000004768372,"domainLookupEnd":134.70000004768372,"domainLookupStart":134.70000004768372,"fetchStart":134.70000004768372,"redirectEnd":0,"redirectStart":0,"requestStart":134.70000004768372,"responseEnd":536.5,"responseStart":536.5,"secureConnectionStart":134.70000004768372},{"duration":405.2999999523163,"initiatorType":"script","name":"https://jira.mariadb.org/s/94c15bff32baef80f4096a08aceae8bc-CDN/lu2bu7/820016/12ta74/c92c0caa9a024ae85b0ebdbed7fb4bd7/_/download/contextbatch/js/atl.global,-_super/batch.js?locale=en","startTime":134.79999995231628,"connectEnd":134.79999995231628,"connectStart":134.79999995231628,"domainLookupEnd":134.79999995231628,"domainLookupStart":134.79999995231628,"fetchStart":134.79999995231628,"redirectEnd":0,"redirectStart":0,"requestStart":134.79999995231628,"responseEnd":540.0999999046326,"responseStart":540.0999999046326,"secureConnectionStart":134.79999995231628},{"duration":405.7000000476837,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bu7/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-en/jira.webresources:calendar-en.js","startTime":135,"connectEnd":135,"connectStart":135,"domainLookupEnd":135,"domainLookupStart":135,"fetchStart":135,"redirectEnd":0,"redirectStart":0,"requestStart":135,"responseEnd":540.7000000476837,"responseStart":540.7000000476837,"secureConnectionStart":135},{"duration":406,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bu7/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-localisation-moment/jira.webresources:calendar-localisation-moment.js","startTime":135.09999990463257,"connectEnd":135.09999990463257,"connectStart":135.09999990463257,"domainLookupEnd":135.09999990463257,"domainLookupStart":135.09999990463257,"fetchStart":135.09999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":135.09999990463257,"responseEnd":541.0999999046326,"responseStart":541.0999999046326,"secureConnectionStart":135.09999990463257},{"duration":406.7999999523163,"initiatorType":"link","name":"https://jira.mariadb.org/s/b04b06a02d1959df322d9cded3aeecc1-CDN/lu2bu7/820016/12ta74/a2ff6aa845ffc9a1d22fe23d9ee791fc/_/download/contextbatch/css/jira.global.look-and-feel,-_super/batch.css","startTime":135.20000004768372,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":135.20000004768372,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":542,"responseStart":0,"secureConnectionStart":0},{"duration":407.2000000476837,"initiatorType":"link","name":"https://jira.mariadb.org/s/3ac36323ba5e4eb0af2aa7ac7211b4bb-CDN/lu2bu7/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/css/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.css?jira.create.linked.issue=true","startTime":135.29999995231628,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":135.29999995231628,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":542.5,"responseStart":0,"secureConnectionStart":0},{"duration":406.40000009536743,"initiatorType":"script","name":"https://jira.mariadb.org/rest/api/1.0/shortcuts/820016/47140b6e0a9bc2e4913da06536125810/shortcuts.js?context=issuenavigation&context=issueaction","startTime":135.29999995231628,"connectEnd":135.29999995231628,"connectStart":135.29999995231628,"domainLookupEnd":135.29999995231628,"domainLookupStart":135.29999995231628,"fetchStart":135.29999995231628,"redirectEnd":0,"redirectStart":0,"requestStart":135.29999995231628,"responseEnd":541.7000000476837,"responseStart":541.7000000476837,"secureConnectionStart":135.29999995231628},{"duration":406.89999985694885,"initiatorType":"script","name":"https://jira.mariadb.org/s/3339d87fa2538a859872f2df449bf8d0-CDN/lu2bu7/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/js/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.js?jira.create.linked.issue=true&locale=en","startTime":135.5,"connectEnd":135.5,"connectStart":135.5,"domainLookupEnd":135.5,"domainLookupStart":135.5,"fetchStart":135.5,"redirectEnd":0,"redirectStart":0,"requestStart":135.5,"responseEnd":542.3999998569489,"responseStart":542.3999998569489,"secureConnectionStart":135.5},{"duration":941.7000000476837,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bu7/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-js/jira.webresources:bigpipe-js.js","startTime":136.39999985694885,"connectEnd":136.39999985694885,"connectStart":136.39999985694885,"domainLookupEnd":136.39999985694885,"domainLookupStart":136.39999985694885,"fetchStart":136.39999985694885,"redirectEnd":0,"redirectStart":0,"requestStart":136.39999985694885,"responseEnd":1078.0999999046326,"responseStart":1078.0999999046326,"secureConnectionStart":136.39999985694885},{"duration":942.1000001430511,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bu7/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-init/jira.webresources:bigpipe-init.js","startTime":136.39999985694885,"connectEnd":136.39999985694885,"connectStart":136.39999985694885,"domainLookupEnd":136.39999985694885,"domainLookupStart":136.39999985694885,"fetchStart":136.39999985694885,"redirectEnd":0,"redirectStart":0,"requestStart":136.39999985694885,"responseEnd":1078.5,"responseStart":1078.5,"secureConnectionStart":136.39999985694885},{"duration":336,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":741.3999998569489,"connectEnd":741.3999998569489,"connectStart":741.3999998569489,"domainLookupEnd":741.3999998569489,"domainLookupStart":741.3999998569489,"fetchStart":741.3999998569489,"redirectEnd":0,"redirectStart":0,"requestStart":741.3999998569489,"responseEnd":1077.3999998569489,"responseStart":1077.2999999523163,"secureConnectionStart":741.3999998569489},{"duration":169.20000004768372,"initiatorType":"script","name":"https://www.google-analytics.com/analytics.js","startTime":909.7999999523163,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":909.7999999523163,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1079,"responseStart":0,"secureConnectionStart":0}],"fetchStart":0,"domainLookupStart":0,"domainLookupEnd":0,"connectStart":0,"connectEnd":0,"requestStart":3,"responseStart":129,"responseEnd":132,"domLoading":132,"domInteractive":1102,"domContentLoadedEventStart":1102,"domContentLoadedEventEnd":1136,"domComplete":2128,"loadEventStart":2128,"loadEventEnd":2130,"userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","marks":[{"name":"bigPipe.sidebar-id.start","time":1086.2000000476837},{"name":"bigPipe.sidebar-id.end","time":1087},{"name":"bigPipe.activity-panel-pipe-id.start","time":1087},{"name":"bigPipe.activity-panel-pipe-id.end","time":1089.2999999523163},{"name":"activityTabFullyLoaded","time":1142.5999999046326}],"measures":[],"correlationId":"2a068da0f3b309","effectiveType":"4g","downlink":10,"rtt":0,"serverDuration":74,"dbReadsTimeInMs":13,"dbConnsTimeInMs":20,"applicationHash":"9d11dbea5f4be3d4cc21f03a88dd11d8c8687422","experiments":[]}}