When we run mariabackup on a galera cluster nodes with a lot of data/databases (over 1500 databases with 700GB of data at the moment), if the servers are under moderate use mariabackup tool fails with the following log messages:
Aug 21 17:44:51 maria3 -innobackupex-backup[25752]: 2018-08-21 17:44:51 140002715002816 [Note] InnoDB: Read redo log up to LSN=1691316337664
|
Aug 21 17:44:51 maria3 -innobackupex-backup[25752]: mariabackup: Error: xtrabackup_copy_logfile() failed.
|
Aug 21 17:44:52 maria3 -wsrep-sst-donor[25825]: mariabackup finished with error: 1. Check /data/maria//innobackup.backup.log
|
Aug 21 17:44:52 maria3 -wsrep-sst-donor[25826]: Cleanup after exit with status:22
|
The cluster is under load, but I think these are happening when schema is being modified (databases created, dropped and so on). It could be related to tables flooded with records, because when we create databases we populate them with a lot of records.
This happens both when running mariabackup manually or when it's triggered by SST.
The log file mentioned in the message "innobackup.backup.log" cannot be found anywhere on the system. So we have no more info about the error.
Unfortunately, this is a production cluster and we can't do much in a way of debugging on these machines.
{"report":{"fcp":775.5,"ttfb":203.5999994277954,"pageVisibility":"visible","entityId":69299,"key":"jira.project.issue.view-issue","isInitial":true,"threshold":1000,"elementTimings":{},"userDeviceMemory":8,"userDeviceProcessors":64,"apdex":1,"journeyId":"526d4a9c-ef46-4ae7-8bd4-a9699233a669","navigationType":0,"readyForUser":848.5,"redirectCount":0,"resourceLoadedEnd":876.5999994277954,"resourceLoadedStart":209,"resourceTiming":[{"duration":17.59999942779541,"initiatorType":"link","name":"https://jira.mariadb.org/s/2c21342762a6a02add1c328bed317ffd-CDN/lu2bu7/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/css/_super/batch.css","startTime":209,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":209,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":226.5999994277954,"responseStart":0,"secureConnectionStart":0},{"duration":17.399999618530273,"initiatorType":"link","name":"https://jira.mariadb.org/s/7ebd35e77e471bc30ff0eba799ebc151-CDN/lu2bu7/820016/12ta74/8679b4946efa1a0bb029a3a22206fb5d/_/download/contextbatch/css/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true","startTime":209.30000019073486,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":209.30000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":226.69999980926514,"responseStart":0,"secureConnectionStart":0},{"duration":74.40000057220459,"initiatorType":"script","name":"https://jira.mariadb.org/s/fbf975c0cce4b1abf04784eeae9ba1f4-CDN/lu2bu7/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/js/_super/batch.js?locale=en","startTime":209.39999961853027,"connectEnd":209.39999961853027,"connectStart":209.39999961853027,"domainLookupEnd":209.39999961853027,"domainLookupStart":209.39999961853027,"fetchStart":209.39999961853027,"redirectEnd":0,"redirectStart":0,"requestStart":209.39999961853027,"responseEnd":283.80000019073486,"responseStart":283.80000019073486,"secureConnectionStart":209.39999961853027},{"duration":142.39999961853027,"initiatorType":"script","name":"https://jira.mariadb.org/s/099b33461394b8015fc36c0a4b96e19f-CDN/lu2bu7/820016/12ta74/8679b4946efa1a0bb029a3a22206fb5d/_/download/contextbatch/js/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true","startTime":209.5,"connectEnd":209.5,"connectStart":209.5,"domainLookupEnd":209.5,"domainLookupStart":209.5,"fetchStart":209.5,"redirectEnd":0,"redirectStart":0,"requestStart":209.5,"responseEnd":351.8999996185303,"responseStart":351.8999996185303,"secureConnectionStart":209.5},{"duration":146.5,"initiatorType":"script","name":"https://jira.mariadb.org/s/94c15bff32baef80f4096a08aceae8bc-CDN/lu2bu7/820016/12ta74/c92c0caa9a024ae85b0ebdbed7fb4bd7/_/download/contextbatch/js/atl.global,-_super/batch.js?locale=en","startTime":209.5999994277954,"connectEnd":209.5999994277954,"connectStart":209.5999994277954,"domainLookupEnd":209.5999994277954,"domainLookupStart":209.5999994277954,"fetchStart":209.5999994277954,"redirectEnd":0,"redirectStart":0,"requestStart":209.5999994277954,"responseEnd":356.0999994277954,"responseStart":356.0999994277954,"secureConnectionStart":209.5999994277954},{"duration":147.19999980926514,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bu7/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-en/jira.webresources:calendar-en.js","startTime":209.69999980926514,"connectEnd":209.69999980926514,"connectStart":209.69999980926514,"domainLookupEnd":209.69999980926514,"domainLookupStart":209.69999980926514,"fetchStart":209.69999980926514,"redirectEnd":0,"redirectStart":0,"requestStart":209.69999980926514,"responseEnd":356.8999996185303,"responseStart":356.8999996185303,"secureConnectionStart":209.69999980926514},{"duration":147.69999980926514,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bu7/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-localisation-moment/jira.webresources:calendar-localisation-moment.js","startTime":209.80000019073486,"connectEnd":209.80000019073486,"connectStart":209.80000019073486,"domainLookupEnd":209.80000019073486,"domainLookupStart":209.80000019073486,"fetchStart":209.80000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":209.80000019073486,"responseEnd":357.5,"responseStart":357.5,"secureConnectionStart":209.80000019073486},{"duration":148.19999980926514,"initiatorType":"link","name":"https://jira.mariadb.org/s/b04b06a02d1959df322d9cded3aeecc1-CDN/lu2bu7/820016/12ta74/a2ff6aa845ffc9a1d22fe23d9ee791fc/_/download/contextbatch/css/jira.global.look-and-feel,-_super/batch.css","startTime":209.89999961853027,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":209.89999961853027,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":358.0999994277954,"responseStart":0,"secureConnectionStart":0},{"duration":148,"initiatorType":"script","name":"https://jira.mariadb.org/rest/api/1.0/shortcuts/820016/47140b6e0a9bc2e4913da06536125810/shortcuts.js?context=issuenavigation&context=issueaction","startTime":210,"connectEnd":210,"connectStart":210,"domainLookupEnd":210,"domainLookupStart":210,"fetchStart":210,"redirectEnd":0,"redirectStart":0,"requestStart":210,"responseEnd":358,"responseStart":358,"secureConnectionStart":210},{"duration":148.4000005722046,"initiatorType":"link","name":"https://jira.mariadb.org/s/3ac36323ba5e4eb0af2aa7ac7211b4bb-CDN/lu2bu7/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/css/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.css?jira.create.linked.issue=true","startTime":210.0999994277954,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":210.0999994277954,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":358.5,"responseStart":0,"secureConnectionStart":0},{"duration":148.39999961853027,"initiatorType":"script","name":"https://jira.mariadb.org/s/3339d87fa2538a859872f2df449bf8d0-CDN/lu2bu7/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/js/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.js?jira.create.linked.issue=true&locale=en","startTime":210.30000019073486,"connectEnd":210.30000019073486,"connectStart":210.30000019073486,"domainLookupEnd":210.30000019073486,"domainLookupStart":210.30000019073486,"fetchStart":210.30000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":210.30000019073486,"responseEnd":358.69999980926514,"responseStart":358.69999980926514,"secureConnectionStart":210.30000019073486},{"duration":554.6000003814697,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bu7/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-js/jira.webresources:bigpipe-js.js","startTime":217.0999994277954,"connectEnd":217.0999994277954,"connectStart":217.0999994277954,"domainLookupEnd":217.0999994277954,"domainLookupStart":217.0999994277954,"fetchStart":217.0999994277954,"redirectEnd":0,"redirectStart":0,"requestStart":217.0999994277954,"responseEnd":771.6999998092651,"responseStart":771.6999998092651,"secureConnectionStart":217.0999994277954},{"duration":554.3000001907349,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bu7/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-init/jira.webresources:bigpipe-init.js","startTime":218,"connectEnd":218,"connectStart":218,"domainLookupEnd":218,"domainLookupStart":218,"fetchStart":218,"redirectEnd":0,"redirectStart":0,"requestStart":218,"responseEnd":772.3000001907349,"responseStart":772.3000001907349,"secureConnectionStart":218},{"duration":208.69999980926514,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":501.80000019073486,"connectEnd":501.80000019073486,"connectStart":501.80000019073486,"domainLookupEnd":501.80000019073486,"domainLookupStart":501.80000019073486,"fetchStart":501.80000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":501.80000019073486,"responseEnd":710.5,"responseStart":710.5,"secureConnectionStart":501.80000019073486},{"duration":104.90000057220459,"initiatorType":"script","name":"https://www.google-analytics.com/analytics.js","startTime":767.5999994277954,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":767.5999994277954,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":872.5,"responseStart":0,"secureConnectionStart":0},{"duration":97.59999942779541,"initiatorType":"link","name":"https://jira.mariadb.org/s/d5715adaadd168a9002b108b2b039b50-CDN/lu2bu7/820016/12ta74/be4b45e9cec53099498fa61c8b7acba4/_/download/contextbatch/css/jira.project.sidebar,-_super,-project.issue.navigator,-jira.general,-jira.browse.project,-jira.view.issue,-jira.global,-atl.general,-com.atlassian.jira.projects.sidebar.init/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true","startTime":776.3000001907349,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":776.3000001907349,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":873.8999996185303,"responseStart":0,"secureConnectionStart":0},{"duration":99.5,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bu7/820016/12ta74/e65b778d185daf5aee24936755b43da6/_/download/contextbatch/js/browser-metrics-plugin.contrib,-_super,-project.issue.navigator,-jira.view.issue,-atl.general/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true","startTime":777.0999994277954,"connectEnd":777.0999994277954,"connectStart":777.0999994277954,"domainLookupEnd":777.0999994277954,"domainLookupStart":777.0999994277954,"fetchStart":777.0999994277954,"redirectEnd":0,"redirectStart":0,"requestStart":777.0999994277954,"responseEnd":876.5999994277954,"responseStart":876.5999994277954,"secureConnectionStart":777.0999994277954},{"duration":128.5,"initiatorType":"script","name":"https://jira.mariadb.org/s/f51ef5507eea4c158f257c66c93b2a3f-CDN/lu2bu7/820016/12ta74/be4b45e9cec53099498fa61c8b7acba4/_/download/contextbatch/js/jira.project.sidebar,-_super,-project.issue.navigator,-jira.general,-jira.browse.project,-jira.view.issue,-jira.global,-atl.general,-com.atlassian.jira.projects.sidebar.init/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true","startTime":777.5999994277954,"connectEnd":777.5999994277954,"connectStart":777.5999994277954,"domainLookupEnd":777.5999994277954,"domainLookupStart":777.5999994277954,"fetchStart":777.5999994277954,"redirectEnd":0,"redirectStart":0,"requestStart":777.5999994277954,"responseEnd":906.0999994277954,"responseStart":906.0999994277954,"secureConnectionStart":777.5999994277954}],"fetchStart":0,"domainLookupStart":0,"domainLookupEnd":0,"connectStart":0,"connectEnd":0,"requestStart":69,"responseStart":203,"responseEnd":222,"domLoading":207,"domInteractive":906,"domContentLoadedEventStart":906,"domContentLoadedEventEnd":946,"domComplete":1213,"loadEventStart":1213,"loadEventEnd":1213,"userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","marks":[{"name":"bigPipe.sidebar-id.start","time":885},{"name":"bigPipe.sidebar-id.end","time":885.8000001907349},{"name":"bigPipe.activity-panel-pipe-id.start","time":885.8999996185303},{"name":"bigPipe.activity-panel-pipe-id.end","time":886.5999994277954},{"name":"activityTabFullyLoaded","time":962.8999996185303}],"measures":[],"correlationId":"61eecaa3240214","effectiveType":"4g","downlink":9,"rtt":0,"serverDuration":81,"dbReadsTimeInMs":14,"dbConnsTimeInMs":22,"applicationHash":"9d11dbea5f4be3d4cc21f03a88dd11d8c8687422","experiments":[]}}
Bug
Critical
MDEV-16791 mariabackup : allow consistent backup, in presence of concurrent DDL, also without --lock-ddl-per-table
I'm going to close it since there is no actionable info about the bug.
The description is vaguely similar to https://jira.mariadb.org/browse/MDEV-16791 . You can reopen (and provide more information, incl the full log of mariabackup run), if it happens after 10.2.18