Read only slave with slave_parallel_threads=10 got deadlocked when mariabackup executed FTWRL and tried to copy non-InnoDB files and remaining part of the redo log. Neither replication, no mariabackup, nor later mariabackup calls could proceed.
In the processlist we see:
...
|
| 10 | system user | | NULL | Slave_IO | 1626750 | Waiting for master to send event | NULL | 0.000 |
|
| 13 | system user | | NULL | Slave_worker | 184973 | Waiting for prior transaction to start commit before starting next transaction | NULL | 0.000 |
|
| 12 | system user | | NULL | Slave_worker | 184973 | Waiting for prior transaction to start commit before starting next transaction | NULL | 0.000 |
|
| 14 | system user | | NULL | Slave_worker | 184973 | Waiting for prior transaction to start commit before starting next transaction | NULL | 0.000 |
|
| 15 | system user | | NULL | Slave_worker | 184973 | Waiting for prior transaction to start commit before starting next transaction | NULL | 0.000 |
|
| 16 | system user | | NULL | Slave_worker | 184973 | Waiting for prior transaction to start commit before starting next transaction | NULL | 0.000 |
|
| 17 | system user | | NULL | Slave_worker | 184973 | Waiting for global read lock | NULL | 0.000 |
|
| 18 | system user | | NULL | Slave_worker | 184973 | Waiting for prior transaction to start commit before starting next transaction | NULL | 0.000 |
|
| 19 | system user | | NULL | Slave_worker | 184973 | Waiting for prior transaction to start commit before starting next transaction | NULL | 0.000 |
|
| 20 | system user | | NULL | Slave_worker | 184973 | Waiting for prior transaction to start commit before starting next transaction | NULL | 0.000 |
|
| 21 | system user | | NULL | Slave_worker | 184973 | Waiting for prior transaction to start commit before starting next transaction | NULL | 0.000 |
|
| 11 | system user | | NULL | Slave_SQL | 187284 | Waiting for room in worker thread event queue | NULL | 0.000 |
|
| 271217 | dbcleaner | some_host:51410 | sa2 | Query | 0 | Waiting in MASTER_GTID_WAIT() (primary waiter) | SELECT MASTER_GTID_WAIT('0-11-5381276620', 4) | 0.000 |
|
...
|
| 402858 | bkpuser | localhost | NULL | Sleep | 184970 | | NULL | 0.000 |
|
| 427762 | bkpuser | localhost | NULL | Query | 98674 | Waiting for worker threads to pause for global read lock | FLUSH TABLES WITH READ LOCK | 0.000 |
|
| 452639 | bkpuser | localhost | NULL | Query | 12636 | Waiting while replication worker thread pool is busy | FLUSH TABLES WITH READ LOCK | 0.000 |
|
| 469510 | dbcleaner | some_host:63978 | sa2 | Sleep | 19 | | NULL | 0.000 |
|
| 469541 | dbcleaner | localhost | NULL | Query | 0 | init | show processlist | 0.000 |
|
See backtrace of all threads attached.
{"report":{"fcp":978.1000003814697,"ttfb":288,"pageVisibility":"visible","entityId":81019,"key":"jira.project.issue.view-issue","isInitial":true,"threshold":1000,"elementTimings":{},"userDeviceMemory":8,"userDeviceProcessors":64,"apdex":0.5,"journeyId":"4ec836dc-faa8-45c3-8d54-301b26c9c76d","navigationType":0,"readyForUser":1061.5,"redirectCount":0,"resourceLoadedEnd":1092.9000005722046,"resourceLoadedStart":293.6000003814697,"resourceTiming":[{"duration":215.80000019073486,"initiatorType":"link","name":"https://jira.mariadb.org/s/2c21342762a6a02add1c328bed317ffd-CDN/lu2bv2/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/css/_super/batch.css","startTime":293.6000003814697,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":293.6000003814697,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":509.4000005722046,"responseStart":0,"secureConnectionStart":0},{"duration":215.89999961853027,"initiatorType":"link","name":"https://jira.mariadb.org/s/7ebd35e77e471bc30ff0eba799ebc151-CDN/lu2bv2/820016/12ta74/2380add21a9a1006587582385952de73/_/download/contextbatch/css/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true","startTime":293.9000005722046,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":293.9000005722046,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":509.80000019073486,"responseStart":0,"secureConnectionStart":0},{"duration":224.5,"initiatorType":"script","name":"https://jira.mariadb.org/s/e9b27a47da5fb0f74a35acd57e9847fb-CDN/lu2bv2/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/js/_super/batch.js?locale=en","startTime":294.1000003814697,"connectEnd":294.1000003814697,"connectStart":294.1000003814697,"domainLookupEnd":294.1000003814697,"domainLookupStart":294.1000003814697,"fetchStart":294.1000003814697,"redirectEnd":0,"redirectStart":0,"requestStart":294.1000003814697,"responseEnd":518.6000003814697,"responseStart":518.6000003814697,"secureConnectionStart":294.1000003814697},{"duration":290.1000003814697,"initiatorType":"script","name":"https://jira.mariadb.org/s/c32eb0da7ad9831253f8397e6cc26afd-CDN/lu2bv2/820016/12ta74/2380add21a9a1006587582385952de73/_/download/contextbatch/js/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true","startTime":294.30000019073486,"connectEnd":294.30000019073486,"connectStart":294.30000019073486,"domainLookupEnd":294.30000019073486,"domainLookupStart":294.30000019073486,"fetchStart":294.30000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":294.30000019073486,"responseEnd":584.4000005722046,"responseStart":584.4000005722046,"secureConnectionStart":294.30000019073486},{"duration":293.80000019073486,"initiatorType":"script","name":"https://jira.mariadb.org/s/bc0bcb146314416123c992714ee00ff7-CDN/lu2bv2/820016/12ta74/c92c0caa9a024ae85b0ebdbed7fb4bd7/_/download/contextbatch/js/atl.global,-_super/batch.js?locale=en","startTime":294.5,"connectEnd":294.5,"connectStart":294.5,"domainLookupEnd":294.5,"domainLookupStart":294.5,"fetchStart":294.5,"redirectEnd":0,"redirectStart":0,"requestStart":294.5,"responseEnd":588.3000001907349,"responseStart":588.3000001907349,"secureConnectionStart":294.5},{"duration":294.1000003814697,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bv2/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-en/jira.webresources:calendar-en.js","startTime":294.80000019073486,"connectEnd":294.80000019073486,"connectStart":294.80000019073486,"domainLookupEnd":294.80000019073486,"domainLookupStart":294.80000019073486,"fetchStart":294.80000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":294.80000019073486,"responseEnd":588.9000005722046,"responseStart":588.9000005722046,"secureConnectionStart":294.80000019073486},{"duration":294.30000019073486,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bv2/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-localisation-moment/jira.webresources:calendar-localisation-moment.js","startTime":295,"connectEnd":295,"connectStart":295,"domainLookupEnd":295,"domainLookupStart":295,"fetchStart":295,"redirectEnd":0,"redirectStart":0,"requestStart":295,"responseEnd":589.3000001907349,"responseStart":589.3000001907349,"secureConnectionStart":295},{"duration":366.5999994277954,"initiatorType":"link","name":"https://jira.mariadb.org/s/b04b06a02d1959df322d9cded3aeecc1-CDN/lu2bv2/820016/12ta74/a2ff6aa845ffc9a1d22fe23d9ee791fc/_/download/contextbatch/css/jira.global.look-and-feel,-_super/batch.css","startTime":295.1000003814697,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":295.1000003814697,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":661.6999998092651,"responseStart":0,"secureConnectionStart":0},{"duration":294.70000076293945,"initiatorType":"script","name":"https://jira.mariadb.org/rest/api/1.0/shortcuts/820016/47140b6e0a9bc2e4913da06536125810/shortcuts.js?context=issuenavigation&context=issueaction","startTime":295.19999980926514,"connectEnd":295.19999980926514,"connectStart":295.19999980926514,"domainLookupEnd":295.19999980926514,"domainLookupStart":295.19999980926514,"fetchStart":295.19999980926514,"redirectEnd":0,"redirectStart":0,"requestStart":295.19999980926514,"responseEnd":589.9000005722046,"responseStart":589.9000005722046,"secureConnectionStart":295.19999980926514},{"duration":366.4000005722046,"initiatorType":"link","name":"https://jira.mariadb.org/s/3ac36323ba5e4eb0af2aa7ac7211b4bb-CDN/lu2bv2/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/css/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.css?jira.create.linked.issue=true","startTime":295.5,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":295.5,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":661.9000005722046,"responseStart":0,"secureConnectionStart":0},{"duration":294.8999996185303,"initiatorType":"script","name":"https://jira.mariadb.org/s/719848dd97ebe0663199f49a3936487a-CDN/lu2bv2/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/js/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.js?jira.create.linked.issue=true&locale=en","startTime":295.6000003814697,"connectEnd":295.6000003814697,"connectStart":295.6000003814697,"domainLookupEnd":295.6000003814697,"domainLookupStart":295.6000003814697,"fetchStart":295.6000003814697,"redirectEnd":0,"redirectStart":0,"requestStart":295.6000003814697,"responseEnd":590.5,"responseStart":590.5,"secureConnectionStart":295.6000003814697},{"duration":716.0999994277954,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bv2/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-js/jira.webresources:bigpipe-js.js","startTime":296.4000005722046,"connectEnd":296.4000005722046,"connectStart":296.4000005722046,"domainLookupEnd":296.4000005722046,"domainLookupStart":296.4000005722046,"fetchStart":296.4000005722046,"redirectEnd":0,"redirectStart":0,"requestStart":296.4000005722046,"responseEnd":1012.5,"responseStart":1012.5,"secureConnectionStart":296.4000005722046},{"duration":796.4000005722046,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bv2/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-init/jira.webresources:bigpipe-init.js","startTime":296.5,"connectEnd":296.5,"connectStart":296.5,"domainLookupEnd":296.5,"domainLookupStart":296.5,"fetchStart":296.5,"redirectEnd":0,"redirectStart":0,"requestStart":296.5,"responseEnd":1092.9000005722046,"responseStart":1092.9000005722046,"secureConnectionStart":296.5},{"duration":341.19999980926514,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":673.6000003814697,"connectEnd":673.6000003814697,"connectStart":673.6000003814697,"domainLookupEnd":673.6000003814697,"domainLookupStart":673.6000003814697,"fetchStart":673.6000003814697,"redirectEnd":0,"redirectStart":0,"requestStart":673.6000003814697,"responseEnd":1014.8000001907349,"responseStart":1014.8000001907349,"secureConnectionStart":673.6000003814697},{"duration":162.0999994277954,"initiatorType":"script","name":"https://www.google-analytics.com/analytics.js","startTime":970.6000003814697,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":970.6000003814697,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1132.6999998092651,"responseStart":0,"secureConnectionStart":0}],"fetchStart":0,"domainLookupStart":0,"domainLookupEnd":0,"connectStart":0,"connectEnd":0,"requestStart":114,"responseStart":287,"responseEnd":289,"domLoading":291,"domInteractive":1161,"domContentLoadedEventStart":1161,"domContentLoadedEventEnd":1204,"domComplete":2073,"loadEventStart":2073,"loadEventEnd":2075,"userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","marks":[{"name":"bigPipe.sidebar-id.start","time":1134.6000003814697},{"name":"bigPipe.sidebar-id.end","time":1135.4000005722046},{"name":"bigPipe.activity-panel-pipe-id.start","time":1135.5},{"name":"bigPipe.activity-panel-pipe-id.end","time":1140.8000001907349},{"name":"activityTabFullyLoaded","time":1249.8000001907349}],"measures":[],"correlationId":"82f50babb8aa69","effectiveType":"4g","downlink":9.2,"rtt":0,"serverDuration":103,"dbReadsTimeInMs":10,"dbConnsTimeInMs":18,"applicationHash":"9d11dbea5f4be3d4cc21f03a88dd11d8c8687422","experiments":[]}}