After update from 10.11.9 to 10.11.10 we observe problem with Rsync SST. It seems that there is a mass rewrite of wsrep_sst_common and if we use dedicated log dir for aria, Rsync SST method crash,
aria_log_dir_path=/var/lib/mysql/logs
|
datadir=/var/lib/mysql/data
|
Problem is in function create_dirs in in wsrep_sst_common which is probably trapped somewhere on other exit code than 0. But if you use custom aria log dir, last exit code is 1 due to condition on line 1866
[ $simplify -ne 0 -a "$ar_log_dir" = "$DATA_DIR" ] && ar_log_dir=""
|
After it scripts flow go to simple_cleanup and exit. If you add on line below, for example echo "1", which has exit code 0, SST can continue and all work.
Output from bash -x
....
|
+ '[' 0 -ne 0 -a /var/lib/mysql/logs = /var/lib/mysql/data ']'
|
+ simple_cleanup
|
+ local estatus=1
|
+ '[' 1 -ne 0 ']'
|
+ wsrep_log_error 'Cleanup after exit with status: 1'
|
+ wsrep_log '[ERROR] Cleanup after exit with status: 1'
|
+ local t
|
+ '[' Linux = Linux ']'
|
++ date '+%Y%m%d %H:%M:%S.%3N'
|
+ t='20241111 19:27:39.773'
|
+ echo 'WSREP_SST: [ERROR] Cleanup after exit with status: 1 (20241111 19:27:39.773)'
|
WSREP_SST: [ERROR] Cleanup after exit with status: 1 (20241111 19:27:39.773)
|
+ '[' -n /var/lib/mysql/data/wsrep_sst.pid ']'
|
++ pwd
|
+ '[' /usr/bin '!=' /usr/bin ']'
|
+ '[' -f /var/lib/mysql/data/wsrep_sst.pid ']'
|
+ rm -f /var/lib/mysql/data/wsrep_sst.pid
|
+ exit 1'
|
How to test:
add different aria and data dir, eg.
aria_log_dir_path=/var/lib/mysql/logs
|
datadir=/var/lib/mysql/data
|
an run rsync SST - problematic part is:.
wsrep_sst_rsync --role 'joiner' --address '10.11.2.31' --datadir '/var/lib/mysql/data/' --parent 2603575 --progress 0 --binlog '/var/lib/mysql/logs/mysql-bin' --binlog-index '/var/lib/mysql/logs/mysql-bin.index' --mysqld-args --wsrep_start_position=00000000-0000-0000-0000-000000000000:-1,0-0-0
|
If you need some more data, let me know.
{"report":{"fcp":907.9000000953674,"ttfb":131.79999995231628,"pageVisibility":"visible","entityId":131555,"key":"jira.project.issue.view-issue","isInitial":true,"threshold":1000,"elementTimings":{},"userDeviceMemory":8,"userDeviceProcessors":64,"apdex":1,"journeyId":"5301f8d5-6cdf-432e-9df2-b6c8f87235b4","navigationType":0,"readyForUser":972.2999999523163,"redirectCount":0,"resourceLoadedEnd":1024.5,"resourceLoadedStart":136.5,"resourceTiming":[{"duration":313.09999990463257,"initiatorType":"link","name":"https://jira.mariadb.org/s/2c21342762a6a02add1c328bed317ffd-CDN/lu2bu7/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/css/_super/batch.css","startTime":136.5,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":136.5,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":449.59999990463257,"responseStart":0,"secureConnectionStart":0},{"duration":313,"initiatorType":"link","name":"https://jira.mariadb.org/s/7ebd35e77e471bc30ff0eba799ebc151-CDN/lu2bu7/820016/12ta74/8679b4946efa1a0bb029a3a22206fb5d/_/download/contextbatch/css/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true","startTime":136.70000004768372,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":136.70000004768372,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":449.7000000476837,"responseStart":0,"secureConnectionStart":0},{"duration":321.2999999523163,"initiatorType":"script","name":"https://jira.mariadb.org/s/fbf975c0cce4b1abf04784eeae9ba1f4-CDN/lu2bu7/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/js/_super/batch.js?locale=en","startTime":136.90000009536743,"connectEnd":136.90000009536743,"connectStart":136.90000009536743,"domainLookupEnd":136.90000009536743,"domainLookupStart":136.90000009536743,"fetchStart":136.90000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":136.90000009536743,"responseEnd":458.2000000476837,"responseStart":458.2000000476837,"secureConnectionStart":136.90000009536743},{"duration":394.2999999523163,"initiatorType":"script","name":"https://jira.mariadb.org/s/099b33461394b8015fc36c0a4b96e19f-CDN/lu2bu7/820016/12ta74/8679b4946efa1a0bb029a3a22206fb5d/_/download/contextbatch/js/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true","startTime":136.90000009536743,"connectEnd":136.90000009536743,"connectStart":136.90000009536743,"domainLookupEnd":136.90000009536743,"domainLookupStart":136.90000009536743,"fetchStart":136.90000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":136.90000009536743,"responseEnd":531.2000000476837,"responseStart":531.2000000476837,"secureConnectionStart":136.90000009536743},{"duration":398,"initiatorType":"script","name":"https://jira.mariadb.org/s/94c15bff32baef80f4096a08aceae8bc-CDN/lu2bu7/820016/12ta74/c92c0caa9a024ae85b0ebdbed7fb4bd7/_/download/contextbatch/js/atl.global,-_super/batch.js?locale=en","startTime":137.09999990463257,"connectEnd":137.09999990463257,"connectStart":137.09999990463257,"domainLookupEnd":137.09999990463257,"domainLookupStart":137.09999990463257,"fetchStart":137.09999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":137.09999990463257,"responseEnd":535.0999999046326,"responseStart":535.0999999046326,"secureConnectionStart":137.09999990463257},{"duration":398.40000009536743,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bu7/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-en/jira.webresources:calendar-en.js","startTime":137.09999990463257,"connectEnd":137.09999990463257,"connectStart":137.09999990463257,"domainLookupEnd":137.09999990463257,"domainLookupStart":137.09999990463257,"fetchStart":137.09999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":137.09999990463257,"responseEnd":535.5,"responseStart":535.5,"secureConnectionStart":137.09999990463257},{"duration":398.59999990463257,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bu7/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-localisation-moment/jira.webresources:calendar-localisation-moment.js","startTime":137.20000004768372,"connectEnd":137.20000004768372,"connectStart":137.20000004768372,"domainLookupEnd":137.20000004768372,"domainLookupStart":137.20000004768372,"fetchStart":137.20000004768372,"redirectEnd":0,"redirectStart":0,"requestStart":137.20000004768372,"responseEnd":535.7999999523163,"responseStart":535.7999999523163,"secureConnectionStart":137.20000004768372},{"duration":399,"initiatorType":"link","name":"https://jira.mariadb.org/s/b04b06a02d1959df322d9cded3aeecc1-CDN/lu2bu7/820016/12ta74/a2ff6aa845ffc9a1d22fe23d9ee791fc/_/download/contextbatch/css/jira.global.look-and-feel,-_super/batch.css","startTime":137.29999995231628,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":137.29999995231628,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":536.2999999523163,"responseStart":0,"secureConnectionStart":0},{"duration":398.90000009536743,"initiatorType":"script","name":"https://jira.mariadb.org/rest/api/1.0/shortcuts/820016/47140b6e0a9bc2e4913da06536125810/shortcuts.js?context=issuenavigation&context=issueaction","startTime":137.29999995231628,"connectEnd":137.29999995231628,"connectStart":137.29999995231628,"domainLookupEnd":137.29999995231628,"domainLookupStart":137.29999995231628,"fetchStart":137.29999995231628,"redirectEnd":0,"redirectStart":0,"requestStart":137.29999995231628,"responseEnd":536.2000000476837,"responseStart":536.2000000476837,"secureConnectionStart":137.29999995231628},{"duration":399,"initiatorType":"link","name":"https://jira.mariadb.org/s/3ac36323ba5e4eb0af2aa7ac7211b4bb-CDN/lu2bu7/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/css/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.css?jira.create.linked.issue=true","startTime":137.5,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":137.5,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":536.5,"responseStart":0,"secureConnectionStart":0},{"duration":399.09999990463257,"initiatorType":"script","name":"https://jira.mariadb.org/s/3339d87fa2538a859872f2df449bf8d0-CDN/lu2bu7/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/js/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.js?jira.create.linked.issue=true&locale=en","startTime":137.70000004768372,"connectEnd":137.70000004768372,"connectStart":137.70000004768372,"domainLookupEnd":137.70000004768372,"domainLookupStart":137.70000004768372,"fetchStart":137.70000004768372,"redirectEnd":0,"redirectStart":0,"requestStart":137.70000004768372,"responseEnd":536.7999999523163,"responseStart":536.7999999523163,"secureConnectionStart":137.70000004768372},{"duration":575,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bu7/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-js/jira.webresources:bigpipe-js.js","startTime":142.79999995231628,"connectEnd":142.79999995231628,"connectStart":142.79999995231628,"domainLookupEnd":142.79999995231628,"domainLookupStart":142.79999995231628,"fetchStart":142.79999995231628,"redirectEnd":0,"redirectStart":0,"requestStart":142.79999995231628,"responseEnd":717.7999999523163,"responseStart":717.7999999523163,"secureConnectionStart":142.79999995231628},{"duration":719,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bu7/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-init/jira.webresources:bigpipe-init.js","startTime":142.79999995231628,"connectEnd":142.79999995231628,"connectStart":142.79999995231628,"domainLookupEnd":142.79999995231628,"domainLookupStart":142.79999995231628,"fetchStart":142.79999995231628,"redirectEnd":0,"redirectStart":0,"requestStart":142.79999995231628,"responseEnd":861.7999999523163,"responseStart":861.7999999523163,"secureConnectionStart":142.79999995231628},{"duration":127.59999990463257,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":667.4000000953674,"connectEnd":667.4000000953674,"connectStart":667.4000000953674,"domainLookupEnd":667.4000000953674,"domainLookupStart":667.4000000953674,"fetchStart":667.4000000953674,"redirectEnd":0,"redirectStart":0,"requestStart":667.4000000953674,"responseEnd":795,"responseStart":795,"secureConnectionStart":667.4000000953674},{"duration":168.5,"initiatorType":"link","name":"https://jira.mariadb.org/s/d5715adaadd168a9002b108b2b039b50-CDN/lu2bu7/820016/12ta74/be4b45e9cec53099498fa61c8b7acba4/_/download/contextbatch/css/jira.project.sidebar,-_super,-project.issue.navigator,-jira.general,-jira.browse.project,-jira.view.issue,-jira.global,-atl.general,-com.atlassian.jira.projects.sidebar.init/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true","startTime":856,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":856,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1024.5,"responseStart":0,"secureConnectionStart":0},{"duration":168.29999995231628,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bu7/820016/12ta74/e65b778d185daf5aee24936755b43da6/_/download/contextbatch/js/browser-metrics-plugin.contrib,-_super,-project.issue.navigator,-jira.view.issue,-atl.general/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true","startTime":856.7000000476837,"connectEnd":856.7000000476837,"connectStart":856.7000000476837,"domainLookupEnd":856.7000000476837,"domainLookupStart":856.7000000476837,"fetchStart":856.7000000476837,"redirectEnd":0,"redirectStart":0,"requestStart":856.7000000476837,"responseEnd":1025,"responseStart":1025,"secureConnectionStart":856.7000000476837},{"duration":173.90000009536743,"initiatorType":"script","name":"https://jira.mariadb.org/s/f51ef5507eea4c158f257c66c93b2a3f-CDN/lu2bu7/820016/12ta74/be4b45e9cec53099498fa61c8b7acba4/_/download/contextbatch/js/jira.project.sidebar,-_super,-project.issue.navigator,-jira.general,-jira.browse.project,-jira.view.issue,-jira.global,-atl.general,-com.atlassian.jira.projects.sidebar.init/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true","startTime":857.0999999046326,"connectEnd":857.0999999046326,"connectStart":857.0999999046326,"domainLookupEnd":857.0999999046326,"domainLookupStart":857.0999999046326,"fetchStart":857.0999999046326,"redirectEnd":0,"redirectStart":0,"requestStart":857.0999999046326,"responseEnd":1031,"responseStart":1031,"secureConnectionStart":857.0999999046326},{"duration":145.89999985694885,"initiatorType":"script","name":"https://www.google-analytics.com/analytics.js","startTime":886.4000000953674,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":886.4000000953674,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1032.2999999523163,"responseStart":0,"secureConnectionStart":0}],"fetchStart":1,"domainLookupStart":1,"domainLookupEnd":1,"connectStart":1,"connectEnd":1,"requestStart":11,"responseStart":132,"responseEnd":137,"domLoading":135,"domInteractive":1025,"domContentLoadedEventStart":1025,"domContentLoadedEventEnd":1059,"domComplete":1778,"loadEventStart":1778,"loadEventEnd":1779,"userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","marks":[{"name":"bigPipe.sidebar-id.start","time":1008.2999999523163},{"name":"bigPipe.sidebar-id.end","time":1009},{"name":"bigPipe.activity-panel-pipe-id.start","time":1009.0999999046326},{"name":"bigPipe.activity-panel-pipe-id.end","time":1011.4000000953674},{"name":"activityTabFullyLoaded","time":1070.0999999046326}],"measures":[],"correlationId":"2a81f73f5b054a","effectiveType":"4g","downlink":9.7,"rtt":0,"serverDuration":68,"dbReadsTimeInMs":8,"dbConnsTimeInMs":15,"applicationHash":"9d11dbea5f4be3d4cc21f03a88dd11d8c8687422","experiments":[]}}