Hey everyone
Rarely, we have a weird issue happening. We have a master <-> master setup. The writes are going to only one of them. Every now and then, the replication lag on the master where we send writes jumps to millions of seconds (ex: 2879736). Then, after 1s, is back to 0. After all, there's no lag, as the writes and the binlog from the other master should be ignored, as this is the server where we write to.
Both servers have the same timezone and ntp running.
Replication is set with GTIDs, and parallel threads.
Sample config:
slave_net_timeout=2
gtid-strict-mode=1
gtid-domain-id=1
server_id = 101
sync_binlog = 1
binlog_format = ROW
log-slave-updates
replicate-same-server-id = 0
log_bin=/var/log/mysql/binlog/mysql-bin.log
log_bin_index=/var/log/mysql/binlog/mysql-bin.log.index
relay_log=/var/log/mysql/binlog/mysql-relay-bin
relay_log_index=/var/log/mysql/binlog/mysql-relay-bin.index
expire_logs_days=5
max_binlog_size=1G
slave_parallel_threads=10
We have a bunch of monitoring scripts looking at that value and triggering actions and alerts and we want to know what can cause this and if there's anything we can do to avoid this in the future.
Thank you
{"report":{"fcp":765,"ttfb":173.60000038146973,"pageVisibility":"visible","entityId":67235,"key":"jira.project.issue.view-issue","isInitial":true,"threshold":1000,"elementTimings":{},"userDeviceMemory":8,"userDeviceProcessors":64,"apdex":1,"journeyId":"4a8eb25a-2161-4776-939b-f5b1b164c5f2","navigationType":0,"readyForUser":840.4000005722046,"redirectCount":0,"resourceLoadedEnd":698.6000003814697,"resourceLoadedStart":200.80000019073486,"resourceTiming":[{"duration":14.90000057220459,"initiatorType":"link","name":"https://jira.mariadb.org/s/2c21342762a6a02add1c328bed317ffd-CDN/lu2bv2/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/css/_super/batch.css","startTime":200.80000019073486,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":200.80000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":215.70000076293945,"responseStart":0,"secureConnectionStart":0},{"duration":14.899999618530273,"initiatorType":"link","name":"https://jira.mariadb.org/s/7ebd35e77e471bc30ff0eba799ebc151-CDN/lu2bv2/820016/12ta74/2380add21a9a1006587582385952de73/_/download/contextbatch/css/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true","startTime":201.10000038146973,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":201.10000038146973,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":216,"responseStart":0,"secureConnectionStart":0},{"duration":68.90000057220459,"initiatorType":"script","name":"https://jira.mariadb.org/s/e9b27a47da5fb0f74a35acd57e9847fb-CDN/lu2bv2/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/js/_super/batch.js?locale=en","startTime":201.30000019073486,"connectEnd":201.30000019073486,"connectStart":201.30000019073486,"domainLookupEnd":201.30000019073486,"domainLookupStart":201.30000019073486,"fetchStart":201.30000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":201.30000019073486,"responseEnd":270.20000076293945,"responseStart":270.20000076293945,"secureConnectionStart":201.30000019073486},{"duration":129.80000019073486,"initiatorType":"script","name":"https://jira.mariadb.org/s/c32eb0da7ad9831253f8397e6cc26afd-CDN/lu2bv2/820016/12ta74/2380add21a9a1006587582385952de73/_/download/contextbatch/js/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true","startTime":201.60000038146973,"connectEnd":201.60000038146973,"connectStart":201.60000038146973,"domainLookupEnd":201.60000038146973,"domainLookupStart":201.60000038146973,"fetchStart":201.60000038146973,"redirectEnd":0,"redirectStart":0,"requestStart":201.60000038146973,"responseEnd":331.4000005722046,"responseStart":331.4000005722046,"secureConnectionStart":201.60000038146973},{"duration":133,"initiatorType":"script","name":"https://jira.mariadb.org/s/bc0bcb146314416123c992714ee00ff7-CDN/lu2bv2/820016/12ta74/c92c0caa9a024ae85b0ebdbed7fb4bd7/_/download/contextbatch/js/atl.global,-_super/batch.js?locale=en","startTime":202.4000005722046,"connectEnd":202.4000005722046,"connectStart":202.4000005722046,"domainLookupEnd":202.4000005722046,"domainLookupStart":202.4000005722046,"fetchStart":202.4000005722046,"redirectEnd":0,"redirectStart":0,"requestStart":202.4000005722046,"responseEnd":335.4000005722046,"responseStart":335.4000005722046,"secureConnectionStart":202.4000005722046},{"duration":133.0999994277954,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bv2/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-en/jira.webresources:calendar-en.js","startTime":202.70000076293945,"connectEnd":202.70000076293945,"connectStart":202.70000076293945,"domainLookupEnd":202.70000076293945,"domainLookupStart":202.70000076293945,"fetchStart":202.70000076293945,"redirectEnd":0,"redirectStart":0,"requestStart":202.70000076293945,"responseEnd":335.80000019073486,"responseStart":335.70000076293945,"secureConnectionStart":202.70000076293945},{"duration":133.10000038146973,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bv2/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-localisation-moment/jira.webresources:calendar-localisation-moment.js","startTime":203,"connectEnd":203,"connectStart":203,"domainLookupEnd":203,"domainLookupStart":203,"fetchStart":203,"redirectEnd":0,"redirectStart":0,"requestStart":203,"responseEnd":336.1000003814697,"responseStart":336.1000003814697,"secureConnectionStart":203},{"duration":214.89999961853027,"initiatorType":"link","name":"https://jira.mariadb.org/s/b04b06a02d1959df322d9cded3aeecc1-CDN/lu2bv2/820016/12ta74/a2ff6aa845ffc9a1d22fe23d9ee791fc/_/download/contextbatch/css/jira.global.look-and-feel,-_super/batch.css","startTime":203.10000038146973,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":203.10000038146973,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":418,"responseStart":0,"secureConnectionStart":0},{"duration":133.30000019073486,"initiatorType":"script","name":"https://jira.mariadb.org/rest/api/1.0/shortcuts/820016/47140b6e0a9bc2e4913da06536125810/shortcuts.js?context=issuenavigation&context=issueaction","startTime":203.30000019073486,"connectEnd":203.30000019073486,"connectStart":203.30000019073486,"domainLookupEnd":203.30000019073486,"domainLookupStart":203.30000019073486,"fetchStart":203.30000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":203.30000019073486,"responseEnd":336.6000003814697,"responseStart":336.6000003814697,"secureConnectionStart":203.30000019073486},{"duration":214.70000076293945,"initiatorType":"link","name":"https://jira.mariadb.org/s/3ac36323ba5e4eb0af2aa7ac7211b4bb-CDN/lu2bv2/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/css/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.css?jira.create.linked.issue=true","startTime":203.5,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":203.5,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":418.20000076293945,"responseStart":0,"secureConnectionStart":0},{"duration":133.60000038146973,"initiatorType":"script","name":"https://jira.mariadb.org/s/719848dd97ebe0663199f49a3936487a-CDN/lu2bv2/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/js/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.js?jira.create.linked.issue=true&locale=en","startTime":203.60000038146973,"connectEnd":203.60000038146973,"connectStart":203.60000038146973,"domainLookupEnd":203.60000038146973,"domainLookupStart":203.60000038146973,"fetchStart":203.60000038146973,"redirectEnd":0,"redirectStart":0,"requestStart":203.60000038146973,"responseEnd":337.20000076293945,"responseStart":337.20000076293945,"secureConnectionStart":203.60000038146973},{"duration":493,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bv2/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-js/jira.webresources:bigpipe-js.js","startTime":204.80000019073486,"connectEnd":204.80000019073486,"connectStart":204.80000019073486,"domainLookupEnd":204.80000019073486,"domainLookupStart":204.80000019073486,"fetchStart":204.80000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":204.80000019073486,"responseEnd":697.8000001907349,"responseStart":697.8000001907349,"secureConnectionStart":204.80000019073486},{"duration":493.6000003814697,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bv2/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-init/jira.webresources:bigpipe-init.js","startTime":205,"connectEnd":205,"connectStart":205,"domainLookupEnd":205,"domainLookupStart":205,"fetchStart":205,"redirectEnd":0,"redirectStart":0,"requestStart":205,"responseEnd":698.6000003814697,"responseStart":698.6000003814697,"secureConnectionStart":205},{"duration":212.4000005722046,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":446.30000019073486,"connectEnd":446.30000019073486,"connectStart":446.30000019073486,"domainLookupEnd":446.30000019073486,"domainLookupStart":446.30000019073486,"fetchStart":446.30000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":446.30000019073486,"responseEnd":658.7000007629395,"responseStart":658.7000007629395,"secureConnectionStart":446.30000019073486}],"fetchStart":0,"domainLookupStart":0,"domainLookupEnd":0,"connectStart":0,"connectEnd":0,"requestStart":13,"responseStart":174,"responseEnd":175,"domLoading":199,"domInteractive":896,"domContentLoadedEventStart":896,"domContentLoadedEventEnd":949,"domComplete":1756,"loadEventStart":1756,"loadEventEnd":1758,"userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","marks":[{"name":"bigPipe.sidebar-id.start","time":875.7000007629395},{"name":"bigPipe.sidebar-id.end","time":876.7000007629395},{"name":"bigPipe.activity-panel-pipe-id.start","time":876.8000001907349},{"name":"bigPipe.activity-panel-pipe-id.end","time":879.3000001907349},{"name":"activityTabFullyLoaded","time":961.9000005722046}],"measures":[],"correlationId":"6b6ab5e60d9d3f","effectiveType":"4g","downlink":9.6,"rtt":0,"serverDuration":106,"dbReadsTimeInMs":16,"dbConnsTimeInMs":25,"applicationHash":"9d11dbea5f4be3d4cc21f03a88dd11d8c8687422","experiments":[]}}