Hi,
I have a setup with 3 Galera nodes on MariaDB 10.4.18.
I have blocking issue with MariaDB stability during concurrent updates on same row (this usually happens for me when multiple client applications tries to modify a record in the database in the same time).
The issues is the following:
1. Multiple customers starts the application and tries to perform an update on the same record.
2. Galera handles a few concurrencies but eventually hangs.
Now, I understand that we need to retry the transaction in the case of a deadlock, the issue is that Galera hangs before the application retries that.
In order to replicate this we perform the following test:
1. Start Galera on 3 nodes
2. Have a ProxySQL with 3 nodes as writers
3. We have a tool that connects to Galera start a tool that sends queries continuously to the database
I'm attaching the script that we use testing and also I'm putting here he Galera configuration:
- MariaDB validation diagram.png - This is the network topology for testing
- MariaDB-server.cnf
- galera-lock-replication.sql - SQLs to create the database and table for testing
- galera-lock-replication.php - the tool that connects to database and send sql update
- threads-tool.php - the tool that starts "galera-lock-replication.php" in multithreading mode to simulate concurrency
Issue replication
1. Create the database using the galera-lock-replication.sql
script provided in attach
2. Copy the 2 php files (galera-lock-replication.php
and threads-tool.php
) provided in attach in /galera-test/ folder under apache root
3. Start a first terminal with:
# php /path/to/galera-test/threads-tool.php
|
- this will start bulks of 100 paralel calls to the frontend webservice in a loop:
http://127.0.0.1/galera-test/galera-lock-replication.php
|
4. Wait few seconds (between 5 and 20)
5. Galera cluster should start refuse connections
- is duplicated by
-
MDEV-25883
Galera Cluster hangs while "DELETE FROM mysql.wsrep_cluster"
-
-
Closed
{"report":{"fcp":723.1999998092651,"ttfb":155.2999997138977,"pageVisibility":"visible","entityId":103612,"key":"jira.project.issue.view-issue","isInitial":true,"threshold":1000,"elementTimings":{},"userDeviceMemory":8,"userDeviceProcessors":64,"apdex":1,"journeyId":"ba16f39f-6249-4863-b598-0217369ace77","navigationType":0,"readyForUser":804.0999999046326,"redirectCount":0,"resourceLoadedEnd":1034,"resourceLoadedStart":161,"resourceTiming":[{"duration":8.799999713897705,"initiatorType":"link","name":"https://jira.mariadb.org/s/2c21342762a6a02add1c328bed317ffd-CDN/lu2bv2/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/css/_super/batch.css","startTime":161,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":161,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":169.7999997138977,"responseStart":0,"secureConnectionStart":0},{"duration":8.800000190734863,"initiatorType":"link","name":"https://jira.mariadb.org/s/7ebd35e77e471bc30ff0eba799ebc151-CDN/lu2bv2/820016/12ta74/2380add21a9a1006587582385952de73/_/download/contextbatch/css/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true","startTime":161.2999997138977,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":161.2999997138977,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":170.09999990463257,"responseStart":0,"secureConnectionStart":0},{"duration":61.59999990463257,"initiatorType":"script","name":"https://jira.mariadb.org/s/e9b27a47da5fb0f74a35acd57e9847fb-CDN/lu2bv2/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/js/_super/batch.js?locale=en","startTime":161.40000009536743,"connectEnd":161.40000009536743,"connectStart":161.40000009536743,"domainLookupEnd":161.40000009536743,"domainLookupStart":161.40000009536743,"fetchStart":161.40000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":161.40000009536743,"responseEnd":223,"responseStart":223,"secureConnectionStart":161.40000009536743},{"duration":148.59999990463257,"initiatorType":"script","name":"https://jira.mariadb.org/s/c32eb0da7ad9831253f8397e6cc26afd-CDN/lu2bv2/820016/12ta74/2380add21a9a1006587582385952de73/_/download/contextbatch/js/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true","startTime":161.59999990463257,"connectEnd":161.59999990463257,"connectStart":161.59999990463257,"domainLookupEnd":161.59999990463257,"domainLookupStart":161.59999990463257,"fetchStart":161.59999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":161.59999990463257,"responseEnd":310.19999980926514,"responseStart":310.19999980926514,"secureConnectionStart":161.59999990463257},{"duration":152.10000038146973,"initiatorType":"script","name":"https://jira.mariadb.org/s/bc0bcb146314416123c992714ee00ff7-CDN/lu2bv2/820016/12ta74/c92c0caa9a024ae85b0ebdbed7fb4bd7/_/download/contextbatch/js/atl.global,-_super/batch.js?locale=en","startTime":161.7999997138977,"connectEnd":161.7999997138977,"connectStart":161.7999997138977,"domainLookupEnd":161.7999997138977,"domainLookupStart":161.7999997138977,"fetchStart":161.7999997138977,"redirectEnd":0,"redirectStart":0,"requestStart":161.7999997138977,"responseEnd":313.90000009536743,"responseStart":313.90000009536743,"secureConnectionStart":161.7999997138977},{"duration":152.80000019073486,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bv2/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-en/jira.webresources:calendar-en.js","startTime":162.09999990463257,"connectEnd":162.09999990463257,"connectStart":162.09999990463257,"domainLookupEnd":162.09999990463257,"domainLookupStart":162.09999990463257,"fetchStart":162.09999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":162.09999990463257,"responseEnd":314.90000009536743,"responseStart":314.90000009536743,"secureConnectionStart":162.09999990463257},{"duration":153.40000009536743,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bv2/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-localisation-moment/jira.webresources:calendar-localisation-moment.js","startTime":162.2999997138977,"connectEnd":162.2999997138977,"connectStart":162.2999997138977,"domainLookupEnd":162.2999997138977,"domainLookupStart":162.2999997138977,"fetchStart":162.2999997138977,"redirectEnd":0,"redirectStart":0,"requestStart":162.2999997138977,"responseEnd":315.69999980926514,"responseStart":315.69999980926514,"secureConnectionStart":162.2999997138977},{"duration":202.69999980926514,"initiatorType":"link","name":"https://jira.mariadb.org/s/b04b06a02d1959df322d9cded3aeecc1-CDN/lu2bv2/820016/12ta74/a2ff6aa845ffc9a1d22fe23d9ee791fc/_/download/contextbatch/css/jira.global.look-and-feel,-_super/batch.css","startTime":162.5,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":162.5,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":365.19999980926514,"responseStart":0,"secureConnectionStart":0},{"duration":154,"initiatorType":"script","name":"https://jira.mariadb.org/rest/api/1.0/shortcuts/820016/47140b6e0a9bc2e4913da06536125810/shortcuts.js?context=issuenavigation&context=issueaction","startTime":162.59999990463257,"connectEnd":162.59999990463257,"connectStart":162.59999990463257,"domainLookupEnd":162.59999990463257,"domainLookupStart":162.59999990463257,"fetchStart":162.59999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":162.59999990463257,"responseEnd":316.59999990463257,"responseStart":316.59999990463257,"secureConnectionStart":162.59999990463257},{"duration":202.60000038146973,"initiatorType":"link","name":"https://jira.mariadb.org/s/3ac36323ba5e4eb0af2aa7ac7211b4bb-CDN/lu2bv2/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/css/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.css?jira.create.linked.issue=true","startTime":162.7999997138977,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":162.7999997138977,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":365.40000009536743,"responseStart":0,"secureConnectionStart":0},{"duration":154.5,"initiatorType":"script","name":"https://jira.mariadb.org/s/719848dd97ebe0663199f49a3936487a-CDN/lu2bv2/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/js/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.js?jira.create.linked.issue=true&locale=en","startTime":163,"connectEnd":163,"connectStart":163,"domainLookupEnd":163,"domainLookupStart":163,"fetchStart":163,"redirectEnd":0,"redirectStart":0,"requestStart":163,"responseEnd":317.5,"responseStart":317.5,"secureConnectionStart":163},{"duration":276.30000019073486,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bv2/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-js/jira.webresources:bigpipe-js.js","startTime":164.09999990463257,"connectEnd":164.09999990463257,"connectStart":164.09999990463257,"domainLookupEnd":164.09999990463257,"domainLookupStart":164.09999990463257,"fetchStart":164.09999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":164.09999990463257,"responseEnd":440.40000009536743,"responseStart":440.40000009536743,"secureConnectionStart":164.09999990463257},{"duration":869.8000001907349,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bv2/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-init/jira.webresources:bigpipe-init.js","startTime":164.19999980926514,"connectEnd":164.19999980926514,"connectStart":164.19999980926514,"domainLookupEnd":164.19999980926514,"domainLookupStart":164.19999980926514,"fetchStart":164.19999980926514,"redirectEnd":0,"redirectStart":0,"requestStart":164.19999980926514,"responseEnd":1034,"responseStart":1034,"secureConnectionStart":164.19999980926514},{"duration":71,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":376.90000009536743,"connectEnd":376.90000009536743,"connectStart":376.90000009536743,"domainLookupEnd":376.90000009536743,"domainLookupStart":376.90000009536743,"fetchStart":376.90000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":376.90000009536743,"responseEnd":447.90000009536743,"responseStart":447.90000009536743,"secureConnectionStart":376.90000009536743}],"fetchStart":0,"domainLookupStart":0,"domainLookupEnd":0,"connectStart":0,"connectEnd":0,"requestStart":2,"responseStart":155,"responseEnd":158,"domLoading":159,"domInteractive":1052,"domContentLoadedEventStart":1052,"domContentLoadedEventEnd":1091,"domComplete":2066,"loadEventStart":2066,"loadEventEnd":2068,"userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","marks":[{"name":"bigPipe.sidebar-id.start","time":1036.1999998092651},{"name":"bigPipe.sidebar-id.end","time":1037.0999999046326},{"name":"bigPipe.activity-panel-pipe-id.start","time":1037.1999998092651},{"name":"bigPipe.activity-panel-pipe-id.end","time":1039.5},{"name":"activityTabFullyLoaded","time":1098.1999998092651}],"measures":[],"correlationId":"7e5fd2961216f3","effectiveType":"4g","downlink":10,"rtt":0,"serverDuration":94,"dbReadsTimeInMs":12,"dbConnsTimeInMs":20,"applicationHash":"9d11dbea5f4be3d4cc21f03a88dd11d8c8687422","experiments":[]}}