Shutting down MariaDB by SIGTERM when it has many (20) replication channels (with parallel replication enabled) seems never-ending. But if I stop all slaves before shutdown, it takes a few seconds. Moreover, mysqld reports errors on I/O threads.
I'd expect that server stops all slaves itself.
# sudo systemctl status mariadb -n 64
|
● mariadb.service - MariaDB server
|
Loaded: loaded (/nix/store/4hm49xrn8md0swvc3sm9n2qhafwllgc1-unit-mariadb.service/mariadb.service)
|
Active: deactivating (stop-sigterm) since Fri 2016-02-19 21:06:29 SGT; 1h 50min ago
|
Process: 3189 ExecStartPre=/nix/store/m8hhisn7gag7iycddw8hflila3gi71h3-unit-script/bin/mariadb-pre-start (code=exited, status=0/SUCCESS)
|
Main PID: 3640 (mysqld)
|
CGroup: /system.slice/mariadb.service
|
└─3640 /nix/store/z7x0ndrckrsmqr3163hp9kgs9hwags61-mariadb-10.1.11/bin/mysqld --defaults-file=/nix/store/8lgvar6h0696ngpsladkvmqwbd5k6x5v-mysqld.cnf
|
|
Feb 19 20:04:23 grumpycat mysqld[3640]: 2016-02-19 20:04:23 139653215652608 [Note] Master 'oms-my': Slave SQL thread initialized, starting replication in l...n: 3761334
|
Feb 19 21:06:29 grumpycat systemd[1]: Stopping MariaDB server...
|
Feb 19 21:06:29 grumpycat mysqld[3640]: 2016-02-19 21:06:29 139652892388096 [Note] /nix/store/z7x0ndrckrsmqr3163hp9kgs9hwags61-mariadb-10.1.11/bin/mysqld: ...l shutdown
|
Feb 19 21:06:29 grumpycat mysqld[3640]: 2016-02-19 21:06:29 139652892388096 [Note] Event Scheduler: Purging the queue. 0 events
|
Feb 19 21:06:29 grumpycat mysqld[3640]: 2016-02-19 21:06:29 139653540129536 [Note] Master 'bob-th': Error reading relay log event: slave SQL thread was killed
|
Feb 19 21:06:29 grumpycat mysqld[3640]: 2016-02-19 21:06:29 139652246162176 [ERROR] Master 'bob-th': Error reading packet from server: Lost connection to M...rrno=2013)
|
Feb 19 21:06:29 grumpycat mysqld[3640]: 2016-02-19 21:06:29 139652246162176 [Note] Master 'bob-th': Slave I/O thread exiting, read up to log 'mysql-bin-cha...on 3256007
|
Feb 19 21:06:29 grumpycat mysqld[3640]: 2016-02-19 21:06:29 139653539826432 [Note] Master 'zops': Error reading relay log event: slave SQL thread was killed
|
Feb 19 21:06:29 grumpycat mysqld[3640]: 2016-02-19 21:06:29 139652894509824 [ERROR] Master 'zops': Error reading packet from server: Lost connection to MyS...rrno=2013)
|
Feb 19 21:06:29 grumpycat mysqld[3640]: 2016-02-19 21:06:29 139652894509824 [Note] Master 'zops': Slave I/O thread exiting, read up to log 'mysql-bin-chang...sition 422
|
Feb 19 21:06:29 grumpycat mysqld[3640]: 2016-02-19 21:06:29 139653999532800 [Note] Master 'bob-my': Error reading relay log event: slave SQL thread was killed
|
Feb 19 21:06:29 grumpycat mysqld[3640]: 2016-02-19 21:06:29 139652570942208 [ERROR] Master 'bob-my': Error reading packet from server: Lost connection to M...rrno=2013)
|
Feb 19 21:06:29 grumpycat mysqld[3640]: 2016-02-19 21:06:29 139652570942208 [Note] Master 'bob-my': Slave I/O thread exiting, read up to log 'mysql-bin-cha...on 3947621
|
Feb 19 21:06:29 grumpycat mysqld[3640]: 2016-02-19 21:06:29 139653538310912 [Note] Master 'bob-hk': Error reading relay log event: slave SQL thread was killed
|
Feb 19 21:06:29 grumpycat mysqld[3640]: 2016-02-19 21:06:29 139652570336000 [ERROR] Master 'bob-hk': Error reading packet from server: Lost connection to M...rrno=2013)
|
Feb 19 21:06:29 grumpycat mysqld[3640]: 2016-02-19 21:06:29 139652570336000 [Note] Master 'bob-hk': Slave I/O thread exiting, read up to log 'mysql-bin-cha...on 1303108
|
Feb 19 21:06:30 grumpycat mysqld[3640]: 2016-02-19 21:06:30 139653217471232 [Note] Master 'bob-ph': Error reading relay log event: slave SQL thread was killed
|
Feb 19 21:06:30 grumpycat mysqld[3640]: 2016-02-19 21:06:30 139652892691200 [ERROR] Master 'bob-ph': Error reading packet from server: Lost connection to M...rrno=2013)
|
Feb 19 21:06:30 grumpycat mysqld[3640]: 2016-02-19 21:06:30 139652892691200 [Note] Master 'bob-ph': Slave I/O thread exiting, read up to log 'mysql-bin-cha...on 8157090
|
...
|
Server has 30G of RAM, and 4 CPUs. iotop shows ~100K/s read and write for mysqld process
[mysqld]
|
basedir = /nix/store/z7x0ndrckrsmqr3163hp9kgs9hwags61-mariadb-10.1.11
|
init_file = /nix/store/71malbvn6kkn29yiwrsalnpirf2b6v62-init
|
pid_file = /run/mysqld/mysqld.pid
|
plugin_load = unix_socket=auth_socket.so
|
datadir = /mariadb/db
|
ignore_db_dirs = lost+found
|
innodb_buffer_pool_instances = 64
|
innodb_buffer_pool_size = 19327352832
|
innodb_file_format = barracuda
|
innodb_file_per_table = ON
|
innodb_flush_log_at_trx_commit = 2
|
innodb_flush_method = O_DIRECT
|
innodb_lock_wait_timeout = 1800
|
innodb_log_file_size = 314572800
|
join_buffer_size = 1048576
|
log_slave_updates = OFF
|
max_allowed_packet = 268435456
|
max_connections = 1000
|
net_read_timeout = 1000
|
net_write_timeout = 1000
|
port = 3306
|
relay_log = /mariadb/relay/cat-bin
|
server_id = 112233
|
skip_log_bin
|
slave_compressed_protocol = ON
|
slave_domain_parallel_threads = 10
|
slave_net_timeout = 600
|
slave_parallel_max_queued = 8388608
|
slave_parallel_threads = 40
|
sort_buffer_size = 4194304
|
ssl_cert = /nix/store/rmlq71y8fnyx1k7sp3qx55fqwg45pl40-cats-cert.pem
|
ssl_key = /run/keys/cats-key.pem
|
table_open_cache = 30000
|
; replicate_*
|
!include /nix/store/czaiav63ddqrz18sizhfy69jvv04nfcn-mysqld-repl.cnf
|
{"report":{"fcp":1118.2999997138977,"ttfb":373.7999997138977,"pageVisibility":"visible","entityId":55629,"key":"jira.project.issue.view-issue","isInitial":true,"threshold":1000,"elementTimings":{},"userDeviceMemory":8,"userDeviceProcessors":64,"apdex":0.5,"journeyId":"f8554d32-483c-43ab-8a4c-6eb781b85518","navigationType":0,"readyForUser":1242,"redirectCount":0,"resourceLoadedEnd":1293.0999999046326,"resourceLoadedStart":379.69999980926514,"resourceTiming":[{"duration":115.5,"initiatorType":"link","name":"https://jira.mariadb.org/s/2c21342762a6a02add1c328bed317ffd-CDN/lu2cib/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/css/_super/batch.css","startTime":379.69999980926514,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":379.69999980926514,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":495.19999980926514,"responseStart":0,"secureConnectionStart":0},{"duration":116,"initiatorType":"link","name":"https://jira.mariadb.org/s/7ebd35e77e471bc30ff0eba799ebc151-CDN/lu2cib/820016/12ta74/494e4c556ecbb29f90a3d3b4f09cb99c/_/download/contextbatch/css/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true&whisper-enabled=true","startTime":380,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":380,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":496,"responseStart":0,"secureConnectionStart":0},{"duration":171.80000019073486,"initiatorType":"script","name":"https://jira.mariadb.org/s/0917945aaa57108d00c5076fea35e069-CDN/lu2cib/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/js/_super/batch.js?locale=en","startTime":380.19999980926514,"connectEnd":380.19999980926514,"connectStart":380.19999980926514,"domainLookupEnd":380.19999980926514,"domainLookupStart":380.19999980926514,"fetchStart":380.19999980926514,"redirectEnd":0,"redirectStart":0,"requestStart":380.19999980926514,"responseEnd":552,"responseStart":552,"secureConnectionStart":380.19999980926514},{"duration":254,"initiatorType":"script","name":"https://jira.mariadb.org/s/2d8175ec2fa4c816e8023260bd8c1786-CDN/lu2cib/820016/12ta74/494e4c556ecbb29f90a3d3b4f09cb99c/_/download/contextbatch/js/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true&whisper-enabled=true","startTime":380.40000009536743,"connectEnd":380.40000009536743,"connectStart":380.40000009536743,"domainLookupEnd":380.40000009536743,"domainLookupStart":380.40000009536743,"fetchStart":380.40000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":380.40000009536743,"responseEnd":634.4000000953674,"responseStart":634.4000000953674,"secureConnectionStart":380.40000009536743},{"duration":257.90000009536743,"initiatorType":"script","name":"https://jira.mariadb.org/s/a9324d6758d385eb45c462685ad88f1d-CDN/lu2cib/820016/12ta74/c92c0caa9a024ae85b0ebdbed7fb4bd7/_/download/contextbatch/js/atl.global,-_super/batch.js?locale=en","startTime":380.59999990463257,"connectEnd":380.59999990463257,"connectStart":380.59999990463257,"domainLookupEnd":380.59999990463257,"domainLookupStart":380.59999990463257,"fetchStart":380.59999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":380.59999990463257,"responseEnd":638.5,"responseStart":638.5,"secureConnectionStart":380.59999990463257},{"duration":258.1000003814697,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-en/jira.webresources:calendar-en.js","startTime":380.7999997138977,"connectEnd":380.7999997138977,"connectStart":380.7999997138977,"domainLookupEnd":380.7999997138977,"domainLookupStart":380.7999997138977,"fetchStart":380.7999997138977,"redirectEnd":0,"redirectStart":0,"requestStart":380.7999997138977,"responseEnd":638.9000000953674,"responseStart":638.9000000953674,"secureConnectionStart":380.7999997138977},{"duration":258.2999997138977,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-localisation-moment/jira.webresources:calendar-localisation-moment.js","startTime":381,"connectEnd":381,"connectStart":381,"domainLookupEnd":381,"domainLookupStart":381,"fetchStart":381,"redirectEnd":0,"redirectStart":0,"requestStart":381,"responseEnd":639.2999997138977,"responseStart":639.2999997138977,"secureConnectionStart":381},{"duration":319,"initiatorType":"link","name":"https://jira.mariadb.org/s/b04b06a02d1959df322d9cded3aeecc1-CDN/lu2cib/820016/12ta74/a2ff6aa845ffc9a1d22fe23d9ee791fc/_/download/contextbatch/css/jira.global.look-and-feel,-_super/batch.css","startTime":381.19999980926514,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":381.19999980926514,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":700.1999998092651,"responseStart":0,"secureConnectionStart":0},{"duration":258.5,"initiatorType":"script","name":"https://jira.mariadb.org/rest/api/1.0/shortcuts/820016/47140b6e0a9bc2e4913da06536125810/shortcuts.js?context=issuenavigation&context=issueaction","startTime":381.2999997138977,"connectEnd":381.2999997138977,"connectStart":381.2999997138977,"domainLookupEnd":381.2999997138977,"domainLookupStart":381.2999997138977,"fetchStart":381.2999997138977,"redirectEnd":0,"redirectStart":0,"requestStart":381.2999997138977,"responseEnd":639.7999997138977,"responseStart":639.7999997138977,"secureConnectionStart":381.2999997138977},{"duration":318.80000019073486,"initiatorType":"link","name":"https://jira.mariadb.org/s/3ac36323ba5e4eb0af2aa7ac7211b4bb-CDN/lu2cib/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/css/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.css?jira.create.linked.issue=true","startTime":381.59999990463257,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":381.59999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":700.4000000953674,"responseStart":0,"secureConnectionStart":0},{"duration":258.7000002861023,"initiatorType":"script","name":"https://jira.mariadb.org/s/5d5e8fe91fbc506585e83ea3b62ccc4b-CDN/lu2cib/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/js/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.js?jira.create.linked.issue=true&locale=en","startTime":381.69999980926514,"connectEnd":381.69999980926514,"connectStart":381.69999980926514,"domainLookupEnd":381.69999980926514,"domainLookupStart":381.69999980926514,"fetchStart":381.69999980926514,"redirectEnd":0,"redirectStart":0,"requestStart":381.69999980926514,"responseEnd":640.4000000953674,"responseStart":640.4000000953674,"secureConnectionStart":381.69999980926514},{"duration":520.9000000953674,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-js/jira.webresources:bigpipe-js.js","startTime":382.5,"connectEnd":382.5,"connectStart":382.5,"domainLookupEnd":382.5,"domainLookupStart":382.5,"fetchStart":382.5,"redirectEnd":0,"redirectStart":0,"requestStart":382.5,"responseEnd":903.4000000953674,"responseStart":903.4000000953674,"secureConnectionStart":382.5},{"duration":910.5999999046326,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-init/jira.webresources:bigpipe-init.js","startTime":382.5,"connectEnd":382.5,"connectStart":382.5,"domainLookupEnd":382.5,"domainLookupStart":382.5,"fetchStart":382.5,"redirectEnd":0,"redirectStart":0,"requestStart":382.5,"responseEnd":1293.0999999046326,"responseStart":1293.0999999046326,"secureConnectionStart":382.5},{"duration":187.59999990463257,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":716.1999998092651,"connectEnd":716.1999998092651,"connectStart":716.1999998092651,"domainLookupEnd":716.1999998092651,"domainLookupStart":716.1999998092651,"fetchStart":716.1999998092651,"redirectEnd":0,"redirectStart":0,"requestStart":716.1999998092651,"responseEnd":903.7999997138977,"responseStart":903.7999997138977,"secureConnectionStart":716.1999998092651}],"fetchStart":1,"domainLookupStart":1,"domainLookupEnd":1,"connectStart":1,"connectEnd":1,"requestStart":200,"responseStart":374,"responseEnd":378,"domLoading":378,"domInteractive":1328,"domContentLoadedEventStart":1328,"domContentLoadedEventEnd":1382,"domComplete":2068,"loadEventStart":2068,"loadEventEnd":2070,"userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","marks":[{"name":"bigPipe.sidebar-id.start","time":1298.4000000953674},{"name":"bigPipe.sidebar-id.end","time":1299.2999997138977},{"name":"bigPipe.activity-panel-pipe-id.start","time":1299.5},{"name":"bigPipe.activity-panel-pipe-id.end","time":1302.1999998092651},{"name":"activityTabFullyLoaded","time":1390.2999997138977}],"measures":[],"correlationId":"79eaef934f07ec","effectiveType":"4g","downlink":10,"rtt":0,"serverDuration":102,"dbReadsTimeInMs":16,"dbConnsTimeInMs":26,"applicationHash":"9d11dbea5f4be3d4cc21f03a88dd11d8c8687422","experiments":[]}}
Bug
Minor