Info = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = Preparing a production change, I've tried to get smarter on how to change the setup of your ClusterControl. I am replacing all VM nodes with Physical. One at a time, drop a new phys into the VM mix, take one VM out. All works like a charm. After this is done, I need to reconfig CC to monitor the new phys nodes in stead. Canøt get it to work in my test setup. Test is a mix now, off 3 Ubuntus ("VM") and the 3 new Centos machines. But CC only knows the Ubuntu ones. I thought the wise thing would be to just delete the cluster in CC, but keep the servers running untouched, so that's what I did. CC knows no cluster now. Trying to (re-)import is a pain - it fails as follows: " [13:16:44]: 192.168.56.201:3306: Failed to determine bind_address: , command: /usr/bin/mysql -NAB -u'root' -pxxxxx -e "show global variables where variable_name = 'bind_address'" 2>&1 [13:16:42]: Detected that skip_name_resolve is used on the target server(s). " I can't figure out what's wrong here - the query will give no result in any of the servers, and that paremeter is (as far as I know) called bind-address, not bind_address ? Can you please shed some light on what I am bashing wrongfully on here :) ?? CC is version: © Copyright Severalnines 2019 All Rights Reserved Controller: 1.7.1.3040, Frontend: 1.7.1.5622-#69b9ab, License: Community Version Here's the job details { "command": "add_cluster", "job_data": { "api_id": 1, "cluster_name": "galera_cluster", "cluster_type": "galera", "config_template": "my.cnf.galera", "datadir": "/var/lib/mysql", "db_password": "paaaswd", "db_user": "root", "disable_firewall": true, "disable_selinux": true, "enable_cluster_autorecovery": false, "enable_information_schema_queries": true, "enable_node_autorecovery": false, "enable_nodes_autodetect": false, "galera_version": "2.x", "generate_token": true, "install_software": true, "monitored_mysql_port": "3306", "nodes": [ { "hostname": "192.168.56.101", "port": "3306" }, { "hostname": "192.168.56.102", "port": "3306" }, { "hostname": "192.168.56.103", "port": "3306" }, { "hostname": "192.168.56.201", "port": "3306" }, { "hostname": "192.168.56.202", "port": "3306" }, { "hostname": "192.168.56.203", "port": "3306" } ], "port": "3306", "ssh_keyfile": "/root/.ssh/id_rsa", "ssh_port": "22", "ssh_user": "root", "sudo_password": "", "tag": "", "user_id": 1, "vendor": "mariadb", "version": "10.2" } } I will attach a screenshot of the job log itself. Thanks Brian = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = Bart Oles (SeveralNines Support engineer) Hello Brian, Thank you for your ticket. The "bind_address" seems to be correct one but you are right in the configuration file it can be called with "-" instead of "_". mysql -e "show global variables where variable_name ='bind_address'" +---------------+----------------+ | Variable_name | Value | +---------------+----------------+ | bind_address | XXX.XX.XXX.XXX | +---------------+----------------+ It could be that ClusterControl is having an issue to gather the status of the SQL query/connect to DB, I see that the same happened with wsrep parameter. Could you please run on your Galera nodes following query: /usr/bin/mysql -NAB -u"root" -pxxxx -e "show global variables where variable_name ='bind_address'" Kind regards, Bart = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = Brian Ryberg Oh, I already tried that, but forgot to put details in the ticket - It's a bit strange, but they all respond like this, empty. root@ubuntuserver-3:~# /usr/bin/mysql -NAB -u"root" -p'pa$$word123' -e "show global variables where variable_name ='bind_address'" [no rows returned] root@ubuntuserver-3:~# /usr/bin/mysql -NAB -u"root" -p'pa$$word123' -e "show global variables where variable_name like '%address%'" wsrep_cluster_address gcomm://192.168.56.101,192.168.56.102,192.168.56.103 wsrep_node_address 192.168.56.103 wsrep_node_incoming_address AUTO wsrep_sst_receive_address AUTO [just to see if something works and is named %address% And here's the content of my config(s) (same on each, apart from IPaddress ;) ) root@ubuntuserver-3:~# grep bind /etc/mysql/my.cnf bind-address = 192.168.56.103 #bind-address=0.0.0.0 root@ubuntuserver-3:~# grep bind /etc/mysql/conf.d/galera.cnf bind-address=192.168.56.103 #bind-address=0.0.0.0 ( And I'm using MariaDB 10.2.24 ) = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = Bart Oles (SeveralNines Support engineer) 1 day ago Brian, this looks like a bug. I've asked development to take a look at this. We will back to you ASAP. In the meanwhile could you please change the bind address variable from bind-address to bind_address? And let us know about the outcome? Thank you in advance, = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = Brian Ryberg Thanks for looking in to this. I am somewhat baffled now. I can't seem to find any signs of that bind[-|_]address anywhere inside mysql/mariadb... Here's details on my attempts from a random of the 6 nodes that all works fine in my Galera testcluster. All have bind-address present in config file, but refuse to let me know about it from inside "mysql"? - config files: grep: /etc/mysql/mariadb.conf.d: Is a directory /etc/mysql/my.cnf:#bind-address = 0.0.0.0 /etc/mysql/my.cnf:bind-address = 192.168.56.101 /etc/mysql/my.cnf:#bind-address=0.0.0.0 root@ubuntuserver-1:~# grep bind /etc/mysql/conf.d/* /etc/mysql/conf.d/galera.cnf:bind-address=192.168.56.101 /etc/mysql/conf.d/galera.cnf:#bind_address=192.168.56.101 /etc/mysql/conf.d/galera.cnf:#bind-address=0.0.0.0 - Any mention in logfile?: root@ubuntuserver-1:~# grep -i bind /var/log/syslog root@ubuntuserver-1:~# grep -i address /var/log/syslog root@ubuntuserver-1:~# grep -i socket /var/log/syslog Jul 11 09:04:04 ubuntuserver-1 mysqld[8772]: 2019-07-11 9:04:04 140102491384000 [Note] Server socket created on IP: '192.168.56.101'. Jul 11 09:04:04 ubuntuserver-1 mysqld[8772]: Version: '10.2.21-MariaDB-10.2.21+maria~xenial-log' socket: '/var/run/mysqld/mysqld.sock' port: 3306 mariadb.org binary distribution Jul 11 09:09:17 ubuntuserver-1 mysqld[9087]: 2019-07-11 9:09:17 139877760112832 [Note] Server socket created on IP: '192.168.56.101'. Jul 11 09:09:17 ubuntuserver-1 mysqld[9087]: Version: '10.2.21-MariaDB-10.2.21+maria~xenial-log' socket: '/var/run/mysqld/mysqld.sock' port: 3306 mariadb.org binary distribution Trying inside "mysql": MariaDB [(none)]> show global variables where variable_name ='bind_address'; Empty set (0.00 sec) MariaDB [(none)]> SHOW GLOBAL VARIABLES like 'bind_address'; Empty set (0.01 sec) MariaDB [(none)]> SHOW GLOBAL VARIABLES like 'bind-address'; Empty set (0.00 sec) - any "skip"-whatnot setting preventing us from seeing bind_adress?: MariaDB [(none)]> SHOW GLOBAL VARIABLES like '%skip%'; +----------------------------------+-----------+ | Variable_name | Value | +----------------------------------+-----------+ | replicate_events_marked_for_skip | REPLICATE | | skip_external_locking | ON | | skip_name_resolve | ON | | skip_networking | OFF | | skip_show_database | OFF | | slave_skip_errors | OFF | | sql_slave_skip_counter | 0 | +----------------------------------+-----------+ 7 rows in set (0.01 sec) MariaDB [(none)]> SHOW GLOBAL VARIABLES like '%address%'; +-----------------------------+------------------------------------------------------+ | Variable_name | Value | +-----------------------------+------------------------------------------------------+ | wsrep_cluster_address | gcomm://192.168.56.101,192.168.56.102,192.168.56.103 | | wsrep_node_address | 192.168.56.101 | | wsrep_node_incoming_address | AUTO | | wsrep_sst_receive_address | AUTO | +-----------------------------+------------------------------------------------------+ 4 rows in set (0.00 sec) I am surely missing something essential here :D ?? = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = Brian Ryberg By the way, I forgot to mention that - yes in both files the bind-address is placed in the [mysqld] section. And it is listening on right port on the right IP: root@ubuntuserver-1:~# netstat -plnt|grep 3306 tcp 0 0 192.168.56.101:3306 0.0.0.0:* LISTEN 9087/mysqld = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = Brian Ryberg I have now cleaned up, so bind-address is only mentioned in galera.cnf. I tried putting in deliberately wrong values like monkey.doesnt.play or 300.400.500.600 Both attempts failed just fine like this: Jul 11 09:55:53 ubuntuserver-1 mysqld[9803]: 2019-07-11 9:55:51 140433421596864 [ERROR] Can't create IP socket: Name or service not known Jul 11 09:55:53 ubuntuserver-1 mysqld[9803]: 2019-07-11 9:55:51 140433421596864 [ERROR] Aborting' Jul 11 09:55:53 ubuntuserver-1 systemd[1]: mariadb.service: Control process exited, code=exited status=1 Jul 11 09:55:53 ubuntuserver-1 systemd[1]: Failed to start MariaDB 10.2.21 database server. Jul 11 09:55:53 ubuntuserver-1 systemd[1]: mariadb.service: Unit entered failed state. Jul 11 09:55:53 ubuntuserver-1 systemd[1]: mariadb.service: Failed with result 'exit-code'. ^C I put back the desired IP address and it started, but still does not show any values named anything like bind-adress/bind_address or socket/ip or whatnot. root@ubuntuserver-1:~# tail -f /var/log/syslog|grep "IP" Jul 11 09:56:54 ubuntuserver-1 mysqld[10303]: 2019-07-11 9:56:54 139847153998016 [Note] Server socket created on IP: '192.168.56.101'. Am I really the first person in the universe to discover that MariaDB developers silently removed that information from under the hood :P ?? = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = Bart Oles (SeveralNines Support engineer) Yes, this is very strange because we didn't notice it in any other environment and in fact, it's base functionality that a lot of users use. Would it be possible for you to upgrade to the recent CC version and if the problem still exist create an error report and attach it to the request? = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = Brian Ryberg Yes, I could do that - but I kind of assume that this problem must lie within MariaDB rather than your CC :) I can see that wsrep_node_address and bind-address seems to have the same function. Can you see any situation, where wrep_node_address could "take over and replace" bind-address. I know it's a silly question, but I'm just grasping at straws here :) If such a thing *could* happen, I would have seen it when first trying to import *any* Galera cluster into CC. And I suspect that you would somehow have stumbled over missing bind_address during development :D Regards Brian = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =