-yes maxscale -stop mcs1 /wait 120 / MCS primary goes to mcs2 -start mcs1 /wait 120/ mcs primary stays on mcs2 All replication is set correctly starting 2022-12-06 17:21:27.445844 test init 2022-12-06 17:21:27.450635 test :::::::::::::: __init__ 2022-12-06 17:21:27.450696 init uos 2022-12-06 17:21:27.450717 rm -r /root/tmp/docker_logs_dbms 2022-12-06 17:21:27.456611 mkdir /root/tmp/docker_logs_dbms 2022-12-06 17:21:27.459262 test :::::::::::::: t1_failover_on_pm1 2022-12-06 17:21:27.459356 connecting 0 localhost 3307 pgmabv99 C0lumnStore! 2022-12-06 17:21:29.553732 Connected ok ihost= 0 2022-12-06 17:21:29.553908 i_host 0 : show status like 'columnstore%'; 2022-12-06 17:21:29.555079 {'Variable_name': 'Columnstore_commit_hash', 'Value': 'a1d89d8-dirty'} 2022-12-06 17:21:29.555212 {'Variable_name': 'Columnstore_version', 'Value': '22.08.4'} 2022-12-06 17:21:29.555306 i_host 0 : show variables where Variable_name like '%hostname%' 2022-12-06 17:21:29.556693 {'Variable_name': 'hostname', 'Value': 'mcs1'} 2022-12-06 17:21:29.556840 i_host 0 : use d1 2022-12-06 17:21:29.557348 ex_sql (1049, "Unknown database 'd1'") 2022-12-06 17:21:29.557439 i_host 0 : drop database if exists d1 2022-12-06 17:21:29.557892 i_host 0 : create database d1 2022-12-06 17:21:29.558487 i_host 0 : use d1 2022-12-06 17:21:29.558894 i_host 0 : create table t1 (f1 integer) engine=columnstore 2022-12-06 17:21:29.908047 docker exec mcs1 bash -c "mcs cluster status | grep 'mcs\|mode'" 2022-12-06 17:21:30.542073 "mcs1": { 2022-12-06 17:21:30.542321 "dbrm_mode": "master", 2022-12-06 17:21:30.542513 "cluster_mode": "readwrite", 2022-12-06 17:21:30.542631 "mcs2": { 2022-12-06 17:21:30.542730 "dbrm_mode": "slave", 2022-12-06 17:21:30.542846 "cluster_mode": "readonly", 2022-12-06 17:21:30.542931 "mcs3": { 2022-12-06 17:21:30.543141 "dbrm_mode": "slave", 2022-12-06 17:21:30.543249 "cluster_mode": "readonly", 2022-12-06 17:21:30.545891 docker exec mx1 bash -c "maxctrl list servers" 2022-12-06 17:21:30.825619 ┌────────┬─────────┬──────┬─────────────┬─────────────────┬────────┬─────────────┐ 2022-12-06 17:21:30.825781 │ Server │ Address │ Port │ Connections │ State │ GTID │ Monitor │ 2022-12-06 17:21:30.825875 ├────────┼─────────┼──────┼─────────────┼─────────────────┼────────┼─────────────┤ 2022-12-06 17:21:30.825935 │ mcs1 │ mcs1 │ 3306 │ 0 │ Master, Running │ 0-2-21 │ mcs_monitor │ 2022-12-06 17:21:30.826026 ├────────┼─────────┼──────┼─────────────┼─────────────────┼────────┼─────────────┤ 2022-12-06 17:21:30.826045 │ mcs2 │ mcs2 │ 3306 │ 0 │ Slave, Running │ 0-2-21 │ mcs_monitor │ 2022-12-06 17:21:30.826146 ├────────┼─────────┼──────┼─────────────┼─────────────────┼────────┼─────────────┤ 2022-12-06 17:21:30.826240 │ mcs3 │ mcs3 │ 3306 │ 0 │ Slave, Running │ 0-2-21 │ mcs_monitor │ 2022-12-06 17:21:30.826261 └────────┴─────────┴──────┴─────────────┴─────────────────┴────────┴─────────────┘ 2022-12-06 17:21:30.834045 i_host 0 : insert into t1 (f1) values ( 1 ) 2022-12-06 17:21:30.979654 i_host 0 : select count(*) from t1 2022-12-06 17:21:31.020872 {'count(*)': 1} 2022-12-06 17:21:31.021213 docker stop mcs1 2022-12-06 17:21:32.477648 mcs1 2022-12-06 17:21:32.478841 docker ps 2022-12-06 17:21:32.498146 CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 2022-12-06 17:21:32.498357 08cbbbb0952c mariadb/maxscale "/usr/bin/tini -- do…" 2 minutes ago Up 2 minutes 0.0.0.0:3311->3306/tcp, :::3311->3306/tcp, 0.0.0.0:8990->8989/tcp, :::8990->8989/tcp mx2 2022-12-06 17:21:32.498481 7fb9ee9a1671 mariadb/maxscale "/usr/bin/tini -- do…" 2 minutes ago Up 2 minutes 0.0.0.0:8989->8989/tcp, :::8989->8989/tcp, 0.0.0.0:3310->3306/tcp, :::3310->3306/tcp mx1 2022-12-06 17:21:32.498594 fa2d73ebdcf0 mylocal "/usr/bin/tini -- do…" 2 minutes ago Up 2 minutes 0.0.0.0:3308->3306/tcp, :::3308->3306/tcp mcs2 2022-12-06 17:21:32.498734 ce6b907a9f01 mylocal "/usr/bin/tini -- do…" 2 minutes ago Up 2 minutes 0.0.0.0:3309->3306/tcp, :::3309->3306/tcp mcs3 2022-12-06 17:21:32.499746 sleep after container stop 120 2022-12-06 17:23:32.560904 docker exec mcs2 bash -c "mcs cluster status | grep 'mcs\|mode'" 2022-12-06 17:23:33.264434 "mcs2": { 2022-12-06 17:23:33.264657 "dbrm_mode": "master", 2022-12-06 17:23:33.264789 "cluster_mode": "readwrite", 2022-12-06 17:23:33.264871 "mcs3": { 2022-12-06 17:23:33.264925 "dbrm_mode": "slave", 2022-12-06 17:23:33.265056 "cluster_mode": "readonly", 2022-12-06 17:23:33.267531 docker exec mx1 bash -c "maxctrl list servers" 2022-12-06 17:23:33.536177 ┌────────┬─────────┬──────┬─────────────┬─────────────────┬────────┬─────────────┐ 2022-12-06 17:23:33.536377 │ Server │ Address │ Port │ Connections │ State │ GTID │ Monitor │ 2022-12-06 17:23:33.536447 ├────────┼─────────┼──────┼─────────────┼─────────────────┼────────┼─────────────┤ 2022-12-06 17:23:33.536530 │ mcs1 │ mcs1 │ 3306 │ 0 │ Down │ 0-2-21 │ mcs_monitor │ 2022-12-06 17:23:33.536677 ├────────┼─────────┼──────┼─────────────┼─────────────────┼────────┼─────────────┤ 2022-12-06 17:23:33.536785 │ mcs2 │ mcs2 │ 3306 │ 0 │ Master, Running │ 0-2-24 │ mcs_monitor │ 2022-12-06 17:23:33.536894 ├────────┼─────────┼──────┼─────────────┼─────────────────┼────────┼─────────────┤ 2022-12-06 17:23:33.536983 │ mcs3 │ mcs3 │ 3306 │ 0 │ Slave, Running │ 0-2-24 │ mcs_monitor │ 2022-12-06 17:23:33.537055 └────────┴─────────┴──────┴─────────────┴─────────────────┴────────┴─────────────┘ 2022-12-06 17:23:33.544953 connecting 1 localhost 3308 pgmabv99 C0lumnStore! 2022-12-06 17:23:35.728444 Connected ok ihost= 1 2022-12-06 17:23:35.728561 i_host 1 : show status like 'columnstore%'; 2022-12-06 17:23:35.729347 {'Variable_name': 'Columnstore_commit_hash', 'Value': 'a1d89d8-dirty'} 2022-12-06 17:23:35.729490 {'Variable_name': 'Columnstore_version', 'Value': '22.08.4'} 2022-12-06 17:23:35.729571 i_host 1 : show variables where Variable_name like '%hostname%' 2022-12-06 17:23:35.730419 {'Variable_name': 'hostname', 'Value': 'mcs2'} 2022-12-06 17:23:35.730511 i_host 1 : use d1 2022-12-06 17:23:35.730868 sleeping 3 2022-12-06 17:23:38.734207 i_host 1 : select count(*) from t1 2022-12-06 17:23:38.836984 {'count(*)': 1} 2022-12-06 17:23:38.837113 SUCCESS !!!!!!!!!!! after 1 attempts, time 3 2022-12-06 17:23:38.837184 sleeping 3 2022-12-06 17:23:41.840614 i_host 1 : drop table if exists t2 2022-12-06 17:23:41.915088 SUCCESS !!!!!!!!!!! after 1 attempts, time 3 2022-12-06 17:23:41.915191 i_host 1 : create table t2 ( f1 int) engine=columnstore 2022-12-06 17:23:42.195353 i_host 1 : insert into t2 (f1) values ( 1 ) 2022-12-06 17:23:42.334461 i_host 1 : select count(*) from t2 2022-12-06 17:23:42.376623 {'count(*)': 1} 2022-12-06 17:23:42.376855 docker start mcs1 2022-12-06 17:23:42.728420 mcs1 2022-12-06 17:23:42.729635 docker ps 2022-12-06 17:23:42.751193 CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 2022-12-06 17:23:42.751372 08cbbbb0952c mariadb/maxscale "/usr/bin/tini -- do…" 4 minutes ago Up 4 minutes 0.0.0.0:3311->3306/tcp, :::3311->3306/tcp, 0.0.0.0:8990->8989/tcp, :::8990->8989/tcp mx2 2022-12-06 17:23:42.751485 7fb9ee9a1671 mariadb/maxscale "/usr/bin/tini -- do…" 4 minutes ago Up 4 minutes 0.0.0.0:8989->8989/tcp, :::8989->8989/tcp, 0.0.0.0:3310->3306/tcp, :::3310->3306/tcp mx1 2022-12-06 17:23:42.751590 864d8092563a mylocal "/usr/bin/tini -- do…" 4 minutes ago Up Less than a second 0.0.0.0:3307->3306/tcp, :::3307->3306/tcp mcs1 2022-12-06 17:23:42.751674 fa2d73ebdcf0 mylocal "/usr/bin/tini -- do…" 4 minutes ago Up 4 minutes 0.0.0.0:3308->3306/tcp, :::3308->3306/tcp mcs2 2022-12-06 17:23:42.751759 ce6b907a9f01 mylocal "/usr/bin/tini -- do…" 4 minutes ago Up 4 minutes 0.0.0.0:3309->3306/tcp, :::3309->3306/tcp mcs3 2022-12-06 17:23:42.753450 sleep after container start 120 2022-12-06 17:25:42.838357 docker exec mcs2 bash -c "mcs cluster status | grep 'mcs\|mode'" 2022-12-06 17:25:48.629267 "mcs2": { 2022-12-06 17:25:48.629416 "dbrm_mode": "master", 2022-12-06 17:25:48.629487 "cluster_mode": "readwrite", 2022-12-06 17:25:48.629556 "mcs3": { 2022-12-06 17:25:48.629631 "dbrm_mode": "slave", 2022-12-06 17:25:48.629715 "cluster_mode": "readonly", 2022-12-06 17:25:48.629787 "mcs1": { 2022-12-06 17:25:48.629846 "dbrm_mode": "slave", 2022-12-06 17:25:48.629926 "cluster_mode": "readonly", 2022-12-06 17:25:48.631934 docker exec mx1 bash -c "maxctrl list servers" 2022-12-06 17:25:48.887541 ┌────────┬─────────┬──────┬─────────────┬─────────────────┬────────┬─────────────┐ 2022-12-06 17:25:48.887746 │ Server │ Address │ Port │ Connections │ State │ GTID │ Monitor │ 2022-12-06 17:25:48.887858 ├────────┼─────────┼──────┼─────────────┼─────────────────┼────────┼─────────────┤ 2022-12-06 17:25:48.887945 │ mcs1 │ mcs1 │ 3306 │ 0 │ Slave, Running │ 0-3-26 │ mcs_monitor │ 2022-12-06 17:25:48.888021 ├────────┼─────────┼──────┼─────────────┼─────────────────┼────────┼─────────────┤ 2022-12-06 17:25:48.888166 │ mcs2 │ mcs2 │ 3306 │ 0 │ Master, Running │ 0-3-26 │ mcs_monitor │ 2022-12-06 17:25:48.888242 ├────────┼─────────┼──────┼─────────────┼─────────────────┼────────┼─────────────┤ 2022-12-06 17:25:48.888318 │ mcs3 │ mcs3 │ 3306 │ 0 │ Slave, Running │ 0-3-26 │ mcs_monitor │ 2022-12-06 17:25:48.888426 └────────┴─────────┴──────┴─────────────┴─────────────────┴────────┴─────────────┘ 2022-12-06 17:25:48.895974 connecting 0 localhost 3307 pgmabv99 C0lumnStore! 2022-12-06 17:25:51.024184 Connected ok ihost= 0 2022-12-06 17:25:51.024540 i_host 0 : show status like 'columnstore%'; 2022-12-06 17:25:51.027325 {'Variable_name': 'Columnstore_commit_hash', 'Value': 'a1d89d8-dirty'} 2022-12-06 17:25:51.027681 {'Variable_name': 'Columnstore_version', 'Value': '22.08.4'} 2022-12-06 17:25:51.028009 i_host 0 : show variables where Variable_name like '%hostname%' 2022-12-06 17:25:51.032048 {'Variable_name': 'hostname', 'Value': 'mcs1'} 2022-12-06 17:25:51.032425 i_host 0 : use d1 2022-12-06 17:25:51.034161 i_host 0 : select count(*) from t1 2022-12-06 17:25:51.119578 {'count(*)': 1} 2022-12-06 17:25:51.119713 i_host 0 : select count(*) from t2 2022-12-06 17:25:52.173752 {'count(*)': 1} 2022-12-06 17:25:52.174060 sleeping 3 2022-12-06 17:25:55.177456 i_host 0 : drop table if exists t3 2022-12-06 17:25:55.260866 SUCCESS !!!!!!!!!!! after 1 attempts, time 3 2022-12-06 17:25:55.260961 i_host 0 : create table t3( f1 int) engine=columnstore 2022-12-06 17:25:55.540591 i_host 0 : insert into t3 (f1) values ( 1 ) 2022-12-06 17:25:55.672922 i_host 0 : select count(*) from t3 2022-12-06 17:25:55.709505 {'count(*)': 1} 2022-12-06 17:25:55.709693 saving logs ++++++++++++++++++++++++++++++++ 2022-12-06 17:25:55.709781 docker exec mcs1 bash -c " ps -aux > /var/log/mariadb/columnstore/zzps_list.log " 2022-12-06 17:25:55.816639 docker exec mcs1 bash -c " dnf list installed > /var/log/mariadb/columnstore/zzpackage_list.log " 2022-12-06 17:25:56.257265 docker exec mcs2 bash -c " ps -aux > /var/log/mariadb/columnstore/zzps_list.log " 2022-12-06 17:25:56.386581 docker exec mcs2 bash -c " dnf list installed > /var/log/mariadb/columnstore/zzpackage_list.log " 2022-12-06 17:25:56.829029 docker exec mcs3 bash -c " ps -aux > /var/log/mariadb/columnstore/zzps_list.log " 2022-12-06 17:25:56.946736 docker exec mcs3 bash -c " dnf list installed > /var/log/mariadb/columnstore/zzpackage_list.log " 2022-12-06 17:25:57.388543 copy from all servers /var/log/mariadb/columnstore 2022-12-06 17:25:57.544369 copy from all servers /etc/columnstore 2022-12-06 17:25:57.695351 copy from all servers /etc/my.cnf.d 2022-12-06 17:25:57.837672 tar -cf /root/tmp/docker-logs-dbms.tar /root/tmp/docker_logs_dbms