thread slave who stay blocked

ALTER TABLE transactionSteps ADD INDEX (`idAccount`, `date`, `idStep`);

MariaDB [(none)]> show processlist;

+---------+-----------------+-------------------+-----------------+-----------+---------+--------------------------------------------------------+------------------------------------------------------------------------------------------------------+----------+

| Id      | User            | Host              | db              | Command   | Time    | State                                                  | Info                                                                                                 | Progress |

+---------+-----------------+-------------------+-----------------+-----------+---------+--------------------------------------------------------+------------------------------------------------------------------------------------------------------+----------+

|       7 | system user     |                   | NULL            | Slave_IO  | 2067845 | Waiting for master to send event                       | NULL                                                                                                 |    0.000 |

|       8 | system user     |                   | pc_aggregation  | Slave_SQL |   54244 | Waiting for table metadata lock                        | ALTER TABLE transactionSteps ADD INDEX (`idAccount`, `date`, `idStep`)                               |    0.000 |

|    1255 | system user     |                   | NULL            | Slave_IO  | 2066489 | Waiting for master to send event                       | NULL                                                                                                 |    0.000 |

|    1256 | system user     |                   | NULL            | Slave_SQL |       0 | Slave has read all relay log; waiting for more updates | NULL                                                                                                 |    0.000 |

MariaDB [(none)]> SELECT  CONCAT('Thread ',P.ID,' executing "',P.INFO,'" IS LOCKED BY Thread ', M.THREAD_ID) WhoLocksWho  FROM INFORMATION_SCHEMA.PROCESSLIST P, INFORMATION_SCHEMA.METADATA_LOCK_INFO M  WHERE LOCATE(lcase(LOCK_TYPE), lcase(STATE))>0;

+-------------------------------------------------------------------------------------------------------------------------+

| WhoLocksWho                                                                                                             |

+-------------------------------------------------------------------------------------------------------------------------+

| Thread 8 executing "ALTER TABLE transactionSteps ADD INDEX (`idAccount`, `date`, `idStep`)" IS LOCKED BY Thread 5259516 |

....

| Thread 8 executing "ALTER TABLE transactionSteps ADD INDEX (`idAccount`, `date`, `idStep`)" IS LOCKED BY Thread 5291866 |

| Thread 8 executing "ALTER TABLE transactionSteps ADD INDEX (`idAccount`, `date`, `idStep`)" IS LOCKED BY Thread 5264199 |

| Thread 8 executing "ALTER TABLE transactionSteps ADD INDEX (`idAccount`, `date`, `idStep`)" IS LOCKED BY Thread 5264205 |

| Thread 8 executing "ALTER TABLE transactionSteps ADD INDEX (`idAccount`, `date`, `idStep`)" IS LOCKED BY Thread 8       |

| Thread 8 executing "ALTER TABLE transactionSteps ADD INDEX (`idAccount`, `date`, `idStep`)" IS LOCKED BY Thread 5280878 |

....

+-------------------------------------------------------------------------------------------------------------------------+

68 rows in set (0.002 sec)

STOP SLAVE;

KILL 8;

SET GLOBAL sql_skip_counter=1;

START SLAVE;

-- and start to working fine again.

# MariaDB database server configuration file.

#

# You can copy this file to one of:

# - "/etc/mysql/my.cnf" to set global options,

# - "~/.my.cnf" to set user-specific options.

# 

# One can use all long options that the program supports.

# Run program with --help to get a list of available options and with

# --print-defaults to see which it would actually understand and use.

#

# For explanations see

# http://dev.mysql.com/doc/mysql/en/server-system-variables.html

# This will be passed to all mysql clients

# It has been reported that passwords should be enclosed with ticks/quotes

# escpecially if they contain "#" chars...

# Remember to edit /etc/mysql/debian.cnf when changing the socket location.

[client]

port            = 3306

socket          = /var/run/mysqld/mysqld.sock

# Here is entries for some specific programs

# The following values assume you have at least 32M ram

# This was formally known as [safe_mysqld]. Both versions are currently parsed.

[mysqld_safe]

socket          = /var/run/mysqld/mysqld.sock

nice            = 0

[mysqld]

#

# * Basic Settings

#

g04.replicate_do_db=pc_alias

g04.replicate_do_db=pc_aliases

g04.replicate_do_db=pc_backoffice

g04.replicate_do_db=pc_batch

g04.replicate_do_db=pc_common

g04.replicate_do_db=pc_data_factory

g04.replicate_do_db=pc_dwhNatixis

g04.replicate_do_db=pc_exchange_rate

g04.replicate_do_db=pc_fraudrate

g04.replicate_do_db=pc_fraudRules

g04.replicate_do_db=pc_invoices

g04.replicate_do_db=pc_marketplaceOnboarding

g04.replicate_do_db=pc_merchOp

g04.replicate_do_db=pc_natixisFlows

innodb_autoextend_increment = 1000

innodb_strict_mode=0

sql_mode=NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION

performance_schema=ON

connect_timeout=10

innodb_rollback_on_timeout=1

wait_timeout=18000

character-set-server  = utf8mb4 

collation-server      = utf8mb4_general_ci 

character_set_server   = utf8mb4

collation-server = utf8mb4_general_ci

#innodb_force_recovery = 1

user            = mysql

pid-file        = /var/run/mysqld/mysqld.pid

socket          = /var/run/mysqld/mysqld.sock

port            = 3306

basedir         = /usr

datadir         = /srv/mysql/data

tmpdir          = /srv/mysql/tmp

lc_messages_dir = /usr/share/mysql

lc_messages     = en_US

plugin_dir = /usr/lib/mysql/plugin/

skip-name-resolve=1

plugin-load=server_audit=server_audit.so

server_audit_logging=1

# do not allow users to uninstall plugin

server_audit=FORCE_PLUS_PERMANENT

# only audit connections and DDL queries

server_audit_events=CONNECT,QUERY_DDL

# flat file

server_audit_output_type=FILE

server_audit_file_path=/srv/mysql/log/audit.log

server_audit_file_rotate_size=1000000

server_audit_file_rotations=9

#logs

log_error=/srv/mysql/log/error.log

#

# Instead of skip-networking the default is now to listen only on

# localhost which is more compatible and is not less secure.

# bind-address           = 127.0.0.1

#

# * Fine Tuning

#

max_connections         = 1000

connect_timeout         = 10

max_allowed_packet      = 256M

thread_cache_size       = 128

sort_buffer_size        = 4M

bulk_insert_buffer_size = 16M

tmp_table_size          = 256M

max_heap_table_size     = 256M

#

# * MyISAM

#

# This replaces the startup script and checks MyISAM tables if needed

# the first time they are touched. On error, make copy and try a repair.

myisam_recover_options = BACKUP

key_buffer_size         = 128M

open-files-limit       = 2000

table_open_cache        = 400

myisam_sort_buffer_size = 512M

concurrent_insert       = 2

read_buffer_size        = 2M

read_rnd_buffer_size    = 1M

key_cache_segments      = 64

#mroonga.replicate_rewrite_db="repl->repl2"

#mroonga.replicate_do_table="repl2.article2"

#

# * Query Cache Configuration

#

# Cache only tiny result sets, so we can fit more in the query cache.

query_cache_limit               = 128K

query_cache_size                = 0

# for more write intensive setups, set to DEMAND or OFF

query_cache_type                = OFF

#

# * Logging and Replication

#

# Both location gets rotated by the cronjob.

# Be aware that this log type is a performance killer.

# As of 5.1 you can enable the log at runtime!

general_log_file        = /srv/mysql/log/general.log

#general_log             = 1

#

# Error logging goes to syslog due to /etc/mysql/conf.d/mysqld_safe_syslog.cnf.

#

# we do want to know about network errors and such

log_warnings            = 2

#

# Enable the slow query log to see queries with especially long duration

#slow_query_log[={0|1}]

slow_query_log_file     = /srv/mysql/mariadb-slow.log

long_query_time = 1

#log_slow_rate_limit    = 1000

log_slow_verbosity      = query_plan

log_slave_updates       = 1

#log-queries-not-using-indexes

#log_slow_admin_statements

#

# The following can be used as easy to replay backup logs or for replication.

# note: if you are setting up a replication slave, see README.Debian about

#       other settings you may need to change.

server_id               = 4043602971

report_host            = gcp-prod-oos-sql-0001-mariadb-data-001 

#auto_increment_increment = 2

#auto_increment_offset  = 1

log_bin                        = /srv/mysql/binlog/mariadb-bin

log_bin_index          = /srv/mysql/binlog/mariadb-bin.index

# not fab for performance, but safer

#sync_binlog            = 1

expire_logs_days        = 10

max_binlog_size         = 1G

# slaves

relay_log              = /srv/mysql/relaylog/relay-bin

relay_log_index        = /srv/mysql/relaylog/relay-bin.index

relay_log_info_file   = /srv/mysql/relaylog/relay-bin.info

log_slave_updates=1

read_only=1

#

# If applications support it, this stricter sql_mode prevents some

# mistakes like inserting invalid dates etc.

# sql_mode               = NO_ENGINE_SUBSTITUTION,TRADITIONAL

#

# * InnoDB

#

# InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/.

# Read the manual for more InnoDB related options. There are many!

default_storage_engine  = InnoDB

# you can't just change log file size, requires special procedure

innodb_log_file_size    = 2G

innodb_buffer_pool_size = 40G

#innodb_buffer_pool_instances=8 ## removed for 10.7

innodb_log_buffer_size  = 8M

innodb_file_per_table   = 1

innodb_open_files       = 400

innodb_io_capacity      = 2000

innodb_flush_method     = O_DIRECT

#

# * Security Features

#

# Read the manual, too, if you want chroot!

# chroot = /var/lib/mysql/

#

# For generating SSL certificates I recommend the OpenSSL GUI "tinyca".

#

# ssl-ca=/etc/mysql/cacert.pem

# ssl-cert=/etc/mysql/server-cert.pem

# ssl-key=/etc/mysql/server-key.pem

#

# * Galera-related settings

#event-scheduler = ON

#

[galera]

# Mandatory settings

wsrep_on=OFF

wsrep_cluster_name='*******'

wsrep_provider=/usr/lib/galera/libgalera_smm.so

wsrep_cluster_address=gcomm://

# The MariaDB configuration file

#

# The MariaDB/MySQL tools read configuration files in the following order:

# 0. "/etc/mysql/my.cnf" symlinks to this file, reason why all the rest is read.

# 1. "/etc/mysql/mariadb.cnf" (this file) to set global defaults,

# 2. "/etc/mysql/conf.d/*.cnf" to set global options.

# 3. "/etc/mysql/mariadb.conf.d/*.cnf" to set MariaDB-only options.

# 4. "~/.my.cnf" to set user-specific options.

#

# If the same option is defined multiple times, the last one will apply.

#

# One can use all long options that the program supports.

# Run program with --help to get a list of available options and with

# --print-defaults to see which it would actually understand and use.

#

# If you are new to MariaDB, check out https://mariadb.com/kb/en/basic-mariadb-articles/

#

# This group is read both by the client and the server

# use it for options that affect everything

#

# Port or socket location where to connect

# port = 3306

socket = /run/mysqld/mysqld.sock

# Import all .cnf files from configuration directory

#wsrep_node_address=10.122.5.201

#wsrep_node_name=gcp-prod-oos-sql-0001-mariadb-data-001

#wsrep_gtid_mode=ON

wsrep_sst_method = xtrabackup-v2

wsrep_sst_auth = 'sst:******'

wsrep_provider_options="gcache.size = 20G"

wsrep_max_ws_rows = 500000

binlog_format=row

default_storage_engine=InnoDB

innodb_autoinc_lock_mode=2

#

# Allow server to accept connections on all interfaces.

#

bind-address=0.0.0.0

#

# Optional setting

wsrep_slave_threads=4

innodb_flush_log_at_trx_commit=2

# DBUG options for wsrep provider

#wsrep_dbug_option

# Generate fake primary keys for non-PK tables (required for multi-master

# and parallel applying operation)

wsrep_certify_nonPK=1

# Location of the directory with data files. Needed for non-mysqldump

# state snapshot transfers. Defaults to mysql_real_data_home.

#wsrep_data_home_dir=

# Maximum number of rows in write set

wsrep_max_ws_rows=131072

# Maximum size of write set

wsrep_max_ws_size=1073741824

# to enable debug level logging, set this to 1

wsrep_debug=0

# convert locking sessions into transactions

wsrep_convert_LOCK_to_trx=0

# how many times to retry deadlocked autocommits

wsrep_retry_autocommit=1

# change auto_increment_increment and auto_increment_offset automatically

wsrep_auto_increment_control=1

# replicate myisam

## wsrep_replicate_myisam=1 #removed in 10.7

# retry autoinc insert, which failed for duplicate key error

wsrep_drupal_282555_workaround=0

# enable "strictly synchronous" semantics for read operations

wsrep_causal_reads=0

# Protocol version to use

# wsrep_protocol_version=

# log conflicts

wsrep_log_conflicts=1

[xtrabackup]

user=sst

password=QSEDWGRg133

databases-exclude=lost+found

[mysqldump]

quick

quote-names

max_allowed_packet      = 256M

[mysql]

#no-auto-rehash # faster start of mysql but no tab completion

[isamchk]

key_buffer              = 16M

!includedir /etc/mysql/conf.d/

               Connection_name: 

               Slave_SQL_State: Waiting for table metadata lock

                Slave_IO_State: Waiting for master to send event

                   Master_Host: 10.122.5.14

                   Master_User: replication_slave

                   Master_Port: 3306

                 Connect_Retry: 60

               Master_Log_File: mariadb-bin.001275

           Read_Master_Log_Pos: 628660033

                Relay_Log_File: relay-bin.002031

                 Relay_Log_Pos: 122108089

         Relay_Master_Log_File: mariadb-bin.001235

              Slave_IO_Running: Yes

             Slave_SQL_Running: Yes

               Replicate_Do_DB: 

           Replicate_Ignore_DB: 

            Replicate_Do_Table: 

        Replicate_Ignore_Table: 

       Replicate_Wild_Do_Table: 

   Replicate_Wild_Ignore_Table: 

                    Last_Errno: 0

                    Last_Error: 

                  Skip_Counter: 0

           Exec_Master_Log_Pos: 122107788

               Relay_Log_Space: 43578835072

               Until_Condition: None

                Until_Log_File: 

                 Until_Log_Pos: 0

            Master_SSL_Allowed: No

            Master_SSL_CA_File: 

            Master_SSL_CA_Path: 

               Master_SSL_Cert: 

             Master_SSL_Cipher: 

                Master_SSL_Key: 

         Seconds_Behind_Master: 55861

 Master_SSL_Verify_Server_Cert: No

                 Last_IO_Errno: 0

                 Last_IO_Error: 

                Last_SQL_Errno: 0

                Last_SQL_Error: 

   Replicate_Ignore_Server_Ids: 

              Master_Server_Id: 4255000500

                Master_SSL_Crl: 

            Master_SSL_Crlpath: 

                    Using_Gtid: No

                   Gtid_IO_Pos: 

       Replicate_Do_Domain_Ids: 

   Replicate_Ignore_Domain_Ids: 

                 Parallel_Mode: optimistic

                     SQL_Delay: 0

           SQL_Remaining_Delay: NULL

       Slave_SQL_Running_State: Waiting for table metadata lock

              Slave_DDL_Groups: 161

Slave_Non_Transactional_Groups: 1

    Slave_Transactional_Groups: 223787868

          Retried_transactions: 0

            Max_relay_log_size: 1073741824

          Executed_log_entries: 1693350495

     Slave_received_heartbeats: 191

        Slave_heartbeat_period: 30.000

                Gtid_Slave_Pos: 0-4216710495-16538644,21-21-44860636,22-22-1942462,31-31-133099508,111-111-8021169341,161-161-2322111997,162-162-2198746