Server is crashing in the replication io thread

[root@ALEPO-DB1-ILB coredumps]# gdb /usr/sbin/mariadbd -c core.mariadbd.3904053

GNU gdb (GDB) Red Hat Enterprise Linux 8.2-19.el8

Copyright (C) 2018 Free Software Foundation, Inc.

License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>

This is free software: you are free to change and redistribute it.

There is NO WARRANTY, to the extent permitted by law.

Type "show copying" and "show warranty" for details.

This GDB was configured as "x86_64-redhat-linux-gnu".

Type "show configuration" for configuration details.

For bug reporting instructions, please see:

<http://www.gnu.org/software/gdb/bugs/>.

Find the GDB manual and other documentation resources online at:

    <http://www.gnu.org/software/gdb/documentation/>.

For help, type "help".

Type "apropos word" to search for commands related to "word"...

Reading symbols from /usr/sbin/mariadbd...Reading symbols from /usr/lib/debug/usr/sbin/mariadbd-10.11.5-1.el8.x86_64.debug...done.

done.

warning: Can't open file (null) during file-backed mapping note processing

[New LWP 904487]

[New LWP 3904059]

[New LWP 3904279]

[New LWP 3904281]

[New LWP 3904297]

[New LWP 3904302]

[New LWP 3904476]

[New LWP 3904289]

[New LWP 3904275]

[New LWP 3904284]

[New LWP 3904280]

[New LWP 3904061]

[New LWP 3904060]

[New LWP 3904276]

[New LWP 3904055]

[New LWP 3904292]

[New LWP 904490]

[New LWP 3904283]

[New LWP 3905019]

[New LWP 3904290]

[New LWP 3904288]

[New LWP 904597]

[New LWP 3904285]

[New LWP 3904277]

[New LWP 3904282]

[New LWP 3904291]

[New LWP 904595]

[New LWP 3904295]

[New LWP 3904477]

[New LWP 887686]

[New LWP 3904286]

[New LWP 904518]

[New LWP 3917479]

[New LWP 214274]

[New LWP 3918505]

[New LWP 3904287]

[New LWP 3904293]

[New LWP 3904475]

[New LWP 904519]

[New LWP 904596]

[New LWP 214273]

[New LWP 3904053]

[Thread debugging using libthread_db enabled]

Using host libthread_db library "/lib64/libthread_db.so.1".

Core was generated by `/usr/sbin/mariadbd'.

Program terminated with signal SIGSEGV, Segmentation fault.

#0  0x00007f5edb451a35 in pthread_kill () from /lib64/libpthread.so.0

[Current thread is 1 (Thread 0x7f5ec420c700 (LWP 904487))]

Missing separate debuginfos, use: yum debuginfo-install glibc-2.28-211.el8.x86_64 libaio-0.3.112-1.el8.x86_64 libblkid-2.32.1-35.el8.x86_64 libcap-2.48-2.el8.x86_64 libgcc-8.5.0-16.el8_7.x86_64 libgcrypt-1.8.5-7.el8_6.x86_64 libgpg-error-1.31-1.el8.x86_64 libmount-2.32.1-35.el8.x86_64 libpmem-1.6.1-1.el8.x86_64 libselinux-2.9-5.el8.x86_64 libstdc++-8.5.0-16.el8_7.x86_64 libuuid-2.32.1-35.el8.x86_64 libxcrypt-4.1.1-6.el8.x86_64 lz4-libs-1.8.3-3.el8_4.x86_64 openssl-libs-1.1.1k-9.el8_7.x86_64 pcre2-10.32-3.el8_6.x86_64 sssd-client-2.6.2-3.el8.x86_64 systemd-libs-239-74.el8_8.x86_64 xz-libs-5.2.4-4.el8_6.x86_64 zlib-1.2.11-21.el8_7.x86_64

(gdb) bt

#0  0x00007f5edb451a35 in pthread_kill () from /lib64/libpthread.so.0

#1  0x000055e00f157f27 in my_write_core (sig=<optimized out>) at /usr/src/debug/MariaDB-/src_0/mysys/stacktrace.c:424

#2  0x000055e00ec515d0 in handle_fatal_signal (sig=11) at /usr/src/debug/MariaDB-/src_0/sql/signal_handler.cc:360

#3  <signal handler called>

#4  0x00007f5eda8311c9 in __memmove_avx_unaligned_erms () from /lib64/libc.so.6

#5  0x000055e00f13c5ec in memcpy (__len=782, __src=0x7f0f8c010a79, __dest=<optimized out>)

    at /usr/include/bits/string_fortified.h:34

#6  my_b_append (info=info@entry=0x55e01269f228, Buffer=0x7f0f8c010a79 "\270\305\377e\030\002", Count=Count@entry=782)

    at /usr/src/debug/MariaDB-/src_0/mysys/mf_iocache.c:1606

#7  0x000055e00ed53dec in MYSQL_BIN_LOG::write_event_buffer (this=0x55e01269ef08, buf=<optimized out>, len=782)

    at /usr/src/debug/MariaDB-/src_0/sql/log.cc:5722

#8  0x000055e00e960bd0 in queue_event(Master_info*, unsigned char const*, unsigned long) ()

    at /usr/src/debug/MariaDB-/src_0/sql/slave.cc:7277

#9  0x000055e00e962f30 in handle_slave_io () at /usr/src/debug/MariaDB-/src_0/sql/slave.cc:5049

#10 0x000055e00ee58c9d in pfs_spawn_thread (arg=0x7f0e70029c48)

    at /usr/src/debug/MariaDB-/src_0/storage/perfschema/pfs.cc:2201

#11 0x00007f5edb44a1ca in start_thread () from /lib64/libpthread.so.0

#12 0x00007f5eda79ae73 in clone () from /lib64/libc.so.6

(gdb) where

#0  0x00007f5edb451a35 in pthread_kill () from /lib64/libpthread.so.0

#1  0x000055e00f157f27 in my_write_core (sig=<optimized out>) at /usr/src/debug/MariaDB-/src_0/mysys/stacktrace.c:424

#2  0x000055e00ec515d0 in handle_fatal_signal (sig=11) at /usr/src/debug/MariaDB-/src_0/sql/signal_handler.cc:360

#3  <signal handler called>

#4  0x00007f5eda8311c9 in __memmove_avx_unaligned_erms () from /lib64/libc.so.6

#5  0x000055e00f13c5ec in memcpy (__len=782, __src=0x7f0f8c010a79, __dest=<optimized out>)

    at /usr/include/bits/string_fortified.h:34

#6  my_b_append (info=info@entry=0x55e01269f228, Buffer=0x7f0f8c010a79 "\270\305\377e\030\002", Count=Count@entry=782)

    at /usr/src/debug/MariaDB-/src_0/mysys/mf_iocache.c:1606

#7  0x000055e00ed53dec in MYSQL_BIN_LOG::write_event_buffer (this=0x55e01269ef08, buf=<optimized out>, len=782)

    at /usr/src/debug/MariaDB-/src_0/sql/log.cc:5722

#8  0x000055e00e960bd0 in queue_event(Master_info*, unsigned char const*, unsigned long) ()

    at /usr/src/debug/MariaDB-/src_0/sql/slave.cc:7277

#9  0x000055e00e962f30 in handle_slave_io () at /usr/src/debug/MariaDB-/src_0/sql/slave.cc:5049

#10 0x000055e00ee58c9d in pfs_spawn_thread (arg=0x7f0e70029c48)

    at /usr/src/debug/MariaDB-/src_0/storage/perfschema/pfs.cc:2201

#11 0x00007f5edb44a1ca in start_thread () from /lib64/libpthread.so.0

#12 0x00007f5eda79ae73 in clone () from /lib64/libc.so.6

(gdb) bt full

#0  0x00007f5edb451a35 in pthread_kill () from /lib64/libpthread.so.0

No symbol table info available.

#1  0x000055e00f157f27 in my_write_core (sig=<optimized out>) at /usr/src/debug/MariaDB-/src_0/mysys/stacktrace.c:424

No locals.

#2  0x000055e00ec515d0 in handle_fatal_signal (sig=11) at /usr/src/debug/MariaDB-/src_0/sql/signal_handler.cc:360

        curr_time = 1711266717

        tm = {tm_sec = 57, tm_min = 51, tm_hour = 12, tm_mday = 24, tm_mon = 2, tm_year = 124, tm_wday = 0, tm_yday = 83,

          tm_isdst = 0, tm_gmtoff = 18000, tm_zone = 0x55e0110bd300 "PKT"}

        thd = 0x7f0f8c0008e8

        print_invalid_query_pointer = true

#3  <signal handler called>

No symbol table info available.

#4  0x00007f5eda8311c9 in __memmove_avx_unaligned_erms () from /lib64/libc.so.6

No symbol table info available.

#5  0x000055e00f13c5ec in memcpy (__len=782, __src=0x7f0f8c010a79, __dest=<optimized out>)

    at /usr/include/bits/string_fortified.h:34

No locals.

#6  my_b_append (info=info@entry=0x55e01269f228, Buffer=0x7f0f8c010a79 "\270\305\377e\030\002", Count=Count@entry=782)

    at /usr/src/debug/MariaDB-/src_0/mysys/mf_iocache.c:1606

        rest_length = <optimized out>

        length = <optimized out>

#7  0x000055e00ed53dec in MYSQL_BIN_LOG::write_event_buffer (this=0x55e01269ef08, buf=<optimized out>, len=782)

    at /usr/src/debug/MariaDB-/src_0/sql/log.cc:5722

        error = true

        ebuf = 0x0

#8  0x000055e00e960bd0 in queue_event(Master_info*, unsigned char const*, unsigned long) ()

    at /usr/src/debug/MariaDB-/src_0/sql/slave.cc:7277

        TIME_FUZZY_DATES = {m_mode = date_conv_mode_t::FUZZY_DATES}

        reconnect_messages = {{0x55e00f2435e0 "Waiting to reconnect after a failed registration on master",

            0x55e00f243620 "Slave I/O thread killed while waiting to reconnect after a failed registration on master",

            0x55e00f243680 "Reconnecting after a failed registration on master",

            0x55e00f2436b8 "failed registering on master, reconnecting to try again, log '%s' at position %llu%s",

--Type <RET> for more, q to quit, c to continue without paging--

            0x55e00f243bbb "COM_REGISTER_SLAVE", 0x55e00f243710 "Slave I/O thread killed during or after reconnect"}, {

            0x55e00f243748 "Waiting to reconnect after a failed binlog dump request",

            0x55e00f243780 "Slave I/O thread killed while retrying master dump",

            0x55e00f2437b8 "Reconnecting after a failed binlog dump request",

            0x55e00f2437e8 "failed dump request, reconnecting to try again, log '%s' at position %llu%s",

            0x55e00f2440c9 "COM_BINLOG_DUMP", 0x55e00f243710 "Slave I/O thread killed during or after reconnect"}, {

            0x55e00f243838 "Waiting to reconnect after a failed master event read",

            0x55e00f243870 "Slave I/O thread killed while waiting to reconnect after a failed read",

            0x55e00f2438b8 "Reconnecting after a failed master event read",

            0x55e00f2438e8 "Slave I/O thread: Failed reading log event, reconnecting to retry, log '%s' at position %llu%s", 0x55e00f3a8a04 "", 0x55e00f243948 "Slave I/O thread killed during or after a reconnect done to recover from failed read"}}

        TIME_TIME_ONLY = {m_mode = date_conv_mode_t::TIME_ONLY}

        RPL_MASTER_INFO = 12

        std::__ioinit = {static _S_refcount = <optimized out>, static _S_synced_with_stdio = <optimized out>}

        key_thread_slave_io = 0

        all_slave_threads = {{m_key = 0x55e00fd61be8 <key_thread_slave_io>, m_name = 0x55e00f2440b6 "slave_io",

            m_flags = 1}, {m_key = 0x55e00fd61be4 <key_thread_slave_sql>, m_name = 0x55e00f2440bf "slave_sql", m_flags = 1}}

        msg_no = <optimized out>

        slave_background_thread_gtid_loaded = true

        msg_yes = <optimized out>

        slave_running = {{str = 0x55e00f243abd "No", length = 2}, {str = 0x55e00f2440ab "Connecting", length = 10}, {

            str = 0x55e00f23a701 "Preparing", length = 9}, {str = 0x55e00f243ac0 "Yes", length = 3}}

        key_thread_slave_sql = 0

        check_temp_dir_result = 0

        gtid_pos_table_definition1 = "CREATE TABLE "

        gtid_pos_table_definition2 = " (domain_id INT UNSIGNED NOT NULL, sub_id BIGINT UNSIGNED NOT NULL, server_id INT UNSIGNED NOT NULL, seq_no BIGINT UNSIGNED NOT NULL, PRIMARY KEY (domain_id, sub_id)) CHARSET=latin1 COMMENT='Replicati"...

        check_temp_dir_run = true

        TIME_INVALID_DATES = {m_mode = date_conv_mode_t::INVALID_DATES}

        TIME_NO_ZERO_DATE = <optimized out>

        sp_data_access_name = <optimized out>

        TIME_NO_ZEROS = <optimized out>

--Type <RET> for more, q to quit, c to continue without paging--

        TIME_INTERVAL_hhmmssff = <optimized out>

        TIME_INTERVAL_DAY = <optimized out>

        TIME_CONV_NONE = <optimized out>

        TIME_NO_ZERO_IN_DATE = <optimized out>

        TIME_MODE_FOR_XXX_TO_DATE = <optimized out>

        TIME_FRAC_NONE = <optimized out>

        TIME_FRAC_TRUNCATE = <optimized out>

        TIME_FRAC_ROUND = <optimized out>

        use_slave_mask = true

        slave_error_mask = {bitmap = 0x55e0110c4528, last_word_ptr = 0x55e0110c4734, last_word_mask = 4294967294,

          n_bits = 4193}

        replicate_same_server_id = 0 '\000'

        slave_load_tmpdir = 0x55e0110c5798 "/tmp"

        default_master_connection_name = {str = 0x55e00f3a8a04 "", length = 0}

        relay_log_index = 0x55e011162108 "/var/lib/mysql/db1-relay.index"

        slave_skip_error_names = 0x55e00fd612e0 <slave_skip_error_names> "1062"

        slave_transaction_retry_errors = 0x55e0110c4758

        master_info_index = 0x55e012120fa0

        opt_read_binlog_speed_limit = 0

        relay_log_basename = 0x55e0111620c8 "/var/lib/mysql/db1-relay"

        relay_log_space_limit = 0

        active_mi = 0x55e01269cda0

        slave_transaction_retry_error_length = 9

        abort_slave_event_count = 0

        disconnect_slave_event_count = 0

        slave_transaction_retry_error_names = 0x55e00fd60ac0 <slave_transaction_retry_error_names> "1158,1159,1160,1161,1205,1213,1429,2013,12701"

#9  0x000055e00e962f30 in handle_slave_io () at /usr/src/debug/MariaDB-/src_0/sql/slave.cc:5049

        TIME_FUZZY_DATES = {m_mode = date_conv_mode_t::FUZZY_DATES}

        reconnect_messages = {{0x55e00f2435e0 "Waiting to reconnect after a failed registration on master",

            0x55e00f243620 "Slave I/O thread killed while waiting to reconnect after a failed registration on master",

            0x55e00f243680 "Reconnecting after a failed registration on master",

--Type <RET> for more, q to quit, c to continue without paging--

            0x55e00f2436b8 "failed registering on master, reconnecting to try again, log '%s' at position %llu%s",

            0x55e00f243bbb "COM_REGISTER_SLAVE", 0x55e00f243710 "Slave I/O thread killed during or after reconnect"}, {

            0x55e00f243748 "Waiting to reconnect after a failed binlog dump request",

            0x55e00f243780 "Slave I/O thread killed while retrying master dump",

            0x55e00f2437b8 "Reconnecting after a failed binlog dump request",

            0x55e00f2437e8 "failed dump request, reconnecting to try again, log '%s' at position %llu%s",

            0x55e00f2440c9 "COM_BINLOG_DUMP", 0x55e00f243710 "Slave I/O thread killed during or after reconnect"}, {

            0x55e00f243838 "Waiting to reconnect after a failed master event read",

            0x55e00f243870 "Slave I/O thread killed while waiting to reconnect after a failed read",

            0x55e00f2438b8 "Reconnecting after a failed master event read",

            0x55e00f2438e8 "Slave I/O thread: Failed reading log event, reconnecting to retry, log '%s' at position %llu%s", 0x55e00f3a8a04 "", 0x55e00f243948 "Slave I/O thread killed during or after a reconnect done to recover from failed read"}}

        TIME_TIME_ONLY = {m_mode = date_conv_mode_t::TIME_ONLY}

        RPL_MASTER_INFO = 12

        std::__ioinit = {static _S_refcount = <optimized out>, static _S_synced_with_stdio = <optimized out>}

        key_thread_slave_io = 0

        all_slave_threads = {{m_key = 0x55e00fd61be8 <key_thread_slave_io>, m_name = 0x55e00f2440b6 "slave_io",

            m_flags = 1}, {m_key = 0x55e00fd61be4 <key_thread_slave_sql>, m_name = 0x55e00f2440bf "slave_sql", m_flags = 1}}

        msg_no = <optimized out>

        slave_background_thread_gtid_loaded = true

        msg_yes = <optimized out>

        slave_running = {{str = 0x55e00f243abd "No", length = 2}, {str = 0x55e00f2440ab "Connecting", length = 10}, {

            str = 0x55e00f23a701 "Preparing", length = 9}, {str = 0x55e00f243ac0 "Yes", length = 3}}

        key_thread_slave_sql = 0

        check_temp_dir_result = 0

        gtid_pos_table_definition1 = "CREATE TABLE "

        gtid_pos_table_definition2 = " (domain_id INT UNSIGNED NOT NULL, sub_id BIGINT UNSIGNED NOT NULL, server_id INT UNSIGNED NOT NULL, seq_no BIGINT UNSIGNED NOT NULL, PRIMARY KEY (domain_id, sub_id)) CHARSET=latin1 COMMENT='Replicati"...

        check_temp_dir_run = true

        TIME_INVALID_DATES = {m_mode = date_conv_mode_t::INVALID_DATES}

        TIME_NO_ZERO_DATE = <optimized out>

        sp_data_access_name = <optimized out>

--Type <RET> for more, q to quit, c to continue without paging--

        TIME_NO_ZEROS = <optimized out>

        TIME_INTERVAL_hhmmssff = <optimized out>

        TIME_INTERVAL_DAY = <optimized out>

        TIME_CONV_NONE = <optimized out>

        TIME_NO_ZERO_IN_DATE = <optimized out>

        TIME_MODE_FOR_XXX_TO_DATE = <optimized out>

        TIME_FRAC_NONE = <optimized out>

        TIME_FRAC_TRUNCATE = <optimized out>

        TIME_FRAC_ROUND = <optimized out>

        use_slave_mask = true

        slave_error_mask = {bitmap = 0x55e0110c4528, last_word_ptr = 0x55e0110c4734, last_word_mask = 4294967294,

          n_bits = 4193}

        replicate_same_server_id = 0 '\000'

        slave_load_tmpdir = 0x55e0110c5798 "/tmp"

        default_master_connection_name = {str = 0x55e00f3a8a04 "", length = 0}

        relay_log_index = 0x55e011162108 "/var/lib/mysql/db1-relay.index"

        slave_skip_error_names = 0x55e00fd612e0 <slave_skip_error_names> "1062"

        slave_transaction_retry_errors = 0x55e0110c4758

        master_info_index = 0x55e012120fa0

        opt_read_binlog_speed_limit = 0

        relay_log_basename = 0x55e0111620c8 "/var/lib/mysql/db1-relay"

        relay_log_space_limit = 0

        active_mi = 0x55e01269cda0

        slave_transaction_retry_error_length = 9

        abort_slave_event_count = 0

        disconnect_slave_event_count = 0

        slave_transaction_retry_error_names = 0x55e00fd60ac0 <slave_transaction_retry_error_names> "1158,1159,1160,1161,1205,1213,1429,2013,12701"

#10 0x000055e00ee58c9d in pfs_spawn_thread (arg=0x7f0e70029c48)

    at /usr/src/debug/MariaDB-/src_0/storage/perfschema/pfs.cc:2201

        typed_arg = 0x7f0e70029c48

        user_arg = 0x55e01269cda0

--Type <RET> for more, q to quit, c to continue without paging--

        user_start_routine = 0x55e00e961e90 <handle_slave_io>

        pfs = <optimized out>

        klass = <optimized out>

#11 0x00007f5edb44a1ca in start_thread () from /lib64/libpthread.so.0

No symbol table info available.

#12 0x00007f5eda79ae73 in clone () from /lib64/libc.so.6

No symbol table info available.

(gdb)

(gdb)