Jul 19 13:38:06 db1132 mysqld[3344701]: 2022-07-19 13:38:06 0 [Warning] mysqld: io_uring_queue_init() failed with ENOMEM: try larger memory locked limit, ulimit -l, or https://mariadb.com/kb/en/systemd/#configuring-limitmemlock under systemd

Jul 19 13:38:06 db1132 mysqld[3344701]: 2022-07-19 13:38:06 0 [Warning] InnoDB: liburing disabled: falling back to innodb_use_native_aio=OFF

#7  mtr_t::page_lock (this=this@entry=0x7f4cecb0fe60, block=block@entry=0x7fac2802c380, rw_latch=rw_latch@entry=1) at /root/mariadb-10.6.8/storage/innobase/mtr/mtr0mtr.cc:1240

        fix_type = MTR_MEMO_PAGE_S_FIX

print *(buf_page_t*)0x7fac2802c380

(gdb) p write_slots

$4 = std::unique_ptr<io_slots> = {get() = 0x564d23f2efd0}

(gdb) p read_slots

$5 = std::unique_ptr<io_slots> = {get() = 0x564d23f06e80}

print *(io_slots*)0x564d23f2efd0

(gdb) p (char[8])0x7fac2802c380

$1 = "\200\303\002(\254\177\000"

\254\177

print *(io_slots*)0x564d23f06e80

Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".

--Type <RET> for more, q to quit, c to continue without paging--c

Core was generated by `/opt/wmf-mariadb106/bin/mysqld'.

#0  0x00007fac896ec3ff in poll () from /lib/x86_64-linux-gnu/libc.so.6

[Current thread is 1 (Thread 0x7fac893631c0 (LWP 3344701))]

(gdb) thread 11

[Switching to thread 11 (Thread 0x7fac808e3700 (LWP 3344713))]

#0  0x00007fac896f19b9 in syscall () from /lib/x86_64-linux-gnu/libc.so.6

(gdb) frame 6

#6  btr_cur_search_to_nth_level_func (index=index@entry=0x7f4c5406d500, level=level@entry=0, tuple=tuple@entry=0x7f49a9189ab0, mode=mode@entry=PAGE_CUR_LE, latch_mode=<optimized out>, latch_mode@entry=33, cursor=cursor@entry=0x7fac808e05f0, ahi_latch=<optimized out>,

    mtr=<optimized out>, autoinc=<optimized out>) at /root/mariadb-10.6.8/storage/innobase/btr/btr0cur.cc:2229

2229	/root/mariadb-10.6.8/storage/innobase/btr/btr0cur.cc: No such file or directory.

(gdb) p index->table->space->chain.start->handle

$1 = {m_file = 133, m_psi = 0x7fac8302afc0}

print *(io_slots*)0x564d23f06e80

gcore $PID

root@db1132.eqiad.wmnet[(none)]> show global variables like 'innodb_adaptive_hash_index';

+----------------------------+-------+

| Variable_name              | Value |

+----------------------------+-------+

| innodb_adaptive_hash_index | OFF   |

+----------------------------+-------+

1 row in set (0.002 sec)

set print static-members off

set print frame-arguments all

thread apply all backtrace full

thread apply all -s -- frame apply level 2 -s print/x *this

p *read_slots

p *write_slots

Could not find operator*.

Could not find operator*.

$46 = {_M_t = {<std::__uniq_ptr_impl<io_slots, std::default_delete<io_slots> >> = {_M_t = {<std::_Tuple_impl<0, io_slots*, std::default_delete<io_slots> >> = {<std::_Tuple_impl<1, std::default_delete<io_slots> >> = {<std::_Head_base<1, std::default_delete<io_slots>, true>> = {<std::default_delete<io_slots>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, io_slots*, false>> = {_M_head_impl = 0x55f7467b5e80}, <No data fields>}, <No data fields>}}, <No data fields>}}

$47 = {_M_t = {<std::__uniq_ptr_impl<io_slots, std::default_delete<io_slots> >> = {_M_t = {<std::_Tuple_impl<0, io_slots*, std::default_delete<io_slots> >> = {<std::_Tuple_impl<1, std::default_delete<io_slots> >> = {<std::_Head_base<1, std::default_delete<io_slots>, true>> = {<std::default_delete<io_slots>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, io_slots*, false>> = {_M_head_impl = 0x55f7467e9fd0}, <No data fields>}, <No data fields>}}, <No data fields>}}

sed -ne 's/^\(#2 .*this=\)/\1/p' < thread_apply_all_backtrace |sort|uniq -c

      1 #2  srw_mutex_impl<true>::wait_and_lock (this=this@entry=0x7eff4d04bd10) at /root/mariadb-10.6.8/storage/innobase/sync/srw_lock.cc:326

     39 #2  srw_mutex_impl<true>::wait_and_lock (this=this@entry=0x7f22741195f8) at /root/mariadb-10.6.8/storage/innobase/sync/srw_lock.cc:326

      1 #2  ssux_lock_impl<true>::wr_wait (this=0x7f22741195f8, lk=<optimized out>) at /root/mariadb-10.6.8/storage/innobase/sync/srw_lock.cc:378

      1 #2  ssux_lock_impl<true>::wr_wait (this=this@entry=0x7f1d1408a538, lk=<optimized out>) at /root/mariadb-10.6.8/storage/innobase/sync/srw_lock.cc:378

Thread 53 (Thread 0x7f00148c1700 (LWP 4008608) "mysqld"):

#2  srw_mutex_impl<true>::wait_and_lock (this=this@entry=0x7eff4d04bd10) at /root/mariadb-10.6.8/storage/innobase/sync/srw_lock.cc:326

$22 = {lock = {<std::__atomic_base<unsigned int>> = {_M_i = 0x80000002}, <No data fields>}}

Thread 65 (Thread 0x7eff7b767700 (LWP 4008909) "mysqld"):

#2  ssux_lock_impl<true>::wr_wait (this=this@entry=0x7f1d1408a538, lk=<optimized out>) at /root/mariadb-10.6.8/storage/innobase/sync/srw_lock.cc:378

$17 = {writer = {lock = {<std::__atomic_base<unsigned int>> = {_M_i = 0x80000001}, <No data fields>}}, readers = {<std::__atomic_base<unsigned int>> = {_M_i = 0x80000001}, <No data fields>}}

Thread 12 (Thread 0x7f5f586ac700 (LWP 2484735) "mysqld"):

#2  ssux_lock_impl<true>::wr_wait (this=0x7f22741195f8, lk=<optimized out>) at /root/mariadb-10.6.8/storage/innobase/sync/srw_lock.cc:378

$45 = {writer = {lock = {<std::__atomic_base<unsigned int>> = {_M_i = 0x80000028}, <No data fields>}}, readers = {<std::__atomic_base<unsigned int>> = {_M_i = 0x80000002}, <No data fields>}}

print/x *(block_lock*)0x7eff4d04bd10

print/x *(block_lock*)0x7f1d1408a538

print/x *(block_lock*)0x7f22741195f8

(gdb) print/x *(block_lock*)0x7eff4d04bd10

$1 = {lock = {writer = {lock = {<std::__atomic_base<unsigned int>> = {static _S_alignment = 0x4,

          _M_i = 0x80000002}, <No data fields>}, static HOLDER = 0x80000000}, readers = {<std::__atomic_base<unsigned int>> = {

        static _S_alignment = 0x4, _M_i = 0x80000000}, <No data fields>}, static WRITER = 0x80000000}, recursive = 0x1,

  writer = {<std::__atomic_base<unsigned long>> = {static _S_alignment = 0x8, _M_i = 0x7f0014ae7700}, <No data fields>},

  static FOR_IO = 0xffffffffffffffff, static RECURSIVE_X = 0x1, static RECURSIVE_U = 0x10000, static RECURSIVE_MAX = <optimized out>}

(gdb) print/x *(block_lock*)0x7f1d1408a538

$2 = {lock = {writer = {lock = {<std::__atomic_base<unsigned int>> = {static _S_alignment = 0x4,

          _M_i = 0x80000001}, <No data fields>}, static HOLDER = 0x80000000}, readers = {<std::__atomic_base<unsigned int>> = {

        static _S_alignment = 0x4, _M_i = 0x80000001}, <No data fields>}, static WRITER = 0x80000000}, recursive = 0x0,

  writer = {<std::__atomic_base<unsigned long>> = {static _S_alignment = 0x8, _M_i = 0x0}, <No data fields>},

  static FOR_IO = 0xffffffffffffffff, static RECURSIVE_X = 0x1, static RECURSIVE_U = 0x10000, static RECURSIVE_MAX = <optimized out>}

(gdb) print/x *(block_lock*)0x7f22741195f8

$3 = {lock = {writer = {lock = {<std::__atomic_base<unsigned int>> = {static _S_alignment = 0x4,

          _M_i = 0x80000028}, <No data fields>}, static HOLDER = 0x80000000}, readers = {<std::__atomic_base<unsigned int>> = {

        static _S_alignment = 0x4, _M_i = 0x80000002}, <No data fields>}, static WRITER = 0x80000000}, recursive = 0x0,

  writer = {<std::__atomic_base<unsigned long>> = {static _S_alignment = 0x8, _M_i = 0x0}, <No data fields>},

  static FOR_IO = 0xffffffffffffffff, static RECURSIVE_X = 0x1, static RECURSIVE_U = 0x10000, static RECURSIVE_MAX = <optimized out>}

thread find 0x7f0014ae7700

Thread 72 (Thread 0x7f0014ae7700 (LWP 4009073) "mysqld"):

#0  0x00007f5faf4b1c61 in clock_nanosleep () from target:/lib/x86_64-linux-gnu/libc.so.6

No symbol table info available.

#1  0x00007f5faf4b7443 in nanosleep () from target:/lib/x86_64-linux-gnu/libc.so.6

No symbol table info available.

#2  0x000055f744758bbb in std::this_thread::sleep_for<long, std::ratio<1l, 1000000l> > (__rtime=<optimized out>, __rtime=<optimized out>) at /usr/include/c++/10/thread:401

        __s = {__r = <optimized out>}

        __ns = {__r = <optimized out>}

        __ts = {tv_sec = 0, tv_nsec = 94044}

        __s = {__r = <optimized out>}

        __ns = {__r = <optimized out>}

        __ts = {tv_sec = <optimized out>, tv_nsec = <optimized out>}

#3  buf_page_get_low (page_id={m_id = 4131762166058}, zip_size=<optimized out>, rw_latch=<optimized out>, guess=<optimized out>, mode=10, mtr=<optimized out>, err=<optimized out>, allow_ibuf_merge=<optimized out>) at /root/mariadb-10.6.8/storage/innobase/buf/buf0buf.cc:2812

	if (UNIV_UNLIKELY(!block->page.frame)) {

		if (!block->page.lock.x_lock_try()) {

			/* The page is being read or written, or

			another thread is executing buf_zip_decompress()

			in buf_page_get_low() on it. */

			block->page.unfix();

			std::this_thread::sleep_for(

				std::chrono::microseconds(100));

			goto loop;

		}

		buf_block_t *new_block = buf_LRU_get_free_block(false);

		buf_block_init_low(new_block);

wait_for_unfix:

		mysql_mutex_lock(&buf_pool.mutex);

		page_hash_latch& hash_lock=buf_pool.page_hash.lock_get(chain);

		/* It does not make sense to use

		transactional_lock_guard here, because buf_relocate()

		would likely make a  memory transaction too large. */

		hash_lock.lock();

		/* block->page.lock implies !block->page.can_relocate() */

		ut_ad(&block->page == buf_pool.page_hash.get(page_id, chain));

		/* Wait for any other threads to release their buffer-fix

		on the compressed-only block descriptor.

		FIXME: Never fix() before acquiring the lock.

		Only in buf_page_get_gen(), buf_page_get_low(), buf_page_free()

		we are violating that principle. */

		state = block->page.state();

		switch (state) {

		case buf_page_t::UNFIXED + 1:

		case buf_page_t::IBUF_EXIST + 1:

		case buf_page_t::REINIT + 1:

			break;

		default:

			ut_ad(state < buf_page_t::READ_FIX);

			if (state < buf_page_t::UNFIXED + 1) {

				ut_ad(state > buf_page_t::FREED);

				ut_ad(mode == BUF_GET_POSSIBLY_FREED

				      || mode == BUF_PEEK_IF_IN_POOL);

				block->page.unfix();

				block->page.lock.x_unlock();

				hash_lock.unlock();

				buf_LRU_block_free_non_file_page(new_block);

				mysql_mutex_unlock(&buf_pool.mutex);

				return nullptr;

			}

			mysql_mutex_unlock(&buf_pool.mutex);

			hash_lock.unlock();

			std::this_thread::sleep_for(

				std::chrono::microseconds(100));

			goto wait_for_unfix;

		}

#6  sux_lock<ssux_lock_impl<true> >::s_lock (this=<optimized out>) at /root/mariadb-10.6.8/storage/innobase/include/sux_lock.h:362

No locals.

#7  buf_page_get_low (page_id={m_id = 4131762166058}, zip_size=<optimized out>, rw_latch=<optimized out>, guess=<optimized out>, mode=10, mtr=<optimized out>, err=<optimized out>, allow_ibuf_merge=<optimized out>) at /root/mariadb-10.6.8/storage/innobase/buf/buf0buf.cc:2698

print *(buf_block_t*)0x7eff4d04bcf8

(gdb) print *(buf_block_t*)0x7eff4d04bcf8

$2 = {page = {id_ = {m_id = 4131762166058}, hash = 0x0, oldest_modification_ = {m = {<std::__atomic_base<unsigned long>> = {static _S_alignment = 8, _M_i = 0}, <No data fields>}}, static NOT_USED = 0, static MEMORY = 1, static REMOVE_HASH = 2, static FREED = 3, static UNFIXED = 536870912, static IBUF_EXIST = 1073741824, static REINIT = 1610612736, static READ_FIX = 2147483648, static WRITE_FIX = 2684354560, static WRITE_FIX_IBUF = 3221225472, static WRITE_FIX_REINIT = 3758096384, static LRU_MASK = 3758096384, lock = {lock = {writer = {lock = {<std::__atomic_base<unsigned int>> = {static _S_alignment = 4, _M_i = 2147483650}, <No data fields>}, static HOLDER = 2147483648}, readers = {<std::__atomic_base<unsigned int>> = {static _S_alignment = 4, _M_i = 2147483648}, <No data fields>}, static WRITER = 2147483648}, recursive = 1, writer = {<std::__atomic_base<unsigned long>> = {static _S_alignment = 8, _M_i = 139638323705600}, <No data fields>}, static FOR_IO = 18446744073709551615, static RECURSIVE_X = 1, static RECURSIVE_U = 65536, static RECURSIVE_MAX = <optimized out>}, frame = 0x0, zip = {data = 0x7f2bc3594000 "4\213p/", m_end = 0, m_nonempty = 0, n_blobs = 0, ssize = 4, fix = {m = {<std::__atomic_base<unsigned int>> = {static _S_alignment = 4, _M_i = 536870914}, <No data fields>}}}, list = {prev = 0x0, next = 0x0}, LRU = {prev = 0x7efa50be5fc8, next = 0x7f5f4c0c7880}, old = 1, freed_page_clock = 0, access_time = {m_counter = {<std::__atomic_base<unsigned int>> = {static _S_alignment = 4, _M_i = 0}, <No data fields>}}}, unzip_LRU = {prev = 0x95, next = 0x0}, modify_clock = 0, n_bytes = 136, n_fields = 0, n_hash_helps = 0, left_side = false, curr_n_fields = 204, curr_n_bytes = 3842, curr_left_side = 0, index = 0x7efda684f868}

root@db1132.eqiad.wmnet[information_schema]> select table_name,row_format from tables where table_schema='enwiki';

+------------------------------+------------+

| table_name                   | row_format |

+------------------------------+------------+

| ores_model                   | Compressed |

| ores_classification          | Compressed |

| babel                        | Compressed |

| bv2017_edits                 | Compressed |

| securepoll_strike            | Compressed |

| page_assessments_projects    | Compressed |

| oldimage                     | Compressed |

| mathoid                      | Compressed |

| page                         | Compressed |

| slots                        | Compressed |

| linter                       | Compressed |

| archive                      | Compressed |

| change_tag_def               | Compressed |

| recentchanges                | Compressed |

| templatelinks                | Compressed |

| content_models               | Compressed |

| job                          | Compressed |

| ip_changes                   | Compressed |

| logging                      | Compressed |

| discussiontools_subscription | Dynamic    |

| revision                     | Compressed |

| actor                        | Compressed |

| comment                      | Compressed |

| global_block_whitelist       | Compressed |

| externallinks                | Compressed |

| linktarget                   | Dynamic    |

| mcdc2021_edits               | Dynamic    |

| page_assessments             | Compressed |

| image                        | Compressed |

| ipblocks                     | Compressed |

| watchlist                    | Compressed |

| watchlist_expiry             | Compressed |

| sites                        | Compressed |

| ucoc_edits                   | Dynamic    |

| content                      | Compressed |

| wikilove_log                 | Compressed |

| bv2022_edits                 | Dynamic    |

| securepoll_votes             | Compressed |

| site_identifiers             | Compressed |

| searchindex                  | Compressed |

| flaggedrevs                  | Compressed |

| user_properties              | Compressed |

| geo_tags                     | Compressed |

| __wmf_checksums              | Compressed |

| abuse_filter_log             | Compressed |

| category                     | Compressed |

| pagelinks                    | Compressed |

| optin_survey_old             | Compressed |

| abuse_filter                 | Compressed |

| abuse_filter_action          | Compressed |

| abuse_filter_history         | Compressed |

| arbcom1_vote                 | Compressed |

| filearchive                  | Compressed |

| bv2009_edits                 | Compressed |

| bv2011_edits                 | Compressed |

| categorylinks                | Compressed |

| imagelinks                   | Compressed |

| site_stats                   | Compressed |

| cu_log                       | Compressed |

| cur                          | Compressed |

| cu_changes                   | Compressed |

| page_restrictions            | Compressed |

| langlinks                    | Compressed |

| uploadstash                  | Compressed |

| exarchive                    | Compressed |

| exrevision                   | Compressed |

| user_newtalk                 | Compressed |

| flaggedpage_config           | Compressed |

| flaggedpage_pending          | Compressed |

| flaggedpages                 | Compressed |

| flaggedrevs_promote          | Compressed |

| page_props                   | Compressed |

| flaggedrevs_statistics       | Compressed |

| flaggedrevs_tracking         | Compressed |

| flaggedtemplates             | Compressed |

| change_tag                   | Compressed |

| hidden                       | Compressed |

| interwiki                    | Compressed |

| l10n_cache                   | Compressed |

| securepoll_cookie_match      | Compressed |

| bv2021_edits                 | Dynamic    |

| log_search                   | Compressed |

| revision_comment_temp        | Compressed |

| module_deps                  | Compressed |

| moodbar_feedback             | Compressed |

| moodbar_feedback_response    | Compressed |

| user                         | Compressed |

| querycache_info              | Compressed |

| pagetriage_page              | Compressed |

| pagetriage_page_tags         | Compressed |

| pagetriage_tags              | Compressed |

| pif_edits                    | Compressed |

| objectcache                  | Compressed |

| slot_roles                   | Compressed |

| protected_titles             | Compressed |

| querycache                   | Compressed |

| bv2015_edits                 | Compressed |

| querycachetwo                | Compressed |

| redirect                     | Compressed |

| securepoll_entity            | Compressed |

| securepoll_lists             | Compressed |

| securepoll_msgs              | Compressed |

| securepoll_properties        | Compressed |

| securepoll_questions         | Compressed |

| securepoll_voters            | Compressed |

| spoofuser                    | Compressed |

| text                         | Compressed |

| transcache                   | Compressed |

| updatelog                    | Compressed |

| user_former_groups           | Compressed |

| user_groups                  | Compressed |

| vote_log                     | Compressed |

| iwlinks                      | Compressed |

| bv2013_edits                 | Compressed |

| securepoll_options           | Compressed |

| wbc_entity_usage             | Compressed |

| betafeatures_user_counts     | Compressed |

| pagetriage_log               | Compressed |

| transcode                    | Compressed |

| securepoll_elections         | Compressed |

| ipblocks_restrictions        | Compressed |

+------------------------------+------------+

	if (state > buf_page_t::READ_FIX && state < buf_page_t::WRITE_FIX) {

		if (mode == BUF_PEEK_IF_IN_POOL) {

ignore_block:

			ut_ad(mode == BUF_GET_POSSIBLY_FREED

			      || mode == BUF_PEEK_IF_IN_POOL);

			block->unfix();

			if (err) {

				*err = DB_CORRUPTION;

			}

			return nullptr;

		}

		/* A read-fix is released after block->page.lock

		in buf_page_t::read_complete() or

		buf_pool_t::corrupted_evict(), or

		after buf_zip_decompress() in this function. */

		block->page.lock.s_lock();