InnoDB stall in shutdown - buf_load waiting for io_slot

#0  0x00007f40c8d519fc in pthread_kill () from /lib/x86_64-linux-gnu/libc.so.6

[Current thread is 1 (Thread 0x7f40c8a2b840 (LWP 137618))]

#0  0x00007f40c8d519fc in pthread_kill () from /lib/x86_64-linux-gnu/libc.so.6

#1  0x0000559185fe1ca0 in handle_fatal_signal (sig=6) at /home/buildbot/amd64-ubuntu-2204-bigtest/build/sql/signal_handler.cc:298

#2  <signal handler called>

#3  0x00007f40c8d4c117 in ?? () from /lib/x86_64-linux-gnu/libc.so.6

#4  0x00007f40c8d4ea41 in pthread_cond_wait () from /lib/x86_64-linux-gnu/libc.so.6

#5  0x0000559186430d4b in tpool::waitable_task::wait (this=this@entry=0x5591877283e0 <buf_dump_load_task>, lk=...) at /home/buildbot/amd64-ubuntu-2204-bigtest/build/tpool/task.cc:63

#6  0x0000559186430dab in tpool::waitable_task::wait (this=this@entry=0x5591877283e0 <buf_dump_load_task>) at /home/buildbot/amd64-ubuntu-2204-bigtest/build/tpool/task.cc:69

#7  0x00005591863ba8e0 in buf_load_dump_end () at /home/buildbot/amd64-ubuntu-2204-bigtest/build/storage/innobase/buf/buf0dump.cc:778

#8  0x0000559185bffd69 in logs_empty_and_mark_files_at_shutdown () at /home/buildbot/amd64-ubuntu-2204-bigtest/build/storage/innobase/log/log0log.cc:1449

#9  0x0000559185c15695 in innodb_shutdown () at /home/buildbot/amd64-ubuntu-2204-bigtest/build/storage/innobase/srv/srv0start.cc:1981

#10 0x00005591862940a7 in innobase_end () at /home/buildbot/amd64-ubuntu-2204-bigtest/build/storage/innobase/handler/ha_innodb.cc:4310

#11 innobase_end () at /home/buildbot/amd64-ubuntu-2204-bigtest/build/storage/innobase/handler/ha_innodb.cc:4297

#12 0x0000559185fe4ec6 in ha_finalize_handlerton (plugin_=0x55919cbf6f20) at /home/buildbot/amd64-ubuntu-2204-bigtest/build/sql/handler.cc:598

#13 0x0000559185da5565 in plugin_deinitialize (plugin=0x55919cbf6f20, ref_check=ref_check@entry=true) at /home/buildbot/amd64-ubuntu-2204-bigtest/build/sql/sql_plugin.cc:1274

#14 0x0000559185da5d26 in reap_plugins () at /home/buildbot/amd64-ubuntu-2204-bigtest/build/sql/sql_plugin.cc:1345

#15 0x0000559185da7522 in plugin_shutdown () at /home/buildbot/amd64-ubuntu-2204-bigtest/build/sql/sql_plugin.cc:2098

#16 0x0000559185c84db2 in clean_up (print_message=<optimized out>) at /home/buildbot/amd64-ubuntu-2204-bigtest/build/sql/mysqld.cc:1989

#17 0x0000559185c900aa in clean_up (print_message=true) at /home/buildbot/amd64-ubuntu-2204-bigtest/build/sql/mysqld.cc:1961

#18 mysqld_main (argc=<optimized out>, argv=<optimized out>) at /home/buildbot/amd64-ubuntu-2204-bigtest/build/sql/mysqld.cc:6061

#19 0x00007f40c8ce4d90 in ?? () from /lib/x86_64-linux-gnu/libc.so.6

#20 0x00007f40c8ce4e40 in __libc_start_main () from /lib/x86_64-linux-gnu/libc.so.6

#21 0x0000559185c83515 in _start ()

Thread 4 (Thread 0x7f40b35ff640 (LWP 137708)):

warning: Section `.reg-xstate/137708' in core file too small.

#0  0x00007f40c8d4c117 in ?? () from /lib/x86_64-linux-gnu/libc.so.6

No symbol table info available.

#1  0x00007f40c8d4ea41 in pthread_cond_wait () from /lib/x86_64-linux-gnu/libc.so.6

No symbol table info available.

#2  0x000055918630240b in tpool::cache<tpool::aiocb>::wait (m=@0x55919cd01300: {m_mutex = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}, m_psi = 0x7f40c43046c0}, this=0x55919cd01300) at /home/buildbot/amd64-ubuntu-2204-bigtest/build/tpool/tpool_structs.h:181

No locals.

#3  tpool::cache<tpool::aiocb>::wait (this=0x55919cd01300) at /home/buildbot/amd64-ubuntu-2204-bigtest/build/tpool/tpool_structs.h:189

No locals.

#4  io_slots::wait (this=<optimized out>) at /home/buildbot/amd64-ubuntu-2204-bigtest/build/storage/innobase/os/os0file.cc:109

No locals.

#5  os_aio_wait_until_no_pending_reads (declare=declare@entry=true) at /home/buildbot/amd64-ubuntu-2204-bigtest/build/storage/innobase/os/os0file.cc:3252

        notify_wait = <optimized out>

#6  0x00005591863bb9b0 in buf_load () at /home/buildbot/amd64-ubuntu-2204-bigtest/build/storage/innobase/buf/buf0dump.cc:669

        full_filename = "/dev/shm/var_auto_r6sZ/36/mysqld.1/data/ib_buffer_pool", '\000' <repeats 2850 times>...

        now = '\000' <repeats 31 times>

        f = <optimized out>

        dump = 0x7f40ac0020c8

        dump_n = 126

        i = 126

        space_id = 0

        page_no = 192

        fscanf_ret = <optimized out>

        cur_space_id = 18

        space = <optimized out>

        zip_size = 0

        pfs_stage_progress = 0x7f40c8305570

#7  0x00005591863bc088 in buf_dump_load_func () at /home/buildbot/amd64-ubuntu-2204-bigtest/build/storage/innobase/buf/buf0dump.cc:716

        first_time = true

#8  0x0000559186430bda in tpool::task::execute (this=0x5591877283e0 <buf_dump_load_task>) at /home/buildbot/amd64-ubuntu-2204-bigtest/build/tpool/task.cc:37

No locals.

#9  tpool::task::execute (this=0x5591877283e0 <buf_dump_load_task>) at /home/buildbot/amd64-ubuntu-2204-bigtest/build/tpool/task.cc:27

No locals.

#10 0x000055918642f097 in tpool::thread_pool_generic::worker_main (this=0x55919ccb9090, thread_var=0x55919ccb93d0) at /home/buildbot/amd64-ubuntu-2204-bigtest/build/tpool/tpool_generic.cc:566

        task = 0x5591877283e0 <buf_dump_load_task>

#11 0x00007f40c90c9253 in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6

No symbol table info available.

#12 0x00007f40c8d4fac3 in ?? () from /lib/x86_64-linux-gnu/libc.so.6

No symbol table info available.

#13 0x00007f40c8de0a04 in clone () from /lib/x86_64-linux-gnu/libc.so.6

No symbol table info available.