close connection hang at THD_count::value() - binlog_dump_thread_count - n_threads_awaiting_ack

while (THD_count::value() - binlog_dump_thread_count -

         n_threads_awaiting_ack)

    my_sleep(1000);

(gdb) t a a bt

Thread 11 (LWP 554186):

#0  0x00007f152de28a3c in pthread_cond_wait () from /usr/lib64/libpthread.so.0

#1  0x000055f0eec8a578 in inline_mysql_cond_wait (that=0x55f0ef9d6800 <mysql_bin_log+3232>, mutex=0x55f0ef9d67d0 <mysql_bin_log+3184>, src_file=0x55f0ef308ab0 "/data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/log.cc", src_line=10341) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/include/mysql/psi/mysql_thread.h:1233

#2  binlog_background_thread (arg=arg@entry=0x0) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/log.cc:10341

#3  0x000055f0eee0899b in pfs_spawn_thread (arg=0x7f13984cfb68) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/storage/perfschema/pfs.cc:2201

#4  0x00007f152de22f4b in ?? () from /usr/lib64/libpthread.so.0

#5  0x00007f152dd5a7ef in clone () from /usr/lib64/libc.so.6

Thread 10 (LWP 553187):

#0  0x00007f152de28df2 in pthread_cond_timedwait () from /usr/lib64/libpthread.so.0

#1  0x000055f0eec85244 in inline_mysql_cond_timedwait (src_file=0x55f0ef308ab0 "/data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/log.cc", src_line=8750, abstime=0x7f152c0f0e30, mutex=0x55f0ef9d5fe0 <mysql_bin_log+1152>, that=0x55f0ef9d60b0 <mysql_bin_log+1360>) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/include/mysql/psi/mysql_thread.h:1270

#2  MYSQL_BIN_LOG::wait_for_update_binlog_end_pos (this=this@entry=0x55f0ef9d5b60 <mysql_bin_log>, thd=0x7f1454000c18, timeout=timeout@entry=0x7f152c0f0e30) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/log.cc:8750

#3  0x000055f0ee99b3f1 in wait_new_events (end_pos_ptr=<synthetic pointer>, binlog_end_pos_filename=0x7f152c0f16f0 "./mysql-bin.000005", linfo=0x7f152c0f18f0, info=0x7f152c0f1b10) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/sql_repl.cc:2535

#4  get_binlog_end_pos (linfo=0x7f152c0f18f0, log=0x7f152c0f1000, info=0x7f152c0f1b10) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/sql_repl.cc:2655

#5  send_one_binlog_file (start_pos=<optimized out>, linfo=0x7f152c0f18f0, log=0x7f152c0f1000, info=0x7f152c0f1b10) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/sql_repl.cc:2815

#6  mysql_binlog_send (thd=thd@entry=0x7f1454000c18, log_ident=<optimized out>, pos=<optimized out>, pos@entry=4, flags=flags@entry=2) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/sql_repl.cc:2960

#7  0x000055f0ee9845f6 in dispatch_command (command=COM_BINLOG_DUMP, thd=0x7f1454000c18, packet=<optimized out>, packet_length=<optimized out>, is_com_multi=<optimized out>, is_next_command=<optimized out>) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/sql_class.h:1182

#8  0x000055f0ee984e13 in do_command (thd=0x7f1454000c18) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/sql_parse.cc:1377

#9  0x000055f0eea80841 in do_handle_one_connection (connect=<optimized out>, connect@entry=0x55f0f45c5a98, put_in_cache=put_in_cache@entry=true) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/sql_connect.cc:1419

#10 0x000055f0eea80ccd in handle_one_connection (arg=arg@entry=0x55f0f45c5a98) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/sql_connect.cc:1313

#11 0x000055f0eee0899b in pfs_spawn_thread (arg=0x55f0f4507b98) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/storage/perfschema/pfs.cc:2201

#12 0x00007f152de22f4b in ?? () from /usr/lib64/libpthread.so.0

#13 0x00007f152dd5a7ef in clone () from /usr/lib64/libc.so.6

Thread 9 (LWP 553391):

#0  0x00007f152de28df2 in pthread_cond_timedwait () from /usr/lib64/libpthread.so.0

#1  0x000055f0eec85244 in inline_mysql_cond_timedwait (src_file=0x55f0ef308ab0 "/data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/log.cc", src_line=8750, abstime=0x7f152c0a5e30, mutex=0x55f0ef9d5fe0 <mysql_bin_log+1152>, that=0x55f0ef9d60b0 <mysql_bin_log+1360>) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/include/mysql/psi/mysql_thread.h:1270

--Type <RET> for more, q to quit, c to continue without paging--

#2  MYSQL_BIN_LOG::wait_for_update_binlog_end_pos (this=this@entry=0x55f0ef9d5b60 <mysql_bin_log>, thd=0x7f1448000c18, timeout=timeout@entry=0x7f152c0a5e30) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/log.cc:8750

#3  0x000055f0ee99b3f1 in wait_new_events (end_pos_ptr=<synthetic pointer>, binlog_end_pos_filename=0x7f152c0a66f0 "./mysql-bin.000005", linfo=0x7f152c0a68f0, info=0x7f152c0a6b10) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/sql_repl.cc:2535

#4  get_binlog_end_pos (linfo=0x7f152c0a68f0, log=0x7f152c0a6000, info=0x7f152c0a6b10) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/sql_repl.cc:2655

#5  send_one_binlog_file (start_pos=<optimized out>, linfo=0x7f152c0a68f0, log=0x7f152c0a6000, info=0x7f152c0a6b10) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/sql_repl.cc:2815

#6  mysql_binlog_send (thd=thd@entry=0x7f1448000c18, log_ident=<optimized out>, pos=<optimized out>, pos@entry=4, flags=flags@entry=2) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/sql_repl.cc:2960

#7  0x000055f0ee9845f6 in dispatch_command (command=COM_BINLOG_DUMP, thd=0x7f1448000c18, packet=<optimized out>, packet_length=<optimized out>, is_com_multi=<optimized out>, is_next_command=<optimized out>) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/sql_class.h:1182

#8  0x000055f0ee984e13 in do_command (thd=0x7f1448000c18) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/sql_parse.cc:1377

#9  0x000055f0eea80841 in do_handle_one_connection (connect=<optimized out>, connect@entry=0x55f0f45953f8, put_in_cache=put_in_cache@entry=true) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/sql_connect.cc:1419

#10 0x000055f0eea80ccd in handle_one_connection (arg=arg@entry=0x55f0f45953f8) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/sql_connect.cc:1313

#11 0x000055f0eee0899b in pfs_spawn_thread (arg=0x55f0f4595468) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/storage/perfschema/pfs.cc:2201

#12 0x00007f152de22f4b in ?? () from /usr/lib64/libpthread.so.0

#13 0x00007f152dd5a7ef in clone () from /usr/lib64/libc.so.6

Thread 8 (LWP 553149):

#0  0x00007f152dc9c4ec in sigtimedwait () from /usr/lib64/libc.so.6

#1  0x000055f0ee8ac153 in my_sigwait (code=<synthetic pointer>, sig=0x7f152c13dd38, set=0x7f152c13dd40) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/include/my_pthread.h:196

#2  signal_hand (arg=arg@entry=0x0) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/mysqld.cc:3005

#3  0x000055f0eee0899b in pfs_spawn_thread (arg=0x55f0f44fe028) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/storage/perfschema/pfs.cc:2201

#4  0x00007f152de22f4b in ?? () from /usr/lib64/libpthread.so.0

#5  0x00007f152dd5a7ef in clone () from /usr/lib64/libc.so.6

Thread 7 (LWP 553143):

#0  0x00007f152de28a3c in pthread_cond_wait () from /usr/lib64/libpthread.so.0

#1  0x000055f0ee97174b in inline_mysql_cond_wait (that=0x55f0ef9a7680 <COND_manager>, mutex=0x55f0ef9a76c0 <LOCK_manager>, src_file=0x55f0ef1c2498 "/data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/sql_manager.cc", src_line=103) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/include/mysql/psi/mysql_thread.h:1233

#2  handle_manager (arg=arg@entry=0x0) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/sql_manager.cc:103

#3  0x000055f0eee0899b in pfs_spawn_thread (arg=0x55f0f450b6f8) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/storage/perfschema/pfs.cc:2201

#4  0x00007f152de22f4b in ?? () from /usr/lib64/libpthread.so.0

#5  0x00007f152dd5a7ef in clone () from /usr/lib64/libc.so.6

Thread 6 (LWP 553136):

#0  0x00007f152de28df2 in pthread_cond_timedwait () from /usr/lib64/libpthread.so.0

--Type <RET> for more, q to quit, c to continue without paging--

#1  0x000055f0ef0c1e42 in __gthread_cond_timedwait (__abs_timeout=0x7f148f7fddd0, __mutex=<optimized out>, __cond=0x55f0f1d55e50) at /opt/hw/gcc-10.3/include/c++/10/x86_64-linux-gnu/bits/gthr-default.h:872

#2  std::condition_variable::__wait_until_impl<std::chrono::duration<long, std::ratio<1l, 1000000000l> > > (__lock=..., __lock=..., __atime=<synthetic pointer>..., this=0x55f0f1d55e50) at /opt/hw/gcc-10.3/include/c++/10/condition_variable:232

#3  std::condition_variable::wait_until<std::chrono::_V2::steady_clock, std::chrono::duration<long, std::ratio<1l, 1000000000l> > > (__atime=..., __lock=..., this=0x55f0f1d55e50) at /opt/hw/gcc-10.3/include/c++/10/condition_variable:141

#4  std::condition_variable::wait_for<long, std::ratio<1l, 1000l> > (__rtime=..., __rtime=..., __lock=..., this=0x55f0f1d55e50) at /opt/hw/gcc-10.3/include/c++/10/condition_variable:172

#5  tpool::thread_pool_generic::wait_for_tasks (this=this@entry=0x55f0f1d461b0, lk=..., thread_data=thread_data@entry=0x55f0f1d55e50) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/tpool/tpool_generic.cc:498

#6  0x000055f0ef0c24fd in tpool::thread_pool_generic::get_task (this=0x55f0f1d461b0, thread_var=0x55f0f1d55e50, t=0x7f148f7fde78) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/tpool/tpool_generic.cc:551

#7  0x000055f0ef0c3b8d in tpool::thread_pool_generic::worker_main (this=0x55f0f1d461b0, thread_var=0x55f0f1d55e50) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/tpool/tpool_generic.cc:596

#8  0x00007f152e08cf80 in std::execute_native_thread_routine (__p=0x7f1528000c00) at ../../../.././libstdc++-v3/src/c++11/thread.cc:80

#9  0x00007f152de22f4b in ?? () from /usr/lib64/libpthread.so.0

#10 0x00007f152dd5a7ef in clone () from /usr/lib64/libc.so.6

Thread 5 (LWP 553131):

#0  0x00007f152de28df2 in pthread_cond_timedwait () from /usr/lib64/libpthread.so.0

#1  0x000055f0ef131d7a in inline_mysql_cond_timedwait (that=0x55f0f0275520 <COND_timer>, mutex=0x55f0f0275560 <LOCK_timer>, src_file=0x55f0ef3b54b0 "/data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/mysys/thr_timer.c", src_line=321, abstime=0x7f152da2ee40) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/include/mysql/psi/mysql_thread.h:1270

#2  timer_handler (arg=<optimized out>) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/mysys/thr_timer.c:321

#3  0x00007f152de22f4b in ?? () from /usr/lib64/libpthread.so.0

#4  0x00007f152dd5a7ef in clone () from /usr/lib64/libc.so.6

Thread 4 (LWP 553133):

#0  0x00007f152dd55169 in syscall () from /usr/lib64/libc.so.6

#1  0x000055f0ef0c53e5 in my_getevents (min_nr=1, nr=256, ev=0x7f151f9bce70, ctx=<optimized out>) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/tpool/aio_linux.cc:107

#2  tpool::aio_linux::getevent_thread_routine (aio=0x55f0f1f059a0) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/tpool/aio_linux.cc:107

#3  0x00007f152e08cf80 in std::execute_native_thread_routine (__p=0x55f0f1f26140) at ../../../.././libstdc++-v3/src/c++11/thread.cc:80

#4  0x00007f152de22f4b in ?? () from /usr/lib64/libpthread.so.0

#5  0x00007f152dd5a7ef in clone () from /usr/lib64/libc.so.6

Thread 3 (LWP 553134):

#0  0x00007f152de28a3c in pthread_cond_wait () from /usr/lib64/libpthread.so.0

#1  0x000055f0ef00fa48 in buf_flush_page_cleaner () at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/storage/innobase/buf/buf0flu.cc:2287

#2  0x00007f152de22f4b in ?? () from /usr/lib64/libpthread.so.0

#3  0x00007f152dd5a7ef in clone () from /usr/lib64/libc.so.6

--Type <RET> for more, q to quit, c to continue without paging--

Thread 2 (LWP 553132):

#0  0x00007f152de28df2 in pthread_cond_timedwait () from /usr/lib64/libpthread.so.0

#1  0x000055f0eed9e159 in inline_mysql_cond_timedwait (src_file=0x55f0ef3387b0 "/data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/storage/maria/ma_servicethread.c", src_line=115, abstime=0x7f15205c0dc0, mutex=0x55f0f01da0e0 <LOCK_checkpoint>, that=0x55f0f01da0a0 <COND_checkpoint>) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/include/mysql/psi/mysql_thread.h:1270

#2  my_service_thread_sleep (control=control@entry=0x55f0ef917360 <checkpoint_control>, sleep_time=sleep_time@entry=30000000000) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/storage/maria/ma_servicethread.c:115

#3  0x000055f0eed964cf in ma_checkpoint_background (arg=0x1e) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/storage/maria/ma_checkpoint.c:725

#4  0x00007f152de22f4b in ?? () from /usr/lib64/libpthread.so.0

#5  0x00007f152dd5a7ef in clone () from /usr/lib64/libc.so.6

Thread 1 (LWP 553129):

#0  0x00007f152de29ff1 in pthread_kill () from /usr/lib64/libpthread.so.0

#1  0x000055f0ef12da19 in my_write_core (sig=<optimized out>) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/mysys/stacktrace.c:424

#2  0x000055f0eeb88d20 in handle_fatal_signal (sig=11) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/signal_handler.cc:344

#3  <signal handler called>

#4  0x00007f152dd51fdf in select () from /usr/lib64/libc.so.6

#5  0x000055f0ef12b7bf in my_sleep (m_seconds=m_seconds@entry=1000) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/mysys/my_sleep.c:29

#6  0x000055f0ee8b57f2 in close_connections () at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/mysqld.cc:1805

#7  mysqld_main (argc=<optimized out>, argv=<optimized out>) at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/mysqld.cc:5749

#8  0x00007f152dc87c57 in __libc_start_main () from /usr/lib64/libc.so.6

#9  0x000055f0ee8a98ee in _start () at /data/fuxi_ci_workspace/6357e590f2f6990b8bd018b0/sql/mysqld.cc:4388