2021-03-18  1:06:37 0 [Warning] InnoDB: A long semaphore wait:
--Thread 140349926676224 has waited at lock0lock.cc line 3882 for 241.00 seconds the semaphore:
Mutex at 0x5587b08404c0, Mutex LOCK_SYS created lock0lock.cc:461, lock var 2
...
2021-03-18  1:18:29 0 [ERROR] [FATAL] InnoDB: Semaphore wait has lasted > 600 seconds. We intentionally crash the server because it appears to be hung.
210318  1:18:29 [ERROR] mysqld got signal 6 ;
...

# grep ^Thread gdb.thread.apply.all.bt  | wc -l
432

# grep '#1 .*_lock' gdb.thread.apply.all.bt
#1  0x00007fc58cdd019c in __GI___pthread_mutex_lock (mutex=mutex@entry=0x7fa36801afe8) at ../nptl/pthread_mutex_lock.c:135

# grep '#[0-9] .*mutex=[^<]' gdb.thread.apply.all.bt |
  sed -e 's/.*mutex=\(mutex@entry=\)\?\(0x[0-9a-f]*\).*/\2/' |
  sort -u |
  sed -e 'i-ex' -e 's/.*/print\/x &/p' -e 'i-ex' -e 's/print\/x \(.*\)/print *(pthread_mutex_t*)\1/' |
  xargs -d'\n' gdb `which mysqld` core.20779-batch

Listing:

  [Current thread is 1 (Thread 0x7fa6094ad700 (LWP 20974))]
  $1 = 0x5587b0812d20
  $2 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 66, __kind = 3, __spins = 87, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "B\000\000\000\003\000\000\000W", '\000' <repeats 18 times>, __align = 0}
  $3 = 0x5587b083b6a0
  $4 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 3, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001\000\000\000\003", '\000' <repeats 22 times>, __align = 0}
  $5 = 0x5587b083f498
  $6 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 3, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001\000\000\000\003", '\000' <repeats 22 times>, __align = 0}
  $7 = 0x5587b1095160
  $8 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}
  $9 = 0x5587b10a4b80
  $10 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 3, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001\000\000\000\003", '\000' <repeats 22 times>, __align = 0}
  $11 = 0x5587b19822d8
  $12 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}
  $13 = 0x5587b198bbb8
  $14 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}
  $15 = 0x5587b198c280
  $16 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 28, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\034", '\000' <repeats 26 times>, __align = 0}
  $17 = 0x5587b198c320
  $18 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 5, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\005", '\000' <repeats 26 times>, __align = 0}
  $19 = 0x5587b1ba9cd0
  $20 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}
  $21 = 0x5587b1ba9d40
  $22 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}
  $23 = 0x5587b1ba9db0
  $24 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}
  $25 = 0x5587b1ba9e20
  $26 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}
  $27 = 0x5587b1ba9f00
  $28 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}
  $29 = 0x5587b1ba9f70
  $30 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}
  $31 = 0x5587b1baa050
  $32 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}
  $33 = 0x5587b1c6b330
  $34 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}
  $35 = 0x5587b1c6b3a0
  $36 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}
  $37 = 0x5587b1c6b410
  $38 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}
  $39 = 0x5587b1c6b480
  $40 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}
  $41 = 0x5587b1c6b4f0
  $42 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}
  $43 = 0x5587b1c6b560
  $44 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}
  $45 = 0x5587b1c6b5d0
  $46 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}
  $47 = 0x5587b1c6b640
  $48 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}
  $49 = 0x5587b1c6b6b0
  $50 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}
  $51 = 0x5587b1c6b720
  $52 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}
  $53 = 0x5587e19cc350
  $54 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}
  $55 = 0x55881a038e70
  $56 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 55, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\067", '\000' <repeats 26 times>, __align = 0}
  $57 = 0x55881a03a200
  $58 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 3, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\003", '\000' <repeats 26 times>, __align = 0}
  $59 = 0x55881a05b160
  $60 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}
  $61 = 0x55881a0d9250
  $62 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}
  $63 = 0x7fa36401e0f0
  $64 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}
  $65 = 0x7fa36801afe8
  $66 = {__data = {__lock = 2, __count = 0, __owner = 21062, __nusers = 1, __kind = 3, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = "\002\000\000\000\000\000\000\000FR\000\000\001\000\000\000\003", '\000' <repeats 22 times>, __align = 2}
  $67 = 0x7fa5c404b090
  $68 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}
  $69 = 0x7fa5eaffcda0
  $70 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 12 times>, "\001", '\000' <repeats 26 times>, __align = 0}

That 0x7fa36801afe8 in is thread 170:

  Thread 170 (Thread 0x7fa5e8ca5700 (LWP 19772)):
  #0  __lll_lock_wait () at ../sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
  #1  0x00007fc58cdd019c in __GI___pthread_mutex_lock (mutex=mutex@entry=0x7fa36801afe8) at ../nptl/pthread_mutex_lock.c:135
  #2  0x00005587af8c3600 in inline_mysql_mutex_lock (src_file=0x5587aff40518 "/home/buildbot/buildbot/build/mariadb-10.3.28/sql/wsrep_thd.cc", src_line=761, that=0x7fa36801afe8) at ./include/mysql/psi/mysql_thread.h:719
  #3  wsrep_thd_is_BF (thd=thd@entry=0x7fa368019568, sync=sync@entry=1 '\001') at ./sql/wsrep_thd.cc:761
  #4  0x00005587af71bd68 in thd_need_ordering_with (thd=0x7fa530000c08, other_thd=0x7fa368019568) at ./sql/sql_class.cc:4990
  #5  0x00005587afb0db39 in lock_rec_has_to_wait (for_locking=<optimized out>, lock_is_on_supremum=<optimized out>, lock2=<optimized out>, type_mode=<optimized out>, trx=<optimized out>) at ./storage/innobase/lock/lock0lock.cc:788
  #6  lock_has_to_wait (lock1=0x7fb8bb961c28, lock2=0x7fb8bb99ae10) at ./storage/innobase/lock/lock0lock.cc:869
  #7  0x00005587afb0ddc9 in DeadlockChecker::search (this=this@entry=0x7fa5e8ca0810) at ./storage/innobase/lock/lock0lock.cc:6811
  #8  0x00005587afb12dac in DeadlockChecker::check_and_resolve (lock=lock@entry=0x7fb8bb99af68, trx=trx@entry=0x7fb8bb99abf0) at ./storage/innobase/lock/lock0lock.cc:6955
  #9  0x00005587afb13253 in lock_rec_enqueue_waiting (c_lock=c_lock@entry=0x7fb8bb961ad0, type_mode=type_mode@entry=2563, block=block@entry=0x7fb03f9b8740, heap_no=heap_no@entry=182, index=index@entry=0x7fa4b40ed338, thr=thr@entry=0x7fa568070a68, prdt=0x0) at ./storage/innobase/lock/lock0lock.cc:1739
  #10 0x00005587afb13902 in lock_rec_insert_check_and_lock (flags=flags@entry=0, rec=<optimized out>, block=0x7fb03f9b8740, index=index@entry=0x7fa4b40ed338, thr=thr@entry=0x7fa568070a68, mtr=0x7fa5e8ca1d60, inherit=<optimized out>) at ./storage/innobase/lock/lock0lock.cc:5324
  #11 0x00005587afc31007 in btr_cur_ins_lock_and_undo (flags=flags@entry=0, cursor=cursor@entry=0x7fa5e8ca0f50, entry=entry@entry=0x7fa568070588, thr=thr@entry=0x7fa568070a68, mtr=mtr@entry=0x7fa5e8ca1d60, inherit=inherit@entry=0x7fa5e8ca0bf7) at ./storage/innobase/btr/btr0cur.cc:3200
  #12 0x00005587afc3e0c2 in btr_cur_optimistic_insert (flags=flags@entry=0, cursor=cursor@entry=0x7fa5e8ca0f50, offsets=offsets@entry=0x7fa5e8ca0e78, heap=heap@entry=0x7fa5e8ca0e68, entry=entry@entry=0x7fa568070588, rec=rec@entry=0x7fa5e8ca0f00, big_rec=0x7fa5e8ca0ff0, n_ext=<optimized out>, thr=0x7fa568070a68, mtr=0x7fa5e8ca1d60) at ./storage/innobase/btr/btr0cur.cc:3437
  #13 0x00005587afb79c8c in row_ins_sec_index_entry_low (flags=flags@entry=0, mode=mode@entry=2, index=index@entry=0x7fa4b40ed338, offsets_heap=<optimized out>, offsets_heap@entry=0x7fa368028120, heap=heap@entry=0x7fa368063fe0, entry=entry@entry=0x7fa568070588, trx_id=<optimized out>, thr=<optimized out>) at ./storage/innobase/row/row0ins.cc:3103
  #14 0x00005587afb7f81c in row_ins_sec_index_entry (index=index@entry=0x7fa4b40ed338, entry=0x7fa568070588, thr=thr@entry=0x7fa568070a68, check_foreign=check_foreign@entry=true) at ./storage/innobase/row/row0ins.cc:3287
  #15 0x00005587afb7fce9 in row_ins_index_entry (thr=0x7fa568070a68, entry=<optimized out>, index=<optimized out>) at ./storage/innobase/row/row0ins.cc:3334
  #16 row_ins_index_entry_step (thr=0x7fa568070a68, node=0x7fa56806abb0) at ./storage/innobase/row/row0ins.cc:3483
  #17 row_ins (thr=<optimized out>, node=0x7fa56806abb0) at ./storage/innobase/row/row0ins.cc:3642
  #18 row_ins_step (thr=thr@entry=0x7fa568070a68) at ./storage/innobase/row/row0ins.cc:3788
  #19 0x00005587afb93538 in row_insert_for_mysql (mysql_rec=mysql_rec@entry=0x7fa568068560 "\364\006@", prebuilt=0x7fa56806a4d8, ins_mode=<optimized out>) at ./storage/innobase/row/row0mysql.cc:1421
  #20 0x00005587afad52e8 in ha_innobase::write_row (this=0x7fa56806e330, record=0x7fa568068560 "\364\006@") at ./storage/innobase/handler/ha_innodb.cc:8014
  #21 0x00005587af94dcfe in handler::ha_write_row (this=0x7fa56806e330, buf=0x7fa568068560 "\364\006@") at ./sql/handler.cc:6476
  #22 0x00005587af72fde0 in write_record (thd=thd@entry=0x7fa368019568, table=table@entry=0x7fa568067988, info=info@entry=0x7fa5e8ca2850) at ./sql/sql_insert.cc:2038
  #23 0x00005587af73a354 in mysql_insert (thd=thd@entry=0x7fa368019568, table_list=<optimized out>, fields=..., values_list=..., update_fields=..., update_values=..., duplic=DUP_ERROR, ignore=false) at ./sql/sql_insert.cc:1072
  #24 0x00005587af760553 in mysql_execute_command (thd=<optimized out>) at ./sql/sql_parse.cc:4482
  #25 0x00005587af77a7ae in Prepared_statement::execute (this=this@entry=0x7fa3680321e8, expanded_query=expanded_query@entry=0x7fa5e8ca4410, open_cursor=open_cursor@entry=false) at ./sql/sql_prepare.cc:5007
  #26 0x00005587af77a8d7 in Prepared_statement::execute_loop (this=0x7fa3680321e8, expanded_query=0x7fa5e8ca4410, open_cursor=<optimized out>, packet=<optimized out>, packet_end=<optimized out>) at ./sql/sql_prepare.cc:4435
  #27 0x00005587af77b94b in mysql_stmt_execute_common (thd=thd@entry=0x7fa368019568, stmt_id=11, packet=packet@entry=0x7fa3680d8a22 "\b", packet_end=packet_end@entry=0x7fa3680e5a67 "", cursor_flags=0, bulk_op=bulk_op@entry=false, read_types=false) at ./sql/sql_prepare.cc:3435
  #28 0x00005587af77ba1a in mysqld_stmt_execute (thd=thd@entry=0x7fa368019568, packet_arg=packet_arg@entry=0x7fa3680d8a19 "\v", packet_length=packet_length@entry=53326) at ./sql/sql_prepare.cc:3225
  #29 0x00005587af768242 in dispatch_command (command=command@entry=COM_STMT_EXECUTE, thd=thd@entry=0x7fa368019568, packet=packet@entry=0x7fa3680d8a19 "\v", packet_length=packet_length@entry=53326, is_com_multi=is_com_multi@entry=false, is_next_command=is_next_command@entry=false) at ./sql/sql_parse.cc:1795
  #30 0x00005587af769ba6 in do_command (thd=0x7fa368019568) at ./sql/sql_parse.cc:1398
  #31 0x00005587af83e097 in do_handle_one_connection (connect=connect@entry=0x55881a131cd8) at ./sql/sql_connect.cc:1403
  #32 0x00005587af83e28d in handle_one_connection (arg=arg@entry=0x55881a131cd8) at ./sql/sql_connect.cc:1308
  #33 0x00005587afe5612a in pfs_spawn_thread (arg=0x55881a131f38) at ./storage/perfschema/pfs.cc:1869
  #34 0x00007fc58cdcd6db in start_thread (arg=0x7fa5e8ca5700) at pthread_create.c:463
  #35 0x00007fc58c3cf71f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

Thread 170 is That is waiting for 0x7fa36801afe8:

  (gdb) print *(pthread_mutex_t*)0x7fa36801afe8
  $2 = {__data = {__lock = 2, __count = 0, __owner = 21062, __nusers = 1, __kind = 3, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}},
    __size = "\002\000\000\000\000\000\000\000FR\000\000\001\000\000\000\003", '\000' <repeats 22 times>, __align = 2}

  (gdb) thread find 21062
  Thread 68 has target id 'Thread 0x7fa5ea7fc700 (LWP 21062)'

Thread 68:

  (gdb) thread 68
  [Switching to thread 68 (Thread 0x7fa5ea7fc700 (LWP 21062))]
  #0  0x00007fc58cdd3ad3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x55881a038ec4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  88	../sysdeps/unix/sysv/linux/futex-internal.h: No such file or directory.

  (gdb) bt
  #0  0x00007fc58cdd3ad3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x55881a038ec4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  #1  __pthread_cond_wait_common (abstime=0x0, mutex=0x55881a038e70, cond=0x55881a038e98) at pthread_cond_wait.c:502
  #2  __pthread_cond_wait (cond=cond@entry=0x55881a038e98, mutex=mutex@entry=0x55881a038e70) at pthread_cond_wait.c:655
  #3  0x00005587afb3f230 in os_event::wait (this=0x55881a038e60) at ./storage/innobase/os/os0event.cc:158
  #4  os_event::wait_low (reset_sig_count=8, this=0x55881a038e60) at ./storage/innobase/os/os0event.cc:325
  #5  os_event_wait_low (event=0x55881a038e60, reset_sig_count=<optimized out>) at ./storage/innobase/os/os0event.cc:502
  #6  0x00005587afbdb82c in sync_array_wait_event (arr=0x5587b1ad5430, cell=@0x7fa5ea7fbcd8: 0x5587b1ad56b0) at ./storage/innobase/sync/sync0arr.cc:471
  #7  0x00005587afadccb7 in TTASEventMutex<GenericPolicy>::enter (line=18772, 
      filename=0x5587b0044130 "/home/buildbot/buildbot/build/mariadb-10.3.28/storage/innobase/handler/ha_innodb.cc", max_delay=4, max_spins=<optimized out>, 
      this=0x5587b08404c0 <lock_sys+64>) at ./storage/innobase/include/ib0mutex.h:471
  #8  PolicyMutex<TTASEventMutex<GenericPolicy> >::enter (this=0x5587b08404c0 <lock_sys+64>, n_spins=30, n_delay=4, 
      name=name@entry=0x5587b0044130 "/home/buildbot/buildbot/build/mariadb-10.3.28/storage/innobase/handler/ha_innodb.cc", line=line@entry=18772)
      at ./storage/innobase/include/ib0mutex.h:592
  #9  0x00005587afad0798 in bg_wsrep_kill_trx (void_arg=0x7fa530046ea0) at ./storage/innobase/handler/ha_innodb.cc:18772
  #10 0x00005587af7565d3 in handle_manager (arg=arg@entry=0x0) at ./sql/sql_manager.cc:112
  #11 0x00005587afe5612a in pfs_spawn_thread (arg=0x55881a187138) at ./storage/perfschema/pfs.cc:1869
  #12 0x00007fc58cdcd6db in start_thread (arg=0x7fa5ea7fc700) at pthread_create.c:463
  #13 0x00007fc58c3cf71f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

Thread 68 is waiting for a condition on 0x55881a038e70, and so are 55 threads:

  # grep '__pthread_cond_wait_common.*0x55881a038e70' gdb.thread.apply.all.bt  | wc -l
  55

  (gdb) print *(pthread_mutex_t*)0x55881a038e70
  $1 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 55, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, 
    __size = '\000' <repeats 12 times>, "\067", '\000' <repeats 26 times>, __align = 0}