Uploaded image for project: 'MariaDB Server'
  1. MariaDB Server
  2. MDEV-35149

Race condition around SET GLOBAL innodb_lru_scan_depth

Details

    Description

      Random order repeating the following SQL in 5000 threads (use --max_connections=10000):

      CREATE TABLE t2 (c1 INTEGER);
      CREATE TEMPORARY TABLE t1(c1 LONGTEXT NULL);
      DROP TABLE t1;
      DROP TABLE t2;
      INSERT INTO t1  VALUES('a');
      SET @@global.innodb_immediate_scrub_data_uncompressed=0;
      SET @@global.innodb_immediate_scrub_data_uncompressed=1;
      SET @@global.innodb_lru_scan_depth = 1536;
      SET @@global.innodb_lru_scan_depth = 86400;
      XA COMMIT 'a';
      XA END 'a';
      XA PREPARE 'a';
      XA START 'a' RESUME;
      XA START 'a';
      

      Will (after approx 1 to 10 minutes on a fast instance) lead to any of the following:

      CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug)

      Core was generated by `/test/MD141024-mariadb-11.7.0-linux-x86_64-dbg/bin/mariadbd --no-defaults --max'.
      Program terminated with signal SIGSEGV, Segmentation fault.
      #0  std::_Rb_tree_insert_and_rebalance (__insert_left=<optimized out>, __x=0x146ee4033100, __x@entry=0x1472740244a0, __p=<optimized out>, __header=...) at ../../../../../src/libstdc++-v3/src/c++98/tree.cc:282
       
      [Current thread is 1 (LWP 4073354)]
      (gdb) bt
      #0  std::_Rb_tree_insert_and_rebalance (__insert_left=<optimized out>, __x=0x146ee4033100, __x@entry=0x1472740244a0, __p=<optimized out>, __header=@0x5636b6ac5910: {_M_color = std::_S_red, _M_parent = 0x0, _M_left = 0x5636b6ac5910, _M_right = 0x5636b6ac5910})at ../../../../../src/libstdc++-v3/src/c++98/tree.cc:282
      #1  0x00005636b3fd1bd4 in std::_Rb_tree<range_t, range_t, std::_Identity<range_t>, range_compare, std::allocator<range_t> >::_M_insert_node (__z=<optimized out>, __p=<optimized out>, __x=<optimized out>, this=<optimized out>) at /usr/include/c++/13/bits/stl_tree.h:2385
      #2  std::_Rb_tree<range_t, range_t, std::_Identity<range_t>, range_compare, std::allocator<range_t> >::_Auto_node::_M_insert (__p=<optimized out>, this=<optimized out>) at /usr/include/c++/13/bits/stl_tree.h:1657
      #3  std::_Rb_tree<range_t, range_t, std::_Identity<range_t>, range_compare, std::allocator<range_t> >::_M_emplace_unique<range_t&> (this=this@entry=0x5636b6ac5908) at /usr/include/c++/13/bits/stl_tree.h:2437
      #4  0x00005636b3fd36af in std::__cxx1998::set<range_t, range_compare, std::allocator<range_t> >::emplace<range_t&> (this=0x5636b6ac5908)at /usr/include/c++/13/bits/stl_set.h:465
      #5  std::__debug::set<range_t, range_compare, std::allocator<range_t> >::emplace<range_t&> (this=0x5636b6ac58f0) at /usr/include/c++/13/debug/set.h:233
      #6  range_set::add_range (this=0x5636b6ac58f0, new_range={first = 3825414400, last = 5230})at /test/11.7_dbg/storage/innobase/include/fil0fil.h:221
      #7  0x00005636b3ff5c84 in fil_space_t::add_free_range (range=<optimized out>, this=<optimized out>)at /test/11.7_dbg/storage/innobase/include/fil0fil.h:908
      #8  mtr_t::commit (this=this@entry=0x14739fbb1540)at /test/11.7_dbg/storage/innobase/mtr/mtr0mtr.cc:517
      #9  0x00005636b41828c9 in btr_drop_temporary_table (table=@0x147184019ee8: {id = 11919, id_hash = 0x0, name = {m_name = 0x147274025828 "tmp/#sql-temptable-3dfb3d-32-192"}, name_hash = 0x0, heap = 0x1470400305b8, data_dir_path = 0x0, space = 0x5636b6ac57e8, space_id = 4294967294, flags = 33, flags2 = 65, skip_alter_undo = 0, file_unreadable = 0, cached = 1, n_def = 4, n_cols = 4, n_t_cols = 4, n_t_def = 4, n_v_def = 0, n_v_cols = 0, persistent_autoinc = 0, can_be_evicted = 0, corrupted = 0, drop_aborted = 0, cols = 0x147274072c28, v_cols = 0x147274072ca8, s_cols = 0x0, instant = 0x0, col_names = 0x147274072d00 "c1", v_col_names = 0x0, vers_start = 0, vers_end = 0, is_system_db = false, dict_frm_mismatch = DICT_FRM_CONSISTENT, fts_doc_id_index = 0x0, indexes = {count = 1, start = 0x147288061558, end = 0x147288061558, node = &dict_index_t::indexes, init = 51966}, freed_indexes = {count = 0, start = 0x0, end = 0x0, node = &dict_index_t::indexes, init = 51966}, foreign_list = {count = 0, start = 0x0, end = 0x0, node = &dict_foreign_t::heap, init = 0}, referenced_list = {count = 0, start = 0x0, end = 0x0, node = &dict_foreign_t::heap, init = 0}, table_LRU = {prev = 0x14724c020ab8, next = 0x1471000239c8}, fk_max_recusive_level = 0 '\000', def_trx_id = 0, bulk_trx_id = {m = std::atomic<unsigned long> = { 0 }}, mdl_name = {m_name = 0x147274025828 "tmp/#sql-temptable-3dfb3d-32-192"}, foreign_set = std::__debug::set with 0 elements, referenced_set = std::__debug::set with 0 elements, stat_initialized = 1, stats_last_recalc = 0, stat_persistent = 4, stats_auto_recalc = 0, stats_sample_pages = 0, stat_n_rows = 2, stat_clustered_index_size = 1, stat_sum_of_other_index_sizes = 0, stat_modified_counter = 2, stats_error_printed = false, autoinc_lock = 0x147274072ca8, autoinc_mutex = {lock = std::atomic<unsigned int> = { 0 }, static HOLDER = 1, static WAITER = 2}, lock_latch = {<srw_lock_impl<false>> = {pfs_psi = 0x0, lock = {writer = {lock = std::atomic<unsigned int> = { 0 }, static HOLDER = 1, static WAITER = 2}, readers = std::atomic<unsigned int> = { 0 }, static WRITER = 2147483648}}, writer = std::atomic<unsigned long> = { 0 }, readers_lock = {lock = std::atomic<unsigned int> = { 0 }, static HOLDER = 1, static WAITER = 2}, readers = std::atomic<std::__debug::unordered_multiset<unsigned long, std::hash<unsigned long>, std::equal_to<unsigned long>, std::allocator<unsigned long> > *> = { 0x14727404ded0 }}, row_id = {m_counter = std::atomic<unsigned long> = { 2 }}, autoinc = 0, autoinc_trx = {m = std::atomic<const trx_t *> = { 0x0 }}, n_waiting_or_granted_auto_inc_locks = 0, n_lock_x_or_s = 0, fts = 0x0, quiesce = QUIESCE_NONE, n_rec_locks = {m_counter = std::atomic<unsigned int> = { 0 }}, n_ref_count = {m_counter = std::atomic<unsigned int> = { 0 }}, locks = {count = 0, start = 0x0, end = 0x0, node = &lock_table_t::locks, init = 51966}, update_time = {m = std::atomic<long> = { 1728943563 }}, query_cache_inv_trx_id = {m = std::atomic<unsigned long> = { 56750 }}, magic_n = 76333786, vc_templ = 0x0})at /test/11.7_dbg/storage/innobase/btr/btr0btr.cc:1132
      #10 0x00005636b3f139e6 in ha_innobase::delete_table (this=<optimized out>, name=<optimized out>)at /test/11.7_dbg/storage/innobase/handler/ha_innodb.cc:13452
      #11 0x00005636b3b9b675 in hton_drop_table (hton=<optimized out>, path=<optimized out>) at /test/11.7_dbg/sql/handler.cc:585
      #12 0x00005636b3af086f in THD::rm_temporary_table (this=<optimized out>, base=0x5636b63d2898, path=0x1472740729a8 "/test/MD141024-mariadb-11.7.0-linux-x86_64-dbg/tmp/#sql-temptable-3dfb3d-32-192") at /test/11.7_dbg/sql/temporary_tables.cc:707
      #13 0x00005636b3af0ff4 in THD::free_tmp_table_share (this=this@entry=0x147274000d58, share=share@entry=0x147274072318, delete_table=delete_table@entry=true)at /test/11.7_dbg/sql/temporary_tables.cc:1488
      #14 0x00005636b3af2819 in THD::drop_temporary_table (this=this@entry=0x147274000d58, table=<optimized out>, is_trans=is_trans@entry=0x14739fbb2477, delete_table=delete_table@entry=true)at /test/11.7_dbg/sql/temporary_tables.cc:675
      #15 0x00005636b394dede in mysql_rm_table_no_locks (thd=thd@entry=0x147274000d58, tables=tables@entry=0x14727401aac8, current_db=current_db@entry=0x147274000df8, ddl_log_state=0x14739fbb24d0, ddl_log_state@entry=0x0, if_exists=if_exists@entry=false, drop_temporary=drop_temporary@entry=false, drop_view=false, drop_sequence=false, dont_log_query=false, dont_free_locks=false)at /test/11.7_dbg/sql/sql_table.cc:1462
      #16 0x00005636b395047f in mysql_rm_table (thd=thd@entry=0x147274000d58, tables=tables@entry=0x14727401aac8, if_exists=false, drop_temporary=false, drop_sequence=false, dont_log_query=dont_log_query@entry=false)at /test/11.7_dbg/sql/sql_table.cc:1210
      #17 0x00005636b38686fb in mysql_execute_command (thd=thd@entry=0x147274000d58, is_called_from_prepared_stmt=is_called_from_prepared_stmt@entry=false)at /test/11.7_dbg/sql/sql_parse.cc:4771
      #18 0x00005636b386cd36 in mysql_parse (thd=thd@entry=0x147274000d58, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0x14739fbb3260)at /test/11.7_dbg/sql/sql_parse.cc:7885
      #19 0x00005636b386f1ee in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0x147274000d58, packet=packet@entry=0x14727400b339 "DROP TABLE t1", packet_length=packet_length@entry=13, blocking=blocking@entry=true)at /test/11.7_dbg/sql/sql_class.h:257
      #20 0x00005636b3871552 in do_command (thd=0x147274000d58, blocking=blocking@entry=true) at /test/11.7_dbg/sql/sql_parse.cc:1405
      #21 0x00005636b39eae5f in do_handle_one_connection (connect=<optimized out>, connect@entry=0x5636b6d3f438, put_in_cache=put_in_cache@entry=true)at /test/11.7_dbg/sql/sql_connect.cc:1448
      #22 0x00005636b39eb173 in handle_one_connection (arg=arg@entry=0x5636b6d3f438)at /test/11.7_dbg/sql/sql_connect.cc:1350
      #23 0x00005636b3e40bcf in pfs_spawn_thread (arg=0x5636b6dda548)at /test/11.7_dbg/storage/perfschema/pfs.cc:2198
      #24 0x00001473b549ca94 in start_thread (arg=<optimized out>)at ./nptl/pthread_create.c:447
      #25 0x00001473b5529c3c in clone3 ()at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78
      

      Or

      CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug)

      mariadbd: /test/11.7_dbg/storage/innobase/buf/buf0lru.cc:400: buf_block_t* buf_LRU_get_free_block(buf_LRU_get): Assertion `LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()' failed.
      

      CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug)

      Core was generated by `/test/MD141024-mariadb-11.7.0-linux-x86_64-dbg/bin/mariadbd --no-defaults --max'.
      Program terminated with signal SIGABRT, Aborted.
      Download failed: Invalid argument.  Continuing without source file ./nptl/./nptl/pthread_kill.c.
      #0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44
       
      [Current thread is 1 (LWP 2573058)]
      (gdb) bt
      #0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44
      #1  __pthread_kill_internal (signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:78
      #2  __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6)at ./nptl/pthread_kill.c:89
      #3  0x00001504d224526e in __GI_raise (sig=sig@entry=6)at ../sysdeps/posix/raise.c:26
      #4  0x00001504d22288ff in __GI_abort () at ./stdlib/abort.c:79
      #5  0x00001504d222881b in __assert_fail_base (fmt=0x1504d23d01e8 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=assertion@entry=0x563ddcc189b0 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=file@entry=0x563ddcc17f48 "/test/11.7_dbg/storage/innobase/buf/buf0lru.cc", line=line@entry=400, function=function@entry=0x563ddcc18a00 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)") at ./assert/assert.c:94
      #6  0x00001504d223b507 in __assert_fail (assertion=0x563ddcc189b0 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=0x563ddcc17f48 "/test/11.7_dbg/storage/innobase/buf/buf0lru.cc", line=400, function=0x563ddcc18a00 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)")at ./assert/assert.c:103
      #7  0x0000563ddc705a0b in buf_LRU_get_free_block (get=get@entry=have_no_mutex)at /test/11.7_dbg/storage/innobase/buf/buf0lru.cc:400
      #8  0x0000563ddc78ca98 in fsp_page_create (space=space@entry=0x563ddf5857e8, offset=offset@entry=137, mtr=mtr@entry=0x1504d01fbd70)at /test/11.7_dbg/storage/innobase/fsp/fsp0fsp.cc:1059
      #9  0x0000563ddc791256 in fsp_alloc_free_page (space=space@entry=0x563ddf5857e8, hint=<optimized out>, hint@entry=0, mtr=mtr@entry=0x1504d01fbd70, init_mtr=init_mtr@entry=0x1504d01fbd70, err=err@entry=0x1504d01fbcdc)at /test/11.7_dbg/storage/innobase/fsp/fsp0fsp.cc:1184
      #10 0x0000563ddc792820 in fseg_alloc_free_page_low (space=space@entry=0x563ddf5857e8, seg_inode=seg_inode@entry=0x1504b78fe8b2 "", iblock=iblock@entry=0x1504b7803190, hint=hint@entry=0, direction=direction@entry=111 'o', has_done_reservation=has_done_reservation@entry=true, mtr=0x1504d01fbd70, init_mtr=0x1504d01fbd70, err=0x1504d01fbcdc)at /test/11.7_dbg/storage/innobase/fsp/fsp0fsp.cc:2167
      #11 0x0000563ddc793fb0 in fseg_create (space=space@entry=0x563ddf5857e8, byte_offset=byte_offset@entry=60, mtr=mtr@entry=0x1504d01fbd70, err=err@entry=0x1504d01fbcdc, has_done_reservation=has_done_reservation@entry=true, block=<optimized out>, block@entry=0x0)at /test/11.7_dbg/storage/innobase/fsp/fsp0fsp.cc:1759
      #12 0x0000563ddc677264 in trx_undo_seg_create (space=0x563ddf5857e8, rseg_hdr=rseg_hdr@entry=0x1504b76005b0, id=id@entry=0x1504d01fbbc8, err=err@entry=0x1504d01fbcdc, mtr=mtr@entry=0x1504d01fbd70)at /test/11.7_dbg/storage/innobase/trx/trx0undo.cc:493
      #13 0x0000563ddc6773cb in trx_undo_create (trx=trx@entry=0x1504d0207f00, rseg=rseg@entry=0x563ddd50ab80 <trx_sys+14976>, undo=undo@entry=0x1504d02088c0, err=err@entry=0x1504d01fbcdc, mtr=mtr@entry=0x1504d01fbd70)at /test/11.7_dbg/storage/innobase/trx/trx0undo.cc:1218
      #14 0x0000563ddc678a99 in trx_undo_assign_low<true> (trx=trx@entry=0x1504d0207f00, rseg=rseg@entry=0x563ddd50ab80 <trx_sys+14976>, undo=undo@entry=0x1504d02088c0, mtr=mtr@entry=0x1504d01fbd70, err=err@entry=0x1504d01fbcdc)at /test/11.7_dbg/storage/innobase/trx/trx0undo.cc:1421
      #15 0x0000563ddc64f8f4 in trx_undo_report_row_operation (thr=thr@entry=0x1503f004ae68, index=index@entry=0x1503f00216d8, clust_entry=clust_entry@entry=0x1503f0028608, update=update@entry=0x0, cmpl_info=cmpl_info@entry=0, rec=rec@entry=0x0, offsets=0x0, roll_ptr=0x1504d01fc1a0)at /test/11.7_dbg/storage/innobase/trx/trx0rec.cc:1890
      #16 0x0000563ddc6a60be in btr_cur_ins_lock_and_undo (flags=flags@entry=2, cursor=cursor@entry=0x1504d01fc680, entry=entry@entry=0x1503f0028608, thr=thr@entry=0x1503f004ae68, mtr=mtr@entry=0x1504d01fcc00, inherit=inherit@entry=0x1504d01fc286)at /test/11.7_dbg/storage/innobase/btr/btr0cur.cc:2200
      #17 0x0000563ddc6ad07a in btr_cur_optimistic_insert (flags=flags@entry=2, cursor=cursor@entry=0x1504d01fc680, offsets=offsets@entry=0x1504d01fc668, heap=heap@entry=0x1504d01fc660, entry=entry@entry=0x1503f0028608, rec=rec@entry=0x1504d01fc678, big_rec=0x1504d01fc658, n_ext=<optimized out>, thr=0x1503f004ae68, mtr=0x1504d01fcc00)at /test/11.7_dbg/storage/innobase/btr/btr0cur.cc:2424
      #18 0x0000563ddc58d350 in row_ins_clust_index_entry_low (flags=flags@entry=2, mode=mode@entry=BTR_MODIFY_LEAF, index=index@entry=0x1503f00216d8, n_uniq=n_uniq@entry=0, entry=entry@entry=0x1503f0028608, n_ext=n_ext@entry=0, thr=0x1503f004ae68)at /test/11.7_dbg/storage/innobase/row/row0ins.cc:2882
      #19 0x0000563ddc58eaca in row_ins_clust_index_entry (index=index@entry=0x1503f00216d8, entry=entry@entry=0x1503f0028608, thr=thr@entry=0x1503f004ae68, n_ext=n_ext@entry=0)at /test/11.7_dbg/storage/innobase/row/row0ins.cc:3267
      #20 0x0000563ddc59096f in row_ins_index_entry (thr=0x1503f004ae68, entry=0x1503f0028608, index=0x1503f00216d8)at /test/11.7_dbg/storage/innobase/row/row0ins.cc:3402
      #21 row_ins_index_entry_step (thr=0x1503f004ae68, node=0x1503f004ac08)at /test/11.7_dbg/storage/innobase/row/row0ins.cc:3570
      #22 row_ins (thr=0x1503f004ae68, node=0x1503f004ac08)at /test/11.7_dbg/storage/innobase/row/row0ins.cc:3687
      #23 row_ins_step (thr=thr@entry=0x1503f004ae68)at /test/11.7_dbg/storage/innobase/row/row0ins.cc:3816
      #24 0x0000563ddc5b6f85 in row_insert_for_mysql (mysql_rec=mysql_rec@entry=0x1503f004ef38 "\376\001", prebuilt=0x1503f004a778, ins_mode=ins_mode@entry=ROW_INS_NORMAL)at /test/11.7_dbg/storage/innobase/row/row0mysql.cc:1290
      #25 0x0000563ddc40e057 in ha_innobase::write_row (this=0x1503f004d250, record=0x1503f004ef38 "\376\001")at /test/11.7_dbg/storage/innobase/handler/ha_innodb.cc:7767
      #26 0x0000563ddc09a995 in handler::ha_write_row (this=0x1503f004d250, buf=0x1503f004ef38 "\376\001") at /test/11.7_dbg/sql/handler.cc:8086
      #27 0x0000563ddbd1e4ef in write_record (thd=thd@entry=0x1503f0000d58, table=table@entry=0x1503f0049c98, info=info@entry=0x1504d01fda50, sink=sink@entry=0x0) at /test/11.7_dbg/sql/sql_insert.cc:2320
      #28 0x0000563ddbd2928b in mysql_insert (thd=thd@entry=0x1503f0000d58, table_list=0x1503f001aaf8, fields=@0x1503f0006198: {<base_list> = {<Sql_alloc> = {<No data fields>}, first = 0x563ddd5cc500 <end_of_list>, last = 0x1503f0006198, elements = 0}, <No data fields>}, values_list=@0x1503f00061e0: {<base_list> = {<Sql_alloc> = {<No data fields>}, first = 0x1503f001b780, last = 0x1503f001b780, elements = 1}, <No data fields>}, update_fields=@0x1503f00061c8: {<base_list> = {<Sql_alloc> = {<No data fields>}, first = 0x563ddd5cc500 <end_of_list>, last = 0x1503f00061c8, elements = 0}, <No data fields>}, update_values=@0x1503f00061b0: {<base_list> = {<Sql_alloc> = {<No data fields>}, first = 0x563ddd5cc500 <end_of_list>, last = 0x1503f00061b0, elements = 0}, <No data fields>}, duplic=DUP_ERROR, ignore=false, result=0x0)at /test/11.7_dbg/sql/sql_insert.cc:1177
      #29 0x0000563ddbd659f8 in mysql_execute_command (thd=thd@entry=0x1503f0000d58, is_called_from_prepared_stmt=is_called_from_prepared_stmt@entry=false)at /test/11.7_dbg/sql/sql_parse.cc:4459
      #30 0x0000563ddbd6ad36 in mysql_parse (thd=thd@entry=0x1503f0000d58, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0x1504d01fe260)at /test/11.7_dbg/sql/sql_parse.cc:7885
      #31 0x0000563ddbd6d1ee in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0x1503f0000d58, packet=packet@entry=0x1503f000b339 "INSERT INTO t1  VALUES('a')", packet_length=packet_length@entry=27, blocking=blocking@entry=true)at /test/11.7_dbg/sql/sql_class.h:257
      #32 0x0000563ddbd6f552 in do_command (thd=0x1503f0000d58, blocking=blocking@entry=true) at /test/11.7_dbg/sql/sql_parse.cc:1405
      #33 0x0000563ddbee8e5f in do_handle_one_connection (connect=<optimized out>, connect@entry=0x563ddf79dd58, put_in_cache=put_in_cache@entry=true)at /test/11.7_dbg/sql/sql_connect.cc:1448
      #34 0x0000563ddbee9173 in handle_one_connection (arg=arg@entry=0x563ddf79dd58)at /test/11.7_dbg/sql/sql_connect.cc:1350
      #35 0x0000563ddc33ebcf in pfs_spawn_thread (arg=0x563ddf79ee88)at /test/11.7_dbg/storage/perfschema/pfs.cc:2198
      #36 0x00001504d229ca94 in start_thread (arg=<optimized out>)at ./nptl/pthread_create.c:447
      #37 0x00001504d2329c3c in clone3 ()at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78
      

      Or

      CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug)

      Core was generated by `/test/MD141024-mariadb-11.7.0-linux-x86_64-dbg/bin/mariadbd --no-defaults --max'.
      Program terminated with signal SIGABRT, Aborted.
      Download failed: Invalid argument.  Continuing without source file ./nptl/./nptl/pthread_kill.c.
      #0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44
       
      [Current thread is 1 (LWP 3056780)]
      (gdb) bt
      #0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44
      #1  __pthread_kill_internal (signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:78
      #2  __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6)at ./nptl/pthread_kill.c:89
      #3  0x000014d48904526e in __GI_raise (sig=sig@entry=6)at ../sysdeps/posix/raise.c:26
      #4  0x000014d4890288ff in __GI_abort () at ./stdlib/abort.c:79
      #5  0x000014d4894a9133 in __gnu_debug::_Error_formatter::_M_error (this=this@entry=0x5588d33ec7e0 <__gnu_debug::_Error_formatter::_S_at(char const*, unsigned int, char const*)::__formatter>)at ../../../../../src/libstdc++-v3/src/c++11/debug.cc:1320
      #6  0x00005588d22ebb28 in __gnu_debug::operator== (__lhs={first = 142, last = 143}, __rhs={first = 0, last = 0})at /usr/include/c++/13/debug/safe_iterator.h:472
      #7  0x00005588d22f4681 in range_set::add_range (this=0x5588d4e198f0, new_range={first = 3056780, last = 0})at /test/11.7_dbg/storage/innobase/include/fil0fil.h:234
      #8  0x00005588d2316c84 in fil_space_t::add_free_range (range=<optimized out>, this=<optimized out>)at /test/11.7_dbg/storage/innobase/include/fil0fil.h:908
      #9  mtr_t::commit (this=this@entry=0x14d473bb1540)at /test/11.7_dbg/storage/innobase/mtr/mtr0mtr.cc:517
      #10 0x00005588d24a38c9 in btr_drop_temporary_table (table=Python Exception <class 'AttributeError'>: 'NoneType' object has no attribute 'pointer'
      Python Exception <class 'AttributeError'>: 'NoneType' object has no attribute 'pointer'
      @0x14d3b00284c8: {id = 468, id_hash = 0x0, name = {m_name = 0x14d3a80274d8 "tmp/#sql-temptable-2e7672-1a-47"}, name_hash = 0x0, heap = 0x14d3b4042bc8, data_dir_path = 0x0, space = 0x5588d4e197e8, space_id = 4294967294, flags = 33, flags2 = 65, skip_alter_undo = 0, file_unreadable = 0, cached = 1, n_def = 4, n_cols = 4, n_t_cols = 4, n_t_def = 4, n_v_def = 0, n_v_cols = 0, persistent_autoinc = 0, can_be_evicted = 0, corrupted = 0, drop_aborted = 0, cols = 0x14d3a803e0e8, v_cols = 0x14d3a803e168, s_cols = 0x0, instant = 0x0, col_names = 0x14d3a803e1c0 "c1", v_col_names = 0x0, vers_start = 0, vers_end = 0, is_system_db = false, dict_frm_mismatch = DICT_FRM_CONSISTENT, fts_doc_id_index = 0x0, indexes = {count = 1, start = 0x14d3b40216d8, end = 0x14d3b40216d8, node = &dict_index_t::indexes, init = 51966}, freed_indexes = {count = 0, start = 0x0, end = 0x0, node = &dict_index_t::indexes, init = 51966}, foreign_list = {count = 0, start = 0x0, end = 0x0, node = &dict_foreign_t::heap, init = 0}, referenced_list = {count = 0, start = 0x0, end = 0x0, node = &dict_foreign_t::heap, init = 0}, table_LRU = {prev = 0x14d3f802a5b8, next = 0x14d3bc0405d8}, fk_max_recusive_level = 0 '\000', def_trx_id = 0, bulk_trx_id = {m = std::atomic<unsigned long> = { 0 }}, mdl_name = {m_name = 0x14d3a80274d8 "tmp/#sql-temptable-2e7672-1a-47"}, foreign_set = std::__debug::set with 0 elements, referenced_set = std::__debug::set with 0 elements, stat_initialized = 1, stats_last_recalc = 0, stat_persistent = 4, stats_auto_recalc = 0, stats_sample_pages = 0, stat_n_rows = 3, stat_clustered_index_size = 1, stat_sum_of_other_index_sizes = 0, stat_modified_counter = 3, stats_error_printed = false, autoinc_lock = 0x14d3a803e168, autoinc_mutex = {lock = std::atomic<unsigned int> = { 0 }, static HOLDER = 1, static WAITER = 2}, lock_latch = {<srw_lock_impl<false>> = {pfs_psi = 0x0, lock = {writer = {lock = std::atomic<unsigned int> = { 0 }, static HOLDER = 1, static WAITER = 2}, readers = std::atomic<unsigned int> = { 0 }, static WRITER = 2147483648}}, writer = std::atomic<unsigned long> = { 0 }, readers_lock = {lock = std::atomic<unsigned int> = { 0 }, static HOLDER = 1, static WAITER = 2}, readers = std::atomic<class std::__debug::unordered_multiset<unsigned long, std::hash<unsigned long>, std::equal_to<unsigned long>, std::allocator<unsigned long> > *> = { 0x14d3a803c4f0 }}, row_id = {m_counter = std::atomic<unsigned long> = { 3 }}, autoinc = 0, autoinc_trx = {m = std::atomic<const struct trx_t *> = { 0x0 }}, n_waiting_or_granted_auto_inc_locks = 0, n_lock_x_or_s = 0, fts = 0x0, quiesce = QUIESCE_NONE, n_rec_locks = {m_counter = std::atomic<unsigned int> = { 0 }}, n_ref_count = {m_counter = std::atomic<unsigned int> = { 0 }}, locks = {count = 0, start = 0x0, end = 0x0, node = &lock_table_t::locks, init = 51966}, update_time = {m = std::atomic<long> = { 1728942858 }}, query_cache_inv_trx_id = {m = std::atomic<unsigned long> = { 2448 }}, magic_n = 76333786, vc_templ = 0x0})at /test/11.7_dbg/storage/innobase/btr/btr0btr.cc:1132
      #11 0x00005588d22349e6 in ha_innobase::delete_table (this=<optimized out>, name=<optimized out>)at /test/11.7_dbg/storage/innobase/handler/ha_innodb.cc:13452
      #12 0x00005588d1ebc675 in hton_drop_table (hton=<optimized out>, path=<optimized out>) at /test/11.7_dbg/sql/handler.cc:585
      #13 0x00005588d1e1186f in THD::rm_temporary_table (this=<optimized out>, base=0x5588d4726898, path=0x14d3a803b568 "/test/MD141024-mariadb-11.7.0-linux-x86_64-dbg/tmp/#sql-temptable-2e7672-1a-47") at /test/11.7_dbg/sql/temporary_tables.cc:707
      #14 0x00005588d1e11ff4 in THD::free_tmp_table_share (this=this@entry=0x14d3a8000d58, share=share@entry=0x14d3a803aed8, delete_table=delete_table@entry=true)at /test/11.7_dbg/sql/temporary_tables.cc:1488
      #15 0x00005588d1e13819 in THD::drop_temporary_table (this=this@entry=0x14d3a8000d58, table=<optimized out>, is_trans=is_trans@entry=0x14d473bb2477, delete_table=delete_table@entry=true)at /test/11.7_dbg/sql/temporary_tables.cc:675
      #16 0x00005588d1c6eede in mysql_rm_table_no_locks (thd=thd@entry=0x14d3a8000d58, tables=tables@entry=0x14d3a801aac8, current_db=current_db@entry=0x14d3a8000df8, ddl_log_state=0x14d473bb24d0, ddl_log_state@entry=0x0, if_exists=if_exists@entry=false, drop_temporary=drop_temporary@entry=false, drop_view=false, drop_sequence=false, dont_log_query=false, dont_free_locks=false)at /test/11.7_dbg/sql/sql_table.cc:1462
      #17 0x00005588d1c7147f in mysql_rm_table (thd=thd@entry=0x14d3a8000d58, tables=tables@entry=0x14d3a801aac8, if_exists=false, drop_temporary=false, drop_sequence=false, dont_log_query=dont_log_query@entry=false)at /test/11.7_dbg/sql/sql_table.cc:1210
      #18 0x00005588d1b896fb in mysql_execute_command (thd=thd@entry=0x14d3a8000d58, is_called_from_prepared_stmt=is_called_from_prepared_stmt@entry=false)at /test/11.7_dbg/sql/sql_parse.cc:4771
      #19 0x00005588d1b8dd36 in mysql_parse (thd=thd@entry=0x14d3a8000d58, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0x14d473bb3260)at /test/11.7_dbg/sql/sql_parse.cc:7885
      #20 0x00005588d1b901ee in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0x14d3a8000d58, packet=packet@entry=0x14d3a800b339 "DROP TABLE t1", packet_length=packet_length@entry=13, blocking=blocking@entry=true)at /test/11.7_dbg/sql/sql_class.h:257
      #21 0x00005588d1b92552 in do_command (thd=0x14d3a8000d58, blocking=blocking@entry=true) at /test/11.7_dbg/sql/sql_parse.cc:1405
      #22 0x00005588d1d0be5f in do_handle_one_connection (connect=<optimized out>, connect@entry=0x5588d5093438, put_in_cache=put_in_cache@entry=true)at /test/11.7_dbg/sql/sql_connect.cc:1448
      #23 0x00005588d1d0c173 in handle_one_connection (arg=arg@entry=0x5588d5093438)at /test/11.7_dbg/sql/sql_connect.cc:1350
      #24 0x00005588d2161bcf in pfs_spawn_thread (arg=0x5588d512b5c8)at /test/11.7_dbg/storage/perfschema/pfs.cc:2198
      #25 0x000014d48909ca94 in start_thread (arg=<optimized out>)at ./nptl/pthread_create.c:447
      #26 0x000014d489129c3c in clone3 ()at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78
      

      Please note that the SIGABRT in __gnu_debug::_Error_formatter::_M_error is definitely single-SQL-input-thread reproducible, given sufficient iterations (given overall sporadicity).

      10.5 Does not seem to be affected.

      Attachments

        Activity

          Roel Roel Van de Paar added a comment - - edited

          Reproductions in various versions (one UniqueID/stack per line):

          CS 10.6.20 cd97caef84a842cf388866cfc0a0ec32b86a9c13 (Debug)

          LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()|SIGABRT|buf_LRU_get_free_block|buf_page_init_for_read|buf_read_page_low|buf_read_page
          

          CS 10.11.10 8a6a4c947a0ca3d2fdca752d7440bdc5c6c83e37 (Debug)

          LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()|SIGABRT|buf_LRU_get_free_block|fsp_header_init|fil_ibd_create|dict_create_index_space
          SIGABRT|__gnu_debug::_Error_formatter::_M_error|__gnu_debug::operator==|range_set::add_range|fil_space_t::add_free_range
          

          CS 11.2.6 12a91b57e27b979819924cf89614e6e51f24b37b (Debug)

          SIGABRT|__gnu_debug::_Error_formatter::_M_error|__gnu_debug::operator==|range_set::add_range|fil_space_t::add_free_range
          

          CS 11.4.4 b53b81e937b7b598ee73657501ec87c6f76334ec (Debug)

          LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()|SIGABRT|buf_LRU_get_free_block|buf_read_page|buf_page_get_gen|fseg_inode_try_get
          

          CS 11.6.2 4e1e9ea6f322dd8b7b7f4f15fa5f0d743f73ea74 (Debug)

          LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()|SIGABRT|buf_LRU_get_free_block|buf_read_page|buf_page_get_gen|btr_drop_temporary_table
          

          Note that the 10.11.10 stack also difers (ref below for other iterations)

          Roel Roel Van de Paar added a comment - - edited Reproductions in various versions (one UniqueID/stack per line): CS 10.6.20 cd97caef84a842cf388866cfc0a0ec32b86a9c13 (Debug) LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()|SIGABRT|buf_LRU_get_free_block|buf_page_init_for_read|buf_read_page_low|buf_read_page CS 10.11.10 8a6a4c947a0ca3d2fdca752d7440bdc5c6c83e37 (Debug) LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()|SIGABRT|buf_LRU_get_free_block|fsp_header_init|fil_ibd_create|dict_create_index_space SIGABRT|__gnu_debug::_Error_formatter::_M_error|__gnu_debug::operator==|range_set::add_range|fil_space_t::add_free_range CS 11.2.6 12a91b57e27b979819924cf89614e6e51f24b37b (Debug) SIGABRT|__gnu_debug::_Error_formatter::_M_error|__gnu_debug::operator==|range_set::add_range|fil_space_t::add_free_range CS 11.4.4 b53b81e937b7b598ee73657501ec87c6f76334ec (Debug) LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()|SIGABRT|buf_LRU_get_free_block|buf_read_page|buf_page_get_gen|fseg_inode_try_get CS 11.6.2 4e1e9ea6f322dd8b7b7f4f15fa5f0d743f73ea74 (Debug) LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()|SIGABRT|buf_LRU_get_free_block|buf_read_page|buf_page_get_gen|btr_drop_temporary_table Note that the 10.11.10 stack also difers (ref below for other iterations)

          Full 11.4 stack as it slightly differs:

          CS 11.4.4 b53b81e937b7b598ee73657501ec87c6f76334ec (Debug)

          mariadbd: /test/11.4_dbg/storage/innobase/buf/buf0lru.cc:400: buf_block_t* buf_LRU_get_free_block(buf_LRU_get): Assertion `LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()' failed.
          

          CS 11.4.4 b53b81e937b7b598ee73657501ec87c6f76334ec (Debug)

          Core was generated by `/test/MD141024-mariadb-11.4.4-linux-x86_64-dbg/bin/mariadbd --no-defaults --max'.
          Program terminated with signal SIGABRT, Aborted.
          Download failed: Invalid argument.  Continuing without source file ./nptl/./nptl/pthread_kill.c.
          #0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44
           
          [Current thread is 1 (LWP 465497)]
          (gdb) bt
          #0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44
          #1  __pthread_kill_internal (signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:78
          #2  __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6)at ./nptl/pthread_kill.c:89
          #3  0x0000127f0444526e in __GI_raise (sig=sig@entry=6)at ../sysdeps/posix/raise.c:26
          #4  0x0000127f044288ff in __GI_abort () at ./stdlib/abort.c:79
          #5  0x0000127f0442881b in __assert_fail_base (fmt=0x127f045d01e8 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=assertion@entry=0x5bd1f99933b0 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=file@entry=0x5bd1f9992948 "/test/11.4_dbg/storage/innobase/buf/buf0lru.cc", line=line@entry=400, function=function@entry=0x5bd1f9993400 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)") at ./assert/assert.c:94
          #6  0x0000127f0443b507 in __assert_fail (assertion=0x5bd1f99933b0 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=0x5bd1f9992948 "/test/11.4_dbg/storage/innobase/buf/buf0lru.cc", line=400, function=0x5bd1f9993400 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)")at ./assert/assert.c:103
          #7  0x00005bd1f949a6c1 in buf_LRU_get_free_block (get=get@entry=have_mutex)at /test/11.4_dbg/storage/innobase/buf/buf0lru.cc:400
          #8  0x00005bd1f949ddf5 in buf_read_page (page_id=page_id@entry={m_id = 18446744065119618873}, chain=@0x5bd1fb9349f8: {first = 0x0})at /test/11.4_dbg/storage/innobase/buf/buf0rea.cc:427
          #9  0x00005bd1f9472026 in buf_page_get_gen (page_id=<optimized out>, zip_size=0, rw_latch=rw_latch@entry=RW_SX_LATCH, guess=guess@entry=0x0, mode=mode@entry=16, mtr=mtr@entry=0x127e1c89d160, err=0x0)at /test/11.4_dbg/storage/innobase/buf/buf0buf.cc:2882
          #10 0x00005bd1f951f892 in fseg_inode_try_get (header=header@entry=0x127f0110c04a "\377\377\377\376", space=space@entry=4294967294, zip_size=<optimized out>, mtr=mtr@entry=0x127e1c89d160, block=block@entry=0x127e1c89d108, err=err@entry=0x0) at /test/11.4_dbg/storage/innobase/fsp/fsp0fsp.cc:1551
          #11 0x00005bd1f9529359 in fseg_free_step (header=0x127f0110c04a "\377\377\377\376", mtr=mtr@entry=0x127e1c89d160, ahi=ahi@entry=false) at /test/11.4_dbg/storage/innobase/fsp/fsp0fsp.cc:2857
          #12 0x00005bd1f94141cb in btr_free_but_not_root (block=block@entry=0x127f010034d0, log_mode=log_mode@entry=MTR_LOG_NO_REDO, ahi=ahi@entry=false) at /test/11.4_dbg/storage/innobase/btr/btr0btr.cc:1021
          #13 0x00005bd1f941555b in btr_drop_temporary_table (table=Python Exception <class 'AttributeError'>: 'NoneType' object has no attribute 'pointer'
          Python Exception <class 'AttributeError'>: 'NoneType' object has no attribute 'pointer'
          @0x127d701cbe18: {id = 4520, id_hash = 0x0, name = {m_name = 0x127d701cc128 "tmp/#sql-temptable-672bf-dc1-0"}, name_hash = 0x0, heap = 0x127d701cbc88, data_dir_path = 0x0, space = 0x5bd1fbba5548, space_id = 4294967294, flags = 33, flags2 = 65, skip_alter_undo = 0, file_unreadable = 0, cached = 1, n_def = 4, n_cols = 4, n_t_cols = 4, n_t_def = 4, n_v_def = 0, n_v_cols = 0, persistent_autoinc = 0, can_be_evicted = 0, corrupted = 0, drop_aborted = 0, cols = 0x127d701cc1e8, v_cols = 0x127d701cc268, s_cols = 0x0, instant = 0x0, col_names = 0x127d701cc2c0 "c1", v_col_names = 0x0, vers_start = 0, vers_end = 0, is_system_db = false, dict_frm_mismatch = DICT_FRM_CONSISTENT, fts_doc_id_index = 0x0, indexes = {count = 1, start = 0x127d701ccdf8, end = 0x127d701ccdf8, node = &dict_index_t::indexes, init = 51966}, freed_indexes = {count = 0, start = 0x0, end = 0x0, node = &dict_index_t::indexes, init = 51966}, foreign_list = {count = 0, start = 0x0, end = 0x0, node = &dict_foreign_t::heap, init = 0}, referenced_list = {count = 0, start = 0x0, end = 0x0, node = &dict_foreign_t::heap, init = 0}, table_LRU = {prev = 0x127ddc1c1c58, next = 0x127d401c4798}, fk_max_recusive_level = 0 '\000', def_trx_id = 0, bulk_trx_id = {m = std::atomic<unsigned long> = { 0 }}, mdl_name = {m_name = 0x127d701cc128 "tmp/#sql-temptable-672bf-dc1-0"}, foreign_set = std::__debug::set with 0 elements, referenced_set = std::__debug::set with 0 elements, stat_initialized = 1, stats_last_recalc = 0, stat_persistent = 4, stats_auto_recalc = 0, stats_sample_pages = 0, stat_n_rows = 1, stat_clustered_index_size = 1, stat_sum_of_other_index_sizes = 0, stat_modified_counter = 1, stats_error_printed = false, autoinc_lock = 0x127d701cc268, autoinc_mutex = {lock = std::atomic<unsigned int> = { 0 }, static HOLDER = 1, static WAITER = 2}, lock_latch = {<srw_lock_impl<false>> = {pfs_psi = 0x0, lock = {writer = {lock = std::atomic<unsigned int> = { 0 }, static HOLDER = 1, static WAITER = 2}, readers = std::atomic<unsigned int> = { 0 }, static WRITER = 2147483648}}, writer = std::atomic<unsigned long> = { 0 }, readers_lock = {lock = std::atomic<unsigned int> = { 0 }, static HOLDER = 1, static WAITER = 2}, readers = std::atomic<class std::__debug::unordered_multiset<unsigned long, std::hash<unsigned long>, std::equal_to<unsigned long>, std::allocator<unsigned long> > *> = { 0x5bd1fc41b4f0 }}, row_id = {m_counter = std::atomic<unsigned long> = { 1 }}, autoinc = 0, autoinc_trx = {m = std::atomic<const struct trx_t *> = { 0x0 }}, n_waiting_or_granted_auto_inc_locks = 0, n_lock_x_or_s = 0, fts = 0x0, quiesce = QUIESCE_NONE, n_rec_locks = {m_counter = std::atomic<unsigned int> = { 0 }}, n_ref_count = {m_counter = std::atomic<unsigned int> = { 0 }}, locks = {count = 0, start = 0x0, end = 0x0, node = &lock_table_t::locks, init = 51966}, update_time = {m = std::atomic<long> = { 1728956557 }}, query_cache_inv_trx_id = {m = std::atomic<unsigned long> = { 15761 }}, magic_n = 76333786, vc_templ = 0x0})at /test/11.4_dbg/storage/innobase/btr/btr0btr.cc:1129
          #14 0x00005bd1f91a69ab in ha_innobase::delete_table (this=<optimized out>, name=<optimized out>)at /test/11.4_dbg/storage/innobase/handler/ha_innodb.cc:13501
          #15 0x00005bd1f8e376d3 in hton_drop_table (hton=<optimized out>, path=<optimized out>) at /test/11.4_dbg/sql/handler.cc:585
          #16 0x00005bd1f8d8840f in THD::rm_temporary_table (this=<optimized out>, base=0x5bd1fb673898, path=0x127d701ca8f8 "/test/MD141024-mariadb-11.4.4-linux-x86_64-dbg/tmp/#sql-temptable-672bf-dc1-0") at /test/11.4_dbg/sql/temporary_tables.cc:706
          #17 0x00005bd1f8d88b18 in THD::free_tmp_table_share (this=this@entry=0x127d701b1978, share=share@entry=0x127d701ca268, delete_table=delete_table@entry=true)at /test/11.4_dbg/sql/temporary_tables.cc:1487
          #18 0x00005bd1f8d8a30f in THD::drop_temporary_table (this=this@entry=0x127d701b1978, table=<optimized out>, is_trans=is_trans@entry=0x127e1c89e477, delete_table=delete_table@entry=true)at /test/11.4_dbg/sql/temporary_tables.cc:674
          #19 0x00005bd1f8bee1d1 in mysql_rm_table_no_locks (thd=thd@entry=0x127d701b1978, tables=tables@entry=0x127d701c4208, current_db=current_db@entry=0x127d701b1a18, ddl_log_state=0x127e1c89e4d0, ddl_log_state@entry=0x0, if_exists=if_exists@entry=false, drop_temporary=drop_temporary@entry=false, drop_view=false, drop_sequence=false, dont_log_query=false, dont_free_locks=false)at /test/11.4_dbg/sql/sql_table.cc:1458
          #20 0x00005bd1f8bf0772 in mysql_rm_table (thd=thd@entry=0x127d701b1978, tables=tables@entry=0x127d701c4208, if_exists=false, drop_temporary=false, drop_sequence=false, dont_log_query=dont_log_query@entry=false)at /test/11.4_dbg/sql/sql_table.cc:1206
          #21 0x00005bd1f8b0db97 in mysql_execute_command (thd=thd@entry=0x127d701b1978, is_called_from_prepared_stmt=is_called_from_prepared_stmt@entry=false)at /test/11.4_dbg/sql/sql_parse.cc:4768
          #22 0x00005bd1f8b121ab in mysql_parse (thd=thd@entry=0x127d701b1978, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0x127e1c89f2a0)at /test/11.4_dbg/sql/sql_parse.cc:7880
          #23 0x00005bd1f8b14663 in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0x127d701b1978, packet=packet@entry=0x127d701bbde9 "DROP TABLE t1", packet_length=packet_length@entry=13, blocking=blocking@entry=true)at /test/11.4_dbg/sql/sql_class.h:254
          #24 0x00005bd1f8b168a8 in do_command (thd=0x127d701b1978, blocking=blocking@entry=true) at /test/11.4_dbg/sql/sql_parse.cc:1407
          #25 0x00005bd1f8c865ab in do_handle_one_connection (connect=<optimized out>, connect@entry=0x5bd1fc41b508, put_in_cache=put_in_cache@entry=true)at /test/11.4_dbg/sql/sql_connect.cc:1439
          #26 0x00005bd1f8c868b3 in handle_one_connection (arg=arg@entry=0x5bd1fc41b508)at /test/11.4_dbg/sql/sql_connect.cc:1341
          #27 0x00005bd1f90d67c0 in pfs_spawn_thread (arg=0x5bd1fc41b578)at /test/11.4_dbg/storage/perfschema/pfs.cc:2201
          #28 0x0000127f0449ca94 in start_thread (arg=<optimized out>)at ./nptl/pthread_create.c:447
          #29 0x0000127f04529c3c in clone3 ()at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78
          

          Roel Roel Van de Paar added a comment - Full 11.4 stack as it slightly differs: CS 11.4.4 b53b81e937b7b598ee73657501ec87c6f76334ec (Debug) mariadbd: /test/11.4_dbg/storage/innobase/buf/buf0lru.cc:400: buf_block_t* buf_LRU_get_free_block(buf_LRU_get): Assertion `LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()' failed. CS 11.4.4 b53b81e937b7b598ee73657501ec87c6f76334ec (Debug) Core was generated by `/test/MD141024-mariadb-11.4.4-linux-x86_64-dbg/bin/mariadbd --no-defaults --max'. Program terminated with signal SIGABRT, Aborted. Download failed: Invalid argument. Continuing without source file ./nptl/./nptl/pthread_kill.c. #0 __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44   [Current thread is 1 (LWP 465497)] (gdb) bt #0 __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44 #1 __pthread_kill_internal (signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:78 #2 __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6)at ./nptl/pthread_kill.c:89 #3 0x0000127f0444526e in __GI_raise (sig=sig@entry=6)at ../sysdeps/posix/raise.c:26 #4 0x0000127f044288ff in __GI_abort () at ./stdlib/abort.c:79 #5 0x0000127f0442881b in __assert_fail_base (fmt=0x127f045d01e8 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=assertion@entry=0x5bd1f99933b0 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=file@entry=0x5bd1f9992948 "/test/11.4_dbg/storage/innobase/buf/buf0lru.cc", line=line@entry=400, function=function@entry=0x5bd1f9993400 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)") at ./assert/assert.c:94 #6 0x0000127f0443b507 in __assert_fail (assertion=0x5bd1f99933b0 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=0x5bd1f9992948 "/test/11.4_dbg/storage/innobase/buf/buf0lru.cc", line=400, function=0x5bd1f9993400 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)")at ./assert/assert.c:103 #7 0x00005bd1f949a6c1 in buf_LRU_get_free_block (get=get@entry=have_mutex)at /test/11.4_dbg/storage/innobase/buf/buf0lru.cc:400 #8 0x00005bd1f949ddf5 in buf_read_page (page_id=page_id@entry={m_id = 18446744065119618873}, chain=@0x5bd1fb9349f8: {first = 0x0})at /test/11.4_dbg/storage/innobase/buf/buf0rea.cc:427 #9 0x00005bd1f9472026 in buf_page_get_gen (page_id=<optimized out>, zip_size=0, rw_latch=rw_latch@entry=RW_SX_LATCH, guess=guess@entry=0x0, mode=mode@entry=16, mtr=mtr@entry=0x127e1c89d160, err=0x0)at /test/11.4_dbg/storage/innobase/buf/buf0buf.cc:2882 #10 0x00005bd1f951f892 in fseg_inode_try_get (header=header@entry=0x127f0110c04a "\377\377\377\376", space=space@entry=4294967294, zip_size=<optimized out>, mtr=mtr@entry=0x127e1c89d160, block=block@entry=0x127e1c89d108, err=err@entry=0x0) at /test/11.4_dbg/storage/innobase/fsp/fsp0fsp.cc:1551 #11 0x00005bd1f9529359 in fseg_free_step (header=0x127f0110c04a "\377\377\377\376", mtr=mtr@entry=0x127e1c89d160, ahi=ahi@entry=false) at /test/11.4_dbg/storage/innobase/fsp/fsp0fsp.cc:2857 #12 0x00005bd1f94141cb in btr_free_but_not_root (block=block@entry=0x127f010034d0, log_mode=log_mode@entry=MTR_LOG_NO_REDO, ahi=ahi@entry=false) at /test/11.4_dbg/storage/innobase/btr/btr0btr.cc:1021 #13 0x00005bd1f941555b in btr_drop_temporary_table (table=Python Exception <class 'AttributeError'>: 'NoneType' object has no attribute 'pointer' Python Exception <class 'AttributeError'>: 'NoneType' object has no attribute 'pointer' @0x127d701cbe18: {id = 4520, id_hash = 0x0, name = {m_name = 0x127d701cc128 "tmp/#sql-temptable-672bf-dc1-0"}, name_hash = 0x0, heap = 0x127d701cbc88, data_dir_path = 0x0, space = 0x5bd1fbba5548, space_id = 4294967294, flags = 33, flags2 = 65, skip_alter_undo = 0, file_unreadable = 0, cached = 1, n_def = 4, n_cols = 4, n_t_cols = 4, n_t_def = 4, n_v_def = 0, n_v_cols = 0, persistent_autoinc = 0, can_be_evicted = 0, corrupted = 0, drop_aborted = 0, cols = 0x127d701cc1e8, v_cols = 0x127d701cc268, s_cols = 0x0, instant = 0x0, col_names = 0x127d701cc2c0 "c1", v_col_names = 0x0, vers_start = 0, vers_end = 0, is_system_db = false, dict_frm_mismatch = DICT_FRM_CONSISTENT, fts_doc_id_index = 0x0, indexes = {count = 1, start = 0x127d701ccdf8, end = 0x127d701ccdf8, node = &dict_index_t::indexes, init = 51966}, freed_indexes = {count = 0, start = 0x0, end = 0x0, node = &dict_index_t::indexes, init = 51966}, foreign_list = {count = 0, start = 0x0, end = 0x0, node = &dict_foreign_t::heap, init = 0}, referenced_list = {count = 0, start = 0x0, end = 0x0, node = &dict_foreign_t::heap, init = 0}, table_LRU = {prev = 0x127ddc1c1c58, next = 0x127d401c4798}, fk_max_recusive_level = 0 '\000', def_trx_id = 0, bulk_trx_id = {m = std::atomic<unsigned long> = { 0 }}, mdl_name = {m_name = 0x127d701cc128 "tmp/#sql-temptable-672bf-dc1-0"}, foreign_set = std::__debug::set with 0 elements, referenced_set = std::__debug::set with 0 elements, stat_initialized = 1, stats_last_recalc = 0, stat_persistent = 4, stats_auto_recalc = 0, stats_sample_pages = 0, stat_n_rows = 1, stat_clustered_index_size = 1, stat_sum_of_other_index_sizes = 0, stat_modified_counter = 1, stats_error_printed = false, autoinc_lock = 0x127d701cc268, autoinc_mutex = {lock = std::atomic<unsigned int> = { 0 }, static HOLDER = 1, static WAITER = 2}, lock_latch = {<srw_lock_impl<false>> = {pfs_psi = 0x0, lock = {writer = {lock = std::atomic<unsigned int> = { 0 }, static HOLDER = 1, static WAITER = 2}, readers = std::atomic<unsigned int> = { 0 }, static WRITER = 2147483648}}, writer = std::atomic<unsigned long> = { 0 }, readers_lock = {lock = std::atomic<unsigned int> = { 0 }, static HOLDER = 1, static WAITER = 2}, readers = std::atomic<class std::__debug::unordered_multiset<unsigned long, std::hash<unsigned long>, std::equal_to<unsigned long>, std::allocator<unsigned long> > *> = { 0x5bd1fc41b4f0 }}, row_id = {m_counter = std::atomic<unsigned long> = { 1 }}, autoinc = 0, autoinc_trx = {m = std::atomic<const struct trx_t *> = { 0x0 }}, n_waiting_or_granted_auto_inc_locks = 0, n_lock_x_or_s = 0, fts = 0x0, quiesce = QUIESCE_NONE, n_rec_locks = {m_counter = std::atomic<unsigned int> = { 0 }}, n_ref_count = {m_counter = std::atomic<unsigned int> = { 0 }}, locks = {count = 0, start = 0x0, end = 0x0, node = &lock_table_t::locks, init = 51966}, update_time = {m = std::atomic<long> = { 1728956557 }}, query_cache_inv_trx_id = {m = std::atomic<unsigned long> = { 15761 }}, magic_n = 76333786, vc_templ = 0x0})at /test/11.4_dbg/storage/innobase/btr/btr0btr.cc:1129 #14 0x00005bd1f91a69ab in ha_innobase::delete_table (this=<optimized out>, name=<optimized out>)at /test/11.4_dbg/storage/innobase/handler/ha_innodb.cc:13501 #15 0x00005bd1f8e376d3 in hton_drop_table (hton=<optimized out>, path=<optimized out>) at /test/11.4_dbg/sql/handler.cc:585 #16 0x00005bd1f8d8840f in THD::rm_temporary_table (this=<optimized out>, base=0x5bd1fb673898, path=0x127d701ca8f8 "/test/MD141024-mariadb-11.4.4-linux-x86_64-dbg/tmp/#sql-temptable-672bf-dc1-0") at /test/11.4_dbg/sql/temporary_tables.cc:706 #17 0x00005bd1f8d88b18 in THD::free_tmp_table_share (this=this@entry=0x127d701b1978, share=share@entry=0x127d701ca268, delete_table=delete_table@entry=true)at /test/11.4_dbg/sql/temporary_tables.cc:1487 #18 0x00005bd1f8d8a30f in THD::drop_temporary_table (this=this@entry=0x127d701b1978, table=<optimized out>, is_trans=is_trans@entry=0x127e1c89e477, delete_table=delete_table@entry=true)at /test/11.4_dbg/sql/temporary_tables.cc:674 #19 0x00005bd1f8bee1d1 in mysql_rm_table_no_locks (thd=thd@entry=0x127d701b1978, tables=tables@entry=0x127d701c4208, current_db=current_db@entry=0x127d701b1a18, ddl_log_state=0x127e1c89e4d0, ddl_log_state@entry=0x0, if_exists=if_exists@entry=false, drop_temporary=drop_temporary@entry=false, drop_view=false, drop_sequence=false, dont_log_query=false, dont_free_locks=false)at /test/11.4_dbg/sql/sql_table.cc:1458 #20 0x00005bd1f8bf0772 in mysql_rm_table (thd=thd@entry=0x127d701b1978, tables=tables@entry=0x127d701c4208, if_exists=false, drop_temporary=false, drop_sequence=false, dont_log_query=dont_log_query@entry=false)at /test/11.4_dbg/sql/sql_table.cc:1206 #21 0x00005bd1f8b0db97 in mysql_execute_command (thd=thd@entry=0x127d701b1978, is_called_from_prepared_stmt=is_called_from_prepared_stmt@entry=false)at /test/11.4_dbg/sql/sql_parse.cc:4768 #22 0x00005bd1f8b121ab in mysql_parse (thd=thd@entry=0x127d701b1978, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0x127e1c89f2a0)at /test/11.4_dbg/sql/sql_parse.cc:7880 #23 0x00005bd1f8b14663 in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0x127d701b1978, packet=packet@entry=0x127d701bbde9 "DROP TABLE t1", packet_length=packet_length@entry=13, blocking=blocking@entry=true)at /test/11.4_dbg/sql/sql_class.h:254 #24 0x00005bd1f8b168a8 in do_command (thd=0x127d701b1978, blocking=blocking@entry=true) at /test/11.4_dbg/sql/sql_parse.cc:1407 #25 0x00005bd1f8c865ab in do_handle_one_connection (connect=<optimized out>, connect@entry=0x5bd1fc41b508, put_in_cache=put_in_cache@entry=true)at /test/11.4_dbg/sql/sql_connect.cc:1439 #26 0x00005bd1f8c868b3 in handle_one_connection (arg=arg@entry=0x5bd1fc41b508)at /test/11.4_dbg/sql/sql_connect.cc:1341 #27 0x00005bd1f90d67c0 in pfs_spawn_thread (arg=0x5bd1fc41b578)at /test/11.4_dbg/storage/perfschema/pfs.cc:2201 #28 0x0000127f0449ca94 in start_thread (arg=<optimized out>)at ./nptl/pthread_create.c:447 #29 0x0000127f04529c3c in clone3 ()at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78

          Full 11.4 stack as it slightly differs:

          CS 11.6.2 4e1e9ea6f322dd8b7b7f4f15fa5f0d743f73ea74 (Debug)

          mariadbd: /test/11.6_dbg/storage/innobase/buf/buf0lru.cc:400: buf_block_t* buf_LRU_get_free_block(buf_LRU_get): Assertion `LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()' failed.
          

          CS 11.6.2 4e1e9ea6f322dd8b7b7f4f15fa5f0d743f73ea74 (Debug)

          Core was generated by `/test/MD141024-mariadb-11.6.2-linux-x86_64-dbg/bin/mariadbd --no-defaults --max'.
          Program terminated with signal SIGABRT, Aborted.
          Download failed: Invalid argument.  Continuing without source file ./nptl/./nptl/pthread_kill.c.
          #0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44
           
          [Current thread is 1 (LWP 671854)]
          (gdb) bt
          #0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44
          #1  __pthread_kill_internal (signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:78
          #2  __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6)at ./nptl/pthread_kill.c:89
          #3  0x00000b6c68c4526e in __GI_raise (sig=sig@entry=6)at ../sysdeps/posix/raise.c:26
          #4  0x00000b6c68c288ff in __GI_abort () at ./stdlib/abort.c:79
          #5  0x00000b6c68c2881b in __assert_fail_base (fmt=0xb6c68dd01e8 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=assertion@entry=0x607b23384878 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=file@entry=0x607b23383e10 "/test/11.6_dbg/storage/innobase/buf/buf0lru.cc", line=line@entry=400, function=function@entry=0x607b233848c8 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)") at ./assert/assert.c:94
          #6  0x00000b6c68c3b507 in __assert_fail (assertion=0x607b23384878 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=0x607b23383e10 "/test/11.6_dbg/storage/innobase/buf/buf0lru.cc", line=400, function=0x607b233848c8 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)")at ./assert/assert.c:103
          #7  0x0000607b22e71ccd in buf_LRU_get_free_block (get=get@entry=have_mutex)at /test/11.6_dbg/storage/innobase/buf/buf0lru.cc:400
          #8  0x0000607b22e75401 in buf_read_page (page_id=page_id@entry={m_id = 18446744065119619602}, chain=@0x607b25330238: {first = 0x0})at /test/11.6_dbg/storage/innobase/buf/buf0rea.cc:427
          #9  0x0000607b22e49626 in buf_page_get_gen (page_id=<optimized out>, zip_size=zip_size@entry=0, rw_latch=rw_latch@entry=RW_X_LATCH, guess=guess@entry=0x0, mode=mode@entry=10, mtr=mtr@entry=0xb6bd48e8540, err=0x0) at /test/11.6_dbg/storage/innobase/buf/buf0buf.cc:2882
          #10 0x0000607b22decbdb in btr_drop_temporary_table (table=Python Exception <class 'AttributeError'>: 'NoneType' object has no attribute 'pointer'
          Python Exception <class 'AttributeError'>: 'NoneType' object has no attribute 'pointer'
          @0xb6bac2fb148: {id = 12050, id_hash = 0x0, name = {m_name = 0xb6bac02ac18 "tmp/#sql-temptable-a0dea-7db-0"}, name_hash = 0x0, heap = 0xb6bac0e6828, data_dir_path = 0x0, space = 0x607b2559b9d8, space_id = 4294967294, flags = 33, flags2 = 65, skip_alter_undo = 0, file_unreadable = 0, cached = 1, n_def = 4, n_cols = 4, n_t_cols = 4, n_t_def = 4, n_v_def = 0, n_v_cols = 0, persistent_autoinc = 0, can_be_evicted = 0, corrupted = 0, drop_aborted = 0, cols = 0xb6bac2fb4d8, v_cols = 0xb6bac2fb558, s_cols = 0x0, instant = 0x0, col_names = 0xb6bac2fb5b0 "c1", v_col_names = 0x0, vers_start = 0, vers_end = 0, is_system_db = false, dict_frm_mismatch = DICT_FRM_CONSISTENT, fts_doc_id_index = 0x0, indexes = {count = 1, start = 0xb6bac2fbc58, end = 0xb6bac2fbc58, node = &dict_index_t::indexes, init = 51966}, freed_indexes = {count = 0, start = 0x0, end = 0x0, node = &dict_index_t::indexes, init = 51966}, foreign_list = {count = 0, start = 0x0, end = 0x0, node = &dict_foreign_t::heap, init = 0}, referenced_list = {count = 0, start = 0x0, end = 0x0, node = &dict_foreign_t::heap, init = 0}, table_LRU = {prev = 0xb6bc42d1838, next = 0xb6b700b0c18}, fk_max_recusive_level = 0 '\000', def_trx_id = 0, bulk_trx_id = {m = std::atomic<unsigned long> = { 0 }}, mdl_name = {m_name = 0xb6bac02ac18 "tmp/#sql-temptable-a0dea-7db-0"}, foreign_set = std::__debug::set with 0 elements, referenced_set = std::__debug::set with 0 elements, stat_initialized = 1, stats_last_recalc = 0, stat_persistent = 4, stats_auto_recalc = 0, stats_sample_pages = 0, stat_n_rows = 8, stat_clustered_index_size = 1, stat_sum_of_other_index_sizes = 0, stat_modified_counter = 8, stats_error_printed = false, autoinc_lock = 0xb6bac2fb558, autoinc_mutex = {lock = std::atomic<unsigned int> = { 0 }, static HOLDER = 1, static WAITER = 2}, lock_latch = {<srw_lock_impl<false>> = {pfs_psi = 0x0, lock = {writer = {lock = std::atomic<unsigned int> = { 0 }, static HOLDER = 1, static WAITER = 2}, readers = std::atomic<unsigned int> = { 0 }, static WRITER = 2147483648}}, writer = std::atomic<unsigned long> = { 0 }, readers_lock = {lock = std::atomic<unsigned int> = { 0 }, static HOLDER = 1, static WAITER = 2}, readers = std::atomic<class std::__debug::unordered_multiset<unsigned long, std::hash<unsigned long>, std::equal_to<unsigned long>, std::allocator<unsigned long> > *> = { 0x607b25bd8f20 }}, row_id = {m_counter = std::atomic<unsigned long> = { 8 }}, autoinc = 0, autoinc_trx = {m = std::atomic<const struct trx_t *> = { 0x0 }}, n_waiting_or_granted_auto_inc_locks = 0, n_lock_x_or_s = 0, fts = 0x0, quiesce = QUIESCE_NONE, n_rec_locks = {m_counter = std::atomic<unsigned int> = { 0 }}, n_ref_count = {m_counter = std::atomic<unsigned int> = { 0 }}, locks = {count = 0, start = 0x0, end = 0x0, node = &lock_table_t::locks, init = 51966}, update_time = {m = std::atomic<long> = { 1728958366 }}, query_cache_inv_trx_id = {m = std::atomic<unsigned long> = { 73135 }}, magic_n = 76333786, vc_templ = 0x0})at /test/11.6_dbg/storage/innobase/btr/btr0btr.cc:1125
          #11 0x0000607b22b7df68 in ha_innobase::delete_table (this=<optimized out>, name=<optimized out>)at /test/11.6_dbg/storage/innobase/handler/ha_innodb.cc:13447
          #12 0x0000607b228062d9 in hton_drop_table (hton=<optimized out>, path=<optimized out>) at /test/11.6_dbg/sql/handler.cc:585
          #13 0x0000607b2275b663 in THD::rm_temporary_table (this=<optimized out>, base=0x607b24ea8898, path=0xb6bac2f9df8 "/test/MD141024-mariadb-11.6.2-linux-x86_64-dbg/tmp/#sql-temptable-a0dea-7db-0") at /test/11.6_dbg/sql/temporary_tables.cc:707
          #14 0x0000607b2275bde8 in THD::free_tmp_table_share (this=this@entry=0xb6bac0f2ad8, share=share@entry=0xb6bac2f9768, delete_table=delete_table@entry=true)at /test/11.6_dbg/sql/temporary_tables.cc:1488
          #15 0x0000607b2275d60d in THD::drop_temporary_table (this=this@entry=0xb6bac0f2ad8, table=<optimized out>, is_trans=is_trans@entry=0xb6bd48e9477, delete_table=delete_table@entry=true)at /test/11.6_dbg/sql/temporary_tables.cc:675
          #16 0x0000607b225b8ed8 in mysql_rm_table_no_locks (thd=thd@entry=0xb6bac0f2ad8, tables=tables@entry=0xb6bac103be8, current_db=current_db@entry=0xb6bac0f2b78, ddl_log_state=0xb6bd48e94d0, ddl_log_state@entry=0x0, if_exists=if_exists@entry=false, drop_temporary=drop_temporary@entry=false, drop_view=false, drop_sequence=false, dont_log_query=false, dont_free_locks=false)at /test/11.6_dbg/sql/sql_table.cc:1462
          #17 0x0000607b225bb479 in mysql_rm_table (thd=thd@entry=0xb6bac0f2ad8, tables=tables@entry=0xb6bac103be8, if_exists=false, drop_temporary=false, drop_sequence=false, dont_log_query=dont_log_query@entry=false)at /test/11.6_dbg/sql/sql_table.cc:1210
          #18 0x0000607b224d36f5 in mysql_execute_command (thd=thd@entry=0xb6bac0f2ad8, is_called_from_prepared_stmt=is_called_from_prepared_stmt@entry=false)at /test/11.6_dbg/sql/sql_parse.cc:4771
          #19 0x0000607b224d7d30 in mysql_parse (thd=thd@entry=0xb6bac0f2ad8, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0xb6bd48ea260)at /test/11.6_dbg/sql/sql_parse.cc:7885
          #20 0x0000607b224da1e8 in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0xb6bac0f2ad8, packet=packet@entry=0xb6bac0f9aa9 "DROP TABLE t1", packet_length=packet_length@entry=13, blocking=blocking@entry=true)at /test/11.6_dbg/sql/sql_class.h:256
          #21 0x0000607b224dc54c in do_command (thd=0xb6bac0f2ad8, blocking=blocking@entry=true) at /test/11.6_dbg/sql/sql_parse.cc:1405
          #22 0x0000607b22655e59 in do_handle_one_connection (connect=<optimized out>, connect@entry=0x607b25bd8f38, put_in_cache=put_in_cache@entry=true)at /test/11.6_dbg/sql/sql_connect.cc:1448
          #23 0x0000607b2265616d in handle_one_connection (arg=arg@entry=0x607b25bd8f38)at /test/11.6_dbg/sql/sql_connect.cc:1350
          #24 0x0000607b22aab5f3 in pfs_spawn_thread (arg=0x607b25bd8fa8)at /test/11.6_dbg/storage/perfschema/pfs.cc:2198
          #25 0x00000b6c68c9ca94 in start_thread (arg=<optimized out>)at ./nptl/pthread_create.c:447
          #26 0x00000b6c68d29c3c in clone3 ()at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78
          

          Roel Roel Van de Paar added a comment - Full 11.4 stack as it slightly differs: CS 11.6.2 4e1e9ea6f322dd8b7b7f4f15fa5f0d743f73ea74 (Debug) mariadbd: /test/11.6_dbg/storage/innobase/buf/buf0lru.cc:400: buf_block_t* buf_LRU_get_free_block(buf_LRU_get): Assertion `LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()' failed. CS 11.6.2 4e1e9ea6f322dd8b7b7f4f15fa5f0d743f73ea74 (Debug) Core was generated by `/test/MD141024-mariadb-11.6.2-linux-x86_64-dbg/bin/mariadbd --no-defaults --max'. Program terminated with signal SIGABRT, Aborted. Download failed: Invalid argument. Continuing without source file ./nptl/./nptl/pthread_kill.c. #0 __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44   [Current thread is 1 (LWP 671854)] (gdb) bt #0 __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44 #1 __pthread_kill_internal (signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:78 #2 __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6)at ./nptl/pthread_kill.c:89 #3 0x00000b6c68c4526e in __GI_raise (sig=sig@entry=6)at ../sysdeps/posix/raise.c:26 #4 0x00000b6c68c288ff in __GI_abort () at ./stdlib/abort.c:79 #5 0x00000b6c68c2881b in __assert_fail_base (fmt=0xb6c68dd01e8 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=assertion@entry=0x607b23384878 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=file@entry=0x607b23383e10 "/test/11.6_dbg/storage/innobase/buf/buf0lru.cc", line=line@entry=400, function=function@entry=0x607b233848c8 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)") at ./assert/assert.c:94 #6 0x00000b6c68c3b507 in __assert_fail (assertion=0x607b23384878 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=0x607b23383e10 "/test/11.6_dbg/storage/innobase/buf/buf0lru.cc", line=400, function=0x607b233848c8 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)")at ./assert/assert.c:103 #7 0x0000607b22e71ccd in buf_LRU_get_free_block (get=get@entry=have_mutex)at /test/11.6_dbg/storage/innobase/buf/buf0lru.cc:400 #8 0x0000607b22e75401 in buf_read_page (page_id=page_id@entry={m_id = 18446744065119619602}, chain=@0x607b25330238: {first = 0x0})at /test/11.6_dbg/storage/innobase/buf/buf0rea.cc:427 #9 0x0000607b22e49626 in buf_page_get_gen (page_id=<optimized out>, zip_size=zip_size@entry=0, rw_latch=rw_latch@entry=RW_X_LATCH, guess=guess@entry=0x0, mode=mode@entry=10, mtr=mtr@entry=0xb6bd48e8540, err=0x0) at /test/11.6_dbg/storage/innobase/buf/buf0buf.cc:2882 #10 0x0000607b22decbdb in btr_drop_temporary_table (table=Python Exception <class 'AttributeError'>: 'NoneType' object has no attribute 'pointer' Python Exception <class 'AttributeError'>: 'NoneType' object has no attribute 'pointer' @0xb6bac2fb148: {id = 12050, id_hash = 0x0, name = {m_name = 0xb6bac02ac18 "tmp/#sql-temptable-a0dea-7db-0"}, name_hash = 0x0, heap = 0xb6bac0e6828, data_dir_path = 0x0, space = 0x607b2559b9d8, space_id = 4294967294, flags = 33, flags2 = 65, skip_alter_undo = 0, file_unreadable = 0, cached = 1, n_def = 4, n_cols = 4, n_t_cols = 4, n_t_def = 4, n_v_def = 0, n_v_cols = 0, persistent_autoinc = 0, can_be_evicted = 0, corrupted = 0, drop_aborted = 0, cols = 0xb6bac2fb4d8, v_cols = 0xb6bac2fb558, s_cols = 0x0, instant = 0x0, col_names = 0xb6bac2fb5b0 "c1", v_col_names = 0x0, vers_start = 0, vers_end = 0, is_system_db = false, dict_frm_mismatch = DICT_FRM_CONSISTENT, fts_doc_id_index = 0x0, indexes = {count = 1, start = 0xb6bac2fbc58, end = 0xb6bac2fbc58, node = &dict_index_t::indexes, init = 51966}, freed_indexes = {count = 0, start = 0x0, end = 0x0, node = &dict_index_t::indexes, init = 51966}, foreign_list = {count = 0, start = 0x0, end = 0x0, node = &dict_foreign_t::heap, init = 0}, referenced_list = {count = 0, start = 0x0, end = 0x0, node = &dict_foreign_t::heap, init = 0}, table_LRU = {prev = 0xb6bc42d1838, next = 0xb6b700b0c18}, fk_max_recusive_level = 0 '\000', def_trx_id = 0, bulk_trx_id = {m = std::atomic<unsigned long> = { 0 }}, mdl_name = {m_name = 0xb6bac02ac18 "tmp/#sql-temptable-a0dea-7db-0"}, foreign_set = std::__debug::set with 0 elements, referenced_set = std::__debug::set with 0 elements, stat_initialized = 1, stats_last_recalc = 0, stat_persistent = 4, stats_auto_recalc = 0, stats_sample_pages = 0, stat_n_rows = 8, stat_clustered_index_size = 1, stat_sum_of_other_index_sizes = 0, stat_modified_counter = 8, stats_error_printed = false, autoinc_lock = 0xb6bac2fb558, autoinc_mutex = {lock = std::atomic<unsigned int> = { 0 }, static HOLDER = 1, static WAITER = 2}, lock_latch = {<srw_lock_impl<false>> = {pfs_psi = 0x0, lock = {writer = {lock = std::atomic<unsigned int> = { 0 }, static HOLDER = 1, static WAITER = 2}, readers = std::atomic<unsigned int> = { 0 }, static WRITER = 2147483648}}, writer = std::atomic<unsigned long> = { 0 }, readers_lock = {lock = std::atomic<unsigned int> = { 0 }, static HOLDER = 1, static WAITER = 2}, readers = std::atomic<class std::__debug::unordered_multiset<unsigned long, std::hash<unsigned long>, std::equal_to<unsigned long>, std::allocator<unsigned long> > *> = { 0x607b25bd8f20 }}, row_id = {m_counter = std::atomic<unsigned long> = { 8 }}, autoinc = 0, autoinc_trx = {m = std::atomic<const struct trx_t *> = { 0x0 }}, n_waiting_or_granted_auto_inc_locks = 0, n_lock_x_or_s = 0, fts = 0x0, quiesce = QUIESCE_NONE, n_rec_locks = {m_counter = std::atomic<unsigned int> = { 0 }}, n_ref_count = {m_counter = std::atomic<unsigned int> = { 0 }}, locks = {count = 0, start = 0x0, end = 0x0, node = &lock_table_t::locks, init = 51966}, update_time = {m = std::atomic<long> = { 1728958366 }}, query_cache_inv_trx_id = {m = std::atomic<unsigned long> = { 73135 }}, magic_n = 76333786, vc_templ = 0x0})at /test/11.6_dbg/storage/innobase/btr/btr0btr.cc:1125 #11 0x0000607b22b7df68 in ha_innobase::delete_table (this=<optimized out>, name=<optimized out>)at /test/11.6_dbg/storage/innobase/handler/ha_innodb.cc:13447 #12 0x0000607b228062d9 in hton_drop_table (hton=<optimized out>, path=<optimized out>) at /test/11.6_dbg/sql/handler.cc:585 #13 0x0000607b2275b663 in THD::rm_temporary_table (this=<optimized out>, base=0x607b24ea8898, path=0xb6bac2f9df8 "/test/MD141024-mariadb-11.6.2-linux-x86_64-dbg/tmp/#sql-temptable-a0dea-7db-0") at /test/11.6_dbg/sql/temporary_tables.cc:707 #14 0x0000607b2275bde8 in THD::free_tmp_table_share (this=this@entry=0xb6bac0f2ad8, share=share@entry=0xb6bac2f9768, delete_table=delete_table@entry=true)at /test/11.6_dbg/sql/temporary_tables.cc:1488 #15 0x0000607b2275d60d in THD::drop_temporary_table (this=this@entry=0xb6bac0f2ad8, table=<optimized out>, is_trans=is_trans@entry=0xb6bd48e9477, delete_table=delete_table@entry=true)at /test/11.6_dbg/sql/temporary_tables.cc:675 #16 0x0000607b225b8ed8 in mysql_rm_table_no_locks (thd=thd@entry=0xb6bac0f2ad8, tables=tables@entry=0xb6bac103be8, current_db=current_db@entry=0xb6bac0f2b78, ddl_log_state=0xb6bd48e94d0, ddl_log_state@entry=0x0, if_exists=if_exists@entry=false, drop_temporary=drop_temporary@entry=false, drop_view=false, drop_sequence=false, dont_log_query=false, dont_free_locks=false)at /test/11.6_dbg/sql/sql_table.cc:1462 #17 0x0000607b225bb479 in mysql_rm_table (thd=thd@entry=0xb6bac0f2ad8, tables=tables@entry=0xb6bac103be8, if_exists=false, drop_temporary=false, drop_sequence=false, dont_log_query=dont_log_query@entry=false)at /test/11.6_dbg/sql/sql_table.cc:1210 #18 0x0000607b224d36f5 in mysql_execute_command (thd=thd@entry=0xb6bac0f2ad8, is_called_from_prepared_stmt=is_called_from_prepared_stmt@entry=false)at /test/11.6_dbg/sql/sql_parse.cc:4771 #19 0x0000607b224d7d30 in mysql_parse (thd=thd@entry=0xb6bac0f2ad8, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0xb6bd48ea260)at /test/11.6_dbg/sql/sql_parse.cc:7885 #20 0x0000607b224da1e8 in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0xb6bac0f2ad8, packet=packet@entry=0xb6bac0f9aa9 "DROP TABLE t1", packet_length=packet_length@entry=13, blocking=blocking@entry=true)at /test/11.6_dbg/sql/sql_class.h:256 #21 0x0000607b224dc54c in do_command (thd=0xb6bac0f2ad8, blocking=blocking@entry=true) at /test/11.6_dbg/sql/sql_parse.cc:1405 #22 0x0000607b22655e59 in do_handle_one_connection (connect=<optimized out>, connect@entry=0x607b25bd8f38, put_in_cache=put_in_cache@entry=true)at /test/11.6_dbg/sql/sql_connect.cc:1448 #23 0x0000607b2265616d in handle_one_connection (arg=arg@entry=0x607b25bd8f38)at /test/11.6_dbg/sql/sql_connect.cc:1350 #24 0x0000607b22aab5f3 in pfs_spawn_thread (arg=0x607b25bd8fa8)at /test/11.6_dbg/storage/perfschema/pfs.cc:2198 #25 0x00000b6c68c9ca94 in start_thread (arg=<optimized out>)at ./nptl/pthread_create.c:447 #26 0x00000b6c68d29c3c in clone3 ()at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78

          Would the failure go away if the TEMPORARY keyword is removed? If yes, this should share a root cause with MDEV-35046.

          Also, I wonder what would happen on a WITH_ASAN=ON build. If there is no heap-use-after-free reported, this could be something else than MDEV-35046.

          marko Marko Mäkelä added a comment - Would the failure go away if the TEMPORARY keyword is removed? If yes, this should share a root cause with MDEV-35046 . Also, I wonder what would happen on a WITH_ASAN=ON build. If there is no heap-use-after-free reported, this could be something else than MDEV-35046 .

          During testing without TEMPORARY,I checked the processlist and noticed nearly all 5k threads were in Opening tables/checking permissions state for a long time:

          CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug)

          | 3287 | root | localhost | test | Query   |  263 | Opening tables                  | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 3288 | root | localhost | test | Query   |  263 | Opening tables                  | INSERT INTO t1  VALUES('a')       |    0.000 |
          | 3289 | root | localhost | test | Query   |  263 | Opening tables                  | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 3291 | root | localhost | test | Query   |  263 | checking permissions            | DROP TABLE t1                     |    0.000 |
          | 3290 | root | localhost | test | Query   |  263 | checking permissions            | DROP TABLE t1                     |    0.000 |
          | 3292 | root | localhost | test | Query   |  263 | Opening tables                  | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 3293 | root | localhost | test | Query   |  263 | checking permissions            | DROP TABLE t1                     |    0.000 |
          

          With a few notable exceptions:

          CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug)

          |  247 | root | localhost | test | Query   |  263 | Query end                       | DROP TABLE t1                     |    0.000 |
          ...
          | 2235 | root | localhost | test | Query   |   78 | checking permissions            | DROP TABLE t1                     |    0.000 |  # 78s
          ...
          | 2565 | root | localhost | test | Query   |  261 | Waiting for table metadata lock | DROP TABLE t1                     |    0.000 |
          ...
          | 4815 | root | localhost | test | Query   |  240 | Waiting for table metadata lock | DROP TABLE t1                     |    0.000 |
          ...
          | 4824 | root | localhost | test | Query   |  240 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          ...
          | 4828 | root | localhost | test | Query   |  240 | Waiting for table metadata lock | INSERT INTO t1  VALUES('a')       |    0.000 |
          ...
          | 4832 | root | localhost | test | Query   |  240 | Waiting for table metadata lock | DROP TABLE t1                     |    0.000 |
          | 4834 | root | localhost | test | Query   |  240 | Waiting for table metadata lock | DROP TABLE t1                     |    0.000 |
          

          A bit later we see an odd mix, including threads with an empty State. Example:

          CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug)

          | 5023 | root | localhost | test | Query   |  673 | Opening tables                  | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 5024 | root | localhost | test | Query   |  673 | Opening tables                  | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 5025 | root | localhost | test | Query   |  171 | Opening tables                  | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 5026 | root | localhost | test | Query   |  171 | Opening tables                  | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 5027 | root | localhost | test | Query   |   71 | Opening tables                  | INSERT INTO t1  VALUES('a')       |    0.000 |
          | 5028 | root | localhost | test | Query   |   95 | Opening tables                  | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 5029 | root | localhost | test | Query   |   95 | checking permissions            | DROP TABLE t1                     |    0.000 |
          | 5030 | root | localhost | test | Query   |   95 | Opening tables                  | INSERT INTO t1  VALUES('a')       |    0.000 |
          | 5031 | root | localhost | test | Query   |   89 | checking permissions            | DROP TABLE t1                     |    0.000 |
          | 5032 | root | localhost | test | Query   |   93 | Opening tables                  | INSERT INTO t1  VALUES('a')       |    0.000 |
          | 5034 | root | localhost | test | Query   |   95 | Opening tables                  | INSERT INTO t1  VALUES('a')       |    0.000 |
          | 5033 | root | localhost | test | Query   |   72 | Opening tables                  | INSERT INTO t1  VALUES('a')       |    0.000 |
          | 5035 | root | localhost | test | Query   |  673 | Opening tables                  | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 5038 | root | localhost | test | Query   |  673 | Opening tables                  | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 5037 | root | localhost | test | Query   |  122 |                                 | INSERT INTO t1  VALUES('a')       |    0.000 |
          | 5036 | root | localhost | test | Query   |  170 |                                 | INSERT INTO t1  VALUES('a')       |    0.000 |
          | 5039 | root | localhost | test | Query   |  171 | Opening tables                  | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 5040 | root | localhost | test | Query   |   95 | Opening tables                  | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 5041 | root | localhost | test | Query   |  170 | Opening tables                  | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 5042 | root | localhost | test | Query   |   95 | Opening tables                  | INSERT INTO t1  VALUES('a')       |    0.000 |
          | 5043 | root | localhost | test | Query   |  673 | Opening tables                  | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 5044 | root | localhost | test | Query   |  103 | Opening tables                  | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 5045 | root | localhost | test | Query   |   82 | checking permissions            | DROP TABLE t1                     |    0.000 |
          | 5046 | root | localhost | test | Query   |   95 | Opening tables                  | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 5047 | root | localhost | test | Query   |  103 | Opening tables                  | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          

          Still later almost all threads are in Waiting for table metadata lock state:

          CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug)

          | 1150 | root | localhost | test | Query   | 1032 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 1151 | root | localhost | test | Query   |    0 | Waiting for table metadata lock | DROP TABLE t1                     |    0.000 |
          | 1153 | root | localhost | test | Query   | 1024 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 1152 | root | localhost | test | Query   | 1010 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 1154 | root | localhost | test | Query   | 1032 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 1155 | root | localhost | test | Query   | 1023 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 1156 | root | localhost | test | Query   |  397 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 1157 | root | localhost | test | Query   | 1032 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 1158 | root | localhost | test | Query   | 1032 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 1159 | root | localhost | test | Query   |  602 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 1160 | root | localhost | test | Query   | 1032 | After create                    | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 1162 | root | localhost | test | Query   | 1024 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 1161 | root | localhost | test | Query   | 1032 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 1164 | root | localhost | test | Query   | 1010 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 1163 | root | localhost | test | Query   | 1000 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 1165 | root | localhost | test | Query   | 1024 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          | 1166 | root | localhost | test | Query   | 1023 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) |    0.000 |
          

          It may be due to the testcase setup/thread interactions, but it would seem odd that almost 50% of the threads would get stuck in the same state for 15+ minutes.

          Shortly thereafter the server crashed with a similar but new stack:

          CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug)

          mariadbd: /test/11.7_dbg/storage/innobase/buf/buf0lru.cc:400: buf_block_t* buf_LRU_get_free_block(buf_LRU_get): Assertion `LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()' failed.
          

          CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug)

          Core was generated by `/test/MD141024-mariadb-11.7.0-linux-x86_64-dbg/bin/mariadbd --no-defaults --max'.
          Program terminated with signal SIGABRT, Aborted.
          Download failed: Invalid argument.  Continuing without source file ./nptl/./nptl/pthread_kill.c.
          #0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44
           
          [Current thread is 1 (LWP 1482047)]
          (gdb) bt
          #0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44
          #1  __pthread_kill_internal (signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:78
          #2  __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6)at ./nptl/pthread_kill.c:89
          #3  0x000011f795c4526e in __GI_raise (sig=sig@entry=6)at ../sysdeps/posix/raise.c:26
          #4  0x000011f795c288ff in __GI_abort () at ./stdlib/abort.c:79
          #5  0x000011f795c2881b in __assert_fail_base (fmt=0x11f795dd01e8 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=assertion@entry=0x609288b6c9b0 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=file@entry=0x609288b6bf48 "/test/11.7_dbg/storage/innobase/buf/buf0lru.cc", line=line@entry=400, function=function@entry=0x609288b6ca00 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)") at ./assert/assert.c:94
          #6  0x000011f795c3b507 in __assert_fail (assertion=0x609288b6c9b0 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=0x609288b6bf48 "/test/11.7_dbg/storage/innobase/buf/buf0lru.cc", line=400, function=0x609288b6ca00 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)")at ./assert/assert.c:103
          #7  0x0000609288659a0b in buf_LRU_get_free_block (get=get@entry=have_mutex)at /test/11.7_dbg/storage/innobase/buf/buf0lru.cc:400
          #8  0x000060928865d13f in buf_read_page (page_id=page_id@entry={m_id = 7}, chain=@0x60928b6f2a10: {first = 0x0})at /test/11.7_dbg/storage/innobase/buf/buf0rea.cc:427
          #9  0x0000609288631364 in buf_page_get_gen (page_id=<optimized out>, page_id@entry={m_id = 7}, zip_size=zip_size@entry=0, rw_latch=rw_latch@entry=RW_X_LATCH, guess=guess@entry=0x0, mode=mode@entry=10, mtr=mtr@entry=0x11f75fa376e0, err=0x0)at /test/11.7_dbg/storage/innobase/buf/buf0buf.cc:2882
          #10 0x00006092886646c1 in dict_hdr_get (mtr=0x11f75fa376e0)at /test/11.7_dbg/storage/innobase/dict/dict0boot.cc:44
          #11 dict_hdr_get_new_id (table_id=table_id@entry=0x11f308312478, index_id=index_id@entry=0x0, space_id=space_id@entry=0x0)at /test/11.7_dbg/storage/innobase/dict/dict0boot.cc:66
          #12 0x000060928866689d in dict_build_table_def_step (node=0x11f57029a578, thr=0x11f1943057c8)at /test/11.7_dbg/storage/innobase/dict/dict0crea.cc:349
          #13 dict_create_table_step (thr=thr@entry=0x11f1943057c8)at /test/11.7_dbg/storage/innobase/dict/dict0crea.cc:1024
          #14 0x0000609288492d93 in que_thr_step (thr=0x11f1943057c8)at /test/11.7_dbg/storage/innobase/que/que0que.cc:562
          #15 que_run_threads_low (thr=0x11f1943057c8)at /test/11.7_dbg/storage/innobase/que/que0que.cc:610
          #16 que_run_threads (thr=thr@entry=0x11f1943057c8)at /test/11.7_dbg/storage/innobase/que/que0que.cc:630
          #17 0x00006092885058ca in row_create_table_for_mysql (table=table@entry=0x11f308312478, trx=0x11f3c8453640)at /test/11.7_dbg/storage/innobase/row/row0mysql.cc:2061
          #18 0x0000609288379d8c in create_table_info_t::create_table_def (this=<optimized out>)at /test/11.7_dbg/storage/innobase/handler/ha_innodb.cc:10834
          #19 0x000060928836d75a in create_table_info_t::create_table (this=this@entry=0x11f75fa38050, create_fk=create_fk@entry=true, strict=true) at /test/11.7_dbg/storage/innobase/handler/ha_innodb.cc:12680
          #20 0x000060928836df24 in ha_innobase::create (this=<optimized out>, name=<optimized out>, form=0x11f75fa38600, create_info=0x11f75fa3a230, file_per_table=<optimized out>, trx=0x11f3c8453640, trx@entry=0x0)at /test/11.7_dbg/storage/innobase/handler/ha_innodb.cc:13188
          #21 0x000060928836f2e7 in ha_innobase::create (this=<optimized out>, name=<optimized out>, form=<optimized out>, create_info=<optimized out>)at /test/11.7_dbg/storage/innobase/handler/ha_innodb.cc:13236
          #22 0x0000609287fe7942 in handler::ha_create (this=0x11f194304000, name=0x11f75fa39bb0 "./test/t2", form=form@entry=0x11f75fa38600, info_arg=info_arg@entry=0x11f75fa3a230)at /test/11.7_dbg/sql/handler.cc:5877
          #23 0x0000609287fed9d0 in ha_create_table (thd=thd@entry=0x11f19401cb58, path=<optimized out>, db=<optimized out>, table_name=<optimized out>, create_info=create_info@entry=0x11f75fa3a230, frm=frm@entry=0x11f75fa39ba0, skip_frm_file=<optimized out>)at /test/11.7_dbg/sql/handler.cc:6364
          #24 0x0000609287db562a in create_table_impl (thd=thd@entry=0x11f19401cb58, ddl_log_state_create=ddl_log_state_create@entry=0x11f75fa39eb0, ddl_log_state_rm=<optimized out>, ddl_log_state_rm@entry=0x11f75fa3a0c0, orig_db=@0x11f1940316f8: {<Lex_ident_fs> = {<Lex_ident<Compare_table_names>> = {<Lex_cstring> = {<st_mysql_const_lex_string> = {str = 0x11f1940316d0 "test", length = 4}, <No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, orig_table_name=@0x11f194031708: {<Lex_ident_fs> = {<Lex_ident<Compare_table_names>> = {<Lex_cstring> = {<st_mysql_const_lex_string> = {str = 0x11f1940316a0 "t2", length = 2}, <No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, db=@0x11f1940316f8: {str = 0x11f1940316d0 "test", length = 4}, table_name=@0x11f194031708: {str = 0x11f1940316a0 "t2", length = 2}, path=@0x11f75fa39b90: {str = 0x11f75fa39bb0 "./test/t2", length = 9}, options=<optimized out>, create_info=0x11f75fa3a230, alter_info=0x11f75fa39f50, create_table_mode=0, is_trans=0x11f75fa39e70, key_info=0x11f75fa39b88, key_count=0x11f75fa39b84, frm=0x11f75fa39ba0)at /test/11.7_dbg/sql/sql_table.cc:4683
          #25 0x0000609287db5b4f in mysql_create_table_no_lock (thd=thd@entry=0x11f19401cb58, ddl_log_state_create=ddl_log_state_create@entry=0x11f75fa39eb0, ddl_log_state_rm=ddl_log_state_rm@entry=0x11f75fa3a0c0, create_info=create_info@entry=0x11f75fa3a230, alter_info=alter_info@entry=0x11f75fa39f50, is_trans=is_trans@entry=0x11f75fa39e70, create_table_mode=0, table_list=0x11f1940316e0) at /test/11.7_dbg/sql/sql_table.cc:4786
          #26 0x0000609287db7c70 in mysql_create_table (alter_info=0x11f75fa39f50, create_info=0x11f75fa3a230, create_table=0x11f1940316e0, thd=0x11f19401cb58) at /test/11.7_dbg/sql/sql_table.cc:5012
          #27 Sql_cmd_create_table_like::execute (this=<optimized out>, thd=0x11f19401cb58) at /test/11.7_dbg/sql/sql_table.cc:13260
          #28 0x0000609287cbdb5b in mysql_execute_command (thd=thd@entry=0x11f19401cb58, is_called_from_prepared_stmt=is_called_from_prepared_stmt@entry=false)at /test/11.7_dbg/sql/sql_parse.cc:5858
          #29 0x0000609287cbed36 in mysql_parse (thd=thd@entry=0x11f19401cb58, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0x11f75fa3c260)at /test/11.7_dbg/sql/sql_parse.cc:7885
          #30 0x0000609287cc11ee in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0x11f19401cb58, packet=packet@entry=0x11f1940270d9 "CREATE TABLE t2 (c1 INTEGER)", packet_length=packet_length@entry=28, blocking=blocking@entry=true)at /test/11.7_dbg/sql/sql_class.h:257
          #31 0x0000609287cc3552 in do_command (thd=0x11f19401cb58, blocking=blocking@entry=true) at /test/11.7_dbg/sql/sql_parse.cc:1405
          #32 0x0000609287e3ce5f in do_handle_one_connection (connect=<optimized out>, connect@entry=0x60928bcb6458, put_in_cache=put_in_cache@entry=true)at /test/11.7_dbg/sql/sql_connect.cc:1448
          #33 0x0000609287e3d173 in handle_one_connection (arg=arg@entry=0x60928bcb6458)at /test/11.7_dbg/sql/sql_connect.cc:1350
          #34 0x0000609288292bcf in pfs_spawn_thread (arg=0x60928bcca848)at /test/11.7_dbg/storage/perfschema/pfs.cc:2198
          #35 0x000011f795c9ca94 in start_thread (arg=<optimized out>)at ./nptl/pthread_create.c:447
          #36 0x000011f795d29c3c in clone3 ()at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78
          

          This was without TEMPORARY.

          Roel Roel Van de Paar added a comment - During testing without TEMPORARY,I checked the processlist and noticed nearly all 5k threads were in Opening tables/checking permissions state for a long time: CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug) | 3287 | root | localhost | test | Query | 263 | Opening tables | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 3288 | root | localhost | test | Query | 263 | Opening tables | INSERT INTO t1 VALUES('a') | 0.000 | | 3289 | root | localhost | test | Query | 263 | Opening tables | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 3291 | root | localhost | test | Query | 263 | checking permissions | DROP TABLE t1 | 0.000 | | 3290 | root | localhost | test | Query | 263 | checking permissions | DROP TABLE t1 | 0.000 | | 3292 | root | localhost | test | Query | 263 | Opening tables | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 3293 | root | localhost | test | Query | 263 | checking permissions | DROP TABLE t1 | 0.000 | With a few notable exceptions: CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug) | 247 | root | localhost | test | Query | 263 | Query end | DROP TABLE t1 | 0.000 | ... | 2235 | root | localhost | test | Query | 78 | checking permissions | DROP TABLE t1 | 0.000 | # 78s ... | 2565 | root | localhost | test | Query | 261 | Waiting for table metadata lock | DROP TABLE t1 | 0.000 | ... | 4815 | root | localhost | test | Query | 240 | Waiting for table metadata lock | DROP TABLE t1 | 0.000 | ... | 4824 | root | localhost | test | Query | 240 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | ... | 4828 | root | localhost | test | Query | 240 | Waiting for table metadata lock | INSERT INTO t1 VALUES('a') | 0.000 | ... | 4832 | root | localhost | test | Query | 240 | Waiting for table metadata lock | DROP TABLE t1 | 0.000 | | 4834 | root | localhost | test | Query | 240 | Waiting for table metadata lock | DROP TABLE t1 | 0.000 | A bit later we see an odd mix, including threads with an empty State. Example: CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug) | 5023 | root | localhost | test | Query | 673 | Opening tables | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 5024 | root | localhost | test | Query | 673 | Opening tables | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 5025 | root | localhost | test | Query | 171 | Opening tables | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 5026 | root | localhost | test | Query | 171 | Opening tables | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 5027 | root | localhost | test | Query | 71 | Opening tables | INSERT INTO t1 VALUES('a') | 0.000 | | 5028 | root | localhost | test | Query | 95 | Opening tables | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 5029 | root | localhost | test | Query | 95 | checking permissions | DROP TABLE t1 | 0.000 | | 5030 | root | localhost | test | Query | 95 | Opening tables | INSERT INTO t1 VALUES('a') | 0.000 | | 5031 | root | localhost | test | Query | 89 | checking permissions | DROP TABLE t1 | 0.000 | | 5032 | root | localhost | test | Query | 93 | Opening tables | INSERT INTO t1 VALUES('a') | 0.000 | | 5034 | root | localhost | test | Query | 95 | Opening tables | INSERT INTO t1 VALUES('a') | 0.000 | | 5033 | root | localhost | test | Query | 72 | Opening tables | INSERT INTO t1 VALUES('a') | 0.000 | | 5035 | root | localhost | test | Query | 673 | Opening tables | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 5038 | root | localhost | test | Query | 673 | Opening tables | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 5037 | root | localhost | test | Query | 122 | | INSERT INTO t1 VALUES('a') | 0.000 | | 5036 | root | localhost | test | Query | 170 | | INSERT INTO t1 VALUES('a') | 0.000 | | 5039 | root | localhost | test | Query | 171 | Opening tables | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 5040 | root | localhost | test | Query | 95 | Opening tables | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 5041 | root | localhost | test | Query | 170 | Opening tables | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 5042 | root | localhost | test | Query | 95 | Opening tables | INSERT INTO t1 VALUES('a') | 0.000 | | 5043 | root | localhost | test | Query | 673 | Opening tables | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 5044 | root | localhost | test | Query | 103 | Opening tables | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 5045 | root | localhost | test | Query | 82 | checking permissions | DROP TABLE t1 | 0.000 | | 5046 | root | localhost | test | Query | 95 | Opening tables | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 5047 | root | localhost | test | Query | 103 | Opening tables | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | Still later almost all threads are in Waiting for table metadata lock state: CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug) | 1150 | root | localhost | test | Query | 1032 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 1151 | root | localhost | test | Query | 0 | Waiting for table metadata lock | DROP TABLE t1 | 0.000 | | 1153 | root | localhost | test | Query | 1024 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 1152 | root | localhost | test | Query | 1010 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 1154 | root | localhost | test | Query | 1032 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 1155 | root | localhost | test | Query | 1023 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 1156 | root | localhost | test | Query | 397 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 1157 | root | localhost | test | Query | 1032 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 1158 | root | localhost | test | Query | 1032 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 1159 | root | localhost | test | Query | 602 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 1160 | root | localhost | test | Query | 1032 | After create | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 1162 | root | localhost | test | Query | 1024 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 1161 | root | localhost | test | Query | 1032 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 1164 | root | localhost | test | Query | 1010 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 1163 | root | localhost | test | Query | 1000 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 1165 | root | localhost | test | Query | 1024 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | | 1166 | root | localhost | test | Query | 1023 | Waiting for table metadata lock | CREATE TABLE t1(c1 LONGTEXT NULL) | 0.000 | It may be due to the testcase setup/thread interactions, but it would seem odd that almost 50% of the threads would get stuck in the same state for 15+ minutes. Shortly thereafter the server crashed with a similar but new stack: CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug) mariadbd: /test/11.7_dbg/storage/innobase/buf/buf0lru.cc:400: buf_block_t* buf_LRU_get_free_block(buf_LRU_get): Assertion `LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()' failed. CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug) Core was generated by `/test/MD141024-mariadb-11.7.0-linux-x86_64-dbg/bin/mariadbd --no-defaults --max'. Program terminated with signal SIGABRT, Aborted. Download failed: Invalid argument. Continuing without source file ./nptl/./nptl/pthread_kill.c. #0 __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44   [Current thread is 1 (LWP 1482047)] (gdb) bt #0 __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44 #1 __pthread_kill_internal (signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:78 #2 __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6)at ./nptl/pthread_kill.c:89 #3 0x000011f795c4526e in __GI_raise (sig=sig@entry=6)at ../sysdeps/posix/raise.c:26 #4 0x000011f795c288ff in __GI_abort () at ./stdlib/abort.c:79 #5 0x000011f795c2881b in __assert_fail_base (fmt=0x11f795dd01e8 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=assertion@entry=0x609288b6c9b0 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=file@entry=0x609288b6bf48 "/test/11.7_dbg/storage/innobase/buf/buf0lru.cc", line=line@entry=400, function=function@entry=0x609288b6ca00 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)") at ./assert/assert.c:94 #6 0x000011f795c3b507 in __assert_fail (assertion=0x609288b6c9b0 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=0x609288b6bf48 "/test/11.7_dbg/storage/innobase/buf/buf0lru.cc", line=400, function=0x609288b6ca00 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)")at ./assert/assert.c:103 #7 0x0000609288659a0b in buf_LRU_get_free_block (get=get@entry=have_mutex)at /test/11.7_dbg/storage/innobase/buf/buf0lru.cc:400 #8 0x000060928865d13f in buf_read_page (page_id=page_id@entry={m_id = 7}, chain=@0x60928b6f2a10: {first = 0x0})at /test/11.7_dbg/storage/innobase/buf/buf0rea.cc:427 #9 0x0000609288631364 in buf_page_get_gen (page_id=<optimized out>, page_id@entry={m_id = 7}, zip_size=zip_size@entry=0, rw_latch=rw_latch@entry=RW_X_LATCH, guess=guess@entry=0x0, mode=mode@entry=10, mtr=mtr@entry=0x11f75fa376e0, err=0x0)at /test/11.7_dbg/storage/innobase/buf/buf0buf.cc:2882 #10 0x00006092886646c1 in dict_hdr_get (mtr=0x11f75fa376e0)at /test/11.7_dbg/storage/innobase/dict/dict0boot.cc:44 #11 dict_hdr_get_new_id (table_id=table_id@entry=0x11f308312478, index_id=index_id@entry=0x0, space_id=space_id@entry=0x0)at /test/11.7_dbg/storage/innobase/dict/dict0boot.cc:66 #12 0x000060928866689d in dict_build_table_def_step (node=0x11f57029a578, thr=0x11f1943057c8)at /test/11.7_dbg/storage/innobase/dict/dict0crea.cc:349 #13 dict_create_table_step (thr=thr@entry=0x11f1943057c8)at /test/11.7_dbg/storage/innobase/dict/dict0crea.cc:1024 #14 0x0000609288492d93 in que_thr_step (thr=0x11f1943057c8)at /test/11.7_dbg/storage/innobase/que/que0que.cc:562 #15 que_run_threads_low (thr=0x11f1943057c8)at /test/11.7_dbg/storage/innobase/que/que0que.cc:610 #16 que_run_threads (thr=thr@entry=0x11f1943057c8)at /test/11.7_dbg/storage/innobase/que/que0que.cc:630 #17 0x00006092885058ca in row_create_table_for_mysql (table=table@entry=0x11f308312478, trx=0x11f3c8453640)at /test/11.7_dbg/storage/innobase/row/row0mysql.cc:2061 #18 0x0000609288379d8c in create_table_info_t::create_table_def (this=<optimized out>)at /test/11.7_dbg/storage/innobase/handler/ha_innodb.cc:10834 #19 0x000060928836d75a in create_table_info_t::create_table (this=this@entry=0x11f75fa38050, create_fk=create_fk@entry=true, strict=true) at /test/11.7_dbg/storage/innobase/handler/ha_innodb.cc:12680 #20 0x000060928836df24 in ha_innobase::create (this=<optimized out>, name=<optimized out>, form=0x11f75fa38600, create_info=0x11f75fa3a230, file_per_table=<optimized out>, trx=0x11f3c8453640, trx@entry=0x0)at /test/11.7_dbg/storage/innobase/handler/ha_innodb.cc:13188 #21 0x000060928836f2e7 in ha_innobase::create (this=<optimized out>, name=<optimized out>, form=<optimized out>, create_info=<optimized out>)at /test/11.7_dbg/storage/innobase/handler/ha_innodb.cc:13236 #22 0x0000609287fe7942 in handler::ha_create (this=0x11f194304000, name=0x11f75fa39bb0 "./test/t2", form=form@entry=0x11f75fa38600, info_arg=info_arg@entry=0x11f75fa3a230)at /test/11.7_dbg/sql/handler.cc:5877 #23 0x0000609287fed9d0 in ha_create_table (thd=thd@entry=0x11f19401cb58, path=<optimized out>, db=<optimized out>, table_name=<optimized out>, create_info=create_info@entry=0x11f75fa3a230, frm=frm@entry=0x11f75fa39ba0, skip_frm_file=<optimized out>)at /test/11.7_dbg/sql/handler.cc:6364 #24 0x0000609287db562a in create_table_impl (thd=thd@entry=0x11f19401cb58, ddl_log_state_create=ddl_log_state_create@entry=0x11f75fa39eb0, ddl_log_state_rm=<optimized out>, ddl_log_state_rm@entry=0x11f75fa3a0c0, orig_db=@0x11f1940316f8: {<Lex_ident_fs> = {<Lex_ident<Compare_table_names>> = {<Lex_cstring> = {<st_mysql_const_lex_string> = {str = 0x11f1940316d0 "test", length = 4}, <No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, orig_table_name=@0x11f194031708: {<Lex_ident_fs> = {<Lex_ident<Compare_table_names>> = {<Lex_cstring> = {<st_mysql_const_lex_string> = {str = 0x11f1940316a0 "t2", length = 2}, <No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, db=@0x11f1940316f8: {str = 0x11f1940316d0 "test", length = 4}, table_name=@0x11f194031708: {str = 0x11f1940316a0 "t2", length = 2}, path=@0x11f75fa39b90: {str = 0x11f75fa39bb0 "./test/t2", length = 9}, options=<optimized out>, create_info=0x11f75fa3a230, alter_info=0x11f75fa39f50, create_table_mode=0, is_trans=0x11f75fa39e70, key_info=0x11f75fa39b88, key_count=0x11f75fa39b84, frm=0x11f75fa39ba0)at /test/11.7_dbg/sql/sql_table.cc:4683 #25 0x0000609287db5b4f in mysql_create_table_no_lock (thd=thd@entry=0x11f19401cb58, ddl_log_state_create=ddl_log_state_create@entry=0x11f75fa39eb0, ddl_log_state_rm=ddl_log_state_rm@entry=0x11f75fa3a0c0, create_info=create_info@entry=0x11f75fa3a230, alter_info=alter_info@entry=0x11f75fa39f50, is_trans=is_trans@entry=0x11f75fa39e70, create_table_mode=0, table_list=0x11f1940316e0) at /test/11.7_dbg/sql/sql_table.cc:4786 #26 0x0000609287db7c70 in mysql_create_table (alter_info=0x11f75fa39f50, create_info=0x11f75fa3a230, create_table=0x11f1940316e0, thd=0x11f19401cb58) at /test/11.7_dbg/sql/sql_table.cc:5012 #27 Sql_cmd_create_table_like::execute (this=<optimized out>, thd=0x11f19401cb58) at /test/11.7_dbg/sql/sql_table.cc:13260 #28 0x0000609287cbdb5b in mysql_execute_command (thd=thd@entry=0x11f19401cb58, is_called_from_prepared_stmt=is_called_from_prepared_stmt@entry=false)at /test/11.7_dbg/sql/sql_parse.cc:5858 #29 0x0000609287cbed36 in mysql_parse (thd=thd@entry=0x11f19401cb58, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0x11f75fa3c260)at /test/11.7_dbg/sql/sql_parse.cc:7885 #30 0x0000609287cc11ee in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0x11f19401cb58, packet=packet@entry=0x11f1940270d9 "CREATE TABLE t2 (c1 INTEGER)", packet_length=packet_length@entry=28, blocking=blocking@entry=true)at /test/11.7_dbg/sql/sql_class.h:257 #31 0x0000609287cc3552 in do_command (thd=0x11f19401cb58, blocking=blocking@entry=true) at /test/11.7_dbg/sql/sql_parse.cc:1405 #32 0x0000609287e3ce5f in do_handle_one_connection (connect=<optimized out>, connect@entry=0x60928bcb6458, put_in_cache=put_in_cache@entry=true)at /test/11.7_dbg/sql/sql_connect.cc:1448 #33 0x0000609287e3d173 in handle_one_connection (arg=arg@entry=0x60928bcb6458)at /test/11.7_dbg/sql/sql_connect.cc:1350 #34 0x0000609288292bcf in pfs_spawn_thread (arg=0x60928bcca848)at /test/11.7_dbg/storage/perfschema/pfs.cc:2198 #35 0x000011f795c9ca94 in start_thread (arg=<optimized out>)at ./nptl/pthread_create.c:447 #36 0x000011f795d29c3c in clone3 ()at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78 This was without TEMPORARY.
          Roel Roel Van de Paar added a comment - - edited

          A 11.7 debug UB+ASAN build (gcc 11) did not reproduce any crash nor produced any UBSAN or ASAN error. The number of InnoDB transaction pools did increase during the test:

          CS 11.7.0 35cebfdc513f92b143b1a7229c480f4f684f1698 (Debug, UBASAN)

          2024-10-17 15:12:56 2118 [Note] InnoDB: Number of transaction pools: 2
          2024-10-17 15:34:27 4641 [Note] InnoDB: Number of transaction pools: 3
          

          Roel Roel Van de Paar added a comment - - edited A 11.7 debug UB+ASAN build (gcc 11) did not reproduce any crash nor produced any UBSAN or ASAN error. The number of InnoDB transaction pools did increase during the test: CS 11.7.0 35cebfdc513f92b143b1a7229c480f4f684f1698 (Debug, UBASAN) 2024-10-17 15:12:56 2118 [Note] InnoDB: Number of transaction pools: 2 2024-10-17 15:34:27 4641 [Note] InnoDB: Number of transaction pools: 3

          Rather than attempt to generate an rr trace for the various bugs seen, I have created a reproducer script which can reproduce the issue in 1-10 minutes. After setting the settings to match your environment, you can run this outright to produce a core, or you can attempt to run this under rr - though the traces will be slow / long. You can also attempt to reduce the number of threads, though it would likely take more time to reproduce in that case (and thus again longer traces).

          #!/bin/bash
          # Start the server with --max_connections=10000
          # ^ If the output from the script stops scrolling, it is due to the server having reached
          #   ${threads} simulateneous connections, which are all processing the input SQL provided
          # ^ Reversely, even if the output continues to scroll (due to terminal backlog: note the
          #   number of queries per thread, ${queries}), the instance may have already crashed
          # Set variables and ensure ramloc is a ramdisk or tmpfs (i.e. /dev/shm)
           
          ramloc="/dev/shm"
          client="./bin/mariadb"
          socket="./socket.sock"
          errorlog="./log/master.err"
          user="root"
          db="test"
          cli_opt='-A --skip-ssl --binary-mode --force'
          background=1    # Set to 0 to debug any failed client connections
          threads=5000    # Number of concurrent threads
          queries=100000  # Number of queries per round
          rounds=999999   # Number of max test rounds
          sql="
          CREATE TABLE t2 (c1 INTEGER);
          CREATE TEMPORARY TABLE t1(c1 LONGTEXT NULL);
          DROP TABLE t1;
          DROP TABLE t2;
          INSERT INTO t1  VALUES('a');
          SET @@global.innodb_immediate_scrub_data_uncompressed=0;
          SET @@global.innodb_immediate_scrub_data_uncompressed=1;
          SET @@global.innodb_lru_scan_depth = 1536;
          SET @@global.innodb_lru_scan_depth = 86400;
          XA COMMIT 'a';
          XA END 'a';
          XA PREPARE 'a';
          XA START 'a' RESUME;
          XA START 'a';
          "
           
          exec_sql(){
            for ((i=0;i<$[ ${queries} / ${sql_len} ];i++)); do
              echo "${sql}" >> ${ramloc}/${prefix}_md_proc_${1}.tmp
            done
            shuf --random-source=/dev/urandom ${ramloc}/${prefix}_md_proc_${1}.tmp > ${ramloc}/${prefix}_md_proc_${1}
            rm -f ${ramloc}/${prefix}_md_proc_${1}.tmp
            ${client} ${cli_opt} -u ${user} -S ${socket} -D ${db} < ${ramloc}/${prefix}_md_proc_${1}
            rm -f ${ramloc}/${prefix}_md_proc_${1}  # Thread done
          }
           
          abort(){ jobs -p | xargs -P100 kill >/dev/null 2>&1; rm -Rf ${ramloc}/${prefix}_md_proc_*; exit 1; }
          trap abort SIGINT
           
          count=0
          sql="$(echo "${sql}" | grep --binary-files=text -vE '^[ \t]*$')"
          RANDOM=$(date +%s%N | cut -b10-19 | sed 's|^[0]\+||')  # Random entropy init
          sql_len="$(echo "${sql}" | wc -l)"
          prefix="$(echo "${RANDOM}${RANDOM}${RANDOM}" | cut -b1-5)"
          echo "Input sql lenght: ${sql_len} lines | Runtime files: ${ramloc}/${prefix}_md_proc_*"
          rm -f ${ramloc}/${prefix}_md_proc_*
          for ((i=0;i<${rounds};i++)); do
            for ((i=0;i<${threads};i++)); do
              if [ ! -r ${ramloc}/${prefix}_md_proc_${i} ]; then  # Thread idle
                touch ${ramloc}/${prefix}_md_proc_${i}  # Thread busy
                if [ ${background} -eq 1 ]; then
                  exec_sql ${i} &
                else
                  exec_sql ${i}
                fi
                count=$[ ${count} + 1 ]
                if [ $[ ${count} % 100 ] -eq 0 ]; then  # Limit disk I/O, check once every new 100 threads
                  echo "${count} threads processing ${queries} queries each: $[ ${count} * ${queries} ] total" | tee lastcount.log
                  TAIL="$(tail -n10 ${errorlog} | tr -d '\n')"
                  if [[ "${TAIL}" == *"ERROR"* ]]; then
                    echo '*** Error found:'
                    grep --binary-files=text -i 'ERROR' log/master.err
                    abort
                  elif [[ "${TAIL}" == *"down complete"* ]]; then
                    echo '*** Server was shutdown'
                    abort
                  elif ! ${client}-admin ping -u ${user} -S ${socket} > /dev/null 2>&1; then
                    echo '*** Server gone (killed/crashed)'
                    abort
                  fi
                fi
              fi
            done
          done
          

          It also works fine if you change CREATE TEMPORARY TABLE t1(c1 LONGTEXT NULL); to CREATE TABLE t1(c1 LONGTEXT NULL); w/o TEMPORARY

          Roel Roel Van de Paar added a comment - Rather than attempt to generate an rr trace for the various bugs seen, I have created a reproducer script which can reproduce the issue in 1-10 minutes. After setting the settings to match your environment, you can run this outright to produce a core, or you can attempt to run this under rr - though the traces will be slow / long. You can also attempt to reduce the number of threads, though it would likely take more time to reproduce in that case (and thus again longer traces). #!/bin/bash # Start the server with --max_connections=10000 # ^ If the output from the script stops scrolling, it is due to the server having reached # ${threads} simulateneous connections, which are all processing the input SQL provided # ^ Reversely, even if the output continues to scroll (due to terminal backlog: note the # number of queries per thread, ${queries}), the instance may have already crashed # Set variables and ensure ramloc is a ramdisk or tmpfs (i.e. /dev/shm) ramloc= "/dev/shm" client= "./bin/mariadb" socket= "./socket.sock" errorlog= "./log/master.err" user= "root" db= "test" cli_opt= '-A --skip-ssl --binary-mode --force' background=1 # Set to 0 to debug any failed client connections threads=5000 # Number of concurrent threads queries=100000 # Number of queries per round rounds=999999 # Number of max test rounds sql=" CREATE TABLE t2 (c1 INTEGER); CREATE TEMPORARY TABLE t1(c1 LONGTEXT NULL); DROP TABLE t1; DROP TABLE t2; INSERT INTO t1 VALUES( 'a' ); SET @@global.innodb_immediate_scrub_data_uncompressed=0; SET @@global.innodb_immediate_scrub_data_uncompressed=1; SET @@global.innodb_lru_scan_depth = 1536; SET @@global.innodb_lru_scan_depth = 86400; XA COMMIT 'a' ; XA END 'a' ; XA PREPARE 'a' ; XA START 'a' RESUME; XA START 'a' ; " exec_sql(){ for ((i=0;i<$[ ${queries} / ${sql_len} ];i++)); do echo "${sql}" >> ${ramloc}/${prefix}_md_proc_${1}.tmp done shuf --random- source = /dev/urandom ${ramloc}/${prefix}_md_proc_${1}.tmp > ${ramloc}/${prefix}_md_proc_${1} rm -f ${ramloc}/${prefix}_md_proc_${1}.tmp ${client} ${cli_opt} -u ${user} -S ${socket} -D ${db} < ${ramloc}/${prefix}_md_proc_${1} rm -f ${ramloc}/${prefix}_md_proc_${1} # Thread done } abort(){ jobs -p | xargs -P100 kill > /dev/null 2>&1; rm -Rf ${ramloc}/${prefix}_md_proc_*; exit 1; } trap abort SIGINT count=0 sql= "$(echo " ${sql} " | grep --binary-files=text -vE '^[ \t]*$')" RANDOM=$( date +%s%N | cut -b10-19 | sed 's|^[0]\+||' ) # Random entropy init sql_len= "$(echo " ${sql} " | wc -l)" prefix= "$(echo " ${RANDOM}${RANDOM}${RANDOM} " | cut -b1-5)" echo "Input sql lenght: ${sql_len} lines | Runtime files: ${ramloc}/${prefix}_md_proc_*" rm -f ${ramloc}/${prefix}_md_proc_* for ((i=0;i<${rounds};i++)); do for ((i=0;i<${threads};i++)); do if [ ! -r ${ramloc}/${prefix}_md_proc_${i} ]; then # Thread idle touch ${ramloc}/${prefix}_md_proc_${i} # Thread busy if [ ${background} - eq 1 ]; then exec_sql ${i} & else exec_sql ${i} fi count=$[ ${count} + 1 ] if [ $[ ${count} % 100 ] - eq 0 ]; then # Limit disk I/O, check once every new 100 threads echo "${count} threads processing ${queries} queries each: $[ ${count} * ${queries} ] total" | tee lastcount.log TAIL= "$(tail -n10 ${errorlog} | tr -d '\n')" if [[ "${TAIL}" == * "ERROR" * ]]; then echo '*** Error found:' grep --binary-files=text -i 'ERROR' log /master .err abort elif [[ "${TAIL}" == * "down complete" * ]]; then echo '*** Server was shutdown' abort elif ! ${client}-admin ping -u ${user} -S ${socket} > /dev/null 2>&1; then echo '*** Server gone (killed/crashed)' abort fi fi fi done done It also works fine if you change CREATE TEMPORARY TABLE t1(c1 LONGTEXT NULL); to CREATE TABLE t1(c1 LONGTEXT NULL); w/o TEMPORARY

          Usually, concurrency problems can be reproduced with 1 to 3 concurrent client connections. 5,000 connections looks like an overkill. That script would easily kill a low-end machine with 4 CPU threads, even when using the minimum innodb_buffer_pool_size=5m. Is this anomaly really not reproducible with a smaller number of concurrent connections, say, 10 or 50?

          Can you attach the output of the following GDB commands after a signal is caught in a debugger:

          set print static-members off
          set print frame-arguments all
          print buf_pool
          thread apply all backtrace full
          

          Also, a copy of a core dump and the generating executables (cmake -DCMAKE_BUILD_TYPE=Debug to avoid MDEV-10814) and shared libraries would be very helpful.

          I have some suspicion that the script could cause a scenario like one that I mentioned in MDEV-33613:

          By design, InnoDB has always hung when permanently running out of buffer pool, for example when several threads are waiting to allocate a block, and all of the buffer pool is buffer-fixed by the active threads.

          As far as I can tell, during the execution of the test (when removing the TEMPORARY keyword) there should be at most 3 pages per table in the buffer pool. However, the number of undo log pages could be much larger.

          marko Marko Mäkelä added a comment - Usually, concurrency problems can be reproduced with 1 to 3 concurrent client connections. 5,000 connections looks like an overkill. That script would easily kill a low-end machine with 4 CPU threads, even when using the minimum innodb_buffer_pool_size=5m . Is this anomaly really not reproducible with a smaller number of concurrent connections, say, 10 or 50? Can you attach the output of the following GDB commands after a signal is caught in a debugger: set print static-members off set print frame-arguments all print buf_pool thread apply all backtrace full Also, a copy of a core dump and the generating executables ( cmake -DCMAKE_BUILD_TYPE=Debug to avoid MDEV-10814 ) and shared libraries would be very helpful. I have some suspicion that the script could cause a scenario like one that I mentioned in MDEV-33613 : By design, InnoDB has always hung when permanently running out of buffer pool, for example when several threads are waiting to allocate a block, and all of the buffer pool is buffer-fixed by the active threads. As far as I can tell, during the execution of the test (when removing the TEMPORARY keyword) there should be at most 3 pages per table in the buffer pool. However, the number of undo log pages could be much larger.
          Roel Roel Van de Paar added a comment - - edited

          Issue was not repeatable with 10 concurrent threads, without TEMPORARY. Tried again with 50 concurrent threads, with TEMPORARY, which produced the

          SIGABRT|__gnu_debug::_Error_formatter::_M_error|__gnu_debug::operator==|range_set::add_range|fil_space_t::add_free_range
          

          issue quickly. In both cases used --innodb_buffer_pool_size=5M. Then again, some later attempts took > 10-15 minutes.

          The crash without TEMPORARY is harder to reproduce.

          Roel Roel Van de Paar added a comment - - edited Issue was not repeatable with 10 concurrent threads, without TEMPORARY. Tried again with 50 concurrent threads, with TEMPORARY, which produced the SIGABRT|__gnu_debug::_Error_formatter::_M_error|__gnu_debug::operator==|range_set::add_range|fil_space_t::add_free_range issue quickly. In both cases used --innodb_buffer_pool_size=5M . Then again, some later attempts took > 10-15 minutes. The crash without TEMPORARY is harder to reproduce.
          Roel Roel Van de Paar added a comment - - edited

          Three additional stacks observed:

          CS 11.4.4 b53b81e937b7b598ee73657501ec87c6f76334ec (Debug)

          Core was generated by `/test/MD141024-mariadb-11.4.4-linux-x86_64-dbg/bin/mariadbd --no-defaults --max'.
          Program terminated with signal SIGABRT, Aborted.
          Download failed: Invalid argument.  Continuing without source file ./nptl/./nptl/pthread_kill.c.
          #0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44
           
          [Current thread is 1 (LWP 3441987)]
          (gdb) bt
          #0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44
          #1  __pthread_kill_internal (signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:78
          #2  __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6)at ./nptl/pthread_kill.c:89
          #3  0x000014a405e4526e in __GI_raise (sig=sig@entry=6)at ../sysdeps/posix/raise.c:26
          #4  0x000014a405e288ff in __GI_abort () at ./stdlib/abort.c:79
          #5  0x000014a4062a9133 in __gnu_debug::_Error_formatter::_M_error (this=this@entry=0x55e63ee35c40 <__gnu_debug::_Error_formatter::_S_at(char const*, unsigned int, char const*)::__formatter>)at ../../../../../src/libstdc++-v3/src/c++11/debug.cc:1320
          #6  0x000055e63dd684dc in std::__debug::set<range_t, range_compare, std::allocator<range_t> >::erase[abi:cxx11](__gnu_debug::_Safe_iterator<std::_Rb_tree_const_iterator<range_t>, std::__debug::set<range_t, range_compare, std::allocator<range_t> >, std::bidirectional_iterator_tag>) (this=this@entry=0x55e640306d30, __position={first = 261, last = 261})at /usr/include/c++/13/debug/set.h:347
          #7  0x000055e63dd68a5f in range_set::add_within_range (this=this@entry=0x55e640306d30, range={first = 261, last = 261}, value=value@entry=262)at /test/11.4_dbg/storage/innobase/include/fil0fil.h:208
          #8  0x000055e63dd6a264 in range_set::add_range (this=0x55e640306d30, new_range={first = 3441987, last = 0})at /test/11.4_dbg/storage/innobase/include/fil0fil.h:247
          #9  0x000055e63dd8c772 in fil_space_t::add_free_range (range=<optimized out>, this=<optimized out>)at /test/11.4_dbg/storage/innobase/include/fil0fil.h:908
          #10 mtr_t::commit (this=this@entry=0x14a3e83fc540)at /test/11.4_dbg/storage/innobase/mtr/mtr0mtr.cc:517
          #11 0x000055e63df1958b in btr_drop_temporary_table (table=Python Exception <class 'AttributeError'>: 'NoneType' object has no attribute 'pointer'
          Python Exception <class 'AttributeError'>: 'NoneType' object has no attribute 'pointer'
          @0x14a38001df58: {id = 1891, id_hash = 0x0, name = {m_name = 0x14a200019e68 "tmp/#sql-temptable-347cae-48-2a"}, name_hash = 0x0, heap = 0x14a1e8038d78, data_dir_path = 0x0, space = 0x55e640306c28, space_id = 4294967294, flags = 33, flags2 = 65, skip_alter_undo = 0, file_unreadable = 0, cached = 1, n_def = 4, n_cols = 4, n_t_cols = 4, n_t_def = 4, n_v_def = 0, n_v_cols = 0, persistent_autoinc = 0, can_be_evicted = 0, corrupted = 0, drop_aborted = 0, cols = 0x14a20003cfb8, v_cols = 0x14a20003d038, s_cols = 0x0, instant = 0x0, col_names = 0x14a20003d090 "c1", v_col_names = 0x0, vers_start = 0, vers_end = 0, is_system_db = false, dict_frm_mismatch = DICT_FRM_CONSISTENT, fts_doc_id_index = 0x0, indexes = {count = 1, start = 0x14a2e80374c8, end = 0x14a2e80374c8, node = &dict_index_t::indexes, init = 51966}, freed_indexes = {count = 0, start = 0x0, end = 0x0, node = &dict_index_t::indexes, init = 51966}, foreign_list = {count = 0, start = 0x0, end = 0x0, node = &dict_foreign_t::heap, init = 0}, referenced_list = {count = 0, start = 0x0, end = 0x0, node = &dict_foreign_t::heap, init = 0}, table_LRU = {prev = 0x14a21801d5a8, next = 0x14a36c023128}, fk_max_recusive_level = 0 '\000', def_trx_id = 0, bulk_trx_id = {m = std::atomic<unsigned long> = { 0 }}, mdl_name = {m_name = 0x14a200019e68 "tmp/#sql-temptable-347cae-48-2a"}, foreign_set = std::__debug::set with 0 elements, referenced_set = std::__debug::set with 0 elements, stat_initialized = 1, stats_last_recalc = 0, stat_persistent = 4, stats_auto_recalc = 0, stats_sample_pages = 0, stat_n_rows = 3, stat_clustered_index_size = 1, stat_sum_of_other_index_sizes = 0, stat_modified_counter = 3, stats_error_printed = false, autoinc_lock = 0x14a20003d038, autoinc_mutex = {lock = std::atomic<unsigned int> = { 0 }, static HOLDER = 1, static WAITER = 2}, lock_latch = {<srw_lock_impl<false>> = {pfs_psi = 0x0, lock = {writer = {lock = std::atomic<unsigned int> = { 0 }, static HOLDER = 1, static WAITER = 2}, readers = std::atomic<unsigned int> = { 0 }, static WRITER = 2147483648}}, writer = std::atomic<unsigned long> = { 0 }, readers_lock = {lock = std::atomic<unsigned int> = { 0 }, static HOLDER = 1, static WAITER = 2}, readers = std::atomic<class std::__debug::unordered_multiset<unsigned long, std::hash<unsigned long>, std::equal_to<unsigned long>, std::allocator<unsigned long> > *> = { 0x14a20001c590 }}, row_id = {m_counter = std::atomic<unsigned long> = { 3 }}, autoinc = 0, autoinc_trx = {m = std::atomic<const struct trx_t *> = { 0x0 }}, n_waiting_or_granted_auto_inc_locks = 0, n_lock_x_or_s = 0, fts = 0x0, quiesce = QUIESCE_NONE, n_rec_locks = {m_counter = std::atomic<unsigned int> = { 0 }}, n_ref_count = {m_counter = std::atomic<unsigned int> = { 0 }}, locks = {count = 0, start = 0x0, end = 0x0, node = &lock_table_t::locks, init = 51966}, update_time = {m = std::atomic<long> = { 1729220816 }}, query_cache_inv_trx_id = {m = std::atomic<unsigned long> = { 14689 }}, magic_n = 76333786, vc_templ = 0x0})at /test/11.4_dbg/storage/innobase/btr/btr0btr.cc:1132
          #12 0x000055e63dcaa9ab in ha_innobase::delete_table (this=<optimized out>, name=<optimized out>)at /test/11.4_dbg/storage/innobase/handler/ha_innodb.cc:13501
          #13 0x000055e63d93b6d3 in hton_drop_table (hton=<optimized out>, path=<optimized out>) at /test/11.4_dbg/sql/handler.cc:585
          #14 0x000055e63d88c40f in THD::rm_temporary_table (this=<optimized out>, base=0x55e63ffee898, path=0x14a20003c558 "/test/MD141024-mariadb-11.4.4-linux-x86_64-dbg/tmp/#sql-temptable-347cae-48-2a") at /test/11.4_dbg/sql/temporary_tables.cc:706
          #15 0x000055e63d88cb18 in THD::free_tmp_table_share (this=this@entry=0x14a200000d58, share=share@entry=0x14a20003bec8, delete_table=delete_table@entry=true)at /test/11.4_dbg/sql/temporary_tables.cc:1487
          #16 0x000055e63d88e30f in THD::drop_temporary_table (this=this@entry=0x14a200000d58, table=<optimized out>, is_trans=is_trans@entry=0x14a3e83fd477, delete_table=delete_table@entry=true)at /test/11.4_dbg/sql/temporary_tables.cc:674
          #17 0x000055e63d6f21d1 in mysql_rm_table_no_locks (thd=thd@entry=0x14a200000d58, tables=tables@entry=0x14a2000136e8, current_db=current_db@entry=0x14a200000df8, ddl_log_state=0x14a3e83fd4d0, ddl_log_state@entry=0x0, if_exists=if_exists@entry=false, drop_temporary=drop_temporary@entry=false, drop_view=false, drop_sequence=false, dont_log_query=false, dont_free_locks=false)at /test/11.4_dbg/sql/sql_table.cc:1458
          #18 0x000055e63d6f4772 in mysql_rm_table (thd=thd@entry=0x14a200000d58, tables=tables@entry=0x14a2000136e8, if_exists=false, drop_temporary=false, drop_sequence=false, dont_log_query=dont_log_query@entry=false)at /test/11.4_dbg/sql/sql_table.cc:1206
          #19 0x000055e63d611b97 in mysql_execute_command (thd=thd@entry=0x14a200000d58, is_called_from_prepared_stmt=is_called_from_prepared_stmt@entry=false)at /test/11.4_dbg/sql/sql_parse.cc:4768
          #20 0x000055e63d6161ab in mysql_parse (thd=thd@entry=0x14a200000d58, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0x14a3e83fe2a0)at /test/11.4_dbg/sql/sql_parse.cc:7880
          #21 0x000055e63d618663 in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0x14a200000d58, packet=packet@entry=0x14a20000b2c9 "DROP TABLE t1", packet_length=packet_length@entry=13, blocking=blocking@entry=true)at /test/11.4_dbg/sql/sql_class.h:254
          #22 0x000055e63d61a8a8 in do_command (thd=0x14a200000d58, blocking=blocking@entry=true) at /test/11.4_dbg/sql/sql_parse.cc:1407
          #23 0x000055e63d78a5ab in do_handle_one_connection (connect=<optimized out>, connect@entry=0x55e64061e698, put_in_cache=put_in_cache@entry=true)at /test/11.4_dbg/sql/sql_connect.cc:1439
          #24 0x000055e63d78a8b3 in handle_one_connection (arg=arg@entry=0x55e64061e698)at /test/11.4_dbg/sql/sql_connect.cc:1341
          #25 0x000055e63dbda7c0 in pfs_spawn_thread (arg=0x55e64061e708)at /test/11.4_dbg/storage/perfschema/pfs.cc:2201
          #26 0x000014a405e9ca94 in start_thread (arg=<optimized out>)at ./nptl/pthread_create.c:447
          #27 0x000014a405f29c3c in clone3 ()at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78
          

          CS 11.4.4 b53b81e937b7b598ee73657501ec87c6f76334ec (Debug)

          Core was generated by `/test/MD141024-mariadb-11.4.4-linux-x86_64-dbg/bin/mariadbd --no-defaults --max'.
          Program terminated with signal SIGABRT, Aborted.
          Download failed: Invalid argument.  Continuing without source file ./nptl/./nptl/pthread_kill.c.
          #0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44
           
          [Current thread is 1 (LWP 2118611)]
          (gdb) bt
          #0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44
          #1  __pthread_kill_internal (signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:78
          #2  __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6)at ./nptl/pthread_kill.c:89
          #3  0x000014c1d004526e in __GI_raise (sig=sig@entry=6)at ../sysdeps/posix/raise.c:26
          #4  0x000014c1d00288ff in __GI_abort () at ./stdlib/abort.c:79
          #5  0x000014c1d04a9133 in __gnu_debug::_Error_formatter::_M_error (this=this@entry=0x5574625e2c40 <__gnu_debug::_Error_formatter::_S_at(char const*, unsigned int, char const*)::__formatter>)at ../../../../../src/libstdc++-v3/src/c++11/debug.cc:1320
          #6  0x0000557461516f90 in __gnu_debug::_Safe_iterator<std::_Rb_tree_const_iterator<range_t>, std::__debug::set<range_t, range_compare, std::allocator<range_t> >, std::bidirectional_iterator_tag>::operator-- (this=0x14c1b171b550)at /usr/include/c++/13/debug/safe_iterator.h:621
          #7  std::__advance<__gnu_debug::_Safe_iterator<std::_Rb_tree_const_iterator<range_t>, std::__debug::set<range_t, range_compare, std::allocator<range_t> >, std::bidirectional_iterator_tag>, long> (__n=0, __i={first = 0, last = 0})at /usr/include/c++/13/bits/stl_iterator_base_funcs.h:179
          #8  std::advance<__gnu_debug::_Safe_iterator<std::_Rb_tree_const_iterator<range_t>, std::__debug::set<range_t, range_compare, std::allocator<range_t> >, std::bidirectional_iterator_tag>, long> (__n=-1, __i={first = 0, last = 0})at /usr/include/c++/13/bits/stl_iterator_base_funcs.h:224
          #9  std::prev<__gnu_debug::_Safe_iterator<std::_Rb_tree_const_iterator<range_t>, std::__debug::set<range_t, range_compare, std::allocator<range_t> >, std::bidirectional_iterator_tag> > (__n=1, __x={first = 0, last = 0})at /usr/include/c++/13/bits/stl_iterator_base_funcs.h:250
          #10 range_set::add_range (this=<optimized out>, new_range={first = 2118611, last = 0})at /test/11.4_dbg/storage/innobase/include/fil0fil.h:228
          #11 0x0000557461539772 in fil_space_t::add_free_range (range=<optimized out>, this=<optimized out>)at /test/11.4_dbg/storage/innobase/include/fil0fil.h:908
          #12 mtr_t::commit (this=this@entry=0x14c1b171b6a0)at /test/11.4_dbg/storage/innobase/mtr/mtr0mtr.cc:517
          #13 0x00005574616a9124 in trx_commit_cleanup (undo=@0x14c1cd430cc0: 0x14bf04043de8)at /test/11.4_dbg/storage/innobase/trx/trx0trx.cc:1371
          #14 0x00005574616aaa3d in trx_t::commit_in_memory (mtr=0x0, this=0x14c1cd430300) at /test/11.4_dbg/storage/innobase/trx/trx0trx.cc:1455
          #15 trx_t::commit_low (this=this@entry=0x14c1cd430300, mtr=0x0)at /test/11.4_dbg/storage/innobase/trx/trx0trx.cc:1596
          #16 0x00005574616ab22d in trx_t::commit_persist (this=this@entry=0x14c1cd430300)at /test/11.4_dbg/storage/innobase/trx/trx0trx.cc:1610
          #17 0x00005574616ab2bf in trx_t::commit (this=this@entry=0x14c1cd430300)at /test/11.4_dbg/storage/innobase/trx/trx0trx.cc:1619
          #18 0x00005574616ab94f in trx_commit_for_mysql (trx=trx@entry=0x14c1cd430300)at /test/11.4_dbg/storage/innobase/trx/trx0trx.cc:1739
          #19 0x000055746143d4c0 in innobase_commit_low (trx=trx@entry=0x14c1cd430300)at /test/11.4_dbg/storage/innobase/handler/ha_innodb.cc:4309
          #20 0x000055746143d50c in innobase_commit_ordered_2 (trx=trx@entry=0x14c1cd430300, thd=thd@entry=0x14bf04000d58)at /test/11.4_dbg/storage/innobase/handler/ha_innodb.cc:4410
          #21 0x000055746143d7c1 in innobase_commit (hton=<optimized out>, thd=0x14bf04000d58, commit_trx=false)at /test/11.4_dbg/storage/innobase/handler/ha_innodb.cc:4527
          #22 0x00005574610d5e36 in commit_one_phase_2 (thd=thd@entry=0x14bf04000d58, all=all@entry=false, trans=trans@entry=0x14bf04004858, is_real_trans=true)at /test/11.4_dbg/sql/handler.cc:2197
          #23 0x00005574610d600e in ha_commit_one_phase (thd=thd@entry=0x14bf04000d58, all=all@entry=false) at /test/11.4_dbg/sql/handler.cc:2150
          #24 0x00005574610d7ce4 in ha_commit_trans (thd=thd@entry=0x14bf04000d58, all=all@entry=false) at /test/11.4_dbg/sql/handler.cc:1944
          #25 0x0000557460f4d315 in trans_commit_stmt (thd=thd@entry=0x14bf04000d58)at /test/11.4_dbg/sql/transaction.cc:497
          #26 0x0000557460dc28f2 in mysql_execute_command (thd=thd@entry=0x14bf04000d58, is_called_from_prepared_stmt=is_called_from_prepared_stmt@entry=false)at /test/11.4_dbg/sql/sql_parse.cc:5937
          #27 0x0000557460dc31ab in mysql_parse (thd=thd@entry=0x14bf04000d58, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0x14c1b171d2a0)at /test/11.4_dbg/sql/sql_parse.cc:7880
          #28 0x0000557460dc5663 in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0x14bf04000d58, packet=packet@entry=0x14bf0400b2c9 "INSERT INTO t1  VALUES('a')", packet_length=packet_length@entry=27, blocking=blocking@entry=true)at /test/11.4_dbg/sql/sql_class.h:254
          #29 0x0000557460dc78a8 in do_command (thd=0x14bf04000d58, blocking=blocking@entry=true) at /test/11.4_dbg/sql/sql_parse.cc:1407
          #30 0x0000557460f375ab in do_handle_one_connection (connect=<optimized out>, connect@entry=0x55746491c388, put_in_cache=put_in_cache@entry=true)at /test/11.4_dbg/sql/sql_connect.cc:1439
          #31 0x0000557460f378b3 in handle_one_connection (arg=arg@entry=0x55746491c388)at /test/11.4_dbg/sql/sql_connect.cc:1341
          #32 0x00005574613877c0 in pfs_spawn_thread (arg=0x55746491c3f8)at /test/11.4_dbg/storage/perfschema/pfs.cc:2201
          #33 0x000014c1d009ca94 in start_thread (arg=<optimized out>)at ./nptl/pthread_create.c:447
          #34 0x000014c1d0129c3c in clone3 ()at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78
          

          CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug)

          mariadbd: /test/11.7_dbg/storage/innobase/buf/buf0lru.cc:400: buf_block_t* buf_LRU_get_free_block(buf_LRU_get): Assertion `LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()' failed.
          

          CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug)

          Core was generated by `/test/MD141024-mariadb-11.7.0-linux-x86_64-dbg/bin/mariadbd --no-defaults --max'.
          Program terminated with signal SIGABRT, Aborted.
          Download failed: Invalid argument.  Continuing without source file ./nptl/./nptl/pthread_kill.c.
          #0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44
           
          [Current thread is 1 (LWP 2445090)]
          (gdb) bt
          #0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44
          #1  __pthread_kill_internal (signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:78
          #2  __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6)at ./nptl/pthread_kill.c:89
          #3  0x00001520bea4526e in __GI_raise (sig=sig@entry=6)at ../sysdeps/posix/raise.c:26
          #4  0x00001520bea288ff in __GI_abort () at ./stdlib/abort.c:79
          #5  0x00001520bea2881b in __assert_fail_base (fmt=0x1520bebd01e8 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=assertion@entry=0x5644ddce29b0 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=file@entry=0x5644ddce1f48 "/test/11.7_dbg/storage/innobase/buf/buf0lru.cc", line=line@entry=400, function=function@entry=0x5644ddce2a00 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)") at ./assert/assert.c:94
          #6  0x00001520bea3b507 in __assert_fail (assertion=0x5644ddce29b0 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=0x5644ddce1f48 "/test/11.7_dbg/storage/innobase/buf/buf0lru.cc", line=400, function=0x5644ddce2a00 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)")at ./assert/assert.c:103
          #7  0x00005644dd7cfa0b in buf_LRU_get_free_block (get=get@entry=have_mutex)at /test/11.7_dbg/storage/innobase/buf/buf0lru.cc:400
          #8  0x00005644dd7d313f in buf_read_page (page_id=page_id@entry={m_id = 4294967343}, chain=@0x5644e088bbe8: {first = 0x0})at /test/11.7_dbg/storage/innobase/buf/buf0rea.cc:427
          #9  0x00005644dd7a7364 in buf_page_get_gen (page_id=<optimized out>, zip_size=zip_size@entry=0, rw_latch=rw_latch@entry=RW_X_LATCH, guess=guess@entry=0x0, mode=mode@entry=10, mtr=mtr@entry=0x15208d748d70, err=0x15208d748cdc) at /test/11.7_dbg/storage/innobase/buf/buf0buf.cc:2882
          #10 0x00005644dd73f225 in trx_undo_reuse_cached (trx=trx@entry=0x1520a8b0d900, rseg=rseg@entry=0x5644de5dcb00 <trx_sys+47616>, pundo=pundo@entry=0x1520a8b0e2b0, mtr=mtr@entry=0x15208d748d70, err=err@entry=0x15208d748cdc)at /test/11.7_dbg/storage/innobase/trx/trx0undo.cc:1291
          #11 0x00005644dd7427f0 in trx_undo_assign_low<false> (trx=trx@entry=0x1520a8b0d900, rseg=rseg@entry=0x5644de5dcb00 <trx_sys+47616>, undo=undo@entry=0x1520a8b0e2b0, mtr=mtr@entry=0x15208d748d70, err=err@entry=0x15208d748cdc)at /test/11.7_dbg/storage/innobase/trx/trx0undo.cc:1416
          #12 0x00005644dd7198a7 in trx_undo_report_row_operation (thr=thr@entry=0x1520540e9f68, index=index@entry=0x151aec0fcf98, clust_entry=clust_entry@entry=0x151c140db6a8, update=update@entry=0x0, cmpl_info=cmpl_info@entry=0, rec=rec@entry=0x0, offsets=0x0, roll_ptr=0x15208d7491a0)at /test/11.7_dbg/storage/innobase/trx/trx0rec.cc:1897
          #13 0x00005644dd7700be in btr_cur_ins_lock_and_undo (flags=flags@entry=0, cursor=cursor@entry=0x15208d749680, entry=entry@entry=0x151c140db6a8, thr=thr@entry=0x1520540e9f68, mtr=mtr@entry=0x15208d749c00, inherit=inherit@entry=0x15208d749286)at /test/11.7_dbg/storage/innobase/btr/btr0cur.cc:2200
          #14 0x00005644dd77707a in btr_cur_optimistic_insert (flags=flags@entry=0, cursor=cursor@entry=0x15208d749680, offsets=offsets@entry=0x15208d749668, heap=heap@entry=0x15208d749660, entry=entry@entry=0x151c140db6a8, rec=rec@entry=0x15208d749678, big_rec=0x15208d749658, n_ext=<optimized out>, thr=0x1520540e9f68, mtr=0x15208d749c00)at /test/11.7_dbg/storage/innobase/btr/btr0cur.cc:2424
          #15 0x00005644dd657350 in row_ins_clust_index_entry_low (flags=flags@entry=0, mode=mode@entry=BTR_MODIFY_LEAF, index=index@entry=0x151aec0fcf98, n_uniq=n_uniq@entry=0, entry=entry@entry=0x151c140db6a8, n_ext=n_ext@entry=0, thr=0x1520540e9f68)at /test/11.7_dbg/storage/innobase/row/row0ins.cc:2882
          #16 0x00005644dd658aca in row_ins_clust_index_entry (index=index@entry=0x151aec0fcf98, entry=entry@entry=0x151c140db6a8, thr=thr@entry=0x1520540e9f68, n_ext=n_ext@entry=0)at /test/11.7_dbg/storage/innobase/row/row0ins.cc:3267
          #17 0x00005644dd65a96f in row_ins_index_entry (thr=0x1520540e9f68, entry=0x151c140db6a8, index=0x151aec0fcf98)at /test/11.7_dbg/storage/innobase/row/row0ins.cc:3402
          #18 row_ins_index_entry_step (thr=0x1520540e9f68, node=0x1520540e9d08)at /test/11.7_dbg/storage/innobase/row/row0ins.cc:3570
          #19 row_ins (thr=0x1520540e9f68, node=0x1520540e9d08)at /test/11.7_dbg/storage/innobase/row/row0ins.cc:3687
          #20 row_ins_step (thr=thr@entry=0x1520540e9f68)at /test/11.7_dbg/storage/innobase/row/row0ins.cc:3816
          #21 0x00005644dd680f85 in row_insert_for_mysql (mysql_rec=mysql_rec@entry=0x1520540c3be8 "\376\001", prebuilt=0x1520540e9878, ins_mode=ins_mode@entry=ROW_INS_NORMAL)at /test/11.7_dbg/storage/innobase/row/row0mysql.cc:1290
          #22 0x00005644dd4d8057 in ha_innobase::write_row (this=0x1520540ee7b0, record=0x1520540c3be8 "\376\001")at /test/11.7_dbg/storage/innobase/handler/ha_innodb.cc:7767
          #23 0x00005644dd164995 in handler::ha_write_row (this=0x1520540ee7b0, buf=0x1520540c3be8 "\376\001") at /test/11.7_dbg/sql/handler.cc:8086
          #24 0x00005644dcde84ef in write_record (thd=thd@entry=0x151ba8000d58, table=table@entry=0x1520540ee368, info=info@entry=0x15208d74aa50, sink=sink@entry=0x0) at /test/11.7_dbg/sql/sql_insert.cc:2320
          #25 0x00005644dcdf328b in mysql_insert (thd=thd@entry=0x151ba8000d58, table_list=0x151ba8015918, fields=@0x151ba8006198: {<base_list> = {<Sql_alloc> = {<No data fields>}, first = 0x5644de696500 <end_of_list>, last = 0x151ba8006198, elements = 0}, <No data fields>}, values_list=@0x151ba80061e0: {<base_list> = {<Sql_alloc> = {<No data fields>}, first = 0x151ba80165a0, last = 0x151ba80165a0, elements = 1}, <No data fields>}, update_fields=@0x151ba80061c8: {<base_list> = {<Sql_alloc> = {<No data fields>}, first = 0x5644de696500 <end_of_list>, last = 0x151ba80061c8, elements = 0}, <No data fields>}, update_values=@0x151ba80061b0: {<base_list> = {<Sql_alloc> = {<No data fields>}, first = 0x5644de696500 <end_of_list>, last = 0x151ba80061b0, elements = 0}, <No data fields>}, duplic=DUP_ERROR, ignore=false, result=0x0)at /test/11.7_dbg/sql/sql_insert.cc:1177
          #26 0x00005644dce2f9f8 in mysql_execute_command (thd=thd@entry=0x151ba8000d58, is_called_from_prepared_stmt=is_called_from_prepared_stmt@entry=false)at /test/11.7_dbg/sql/sql_parse.cc:4459
          #27 0x00005644dce34d36 in mysql_parse (thd=thd@entry=0x151ba8000d58, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0x15208d74b260)at /test/11.7_dbg/sql/sql_parse.cc:7885
          #28 0x00005644dce371ee in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0x151ba8000d58, packet=packet@entry=0x151ba800b339 "INSERT INTO t1  VALUES('a')", packet_length=packet_length@entry=27, blocking=blocking@entry=true)at /test/11.7_dbg/sql/sql_class.h:257
          #29 0x00005644dce39552 in do_command (thd=0x151ba8000d58, blocking=blocking@entry=true) at /test/11.7_dbg/sql/sql_parse.cc:1405
          #30 0x00005644dcfb2e5f in do_handle_one_connection (connect=<optimized out>, connect@entry=0x5644e0e41d18, put_in_cache=put_in_cache@entry=true)at /test/11.7_dbg/sql/sql_connect.cc:1448
          #31 0x00005644dcfb3173 in handle_one_connection (arg=arg@entry=0x5644e0e41d18)at /test/11.7_dbg/sql/sql_connect.cc:1350
          #32 0x00005644dd408bcf in pfs_spawn_thread (arg=0x5644e0e41d88)at /test/11.7_dbg/storage/perfschema/pfs.cc:2198
          #33 0x00001520bea9ca94 in start_thread (arg=<optimized out>)at ./nptl/pthread_create.c:447
          #34 0x00001520beb29c3c in clone3 ()at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78
          

          There were all produced using both non-temporary and temporary table for t1 simultaneously in the input SQL (i.e. adding an additional line):

          CREATE TABLE t1(c1 LONGTEXT NULL);
          CREATE TEMPORARY TABLE t1(c1 LONGTEXT NULL);
          

          It makes the crashes more readily repeatable. In general, the SIGABRT in range_set::add_range and in from_range_set::add_range seems reproducible with a lower number of threads. For the LRU_size... assert, a higher number of threads (~1000) seems required.
          For 11.4, 100 threads were used, and --innodb_buffer_pool_size=5M
          For 11.7, 1000 threads were used, and a default innodb_buffer_pool_size size
          The second 11.4 stack above was also observed on 11.7

          Roel Roel Van de Paar added a comment - - edited Three additional stacks observed: CS 11.4.4 b53b81e937b7b598ee73657501ec87c6f76334ec (Debug) Core was generated by `/test/MD141024-mariadb-11.4.4-linux-x86_64-dbg/bin/mariadbd --no-defaults --max'. Program terminated with signal SIGABRT, Aborted. Download failed: Invalid argument. Continuing without source file ./nptl/./nptl/pthread_kill.c. #0 __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44   [Current thread is 1 (LWP 3441987)] (gdb) bt #0 __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44 #1 __pthread_kill_internal (signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:78 #2 __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6)at ./nptl/pthread_kill.c:89 #3 0x000014a405e4526e in __GI_raise (sig=sig@entry=6)at ../sysdeps/posix/raise.c:26 #4 0x000014a405e288ff in __GI_abort () at ./stdlib/abort.c:79 #5 0x000014a4062a9133 in __gnu_debug::_Error_formatter::_M_error (this=this@entry=0x55e63ee35c40 <__gnu_debug::_Error_formatter::_S_at(char const*, unsigned int, char const*)::__formatter>)at ../../../../../src/libstdc++-v3/src/c++11/debug.cc:1320 #6 0x000055e63dd684dc in std::__debug::set<range_t, range_compare, std::allocator<range_t> >::erase[abi:cxx11](__gnu_debug::_Safe_iterator<std::_Rb_tree_const_iterator<range_t>, std::__debug::set<range_t, range_compare, std::allocator<range_t> >, std::bidirectional_iterator_tag>) (this=this@entry=0x55e640306d30, __position={first = 261, last = 261})at /usr/include/c++/13/debug/set.h:347 #7 0x000055e63dd68a5f in range_set::add_within_range (this=this@entry=0x55e640306d30, range={first = 261, last = 261}, value=value@entry=262)at /test/11.4_dbg/storage/innobase/include/fil0fil.h:208 #8 0x000055e63dd6a264 in range_set::add_range (this=0x55e640306d30, new_range={first = 3441987, last = 0})at /test/11.4_dbg/storage/innobase/include/fil0fil.h:247 #9 0x000055e63dd8c772 in fil_space_t::add_free_range (range=<optimized out>, this=<optimized out>)at /test/11.4_dbg/storage/innobase/include/fil0fil.h:908 #10 mtr_t::commit (this=this@entry=0x14a3e83fc540)at /test/11.4_dbg/storage/innobase/mtr/mtr0mtr.cc:517 #11 0x000055e63df1958b in btr_drop_temporary_table (table=Python Exception <class 'AttributeError'>: 'NoneType' object has no attribute 'pointer' Python Exception <class 'AttributeError'>: 'NoneType' object has no attribute 'pointer' @0x14a38001df58: {id = 1891, id_hash = 0x0, name = {m_name = 0x14a200019e68 "tmp/#sql-temptable-347cae-48-2a"}, name_hash = 0x0, heap = 0x14a1e8038d78, data_dir_path = 0x0, space = 0x55e640306c28, space_id = 4294967294, flags = 33, flags2 = 65, skip_alter_undo = 0, file_unreadable = 0, cached = 1, n_def = 4, n_cols = 4, n_t_cols = 4, n_t_def = 4, n_v_def = 0, n_v_cols = 0, persistent_autoinc = 0, can_be_evicted = 0, corrupted = 0, drop_aborted = 0, cols = 0x14a20003cfb8, v_cols = 0x14a20003d038, s_cols = 0x0, instant = 0x0, col_names = 0x14a20003d090 "c1", v_col_names = 0x0, vers_start = 0, vers_end = 0, is_system_db = false, dict_frm_mismatch = DICT_FRM_CONSISTENT, fts_doc_id_index = 0x0, indexes = {count = 1, start = 0x14a2e80374c8, end = 0x14a2e80374c8, node = &dict_index_t::indexes, init = 51966}, freed_indexes = {count = 0, start = 0x0, end = 0x0, node = &dict_index_t::indexes, init = 51966}, foreign_list = {count = 0, start = 0x0, end = 0x0, node = &dict_foreign_t::heap, init = 0}, referenced_list = {count = 0, start = 0x0, end = 0x0, node = &dict_foreign_t::heap, init = 0}, table_LRU = {prev = 0x14a21801d5a8, next = 0x14a36c023128}, fk_max_recusive_level = 0 '\000', def_trx_id = 0, bulk_trx_id = {m = std::atomic<unsigned long> = { 0 }}, mdl_name = {m_name = 0x14a200019e68 "tmp/#sql-temptable-347cae-48-2a"}, foreign_set = std::__debug::set with 0 elements, referenced_set = std::__debug::set with 0 elements, stat_initialized = 1, stats_last_recalc = 0, stat_persistent = 4, stats_auto_recalc = 0, stats_sample_pages = 0, stat_n_rows = 3, stat_clustered_index_size = 1, stat_sum_of_other_index_sizes = 0, stat_modified_counter = 3, stats_error_printed = false, autoinc_lock = 0x14a20003d038, autoinc_mutex = {lock = std::atomic<unsigned int> = { 0 }, static HOLDER = 1, static WAITER = 2}, lock_latch = {<srw_lock_impl<false>> = {pfs_psi = 0x0, lock = {writer = {lock = std::atomic<unsigned int> = { 0 }, static HOLDER = 1, static WAITER = 2}, readers = std::atomic<unsigned int> = { 0 }, static WRITER = 2147483648}}, writer = std::atomic<unsigned long> = { 0 }, readers_lock = {lock = std::atomic<unsigned int> = { 0 }, static HOLDER = 1, static WAITER = 2}, readers = std::atomic<class std::__debug::unordered_multiset<unsigned long, std::hash<unsigned long>, std::equal_to<unsigned long>, std::allocator<unsigned long> > *> = { 0x14a20001c590 }}, row_id = {m_counter = std::atomic<unsigned long> = { 3 }}, autoinc = 0, autoinc_trx = {m = std::atomic<const struct trx_t *> = { 0x0 }}, n_waiting_or_granted_auto_inc_locks = 0, n_lock_x_or_s = 0, fts = 0x0, quiesce = QUIESCE_NONE, n_rec_locks = {m_counter = std::atomic<unsigned int> = { 0 }}, n_ref_count = {m_counter = std::atomic<unsigned int> = { 0 }}, locks = {count = 0, start = 0x0, end = 0x0, node = &lock_table_t::locks, init = 51966}, update_time = {m = std::atomic<long> = { 1729220816 }}, query_cache_inv_trx_id = {m = std::atomic<unsigned long> = { 14689 }}, magic_n = 76333786, vc_templ = 0x0})at /test/11.4_dbg/storage/innobase/btr/btr0btr.cc:1132 #12 0x000055e63dcaa9ab in ha_innobase::delete_table (this=<optimized out>, name=<optimized out>)at /test/11.4_dbg/storage/innobase/handler/ha_innodb.cc:13501 #13 0x000055e63d93b6d3 in hton_drop_table (hton=<optimized out>, path=<optimized out>) at /test/11.4_dbg/sql/handler.cc:585 #14 0x000055e63d88c40f in THD::rm_temporary_table (this=<optimized out>, base=0x55e63ffee898, path=0x14a20003c558 "/test/MD141024-mariadb-11.4.4-linux-x86_64-dbg/tmp/#sql-temptable-347cae-48-2a") at /test/11.4_dbg/sql/temporary_tables.cc:706 #15 0x000055e63d88cb18 in THD::free_tmp_table_share (this=this@entry=0x14a200000d58, share=share@entry=0x14a20003bec8, delete_table=delete_table@entry=true)at /test/11.4_dbg/sql/temporary_tables.cc:1487 #16 0x000055e63d88e30f in THD::drop_temporary_table (this=this@entry=0x14a200000d58, table=<optimized out>, is_trans=is_trans@entry=0x14a3e83fd477, delete_table=delete_table@entry=true)at /test/11.4_dbg/sql/temporary_tables.cc:674 #17 0x000055e63d6f21d1 in mysql_rm_table_no_locks (thd=thd@entry=0x14a200000d58, tables=tables@entry=0x14a2000136e8, current_db=current_db@entry=0x14a200000df8, ddl_log_state=0x14a3e83fd4d0, ddl_log_state@entry=0x0, if_exists=if_exists@entry=false, drop_temporary=drop_temporary@entry=false, drop_view=false, drop_sequence=false, dont_log_query=false, dont_free_locks=false)at /test/11.4_dbg/sql/sql_table.cc:1458 #18 0x000055e63d6f4772 in mysql_rm_table (thd=thd@entry=0x14a200000d58, tables=tables@entry=0x14a2000136e8, if_exists=false, drop_temporary=false, drop_sequence=false, dont_log_query=dont_log_query@entry=false)at /test/11.4_dbg/sql/sql_table.cc:1206 #19 0x000055e63d611b97 in mysql_execute_command (thd=thd@entry=0x14a200000d58, is_called_from_prepared_stmt=is_called_from_prepared_stmt@entry=false)at /test/11.4_dbg/sql/sql_parse.cc:4768 #20 0x000055e63d6161ab in mysql_parse (thd=thd@entry=0x14a200000d58, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0x14a3e83fe2a0)at /test/11.4_dbg/sql/sql_parse.cc:7880 #21 0x000055e63d618663 in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0x14a200000d58, packet=packet@entry=0x14a20000b2c9 "DROP TABLE t1", packet_length=packet_length@entry=13, blocking=blocking@entry=true)at /test/11.4_dbg/sql/sql_class.h:254 #22 0x000055e63d61a8a8 in do_command (thd=0x14a200000d58, blocking=blocking@entry=true) at /test/11.4_dbg/sql/sql_parse.cc:1407 #23 0x000055e63d78a5ab in do_handle_one_connection (connect=<optimized out>, connect@entry=0x55e64061e698, put_in_cache=put_in_cache@entry=true)at /test/11.4_dbg/sql/sql_connect.cc:1439 #24 0x000055e63d78a8b3 in handle_one_connection (arg=arg@entry=0x55e64061e698)at /test/11.4_dbg/sql/sql_connect.cc:1341 #25 0x000055e63dbda7c0 in pfs_spawn_thread (arg=0x55e64061e708)at /test/11.4_dbg/storage/perfschema/pfs.cc:2201 #26 0x000014a405e9ca94 in start_thread (arg=<optimized out>)at ./nptl/pthread_create.c:447 #27 0x000014a405f29c3c in clone3 ()at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78 CS 11.4.4 b53b81e937b7b598ee73657501ec87c6f76334ec (Debug) Core was generated by `/test/MD141024-mariadb-11.4.4-linux-x86_64-dbg/bin/mariadbd --no-defaults --max'. Program terminated with signal SIGABRT, Aborted. Download failed: Invalid argument. Continuing without source file ./nptl/./nptl/pthread_kill.c. #0 __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44   [Current thread is 1 (LWP 2118611)] (gdb) bt #0 __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44 #1 __pthread_kill_internal (signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:78 #2 __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6)at ./nptl/pthread_kill.c:89 #3 0x000014c1d004526e in __GI_raise (sig=sig@entry=6)at ../sysdeps/posix/raise.c:26 #4 0x000014c1d00288ff in __GI_abort () at ./stdlib/abort.c:79 #5 0x000014c1d04a9133 in __gnu_debug::_Error_formatter::_M_error (this=this@entry=0x5574625e2c40 <__gnu_debug::_Error_formatter::_S_at(char const*, unsigned int, char const*)::__formatter>)at ../../../../../src/libstdc++-v3/src/c++11/debug.cc:1320 #6 0x0000557461516f90 in __gnu_debug::_Safe_iterator<std::_Rb_tree_const_iterator<range_t>, std::__debug::set<range_t, range_compare, std::allocator<range_t> >, std::bidirectional_iterator_tag>::operator-- (this=0x14c1b171b550)at /usr/include/c++/13/debug/safe_iterator.h:621 #7 std::__advance<__gnu_debug::_Safe_iterator<std::_Rb_tree_const_iterator<range_t>, std::__debug::set<range_t, range_compare, std::allocator<range_t> >, std::bidirectional_iterator_tag>, long> (__n=0, __i={first = 0, last = 0})at /usr/include/c++/13/bits/stl_iterator_base_funcs.h:179 #8 std::advance<__gnu_debug::_Safe_iterator<std::_Rb_tree_const_iterator<range_t>, std::__debug::set<range_t, range_compare, std::allocator<range_t> >, std::bidirectional_iterator_tag>, long> (__n=-1, __i={first = 0, last = 0})at /usr/include/c++/13/bits/stl_iterator_base_funcs.h:224 #9 std::prev<__gnu_debug::_Safe_iterator<std::_Rb_tree_const_iterator<range_t>, std::__debug::set<range_t, range_compare, std::allocator<range_t> >, std::bidirectional_iterator_tag> > (__n=1, __x={first = 0, last = 0})at /usr/include/c++/13/bits/stl_iterator_base_funcs.h:250 #10 range_set::add_range (this=<optimized out>, new_range={first = 2118611, last = 0})at /test/11.4_dbg/storage/innobase/include/fil0fil.h:228 #11 0x0000557461539772 in fil_space_t::add_free_range (range=<optimized out>, this=<optimized out>)at /test/11.4_dbg/storage/innobase/include/fil0fil.h:908 #12 mtr_t::commit (this=this@entry=0x14c1b171b6a0)at /test/11.4_dbg/storage/innobase/mtr/mtr0mtr.cc:517 #13 0x00005574616a9124 in trx_commit_cleanup (undo=@0x14c1cd430cc0: 0x14bf04043de8)at /test/11.4_dbg/storage/innobase/trx/trx0trx.cc:1371 #14 0x00005574616aaa3d in trx_t::commit_in_memory (mtr=0x0, this=0x14c1cd430300) at /test/11.4_dbg/storage/innobase/trx/trx0trx.cc:1455 #15 trx_t::commit_low (this=this@entry=0x14c1cd430300, mtr=0x0)at /test/11.4_dbg/storage/innobase/trx/trx0trx.cc:1596 #16 0x00005574616ab22d in trx_t::commit_persist (this=this@entry=0x14c1cd430300)at /test/11.4_dbg/storage/innobase/trx/trx0trx.cc:1610 #17 0x00005574616ab2bf in trx_t::commit (this=this@entry=0x14c1cd430300)at /test/11.4_dbg/storage/innobase/trx/trx0trx.cc:1619 #18 0x00005574616ab94f in trx_commit_for_mysql (trx=trx@entry=0x14c1cd430300)at /test/11.4_dbg/storage/innobase/trx/trx0trx.cc:1739 #19 0x000055746143d4c0 in innobase_commit_low (trx=trx@entry=0x14c1cd430300)at /test/11.4_dbg/storage/innobase/handler/ha_innodb.cc:4309 #20 0x000055746143d50c in innobase_commit_ordered_2 (trx=trx@entry=0x14c1cd430300, thd=thd@entry=0x14bf04000d58)at /test/11.4_dbg/storage/innobase/handler/ha_innodb.cc:4410 #21 0x000055746143d7c1 in innobase_commit (hton=<optimized out>, thd=0x14bf04000d58, commit_trx=false)at /test/11.4_dbg/storage/innobase/handler/ha_innodb.cc:4527 #22 0x00005574610d5e36 in commit_one_phase_2 (thd=thd@entry=0x14bf04000d58, all=all@entry=false, trans=trans@entry=0x14bf04004858, is_real_trans=true)at /test/11.4_dbg/sql/handler.cc:2197 #23 0x00005574610d600e in ha_commit_one_phase (thd=thd@entry=0x14bf04000d58, all=all@entry=false) at /test/11.4_dbg/sql/handler.cc:2150 #24 0x00005574610d7ce4 in ha_commit_trans (thd=thd@entry=0x14bf04000d58, all=all@entry=false) at /test/11.4_dbg/sql/handler.cc:1944 #25 0x0000557460f4d315 in trans_commit_stmt (thd=thd@entry=0x14bf04000d58)at /test/11.4_dbg/sql/transaction.cc:497 #26 0x0000557460dc28f2 in mysql_execute_command (thd=thd@entry=0x14bf04000d58, is_called_from_prepared_stmt=is_called_from_prepared_stmt@entry=false)at /test/11.4_dbg/sql/sql_parse.cc:5937 #27 0x0000557460dc31ab in mysql_parse (thd=thd@entry=0x14bf04000d58, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0x14c1b171d2a0)at /test/11.4_dbg/sql/sql_parse.cc:7880 #28 0x0000557460dc5663 in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0x14bf04000d58, packet=packet@entry=0x14bf0400b2c9 "INSERT INTO t1 VALUES('a')", packet_length=packet_length@entry=27, blocking=blocking@entry=true)at /test/11.4_dbg/sql/sql_class.h:254 #29 0x0000557460dc78a8 in do_command (thd=0x14bf04000d58, blocking=blocking@entry=true) at /test/11.4_dbg/sql/sql_parse.cc:1407 #30 0x0000557460f375ab in do_handle_one_connection (connect=<optimized out>, connect@entry=0x55746491c388, put_in_cache=put_in_cache@entry=true)at /test/11.4_dbg/sql/sql_connect.cc:1439 #31 0x0000557460f378b3 in handle_one_connection (arg=arg@entry=0x55746491c388)at /test/11.4_dbg/sql/sql_connect.cc:1341 #32 0x00005574613877c0 in pfs_spawn_thread (arg=0x55746491c3f8)at /test/11.4_dbg/storage/perfschema/pfs.cc:2201 #33 0x000014c1d009ca94 in start_thread (arg=<optimized out>)at ./nptl/pthread_create.c:447 #34 0x000014c1d0129c3c in clone3 ()at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78 CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug) mariadbd: /test/11.7_dbg/storage/innobase/buf/buf0lru.cc:400: buf_block_t* buf_LRU_get_free_block(buf_LRU_get): Assertion `LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()' failed. CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug) Core was generated by `/test/MD141024-mariadb-11.7.0-linux-x86_64-dbg/bin/mariadbd --no-defaults --max'. Program terminated with signal SIGABRT, Aborted. Download failed: Invalid argument. Continuing without source file ./nptl/./nptl/pthread_kill.c. #0 __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44   [Current thread is 1 (LWP 2445090)] (gdb) bt #0 __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44 #1 __pthread_kill_internal (signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:78 #2 __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6)at ./nptl/pthread_kill.c:89 #3 0x00001520bea4526e in __GI_raise (sig=sig@entry=6)at ../sysdeps/posix/raise.c:26 #4 0x00001520bea288ff in __GI_abort () at ./stdlib/abort.c:79 #5 0x00001520bea2881b in __assert_fail_base (fmt=0x1520bebd01e8 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=assertion@entry=0x5644ddce29b0 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=file@entry=0x5644ddce1f48 "/test/11.7_dbg/storage/innobase/buf/buf0lru.cc", line=line@entry=400, function=function@entry=0x5644ddce2a00 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)") at ./assert/assert.c:94 #6 0x00001520bea3b507 in __assert_fail (assertion=0x5644ddce29b0 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=0x5644ddce1f48 "/test/11.7_dbg/storage/innobase/buf/buf0lru.cc", line=400, function=0x5644ddce2a00 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)")at ./assert/assert.c:103 #7 0x00005644dd7cfa0b in buf_LRU_get_free_block (get=get@entry=have_mutex)at /test/11.7_dbg/storage/innobase/buf/buf0lru.cc:400 #8 0x00005644dd7d313f in buf_read_page (page_id=page_id@entry={m_id = 4294967343}, chain=@0x5644e088bbe8: {first = 0x0})at /test/11.7_dbg/storage/innobase/buf/buf0rea.cc:427 #9 0x00005644dd7a7364 in buf_page_get_gen (page_id=<optimized out>, zip_size=zip_size@entry=0, rw_latch=rw_latch@entry=RW_X_LATCH, guess=guess@entry=0x0, mode=mode@entry=10, mtr=mtr@entry=0x15208d748d70, err=0x15208d748cdc) at /test/11.7_dbg/storage/innobase/buf/buf0buf.cc:2882 #10 0x00005644dd73f225 in trx_undo_reuse_cached (trx=trx@entry=0x1520a8b0d900, rseg=rseg@entry=0x5644de5dcb00 <trx_sys+47616>, pundo=pundo@entry=0x1520a8b0e2b0, mtr=mtr@entry=0x15208d748d70, err=err@entry=0x15208d748cdc)at /test/11.7_dbg/storage/innobase/trx/trx0undo.cc:1291 #11 0x00005644dd7427f0 in trx_undo_assign_low<false> (trx=trx@entry=0x1520a8b0d900, rseg=rseg@entry=0x5644de5dcb00 <trx_sys+47616>, undo=undo@entry=0x1520a8b0e2b0, mtr=mtr@entry=0x15208d748d70, err=err@entry=0x15208d748cdc)at /test/11.7_dbg/storage/innobase/trx/trx0undo.cc:1416 #12 0x00005644dd7198a7 in trx_undo_report_row_operation (thr=thr@entry=0x1520540e9f68, index=index@entry=0x151aec0fcf98, clust_entry=clust_entry@entry=0x151c140db6a8, update=update@entry=0x0, cmpl_info=cmpl_info@entry=0, rec=rec@entry=0x0, offsets=0x0, roll_ptr=0x15208d7491a0)at /test/11.7_dbg/storage/innobase/trx/trx0rec.cc:1897 #13 0x00005644dd7700be in btr_cur_ins_lock_and_undo (flags=flags@entry=0, cursor=cursor@entry=0x15208d749680, entry=entry@entry=0x151c140db6a8, thr=thr@entry=0x1520540e9f68, mtr=mtr@entry=0x15208d749c00, inherit=inherit@entry=0x15208d749286)at /test/11.7_dbg/storage/innobase/btr/btr0cur.cc:2200 #14 0x00005644dd77707a in btr_cur_optimistic_insert (flags=flags@entry=0, cursor=cursor@entry=0x15208d749680, offsets=offsets@entry=0x15208d749668, heap=heap@entry=0x15208d749660, entry=entry@entry=0x151c140db6a8, rec=rec@entry=0x15208d749678, big_rec=0x15208d749658, n_ext=<optimized out>, thr=0x1520540e9f68, mtr=0x15208d749c00)at /test/11.7_dbg/storage/innobase/btr/btr0cur.cc:2424 #15 0x00005644dd657350 in row_ins_clust_index_entry_low (flags=flags@entry=0, mode=mode@entry=BTR_MODIFY_LEAF, index=index@entry=0x151aec0fcf98, n_uniq=n_uniq@entry=0, entry=entry@entry=0x151c140db6a8, n_ext=n_ext@entry=0, thr=0x1520540e9f68)at /test/11.7_dbg/storage/innobase/row/row0ins.cc:2882 #16 0x00005644dd658aca in row_ins_clust_index_entry (index=index@entry=0x151aec0fcf98, entry=entry@entry=0x151c140db6a8, thr=thr@entry=0x1520540e9f68, n_ext=n_ext@entry=0)at /test/11.7_dbg/storage/innobase/row/row0ins.cc:3267 #17 0x00005644dd65a96f in row_ins_index_entry (thr=0x1520540e9f68, entry=0x151c140db6a8, index=0x151aec0fcf98)at /test/11.7_dbg/storage/innobase/row/row0ins.cc:3402 #18 row_ins_index_entry_step (thr=0x1520540e9f68, node=0x1520540e9d08)at /test/11.7_dbg/storage/innobase/row/row0ins.cc:3570 #19 row_ins (thr=0x1520540e9f68, node=0x1520540e9d08)at /test/11.7_dbg/storage/innobase/row/row0ins.cc:3687 #20 row_ins_step (thr=thr@entry=0x1520540e9f68)at /test/11.7_dbg/storage/innobase/row/row0ins.cc:3816 #21 0x00005644dd680f85 in row_insert_for_mysql (mysql_rec=mysql_rec@entry=0x1520540c3be8 "\376\001", prebuilt=0x1520540e9878, ins_mode=ins_mode@entry=ROW_INS_NORMAL)at /test/11.7_dbg/storage/innobase/row/row0mysql.cc:1290 #22 0x00005644dd4d8057 in ha_innobase::write_row (this=0x1520540ee7b0, record=0x1520540c3be8 "\376\001")at /test/11.7_dbg/storage/innobase/handler/ha_innodb.cc:7767 #23 0x00005644dd164995 in handler::ha_write_row (this=0x1520540ee7b0, buf=0x1520540c3be8 "\376\001") at /test/11.7_dbg/sql/handler.cc:8086 #24 0x00005644dcde84ef in write_record (thd=thd@entry=0x151ba8000d58, table=table@entry=0x1520540ee368, info=info@entry=0x15208d74aa50, sink=sink@entry=0x0) at /test/11.7_dbg/sql/sql_insert.cc:2320 #25 0x00005644dcdf328b in mysql_insert (thd=thd@entry=0x151ba8000d58, table_list=0x151ba8015918, fields=@0x151ba8006198: {<base_list> = {<Sql_alloc> = {<No data fields>}, first = 0x5644de696500 <end_of_list>, last = 0x151ba8006198, elements = 0}, <No data fields>}, values_list=@0x151ba80061e0: {<base_list> = {<Sql_alloc> = {<No data fields>}, first = 0x151ba80165a0, last = 0x151ba80165a0, elements = 1}, <No data fields>}, update_fields=@0x151ba80061c8: {<base_list> = {<Sql_alloc> = {<No data fields>}, first = 0x5644de696500 <end_of_list>, last = 0x151ba80061c8, elements = 0}, <No data fields>}, update_values=@0x151ba80061b0: {<base_list> = {<Sql_alloc> = {<No data fields>}, first = 0x5644de696500 <end_of_list>, last = 0x151ba80061b0, elements = 0}, <No data fields>}, duplic=DUP_ERROR, ignore=false, result=0x0)at /test/11.7_dbg/sql/sql_insert.cc:1177 #26 0x00005644dce2f9f8 in mysql_execute_command (thd=thd@entry=0x151ba8000d58, is_called_from_prepared_stmt=is_called_from_prepared_stmt@entry=false)at /test/11.7_dbg/sql/sql_parse.cc:4459 #27 0x00005644dce34d36 in mysql_parse (thd=thd@entry=0x151ba8000d58, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0x15208d74b260)at /test/11.7_dbg/sql/sql_parse.cc:7885 #28 0x00005644dce371ee in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0x151ba8000d58, packet=packet@entry=0x151ba800b339 "INSERT INTO t1 VALUES('a')", packet_length=packet_length@entry=27, blocking=blocking@entry=true)at /test/11.7_dbg/sql/sql_class.h:257 #29 0x00005644dce39552 in do_command (thd=0x151ba8000d58, blocking=blocking@entry=true) at /test/11.7_dbg/sql/sql_parse.cc:1405 #30 0x00005644dcfb2e5f in do_handle_one_connection (connect=<optimized out>, connect@entry=0x5644e0e41d18, put_in_cache=put_in_cache@entry=true)at /test/11.7_dbg/sql/sql_connect.cc:1448 #31 0x00005644dcfb3173 in handle_one_connection (arg=arg@entry=0x5644e0e41d18)at /test/11.7_dbg/sql/sql_connect.cc:1350 #32 0x00005644dd408bcf in pfs_spawn_thread (arg=0x5644e0e41d88)at /test/11.7_dbg/storage/perfschema/pfs.cc:2198 #33 0x00001520bea9ca94 in start_thread (arg=<optimized out>)at ./nptl/pthread_create.c:447 #34 0x00001520beb29c3c in clone3 ()at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78 There were all produced using both non-temporary and temporary table for t1 simultaneously in the input SQL (i.e. adding an additional line): CREATE TABLE t1(c1 LONGTEXT NULL ); CREATE TEMPORARY TABLE t1(c1 LONGTEXT NULL ); It makes the crashes more readily repeatable. In general, the SIGABRT in range_set::add_range and in from_range_set::add_range seems reproducible with a lower number of threads. For the LRU_size... assert, a higher number of threads (~1000) seems required. For 11.4, 100 threads were used, and --innodb_buffer_pool_size=5M For 11.7, 1000 threads were used, and a default innodb_buffer_pool_size size The second 11.4 stack above was also observed on 11.7

          I am not interested in crashes that involve XA PREPARE and TEMPORARY TABLE as long as MDEV-35046 has not been fixed. That will obviously lead to a "garbage in, garbage out" territory. I will try again with 100 concurrent connections and innodb_buffer_pool_size=5m.

          marko Marko Mäkelä added a comment - I am not interested in crashes that involve XA PREPARE and TEMPORARY TABLE as long as MDEV-35046 has not been fixed. That will obviously lead to a "garbage in, garbage out" territory. I will try again with 100 concurrent connections and innodb_buffer_pool_size=5m .

          I understand, though I am not sure they are connected. This issue is reproducible without TEMPORARY also (though less easy), and no SAN issues were discovered with the testcase.

          As a validation of reproducibility, you can try reproducing the TEMPORARY TABLE version (or even better both statements as mentioned above) first, as it is much easier to reproduce. For the LRU_size reproduction, I usually now use a high number of threads (1000) and default buffer pool size. The SIGABRT is much easier to reproduce using TEMPORARY with 50 threads and a 5m buffer pool.

          Issue occurence seems connected to the server starting to stall (not sure if it is due to buffer pool filling up), i.e. [some or all] threads starting to get long State durations like 300-1200 seconds.

          Roel Roel Van de Paar added a comment - I understand, though I am not sure they are connected. This issue is reproducible without TEMPORARY also (though less easy), and no SAN issues were discovered with the testcase. As a validation of reproducibility, you can try reproducing the TEMPORARY TABLE version (or even better both statements as mentioned above) first, as it is much easier to reproduce. For the LRU_size reproduction, I usually now use a high number of threads (1000) and default buffer pool size. The SIGABRT is much easier to reproduce using TEMPORARY with 50 threads and a 5m buffer pool. Issue occurence seems connected to the server starting to stall (not sure if it is due to buffer pool filling up), i.e. [some or all] threads starting to get long State durations like 300-1200 seconds.
          Roel Roel Van de Paar added a comment - - edited

          Reproduced

          CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug)

          LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()|SIGABRT|buf_LRU_get_free_block|buf_read_page|buf_page_get_gen|trx_undo_reuse_cached
          

          With 100 threads, default buffer pool, and without TEMPORARY, using this somewhat altered testcase:

          #!/bin/bash
          # Set variables and ensure ramloc is a ramdisk or tmpfs (i.e. /dev/shm)
           
          ramloc="/dev/shm"
          client="./bin/mariadb"
          socket="./socket.sock"
          errorlog="./log/master.err"
          user="root"
          db="test"
          cli_opt='-A --skip-ssl --binary-mode --force'  # Do not modify
          background=1    # Set to 0 to debug any failed client connections
          threads=100     # Number of concurrent threads
          queries=100000  # Number of queries per round
          rounds=999999   # Number of max test rounds
          sql="
          CREATE TABLE t1(c1 LONGTEXT NULL, c2 LONGTEXT NULL);
          CREATE TABLE t2(c1 LONGTEXT NULL, c2 LONGTEXT NULL);
          CREATE TABLE t3(c1 LONGTEXT NULL, c2 LONGTEXT NULL);
          DROP TABLE t1;
          DROP TABLE t2;
          DROP TABLE t3;
          INSERT INTO t1 VALUES('a');
          INSERT INTO t2 VALUES('a');
          INSERT INTO t3 VALUES('a');
          SET @@global.innodb_immediate_scrub_data_uncompressed=0;
          SET @@global.innodb_immediate_scrub_data_uncompressed=1;
          SET @@global.innodb_lru_scan_depth = 1536;
          SET @@global.innodb_lru_scan_depth = 86400;
          XA COMMIT 'a';
          XA END 'a';
          XA PREPARE 'a';
          XA START 'a' RESUME;
          XA START 'a';
          "
           
          exec_sql(){
            for ((i=0;i<$[ ${queries} / ${sql_len} ];i++)); do
              echo "${sql}" >> ${ramloc}/${prefix}_md_proc_${1}.tmp
            done
            shuf --random-source=/dev/urandom ${ramloc}/${prefix}_md_proc_${1}.tmp > ${ramloc}/${prefix}_md_proc_${1}
            rm -f ${ramloc}/${prefix}_md_proc_${1}.tmp
            ${client} ${cli_opt} -u ${user} -S ${socket} -D ${db} < ${ramloc}/${prefix}_md_proc_${1} >/dev/null 2>&1
            rm -f ${ramloc}/${prefix}_md_proc_${1}  # Thread done
          }
           
          abort(){ jobs -p | xargs -P50 kill >/dev/null 2>&1; rm -Rf ${ramloc}/${prefix}_md_proc_*; exit 1; }
          trap abort SIGINT
           
          count=0
          sql="$(echo "${sql}" | grep --binary-files=text -vE '^[ \t]*$')"
          RANDOM=$(date +%s%N | cut -b10-19 | sed 's|^[0]\+||')  # Random entropy init
          sql_len="$(echo "${sql}" | wc -l)"
          prefix="$(echo "${RANDOM}${RANDOM}${RANDOM}" | cut -b1-5)"
          echo "Input sql lenght: ${sql_len} lines | Runtime files: ${ramloc}/${prefix}_md_proc_*"
          rm -f ${ramloc}/${prefix}_md_proc_*
          for ((i=0;i<${rounds};i++)); do
            for ((i=0;i<${threads};i++)); do
              if [ ! -r ${ramloc}/${prefix}_md_proc_${i} ]; then  # Thread idle
                touch ${ramloc}/${prefix}_md_proc_${i}  # Thread busy
                if [ ${background} -eq 1 ]; then
                  exec_sql ${i} &
                else
                  exec_sql ${i}
                fi
                count=$[ ${count} + 1 ]
                if [ $[ ${count} % 100 ] -eq 0 ]; then  # Limit disk I/O, check once every new 100 threads
                  echo "${count} threads processing ${queries} queries each: $[ ${count} * ${queries} ] total" | tee lastcount.log
                  TAIL="$(tail -n10 ${errorlog} | tr -d '\n')"
                  if [[ "${TAIL}" == *"ERROR"* ]]; then
                    echo '*** Error found:'
                    grep --binary-files=text -i 'ERROR' log/master.err
                    abort
                  elif [[ "${TAIL}" == *"down complete"* ]]; then
                    echo '*** Server was shutdown'
                    abort
                  elif ! ${client}-admin ping -u ${user} -S ${socket} > /dev/null 2>&1; then
                    echo '*** Server gone (killed/crashed)'
                    abort
                  fi
                fi
              fi
            done
          done
          

          i.e. 3 Tables, two columns and inserts on all tables. This script also has significantly less output (only thread counts, no errors - please remove the >/dev/null 2>&1 for the client call if you want to debug/see any client output.

          Update: realized that the INSERT statements are not valid (due to single col insert vs double col def), so the INSERTs are likely not needed, making the issue more interesting.

          Roel Roel Van de Paar added a comment - - edited Reproduced CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug) LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()|SIGABRT|buf_LRU_get_free_block|buf_read_page|buf_page_get_gen|trx_undo_reuse_cached With 100 threads, default buffer pool, and without TEMPORARY , using this somewhat altered testcase: #!/bin/bash # Set variables and ensure ramloc is a ramdisk or tmpfs (i.e. /dev/shm) ramloc= "/dev/shm" client= "./bin/mariadb" socket= "./socket.sock" errorlog= "./log/master.err" user= "root" db= "test" cli_opt= '-A --skip-ssl --binary-mode --force' # Do not modify background=1 # Set to 0 to debug any failed client connections threads=100 # Number of concurrent threads queries=100000 # Number of queries per round rounds=999999 # Number of max test rounds sql=" CREATE TABLE t1(c1 LONGTEXT NULL, c2 LONGTEXT NULL); CREATE TABLE t2(c1 LONGTEXT NULL, c2 LONGTEXT NULL); CREATE TABLE t3(c1 LONGTEXT NULL, c2 LONGTEXT NULL); DROP TABLE t1; DROP TABLE t2; DROP TABLE t3; INSERT INTO t1 VALUES( 'a' ); INSERT INTO t2 VALUES( 'a' ); INSERT INTO t3 VALUES( 'a' ); SET @@global.innodb_immediate_scrub_data_uncompressed=0; SET @@global.innodb_immediate_scrub_data_uncompressed=1; SET @@global.innodb_lru_scan_depth = 1536; SET @@global.innodb_lru_scan_depth = 86400; XA COMMIT 'a' ; XA END 'a' ; XA PREPARE 'a' ; XA START 'a' RESUME; XA START 'a' ; " exec_sql(){ for ((i=0;i<$[ ${queries} / ${sql_len} ];i++)); do echo "${sql}" >> ${ramloc}/${prefix}_md_proc_${1}.tmp done shuf --random- source = /dev/urandom ${ramloc}/${prefix}_md_proc_${1}.tmp > ${ramloc}/${prefix}_md_proc_${1} rm -f ${ramloc}/${prefix}_md_proc_${1}.tmp ${client} ${cli_opt} -u ${user} -S ${socket} -D ${db} < ${ramloc}/${prefix}_md_proc_${1} > /dev/null 2>&1 rm -f ${ramloc}/${prefix}_md_proc_${1} # Thread done } abort(){ jobs -p | xargs -P50 kill > /dev/null 2>&1; rm -Rf ${ramloc}/${prefix}_md_proc_*; exit 1; } trap abort SIGINT count=0 sql= "$(echo " ${sql} " | grep --binary-files=text -vE '^[ \t]*$')" RANDOM=$( date +%s%N | cut -b10-19 | sed 's|^[0]\+||' ) # Random entropy init sql_len= "$(echo " ${sql} " | wc -l)" prefix= "$(echo " ${RANDOM}${RANDOM}${RANDOM} " | cut -b1-5)" echo "Input sql lenght: ${sql_len} lines | Runtime files: ${ramloc}/${prefix}_md_proc_*" rm -f ${ramloc}/${prefix}_md_proc_* for ((i=0;i<${rounds};i++)); do for ((i=0;i<${threads};i++)); do if [ ! -r ${ramloc}/${prefix}_md_proc_${i} ]; then # Thread idle touch ${ramloc}/${prefix}_md_proc_${i} # Thread busy if [ ${background} - eq 1 ]; then exec_sql ${i} & else exec_sql ${i} fi count=$[ ${count} + 1 ] if [ $[ ${count} % 100 ] - eq 0 ]; then # Limit disk I/O, check once every new 100 threads echo "${count} threads processing ${queries} queries each: $[ ${count} * ${queries} ] total" | tee lastcount.log TAIL= "$(tail -n10 ${errorlog} | tr -d '\n')" if [[ "${TAIL}" == * "ERROR" * ]]; then echo '*** Error found:' grep --binary-files=text -i 'ERROR' log /master .err abort elif [[ "${TAIL}" == * "down complete" * ]]; then echo '*** Server was shutdown' abort elif ! ${client}-admin ping -u ${user} -S ${socket} > /dev/null 2>&1; then echo '*** Server gone (killed/crashed)' abort fi fi fi done done i.e. 3 Tables, two columns and inserts on all tables . This script also has significantly less output (only thread counts, no errors - please remove the >/dev/null 2>&1 for the client call if you want to debug/see any client output. Update: realized that the INSERT statements are not valid (due to single col insert vs double col def), so the INSERTs are likely not needed, making the issue more interesting.
          Roel Roel Van de Paar added a comment - - edited

          Confirmed that INSERT statements are not needed: produced new stack:

          CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug)

          LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()|SIGABRT|buf_LRU_get_free_block|buf_read_page|buf_page_get_gen|trx_rseg_t::get
          

          With the testcase in the last comment, with the sql variable set to:

          sql="
          CREATE TABLE t1(c1 LONGTEXT NULL, c2 LONGTEXT NULL);
          CREATE TABLE t2(c1 LONGTEXT NULL, c2 LONGTEXT NULL);
          CREATE TABLE t3(c1 LONGTEXT NULL, c2 LONGTEXT NULL);
          DROP TABLE t1;
          DROP TABLE t2;
          DROP TABLE t3;
          SET @@global.innodb_immediate_scrub_data_uncompressed=0;
          SET @@global.innodb_immediate_scrub_data_uncompressed=1;
          SET @@global.innodb_lru_scan_depth = 1536;
          SET @@global.innodb_lru_scan_depth = 86400;
          XA COMMIT 'a';
          XA END 'a';
          XA PREPARE 'a';
          XA START 'a' RESUME;
          XA START 'a';
          "
          

          Issue production also seems faster now at about 1-5 minutes. Full stack for this new offset trace:

          CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug)

          mariadbd: /test/11.7_dbg/storage/innobase/buf/buf0lru.cc:400: buf_block_t* buf_LRU_get_free_block(buf_LRU_get): Assertion `LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()' failed.
          

          CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug)

          Core was generated by `/test/MD141024-mariadb-11.7.0-linux-x86_64-dbg/bin/mariadbd --no-defaults --max'.
          Program terminated with signal SIGABRT, Aborted.
          Download failed: Invalid argument.  Continuing without source file ./nptl/./nptl/pthread_kill.c.
          #0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44
           
          [Current thread is 1 (LWP 580385)]
          (gdb) bt
          #0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44
          #1  __pthread_kill_internal (signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:78
          #2  __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6)at ./nptl/pthread_kill.c:89
          #3  0x00001506aa64526e in __GI_raise (sig=sig@entry=6)at ../sysdeps/posix/raise.c:26
          #4  0x00001506aa6288ff in __GI_abort () at ./stdlib/abort.c:79
          #5  0x00001506aa62881b in __assert_fail_base (fmt=0x1506aa7d01e8 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=assertion@entry=0x5618e4e189b0 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=file@entry=0x5618e4e17f48 "/test/11.7_dbg/storage/innobase/buf/buf0lru.cc", line=line@entry=400, function=function@entry=0x5618e4e18a00 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)") at ./assert/assert.c:94
          #6  0x00001506aa63b507 in __assert_fail (assertion=0x5618e4e189b0 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=0x5618e4e17f48 "/test/11.7_dbg/storage/innobase/buf/buf0lru.cc", line=400, function=0x5618e4e18a00 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)")at ./assert/assert.c:103
          #7  0x00005618e4905a0b in buf_LRU_get_free_block (get=get@entry=have_mutex)at /test/11.7_dbg/storage/innobase/buf/buf0lru.cc:400
          #8  0x00005618e490913f in buf_read_page (page_id=page_id@entry={m_id = 12884901924}, chain=@0x5618e6b46b50: {first = 0x0})at /test/11.7_dbg/storage/innobase/buf/buf0rea.cc:427
          #9  0x00005618e48dd364 in buf_page_get_gen (page_id=<optimized out>, zip_size=zip_size@entry=0, rw_latch=rw_latch@entry=RW_X_LATCH, guess=guess@entry=0x0, mode=mode@entry=10, mtr=mtr@entry=0x150694e820f0, err=0x0) at /test/11.7_dbg/storage/innobase/buf/buf0buf.cc:2882
          #10 0x00005618e485995f in trx_rseg_t::get (this=this@entry=0x5618e5711f00 <trx_sys+44544>, mtr=mtr@entry=0x150694e820f0, err=err@entry=0x0)at /test/11.7_dbg/storage/innobase/trx/trx0rseg.cc:306
          #11 0x00005618e4835a1c in trx_purge_add_undo_to_history (trx=trx@entry=0x150694a49580, undo=@0x150694a49f30: 0x15051c033988, mtr=mtr@entry=0x150694e820f0)at /test/11.7_dbg/storage/innobase/trx/trx0purge.cc:179
          #12 0x00005618e486d468 in trx_t::write_serialisation_history (this=this@entry=0x150694a49580, mtr=mtr@entry=0x150694e820f0)at /test/11.7_dbg/storage/innobase/trx/trx0trx.cc:1177
          #13 0x00005618e48653e6 in trx_t::commit_low (this=this@entry=0x150694a49580, mtr=0x150694e820f0) at /test/11.7_dbg/storage/innobase/trx/trx0trx.cc:1588
          #14 0x00005618e486556b in trx_t::commit_persist (this=this@entry=0x150694a49580)at /test/11.7_dbg/storage/innobase/trx/trx0trx.cc:1616
          #15 0x00005618e48655fd in trx_t::commit (this=this@entry=0x150694a49580)at /test/11.7_dbg/storage/innobase/trx/trx0trx.cc:1625
          #16 0x00005618e4959e9c in dict_stats_save (table_orig=table_orig@entry=0x1505e802de68, only_for_index=only_for_index@entry=0x0)at /test/11.7_dbg/storage/innobase/dict/dict0stats.cc:3409
          #17 0x00005618e495c79b in dict_stats_update (table=0x1505e802de68, stats_upd_option=stats_upd_option@entry=DICT_STATS_EMPTY_TABLE)at /test/11.7_dbg/storage/innobase/dict/dict0stats.cc:4074
          #18 0x00005618e4619ae5 in ha_innobase::create (this=<optimized out>, name=<optimized out>, form=0x150694e83600, create_info=0x150694e85230, file_per_table=<optimized out>, trx=0x150694a48a00, trx@entry=0x0)at /test/11.7_dbg/storage/innobase/handler/ha_innodb.h:713
          #19 0x00005618e461b2e7 in ha_innobase::create (this=<optimized out>, name=<optimized out>, form=<optimized out>, create_info=<optimized out>)at /test/11.7_dbg/storage/innobase/handler/ha_innodb.cc:13236
          #20 0x00005618e4293942 in handler::ha_create (this=0x1505cc02dfb0, name=0x150694e84bb0 "./test/t2", form=form@entry=0x150694e83600, info_arg=info_arg@entry=0x150694e85230)at /test/11.7_dbg/sql/handler.cc:5877
          #21 0x00005618e42999d0 in ha_create_table (thd=thd@entry=0x1505cc000d58, path=<optimized out>, db=<optimized out>, table_name=<optimized out>, create_info=create_info@entry=0x150694e85230, frm=frm@entry=0x150694e84ba0, skip_frm_file=<optimized out>)at /test/11.7_dbg/sql/handler.cc:6364
          #22 0x00005618e406162a in create_table_impl (thd=thd@entry=0x1505cc000d58, ddl_log_state_create=ddl_log_state_create@entry=0x150694e84eb0, ddl_log_state_rm=<optimized out>, ddl_log_state_rm@entry=0x150694e850c0, orig_db=@0x1505cc015988: {<Lex_ident_fs> = {<Lex_ident<Compare_table_names>> = {<Lex_cstring> = {<st_mysql_const_lex_string> = {str = 0x1505cc015960 "test", length = 4}, <No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, orig_table_name=@0x1505cc015998: {<Lex_ident_fs> = {<Lex_ident<Compare_table_names>> = {<Lex_cstring> = {<st_mysql_const_lex_string> = {str = 0x1505cc015930 "t2", length = 2}, <No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, db=@0x1505cc015988: {str = 0x1505cc015960 "test", length = 4}, table_name=@0x1505cc015998: {str = 0x1505cc015930 "t2", length = 2}, path=@0x150694e84b90: {str = 0x150694e84bb0 "./test/t2", length = 9}, options=<optimized out>, create_info=0x150694e85230, alter_info=0x150694e84f50, create_table_mode=0, is_trans=0x150694e84e70, key_info=0x150694e84b88, key_count=0x150694e84b84, frm=0x150694e84ba0)at /test/11.7_dbg/sql/sql_table.cc:4683
          #23 0x00005618e4061b4f in mysql_create_table_no_lock (thd=thd@entry=0x1505cc000d58, ddl_log_state_create=ddl_log_state_create@entry=0x150694e84eb0, ddl_log_state_rm=ddl_log_state_rm@entry=0x150694e850c0, create_info=create_info@entry=0x150694e85230, alter_info=alter_info@entry=0x150694e84f50, is_trans=is_trans@entry=0x150694e84e70, create_table_mode=0, table_list=0x1505cc015970) at /test/11.7_dbg/sql/sql_table.cc:4786
          #24 0x00005618e4063c70 in mysql_create_table (alter_info=0x150694e84f50, create_info=0x150694e85230, create_table=0x1505cc015970, thd=0x1505cc000d58) at /test/11.7_dbg/sql/sql_table.cc:5012
          #25 Sql_cmd_create_table_like::execute (this=<optimized out>, thd=0x1505cc000d58) at /test/11.7_dbg/sql/sql_table.cc:13260
          #26 0x00005618e3f69b5b in mysql_execute_command (thd=thd@entry=0x1505cc000d58, is_called_from_prepared_stmt=is_called_from_prepared_stmt@entry=false)at /test/11.7_dbg/sql/sql_parse.cc:5858
          #27 0x00005618e3f6ad36 in mysql_parse (thd=thd@entry=0x1505cc000d58, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0x150694e87260)at /test/11.7_dbg/sql/sql_parse.cc:7885
          #28 0x00005618e3f6d1ee in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0x1505cc000d58, packet=packet@entry=0x1505cc00b339 "CREATE TABLE t2(c1 LONGTEXT NULL, c2 LONGTEXT NULL)", packet_length=packet_length@entry=51, blocking=blocking@entry=true) at /test/11.7_dbg/sql/sql_class.h:257
          #29 0x00005618e3f6f552 in do_command (thd=0x1505cc000d58, blocking=blocking@entry=true) at /test/11.7_dbg/sql/sql_parse.cc:1405
          #30 0x00005618e40e8e5f in do_handle_one_connection (connect=<optimized out>, connect@entry=0x5618e6fc0848, put_in_cache=put_in_cache@entry=true)at /test/11.7_dbg/sql/sql_connect.cc:1448
          #31 0x00005618e40e9173 in handle_one_connection (arg=arg@entry=0x5618e6fc0848)at /test/11.7_dbg/sql/sql_connect.cc:1350
          #32 0x00005618e453ebcf in pfs_spawn_thread (arg=0x5618e70bbec8)at /test/11.7_dbg/storage/perfschema/pfs.cc:2198
          #33 0x00001506aa69ca94 in start_thread (arg=<optimized out>)at ./nptl/pthread_create.c:447
          #34 0x00001506aa729c3c in clone3 ()at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78
          

          Roel Roel Van de Paar added a comment - - edited Confirmed that INSERT statements are not needed: produced new stack: CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug) LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()|SIGABRT|buf_LRU_get_free_block|buf_read_page|buf_page_get_gen|trx_rseg_t::get With the testcase in the last comment, with the sql variable set to: sql=" CREATE TABLE t1(c1 LONGTEXT NULL, c2 LONGTEXT NULL); CREATE TABLE t2(c1 LONGTEXT NULL, c2 LONGTEXT NULL); CREATE TABLE t3(c1 LONGTEXT NULL, c2 LONGTEXT NULL); DROP TABLE t1; DROP TABLE t2; DROP TABLE t3; SET @@global.innodb_immediate_scrub_data_uncompressed=0; SET @@global.innodb_immediate_scrub_data_uncompressed=1; SET @@global.innodb_lru_scan_depth = 1536; SET @@global.innodb_lru_scan_depth = 86400; XA COMMIT 'a' ; XA END 'a' ; XA PREPARE 'a' ; XA START 'a' RESUME; XA START 'a' ; " Issue production also seems faster now at about 1-5 minutes. Full stack for this new offset trace: CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug) mariadbd: /test/11.7_dbg/storage/innobase/buf/buf0lru.cc:400: buf_block_t* buf_LRU_get_free_block(buf_LRU_get): Assertion `LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()' failed. CS 11.7.0 4016c905cbabea7f29ed282dc2125254c7c0d419 (Debug) Core was generated by `/test/MD141024-mariadb-11.7.0-linux-x86_64-dbg/bin/mariadbd --no-defaults --max'. Program terminated with signal SIGABRT, Aborted. Download failed: Invalid argument. Continuing without source file ./nptl/./nptl/pthread_kill.c. #0 __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44   [Current thread is 1 (LWP 580385)] (gdb) bt #0 __pthread_kill_implementation (no_tid=0, signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:44 #1 __pthread_kill_internal (signo=6, threadid=<optimized out>)at ./nptl/pthread_kill.c:78 #2 __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6)at ./nptl/pthread_kill.c:89 #3 0x00001506aa64526e in __GI_raise (sig=sig@entry=6)at ../sysdeps/posix/raise.c:26 #4 0x00001506aa6288ff in __GI_abort () at ./stdlib/abort.c:79 #5 0x00001506aa62881b in __assert_fail_base (fmt=0x1506aa7d01e8 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=assertion@entry=0x5618e4e189b0 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=file@entry=0x5618e4e17f48 "/test/11.7_dbg/storage/innobase/buf/buf0lru.cc", line=line@entry=400, function=function@entry=0x5618e4e18a00 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)") at ./assert/assert.c:94 #6 0x00001506aa63b507 in __assert_fail (assertion=0x5618e4e189b0 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=0x5618e4e17f48 "/test/11.7_dbg/storage/innobase/buf/buf0lru.cc", line=400, function=0x5618e4e18a00 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)")at ./assert/assert.c:103 #7 0x00005618e4905a0b in buf_LRU_get_free_block (get=get@entry=have_mutex)at /test/11.7_dbg/storage/innobase/buf/buf0lru.cc:400 #8 0x00005618e490913f in buf_read_page (page_id=page_id@entry={m_id = 12884901924}, chain=@0x5618e6b46b50: {first = 0x0})at /test/11.7_dbg/storage/innobase/buf/buf0rea.cc:427 #9 0x00005618e48dd364 in buf_page_get_gen (page_id=<optimized out>, zip_size=zip_size@entry=0, rw_latch=rw_latch@entry=RW_X_LATCH, guess=guess@entry=0x0, mode=mode@entry=10, mtr=mtr@entry=0x150694e820f0, err=0x0) at /test/11.7_dbg/storage/innobase/buf/buf0buf.cc:2882 #10 0x00005618e485995f in trx_rseg_t::get (this=this@entry=0x5618e5711f00 <trx_sys+44544>, mtr=mtr@entry=0x150694e820f0, err=err@entry=0x0)at /test/11.7_dbg/storage/innobase/trx/trx0rseg.cc:306 #11 0x00005618e4835a1c in trx_purge_add_undo_to_history (trx=trx@entry=0x150694a49580, undo=@0x150694a49f30: 0x15051c033988, mtr=mtr@entry=0x150694e820f0)at /test/11.7_dbg/storage/innobase/trx/trx0purge.cc:179 #12 0x00005618e486d468 in trx_t::write_serialisation_history (this=this@entry=0x150694a49580, mtr=mtr@entry=0x150694e820f0)at /test/11.7_dbg/storage/innobase/trx/trx0trx.cc:1177 #13 0x00005618e48653e6 in trx_t::commit_low (this=this@entry=0x150694a49580, mtr=0x150694e820f0) at /test/11.7_dbg/storage/innobase/trx/trx0trx.cc:1588 #14 0x00005618e486556b in trx_t::commit_persist (this=this@entry=0x150694a49580)at /test/11.7_dbg/storage/innobase/trx/trx0trx.cc:1616 #15 0x00005618e48655fd in trx_t::commit (this=this@entry=0x150694a49580)at /test/11.7_dbg/storage/innobase/trx/trx0trx.cc:1625 #16 0x00005618e4959e9c in dict_stats_save (table_orig=table_orig@entry=0x1505e802de68, only_for_index=only_for_index@entry=0x0)at /test/11.7_dbg/storage/innobase/dict/dict0stats.cc:3409 #17 0x00005618e495c79b in dict_stats_update (table=0x1505e802de68, stats_upd_option=stats_upd_option@entry=DICT_STATS_EMPTY_TABLE)at /test/11.7_dbg/storage/innobase/dict/dict0stats.cc:4074 #18 0x00005618e4619ae5 in ha_innobase::create (this=<optimized out>, name=<optimized out>, form=0x150694e83600, create_info=0x150694e85230, file_per_table=<optimized out>, trx=0x150694a48a00, trx@entry=0x0)at /test/11.7_dbg/storage/innobase/handler/ha_innodb.h:713 #19 0x00005618e461b2e7 in ha_innobase::create (this=<optimized out>, name=<optimized out>, form=<optimized out>, create_info=<optimized out>)at /test/11.7_dbg/storage/innobase/handler/ha_innodb.cc:13236 #20 0x00005618e4293942 in handler::ha_create (this=0x1505cc02dfb0, name=0x150694e84bb0 "./test/t2", form=form@entry=0x150694e83600, info_arg=info_arg@entry=0x150694e85230)at /test/11.7_dbg/sql/handler.cc:5877 #21 0x00005618e42999d0 in ha_create_table (thd=thd@entry=0x1505cc000d58, path=<optimized out>, db=<optimized out>, table_name=<optimized out>, create_info=create_info@entry=0x150694e85230, frm=frm@entry=0x150694e84ba0, skip_frm_file=<optimized out>)at /test/11.7_dbg/sql/handler.cc:6364 #22 0x00005618e406162a in create_table_impl (thd=thd@entry=0x1505cc000d58, ddl_log_state_create=ddl_log_state_create@entry=0x150694e84eb0, ddl_log_state_rm=<optimized out>, ddl_log_state_rm@entry=0x150694e850c0, orig_db=@0x1505cc015988: {<Lex_ident_fs> = {<Lex_ident<Compare_table_names>> = {<Lex_cstring> = {<st_mysql_const_lex_string> = {str = 0x1505cc015960 "test", length = 4}, <No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, orig_table_name=@0x1505cc015998: {<Lex_ident_fs> = {<Lex_ident<Compare_table_names>> = {<Lex_cstring> = {<st_mysql_const_lex_string> = {str = 0x1505cc015930 "t2", length = 2}, <No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, db=@0x1505cc015988: {str = 0x1505cc015960 "test", length = 4}, table_name=@0x1505cc015998: {str = 0x1505cc015930 "t2", length = 2}, path=@0x150694e84b90: {str = 0x150694e84bb0 "./test/t2", length = 9}, options=<optimized out>, create_info=0x150694e85230, alter_info=0x150694e84f50, create_table_mode=0, is_trans=0x150694e84e70, key_info=0x150694e84b88, key_count=0x150694e84b84, frm=0x150694e84ba0)at /test/11.7_dbg/sql/sql_table.cc:4683 #23 0x00005618e4061b4f in mysql_create_table_no_lock (thd=thd@entry=0x1505cc000d58, ddl_log_state_create=ddl_log_state_create@entry=0x150694e84eb0, ddl_log_state_rm=ddl_log_state_rm@entry=0x150694e850c0, create_info=create_info@entry=0x150694e85230, alter_info=alter_info@entry=0x150694e84f50, is_trans=is_trans@entry=0x150694e84e70, create_table_mode=0, table_list=0x1505cc015970) at /test/11.7_dbg/sql/sql_table.cc:4786 #24 0x00005618e4063c70 in mysql_create_table (alter_info=0x150694e84f50, create_info=0x150694e85230, create_table=0x1505cc015970, thd=0x1505cc000d58) at /test/11.7_dbg/sql/sql_table.cc:5012 #25 Sql_cmd_create_table_like::execute (this=<optimized out>, thd=0x1505cc000d58) at /test/11.7_dbg/sql/sql_table.cc:13260 #26 0x00005618e3f69b5b in mysql_execute_command (thd=thd@entry=0x1505cc000d58, is_called_from_prepared_stmt=is_called_from_prepared_stmt@entry=false)at /test/11.7_dbg/sql/sql_parse.cc:5858 #27 0x00005618e3f6ad36 in mysql_parse (thd=thd@entry=0x1505cc000d58, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0x150694e87260)at /test/11.7_dbg/sql/sql_parse.cc:7885 #28 0x00005618e3f6d1ee in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0x1505cc000d58, packet=packet@entry=0x1505cc00b339 "CREATE TABLE t2(c1 LONGTEXT NULL, c2 LONGTEXT NULL)", packet_length=packet_length@entry=51, blocking=blocking@entry=true) at /test/11.7_dbg/sql/sql_class.h:257 #29 0x00005618e3f6f552 in do_command (thd=0x1505cc000d58, blocking=blocking@entry=true) at /test/11.7_dbg/sql/sql_parse.cc:1405 #30 0x00005618e40e8e5f in do_handle_one_connection (connect=<optimized out>, connect@entry=0x5618e6fc0848, put_in_cache=put_in_cache@entry=true)at /test/11.7_dbg/sql/sql_connect.cc:1448 #31 0x00005618e40e9173 in handle_one_connection (arg=arg@entry=0x5618e6fc0848)at /test/11.7_dbg/sql/sql_connect.cc:1350 #32 0x00005618e453ebcf in pfs_spawn_thread (arg=0x5618e70bbec8)at /test/11.7_dbg/storage/perfschema/pfs.cc:2198 #33 0x00001506aa69ca94 in start_thread (arg=<optimized out>)at ./nptl/pthread_create.c:447 #34 0x00001506aa729c3c in clone3 ()at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78

          In the file SIGABRT_from_range_set::add_within_range_11.4/gdb.txt that you shared along with a core dump, the error seems to be related to a dropping a TEMPORARY TABLE (btr_drop_temporary_table() invoking fil_space_t::add_free_range(), which would trip some assertion that is probably enabled by _GLIBCXX_ASSERTIONS (MDEV-33974). That one I would disqualify because of MDEV-35046. I suspect that this bug will result in something like a double free. With an rr replay trace it could be debugged in more detail.

          I have some trouble accessing the core dump and the libraries, because the directory structure was not preserved. I think that the following would be a step to the right direction:

          gdb --ex 'set solib-absolute-prefix /dev/shm/mariadbd_and_libs_11.4' --ex 'file mariadbd_and_libs_11.4/mariadbd' --ex 'core-file SIGABRT_from_range_set::add_range_11.4/core'
          

          This would fail to load the correct libraries, as follows:

          warning: Can't open file /usr/lib/x86_64-linux-gnu/libzstd.so.1.5.5 during file-backed mapping note processing
          warning: Can't open file /usr/lib/x86_64-linux-gnu/libc.so.6 during file-backed mapping note processing
          warning: Can't open file /usr/lib/x86_64-linux-gnu/libgpg-error.so.0.34.0 during file-backed mapping note processing
          warning: Can't open file /usr/lib/x86_64-linux-gnu/libgcrypt.so.20.4.3 during file-backed mapping note processing
          warning: Can't open file /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.33 during file-backed mapping note processing
          warning: Can't open file /usr/lib/x86_64-linux-gnu/libgcc_s.so.1 during file-backed mapping note processing
          warning: Can't open file /usr/lib/x86_64-linux-gnu/liblzma.so.5.4.5 during file-backed mapping note processing
          warning: Can't open file /usr/lib/x86_64-linux-gnu/liblz4.so.1.9.4 during file-backed mapping note processing
          warning: Can't open file /usr/lib/x86_64-linux-gnu/libcap.so.2.66 during file-backed mapping note processing
          warning: Can't open file /usr/lib/x86_64-linux-gnu/libm.so.6 during file-backed mapping note processing
          warning: Can't open file /usr/lib/x86_64-linux-gnu/libsystemd.so.0.38.0 during file-backed mapping note processing
          warning: Can't open file /usr/lib/liburing.so.2.2 during file-backed mapping note processing
          warning: Can't open file /usr/lib/x86_64-linux-gnu/libcrypt.so.1.1.0 during file-backed mapping note processing
          warning: Can't open file /test/MD141024-mariadb-11.4.4-linux-x86_64-dbg/data/tc.log during file-backed mapping note processing
          warning: Can't open file /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 during file-backed mapping note processing
          warning: Can't open file /test/MD141024-mariadb-11.4.4-linux-x86_64-dbg/bin/mariadbd during file-backed mapping note processing
          

          To fix this (trying to access another core file that was in the archive), I created an expected directory structure as follows:

          mkdir -p 11.7/test/MD141024-mariadb-11.7.0-linux-x86_64-dbg/bin 11.7/usr/lib/x86_64-linux-gnu
          mv mariadbd_and_libs_11.7/mariadbd 11.7/test/MD141024-mariadb-11.7.0-linux-x86_64-dbg/bin/
          mv mariadbd_and_libs_11.7/l* 11.7/usr/lib/x86_64-linux-gnu
          gdb --ex 'set solib-absolute-prefix /dev/shm/11.7' --ex 'file 11.7/test/MD141024-mariadb-11.7.0-linux-x86_64-dbg/bin/mariadbd' --ex 'core-file SIGABRT_from_range_set::add_range_11.7/core'
          

          This is still not enough, because the file names are too short:

          warning: Can't open file /usr/lib/x86_64-linux-gnu/libgcrypt.so.20.4.3 during file-backed mapping note processing
          warning: Can't open file /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.33 during file-backed mapping note processing
          warning: Can't open file /usr/lib/x86_64-linux-gnu/libgpg-error.so.0.34.0 during file-backed mapping note processing
          warning: Can't open file /usr/lib/x86_64-linux-gnu/liblzma.so.5.4.5 during file-backed mapping note processing
          warning: Can't open file /usr/lib/x86_64-linux-gnu/liblz4.so.1.9.4 during file-backed mapping note processing
          warning: Can't open file /usr/lib/x86_64-linux-gnu/libcap.so.2.66 during file-backed mapping note processing
          warning: Can't open file /usr/lib/x86_64-linux-gnu/libsystemd.so.0.38.0 during file-backed mapping note processing
          warning: Can't open file /usr/lib/liburing.so.2.2 during file-backed mapping note processing
          warning: Can't open file /usr/lib/x86_64-linux-gnu/libcrypt.so.1.1.0 during file-backed mapping note processing
          warning: Can't open file /test/MD141024-mariadb-11.7.0-linux-x86_64-dbg/data/tc.log during file-backed mapping note processing
          warning: Can't open file /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 during file-backed mapping note processing
          

          So, I will have to rename some libraries and hope that hey are correct:

          mv 11.7/usr/lib/x86_64-linux-gnu/libcrypt.so.1{,.1.0}
          mv 11.7/usr/lib/x86_64-linux-gnu/libgcrypt.so.20{,.4.3}
          mv 11.7/usr/lib/x86_64-linux-gnu/libstdc++.so.6{,.0.33}
          mv 11.7/usr/lib/x86_64-linux-gnu/libgpg-error.so.0{,.34.0}
          mv 11.7/usr/lib/x86_64-linux-gnu/liblzma.so.5{,.4.5}
          mv 11.7/usr/lib/x86_64-linux-gnu/liblz4.so.1{,.9.4}
          mv 11.7/usr/lib/x86_64-linux-gnu/libzstd.so.1{,.5.5}
          mv 11.7/usr/lib/x86_64-linux-gnu/libcap.so.2{,.66}
          mv 11.7/usr/lib/x86_64-linux-gnu/libsystemd.so.0{,.38.0}
          mv 11.7/usr/lib/x86_64-linux-gnu/liburing.so 11.7/usr/lib/liburing.so.2.2
          

          After all these adjustments, GDB would no longer complain about any misplaced libraries, but info sharedlibrary expectedly reports that no debugging symbols were loaded (or provided in the archive). Most output in thread apply all backtrace is garbage, including for the crashing thread. There are only a few threads with a partly readable stack trace:

          Thread 6 (LWP 472254):
          #0  0x000014585ea98d61 in ?? ()
          #1  0x0000000000000818 in ?? ()
          #2  0x00001458491ffb70 in ?? ()
          #3  0x000055cbff009988 in COND_checkpoint ()
          #4  0x000001895eaad6e4 in ?? ()
          #5  0x00001458491ffa30 in ?? ()
          #6  0x000055cbff009960 in ?? ()
          #7  0x000055cbff009984 in COND_checkpoint ()
          #8  0x0000000000000000 in ?? ()
          

          This I suspect is because some of the libraries had been built with -fomit-frame-pointer and therefore a successful stack unwinding would require the corresponding -dbgsym or -debuginfo packages to be available, for example, in /usr/lib/debug/.build-id/.

          The file Assertion_LRU_size_etc_11.7/gdb.txt inside the archive is more promising. I see that InnoDB has not run out of the buffer pool, so this should be some previously unknown bug. The following is the interesting part of the stack trace:

          #6  0x00001520bea3b507 in __assert_fail (assertion=0x5644ddce29b0 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=0x5644ddce1f48 "/test/11.7_dbg/storage/innobase/buf/buf0lru.cc", line=400, function=0x5644ddce2a00 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)") at ./assert/assert.c:103
          No locals.
          #7  0x00005644dd7cfa0b in buf_LRU_get_free_block (get=get@entry=have_mutex) at /test/11.7_dbg/storage/innobase/buf/buf0lru.cc:400
                  LRU_size = <optimized out>
                  available = <optimized out>
                  scan_depth = <optimized out>
                  signalled = <optimized out>
                  waited = <optimized out>
                  block = 0x1520a8202150
                  __PRETTY_FUNCTION__ = "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)"
          #8  0x00005644dd7d313f in buf_read_page (page_id=page_id@entry={m_id = 4294967343}, chain=@0x5644e088bbe8: {first = 0x0}) at /test/11.7_dbg/storage/innobase/buf/buf0rea.cc:427
                  space = 0x5644e0865488
                  block = 0x0
                  zip_size = 0
                  err = <optimized out>
          #9  0x00005644dd7a7364 in buf_page_get_gen (page_id=<optimized out>, zip_size=zip_size@entry=0, rw_latch=rw_latch@entry=RW_X_LATCH, guess=guess@entry=0x0, mode=mode@entry=10, mtr=mtr@entry=0x15208d748d70, err=0x15208d748cdc) at /test/11.7_dbg/storage/innobase/buf/buf0buf.cc:2882
                  local_err = <optimized out>
                  loop = <optimized out>
                  retries = 0
                  __PRETTY_FUNCTION__ = "buf_block_t* buf_page_get_gen(page_id_t, ulint, rw_lock_type_t, buf_block_t*, ulint, mtr_t*, dberr_t*)"
                  stats = 0x0
                  chain = @0x5644e088bbe8: {first = 0x0}
                  hash_lock = @0x5644e088bbc0: {lk = {writer = {lock = std::atomic<unsigned int> = { 0 }}, readers = std::atomic<unsigned int> = { 0 }}}
                  block = 0x0
                  state = <optimized out>
          #10 0x00005644dd73f225 in trx_undo_reuse_cached (trx=trx@entry=0x1520a8b0d900, rseg=rseg@entry=0x5644de5dcb00 <trx_sys+47616>, pundo=pundo@entry=0x1520a8b0e2b0, mtr=mtr@entry=0x15208d748d70, err=err@entry=0x15208d748cdc) at /test/11.7_dbg/storage/innobase/trx/trx0undo.cc:1291
                  __PRETTY_FUNCTION__ = "buf_block_t* trx_undo_reuse_cached(trx_t*, trx_rseg_t*, trx_undo_t**, mtr_t*, dberr_t*)"
                  undo = 0x1520780b0628
                  block = <optimized out>
                  offset = <optimized out>
          

          The interesting variables are not part of the output, and I don’t think they would even be available in the CPU registers. They might be if the stack traces could be unwound (and all needed debugging information would have been included).

          I will try to reproduce the failure myself.

          marko Marko Mäkelä added a comment - In the file SIGABRT_from_range_set::add_within_range_11.4/gdb.txt that you shared along with a core dump, the error seems to be related to a dropping a TEMPORARY TABLE ( btr_drop_temporary_table() invoking fil_space_t::add_free_range() , which would trip some assertion that is probably enabled by _GLIBCXX_ASSERTIONS ( MDEV-33974 ). That one I would disqualify because of MDEV-35046 . I suspect that this bug will result in something like a double free. With an rr replay trace it could be debugged in more detail. I have some trouble accessing the core dump and the libraries, because the directory structure was not preserved. I think that the following would be a step to the right direction: gdb --ex 'set solib-absolute-prefix /dev/shm/mariadbd_and_libs_11.4' --ex 'file mariadbd_and_libs_11.4/mariadbd' --ex 'core-file SIGABRT_from_range_set::add_range_11.4/core' This would fail to load the correct libraries, as follows: warning: Can't open file /usr/lib/x86_64-linux-gnu/libzstd.so.1.5.5 during file-backed mapping note processing warning: Can't open file /usr/lib/x86_64-linux-gnu/libc.so.6 during file-backed mapping note processing warning: Can't open file /usr/lib/x86_64-linux-gnu/libgpg-error.so.0.34.0 during file-backed mapping note processing warning: Can't open file /usr/lib/x86_64-linux-gnu/libgcrypt.so.20.4.3 during file-backed mapping note processing warning: Can't open file /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.33 during file-backed mapping note processing warning: Can't open file /usr/lib/x86_64-linux-gnu/libgcc_s.so.1 during file-backed mapping note processing warning: Can't open file /usr/lib/x86_64-linux-gnu/liblzma.so.5.4.5 during file-backed mapping note processing warning: Can't open file /usr/lib/x86_64-linux-gnu/liblz4.so.1.9.4 during file-backed mapping note processing warning: Can't open file /usr/lib/x86_64-linux-gnu/libcap.so.2.66 during file-backed mapping note processing warning: Can't open file /usr/lib/x86_64-linux-gnu/libm.so.6 during file-backed mapping note processing warning: Can't open file /usr/lib/x86_64-linux-gnu/libsystemd.so.0.38.0 during file-backed mapping note processing warning: Can't open file /usr/lib/liburing.so.2.2 during file-backed mapping note processing warning: Can't open file /usr/lib/x86_64-linux-gnu/libcrypt.so.1.1.0 during file-backed mapping note processing warning: Can't open file /test/MD141024-mariadb-11.4.4-linux-x86_64-dbg/data/tc.log during file-backed mapping note processing warning: Can't open file /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 during file-backed mapping note processing warning: Can't open file /test/MD141024-mariadb-11.4.4-linux-x86_64-dbg/bin/mariadbd during file-backed mapping note processing To fix this (trying to access another core file that was in the archive), I created an expected directory structure as follows: mkdir -p 11.7/test/MD141024-mariadb-11.7.0-linux-x86_64-dbg/bin 11.7/usr/lib/x86_64-linux-gnu mv mariadbd_and_libs_11.7/mariadbd 11.7/test/MD141024-mariadb-11.7.0-linux-x86_64-dbg/bin/ mv mariadbd_and_libs_11.7/l* 11.7/usr/lib/x86_64-linux-gnu gdb --ex 'set solib-absolute-prefix /dev/shm/11.7' --ex 'file 11.7/test/MD141024-mariadb-11.7.0-linux-x86_64-dbg/bin/mariadbd' --ex 'core-file SIGABRT_from_range_set::add_range_11.7/core' This is still not enough, because the file names are too short: warning: Can't open file /usr/lib/x86_64-linux-gnu/libgcrypt.so.20.4.3 during file-backed mapping note processing warning: Can't open file /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.33 during file-backed mapping note processing warning: Can't open file /usr/lib/x86_64-linux-gnu/libgpg-error.so.0.34.0 during file-backed mapping note processing warning: Can't open file /usr/lib/x86_64-linux-gnu/liblzma.so.5.4.5 during file-backed mapping note processing warning: Can't open file /usr/lib/x86_64-linux-gnu/liblz4.so.1.9.4 during file-backed mapping note processing warning: Can't open file /usr/lib/x86_64-linux-gnu/libcap.so.2.66 during file-backed mapping note processing warning: Can't open file /usr/lib/x86_64-linux-gnu/libsystemd.so.0.38.0 during file-backed mapping note processing warning: Can't open file /usr/lib/liburing.so.2.2 during file-backed mapping note processing warning: Can't open file /usr/lib/x86_64-linux-gnu/libcrypt.so.1.1.0 during file-backed mapping note processing warning: Can't open file /test/MD141024-mariadb-11.7.0-linux-x86_64-dbg/data/tc.log during file-backed mapping note processing warning: Can't open file /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 during file-backed mapping note processing So, I will have to rename some libraries and hope that hey are correct: mv 11.7 /usr/lib/x86_64-linux-gnu/libcrypt .so.1{,.1.0} mv 11.7 /usr/lib/x86_64-linux-gnu/libgcrypt .so.20{,.4.3} mv 11.7 /usr/lib/x86_64-linux-gnu/libstdc ++.so.6{,.0.33} mv 11.7 /usr/lib/x86_64-linux-gnu/libgpg-error .so.0{,.34.0} mv 11.7 /usr/lib/x86_64-linux-gnu/liblzma .so.5{,.4.5} mv 11.7 /usr/lib/x86_64-linux-gnu/liblz4 .so.1{,.9.4} mv 11.7 /usr/lib/x86_64-linux-gnu/libzstd .so.1{,.5.5} mv 11.7 /usr/lib/x86_64-linux-gnu/libcap .so.2{,.66} mv 11.7 /usr/lib/x86_64-linux-gnu/libsystemd .so.0{,.38.0} mv 11.7 /usr/lib/x86_64-linux-gnu/liburing .so 11.7 /usr/lib/liburing .so.2.2 After all these adjustments, GDB would no longer complain about any misplaced libraries, but info sharedlibrary expectedly reports that no debugging symbols were loaded (or provided in the archive). Most output in thread apply all backtrace is garbage, including for the crashing thread. There are only a few threads with a partly readable stack trace: Thread 6 (LWP 472254): #0 0x000014585ea98d61 in ?? () #1 0x0000000000000818 in ?? () #2 0x00001458491ffb70 in ?? () #3 0x000055cbff009988 in COND_checkpoint () #4 0x000001895eaad6e4 in ?? () #5 0x00001458491ffa30 in ?? () #6 0x000055cbff009960 in ?? () #7 0x000055cbff009984 in COND_checkpoint () #8 0x0000000000000000 in ?? () This I suspect is because some of the libraries had been built with -fomit-frame-pointer and therefore a successful stack unwinding would require the corresponding -dbgsym or -debuginfo packages to be available, for example, in /usr/lib/debug/.build-id/ . The file Assertion_LRU_size_etc_11.7/gdb.txt inside the archive is more promising. I see that InnoDB has not run out of the buffer pool, so this should be some previously unknown bug. The following is the interesting part of the stack trace: #6 0x00001520bea3b507 in __assert_fail (assertion=0x5644ddce29b0 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=0x5644ddce1f48 "/test/11.7_dbg/storage/innobase/buf/buf0lru.cc", line=400, function=0x5644ddce2a00 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)") at ./assert/assert.c:103 No locals. #7 0x00005644dd7cfa0b in buf_LRU_get_free_block (get=get@entry=have_mutex) at /test/11.7_dbg/storage/innobase/buf/buf0lru.cc:400 LRU_size = <optimized out> available = <optimized out> scan_depth = <optimized out> signalled = <optimized out> waited = <optimized out> block = 0x1520a8202150 __PRETTY_FUNCTION__ = "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)" #8 0x00005644dd7d313f in buf_read_page (page_id=page_id@entry={m_id = 4294967343}, chain=@0x5644e088bbe8: {first = 0x0}) at /test/11.7_dbg/storage/innobase/buf/buf0rea.cc:427 space = 0x5644e0865488 block = 0x0 zip_size = 0 err = <optimized out> #9 0x00005644dd7a7364 in buf_page_get_gen (page_id=<optimized out>, zip_size=zip_size@entry=0, rw_latch=rw_latch@entry=RW_X_LATCH, guess=guess@entry=0x0, mode=mode@entry=10, mtr=mtr@entry=0x15208d748d70, err=0x15208d748cdc) at /test/11.7_dbg/storage/innobase/buf/buf0buf.cc:2882 local_err = <optimized out> loop = <optimized out> retries = 0 __PRETTY_FUNCTION__ = "buf_block_t* buf_page_get_gen(page_id_t, ulint, rw_lock_type_t, buf_block_t*, ulint, mtr_t*, dberr_t*)" stats = 0x0 chain = @0x5644e088bbe8: {first = 0x0} hash_lock = @0x5644e088bbc0: {lk = {writer = {lock = std::atomic<unsigned int> = { 0 }}, readers = std::atomic<unsigned int> = { 0 }}} block = 0x0 state = <optimized out> #10 0x00005644dd73f225 in trx_undo_reuse_cached (trx=trx@entry=0x1520a8b0d900, rseg=rseg@entry=0x5644de5dcb00 <trx_sys+47616>, pundo=pundo@entry=0x1520a8b0e2b0, mtr=mtr@entry=0x15208d748d70, err=err@entry=0x15208d748cdc) at /test/11.7_dbg/storage/innobase/trx/trx0undo.cc:1291 __PRETTY_FUNCTION__ = "buf_block_t* trx_undo_reuse_cached(trx_t*, trx_rseg_t*, trx_undo_t**, mtr_t*, dberr_t*)" undo = 0x1520780b0628 block = <optimized out> offset = <optimized out> The interesting variables are not part of the output, and I don’t think they would even be available in the CPU registers. They might be if the stack traces could be unwound (and all needed debugging information would have been included). I will try to reproduce the failure myself.

          I tried to reproduce this on 11.2 ebefef658ebff14d55ebb068a0e4af3a6b647d44 on big iron (152 hardware threads) using the script that you posted 7 hours ago. The optimized CMAKE_BUILD_TYPE=Debug build of mariadbd has been occupying between 300% and 330% of one CPU. I interrupted it after 145 CPU minutes (about 46 minutes wall-clock time) because there was no failure.

          Can you reproduce this on 10.11 or 11.2? I would pretty much need an rr replay trace to debug this, and for that I would suggest to use much fewer concurrent connections. Based on my test, the server should be kept equally busy with 3 or 4 concurrent connections.

          marko Marko Mäkelä added a comment - I tried to reproduce this on 11.2 ebefef658ebff14d55ebb068a0e4af3a6b647d44 on big iron (152 hardware threads) using the script that you posted 7 hours ago. The optimized CMAKE_BUILD_TYPE=Debug build of mariadbd has been occupying between 300% and 330% of one CPU. I interrupted it after 145 CPU minutes (about 46 minutes wall-clock time) because there was no failure. Can you reproduce this on 10.11 or 11.2? I would pretty much need an rr replay trace to debug this, and for that I would suggest to use much fewer concurrent connections. Based on my test, the server should be kept equally busy with 3 or 4 concurrent connections.
          Roel Roel Van de Paar added a comment - - edited

          Thank you for testing. I setup the scripts on the shared `rr` server and reproduced the issue there on 11.2 and 11.7 (tested reproducibility without rr to start with). On 11.7 yet another offset stack was observed:

          CS 11.7.0 a812dba6dc881a8a7fbd9bcc1451d2e54c92575c (Debug)

          mariadbd: /test/11.7_dbg/storage/innobase/buf/buf0lru.cc:400: buf_block_t* buf_LRU_get_free_block(buf_LRU_get): Assertion `LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()' failed.
          

          CS 11.7.0 a812dba6dc881a8a7fbd9bcc1451d2e54c92575c (Debug)

          Core was generated by `/test/MD191024-mariadb-11.7.0-linux-x86_64-dbg/bin/mariadbd --no-defaults --max'.
          Program terminated with signal SIGABRT, Aborted.
          #0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50
          [Current thread is 1 (Thread 0x14b56aa42700 (LWP 1782819))]
          (gdb) bt
          #0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50
          #1  0x000014b6213fd859 in __GI_abort () at abort.c:79
          #2  0x000014b6213fd729 in __assert_fail_base (fmt=0x14b621593588 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=0x55acf1f7e250 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=0x55acf1f7d7c8 "/test/11.7_dbg/storage/innobase/buf/buf0lru.cc", line=400, function=<optimized out>) at assert.c:92
          #3  0x000014b62140efd6 in __GI___assert_fail (assertion=assertion@entry=0x55acf1f7e250 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=file@entry=0x55acf1f7d7c8 "/test/11.7_dbg/storage/innobase/buf/buf0lru.cc", line=line@entry=400, function=function@entry=0x55acf1f7e2a0 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)") at assert.c:101
          #4  0x000055acf190bc57 in buf_LRU_get_free_block (get=get@entry=have_mutex) at /test/11.7_dbg/storage/innobase/buf/buf0lru.cc:400
          #5  0x000055acf1910dce in buf_read_page (page_id=page_id@entry={m_id = 4294967357}, chain=@0x55acf3d51538: {first = 0x0}) at /test/11.7_dbg/storage/innobase/buf/buf0rea.cc:427
          #6  0x000055acf18e47e8 in buf_page_get_gen (page_id=<optimized out>, zip_size=zip_size@entry=0, rw_latch=rw_latch@entry=RW_X_LATCH, guess=guess@entry=0x0, mode=mode@entry=10, mtr=mtr@entry=0x14b56aa3a400, err=0x14b56aa3a3c4) at /test/11.7_dbg/storage/innobase/buf/buf0buf.cc:2882
          #7  0x000055acf1873f7e in trx_undo_reuse_cached (trx=trx@entry=0x14b60bc44780, rseg=rseg@entry=0x55acf269c580 <trx_sys+25728>, pundo=pundo@entry=0x14b60bc450d0, mtr=mtr@entry=0x14b56aa3a400, err=err@entry=0x14b56aa3a3c4) at /test/11.7_dbg/storage/innobase/include/buf0types.h:95
          #8  0x000055acf187820b in trx_undo_assign_low<false> (trx=trx@entry=0x14b60bc44780, rseg=rseg@entry=0x55acf269c580 <trx_sys+25728>, undo=undo@entry=0x14b60bc450d0, mtr=mtr@entry=0x14b56aa3a400, err=err@entry=0x14b56aa3a3c4) at /test/11.7_dbg/storage/innobase/include/mtr0mtr.h:153
          #9  0x000055acf184c8a9 in trx_undo_report_row_operation (thr=thr@entry=0x14b4a402abe8, index=index@entry=0x55acf3d264e8, clust_entry=clust_entry@entry=0x0, update=update@entry=0x14b4a4033398, cmpl_info=cmpl_info@entry=0, rec=rec@entry=0x14b60b8d4388 "test/t1", offsets=0x14b56aa3be10, roll_ptr=0x14b56aa3b4b8) at /test/11.7_dbg/storage/innobase/trx/trx0rec.cc:1897
          #10 0x000055acf18a1b17 in btr_cur_upd_lock_and_undo (flags=flags@entry=0, cursor=cursor@entry=0x14b56aa3bd50, offsets=offsets@entry=0x14b56aa3be10, update=update@entry=0x14b4a4033398, cmpl_info=cmpl_info@entry=0, thr=thr@entry=0x14b4a402abe8, mtr=0x14b56aa3c2d0, roll_ptr=0x14b56aa3b4b8) at /test/11.7_dbg/storage/innobase/btr/btr0cur.cc:2738
          #11 0x000055acf18b05e5 in btr_cur_update_in_place (flags=flags@entry=0, cursor=cursor@entry=0x14b56aa3bd50, offsets=0x14b56aa3be10, update=update@entry=0x14b4a4033398, cmpl_info=cmpl_info@entry=0, thr=thr@entry=0x14b4a402abe8, trx_id=132883, mtr=0x14b56aa3c2d0) at /test/11.7_dbg/storage/innobase/btr/btr0cur.cc:3197
          #12 0x000055acf18b160d in btr_cur_optimistic_update (flags=flags@entry=0, cursor=cursor@entry=0x14b56aa3bd50, offsets=offsets@entry=0x14b56aa3bd38, heap=heap@entry=0x14b56aa3bd30, update=update@entry=0x14b4a4033398, cmpl_info=cmpl_info@entry=0, thr=0x14b4a402abe8, trx_id=132883, mtr=0x14b56aa3c2d0) at /test/11.7_dbg/storage/innobase/btr/btr0cur.cc:3479
          #13 0x000055acf177fca5 in row_ins_clust_index_entry_by_modify (pcur=pcur@entry=0x14b56aa3bd50, flags=flags@entry=0, mode=mode@entry=2, offsets=offsets@entry=0x14b56aa3bd38, offsets_heap=offsets_heap@entry=0x14b56aa3bd30, heap=heap@entry=0x14b4a4033318, entry=0x14b4a401f608, thr=0x14b4a402abe8, mtr=0x14b56aa3c2d0) at /test/11.7_dbg/storage/innobase/include/que0que.inl:37
          #14 0x000055acf178d4a0 in row_ins_clust_index_entry_low (flags=flags@entry=0, mode=<optimized out>, mode@entry=BTR_MODIFY_LEAF, index=index@entry=0x55acf3d264e8, n_uniq=n_uniq@entry=1, entry=entry@entry=0x14b4a401f608, n_ext=n_ext@entry=0, thr=<optimized out>) at /test/11.7_dbg/storage/innobase/row/row0ins.cc:2865
          #15 0x000055acf178e1d2 in row_ins_clust_index_entry (index=index@entry=0x55acf3d264e8, entry=entry@entry=0x14b4a401f608, thr=thr@entry=0x14b4a402abe8, n_ext=n_ext@entry=0) at /test/11.7_dbg/storage/innobase/row/row0ins.cc:3267
          #16 0x000055acf178f963 in row_ins_index_entry (thr=0x14b4a402abe8, entry=0x14b4a401f608, index=0x55acf3d264e8) at /test/11.7_dbg/storage/innobase/row/row0ins.cc:3402
          #17 row_ins_index_entry_step (thr=0x14b4a402abe8, node=<optimized out>) at /test/11.7_dbg/storage/innobase/row/row0ins.cc:3570
          #18 row_ins (thr=0x14b4a402abe8, node=<optimized out>) at /test/11.7_dbg/storage/innobase/row/row0ins.cc:3687
          #19 row_ins_step (thr=thr@entry=0x14b4a402abe8) at /test/11.7_dbg/storage/innobase/row/row0ins.cc:3816
          #20 0x000055acf1744c53 in que_thr_step (thr=0x14b4a402abe8) at /test/11.7_dbg/storage/innobase/que/que0que.cc:533
          #21 que_run_threads_low (thr=0x14b4a402abe8) at /test/11.7_dbg/storage/innobase/que/que0que.cc:610
          #22 que_run_threads (thr=thr@entry=0x14b4a402abe8) at /test/11.7_dbg/storage/innobase/que/que0que.cc:630
          #23 0x000055acf17ae39f in row_create_table_for_mysql (table=table@entry=0x14b59c035358, trx=0x14b60bc44780) at /test/11.7_dbg/storage/innobase/row/row0mysql.cc:2061
          #24 0x000055acf164e00b in create_table_info_t::create_table_def (this=<optimized out>) at /test/11.7_dbg/storage/innobase/handler/ha_innodb.cc:10834
          #25 0x000055acf164519d in create_table_info_t::create_table (this=this@entry=0x14b56aa3d000, create_fk=create_fk@entry=true, strict=<optimized out>) at /test/11.7_dbg/storage/innobase/handler/ha_innodb.cc:12680
          #26 0x000055acf16458f1 in ha_innobase::create (this=<optimized out>, name=0x14b56aa3eb80 "./test/t1", form=0x14b56aa3d5b0, create_info=0x14b56aa3f200, file_per_table=<optimized out>, trx=0x14b60bc44780, trx@entry=0x0) at /test/11.7_dbg/storage/innobase/handler/ha_innodb.cc:13188
          #27 0x000055acf1646bf7 in ha_innobase::create (this=<optimized out>, name=<optimized out>, form=<optimized out>, create_info=<optimized out>) at /test/11.7_dbg/storage/innobase/handler/ha_innodb.cc:13236
          #28 0x000055acf129f8dc in handler::ha_create (this=0x14b4a402dce0, name=0x14b56aa3eb80 "./test/t1", form=form@entry=0x14b56aa3d5b0, info_arg=info_arg@entry=0x14b56aa3f200) at /test/11.7_dbg/sql/handler.cc:5877
          #29 0x000055acf12a06bd in ha_create_table (thd=thd@entry=0x14b4a4000d48, path=<optimized out>, db=0x14b4a40158a0 "test", table_name=0x14b4a4015870 "t1", create_info=create_info@entry=0x14b56aa3f200, frm=frm@entry=0x14b56aa3eb70, skip_frm_file=false) at /test/11.7_dbg/sql/handler.cc:6364
          #30 0x000055acf104a7bd in create_table_impl (thd=thd@entry=0x14b4a4000d48, ddl_log_state_create=ddl_log_state_create@entry=0x14b56aa3ee80, ddl_log_state_rm=<optimized out>, ddl_log_state_rm@entry=0x14b56aa3f090, orig_db=@0x14b4a40158c8: {<Lex_ident_fs> = {<Lex_ident<Compare_table_names>> = {<Lex_cstring> = {<st_mysql_const_lex_string> = {str = 0x14b4a40158a0 "test", length = 4}, <No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, orig_table_name=@0x14b4a40158d8: {<Lex_ident_fs> = {<Lex_ident<Compare_table_names>> = {<Lex_cstring> = {<st_mysql_const_lex_string> = {str = 0x14b4a4015870 "t1", length = 2}, <No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, db=@0x14b4a40158c8: {str = 0x14b4a40158a0 "test", length = 4}, table_name=@0x14b4a40158d8: {str = 0x14b4a4015870 "t1", length = 2}, path=@0x14b56aa3eb60: {str = 0x14b56aa3eb80 "./test/t1", length = 9}, options=<optimized out>, create_info=0x14b56aa3f200, alter_info=0x14b56aa3ef20, create_table_mode=0, is_trans=0x14b56aa3ee60, key_info=0x14b56aa3eb58, key_count=0x14b56aa3eb54, frm=0x14b56aa3eb70) at /test/11.7_dbg/sql/sql_table.cc:4683
          #31 0x000055acf104acdc in mysql_create_table_no_lock (thd=thd@entry=0x14b4a4000d48, ddl_log_state_create=ddl_log_state_create@entry=0x14b56aa3ee80, ddl_log_state_rm=ddl_log_state_rm@entry=0x14b56aa3f090, create_info=create_info@entry=0x14b56aa3f200, alter_info=alter_info@entry=0x14b56aa3ef20, is_trans=is_trans@entry=0x14b56aa3ee60, create_table_mode=0, table_list=0x14b4a40158b0) at /test/11.7_dbg/sql/sql_table.cc:4786
          #32 0x000055acf104cead in mysql_create_table (alter_info=0x14b56aa3ef20, create_info=0x14b56aa3f200, create_table=0x14b4a40158b0, thd=0x14b4a4000d48) at /test/11.7_dbg/sql/sql_table.cc:5012
          #33 Sql_cmd_create_table_like::execute (this=<optimized out>, thd=0x14b4a4000d48) at /test/11.7_dbg/sql/sql_table.cc:13260
          #34 0x000055acf0f4c388 in mysql_execute_command (thd=thd@entry=0x14b4a4000d48, is_called_from_prepared_stmt=is_called_from_prepared_stmt@entry=false) at /test/11.7_dbg/sql/sql_parse.cc:5862
          #35 0x000055acf0f36c7c in mysql_parse (thd=thd@entry=0x14b4a4000d48, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0x14b56aa41260) at /test/11.7_dbg/sql/sql_parse.cc:7889
          #36 0x000055acf0f4ea37 in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0x14b4a4000d48, packet=packet@entry=0x14b4a400b279 "CREATE TABLE t1(c1 LONGTEXT NULL, c2 LONGTEXT NULL)", packet_length=packet_length@entry=51, blocking=blocking@entry=true) at /test/11.7_dbg/sql/sql_class.h:1641
          #37 0x000055acf0f5154d in do_command (thd=0x14b4a4000d48, blocking=blocking@entry=true) at /test/11.7_dbg/sql/sql_parse.cc:1405
          #38 0x000055acf10ddd5c in do_handle_one_connection (connect=<optimized out>, connect@entry=0x55acf42dcf18, put_in_cache=put_in_cache@entry=true) at /test/11.7_dbg/sql/sql_connect.cc:1448
          #39 0x000055acf10de325 in handle_one_connection (arg=arg@entry=0x55acf42dcf18) at /test/11.7_dbg/sql/sql_connect.cc:1350
          #40 0x000055acf156f966 in pfs_spawn_thread (arg=0x55acf42dcf88) at /test/11.7_dbg/storage/perfschema/pfs.cc:2198
          #41 0x000014b62190e609 in start_thread (arg=<optimized out>) at pthread_create.c:477
          #42 0x000014b6214fa133 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
          

          For this, the following changed SQL was used:

          sql="
          CREATE TABLE t1(c1 LONGTEXT NULL, c2 LONGTEXT NULL);
          CREATE TABLE t2(c1 LONGTEXT NULL, c2 LONGTEXT NULL);
          CREATE TABLE t3(c1 LONGTEXT NULL, c2 LONGTEXT NULL);
          CREATE TABLE t4(c1 LONGTEXT NULL, c2 LONGTEXT NULL);
          CREATE TABLE t5(c1 LONGTEXT NULL, c2 LONGTEXT NULL);
          DROP TABLE t1;
          DROP TABLE t2;
          DROP TABLE t3;
          DROP TABLE t4;
          DROP TABLE t5;
          SET @@global.innodb_immediate_scrub_data_uncompressed=0;
          SET @@global.innodb_immediate_scrub_data_uncompressed=1;
          SET @@global.innodb_lru_scan_depth = 1536;
          SET @@global.innodb_lru_scan_depth = 86400;
          XA COMMIT 'a';
          XA END 'a';
          XA PREPARE 'a';
          XA START 'a' RESUME;
          XA START 'a';
          "
          

          I am now attempting to get rr traces, hopefully with a low number of threads.

          Roel Roel Van de Paar added a comment - - edited Thank you for testing. I setup the scripts on the shared `rr` server and reproduced the issue there on 11.2 and 11.7 (tested reproducibility without rr to start with). On 11.7 yet another offset stack was observed: CS 11.7.0 a812dba6dc881a8a7fbd9bcc1451d2e54c92575c (Debug) mariadbd: /test/11.7_dbg/storage/innobase/buf/buf0lru.cc:400: buf_block_t* buf_LRU_get_free_block(buf_LRU_get): Assertion `LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()' failed. CS 11.7.0 a812dba6dc881a8a7fbd9bcc1451d2e54c92575c (Debug) Core was generated by `/test/MD191024-mariadb-11.7.0-linux-x86_64-dbg/bin/mariadbd --no-defaults --max'. Program terminated with signal SIGABRT, Aborted. #0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50 [Current thread is 1 (Thread 0x14b56aa42700 (LWP 1782819))] (gdb) bt #0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50 #1 0x000014b6213fd859 in __GI_abort () at abort.c:79 #2 0x000014b6213fd729 in __assert_fail_base (fmt=0x14b621593588 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=0x55acf1f7e250 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=0x55acf1f7d7c8 "/test/11.7_dbg/storage/innobase/buf/buf0lru.cc", line=400, function=<optimized out>) at assert.c:92 #3 0x000014b62140efd6 in __GI___assert_fail (assertion=assertion@entry=0x55acf1f7e250 "LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()", file=file@entry=0x55acf1f7d7c8 "/test/11.7_dbg/storage/innobase/buf/buf0lru.cc", line=line@entry=400, function=function@entry=0x55acf1f7e2a0 "buf_block_t* buf_LRU_get_free_block(buf_LRU_get)") at assert.c:101 #4 0x000055acf190bc57 in buf_LRU_get_free_block (get=get@entry=have_mutex) at /test/11.7_dbg/storage/innobase/buf/buf0lru.cc:400 #5 0x000055acf1910dce in buf_read_page (page_id=page_id@entry={m_id = 4294967357}, chain=@0x55acf3d51538: {first = 0x0}) at /test/11.7_dbg/storage/innobase/buf/buf0rea.cc:427 #6 0x000055acf18e47e8 in buf_page_get_gen (page_id=<optimized out>, zip_size=zip_size@entry=0, rw_latch=rw_latch@entry=RW_X_LATCH, guess=guess@entry=0x0, mode=mode@entry=10, mtr=mtr@entry=0x14b56aa3a400, err=0x14b56aa3a3c4) at /test/11.7_dbg/storage/innobase/buf/buf0buf.cc:2882 #7 0x000055acf1873f7e in trx_undo_reuse_cached (trx=trx@entry=0x14b60bc44780, rseg=rseg@entry=0x55acf269c580 <trx_sys+25728>, pundo=pundo@entry=0x14b60bc450d0, mtr=mtr@entry=0x14b56aa3a400, err=err@entry=0x14b56aa3a3c4) at /test/11.7_dbg/storage/innobase/include/buf0types.h:95 #8 0x000055acf187820b in trx_undo_assign_low<false> (trx=trx@entry=0x14b60bc44780, rseg=rseg@entry=0x55acf269c580 <trx_sys+25728>, undo=undo@entry=0x14b60bc450d0, mtr=mtr@entry=0x14b56aa3a400, err=err@entry=0x14b56aa3a3c4) at /test/11.7_dbg/storage/innobase/include/mtr0mtr.h:153 #9 0x000055acf184c8a9 in trx_undo_report_row_operation (thr=thr@entry=0x14b4a402abe8, index=index@entry=0x55acf3d264e8, clust_entry=clust_entry@entry=0x0, update=update@entry=0x14b4a4033398, cmpl_info=cmpl_info@entry=0, rec=rec@entry=0x14b60b8d4388 "test/t1", offsets=0x14b56aa3be10, roll_ptr=0x14b56aa3b4b8) at /test/11.7_dbg/storage/innobase/trx/trx0rec.cc:1897 #10 0x000055acf18a1b17 in btr_cur_upd_lock_and_undo (flags=flags@entry=0, cursor=cursor@entry=0x14b56aa3bd50, offsets=offsets@entry=0x14b56aa3be10, update=update@entry=0x14b4a4033398, cmpl_info=cmpl_info@entry=0, thr=thr@entry=0x14b4a402abe8, mtr=0x14b56aa3c2d0, roll_ptr=0x14b56aa3b4b8) at /test/11.7_dbg/storage/innobase/btr/btr0cur.cc:2738 #11 0x000055acf18b05e5 in btr_cur_update_in_place (flags=flags@entry=0, cursor=cursor@entry=0x14b56aa3bd50, offsets=0x14b56aa3be10, update=update@entry=0x14b4a4033398, cmpl_info=cmpl_info@entry=0, thr=thr@entry=0x14b4a402abe8, trx_id=132883, mtr=0x14b56aa3c2d0) at /test/11.7_dbg/storage/innobase/btr/btr0cur.cc:3197 #12 0x000055acf18b160d in btr_cur_optimistic_update (flags=flags@entry=0, cursor=cursor@entry=0x14b56aa3bd50, offsets=offsets@entry=0x14b56aa3bd38, heap=heap@entry=0x14b56aa3bd30, update=update@entry=0x14b4a4033398, cmpl_info=cmpl_info@entry=0, thr=0x14b4a402abe8, trx_id=132883, mtr=0x14b56aa3c2d0) at /test/11.7_dbg/storage/innobase/btr/btr0cur.cc:3479 #13 0x000055acf177fca5 in row_ins_clust_index_entry_by_modify (pcur=pcur@entry=0x14b56aa3bd50, flags=flags@entry=0, mode=mode@entry=2, offsets=offsets@entry=0x14b56aa3bd38, offsets_heap=offsets_heap@entry=0x14b56aa3bd30, heap=heap@entry=0x14b4a4033318, entry=0x14b4a401f608, thr=0x14b4a402abe8, mtr=0x14b56aa3c2d0) at /test/11.7_dbg/storage/innobase/include/que0que.inl:37 #14 0x000055acf178d4a0 in row_ins_clust_index_entry_low (flags=flags@entry=0, mode=<optimized out>, mode@entry=BTR_MODIFY_LEAF, index=index@entry=0x55acf3d264e8, n_uniq=n_uniq@entry=1, entry=entry@entry=0x14b4a401f608, n_ext=n_ext@entry=0, thr=<optimized out>) at /test/11.7_dbg/storage/innobase/row/row0ins.cc:2865 #15 0x000055acf178e1d2 in row_ins_clust_index_entry (index=index@entry=0x55acf3d264e8, entry=entry@entry=0x14b4a401f608, thr=thr@entry=0x14b4a402abe8, n_ext=n_ext@entry=0) at /test/11.7_dbg/storage/innobase/row/row0ins.cc:3267 #16 0x000055acf178f963 in row_ins_index_entry (thr=0x14b4a402abe8, entry=0x14b4a401f608, index=0x55acf3d264e8) at /test/11.7_dbg/storage/innobase/row/row0ins.cc:3402 #17 row_ins_index_entry_step (thr=0x14b4a402abe8, node=<optimized out>) at /test/11.7_dbg/storage/innobase/row/row0ins.cc:3570 #18 row_ins (thr=0x14b4a402abe8, node=<optimized out>) at /test/11.7_dbg/storage/innobase/row/row0ins.cc:3687 #19 row_ins_step (thr=thr@entry=0x14b4a402abe8) at /test/11.7_dbg/storage/innobase/row/row0ins.cc:3816 #20 0x000055acf1744c53 in que_thr_step (thr=0x14b4a402abe8) at /test/11.7_dbg/storage/innobase/que/que0que.cc:533 #21 que_run_threads_low (thr=0x14b4a402abe8) at /test/11.7_dbg/storage/innobase/que/que0que.cc:610 #22 que_run_threads (thr=thr@entry=0x14b4a402abe8) at /test/11.7_dbg/storage/innobase/que/que0que.cc:630 #23 0x000055acf17ae39f in row_create_table_for_mysql (table=table@entry=0x14b59c035358, trx=0x14b60bc44780) at /test/11.7_dbg/storage/innobase/row/row0mysql.cc:2061 #24 0x000055acf164e00b in create_table_info_t::create_table_def (this=<optimized out>) at /test/11.7_dbg/storage/innobase/handler/ha_innodb.cc:10834 #25 0x000055acf164519d in create_table_info_t::create_table (this=this@entry=0x14b56aa3d000, create_fk=create_fk@entry=true, strict=<optimized out>) at /test/11.7_dbg/storage/innobase/handler/ha_innodb.cc:12680 #26 0x000055acf16458f1 in ha_innobase::create (this=<optimized out>, name=0x14b56aa3eb80 "./test/t1", form=0x14b56aa3d5b0, create_info=0x14b56aa3f200, file_per_table=<optimized out>, trx=0x14b60bc44780, trx@entry=0x0) at /test/11.7_dbg/storage/innobase/handler/ha_innodb.cc:13188 #27 0x000055acf1646bf7 in ha_innobase::create (this=<optimized out>, name=<optimized out>, form=<optimized out>, create_info=<optimized out>) at /test/11.7_dbg/storage/innobase/handler/ha_innodb.cc:13236 #28 0x000055acf129f8dc in handler::ha_create (this=0x14b4a402dce0, name=0x14b56aa3eb80 "./test/t1", form=form@entry=0x14b56aa3d5b0, info_arg=info_arg@entry=0x14b56aa3f200) at /test/11.7_dbg/sql/handler.cc:5877 #29 0x000055acf12a06bd in ha_create_table (thd=thd@entry=0x14b4a4000d48, path=<optimized out>, db=0x14b4a40158a0 "test", table_name=0x14b4a4015870 "t1", create_info=create_info@entry=0x14b56aa3f200, frm=frm@entry=0x14b56aa3eb70, skip_frm_file=false) at /test/11.7_dbg/sql/handler.cc:6364 #30 0x000055acf104a7bd in create_table_impl (thd=thd@entry=0x14b4a4000d48, ddl_log_state_create=ddl_log_state_create@entry=0x14b56aa3ee80, ddl_log_state_rm=<optimized out>, ddl_log_state_rm@entry=0x14b56aa3f090, orig_db=@0x14b4a40158c8: {<Lex_ident_fs> = {<Lex_ident<Compare_table_names>> = {<Lex_cstring> = {<st_mysql_const_lex_string> = {str = 0x14b4a40158a0 "test", length = 4}, <No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, orig_table_name=@0x14b4a40158d8: {<Lex_ident_fs> = {<Lex_ident<Compare_table_names>> = {<Lex_cstring> = {<st_mysql_const_lex_string> = {str = 0x14b4a4015870 "t1", length = 2}, <No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, db=@0x14b4a40158c8: {str = 0x14b4a40158a0 "test", length = 4}, table_name=@0x14b4a40158d8: {str = 0x14b4a4015870 "t1", length = 2}, path=@0x14b56aa3eb60: {str = 0x14b56aa3eb80 "./test/t1", length = 9}, options=<optimized out>, create_info=0x14b56aa3f200, alter_info=0x14b56aa3ef20, create_table_mode=0, is_trans=0x14b56aa3ee60, key_info=0x14b56aa3eb58, key_count=0x14b56aa3eb54, frm=0x14b56aa3eb70) at /test/11.7_dbg/sql/sql_table.cc:4683 #31 0x000055acf104acdc in mysql_create_table_no_lock (thd=thd@entry=0x14b4a4000d48, ddl_log_state_create=ddl_log_state_create@entry=0x14b56aa3ee80, ddl_log_state_rm=ddl_log_state_rm@entry=0x14b56aa3f090, create_info=create_info@entry=0x14b56aa3f200, alter_info=alter_info@entry=0x14b56aa3ef20, is_trans=is_trans@entry=0x14b56aa3ee60, create_table_mode=0, table_list=0x14b4a40158b0) at /test/11.7_dbg/sql/sql_table.cc:4786 #32 0x000055acf104cead in mysql_create_table (alter_info=0x14b56aa3ef20, create_info=0x14b56aa3f200, create_table=0x14b4a40158b0, thd=0x14b4a4000d48) at /test/11.7_dbg/sql/sql_table.cc:5012 #33 Sql_cmd_create_table_like::execute (this=<optimized out>, thd=0x14b4a4000d48) at /test/11.7_dbg/sql/sql_table.cc:13260 #34 0x000055acf0f4c388 in mysql_execute_command (thd=thd@entry=0x14b4a4000d48, is_called_from_prepared_stmt=is_called_from_prepared_stmt@entry=false) at /test/11.7_dbg/sql/sql_parse.cc:5862 #35 0x000055acf0f36c7c in mysql_parse (thd=thd@entry=0x14b4a4000d48, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0x14b56aa41260) at /test/11.7_dbg/sql/sql_parse.cc:7889 #36 0x000055acf0f4ea37 in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0x14b4a4000d48, packet=packet@entry=0x14b4a400b279 "CREATE TABLE t1(c1 LONGTEXT NULL, c2 LONGTEXT NULL)", packet_length=packet_length@entry=51, blocking=blocking@entry=true) at /test/11.7_dbg/sql/sql_class.h:1641 #37 0x000055acf0f5154d in do_command (thd=0x14b4a4000d48, blocking=blocking@entry=true) at /test/11.7_dbg/sql/sql_parse.cc:1405 #38 0x000055acf10ddd5c in do_handle_one_connection (connect=<optimized out>, connect@entry=0x55acf42dcf18, put_in_cache=put_in_cache@entry=true) at /test/11.7_dbg/sql/sql_connect.cc:1448 #39 0x000055acf10de325 in handle_one_connection (arg=arg@entry=0x55acf42dcf18) at /test/11.7_dbg/sql/sql_connect.cc:1350 #40 0x000055acf156f966 in pfs_spawn_thread (arg=0x55acf42dcf88) at /test/11.7_dbg/storage/perfschema/pfs.cc:2198 #41 0x000014b62190e609 in start_thread (arg=<optimized out>) at pthread_create.c:477 #42 0x000014b6214fa133 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 For this, the following changed SQL was used: sql=" CREATE TABLE t1(c1 LONGTEXT NULL, c2 LONGTEXT NULL); CREATE TABLE t2(c1 LONGTEXT NULL, c2 LONGTEXT NULL); CREATE TABLE t3(c1 LONGTEXT NULL, c2 LONGTEXT NULL); CREATE TABLE t4(c1 LONGTEXT NULL, c2 LONGTEXT NULL); CREATE TABLE t5(c1 LONGTEXT NULL, c2 LONGTEXT NULL); DROP TABLE t1; DROP TABLE t2; DROP TABLE t3; DROP TABLE t4; DROP TABLE t5; SET @@global.innodb_immediate_scrub_data_uncompressed=0; SET @@global.innodb_immediate_scrub_data_uncompressed=1; SET @@global.innodb_lru_scan_depth = 1536; SET @@global.innodb_lru_scan_depth = 86400; XA COMMIT 'a'; XA END 'a'; XA PREPARE 'a'; XA START 'a' RESUME; XA START 'a'; " I am now attempting to get rr traces, hopefully with a low number of threads.

          The issue does regrettably not reproduce under rr.

          I however do have double good news; using the SQL from the last comment various iterations of the LRU_size <= 256 assertion were reproduced on rr, including another new stack variation, on both 11.2 and 11.7 using only 3 or 5 threads, and no TEMPORARY. Core dumps as follows:

          cd /test/MD191024-mariadb-11.7.0-linux-x86_64-dbg_MDEV35149
          gdb ./bin/mariadbd ./data/core
          

          Used 3 threads. SQL as above (i.e. 5 tables CREATE/DROP, no INSERT, no TEMPORARY). Resulting stack:

          MDEV35149 CS 11.7.0 a812dba6dc881a8a7fbd9bcc1451d2e54c92575c (Debug)

          LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()|SIGABRT|buf_LRU_get_free_block|buf_read_page|buf_page_get_gen|trx_undo_reuse_cached
          

          cd /test/MD191024-mariadb-11.7.0-linux-x86_64-dbg
          gdb ./bin/mariadbd ./data/core
          

          Used 5 threads. SQL as above (idem). Resulting stack:

          CS 11.7.0 a812dba6dc881a8a7fbd9bcc1451d2e54c92575c (Debug)

          LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()|SIGABRT|buf_LRU_get_free_block|fsp_page_create|fsp_alloc_free_page|fseg_alloc_free_page_low
          

          cd /test/MD191024-mariadb-11.2.6-linux-x86_64-dbg-MDEV-35149
          gdb ./bin/mariadbd ./data/core
          

          Used 3 threads. SQL as above (idem). Resulting stack:

          MDEV-35149 CS 11.2.6 ebefef658ebff14d55ebb068a0e4af3a6b647d44 (Debug)

          LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()|SIGABRT|buf_LRU_get_free_block|buf_read_page|buf_page_get_gen|trx_undo_reuse_cached
          

          cd /test/MD191024-mariadb-11.2.6-linux-x86_64-dbg
          gdb ./bin/mariadbd ./data/core
          

          Used 5 threads. SQL as above (idem). Resulting stack:

          CS 11.2.6 ebefef658ebff14d55ebb068a0e4af3a6b647d44 (Debug)

          LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()|SIGABRT|buf_LRU_get_free_block|buf_read_page|buf_page_get_gen|trx_rseg_t::get
          

          The issue also reproduces in about 1 minute consistently now (on the rr instance).

          Roel Roel Van de Paar added a comment - The issue does regrettably not reproduce under rr. I however do have double good news; using the SQL from the last comment various iterations of the LRU_size <= 256 assertion were reproduced on rr , including another new stack variation, on both 11.2 and 11.7 using only 3 or 5 threads, and no TEMPORARY . Core dumps as follows: cd /test/MD191024-mariadb-11 .7.0-linux-x86_64-dbg_MDEV35149 gdb . /bin/mariadbd . /data/core Used 3 threads. SQL as above (i.e. 5 tables CREATE/DROP, no INSERT, no TEMPORARY). Resulting stack: MDEV35149 CS 11.7.0 a812dba6dc881a8a7fbd9bcc1451d2e54c92575c (Debug) LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()|SIGABRT|buf_LRU_get_free_block|buf_read_page|buf_page_get_gen|trx_undo_reuse_cached cd /test/MD191024-mariadb-11 .7.0-linux-x86_64-dbg gdb . /bin/mariadbd . /data/core Used 5 threads. SQL as above (idem). Resulting stack: CS 11.7.0 a812dba6dc881a8a7fbd9bcc1451d2e54c92575c (Debug) LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()|SIGABRT|buf_LRU_get_free_block|fsp_page_create|fsp_alloc_free_page|fseg_alloc_free_page_low cd /test/MD191024-mariadb-11 .2.6-linux-x86_64-dbg-MDEV-35149 gdb . /bin/mariadbd . /data/core Used 3 threads. SQL as above (idem). Resulting stack: MDEV-35149 CS 11.2.6 ebefef658ebff14d55ebb068a0e4af3a6b647d44 (Debug) LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()|SIGABRT|buf_LRU_get_free_block|buf_read_page|buf_page_get_gen|trx_undo_reuse_cached cd /test/MD191024-mariadb-11 .2.6-linux-x86_64-dbg gdb . /bin/mariadbd . /data/core Used 5 threads. SQL as above (idem). Resulting stack: CS 11.2.6 ebefef658ebff14d55ebb068a0e4af3a6b647d44 (Debug) LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()|SIGABRT|buf_LRU_get_free_block|buf_read_page|buf_page_get_gen|trx_rseg_t::get The issue also reproduces in about 1 minute consistently now (on the rr instance).
          Roel Roel Van de Paar added a comment - - edited

          I have one more core to share with an interesting change: I removed all XA commands except XA BEGIN 'a'; (without it, using BEGIN; only, the bug would not reproduce). This still produced the following stack:

          cd /test/MD191024-mariadb-11.2.6-linux-x86_64-dbg-MDEV-35149_2
          gdb ./bin/mariadbd ./data/core
          

          Used 3 threads. SQL as full last paste above, without all XA commands, except XA BEGIN 'a';. Resulting stack:

          MDEV-35149 CS 11.2.6 ebefef658ebff14d55ebb068a0e4af3a6b647d44 (Debug)

          LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()|SIGABRT|buf_LRU_get_free_block|buf_read_page|buf_page_get_gen|trx_undo_reuse_cached
          

          Roel Roel Van de Paar added a comment - - edited I have one more core to share with an interesting change: I removed all XA commands except XA BEGIN 'a'; (without it, using BEGIN; only, the bug would not reproduce). This still produced the following stack: cd /test/MD191024-mariadb-11 .2.6-linux-x86_64-dbg-MDEV-35149_2 gdb . /bin/mariadbd . /data/core Used 3 threads. SQL as full last paste above, without all XA commands, except XA BEGIN 'a'; . Resulting stack: MDEV-35149 CS 11.2.6 ebefef658ebff14d55ebb068a0e4af3a6b647d44 (Debug) LRU_size <= 256 || available >= scan_depth || buf_pool.need_LRU_eviction()|SIGABRT|buf_LRU_get_free_block|buf_read_page|buf_page_get_gen|trx_undo_reuse_cached
          marko Marko Mäkelä added a comment - - edited

          This is a simple race condition: SET GLOBAL innodb_lru_scan_depth is not protected by buf_pool.mutex, as it should be. As far as I can tell, the race condition is present in 10.5 as well, but there should be no debug assertion failing, and possibly no observable ill effect.
          In the following:

          ssh rr
          gdb -c /test/MD191024-mariadb-11.7.0-linux-x86_64-dbg_MDEV35149/{data/core,bin/mariadbd}
          

          I can see that buf_pool.LRU.count is 2+BUF_LRU_MIN_LEN, but buf_pool.free.count is 7805, which is well beyond the current value of srv_LRU_scan_depth / 2. In other words, available >= scan_depth should actually hold at the time of the crash.

          I realize that the test is constantly fiddling with srv_LRU_scan_depth. It seems to me that any change of this parameter must be protected by buf_pool.mutex. That should fix this race condition. While fixing this, I’d make the variable a member of the static buf_pool object in order to improve locality of reference and to reduce unnecessary obfuscation.

          The caller of the function was trying to write the very first undo log record for a CREATE TABLE operation:

          main a812dba6dc881a8a7fbd9bcc1451d2e54c92575c

          #4  0x000055f17f28ec57 in buf_LRU_get_free_block (get=get@entry=have_mutex) at /test/11.7_dbg/storage/innobase/buf/buf0lru.cc:400
          #5  0x000055f17f293dce in buf_read_page (page_id=page_id@entry={m_id = 0x30000004b}, chain=@0x55f182bebb60: {first = 0x0}) at /test/11.7_dbg/storage/innobase/buf/buf0rea.cc:427
          #6  0x000055f17f2677e8 in buf_page_get_gen (page_id=<optimized out>, zip_size=zip_size@entry=0x0, rw_latch=rw_latch@entry=RW_X_LATCH, guess=guess@entry=0x0, mode=mode@entry=0xa, mtr=mtr@entry=0x1496600b6400, err=0x1496600b63c4) at /test/11.7_dbg/storage/innobase/buf/buf0buf.cc:2882
          #7  0x000055f17f1f6f7e in trx_undo_reuse_cached (trx=trx@entry=0x1496758a1c80, rseg=rseg@entry=0x55f180020000 <trx_sys+28416>, pundo=pundo@entry=0x1496758a25d0, mtr=mtr@entry=0x1496600b6400, err=err@entry=0x1496600b63c4) at /test/11.7_dbg/storage/innobase/include/buf0types.h:95
          #8  0x000055f17f1fb20b in trx_undo_assign_low<false> (trx=trx@entry=0x1496758a1c80, rseg=rseg@entry=0x55f180020000 <trx_sys+28416>, undo=undo@entry=0x1496758a25d0, mtr=mtr@entry=0x1496600b6400, err=err@entry=0x1496600b63c4) at /test/11.7_dbg/storage/innobase/include/mtr0mtr.h:153
          #9  0x000055f17f1cf8a9 in trx_undo_report_row_operation (thr=thr@entry=0x149628071408, index=index@entry=0x55f182bb71f8, clust_entry=clust_entry@entry=0x0, update=update@entry=0x14962807a128, cmpl_info=cmpl_info@entry=0x0, rec=rec@entry=0x1496751f43cc "test/t5", offsets=0x1496600b7e10, roll_ptr=0x1496600b74b8) at /test/11.7_dbg/storage/innobase/trx/trx0rec.cc:1897
          

          At the time of this assertion failure, the buf_flush_page_cleaner thread is in an untimed wait.
          In the other failure

          ssh rr
          gdb -c /test/MD191024-mariadb-11.2.6-linux-x86_64-dbg-MDEV-35149_2/{data/core,bin/mariadbd}
          

          it is similar, but the assertion failure is triggered a little later, inside btr_create() as part of executing a CREATE TABLE statement.

          buf_pool.try_LRU_scan holds in both core dumps, meaning that the server does not "think" that any LRU eviction is currently needed.

          marko Marko Mäkelä added a comment - - edited This is a simple race condition: SET GLOBAL innodb_lru_scan_depth is not protected by buf_pool.mutex , as it should be. As far as I can tell, the race condition is present in 10.5 as well, but there should be no debug assertion failing, and possibly no observable ill effect. In the following: ssh rr gdb -c /test/MD191024-mariadb-11 .7.0-linux-x86_64-dbg_MDEV35149/{data /core ,bin /mariadbd } I can see that buf_pool.LRU.count is 2+BUF_LRU_MIN_LEN , but buf_pool.free.count is 7805, which is well beyond the current value of srv_LRU_scan_depth / 2 . In other words, available >= scan_depth should actually hold at the time of the crash. I realize that the test is constantly fiddling with srv_LRU_scan_depth . It seems to me that any change of this parameter must be protected by buf_pool.mutex . That should fix this race condition. While fixing this, I’d make the variable a member of the static buf_pool object in order to improve locality of reference and to reduce unnecessary obfuscation. The caller of the function was trying to write the very first undo log record for a CREATE TABLE operation: main a812dba6dc881a8a7fbd9bcc1451d2e54c92575c #4 0x000055f17f28ec57 in buf_LRU_get_free_block (get=get@entry=have_mutex) at /test/11.7_dbg/storage/innobase/buf/buf0lru.cc:400 #5 0x000055f17f293dce in buf_read_page (page_id=page_id@entry={m_id = 0x30000004b}, chain=@0x55f182bebb60: {first = 0x0}) at /test/11.7_dbg/storage/innobase/buf/buf0rea.cc:427 #6 0x000055f17f2677e8 in buf_page_get_gen (page_id=<optimized out>, zip_size=zip_size@entry=0x0, rw_latch=rw_latch@entry=RW_X_LATCH, guess=guess@entry=0x0, mode=mode@entry=0xa, mtr=mtr@entry=0x1496600b6400, err=0x1496600b63c4) at /test/11.7_dbg/storage/innobase/buf/buf0buf.cc:2882 #7 0x000055f17f1f6f7e in trx_undo_reuse_cached (trx=trx@entry=0x1496758a1c80, rseg=rseg@entry=0x55f180020000 <trx_sys+28416>, pundo=pundo@entry=0x1496758a25d0, mtr=mtr@entry=0x1496600b6400, err=err@entry=0x1496600b63c4) at /test/11.7_dbg/storage/innobase/include/buf0types.h:95 #8 0x000055f17f1fb20b in trx_undo_assign_low<false> (trx=trx@entry=0x1496758a1c80, rseg=rseg@entry=0x55f180020000 <trx_sys+28416>, undo=undo@entry=0x1496758a25d0, mtr=mtr@entry=0x1496600b6400, err=err@entry=0x1496600b63c4) at /test/11.7_dbg/storage/innobase/include/mtr0mtr.h:153 #9 0x000055f17f1cf8a9 in trx_undo_report_row_operation (thr=thr@entry=0x149628071408, index=index@entry=0x55f182bb71f8, clust_entry=clust_entry@entry=0x0, update=update@entry=0x14962807a128, cmpl_info=cmpl_info@entry=0x0, rec=rec@entry=0x1496751f43cc "test/t5", offsets=0x1496600b7e10, roll_ptr=0x1496600b74b8) at /test/11.7_dbg/storage/innobase/trx/trx0rec.cc:1897 At the time of this assertion failure, the buf_flush_page_cleaner thread is in an untimed wait. In the other failure ssh rr gdb -c /test/MD191024-mariadb-11 .2.6-linux-x86_64-dbg-MDEV-35149_2/{data /core ,bin /mariadbd } it is similar, but the assertion failure is triggered a little later, inside btr_create() as part of executing a CREATE TABLE statement. buf_pool.try_LRU_scan holds in both core dumps, meaning that the server does not "think" that any LRU eviction is currently needed.
          Roel Roel Van de Paar added a comment - - edited

          Thank you for the fix Marko and the review Debarun. Testing is not immediately showing the issue, will let it run overnight.

          Roel Roel Van de Paar added a comment - - edited Thank you for the fix Marko and the review Debarun. Testing is not immediately showing the issue, will let it run overnight.

          Testcase ran bit longer than intended, issue did not show in 1.3T queries. Bug confirmed fixed with thanks.

          Roel Roel Van de Paar added a comment - Testcase ran bit longer than intended, issue did not show in 1.3T queries. Bug confirmed fixed with thanks.

          People

            marko Marko Mäkelä
            Roel Roel Van de Paar
            Votes:
            0 Vote for this issue
            Watchers:
            4 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved:

              Git Integration

                Error rendering 'com.xiplink.jira.git.jira_git_plugin:git-issue-webpanel'. Please contact your Jira administrators.