Uploaded image for project: 'MariaDB Server'
  1. MariaDB Server
  2. MDEV-29799

SIGSEGV in lock_sys_t::hash_table::resize from buf_resize_callback in tpool::task::execute during InnoDB Buffer Pool resize

    XMLWordPrintable

    Details

      Description

      Start by applying the attached patch to 10.11, revision 6ebdd3013a18b01dbecec76b870810329eb76586. It contains 17 additional random usleeps in InnoDB's locking code in storage/innobase/lock/lock0lock.cc (with a few matching extra brackets were needed). Then run the following code using a C based client but not the CLI.
      Note: Most of the code in this testcase is repeated to trigger the issue more readily. Using a patched tree, the issue should show as good as every run.

      # mysqld options possibly required for replay:  --innodb_change_buffering=inserts --innodb_fast_shutdown=1 --innodb_file_per_table=1
      CREATE DEFINER=root@localhost EVENT e1 ON SCHEDULE EVERY '1' SECOND COMMENT 'a' DO DELETE FROM t;
      SELECT SLEEP(0.2);
      CREATE DEFINER=root@localhost EVENT e2 ON SCHEDULE EVERY '1' SECOND COMMENT 'a' DO DELETE FROM t;
      SELECT SLEEP(0.2);
      CREATE DEFINER=root@localhost EVENT e3 ON SCHEDULE EVERY '1' SECOND COMMENT 'a' DO DELETE FROM t;
      SELECT SLEEP(0.2);
      CREATE DEFINER=root@localhost EVENT e4 ON SCHEDULE EVERY '1' SECOND COMMENT 'a' DO DELETE FROM t;
      SELECT SLEEP(0.1);
      CREATE DEFINER=root@localhost EVENT e4 ON SCHEDULE EVERY '1' SECOND COMMENT 'a' DO DELETE FROM t;
      SET GLOBAL event_scheduler=ON;
      CREATE TABLE t (id INT);
      INSERT INTO t VALUES (1),(1);
      SET GLOBAL innodb_buffer_pool_size=21474836480;
      SELECT SLEEP(3);
      SET GLOBAL innodb_buffer_pool_size=134217728;
      SELECT SLEEP(3);
      SET GLOBAL innodb_buffer_pool_size=21474836480;
      SELECT SLEEP(3);
      SET GLOBAL innodb_buffer_pool_size=134217728;
      SELECT SLEEP(3);
      SET GLOBAL innodb_buffer_pool_size=21474836480;
      SELECT SLEEP(3);
      SET GLOBAL innodb_buffer_pool_size=134217728;
      SELECT SLEEP(3);
      SET GLOBAL innodb_buffer_pool_size=21474836480;
      SELECT SLEEP(3);
      SET GLOBAL innodb_buffer_pool_size=134217728;
      SELECT SLEEP(3);
      SET GLOBAL innodb_buffer_pool_size=21474836480;
      SELECT SLEEP(3);
      SET GLOBAL innodb_buffer_pool_size=134217728;
      SELECT SLEEP(3);
      SET GLOBAL innodb_buffer_pool_size=21474836480;
      SELECT SLEEP(3);
      

      Leads to:

      10.11.0 ba875e939619baefb08936863a889830f595e426 (Debug)

      Core was generated by `/test/SLEEP_MD071022-mariadb-10.11.0-linux-x86_64-dbg/bin/mysqld --no-defaults'.
      Program terminated with signal SIGSEGV, Segmentation fault.
      #0  lock_sys_t::hash_table::resize (
          this=this@entry=0x55dcb133ba98 <lock_sys+88>, n=n@entry=6553600)
          at /test/10.11_dbg/storage/innobase/lock/lock0lock.cc:123
      [Current thread is 1 (Thread 0x15192d1d4700 (LWP 2940395))]
      (gdb) bt
      #0  lock_sys_t::hash_table::resize (this=this@entry=0x55dcb133ba98 <lock_sys+88>, n=n@entry=6553600) at /test/10.11_dbg/storage/innobase/lock/lock0lock.cc:123
      #1  0x000055dcafbc307b in lock_sys_t::resize (this=0x55dcb133ba40 <lock_sys>, n_cells=6553600) at /test/10.11_dbg/storage/innobase/lock/lock0lock.cc:464
      #2  0x000055dcafdecf88 in buf_pool_t::resize (this=this@entry=0x55dcb0a1f000 <buf_pool>) at /test/10.11_dbg/storage/innobase/buf/buf0buf.cc:1813
      #3  0x000055dcafde1065 in buf_resize_callback () at /test/10.11_dbg/storage/innobase/buf/buf0buf.cc:1860
      #4  0x000055dcafefbe0d in tpool::task_group::execute (this=0x55dcb1365be0 <single_threaded_group>, t=t@entry=0x55dcb1365b40 <buf_resize_task>) at /test/10.11_dbg/tpool/task_group.cc:70
      #5  0x000055dcafefbe95 in tpool::task::execute (this=0x55dcb1365b40 <buf_resize_task>) at /test/10.11_dbg/tpool/task.cc:32
      #6  0x000055dcafef98ab in tpool::thread_pool_generic::worker_main (this=0x55dcb18a14b0, thread_var=0x55dcb18a1f60) at /test/10.11_dbg/tpool/tpool_generic.cc:580
      #7  0x000055dcafefafde in std::__invoke_impl<void, void (tpool::thread_pool_generic::*)(tpool::worker_data*), tpool::thread_pool_generic*, tpool::worker_data*> (__t=<optimized out>, __f=<optimized out>) at /usr/include/c++/9/bits/invoke.h:89
      #8  std::__invoke<void (tpool::thread_pool_generic::*)(tpool::worker_data*), tpool::thread_pool_generic*, tpool::worker_data*> (__fn=<optimized out>) at /usr/include/c++/9/bits/invoke.h:95
      #9  std::thread::_Invoker<std::tuple<void (tpool::thread_pool_generic::*)(tpool::worker_data*), tpool::thread_pool_generic*, tpool::worker_data*> >::_M_invoke<0ul, 1ul, 2ul> (this=<optimized out>) at /usr/include/c++/9/thread:244
      #10 std::thread::_Invoker<std::tuple<void (tpool::thread_pool_generic::*)(tpool::worker_data*), tpool::thread_pool_generic*, tpool::worker_data*> >::operator() (this=<optimized out>) at /usr/include/c++/9/thread:251
      #11 std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (tpool::thread_pool_generic::*)(tpool::worker_data*), tpool::thread_pool_generic*, tpool::worker_data*> > >::_M_run (this=<optimized out>) at /usr/include/c++/9/thread:195
      #12 0x0000151946c11de4 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
      #13 0x0000151946d2b609 in start_thread (arg=<optimized out>) at pthread_create.c:477
      #14 0x0000151946917133 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
      

      10.11.0 ba875e939619baefb08936863a889830f595e426 (Debug)

      2022-10-15 14:37:32 0 [Note] /test/SLEEP_MD071022-mariadb-10.11.0-linux-x86_64-dbg/bin/mysqld: ready for connections.
      Version: '10.11.0-MariaDB-debug'  socket: '/test/SLEEP_MD071022-mariadb-10.11.0-linux-x86_64-dbg/socket.sock'  port: 12293  MariaDB Server
      2022-10-15 14:37:33 6 [Note] Event Scheduler: scheduler thread started with id 6
      2022-10-15 14:37:33 0 [Note] InnoDB: Resizing buffer pool from 128.000MiB to 20.000GiB (unit = 2.000MiB).
      2022-10-15 14:37:33 0 [Note] InnoDB: Disabling adaptive hash index.
      2022-10-15 14:37:33 0 [Note] InnoDB: Withdrawing blocks to be shrunken.
      2022-10-15 14:37:33 0 [Note] InnoDB: Latching entire buffer pool.
      2022-10-15 14:37:33 0 [Note] InnoDB: Resizing buffer pool from 64 chunks to 10240 chunks.
      2022-10-15 14:37:34 0 [Note] InnoDB: 10176 chunks (1282170 blocks) were added.
      2022-10-15 14:37:34 0 [Note] InnoDB: Resizing other hash tables.
      -EOL-
      

        Attachments

          Activity

            People

            Assignee:
            Roel Roel Van de Paar
            Reporter:
            Roel Roel Van de Paar
            Votes:
            0 Vote for this issue
            Watchers:
            3 Start watching this issue

              Dates

              Created:
              Updated:
              Resolved:

                Git Integration

                Error rendering 'com.xiplink.jira.git.jira_git_plugin:git-issue-webpanel'. Please contact your Jira administrators.