Uploaded image for project: 'MariaDB Server'
  1. MariaDB Server
  2. MDEV-31813

SET GLOBAL innodb_max_purge_lag_wait=… hangs if innodb_read_only=ON

Details

    Description

      # # mysqld options required for replay:  --innodb-force-recovery=6
      SET GLOBAL innodb_undo_log_truncate=1;
      

      Leads to

      10.5.22 5f2a77cef1cced322d3a6e6a48f4f4e5480283dc (Debug)

      mariadbd: /test/10.5_dbg/storage/innobase/srv/srv0srv.cc:1351: void srv_wake_purge_thread_if_not_active(): Assertion `!srv_read_only_mode' failed.
      

      10.5.22 5f2a77cef1cced322d3a6e6a48f4f4e5480283dc (Debug)

      Core was generated by `/test/MD040723-mariadb-10.5.22-linux-x86_64-dbg/bin/mariadbd --no-defaults --in'.
      Program terminated with signal SIGABRT, Aborted.
      #0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50
      [Current thread is 1 (Thread 0x1503332bd700 (LWP 1968710))]
      (gdb) bt
      #0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50
      #1  0x000015035285b859 in __GI_abort () at abort.c:79
      #2  0x000015035285b729 in __assert_fail_base (fmt=0x1503529f1588 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=0x55d0439638dd "!srv_read_only_mode", file=0x55d0439b7b00 "/test/10.5_dbg/storage/innobase/srv/srv0srv.cc", line=1351, function=<optimized out>) at assert.c:92
      #3  0x000015035286cfd6 in __GI___assert_fail (assertion=assertion@entry=0x55d0439638dd "!srv_read_only_mode", file=file@entry=0x55d0439b7b00 "/test/10.5_dbg/storage/innobase/srv/srv0srv.cc", line=line@entry=1351, function=function@entry=0x55d0439b7b78 "void srv_wake_purge_thread_if_not_active()") at assert.c:101
      #4  0x000055d043341033 in srv_wake_purge_thread_if_not_active () at /test/10.5_dbg/storage/innobase/srv/srv0srv.cc:1351
      #5  0x000055d04312ed31 in innodb_undo_log_truncate_update (thd=<optimized out>, save=<optimized out>) at /test/10.5_dbg/storage/innobase/handler/ha_innodb.cc:19691
      #6  0x000055d042b7e485 in sys_var_pluginvar::global_update (this=0x55d0464104e8, thd=0x15031c000d48, var=0x15031c012f38) at /test/10.5_dbg/sql/sql_plugin.cc:3628
      #7  0x000055d042a94a9a in sys_var::update (this=0x55d0464104e8, thd=0x15031c000d48, var=0x15031c012f38) at /test/10.5_dbg/sql/set_var.cc:207
      #8  0x000055d042a94fe7 in set_var::update (this=<optimized out>, thd=<optimized out>) at /test/10.5_dbg/sql/set_var.cc:859
      #9  0x000055d042a96279 in sql_set_variables (thd=thd@entry=0x15031c000d48, var_list=var_list@entry=0x15031c005c88, free=free@entry=true) at /test/10.5_dbg/sql/set_var.cc:746
      #10 0x000055d042b753be in mysql_execute_command (thd=thd@entry=0x15031c000d48) at /test/10.5_dbg/sql/sql_parse.cc:5112
      #11 0x000055d042b5fe55 in mysql_parse (thd=thd@entry=0x15031c000d48, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0x1503332bc310, is_com_multi=is_com_multi@entry=false, is_next_command=is_next_command@entry=false) at /test/10.5_dbg/sql/sql_parse.cc:8106
      #12 0x000055d042b6db08 in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0x15031c000d48, packet=packet@entry=0x15031c00a9e9 "SET GLOBAL innodb_undo_log_truncate=1", packet_length=packet_length@entry=37, is_com_multi=is_com_multi@entry=false, is_next_command=is_next_command@entry=false) at /test/10.5_dbg/sql/sql_class.h:1320
      #13 0x000055d042b703c6 in do_command (thd=0x15031c000d48) at /test/10.5_dbg/sql/sql_parse.cc:1375
      #14 0x000055d042cab37e in do_handle_one_connection (connect=<optimized out>, connect@entry=0x55d046b98258, put_in_cache=put_in_cache@entry=true) at /test/10.5_dbg/sql/sql_connect.cc:1416
      #15 0x000055d042cab84c in handle_one_connection (arg=0x55d046b98258) at /test/10.5_dbg/sql/sql_connect.cc:1318
      #16 0x0000150352d6c609 in start_thread (arg=<optimized out>) at pthread_create.c:477
      #17 0x0000150352958133 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
      

      11.2.0 e81fa345020ec6a067583db6a7019d6404b26f93 (Debug)

      mariadbd: /test/11.2_dbg/storage/innobase/srv/srv0srv.cc:1127: void srv_wake_purge_thread_if_not_active(): Assertion `!srv_read_only_mode' failed.
      

      11.2.0 e81fa345020ec6a067583db6a7019d6404b26f93 (Debug)

      Core was generated by `/test/MD270723-mariadb-11.2.0-linux-x86_64-dbg/bin/mariadbd --no-defaults --inn'.
      Program terminated with signal SIGABRT, Aborted.
      #0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50
      [Current thread is 1 (Thread 0x146b81a2b700 (LWP 598206))]
      (gdb) bt
      #0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50
      #1  0x0000146ba0fbf859 in __GI_abort () at abort.c:79
      #2  0x0000146ba0fbf729 in __assert_fail_base (fmt=0x146ba1155588 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=0x562a3efbe0ee "!srv_read_only_mode", file=0x562a3f00ce10 "/test/11.2_dbg/storage/innobase/srv/srv0srv.cc", line=1127, function=<optimized out>) at assert.c:92
      #3  0x0000146ba0fd0fd6 in __GI___assert_fail (assertion=assertion@entry=0x562a3efbe0ee "!srv_read_only_mode", file=file@entry=0x562a3f00ce10 "/test/11.2_dbg/storage/innobase/srv/srv0srv.cc", line=line@entry=1127, function=function@entry=0x562a3f00d390 "void srv_wake_purge_thread_if_not_active()") at assert.c:101
      #4  0x0000562a3e9b25bd in srv_wake_purge_thread_if_not_active () at /test/11.2_dbg/storage/innobase/srv/srv0srv.cc:1127
      #5  0x0000562a3e7bf696 in innodb_undo_log_truncate_update (thd=<optimized out>, save=<optimized out>) at /test/11.2_dbg/storage/innobase/handler/ha_innodb.cc:19340
      #6  0x0000562a3e1f690d in sys_var_pluginvar::global_update (this=0x562a42100810, thd=0x146b6c000d48, var=0x146b6c013410) at /test/11.2_dbg/sql/sql_plugin.cc:3635
      #7  0x0000562a3e0f7ae0 in sys_var::update (this=0x562a42100810, thd=0x146b6c000d48, var=0x146b6c013410) at /test/11.2_dbg/sql/set_var.cc:207
      #8  0x0000562a3e0f8023 in set_var::update (this=<optimized out>, thd=<optimized out>) at /test/11.2_dbg/sql/set_var.cc:864
      #9  0x0000562a3e0f924a in sql_set_variables (thd=thd@entry=0x146b6c000d48, var_list=var_list@entry=0x146b6c005fe8, free=free@entry=true) at /test/11.2_dbg/sql/set_var.cc:746
      #10 0x0000562a3e1db77a in mysql_execute_command (thd=thd@entry=0x146b6c000d48, is_called_from_prepared_stmt=is_called_from_prepared_stmt@entry=false) at /test/11.2_dbg/sql/sql_parse.cc:4803
      #11 0x0000562a3e1c773b in mysql_parse (thd=thd@entry=0x146b6c000d48, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0x146b81a2a2c0) at /test/11.2_dbg/sql/sql_parse.cc:7800
      #12 0x0000562a3e1d4c00 in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0x146b6c000d48, packet=packet@entry=0x146b6c00af49 "SET GLOBAL innodb_undo_log_truncate=1", packet_length=packet_length@entry=37, blocking=blocking@entry=true) at /test/11.2_dbg/sql/sql_class.h:1374
      #13 0x0000562a3e1d70b8 in do_command (thd=0x146b6c000d48, blocking=blocking@entry=true) at /test/11.2_dbg/sql/sql_parse.cc:1405
      #14 0x0000562a3e33c717 in do_handle_one_connection (connect=<optimized out>, connect@entry=0x562a4236a848, put_in_cache=put_in_cache@entry=true) at /test/11.2_dbg/sql/sql_connect.cc:1445
      #15 0x0000562a3e33cbe6 in handle_one_connection (arg=0x562a4236a848) at /test/11.2_dbg/sql/sql_connect.cc:1347
      #16 0x0000146ba14d0609 in start_thread (arg=<optimized out>) at pthread_create.c:477
      #17 0x0000146ba10bc133 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
      

      Bug confirmed present in:
      MariaDB: 10.5.22 (dbg), 10.6.15 (dbg), 10.9.8 (dbg), 10.10.6 (dbg), 10.11.4 (dbg), 11.0.2 (dbg), 11.1.2 (dbg), 11.2.0 (dbg)

      Bug (or feature/syntax) confirmed not present in:
      MariaDB: 10.4.31 (dbg), 10.4.31 (opt), 10.5.22 (opt), 10.6.15 (opt), 10.9.8 (opt), 10.10.6 (opt), 10.11.4 (opt), 11.0.2 (opt), 11.1.2 (opt), 11.2.0 (opt)

      Attachments

        Issue Links

          Activity

            Following test case crashes the server with a slightly different stack.

            # # mysqld options required for replay:  --innodb-read-only=1
            SET GLOBAL innodb_max_purge_lag_wait=1;
            

            Leads to

            11.2.0 00089ead504c63b99063236f5a870615240b5c00 (Debug)

            mariadbd: /test/11.2_dbg/storage/innobase/srv/srv0srv.cc:1127: void srv_wake_purge_thread_if_not_active(): Assertion `!srv_read_only_mode' failed.
            

            11.2.0 00089ead504c63b99063236f5a870615240b5c00 (Debug)

            Core was generated by `/test/MD090823-mariadb-11.2.0-linux-x86_64-dbg/bin/mariadbd --no-defaults --inn'.
            Program terminated with signal SIGABRT, Aborted.
            #0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50
            [Current thread is 1 (Thread 0x14dead971700 (LWP 876623))]
            (gdb) bt
            #0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50
            #1  0x000014deccf17859 in __GI_abort () at abort.c:79
            #2  0x000014deccf17729 in __assert_fail_base (fmt=0x14decd0ad588 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=0x55c8da77416e "!srv_read_only_mode", file=0x55c8da7c2f70 "/test/11.2_dbg/storage/innobase/srv/srv0srv.cc", line=1127, function=<optimized out>) at assert.c:92
            #3  0x000014deccf28fd6 in __GI___assert_fail (assertion=assertion@entry=0x55c8da77416e "!srv_read_only_mode", file=file@entry=0x55c8da7c2f70 "/test/11.2_dbg/storage/innobase/srv/srv0srv.cc", line=line@entry=1127, function=function@entry=0x55c8da7c34f0 "void srv_wake_purge_thread_if_not_active()") at assert.c:101
            #4  0x000055c8da1667bf in srv_wake_purge_thread_if_not_active () at /test/11.2_dbg/storage/innobase/srv/srv0srv.cc:1127
            #5  0x000055c8d9f754af in innodb_max_purge_lag_wait_update (thd=0x14de98000d48, limit=<optimized out>) at /test/11.2_dbg/storage/innobase/handler/ha_innodb.cc:249
            #6  0x000055c8d99a990d in sys_var_pluginvar::global_update (this=0x55c8dbf74f90, thd=0x14de98000d48, var=0x14de98013410) at /test/11.2_dbg/sql/sql_plugin.cc:3635
            #7  0x000055c8d98aaae0 in sys_var::update (this=0x55c8dbf74f90, thd=0x14de98000d48, var=0x14de98013410) at /test/11.2_dbg/sql/set_var.cc:207
            #8  0x000055c8d98ab023 in set_var::update (this=<optimized out>, thd=<optimized out>) at /test/11.2_dbg/sql/set_var.cc:864
            #9  0x000055c8d98ac24a in sql_set_variables (thd=thd@entry=0x14de98000d48, var_list=var_list@entry=0x14de98005ff0, free=free@entry=true) at /test/11.2_dbg/sql/set_var.cc:746
            #10 0x000055c8d998e77a in mysql_execute_command (thd=thd@entry=0x14de98000d48, is_called_from_prepared_stmt=is_called_from_prepared_stmt@entry=false) at /test/11.2_dbg/sql/sql_parse.cc:4803
            #11 0x000055c8d997a73b in mysql_parse (thd=thd@entry=0x14de98000d48, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0x14dead9702c0) at /test/11.2_dbg/sql/sql_parse.cc:7800
            #12 0x000055c8d9987c00 in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0x14de98000d48, packet=packet@entry=0x14de9800af49 "SET GLOBAL innodb_max_purge_lag_wait=1", packet_length=packet_length@entry=38, blocking=blocking@entry=true) at /test/11.2_dbg/sql/sql_class.h:1375
            #13 0x000055c8d998a0b8 in do_command (thd=0x14de98000d48, blocking=blocking@entry=true) at /test/11.2_dbg/sql/sql_parse.cc:1405
            #14 0x000055c8d9aef709 in do_handle_one_connection (connect=<optimized out>, connect@entry=0x55c8dc1a2058, put_in_cache=put_in_cache@entry=true) at /test/11.2_dbg/sql/sql_connect.cc:1445
            #15 0x000055c8d9aefbd8 in handle_one_connection (arg=0x55c8dc1a2058) at /test/11.2_dbg/sql/sql_connect.cc:1347
            #16 0x000014decd428609 in start_thread (arg=<optimized out>) at pthread_create.c:477
            #17 0x000014decd014133 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
            

            ramesh Ramesh Sivaraman added a comment - Following test case crashes the server with a slightly different stack. # # mysqld options required for replay: --innodb-read-only=1 SET GLOBAL innodb_max_purge_lag_wait=1; Leads to 11.2.0 00089ead504c63b99063236f5a870615240b5c00 (Debug) mariadbd: /test/11.2_dbg/storage/innobase/srv/srv0srv.cc:1127: void srv_wake_purge_thread_if_not_active(): Assertion `!srv_read_only_mode' failed. 11.2.0 00089ead504c63b99063236f5a870615240b5c00 (Debug) Core was generated by `/test/MD090823-mariadb-11.2.0-linux-x86_64-dbg/bin/mariadbd --no-defaults --inn'. Program terminated with signal SIGABRT, Aborted. #0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50 [Current thread is 1 (Thread 0x14dead971700 (LWP 876623))] (gdb) bt #0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50 #1 0x000014deccf17859 in __GI_abort () at abort.c:79 #2 0x000014deccf17729 in __assert_fail_base (fmt=0x14decd0ad588 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=0x55c8da77416e "!srv_read_only_mode", file=0x55c8da7c2f70 "/test/11.2_dbg/storage/innobase/srv/srv0srv.cc", line=1127, function=<optimized out>) at assert.c:92 #3 0x000014deccf28fd6 in __GI___assert_fail (assertion=assertion@entry=0x55c8da77416e "!srv_read_only_mode", file=file@entry=0x55c8da7c2f70 "/test/11.2_dbg/storage/innobase/srv/srv0srv.cc", line=line@entry=1127, function=function@entry=0x55c8da7c34f0 "void srv_wake_purge_thread_if_not_active()") at assert.c:101 #4 0x000055c8da1667bf in srv_wake_purge_thread_if_not_active () at /test/11.2_dbg/storage/innobase/srv/srv0srv.cc:1127 #5 0x000055c8d9f754af in innodb_max_purge_lag_wait_update (thd=0x14de98000d48, limit=<optimized out>) at /test/11.2_dbg/storage/innobase/handler/ha_innodb.cc:249 #6 0x000055c8d99a990d in sys_var_pluginvar::global_update (this=0x55c8dbf74f90, thd=0x14de98000d48, var=0x14de98013410) at /test/11.2_dbg/sql/sql_plugin.cc:3635 #7 0x000055c8d98aaae0 in sys_var::update (this=0x55c8dbf74f90, thd=0x14de98000d48, var=0x14de98013410) at /test/11.2_dbg/sql/set_var.cc:207 #8 0x000055c8d98ab023 in set_var::update (this=<optimized out>, thd=<optimized out>) at /test/11.2_dbg/sql/set_var.cc:864 #9 0x000055c8d98ac24a in sql_set_variables (thd=thd@entry=0x14de98000d48, var_list=var_list@entry=0x14de98005ff0, free=free@entry=true) at /test/11.2_dbg/sql/set_var.cc:746 #10 0x000055c8d998e77a in mysql_execute_command (thd=thd@entry=0x14de98000d48, is_called_from_prepared_stmt=is_called_from_prepared_stmt@entry=false) at /test/11.2_dbg/sql/sql_parse.cc:4803 #11 0x000055c8d997a73b in mysql_parse (thd=thd@entry=0x14de98000d48, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0x14dead9702c0) at /test/11.2_dbg/sql/sql_parse.cc:7800 #12 0x000055c8d9987c00 in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0x14de98000d48, packet=packet@entry=0x14de9800af49 "SET GLOBAL innodb_max_purge_lag_wait=1", packet_length=packet_length@entry=38, blocking=blocking@entry=true) at /test/11.2_dbg/sql/sql_class.h:1375 #13 0x000055c8d998a0b8 in do_command (thd=0x14de98000d48, blocking=blocking@entry=true) at /test/11.2_dbg/sql/sql_parse.cc:1405 #14 0x000055c8d9aef709 in do_handle_one_connection (connect=<optimized out>, connect@entry=0x55c8dc1a2058, put_in_cache=put_in_cache@entry=true) at /test/11.2_dbg/sql/sql_connect.cc:1445 #15 0x000055c8d9aefbd8 in handle_one_connection (arg=0x55c8dc1a2058) at /test/11.2_dbg/sql/sql_connect.cc:1347 #16 0x000014decd428609 in start_thread (arg=<optimized out>) at pthread_create.c:477 #17 0x000014decd014133 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

            The debug assertion is redundant, because srv_read_only_mode implies !purge_sys.enabled(). That is, the function srv_wake_purge_thread_if_not_active() will do nothing else than evaluate purge_sys.enabled() if it is being invoked while InnoDB is in read-only mode.

            Removing the debug assertion will also allow us to remove an unlikely condition from the end of trx_t::commit_in_memory() in 10.4 and 10.5. Starting with MDEV-26193 in 10.6, this function will not call srv_wake_purge_thread_if_not_active() anymore.

            I reproduced a hang of 10.4 using this patched test:

            diff --git a/mysql-test/suite/innodb/t/read_only_recovery.test b/mysql-test/suite/innodb/t/read_only_recovery.test
            index 7da012efb74..7cb6335a1bb 100644
            --- a/mysql-test/suite/innodb/t/read_only_recovery.test
            +++ b/mysql-test/suite/innodb/t/read_only_recovery.test
            @@ -37,6 +37,7 @@ UPDATE t SET a=3 WHERE a=1;
             SELECT * FROM t;
             SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
             SELECT * FROM t;
            +SET GLOBAL innodb_max_purge_lag_wait=0;
             --let $restart_parameters=
             --source include/restart_mysqld.inc
             SELECT * FROM t;
            

            The statement would only be interruptible by KILL or SHUTDOWN. The fix is to make innodb_max_purge_lag_wait_update() return immediately if InnoDB is executing in read-only mode.

            marko Marko Mäkelä added a comment - The debug assertion is redundant, because srv_read_only_mode implies !purge_sys.enabled() . That is, the function srv_wake_purge_thread_if_not_active() will do nothing else than evaluate purge_sys.enabled() if it is being invoked while InnoDB is in read-only mode. Removing the debug assertion will also allow us to remove an unlikely condition from the end of trx_t::commit_in_memory() in 10.4 and 10.5. Starting with MDEV-26193 in 10.6, this function will not call srv_wake_purge_thread_if_not_active() anymore. I reproduced a hang of 10.4 using this patched test: diff --git a/mysql-test/suite/innodb/t/read_only_recovery.test b/mysql-test/suite/innodb/t/read_only_recovery.test index 7da012efb74..7cb6335a1bb 100644 --- a/mysql-test/suite/innodb/t/read_only_recovery.test +++ b/mysql-test/suite/innodb/t/read_only_recovery.test @@ -37,6 +37,7 @@ UPDATE t SET a=3 WHERE a=1; SELECT * FROM t; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; SELECT * FROM t; +SET GLOBAL innodb_max_purge_lag_wait=0; --let $restart_parameters= --source include/restart_mysqld.inc SELECT * FROM t; The statement would only be interruptible by KILL or SHUTDOWN . The fix is to make innodb_max_purge_lag_wait_update() return immediately if InnoDB is executing in read-only mode.

            People

              marko Marko Mäkelä
              ramesh Ramesh Sivaraman
              Votes:
              0 Vote for this issue
              Watchers:
              2 Start watching this issue

              Dates

                Created:
                Updated:
                Resolved:

                Git Integration

                  Error rendering 'com.xiplink.jira.git.jira_git_plugin:git-issue-webpanel'. Please contact your Jira administrators.