Uploaded image for project: 'MariaDB Server'
  1. MariaDB Server
  2. MDEV-15798

Mutex leak on accessing INFORMATION_SCHEMA.INNODB_MUTEXES

Details

    Description

      https://api.travis-ci.org/v3/job/362727661/log.txt

      10.3 a1a966fc0e084494da0f4ff88d41908e094865b9

      		 
      mysqld: /home/travis/src/storage/innobase/include/sync0policy.ic:64: void MutexDebug<Mutex>::enter(const Mutex*, const char*, unsigned int) [with Mutex = TTASEventMutex<GenericPolicy>]: Assertion `!is_owned()' failed.
      		 
      180405 20:56:41 [ERROR] mysqld got signal 6 ;
      		 
       
      		 
      Query (0x7f4760013bf8): /* GenTest::Transform::OrderBy */  SELECT * FROM INFORMATION_SCHEMA.`INNODB_MUTEXES` AS table1 WHERE table1.`CREATE_LINE` <> 9  /* TRANSFORM_OUTCOME_SUPERSET */  /* QNO 8304 CON_ID 17 */
      		 
      Connection ID (thread ID): 17
      		 
      Status: NOT_KILLED


      #6  0x00007f47bc552bf6 in __assert_fail_base (fmt=0x7f47bc6a7058 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=assertion@entry=0x55c93e665da9 "!is_owned()", file=file@entry=0x55c93e665d70 "/home/travis/src/storage/innobase/include/sync0policy.ic", line=line@entry=64, function=function@entry=0x55c93e666a20 <MutexDebug<TTASEventMutex<GenericPolicy> >::enter(TTASEventMutex<GenericPolicy> const*, char const*, unsigned int)::__PRETTY_FUNCTION__> "void MutexDebug<Mutex>::enter(const Mutex*, const char*, unsigned int) [with Mutex = TTASEventMutex<GenericPolicy>]") at assert.c:92
      		 
      #7  0x00007f47bc552ca2 in __GI___assert_fail (assertion=0x55c93e665da9 "!is_owned()", file=0x55c93e665d70 "/home/travis/src/storage/innobase/include/sync0policy.ic", line=64, function=0x55c93e666a20 <MutexDebug<TTASEventMutex<GenericPolicy> >::enter(TTASEventMutex<GenericPolicy> const*, char const*, unsigned int)::__PRETTY_FUNCTION__> "void MutexDebug<Mutex>::enter(const Mutex*, const char*, unsigned int) [with Mutex = TTASEventMutex<GenericPolicy>]") at assert.c:101
      		 
      #8  0x000055c93de74016 in MutexDebug<TTASEventMutex<GenericPolicy> >::enter (this=0x55c93f0eb690 <rw_lock_list_mutex+16>, mutex=0x55c93f0eb680 <rw_lock_list_mutex>, name=0x55c93e674a98 "/home/travis/src/storage/innobase/handler/i_s.cc", line=9102) at /home/travis/src/storage/innobase/include/sync0policy.ic:64
      		 
      #9  0x000055c93de72ca0 in GenericPolicy<TTASEventMutex<GenericPolicy> >::enter (this=0x55c93f0eb690 <rw_lock_list_mutex+16>, mutex=..., filename=0x55c93e674a98 "/home/travis/src/storage/innobase/handler/i_s.cc", line=9102) at /home/travis/src/storage/innobase/include/sync0policy.h:348
      		 
      #10 0x000055c93de71336 in PolicyMutex<TTASEventMutex<GenericPolicy> >::enter (this=0x55c93f0eb680 <rw_lock_list_mutex>, n_spins=30, n_delay=4, name=0x55c93e674a98 "/home/travis/src/storage/innobase/handler/i_s.cc", line=9102) at /home/travis/src/storage/innobase/include/ib0mutex.h:635
      		 
      #11 0x000055c93deb09e2 in i_s_innodb_mutexes_fill_table (thd=0x7f4760000d90, tables=0x7f4760013fa0) at /home/travis/src/storage/innobase/handler/i_s.cc:9102
      		 
      #12 0x000055c93da0305f in get_schema_tables_result (join=0x7f4760014f40, executed_place=PROCESSED_BY_JOIN_EXEC) at /home/travis/src/sql/sql_show.cc:8872
      		 
      #13 0x000055c93d99cb94 in JOIN::exec_inner (this=0x7f4760014f40) at /home/travis/src/sql/sql_select.cc:3952
      		 
      #14 0x000055c93d99c23a in JOIN::exec (this=0x7f4760014f40) at /home/travis/src/sql/sql_select.cc:3783
      		 
      #15 0x000055c93d99d4ae in mysql_select (thd=0x7f4760000d90, tables=0x7f4760013fa0, wild_num=1, fields=..., conds=0x7f47600147e8, og_num=0, order=0x0, group=0x0, having=0x0, proc_param=0x0, select_options=552440433408, result=0x7f4760014f20, unit=0x7f4760004bd8, select_lex=0x7f4760005350) at /home/travis/src/sql/sql_select.cc:4188
      		 
      #16 0x000055c93d98f747 in handle_select (thd=0x7f4760000d90, lex=0x7f4760004b10, result=0x7f4760014f20, setup_tables_done_option=0) at /home/travis/src/sql/sql_select.cc:382
      		 
      #17 0x000055c93d95a8b4 in execute_sqlcom_select (thd=0x7f4760000d90, all_tables=0x7f4760013fa0) at /home/travis/src/sql/sql_parse.cc:6556
      		 
      #18 0x000055c93d950f20 in mysql_execute_command (thd=0x7f4760000d90) at /home/travis/src/sql/sql_parse.cc:3770
      		 
      #19 0x000055c93d95e288 in mysql_parse (thd=0x7f4760000d90, rawbuf=0x7f4760013bf8 "/* GenTest::Transform::OrderBy */  SELECT * FROM INFORMATION_SCHEMA.`INNODB_MUTEXES` AS table1 WHERE table1.`CREATE_LINE` <> 9  /* TRANSFORM_OUTCOME_SUPERSET */  /* QNO 8304 CON_ID 17 */", length=186, parser_state=0x7f47b8e995e0, is_com_multi=false, is_next_command=false) at /home/travis/src/sql/sql_parse.cc:8013
      		 
      #20 0x000055c93d94b97e in dispatch_command (command=COM_QUERY, thd=0x7f4760000d90, packet=0x7f476000b431 "/* GenTest::Transform::OrderBy */  SELECT * FROM INFORMATION_SCHEMA.`INNODB_MUTEXES` AS table1 WHERE table1.`CREATE_LINE` <> 9  /* TRANSFORM_OUTCOME_SUPERSET */  /* QNO 8304 CON_ID 17 */ ", packet_length=187, is_com_multi=false, is_next_command=false) at /home/travis/src/sql/sql_parse.cc:1842
      		 
      #21 0x000055c93d94a3ca in do_command (thd=0x7f4760000d90) at /home/travis/src/sql/sql_parse.cc:1387
      		 
      #22 0x000055c93daafb71 in do_handle_one_connection (connect=0x55c940a254c0) at /home/travis/src/sql/sql_connect.cc:1402
      		 
      #23 0x000055c93daaf8fe in handle_one_connection (arg=0x55c940a254c0) at /home/travis/src/sql/sql_connect.cc:1308
      		 
      #24 0x00007f47bd114184 in start_thread (arg=0x7f47b8e9a700) at pthread_create.c:312
      		 
      #25 0x00007f47bc62103d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

      experimental 4feb78e17fbecf7fc7f2847b49b8c66b54879629

      		 
      perl /home/travis/rqg/runall-new.pl --duration=350 --threads=6 --seed=1522961592 --reporters=Backtrace,ErrorLog,Deadlock --validators=TransformerNoComparator --transformers=ConvertSubqueriesToViews,ConvertTablesToDerived,Count,DisableOptimizations,Distinct,EnableOptimizations,ExecuteAsCTE,ExecuteAsDeleteReturning,ExecuteAsDerived,ExecuteAsExcept,ExecuteAsExecuteImmediate,ExecuteAsInsertSelect,ExecuteAsIntersect,ExecuteAsSelectItem,ExecuteAsUnion,ExecuteAsUpdateDelete,ExecuteAsView,ExecuteAsWhereSubquery,Having,InlineSubqueries,LimitRowsExamined,OrderBy,StraightJoin,ExecuteAsPreparedTwice,ExecuteAsTrigger,ExecuteAsSPTwice,ExecuteAsFunctionTwice --redefine=conf/mariadb/general-workarounds.yy --mysqld=--log_output=FILE --mysqld=--log_bin_trust_function_creators=1 --mysqld=--log-bin --mysqld=--max-statement-time=30 --mysqld=--loose-debug_assert_on_not_freed_memory=0 --views --redefine=conf/mariadb/versioning.yy --basedir=/home/travis/server --grammar=conf/runtime/information_schema.yy --engine=InnoDB --vardir=/home/travis/logs/vardir1_2

      Not reproducible right away.

      Attachments

        Issue Links

          is caused by

          Task - A task that needs to be done. MDEV-7399 Add support for INFORMATION_SCHEMA.INNODB_MUTEXES

          • Minor - Minor loss of function, or other problem where easy workaround is present.
          • Closed
          relates to

          Task - A task that needs to be done. MDEV-6114 merge new P_S instrumentation and tables from 5.7

          • Critical - Crashes, loss of data, severe memory leak.
          • Closed

          Bug - A problem which impairs or prevents the functions of the product. MDEV-24142 rw_lock_t has unnecessarily complex wait logic

          • Major - Major loss of function.
          • Closed

          Bug - A problem which impairs or prevents the functions of the product. MDEV-18343 InnoDB: DEADLOCK of threads detected: Mutex RW_LOCK_LIST / Mutex DICT_SYS upon CREATE TABLE .. SELECT * FROM INFORMATION_SCHEMA.`INNODB_MUTEXES`

          • Major - Major loss of function.
          • Closed

          Activity

            Ascending order - Click to sort in descending order
            marko Marko Mäkelä added a comment -

            I think that the bug is pretty obvious in i_s_innodb_mutexes_fill_table(), which was introduced in MDEV-7399 in MariaDB 10.1.3:

            	mutex_enter(&rw_lock_list_mutex);
            		 
            	for (lock = UT_LIST_GET_FIRST(rw_lock_list); lock != NULL;
            		 
            	     lock = UT_LIST_GET_NEXT(list, lock)) {
            		 
            …
            		 
            		OK(schema_table_store_record(thd, tables->table));
            		 
            	}
            		 
            …
            		 
            	mutex_exit(&rw_lock_list_mutex);
            		 
             
            		 
            	DBUG_RETURN(0);

            If the condition in any of the OK macros holds, then we would return from the function without releasing the mutex:

            #define OK(expr)		\
            		 
            	if ((expr) != 0) {	\
            		 
            		DBUG_RETURN(1);	\
            		 
            	}

            We should review all of the file i_s.cc for similar errors. One way to ensure the release of resources on leaving a scope would be to use the RAII paradigm.

            marko Marko Mäkelä added a comment - I think that the bug is pretty obvious in i_s_innodb_mutexes_fill_table() , which was introduced in MDEV-7399 in MariaDB 10.1.3 : mutex_enter(&rw_lock_list_mutex); for (lock = UT_LIST_GET_FIRST(rw_lock_list); lock != NULL; lock = UT_LIST_GET_NEXT(list, lock)) { … OK(schema_table_store_record(thd, tables->table)); } … mutex_exit(&rw_lock_list_mutex);   DBUG_RETURN(0); If the condition in any of the OK macros holds, then we would return from the function without releasing the mutex: #define OK(expr) \ if ((expr) != 0) { \ DBUG_RETURN(1); \ } We should review all of the file i_s.cc for similar errors. One way to ensure the release of resources on leaving a scope would be to use the RAII paradigm.
            marko Marko Mäkelä added a comment -

            The reported bug was introduced in MDEV-7399.

            The same commit also added MDEV-7109 (INFORMATION_SCHEMA.INNODB_SEMAPHORE_WAITS), but it does not seem to suffer from the same type of problem. Instead, it appears to access some global data structures without any concurrency control (prone to return inconsistent data due to inherent race conditions).

            marko Marko Mäkelä added a comment - The reported bug was introduced in MDEV-7399 . The same commit also added MDEV-7109 ( INFORMATION_SCHEMA.INNODB_SEMAPHORE_WAITS ), but it does not seem to suffer from the same type of problem. Instead, it appears to access some global data structures without any concurrency control (prone to return inconsistent data due to inherent race conditions).
            marko Marko Mäkelä added a comment -

            Remove-rw_lock_list.patch would remove the view altogether, along with removing SHOW ENGINE INNODB MUTEX output.
            This is probably too extreme before we start exposing this information via performance_schema (MDEV-6114).

            marko Marko Mäkelä added a comment - Remove-rw_lock_list.patch would remove the view altogether, along with removing SHOW ENGINE INNODB MUTEX output. This is probably too extreme before we start exposing this information via performance_schema ( MDEV-6114 ).
            marko Marko Mäkelä added a comment -

            I rewrote the code to use the C++ RAII pattern, so that neither mutex_list_mutex nor rw_lock_list_mutex will be ‘stuck’ if the function returns prematurely.

            marko Marko Mäkelä added a comment - I rewrote the code to use the C++ RAII pattern, so that neither mutex_list_mutex nor rw_lock_list_mutex will be ‘stuck’ if the function returns prematurely.
            marko Marko Mäkelä added a comment -

            It turns out that starting with MariaDB 10.2.2, information_schema.innodb_mutexes does not output any information about mutexes, only about rw-locks. The mutex information is available via the old SHOW ENGINE INNODB MUTEX interface, which this view is duplicating.

            marko Marko Mäkelä added a comment - It turns out that starting with MariaDB 10.2.2, information_schema.innodb_mutexes does not output any information about mutexes, only about rw-locks. The mutex information is available via the old SHOW ENGINE INNODB MUTEX interface, which this view is duplicating.
            marko Marko Mäkelä added a comment -

            MDEV-24142 in MariaDB Server 10.6.0 removed INFORMATION_SCHEMA.INNODB_MUTEXES altogether.

            marko Marko Mäkelä added a comment - MDEV-24142 in MariaDB Server 10.6.0 removed INFORMATION_SCHEMA.INNODB_MUTEXES altogether.

            People

              marko Marko Mäkelä
              elenst Elena Stepanova
              Votes:
              0 Vote for this issue
              Watchers:
              3 Start watching this issue

              Dates

                Created:
                Updated:
                Resolved:

                Git Integration

                  Error rendering 'com.xiplink.jira.git.jira_git_plugin:git-issue-webpanel'. Please contact your Jira administrators.