Uploaded image for project: 'MariaDB Server'
  1. MariaDB Server
  2. MDEV-15174

Server crash in toku_memory_startup or Assertion `r==0' failed (errno=0) in tokudb/PerconaFT/util/partitioned_counter.cc:146 pk_create upon query from ALL_PLUGINS

    XMLWordPrintable

Details

    Description

      Note: the problem looks similar to MDEV-5948, but with important difference: there is nothing in the plugin dir that shouldn't be there, the culprit is not the extra library anymore, it's ha_tokudb.so itself.

      Both the assertion failure and SIGSEGV happen both on debug and release builds, the difference seems to be in the plugins which are loaded at startup, or in server startup options; e.g. with MTR default options it's the assertion failure, and with server default options it's SIGSEGV. One or another happens all the time though, and It's not random: on two machines where I tried, the assertion failure always happens upon 339th execution of the query, and SIGSEGV upon 340th.

      10.2 859fe1f24d499

      /data/src/10.2/storage/tokudb/PerconaFT/util/scoped_malloc.cc:199 toku_scoped_malloc_init: Assertion `r == 0' failed (errno=0) (r=11)
      : Success
      Backtrace: (Note: toku_do_assert=0x0x7f69cd65f941)
       
      #5  0x00007f69d36783fa in abort () from /lib/x86_64-linux-gnu/libc.so.6
      #6  0x00007f69cd65f6b3 in toku_do_backtrace_abort () at /data/src/10.2/storage/tokudb/PerconaFT/portability/toku_assert.cc:147
      #7  0x00007f69cd65f876 in toku_do_assert_zero_fail (expr=11, expr_as_string=0x7f69cd691f7b "r", function=0x7f69cd6922b0 <toku_scoped_malloc_init()::__FUNCTION__> "toku_scoped_malloc_init", file=0x7f69cd691fe0 "/data/src/10.2/storage/tokudb/PerconaFT/util/scoped_malloc.cc", line=199, caller_errno=0) at /data/src/10.2/storage/tokudb/PerconaFT/portability/toku_assert.cc:177
      #8  0x00007f69cd62c4d7 in toku_scoped_malloc_init () at /data/src/10.2/storage/tokudb/PerconaFT/util/scoped_malloc.cc:199
      #9  0x00007f69cd59548c in toku_ft_layer_init () at /data/src/10.2/storage/tokudb/PerconaFT/ft/ft-ops.cc:4632
      #10 0x00007f69cd556138 in toku_ydb_init () at /data/src/10.2/storage/tokudb/PerconaFT/src/ydb.cc:182
      #11 0x00007f69cd555dfc in libtokuft_init () at /data/src/10.2/storage/tokudb/PerconaFT/src/ydb_lib.cc:49
      #12 0x00007f69d556b64a in call_init.part () from /lib64/ld-linux-x86-64.so.2
      #13 0x00007f69d556b75b in _dl_init () from /lib64/ld-linux-x86-64.so.2
      #14 0x00007f69d556fcd8 in dl_open_worker () from /lib64/ld-linux-x86-64.so.2
      #15 0x00007f69d556b4f4 in _dl_catch_error () from /lib64/ld-linux-x86-64.so.2
      #16 0x00007f69d556f489 in _dl_open () from /lib64/ld-linux-x86-64.so.2
      #17 0x00007f69d427eee9 in dlopen_doit () from /lib/x86_64-linux-gnu/libdl.so.2
      #18 0x00007f69d556b4f4 in _dl_catch_error () from /lib64/ld-linux-x86-64.so.2
      #19 0x00007f69d427f531 in _dlerror_run () from /lib/x86_64-linux-gnu/libdl.so.2
      #20 0x00007f69d427ef82 in dlopen () from /lib/x86_64-linux-gnu/libdl.so.2
      #21 0x0000559324fda0d2 in plugin_dl_add (dl=0x7f69cd949e10, report=2) at /data/src/10.2/sql/sql_plugin.cc:765
      #22 0x0000559324fdf5b7 in plugin_dl_foreach (thd=0x7f69bc000b00, dl=0x7f69cd949e10, func=0x559325050b5b <show_plugins(THD*, plugin_ref, void*)>, arg=0x7f69bc10f5f8) at /data/src/10.2/sql/sql_plugin.cc:2437
      #23 0x000055932505182d in fill_all_plugins (thd=0x7f69bc000b00, tables=0x7f69bc012720, cond=0x0) at /data/src/10.2/sql/sql_show.cc:344
      #24 0x000055932506f1ab in get_schema_tables_result (join=0x7f69bc013d70, executed_place=PROCESSED_BY_JOIN_EXEC) at /data/src/10.2/sql/sql_show.cc:8343
      #25 0x000055932500cbf5 in JOIN::exec_inner (this=0x7f69bc013d70) at /data/src/10.2/sql/sql_select.cc:3504
      #26 0x000055932500c2b0 in JOIN::exec (this=0x7f69bc013d70) at /data/src/10.2/sql/sql_select.cc:3335
      #27 0x000055932500d479 in mysql_select (thd=0x7f69bc000b00, tables=0x7f69bc012720, wild_num=1, fields=..., conds=0x0, og_num=0, order=0x0, group=0x0, having=0x0, proc_param=0x0, select_options=2684619520, result=0x7f69bc013d50, unit=0x7f69bc0046a0, select_lex=0x7f69bc004dd8) at /data/src/10.2/sql/sql_select.cc:3735
      #28 0x0000559325001bee in handle_select (thd=0x7f69bc000b00, lex=0x7f69bc0045d8, result=0x7f69bc013d50, setup_tables_done_option=0) at /data/src/10.2/sql/sql_select.cc:373
      #29 0x0000559324fcd9dd in execute_sqlcom_select (thd=0x7f69bc000b00, all_tables=0x7f69bc012720) at /data/src/10.2/sql/sql_parse.cc:6456
      #30 0x0000559324fc3a64 in mysql_execute_command (thd=0x7f69bc000b00) at /data/src/10.2/sql/sql_parse.cc:3467
      #31 0x0000559324fd13c2 in mysql_parse (thd=0x7f69bc000b00, rawbuf=0x7f69bc0124e8 "SELECT * FROM INFORMATION_SCHEMA.ALL_PLUGINS", length=44, parser_state=0x7f69cd94b200, is_com_multi=false, is_next_command=false) at /data/src/10.2/sql/sql_parse.cc:7897
      #32 0x0000559324fbf335 in dispatch_command (command=COM_QUERY, thd=0x7f69bc000b00, packet=0x7f69bc1703b1 "SELECT * FROM INFORMATION_SCHEMA.ALL_PLUGINS", packet_length=44, is_com_multi=false, is_next_command=false) at /data/src/10.2/sql/sql_parse.cc:1806
      #33 0x0000559324fbdc98 in do_command (thd=0x7f69bc000b00) at /data/src/10.2/sql/sql_parse.cc:1360
      #34 0x000055932510bf28 in do_handle_one_connection (connect=0x559328416110) at /data/src/10.2/sql/sql_connect.cc:1335
      #35 0x000055932510bcb5 in handle_one_connection (arg=0x559328416110) at /data/src/10.2/sql/sql_connect.cc:1241
      #36 0x000055932552b47e in pfs_spawn_thread (arg=0x559328375000) at /data/src/10.2/storage/perfschema/pfs.cc:1862
      #37 0x00007f69d5346494 in start_thread (arg=0x7f69cd94c700) at pthread_create.c:333
      #38 0x00007f69d372c93f in clone () from /lib/x86_64-linux-gnu/libc.so.6
      

      #3  <signal handler called>
      #4  0x00007f36fc2b302c in ?? () from /usr/lib/x86_64-linux-gnu/libjemalloc.so.1
      #5  0x00007f36fc2b7d13 in ?? () from /usr/lib/x86_64-linux-gnu/libjemalloc.so.1
      #6  0x00007f36fc2be8fd in ?? () from /usr/lib/x86_64-linux-gnu/libjemalloc.so.1
      #7  0x00007f36fc2c09d1 in ?? () from /usr/lib/x86_64-linux-gnu/libjemalloc.so.1
      #8  0x00007f3710292b72 in toku_memory_startup () at /data/src/10.2/storage/tokudb/PerconaFT/portability/memory.cc:102
      #9  0x00007f371029438f in toku_portability_init () at /data/src/10.2/storage/tokudb/PerconaFT/portability/portability.cc:87
      #10 0x00007f37101cb43b in toku_ft_layer_init () at /data/src/10.2/storage/tokudb/PerconaFT/ft/ft-ops.cc:4621
      #11 0x00007f371018c138 in toku_ydb_init () at /data/src/10.2/storage/tokudb/PerconaFT/src/ydb.cc:182
      #12 0x00007f371018bdfc in libtokuft_init () at /data/src/10.2/storage/tokudb/PerconaFT/src/ydb_lib.cc:49
      #13 0x00007f3715b4c64a in call_init.part () from /lib64/ld-linux-x86-64.so.2
      #14 0x00007f3715b4c75b in _dl_init () from /lib64/ld-linux-x86-64.so.2
      #15 0x00007f3715b50cd8 in dl_open_worker () from /lib64/ld-linux-x86-64.so.2
      #16 0x00007f3715b4c4f4 in _dl_catch_error () from /lib64/ld-linux-x86-64.so.2
      #17 0x00007f3715b50489 in _dl_open () from /lib64/ld-linux-x86-64.so.2
      #18 0x00007f371485fee9 in dlopen_doit () from /lib/x86_64-linux-gnu/libdl.so.2
      #19 0x00007f3715b4c4f4 in _dl_catch_error () from /lib64/ld-linux-x86-64.so.2
      #20 0x00007f3714860531 in _dlerror_run () from /lib/x86_64-linux-gnu/libdl.so.2
      #21 0x00007f371485ff82 in dlopen () from /lib/x86_64-linux-gnu/libdl.so.2
      #22 0x00005561a7f110d2 in plugin_dl_add (dl=0x7f37108c3e60, report=2) at /data/src/10.2/sql/sql_plugin.cc:765
      #23 0x00005561a7f165b7 in plugin_dl_foreach (thd=0x7f36ac000b00, dl=0x7f37108c3e60, func=0x5561a7f87b5b <show_plugins(THD*, plugin_ref, void*)>, arg=0x7f36ac45e2d8) at /data/src/10.2/sql/sql_plugin.cc:2437
      #24 0x00005561a7f8882d in fill_all_plugins (thd=0x7f36ac000b00, tables=0x7f36ac028920, cond=0x0) at /data/src/10.2/sql/sql_show.cc:344
      #25 0x00005561a7fa61ab in get_schema_tables_result (join=0x7f36ac029f70, executed_place=PROCESSED_BY_JOIN_EXEC) at /data/src/10.2/sql/sql_show.cc:8343
      #26 0x00005561a7f43bf5 in JOIN::exec_inner (this=0x7f36ac029f70) at /data/src/10.2/sql/sql_select.cc:3504
      #27 0x00005561a7f432b0 in JOIN::exec (this=0x7f36ac029f70) at /data/src/10.2/sql/sql_select.cc:3335
      #28 0x00005561a7f44479 in mysql_select (thd=0x7f36ac000b00, tables=0x7f36ac028920, wild_num=1, fields=..., conds=0x0, og_num=0, order=0x0, group=0x0, having=0x0, proc_param=0x0, select_options=2684619520, result=0x7f36ac029f50, unit=0x7f36ac0046a0, select_lex=0x7f36ac004dd8) at /data/src/10.2/sql/sql_select.cc:3735
      #29 0x00005561a7f38bee in handle_select (thd=0x7f36ac000b00, lex=0x7f36ac0045d8, result=0x7f36ac029f50, setup_tables_done_option=0) at /data/src/10.2/sql/sql_select.cc:373
      #30 0x00005561a7f049dd in execute_sqlcom_select (thd=0x7f36ac000b00, all_tables=0x7f36ac028920) at /data/src/10.2/sql/sql_parse.cc:6456
      #31 0x00005561a7efaa64 in mysql_execute_command (thd=0x7f36ac000b00) at /data/src/10.2/sql/sql_parse.cc:3467
      #32 0x00005561a7f083c2 in mysql_parse (thd=0x7f36ac000b00, rawbuf=0x7f36ac0286e8 "SELECT * FROM INFORMATION_SCHEMA.ALL_PLUGINS", length=44, parser_state=0x7f37108c5250, is_com_multi=false, is_next_command=false) at /data/src/10.2/sql/sql_parse.cc:7897
      #33 0x00005561a7ef6335 in dispatch_command (command=COM_QUERY, thd=0x7f36ac000b00, packet=0x7f36ac008941 "SELECT * FROM INFORMATION_SCHEMA.ALL_PLUGINS", packet_length=44, is_com_multi=false, is_next_command=false) at /data/src/10.2/sql/sql_parse.cc:1806
      #34 0x00005561a7ef4c98 in do_command (thd=0x7f36ac000b00) at /data/src/10.2/sql/sql_parse.cc:1360
      #35 0x00005561a8042f28 in do_handle_one_connection (connect=0x5561ac6ceaa0) at /data/src/10.2/sql/sql_connect.cc:1335
      #36 0x00005561a8042cb5 in handle_one_connection (arg=0x5561ac6ceaa0) at /data/src/10.2/sql/sql_connect.cc:1241
      #37 0x00007f3715927494 in start_thread (arg=0x7f37108c6700) at pthread_create.c:333
      #38 0x00007f3713d0d93f in clone () from /lib/x86_64-linux-gnu/libc.so.6
      

      --let $num= 1
       
      --disable_query_log
      --disable_result_log
      while ($num < 341)
      {
        --echo # Query no. $num
        SELECT * FROM INFORMATION_SCHEMA.ALL_PLUGINS;
        --inc $num
      }
      --enable_query_log
      --enable_result_log
       
      --echo # All done.
      

      Reproducible on 10.2 and 10.3. Couldn't reproduce on 10.1.

      Attachments

        Issue Links

          Activity

            People

              serg Sergei Golubchik
              elenst Elena Stepanova
              Votes:
              0 Vote for this issue
              Watchers:
              2 Start watching this issue

              Dates

                Created:
                Updated:

                Git Integration

                  Error rendering 'com.xiplink.jira.git.jira_git_plugin:git-issue-webpanel'. Please contact your Jira administrators.