Details
-
Bug
-
Status: Closed (View Workflow)
-
Critical
-
Resolution: Fixed
-
10.6
-
None
Description
This is a non-standard issue, quite possibly it's a flaw in our support of MSAN, or in the way I build it.
The test case below causes a crash on my MSAN builds – so far, I had it only on MSAN builds; however, it is not a MemorySanitizer error which results in a server abort, it is a direct SIGSEGV. The server crashes without any visible attempt to write a crash report in the error log, however a coredump is created, if the system allows it.
MTR cannot handle a coredump if it's run with MSAN libraries in LD_LIBRARY_PATH (gdb doesn't start). But the coredump itself is usable and it reveals the stack trace pasted below.
--source include/have_innodb.inc
|
|
|
CREATE TABLE t (pk INT PRIMARY KEY) ENGINE=InnoDB; |
INSERT INTO t VALUES (1),(2); |
|
|
CREATE PROCEDURE sp1() UPDATE t SET pk = 1; |
CREATE PROCEDURE sp2() CALL sp1; |
--error ER_DUP_ENTRY
|
CALL sp2;
|
|
|
# Cleanup
|
DROP PROCEDURE sp2; |
DROP PROCEDURE sp1; |
DROP TABLE t; |
|
10.6 0df4651c13d68f1c228986f940d8ea4a226132f5 |
Program terminated with signal SIGSEGV, Segmentation fault.
|
#0 0x00005595827db261 in debug_sync (thd=thd@entry=0x0, sync_point_name=0x559585f00a47 "ha_innobase_info_low", name_len=name_len@entry=20) at /data/src/10.6-msan/sql/debug_sync.cc:1757
|
1757 {
|
[Current thread is 1 (Thread 0x7fe7c0ee76c0 (LWP 2125231))]
|
|
|
#0 0x00005595827db261 in debug_sync (thd=thd@entry=0x0, sync_point_name=0x559585f00a47 "ha_innobase_info_low", name_len=name_len@entry=20) at /data/src/10.6-msan/sql/debug_sync.cc:1757
|
#1 0x00005595844ca918 in ha_innobase::info_low (this=0x71d0001a8630, flag=34, is_analyze=false) at /data/src/10.6-msan/storage/innobase/handler/ha_innodb.cc:14709
|
#2 0x0000559583453af8 in handler::get_dup_key (this=0x71d0001a8630, error=121) at /data/src/10.6-msan/sql/handler.cc:4890
|
#3 0x0000559582bb189d in prepare_record_for_error_message (error=<optimized out>, table=table@entry=0x719000032018) at /data/src/10.6-msan/sql/sql_update.cc:275
|
#4 0x0000559582bad215 in mysql_update (thd=<optimized out>, thd@entry=0x72b00007e018, table_list=<optimized out>, fields=..., values=..., conds=<optimized out>, order_num=<optimized out>, order=<optimized out>, limit=<optimized out>, ignore=<optimized out>, found_return=<optimized out>, updated_return=<optimized out>) at /data/src/10.6-msan/sql/sql_update.cc:1148
|
#5 0x000055958272eac4 in mysql_execute_command (thd=<optimized out>, is_called_from_prepared_stmt=true) at /data/src/10.6-msan/sql/sql_parse.cc:4444
|
#6 0x00005595823a87ce in sp_instr_stmt::exec_core (this=0x70a000036270, thd=0x0, nextp=0x7fe7c0ee0a98) at /data/src/10.6-msan/sql/sp_head.cc:3843
|
#7 0x00005595823a3976 in sp_lex_keeper::reset_lex_and_exec_core (this=<optimized out>, this@entry=0x70a0000362b8, thd=<optimized out>, thd@entry=0x72b00007e018, nextp=<optimized out>, nextp@entry=0x7fe7c0ee0a98, open_tables=false, instr=<optimized out>, instr@entry=0x70a000036270) at /data/src/10.6-msan/sql/sp_head.cc:3568
|
#8 0x00005595823a648a in sp_instr_stmt::execute (this=<optimized out>, thd=<optimized out>, nextp=<optimized out>) at /data/src/10.6-msan/sql/sp_head.cc:3749
|
#9 0x00005595823815bd in sp_head::execute (this=this@entry=0x71e000012c30, thd=thd@entry=0x72b00007e018, merge_da_on_success=true) at /data/src/10.6-msan/sql/sp_head.cc:1442
|
#10 0x000055958238b827 in sp_head::execute_procedure (this=<optimized out>, thd=0x72b00007e018, args=<optimized out>) at /data/src/10.6-msan/sql/sp_head.cc:2485
|
#11 0x0000559582720e03 in do_execute_sp (thd=<optimized out>, sp=<optimized out>, sp@entry=0x71e000012c30) at /data/src/10.6-msan/sql/sql_parse.cc:3057
|
#12 0x000055958271fe6b in Sql_cmd_call::execute (this=<optimized out>, thd=<optimized out>) at /data/src/10.6-msan/sql/sql_parse.cc:3303
|
#13 0x00005595827271ae in mysql_execute_command (thd=<optimized out>, is_called_from_prepared_stmt=false) at /data/src/10.6-msan/sql/sql_parse.cc:6117
|
#14 0x00005595823a87ce in sp_instr_stmt::exec_core (this=0x70a000035e10, thd=0x0, nextp=0x7fe7c0ee3048) at /data/src/10.6-msan/sql/sp_head.cc:3843
|
#15 0x00005595823a3976 in sp_lex_keeper::reset_lex_and_exec_core (this=<optimized out>, this@entry=0x70a000035e58, thd=<optimized out>, thd@entry=0x72b00007e018, nextp=<optimized out>, nextp@entry=0x7fe7c0ee3048, open_tables=false, instr=<optimized out>, instr@entry=0x70a000035e10) at /data/src/10.6-msan/sql/sp_head.cc:3568
|
#16 0x00005595823a648a in sp_instr_stmt::execute (this=<optimized out>, thd=<optimized out>, nextp=<optimized out>) at /data/src/10.6-msan/sql/sp_head.cc:3749
|
#17 0x00005595823815bd in sp_head::execute (this=this@entry=0x71e000011430, thd=thd@entry=0x72b00007e018, merge_da_on_success=true) at /data/src/10.6-msan/sql/sp_head.cc:1442
|
#18 0x000055958238b827 in sp_head::execute_procedure (this=<optimized out>, thd=0x72b00007e018, args=<optimized out>) at /data/src/10.6-msan/sql/sp_head.cc:2485
|
#19 0x0000559582720e03 in do_execute_sp (thd=<optimized out>, sp=<optimized out>, sp@entry=0x71e000011430) at /data/src/10.6-msan/sql/sql_parse.cc:3057
|
#20 0x000055958271fe6b in Sql_cmd_call::execute (this=<optimized out>, thd=<optimized out>) at /data/src/10.6-msan/sql/sql_parse.cc:3303
|
#21 0x00005595827271ae in mysql_execute_command (thd=thd@entry=0x72b00007e018, is_called_from_prepared_stmt=false) at /data/src/10.6-msan/sql/sql_parse.cc:6117
|
#22 0x000055958270ef0b in mysql_parse (thd=thd@entry=0x72b00007e018, rawbuf=0x72b000082320 "\200\324Z\206\225U", length=2147486200, parser_state=parser_state@entry=0x7fe7c0ee5340) at /data/src/10.6-msan/sql/sql_parse.cc:8143
|
#23 0x00005595827029c1 in dispatch_command (command=COM_QUERY, thd=thd@entry=0x72b00007e018, packet=0x0, packet_length=531704, blocking=false) at /data/src/10.6-msan/sql/sql_parse.cc:1896
|
#24 0x0000559582711301 in do_command (thd=<optimized out>, blocking=true) at /data/src/10.6-msan/sql/sql_parse.cc:1409
|
#25 0x0000559582d4e300 in do_handle_one_connection (connect=<optimized out>, put_in_cache=true) at /data/src/10.6-msan/sql/sql_connect.cc:1415
|
#26 0x0000559582d4d8e6 in handle_one_connection (arg=0x0) at /data/src/10.6-msan/sql/sql_connect.cc:1317
|
#27 0x00005595841c4ebb in pfs_spawn_thread (arg=0x716000003618) at /data/src/10.6-msan/storage/perfschema/pfs.cc:2201
|
#28 0x00007fe7c8ecb044 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
|
#29 0x00007fe7c8f4b61c in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81
|
|
10.6-enterprise 249b7e2a0440ae1eb3ba14c607bdb61393bd464e |
Program terminated with signal SIGSEGV, Segmentation fault.
|
#0 0x000056196ac1291e in _my_thread_var () at /data/bld/10.6-enterprise-msan/mysys/my_thr_init.c:387
|
387 return my_pthread_getspecific(struct st_my_thread_var*,THR_KEY_mysys);
|
[Current thread is 1 (Thread 0x7fc82e5a16c0 (LWP 2088429))]
|
|
|
Thread 1 (Thread 0x7fc82e5a16c0 (LWP 2088429)):
|
#0 0x000056196ac1291e in _my_thread_var () at /data/bld/10.6-enterprise-msan/mysys/my_thr_init.c:387
|
#1 my_thread_var_dbug () at /data/bld/10.6-enterprise-msan/mysys/my_thr_init.c:437
|
#2 0x000056196acaf1f1 in code_state () at /data/bld/10.6-enterprise-msan/dbug/dbug.c:386
|
#3 0x000056196acb8025 in _db_enter_ (_func_=0x56196b5e0334 "info", _file_=0x56196b63c004 "/data/bld/10.6-enterprise-msan/storage/innobase/handler/ha_innodb.cc", _line_=14749, _stack_frame_=0x7fc82e558210) at /data/bld/10.6-enterprise-msan/dbug/dbug.c:1140
|
#4 0x0000561969bcdb1a in ha_innobase::info_low (this=0x71e000032430, flag=34, is_analyze=false) at /data/bld/10.6-enterprise-msan/storage/innobase/handler/ha_innodb.cc:14749
|
#5 0x0000561968b69d88 in handler::get_dup_key (this=0x71e000032430, error=121) at /data/bld/10.6-enterprise-msan/sql/handler.cc:4897
|
#6 0x0000561968287acd in prepare_record_for_error_message (error=<optimized out>, table=table@entry=0x719000041f18) at /data/bld/10.6-enterprise-msan/sql/sql_update.cc:275
|
#7 0x0000561968283429 in mysql_update (thd=<optimized out>, thd@entry=0x72b000149018, table_list=<optimized out>, fields=..., values=..., conds=<optimized out>, order_num=<optimized out>, order=<optimized out>, limit=<optimized out>, ignore=<optimized out>, found_return=<optimized out>, updated_return=<optimized out>) at /data/bld/10.6-enterprise-msan/sql/sql_update.cc:1154
|
#8 0x0000561967da5073 in mysql_execute_command (thd=<optimized out>, is_called_from_prepared_stmt=true) at /data/bld/10.6-enterprise-msan/sql/sql_parse.cc:4458
|
#9 0x0000561967a1c51e in sp_instr_stmt::exec_core (this=0x70a000036270, thd=0xb, nextp=0x7fc82e59aa58) at /data/bld/10.6-enterprise-msan/sql/sp_head.cc:3846
|
#10 0x0000561967a176b5 in sp_lex_keeper::reset_lex_and_exec_core (this=<optimized out>, this@entry=0x70a0000362b8, thd=<optimized out>, thd@entry=0x72b000149018, nextp=0xffffffffffffedf0, nextp@entry=0x7fc82e59aa58, open_tables=false, instr=<optimized out>, instr@entry=0x70a000036270) at /data/bld/10.6-enterprise-msan/sql/sp_head.cc:3571
|
#11 0x0000561967a1a1da in sp_instr_stmt::execute (this=<optimized out>, thd=<optimized out>, nextp=<optimized out>) at /data/bld/10.6-enterprise-msan/sql/sp_head.cc:3752
|
#12 0x00005619679f4ba5 in sp_head::execute (this=this@entry=0x71e000048c30, thd=thd@entry=0x72b000149018, merge_da_on_success=true) at /data/bld/10.6-enterprise-msan/sql/sp_head.cc:1442
|
#13 0x00005619679ff457 in sp_head::execute_procedure (this=<optimized out>, thd=0x72b000149018, args=<optimized out>) at /data/bld/10.6-enterprise-msan/sql/sp_head.cc:2485
|
#14 0x0000561967d96d43 in do_execute_sp (thd=<optimized out>, sp=<optimized out>, sp@entry=0x71e000048c30) at /data/bld/10.6-enterprise-msan/sql/sql_parse.cc:3069
|
#15 0x0000561967d95dab in Sql_cmd_call::execute (this=<optimized out>, thd=<optimized out>) at /data/bld/10.6-enterprise-msan/sql/sql_parse.cc:3315
|
#16 0x0000561967d9d2fe in mysql_execute_command (thd=<optimized out>, is_called_from_prepared_stmt=false) at /data/bld/10.6-enterprise-msan/sql/sql_parse.cc:6097
|
#17 0x0000561967a1c51e in sp_instr_stmt::exec_core (this=0x70a000035e10, thd=0xb, nextp=0x7fc82e59d028) at /data/bld/10.6-enterprise-msan/sql/sp_head.cc:3846
|
#18 0x0000561967a176b5 in sp_lex_keeper::reset_lex_and_exec_core (this=<optimized out>, this@entry=0x70a000035e58, thd=<optimized out>, thd@entry=0x72b000149018, nextp=0xffffffffffffedf0, nextp@entry=0x7fc82e59d028, open_tables=false, instr=<optimized out>, instr@entry=0x70a000035e10) at /data/bld/10.6-enterprise-msan/sql/sp_head.cc:3571
|
#19 0x0000561967a1a1da in sp_instr_stmt::execute (this=<optimized out>, thd=<optimized out>, nextp=<optimized out>) at /data/bld/10.6-enterprise-msan/sql/sp_head.cc:3752
|
#20 0x00005619679f4ba5 in sp_head::execute (this=this@entry=0x71e000044430, thd=thd@entry=0x72b000149018, merge_da_on_success=true) at /data/bld/10.6-enterprise-msan/sql/sp_head.cc:1442
|
#21 0x00005619679ff457 in sp_head::execute_procedure (this=<optimized out>, thd=0x72b000149018, args=<optimized out>) at /data/bld/10.6-enterprise-msan/sql/sp_head.cc:2485
|
#22 0x0000561967d96d43 in do_execute_sp (thd=<optimized out>, sp=<optimized out>, sp@entry=0x71e000044430) at /data/bld/10.6-enterprise-msan/sql/sql_parse.cc:3069
|
#23 0x0000561967d95dab in Sql_cmd_call::execute (this=<optimized out>, thd=<optimized out>) at /data/bld/10.6-enterprise-msan/sql/sql_parse.cc:3315
|
#24 0x0000561967d9d2fe in mysql_execute_command (thd=thd@entry=0x72b000149018, is_called_from_prepared_stmt=false) at /data/bld/10.6-enterprise-msan/sql/sql_parse.cc:6097
|
#25 0x0000561967d83d1b in mysql_parse (thd=thd@entry=0x72b000149018, rawbuf=0x72b00014d320 "\360$\320k\031V", length=2147486392, parser_state=parser_state@entry=0x7fc82e59f330) at /data/bld/10.6-enterprise-msan/sql/sql_parse.cc:8144
|
#26 0x0000561967d77575 in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0x72b000149018, packet=0x0, packet_length=1363192, blocking=false) at /data/bld/10.6-enterprise-msan/sql/sql_parse.cc:1903
|
#27 0x0000561967d86181 in do_command (thd=0x72b000149018, blocking=true) at /data/bld/10.6-enterprise-msan/sql/sql_parse.cc:1416
|
#28 0x0000561968426190 in do_handle_one_connection (connect=<optimized out>, put_in_cache=true) at /data/bld/10.6-enterprise-msan/sql/sql_connect.cc:1415
|
#29 0x0000561968425776 in handle_one_connection (arg=0xb) at /data/bld/10.6-enterprise-msan/sql/sql_connect.cc:1317
|
#30 0x0000561969899adb in pfs_spawn_thread (arg=0x716000003f18) at /data/bld/10.6-enterprise-msan/storage/perfschema/pfs.cc:2201
|
#31 0x00007fc83abdc044 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
|
#32 0x00007fc83ac5c61c in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81
|
A valgrind build also fails, but in a very different manner.
At least in my builds, the failure starts happening with this commit in 10.6:
commit b5d65fc105b6efa07cbbafcf053f6ce46f02dbc0 (HEAD)
|
Author: Monty
|
Date: Sun Feb 18 17:30:01 2024 +0200
|
|
|
Optimize performance of my_bitmap
|
Attachments
Activity
True, it is not reproducible on b3d507ff13311a4c2c40a0ff614494084a1eb6e9, more precisely it stops failing for me from the commit
commit f838b2d7998f18ac2a1bb9d56081aac6e563de1e (HEAD)
|
Author: Monty
|
Date: Fri Mar 8 15:18:21 2024 +0200
|
|
|
MDEV-33623 Partitioning is broken on big endian architectures
|
|
MDEV-33502 Slowdown when running nested statement with many partitions
|
caused this error as I failed to take into account bigendian architectures.
|
|
This patch also introduces bitmap_import() and bitmap_export() to be used
|
when one wants to store bitmaps in files/logs in a portable way.
|
that is, two commits after it was introduced.
On the "bad" commits, the failure indeed stops happening if the server is built with -DWITH_DBUG_TRACE=OFF.
The only failure that I am able to reproduce is the one on shutdown. The MSAN builder on our CI should be unaffected by that, because it is configured with cmake -DWITH_DBUG_TRACE=OFF.
I couldn’t reproduce this crash when building clang 18.1.0 and -DWITH_DBUG_TRACE=OFF (
MDEV-29613). When it comes to invoking gdb, I see that about a year ago (possibly related toMDEV-22083), I had written a wrapper script called gdb that is in my $PATH before /usr/bin:$HOME/bin/gdb
#!/bin/shunset LD_LIBRARY_PATHexec /usr/bin/gdb "$@"However, I can reproduce something when I use the default setting -DWITH_DBUG_TRACE=ON:
10.6 b3d507ff13311a4c2c40a0ff614494084a1eb6e9
2024-03-11 7:55:49 0 [Note] InnoDB: Shutdown completed; log sequence number 59919; transaction id 282024-03-11 7:55:49 0 [Note] Debug sync points hit: 25782024-03-11 7:55:49 0 [Note] Debug sync points executed: 02024-03-11 7:55:49 0 [Note] Debug sync points max active per thread: 02024-03-11 7:55:49 0 [Note] /dev/shm/10.6msan/sql/mariadbd: Shutdown completeMemorySanitizer:DEADLYSIGNAL==84865==ERROR: MemorySanitizer: SEGV on unknown address 0x2fd90b17faa8 (pc 0x2fd90b17faa8 bp 0x7fd90b17f580 sp 0x7fd90b17f398 T85187)==84865==The signal is caused by a READ memory access.==84865==Hint: PC is at a non-executable region. Maybe a wild jump?At the time of the crash, only two threads exist: the main thread, and the signal handler thread. The crash occurs somewhere deep inside pthread_exit() in the signal handler thread while the main thread is waiting:
10.6 b3d507ff13311a4c2c40a0ff614494084a1eb6e9
#11 0x000055c0d6c74459 in my_sleep (m_seconds=<optimized out>) at /mariadb/10.6/mysys/my_sleep.c:29#12 0x000055c0d4943b95 in wait_for_signal_thread_to_end () at /mariadb/10.6/sql/mysqld.cc:2073#13 mysqld_exit (exit_code=exit_code@entry=0) at /mariadb/10.6/sql/mysqld.cc:1914#14 0x000055c0d4949653 in mysqld_main (argc=159, argv=0x71a000000070) at /mariadb/10.6/sql/mysqld.cc:5955#15 0x00007fd9122076ca in __libc_start_call_main (main=main@entry=0x55c0d4940740 <main(int, char**)>, argc=argc@entry=27, argv=argv@entry=0x7ffd689dee28) at ../sysdeps/nptl/libc_start_call_main.h:58#16 0x00007fd912207785 in __libc_start_main_impl (main=0x55c0d4940740 <main(int, char**)>, argc=27, argv=0x7ffd689dee28, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7ffd689dee18) at ../csu/libc-start.c:360#17 0x000055c0d48acd91 in _start ()The following patch would fix this crash in my environment:
diff --git a/sql/mysqld.cc b/sql/mysqld.ccindex e224871795e..9253feb8af9 100644--- a/sql/mysqld.cc+++ b/sql/mysqld.cc@@ -3124,7 +3124,6 @@ pthread_handler_t signal_hand(void *arg __attribute__((unused)))sigset_t set;int sig;my_thread_init(); // Init new thread- DBUG_ENTER("signal_hand");signal_thread_in_use= 1;/*@@ -3178,7 +3177,6 @@ pthread_handler_t signal_hand(void *arg __attribute__((unused))){DBUG_PRINT("quit",("signal_handler: calling my_thread_end()"));my_thread_end();- DBUG_LEAVE; // Must match DBUG_ENTER()signal_thread_in_use= 0;pthread_exit(0); // SafetyHowever, your crash is something quite different. Can you reproduce it in an environment that I can access remotely?