Details
Description
We have two servers running MariaDB with galera for replication. Every few weeks we get alerts that MariaDB has crashed with a segfault. We were on older 10.X versions MariaDB ( https://serverfault.com/questions/1016977/mariadb-crashing ) and had the same issues. I am not sure if it is a specific query that is causing MariaDB to crash or an issue elsewhere. Below is what I am seeing with a back trace
[root@mon2 ccpp-2020-11-05-08:55:49-43159]# gdb /usr/libexec/mysqld coredump
|
GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-119.el7
|
Copyright (C) 2013 Free Software Foundation, Inc.
|
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
|
This is free software: you are free to change and redistribute it.
|
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
|
and "show warranty" for details.
|
This GDB was configured as "x86_64-redhat-linux-gnu".
|
For bug reporting instructions, please see:
|
<http://www.gnu.org/software/gdb/bugs/>...
|
Reading symbols from /usr/libexec/mysqld...Reading symbols from /usr/libexec/mysqld...(no debugging symbols found)...done.
|
(no debugging symbols found)...done.
|
[New LWP 13153]
|
[New LWP 43164]
|
[New LWP 43163]
|
[New LWP 44190]
|
[New LWP 44193]
|
[New LWP 26180]
|
[New LWP 44196]
|
[New LWP 44051]
|
[New LWP 44186]
|
[New LWP 44194]
|
[New LWP 44192]
|
[New LWP 44189]
|
[New LWP 44188]
|
[New LWP 44208]
|
[New LWP 44195]
|
[New LWP 44191]
|
[New LWP 44200]
|
[New LWP 43159]
|
[New LWP 43166]
|
[New LWP 43162]
|
[New LWP 43161]
|
[New LWP 44187]
|
[New LWP 43165]
|
[New LWP 44209]
|
[New LWP 44210]
|
[New LWP 44211]
|
[New LWP 44197]
|
[New LWP 44216]
|
[New LWP 44215]
|
[New LWP 44198]
|
[New LWP 44214]
|
[New LWP 44217]
|
[New LWP 44218]
|
[New LWP 44212]
|
[New LWP 44224]
|
[New LWP 44261]
|
[New LWP 44213]
|
[New LWP 44207]
|
[New LWP 44223]
|
[New LWP 44222]
|
[New LWP 44220]
|
[Thread debugging using libthread_db enabled]
|
Using host libthread_db library "/lib64/libthread_db.so.1".
|
Core was generated by `/usr/libexec/mysqld --basedir=/usr'.
|
Program terminated with signal 11, Segmentation fault.
|
#0 0x00007f4375bac0b8 in ?? () from /lib64/libgcc_s.so.1
|
Missing separate debuginfos, use: debuginfo-install mariadb103-server-10.3.21-2.el7.ius.x86_64
|
(gdb) bt
|
#0 0x00007f4375bac0b8 in ?? () from /lib64/libgcc_s.so.1
|
#1 0x00007f4375bacfb9 in _Unwind_Backtrace () from /lib64/libgcc_s.so.1
|
#2 0x00007f4376fefaa6 in backtrace () from /lib64/libc.so.6
|
#3 0x000055d479977c3d in my_print_stacktrace ()
|
#4 0x000055d479458637 in handle_fatal_signal ()
|
#5 <signal handler called>
|
#6 0x0000000000000051 in ?? ()
|
#7 0x000055d4794c3fc5 in Item_func_release_lock::val_int() ()
|
#8 0x000055d4791d41fc in Item::update_null_value() ()
|
#9 0x000055d47923d215 in Item_func::is_null() ()
|
#10 0x000055d47959bde9 in mysql_do(THD*, List<Item>&) ()
|
#11 0x000055d47927ea66 in mysql_execute_command(THD*) ()
|
#12 0x000055d4791ead76 in sp_instr_stmt::exec_core(THD*, unsigned int*) ()
|
#13 0x000055d4791f2949 in sp_lex_keeper::reset_lex_and_exec_core(THD*, unsigned int*, bool, sp_instr*) ()
|
#14 0x000055d4791f337c in sp_instr_stmt::execute(THD*, unsigned int*) ()
|
#15 0x000055d4791ee6c0 in sp_head::execute(THD*, bool) ()
|
#16 0x000055d4791ef91d in sp_head::execute_procedure(THD*, List<Item>*) ()
|
#17 0x000055d479270df2 in do_execute_sp(THD*, sp_head*) ()
|
#18 0x000055d4792722e6 in Sql_cmd_call::execute(THD*) [clone .part.293] ()
|
#19 0x000055d479272b60 in Sql_cmd_call::execute(THD*) ()
|
#20 0x000055d47927c2b8 in mysql_execute_command(THD*) ()
|
#21 0x000055d47928120b in mysql_parse(THD*, char*, unsigned int, Parser_state*, bool, bool) ()
|
#22 0x000055d479281b81 in wsrep_mysql_parse(THD*, char*, unsigned int, Parser_state*, bool, bool) ()
|
#23 0x000055d479283306 in dispatch_command(enum_server_command, THD*, char*, unsigned int, bool, bool) ()
|
#24 0x000055d479284cae in do_command(THD*) ()
|
#25 0x000055d4793577d1 in do_handle_one_connection(CONNECT*) ()
|
#26 0x000055d47935789d in handle_one_connection ()
|
#27 0x00007f4378e4dea5 in start_thread () from /lib64/libpthread.so.0
|
#28 0x00007f4376fd98dd in clone () from /lib64/libc.so.6
|
Attached all the traces besides the backtrace (it's 4.3GB) and the hot name file. We do have all queries stored so we can pull them for the time of the crash if needed.
Attachments
Issue Links
- relates to
-
MDEV-17547 MariaDB 10.2 and Galera crashing with segfault
-
- Closed
-
-
-
- Closed
-
-
-
- Closed
-
Activity
| Field | Original Value | New Value |
|---|---|---|
| Attachment | component [ 54664 ] | |
| Attachment | core_backtrace [ 54665 ] | |
| Attachment | count [ 54666 ] | |
| Attachment | dso_list [ 54667 ] | |
| Attachment | environ [ 54668 ] | |
| Attachment | executable [ 54669 ] | |
| Attachment | exploitable [ 54670 ] | |
| Attachment | global_pid [ 54671 ] | |
| Attachment | kernel [ 54672 ] | |
| Attachment | last_occurrence [ 54673 ] | |
| Attachment | limits [ 54674 ] | |
| Attachment | machineid [ 54675 ] | |
| Attachment | maps [ 54676 ] | |
| Attachment | open_fds [ 54677 ] | |
| Attachment | os_info [ 54678 ] | |
| Attachment | os_release [ 54679 ] | |
| Attachment | package [ 54680 ] | |
| Attachment | pid [ 54681 ] | |
| Attachment | pkg_arch [ 54682 ] | |
| Attachment | pkg_epoch [ 54683 ] | |
| Attachment | pkg_fingerprint [ 54684 ] | |
| Attachment | pkg_name [ 54685 ] | |
| Attachment | pkg_release [ 54686 ] | |
| Attachment | pkg_vendor [ 54687 ] | |
| Attachment | pkg_version [ 54688 ] | |
| Attachment | proc_pid_status [ 54689 ] | |
| Attachment | pwd [ 54690 ] | |
| Attachment | reason [ 54691 ] | |
| Attachment | runlevel [ 54692 ] | |
| Attachment | time [ 54693 ] | |
| Attachment | type [ 54694 ] | |
| Attachment | uid [ 54695 ] | |
| Attachment | username [ 54696 ] | |
| Attachment | uuid [ 54697 ] | |
| Attachment | var_log_messages [ 54698 ] | |
| Attachment | abrt_version [ 54699 ] | |
| Attachment | analyzer [ 54700 ] | |
| Attachment | architecture [ 54701 ] | |
| Attachment | cgroup [ 54702 ] | |
| Attachment | cmdline [ 54703 ] |
| Link |
This issue relates to |
And this is the query that was ran on the other galera system (we only have two servers in a MariaDB Galera Cluster). Again IP's switched for privacy
201105 11:13:35 317 Query drop procedure if exists __insert_10S2
|
317 Query create procedure __insert_10S2()
|
BEGIN
|
INSERT IGNORE INTO rtp_stat ( `id_sensor`,`time`,`saddr`,`mosf1_min`,`mosf1_avg`,`mosf2_min`,`mosf2_avg`,`mosAD_min`,`mosAD_avg`,`jitter_max`,`jitter_avg`,`loss_max_mult10`,`loss_avg_mult10`,`counter` ) VALUES ( '2','2020-11-05 11:12:50',inet6_aton('46.116.65.84'),'45','45','45','45','45','45','6','6','0','0','1' ),( '2','2020-11-05 11:12:50',inet6_aton('192.73.251.107'),'45','45','45','45','45','45','0','0','0','0','1' ),( '2','2020-11-05 11:12:50',inet6_aton('192.168.31.32'),'45','45','45','45','45','45','7','3','0','0','2' );
|
INSERT IGNORE INTO rtp_stat ( `id_sensor`,`time`,`saddr`,`mosf1_min`,`mosf1_avg`,`mosf2_min`,`mosf2_avg`,`mosAD_min`,`mosAD_avg`,`jitter_max`,`jitter_avg`,`loss_max_mult10`,`loss_avg_mult10`,`counter` ) VALUES ( '2','2020-11-05 11:13:00',inet6_aton('46.116.65.84'),'45','45','45','45','45','45','7','7','0','0','1' ),( '2','2020-11-05 11:13:00',inet6_aton('192.73.251.107'),'45','45','45','45','45','45','0','0','0','0','1' ),( '2','2020-11-05 11:13:00',inet6_aton('192.168.31.32'),'45','45','45','45','45','45','7','3','0','0','2' );
|
do get_lock('vm_cdr_callid_57351970fdffd984ef4fd91cc55980da', 60);
|
set @exists_call_id = coalesce(
|
(select cdr.ID from cdr
|
join cdr_next on (cdr_next.cdr_ID = cdr.ID and cdr_next.calldate = cdr.calldate)
|
where cdr.calldate > ('2020-11-05 11:12:17' - interval 1 hour) and
|
cdr.calldate < ('2020-11-05 11:12:17' + interval 1 hour) and
|
fbasename = 'KtisOgVrJU3dGa1YukpCI3..' limit 1), 0);
|
set @exists_rtp =
|
if(@exists_call_id,
|
exists (select * from cdr_rtp where cdr_id = @exists_call_id),
|
0);
|
if @exists_call_id and not @exists_rtp and 0 then
|
delete from cdr where id = @exists_call_id;
|
delete from cdr_next where cdr_id = @exists_call_id;
|
delete from cdr_next_1 where cdr_ID = @exists_call_id;
|
delete from cdr_country_code where cdr_id = @exists_call_id;
|
delete from cdr_rtp where cdr_id = @exists_call_id;
|
delete from cdr_dtmf where cdr_id = @exists_call_id;
|
delete from cdr_sipresp where cdr_id = @exists_call_id;
|
delete from cdr_tar_part where cdr_id = @exists_call_id;
|
set @exists_call_id = 0;
|
end if;
|
if not @exists_call_id then
|
INSERT INTO cdr ( `id_sensor`,`caller`,`caller_reverse`,`called`,`called_reverse`,`caller_domain`,`called_domain`,`callername`,`callername_reverse`,`sipcallerip`,`sipcalledip`,`sipcallerport`,`sipcalledport`,`duration`,`vlan`,`calldate`,`callend`,`sighup`,`lastSIPresponseNum`,`response_time_xxx`,`bye`,`dscp`,`flags`,`lastSIPresponse_id`,`a_ua_id` ) VALUES ( '2','1316','6131','000390237920793','397029732093000','192.168.31.15','192.168.31.15','','',inet6_aton('103.145.13.60'),inet6_aton('192.168.31.66'),'17824','5060','0','2','2020-11-05 11:12:17','2020-11-05 11:12:17','0','407','3.000000','0','262144','1280','2','46' );
|
if row_count() > 0 then
|
set @MI_NEW_ID = last_insert_id();
|
INSERT INTO cdr_next ( `fbasename`,`calldate`,`cdr_ID` ) VALUES ( 'KtisOgVrJU3dGa1YukpCI3..','2020-11-05 11:12:17',@MI_NEW_ID );
|
do release_lock('vm_cdr_callid_57351970fdffd984ef4fd91cc55980da');
|
INSERT INTO cdr_country_code ( `sipcallerip_country_code`,`sipcalledip_country_code`,`caller_number_country_code`,`called_number_country_code`,`calldate`,`cdr_ID` ) VALUES ( 'NL','US','','','2020-11-05 11:12:17',@MI_NEW_ID );
|
INSERT INTO cdr_proxy ( `cdr_ID`,`calldate`,`dst` ) VALUES ( @MI_NEW_ID,'2020-11-05 11:12:17',inet6_aton('192.168.31.15') ),( @MI_NEW_ID,'2020-11-05 11:12:17',inet6_aton('192.168.31.29') );
|
INSERT INTO cdr_sdp ( `cdr_ID`,`ip`,`port`,`is_caller`,`calldate` ) VALUES ( @MI_NEW_ID,inet6_aton('189.144.116.105'),'8000','0','2020-11-05 11:12:17' );
|
INSERT INTO cdr_tar_part ( `cdr_ID`,`type`,`pos`,`calldate` ) VALUES ( @MI_NEW_ID,'1','0','2020-11-05 11:12:17' );
|
end if;
|
end if;
|
do release_lock('vm_cdr_callid_57351970fdffd984ef4fd91cc55980da');
|
END
|
317 Query call __insert_10S2()
|
201105 11:13:45 12 Query drop procedure if exists __insert_50S2
|
12 Query create procedure __insert_50S2()
|
BEGIN
|
INSERT IGNORE INTO register_failed ( `created_at`,`sipcallerip`,`sipcalledip`,`from_num`,`to_num`,`contact_num`,`contact_domain`,`to_domain`,`digestusername`,`fname`,`counter`,`ID`,`vlan`,`id_sensor`,`ua_id` ) VALUES ( '2020-11-05 11:13:14',inet6_aton('45.56.137.137'),inet6_aton('192.168.31.10'),'949','949','949','100.64.36.26','192.168.31.10','','0','1','710717000002','2','2','55' );
|
INSERT IGNORE INTO register_failed ( `created_at`,`sipcallerip`,`sipcalledip`,`from_num`,`to_num`,`contact_num`,`contact_domain`,`to_domain`,`digestusername`,`fname`,`counter`,`ID`,`vlan`,`id_sensor`,`ua_id` ) VALUES ( '2020-11-05 11:13:26',inet6_aton('45.56.137.137'),inet6_aton('192.168.31.66'),'1094','1094','1094','100.64.36.10','192.168.31.66','1094','0','1','710717100002','2','2','55' );
|
|
|
END
|
12 Query call __insert_50S2()
|
201105 11:13:52 317 Query drop procedure if exists __insert_10S2
|
| Summary | MariaDB "randomly" creashing | MariaDB "randomly" crashing |
| Description |
We have two servers running MariaDB with galera for replication. Every few weeks we get alerts that MariaDB has crashed with a segfault. We were on older 10.X versions MariaDB ( https://serverfault.com/questions/1016977/mariadb-crashing ) and had the same issues. I am not sure if it is a specific query that is causing MariaDB to crash or an issue elsewhere. Below is what I am seeing with a back trace
[root@mon2 ccpp-2020-11-05-08:55:49-43159]# gdb /usr/libexec/mysqld coredump GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-119.el7 Copyright (C) 2013 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-redhat-linux-gnu". For bug reporting instructions, please see: <http://www.gnu.org/software/gdb/bugs/>... Reading symbols from /usr/libexec/mysqld...Reading symbols from /usr/libexec/mysqld...(no debugging symbols found)...done. (no debugging symbols found)...done. [New LWP 13153] [New LWP 43164] [New LWP 43163] [New LWP 44190] [New LWP 44193] [New LWP 26180] [New LWP 44196] [New LWP 44051] [New LWP 44186] [New LWP 44194] [New LWP 44192] [New LWP 44189] [New LWP 44188] [New LWP 44208] [New LWP 44195] [New LWP 44191] [New LWP 44200] [New LWP 43159] [New LWP 43166] [New LWP 43162] [New LWP 43161] [New LWP 44187] [New LWP 43165] [New LWP 44209] [New LWP 44210] [New LWP 44211] [New LWP 44197] [New LWP 44216] [New LWP 44215] [New LWP 44198] [New LWP 44214] [New LWP 44217] [New LWP 44218] [New LWP 44212] [New LWP 44224] [New LWP 44261] [New LWP 44213] [New LWP 44207] [New LWP 44223] [New LWP 44222] [New LWP 44220] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib64/libthread_db.so.1". Core was generated by `/usr/libexec/mysqld --basedir=/usr'. Program terminated with signal 11, Segmentation fault. #0 0x00007f4375bac0b8 in ?? () from /lib64/libgcc_s.so.1 Missing separate debuginfos, use: debuginfo-install mariadb103-server-10.3.21-2.el7.ius.x86_64 (gdb) bt #0 0x00007f4375bac0b8 in ?? () from /lib64/libgcc_s.so.1 #1 0x00007f4375bacfb9 in _Unwind_Backtrace () from /lib64/libgcc_s.so.1 #2 0x00007f4376fefaa6 in backtrace () from /lib64/libc.so.6 #3 0x000055d479977c3d in my_print_stacktrace () #4 0x000055d479458637 in handle_fatal_signal () #5 <signal handler called> #6 0x0000000000000051 in ?? () #7 0x000055d4794c3fc5 in Item_func_release_lock::val_int() () #8 0x000055d4791d41fc in Item::update_null_value() () #9 0x000055d47923d215 in Item_func::is_null() () #10 0x000055d47959bde9 in mysql_do(THD*, List<Item>&) () #11 0x000055d47927ea66 in mysql_execute_command(THD*) () #12 0x000055d4791ead76 in sp_instr_stmt::exec_core(THD*, unsigned int*) () #13 0x000055d4791f2949 in sp_lex_keeper::reset_lex_and_exec_core(THD*, unsigned int*, bool, sp_instr*) () #14 0x000055d4791f337c in sp_instr_stmt::execute(THD*, unsigned int*) () #15 0x000055d4791ee6c0 in sp_head::execute(THD*, bool) () #16 0x000055d4791ef91d in sp_head::execute_procedure(THD*, List<Item>*) () #17 0x000055d479270df2 in do_execute_sp(THD*, sp_head*) () #18 0x000055d4792722e6 in Sql_cmd_call::execute(THD*) [clone .part.293] () #19 0x000055d479272b60 in Sql_cmd_call::execute(THD*) () #20 0x000055d47927c2b8 in mysql_execute_command(THD*) () #21 0x000055d47928120b in mysql_parse(THD*, char*, unsigned int, Parser_state*, bool, bool) () #22 0x000055d479281b81 in wsrep_mysql_parse(THD*, char*, unsigned int, Parser_state*, bool, bool) () #23 0x000055d479283306 in dispatch_command(enum_server_command, THD*, char*, unsigned int, bool, bool) () #24 0x000055d479284cae in do_command(THD*) () #25 0x000055d4793577d1 in do_handle_one_connection(CONNECT*) () #26 0x000055d47935789d in handle_one_connection () #27 0x00007f4378e4dea5 in start_thread () from /lib64/libpthread.so.0 #28 0x00007f4376fd98dd in clone () from /lib64/libc.so.6 Attached all the traces besides the backtrace (it's 4.3GB) and the hot name file. We do have all queries stored so we can pull them for the time of the crash if needed. |
We have two servers running MariaDB with galera for replication. Every few weeks we get alerts that MariaDB has crashed with a segfault. We were on older 10.X versions MariaDB ( https://serverfault.com/questions/1016977/mariadb-crashing ) and had the same issues. I am not sure if it is a specific query that is causing MariaDB to crash or an issue elsewhere. Below is what I am seeing with a back trace
{noformat} [root@mon2 ccpp-2020-11-05-08:55:49-43159]# gdb /usr/libexec/mysqld coredump GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-119.el7 Copyright (C) 2013 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-redhat-linux-gnu". For bug reporting instructions, please see: <http://www.gnu.org/software/gdb/bugs/>... Reading symbols from /usr/libexec/mysqld...Reading symbols from /usr/libexec/mysqld...(no debugging symbols found)...done. (no debugging symbols found)...done. [New LWP 13153] [New LWP 43164] [New LWP 43163] [New LWP 44190] [New LWP 44193] [New LWP 26180] [New LWP 44196] [New LWP 44051] [New LWP 44186] [New LWP 44194] [New LWP 44192] [New LWP 44189] [New LWP 44188] [New LWP 44208] [New LWP 44195] [New LWP 44191] [New LWP 44200] [New LWP 43159] [New LWP 43166] [New LWP 43162] [New LWP 43161] [New LWP 44187] [New LWP 43165] [New LWP 44209] [New LWP 44210] [New LWP 44211] [New LWP 44197] [New LWP 44216] [New LWP 44215] [New LWP 44198] [New LWP 44214] [New LWP 44217] [New LWP 44218] [New LWP 44212] [New LWP 44224] [New LWP 44261] [New LWP 44213] [New LWP 44207] [New LWP 44223] [New LWP 44222] [New LWP 44220] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib64/libthread_db.so.1". Core was generated by `/usr/libexec/mysqld --basedir=/usr'. Program terminated with signal 11, Segmentation fault. #0 0x00007f4375bac0b8 in ?? () from /lib64/libgcc_s.so.1 Missing separate debuginfos, use: debuginfo-install mariadb103-server-10.3.21-2.el7.ius.x86_64 (gdb) bt #0 0x00007f4375bac0b8 in ?? () from /lib64/libgcc_s.so.1 #1 0x00007f4375bacfb9 in _Unwind_Backtrace () from /lib64/libgcc_s.so.1 #2 0x00007f4376fefaa6 in backtrace () from /lib64/libc.so.6 #3 0x000055d479977c3d in my_print_stacktrace () #4 0x000055d479458637 in handle_fatal_signal () #5 <signal handler called> #6 0x0000000000000051 in ?? () #7 0x000055d4794c3fc5 in Item_func_release_lock::val_int() () #8 0x000055d4791d41fc in Item::update_null_value() () #9 0x000055d47923d215 in Item_func::is_null() () #10 0x000055d47959bde9 in mysql_do(THD*, List<Item>&) () #11 0x000055d47927ea66 in mysql_execute_command(THD*) () #12 0x000055d4791ead76 in sp_instr_stmt::exec_core(THD*, unsigned int*) () #13 0x000055d4791f2949 in sp_lex_keeper::reset_lex_and_exec_core(THD*, unsigned int*, bool, sp_instr*) () #14 0x000055d4791f337c in sp_instr_stmt::execute(THD*, unsigned int*) () #15 0x000055d4791ee6c0 in sp_head::execute(THD*, bool) () #16 0x000055d4791ef91d in sp_head::execute_procedure(THD*, List<Item>*) () #17 0x000055d479270df2 in do_execute_sp(THD*, sp_head*) () #18 0x000055d4792722e6 in Sql_cmd_call::execute(THD*) [clone .part.293] () #19 0x000055d479272b60 in Sql_cmd_call::execute(THD*) () #20 0x000055d47927c2b8 in mysql_execute_command(THD*) () #21 0x000055d47928120b in mysql_parse(THD*, char*, unsigned int, Parser_state*, bool, bool) () #22 0x000055d479281b81 in wsrep_mysql_parse(THD*, char*, unsigned int, Parser_state*, bool, bool) () #23 0x000055d479283306 in dispatch_command(enum_server_command, THD*, char*, unsigned int, bool, bool) () #24 0x000055d479284cae in do_command(THD*) () #25 0x000055d4793577d1 in do_handle_one_connection(CONNECT*) () #26 0x000055d47935789d in handle_one_connection () #27 0x00007f4378e4dea5 in start_thread () from /lib64/libpthread.so.0 #28 0x00007f4376fd98dd in clone () from /lib64/libc.so.6 {noformat} Attached all the traces besides the backtrace (it's 4.3GB) and the hot name file. We do have all queries stored so we can pull them for the time of the crash if needed. |
We had another crash today. If you would like the logs from the latest one please let me know.
I also saw this as well in /var/log/messages
201110 14:48:35 [ERROR] mysqld got signal 11 ;
This could be because you hit a bug. It is also possible that this binary
or one of the libraries it was linked against is corrupt, improperly built,
or misconfigured. This error can also be caused by malfunctioning hardware.
To report this bug, see https://mariadb.com/kb/en/reporting-bugs
We will try our best to scrape up some info that will hopefully help
diagnose the problem, but since we have already crashed,
something is definitely wrong and this may fail.
Server version: 10.3.21-MariaDB-log
key_buffer_size=134217728
read_buffer_size=131072
max_used_connections=4
max_threads=153
thread_count=12
It is possible that mysqld could use up to
key_buffer_size + (read_buffer_size + sort_buffer_size)*max_threads = 467424 K bytes of memory
Hope that's ok; if not, decrease some variables in the equation.
Thread pointer: 0x7fb7a40009a8
Attempting backtrace. You can use the following information to find out
where mysqld died. If you see no messages after this, something went
terribly wrong...
stack_bottom = 0x7fc924397d30 thread_stack 0x49000
/usr/libexec/mysqld(my_print_stacktrace+0x3d)[0x563b1cdd9c3d]
/usr/libexec/mysqld(handle_fatal_signal+0x327)[0x563b1c8ba637]
/lib64/libpthread.so.0(+0xf630)[0x7fc94379c630]
/usr/libexec/mysqld(+0x695513)[0x563b1c7be513]
/usr/libexec/mysqld(_ZN8MDL_lock13remove_ticketEP7LF_PINSMS_NS_11Ticket_listEP10MDL_ticket+0x38)[0x563b1c7c0198]
/usr/libexec/mysqld(_ZN11MDL_context12release_lockE17enum_mdl_durationP10MDL_ticket+0x24)[0x563b1c7c12c4]
/usr/libexec/mysqld(_Z17mysql_ull_cleanupP3THD+0x49)[0x563b1c92f089]
/usr/libexec/mysqld(_ZN3THD7cleanupEv+0x1c4)[0x563b1c69b9e4]
/usr/libexec/mysqld(_Z10unlink_thdP3THD+0xf)[0x563b1c622cff]
/usr/libexec/mysqld(_Z29one_thread_per_connection_endP3THDb+0x38)[0x563b1c622e28]
/usr/libexec/mysqld(_Z24do_handle_one_connectionP7CONNECT+0x15e)[0x563b1c7b96ee]
/usr/libexec/mysqld(handle_one_connection+0x3d)[0x563b1c7b989d]
/lib64/libpthread.so.0(+0x7ea5)[0x7fc943794ea5]
/lib64/libc.so.6(clone+0x6d)[0x7fc9419208dd]
Trying to get some variables.
Some pointers may be invalid and cause the dump to abort.
Query (0x0):
Connection ID (thread ID): 115
Status: KILL_CONNECTION
Optimizer switch: index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,index_merge_sort_intersection=off,engine_condition_pushdown=off,index_condition_pushdown=on,derived_merge=on,derived_with_keys=on,firstmatch=on,loosescan=on,materialization=on,in_to_exists=on,semijoin=on,partial_match_rowid_merge=on,partial_match_table_scan=on,subquery_cache=on,mrr=off,mrr_cost_based=off,mrr_sort_keys=off,outer_join_with_cache=on,semijoin_with_cache=on,join_cache_incremental=on,join_cache_hashed=on,join_cache_bka=on,optimize_join_buffer_size=off,table_elimination=on,extended_keys=on,exists_to_in=on,orderby_uses_equalities=on,condition_pushdown_for_derived=on,split_materialized=on
The manual page at http://dev.mysql.com/doc/mysql/en/crashing.html contains
information that should help you find out what is causing the crash.
Writing a core file...
Working directory at /var/lib/mysql
Resource Limits:
Limit Soft Limit Hard Limit Units
Max cpu time unlimited unlimited seconds
Max file size unlimited unlimited bytes
Max data size unlimited unlimited bytes
Max stack size 8388608 unlimited bytes
Max core file size 0 unlimited bytes
Max resident set unlimited unlimited bytes
Max processes 514564 514564 processes
Max open files 65535 65535 files
Max locked memory 65536 65536 bytes
Max address space unlimited unlimited bytes
Max file locks unlimited unlimited locks
Max pending signals 514564 514564 signals
Max msgqueue size 819200 819200 bytes
Max nice priority 0 0
Max realtime priority 0 0
Max realtime timeout unlimited unlimited us
Core pattern: |/usr/libexec/abrt-hook-ccpp %s %c %p %u %g %t e %P %I %h
Thanks for the report.
To reproduce:
git clone https://github.com/MariaDB/randgen --branch mdev24143 rqg-mdev24143
|
cd rqg-mdev24143
|
. ./mdev24143.cmd <basedir> <path to Galera 3 library>
|
|
10.3 e59c1cef RelWithDebInfo |
#3 <signal handler called>
|
#4 0x00007fd8d80008d0 in ?? ()
|
#5 0x0000557d54234c7b in Item_func_release_lock::val_int (this=0x7fd8d800f818) at /data/src/10.3/sql/item_func.cc:4179
|
#6 0x0000557d53f28c3d in Item::update_null_value (this=0x7fd8d800f818) at /data/src/10.3/sql/item.h:1628
|
#7 0x0000557d53f94439 in Item_func::is_null (this=0x7fd8d800f818) at /data/src/10.3/sql/item_func.h:184
|
#8 0x0000557d5431a5f9 in mysql_do (thd=thd@entry=0x7fd8d8000c48, values=...) at /data/src/10.3/sql/sql_do.cc:35
|
#9 0x0000557d53fd5ed0 in mysql_execute_command (thd=0x7fd8d8000c48) at /data/src/10.3/sql/sql_parse.cc:3876
|
#10 0x0000557d53fdb913 in mysql_parse (thd=thd@entry=0x7fd8d8000c48, rawbuf=rawbuf@entry=0x7fd8d800f670 "DO RELEASE_LOCK('a') /* QNO 37578 CON_ID 14 */", length=length@entry=46, parser_state=parser_state@entry=0x7fd94561b5c0, is_com_multi=is_com_multi@entry=false, is_next_command=is_next_command@entry=false) at /data/src/10.3/sql/sql_parse.cc:7840
|
#11 0x0000557d53fdc3ba in wsrep_mysql_parse (thd=0x7fd8d8000c48, rawbuf=0x7fd8d800f670 "DO RELEASE_LOCK('a') /* QNO 37578 CON_ID 14 */", length=46, parser_state=0x7fd94561b5c0, is_com_multi=false, is_next_command=false) at /data/src/10.3/sql/sql_parse.cc:7632
|
#12 0x0000557d53fde071 in dispatch_command (command=COM_QUERY, thd=0x7fd8d8000c48, packet=<optimized out>, packet_length=<optimized out>, is_com_multi=<optimized out>, is_next_command=<optimized out>) at /data/src/10.3/sql/sql_parse.cc:1938
|
#13 0x0000557d53fdfcbd in do_command (thd=0x7fd8d8000c48) at /data/src/10.3/sql/sql_parse.cc:1398
|
#14 0x0000557d540c74f6 in do_handle_one_connection (connect=connect@entry=0x557d576e7b88) at /data/src/10.3/sql/sql_connect.cc:1403
|
#15 0x0000557d540c76cf in handle_one_connection (arg=0x557d576e7b88) at /data/src/10.3/sql/sql_connect.cc:1308
|
#16 0x00007fd96eae0609 in start_thread (arg=<optimized out>) at pthread_create.c:477
|
#17 0x00007fd96e6d5293 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
|
|
10.4 e59c1cef debug build |
#3 <signal handler called>
|
#4 0x000055e0f9e946da in MDL_key::length (this=0x0) at /data/src/10.3/sql/mdl.h:311
|
#5 0x000055e0fa2dc7c9 in ull_get_key (ptr=0x7f15b01e3310 "\260F\004\260\025\177", length=0x7f16408ac4d0, not_used=1 '\001') at /data/src/10.3/sql/item_func.cc:3925
|
#6 0x000055e0faaa40a5 in my_hash_key (hash=0x7f15b0003108, record=0x7f15b01e3310 "\260F\004\260\025\177", length=0x7f16408ac4d0, first=1 '\001') at /data/src/10.3/mysys/hash.c:196
|
#7 0x000055e0faaa4560 in hashcmp (hash=0x7f15b0003108, pos=0x7f15b044de70, key=0x7f16408ac648 "\ta", length=4) at /data/src/10.3/mysys/hash.c:371
|
#8 0x000055e0faaa436c in my_hash_first_from_hash_value (hash=0x7f15b0003108, hash_value=1048460927, key=0x7f16408ac648 "\ta", length=4, current_record=0x7f16408ac5dc) at /data/src/10.3/mysys/hash.c:288
|
#9 0x000055e0faaa42bd in my_hash_first (hash=0x7f15b0003108, key=0x7f16408ac648 "\ta", length=4, current_record=0x7f16408ac5dc) at /data/src/10.3/mysys/hash.c:262
|
#10 0x000055e0faaa41b7 in my_hash_search (hash=0x7f15b0003108, key=0x7f16408ac648 "\ta", length=4) at /data/src/10.3/mysys/hash.c:235
|
#11 0x000055e0fa2dd3a9 in Item_func_release_lock::val_int (this=0x7f15b0011790) at /data/src/10.3/sql/item_func.cc:4168
|
#12 0x000055e0f9e071fa in Item::update_null_value (this=0x7f15b0011790) at /data/src/10.3/sql/item.h:1628
|
#13 0x000055e0f9ec0d52 in Item_func::is_null (this=0x7f15b0011790) at /data/src/10.3/sql/item_func.h:184
|
#14 0x000055e0fa40a8c6 in mysql_do (thd=0x7f15b0000d90, values=...) at /data/src/10.3/sql/sql_do.cc:35
|
#15 0x000055e0f9f1ce3f in mysql_execute_command (thd=0x7f15b0000d90) at /data/src/10.3/sql/sql_parse.cc:3876
|
#16 0x000055e0f9f2a8d0 in mysql_parse (thd=0x7f15b0000d90, rawbuf=0x7f15b00115e8 "DO RELEASE_LOCK('a') /* QNO 252 CON_ID 14 */", length=44, parser_state=0x7f16408ad610, is_com_multi=false, is_next_command=false) at /data/src/10.3/sql/sql_parse.cc:7840
|
#17 0x000055e0f9f29f81 in wsrep_mysql_parse (thd=0x7f15b0000d90, rawbuf=0x7f15b00115e8 "DO RELEASE_LOCK('a') /* QNO 252 CON_ID 14 */", length=44, parser_state=0x7f16408ad610, is_com_multi=false, is_next_command=false) at /data/src/10.3/sql/sql_parse.cc:7632
|
#18 0x000055e0f9f16fde in dispatch_command (command=COM_QUERY, thd=0x7f15b0000d90, packet=0x7f15b0008e11 "DO RELEASE_LOCK('a') /* QNO 252 CON_ID 14 */ ", packet_length=45, is_com_multi=false, is_next_command=false) at /data/src/10.3/sql/sql_parse.cc:1849
|
#19 0x000055e0f9f159d4 in do_command (thd=0x7f15b0000d90) at /data/src/10.3/sql/sql_parse.cc:1398
|
#20 0x000055e0fa093ed1 in do_handle_one_connection (connect=0x55e0fe83cb50) at /data/src/10.3/sql/sql_connect.cc:1403
|
#21 0x000055e0fa093c2d in handle_one_connection (arg=0x55e0fe83cb50) at /data/src/10.3/sql/sql_connect.cc:1308
|
#22 0x00007f16522cd609 in start_thread (arg=<optimized out>) at pthread_create.c:477
|
#23 0x00007f1651ea9293 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
|
|
10.3 e59c1cef ASAN-debug |
==1927354==ERROR: AddressSanitizer: heap-use-after-free on address 0x606000a80e98 at pc 0x564f3047d3ef bp 0x7f9b3577cf70 sp 0x7f9b3577cf60
|
READ of size 8 at 0x606000a80e98 thread T39
|
#0 0x564f3047d3ee in MDL_ticket::get_key() const /data/src/10.3/sql/mdl.cc:2812
|
#1 0x564f3098229f in ull_get_key(unsigned char const*, unsigned long*, char) /data/src/10.3/sql/item_func.cc:3924
|
#2 0x564f31b287d9 in my_hash_key /data/src/10.3/mysys/hash.c:196
|
#3 0x564f31b296e1 in hashcmp /data/src/10.3/mysys/hash.c:371
|
#4 0x564f31b29070 in my_hash_first_from_hash_value /data/src/10.3/mysys/hash.c:288
|
#5 0x564f31b28e8b in my_hash_first /data/src/10.3/mysys/hash.c:262
|
#6 0x564f31b28b7e in my_hash_search /data/src/10.3/mysys/hash.c:235
|
#7 0x564f3098402a in Item_func_release_lock::val_int() /data/src/10.3/sql/item_func.cc:4168
|
#8 0x564f2fe178bd in Item::update_null_value() /data/src/10.3/sql/item.h:1628
|
#9 0x564f2ffc49de in Item_func::is_null() /data/src/10.3/sql/item_func.h:184
|
#10 0x564f30c71b6a in mysql_do(THD*, List<Item>&) /data/src/10.3/sql/sql_do.cc:35
|
#11 0x564f300a330c in mysql_execute_command(THD*) /data/src/10.3/sql/sql_parse.cc:3876
|
#12 0x564f300bf41d in mysql_parse(THD*, char*, unsigned int, Parser_state*, bool, bool) /data/src/10.3/sql/sql_parse.cc:7840
|
#13 0x564f300bdef0 in wsrep_mysql_parse /data/src/10.3/sql/sql_parse.cc:7632
|
#14 0x564f30096200 in dispatch_command(enum_server_command, THD*, char*, unsigned int, bool, bool) /data/src/10.3/sql/sql_parse.cc:1849
|
#15 0x564f30092d98 in do_command(THD*) /data/src/10.3/sql/sql_parse.cc:1398
|
#16 0x564f3045d61b in do_handle_one_connection(CONNECT*) /data/src/10.3/sql/sql_connect.cc:1403
|
#17 0x564f3045ced5 in handle_one_connection /data/src/10.3/sql/sql_connect.cc:1308
|
#18 0x7f9b6f6f7608 in start_thread /build/glibc-ZN95T4/glibc-2.31/nptl/pthread_create.c:477
|
#19 0x7f9b6f2d1292 in __clone (/lib/x86_64-linux-gnu/libc.so.6+0x122292)
|
|
|
0x606000a80e98 is located 56 bytes inside of 64-byte region [0x606000a80e60,0x606000a80ea0)
|
freed by thread T39 here:
|
#0 0x7f9b6fc62025 in operator delete(void*, unsigned long) (/lib/x86_64-linux-gnu/libasan.so.5+0x111025)
|
#1 0x564f304812b8 in MDL_ticket::~MDL_ticket() /data/src/10.3/sql/mdl.h:593
|
#2 0x564f30475c7a in MDL_ticket::destroy(MDL_ticket*) /data/src/10.3/sql/mdl.cc:965
|
#3 0x564f3047c56f in MDL_context::release_lock(enum_mdl_duration, MDL_ticket*) /data/src/10.3/sql/mdl.cc:2633
|
#4 0x564f3047c977 in MDL_context::release_locks_stored_before(enum_mdl_duration, MDL_ticket*) /data/src/10.3/sql/mdl.cc:2680
|
#5 0x564f3047e493 in MDL_context::release_explicit_locks() /data/src/10.3/sql/mdl.cc:3003
|
#6 0x564f30633c6f in wsrep_client_rollback(THD*) /data/src/10.3/sql/wsrep_thd.cc:81
|
#7 0x564f300be018 in wsrep_mysql_parse /data/src/10.3/sql/sql_parse.cc:7639
|
#8 0x564f30096200 in dispatch_command(enum_server_command, THD*, char*, unsigned int, bool, bool) /data/src/10.3/sql/sql_parse.cc:1849
|
#9 0x564f30092d98 in do_command(THD*) /data/src/10.3/sql/sql_parse.cc:1398
|
#10 0x564f3045d61b in do_handle_one_connection(CONNECT*) /data/src/10.3/sql/sql_connect.cc:1403
|
#11 0x564f3045ced5 in handle_one_connection /data/src/10.3/sql/sql_connect.cc:1308
|
#12 0x7f9b6f6f7608 in start_thread /build/glibc-ZN95T4/glibc-2.31/nptl/pthread_create.c:477
|
|
|
previously allocated by thread T39 here:
|
#0 0x7f9b6fc60d3f in operator new(unsigned long, std::nothrow_t const&) (/lib/x86_64-linux-gnu/libasan.so.5+0x10fd3f)
|
#1 0x564f30475bea in MDL_ticket::create(MDL_context*, enum_mdl_type, enum_mdl_duration) /data/src/10.3/sql/mdl.cc:959
|
#2 0x564f30478ef4 in MDL_context::try_acquire_lock_impl(MDL_request*, MDL_ticket**) /data/src/10.3/sql/mdl.cc:1909
|
#3 0x564f30479aaa in MDL_context::acquire_lock(MDL_request*, double) /data/src/10.3/sql/mdl.cc:2069
|
#4 0x564f309835fe in Item_func_get_lock::val_int() /data/src/10.3/sql/item_func.cc:4108
|
#5 0x564f2fe178bd in Item::update_null_value() /data/src/10.3/sql/item.h:1628
|
#6 0x564f2ffc49de in Item_func::is_null() /data/src/10.3/sql/item_func.h:184
|
#7 0x564f30c71b6a in mysql_do(THD*, List<Item>&) /data/src/10.3/sql/sql_do.cc:35
|
#8 0x564f300a330c in mysql_execute_command(THD*) /data/src/10.3/sql/sql_parse.cc:3876
|
#9 0x564f300bf41d in mysql_parse(THD*, char*, unsigned int, Parser_state*, bool, bool) /data/src/10.3/sql/sql_parse.cc:7840
|
#10 0x564f300bdef0 in wsrep_mysql_parse /data/src/10.3/sql/sql_parse.cc:7632
|
#11 0x564f30096200 in dispatch_command(enum_server_command, THD*, char*, unsigned int, bool, bool) /data/src/10.3/sql/sql_parse.cc:1849
|
#12 0x564f30092d98 in do_command(THD*) /data/src/10.3/sql/sql_parse.cc:1398
|
#13 0x564f3045d61b in do_handle_one_connection(CONNECT*) /data/src/10.3/sql/sql_connect.cc:1403
|
#14 0x564f3045ced5 in handle_one_connection /data/src/10.3/sql/sql_connect.cc:1308
|
#15 0x7f9b6f6f7608 in start_thread /build/glibc-ZN95T4/glibc-2.31/nptl/pthread_create.c:477
|
|
|
Thread T39 created by T0 here:
|
#0 0x7f9b6fb8b805 in pthread_create (/lib/x86_64-linux-gnu/libasan.so.5+0x3a805)
|
#1 0x564f31bf18bd in spawn_thread_noop /data/src/10.3/mysys/psi_noop.c:187
|
#2 0x564f2fdbe15e in inline_mysql_thread_create /data/src/10.3/include/mysql/psi/mysql_thread.h:1275
|
#3 0x564f2fdd6b28 in create_thread_to_handle_connection(CONNECT*) /data/src/10.3/sql/mysqld.cc:6658
|
#4 0x564f2fdd72c3 in create_new_thread /data/src/10.3/sql/mysqld.cc:6728
|
#5 0x564f2fdd8455 in handle_connections_sockets() /data/src/10.3/sql/mysqld.cc:6986
|
#6 0x564f2fdd5e19 in mysqld_main(int, char**) /data/src/10.3/sql/mysqld.cc:6280
|
#7 0x564f2fdbc95c in main /data/src/10.3/sql/main.cc:25
|
#8 0x7f9b6f1d60b2 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x270b2)
|
|
|
SUMMARY: AddressSanitizer: heap-use-after-free /data/src/10.3/sql/mdl.cc:2812 in MDL_ticket::get_key() const
|
Shadow bytes around the buggy address:
|
0x0c0c80148180: fa fa fa fa fd fd fd fd fd fd fd fa fa fa fa fa
|
0x0c0c80148190: fd fd fd fd fd fd fd fa fa fa fa fa fd fd fd fd
|
0x0c0c801481a0: fd fd fd fa fa fa fa fa fd fd fd fd fd fd fd fa
|
0x0c0c801481b0: fa fa fa fa fd fd fd fd fd fd fd fa fa fa fa fa
|
0x0c0c801481c0: fd fd fd fd fd fd fd fa fa fa fa fa fd fd fd fd
|
=>0x0c0c801481d0: fd fd fd[fd]fa fa fa fa fd fd fd fd fd fd fd fd
|
0x0c0c801481e0: fa fa fa fa fd fd fd fd fd fd fd fd fa fa fa fa
|
0x0c0c801481f0: fd fd fd fd fd fd fd fa fa fa fa fa fd fd fd fd
|
0x0c0c80148200: fd fd fd fa fa fa fa fa fd fd fd fd fd fd fd fa
|
0x0c0c80148210: fa fa fa fa fd fd fd fd fd fd fd fa fa fa fa fa
|
0x0c0c80148220: fd fd fd fd fd fd fd fa fa fa fa fa fd fd fd fd
|
Shadow byte legend (one shadow byte represents 8 application bytes):
|
Addressable: 00
|
Partially addressable: 01 02 03 04 05 06 07
|
Heap left redzone: fa
|
Freed heap region: fd
|
Stack left redzone: f1
|
Stack mid redzone: f2
|
Stack right redzone: f3
|
Stack after return: f5
|
Stack use after scope: f8
|
Global redzone: f9
|
Global init order: f6
|
Poisoned by user: f7
|
Container overflow: fc
|
Array cookie: ac
|
Intra object redzone: bb
|
ASan internal: fe
|
Left alloca redzone: ca
|
Right alloca redzone: cb
|
Shadow gap: cc
|
==1927354==ABORTING
|
| Fix Version/s | 10.2 [ 14601 ] | |
| Fix Version/s | 10.3 [ 22126 ] | |
| Affects Version/s | 10.2 [ 14601 ] | |
| Assignee | Jan Lindström [ jplindst ] |
| Status | Open [ 1 ] | Confirmed [ 10101 ] |
| Summary | MariaDB "randomly" crashing | Galera nodes "randomly" crashing in Item_func_release_lock::val_int |
Original issue does not really look like Galera related so I will reassign this.
| Assignee | Jan Lindström [ jplindst ] | Oleksandr Byelkin [ sanja ] |
| Assignee | Oleksandr Byelkin [ sanja ] | Vladislav Vaintroub [ wlad ] |
| Assignee | Vladislav Vaintroub [ wlad ] | Jan Lindström [ jplindst ] |
It is very much Galera related. If you look at ASAN errors, they are caused by accessing freed memory, freed by *wsrep_client_rollback *(previous entry, 10.4 e59c1cef debug build)
=1927354==ERROR: AddressSanitizer: heap-use-after-free on address 0x606000a80e98 at pc 0x564f3047d3ef bp 0x7f9b3577cf70 sp 0x7f9b3577cf60
|
READ of size 8 at 0x606000a80e98 thread T39
|
#0 0x564f3047d3ee in MDL_ticket::get_key() const /data/src/10.3/sql/mdl.cc:2812
|
|
0x606000a80e98 is located 56 bytes inside of 64-byte region [0x606000a80e60,0x606000a80ea0)
|
freed by thread T39 here:
|
#0 0x7f9b6fc62025 in operator delete(void*, unsigned long) (/lib/x86_64-linux-gnu/libasan.so.5+0x111025)
|
#1 0x564f304812b8 in MDL_ticket::~MDL_ticket() /data/src/10.3/sql/mdl.h:593
|
#2 0x564f30475c7a in MDL_ticket::destroy(MDL_ticket*) /data/src/10.3/sql/mdl.cc:965
|
#3 0x564f3047c56f in MDL_context::release_lock(enum_mdl_duration, MDL_ticket*) /data/src/10.3/sql/mdl.cc:2633
|
#4 0x564f3047c977 in MDL_context::release_locks_stored_before(enum_mdl_duration, MDL_ticket*) /data/src/10.3/sql/mdl.cc:2680
|
#5 0x564f3047e493 in MDL_context::release_explicit_locks() /data/src/10.3/sql/mdl.cc:3003
|
#6 0x564f30633c6f in wsrep_client_rollback(THD*) /data/src/10.3/sql/wsrep_thd.cc:81
|
#7 0x564f300be018 in wsrep_mysql_parse /data/src/10.3/sql/sql_parse.cc:7639
|
|
Since Galera inserts itself into everything, including MDL, if should also be fixing it
| Assignee | Jan Lindström [ jplindst ] | Seppo Jaakola [ seppo ] |
| Status | Confirmed [ 10101 ] | In Progress [ 3 ] |
The stored procedure has get_lock() / release_lock() function calls, and these are not supported in galera replication (also noted in KB limitations). However, although not safe, they probably should nevertheless work in topologies where all writes go to same dedicated node, i.e. cluster write conflict would not happen. Dovid does your application/load balancer direct all writes to same node?
A crash is not a good reaction to use of non supported feature, so some work remains for fixing this bug. Preferably by rejecting the use of get_lock() & release_lock() functions. But, this does not help this application's use case. Dovid is it possible to change the stored procedure definition to not use these functions?
It could be possible to support locking functions as new feature, e.g. streaming replication might be helpful technology for it.
| Labels | need_feedback |
| Link |
This issue relates to |
| Workflow | MariaDB v3 [ 115476 ] | MariaDB v4 [ 144575 ] |
| Status | In Progress [ 3 ] | Needs Feedback [ 10501 ] |
| Labels | need_feedback |
| Status | Needs Feedback [ 10501 ] | Open [ 1 ] |
Even though there is no answer to your question from the reporter, I assume you still want to handle the crash, so I'm not closing it as incomplete.
| Link |
This issue relates to |
Found similar crash with slightly different stack.
Test case
CREATE TABLE t1 (c1 BIGINT NOT NULL PRIMARY KEY, c2 BINARY (10), c3 DATETIME); |
SELECT get_lock ('test2', 0); |
DROP TABLE t1; |
CREATE TABLE t1 (c1 SMALLINT NOT NULL AUTO_INCREMENT PRIMARY KEY); |
INSERT INTO t1 VALUES (1); |
SET SESSION wsrep_trx_fragment_size=10; |
SET SESSION autocommit=0; |
SELECT * FROM t1 WHERE c1 <=0 ORDER BY c1 DESC; |
INSERT INTO t1 VALUES (4),(3),(1),(2); |
CREATE TABLE t1 (pk INT PRIMARY KEY, b INT) ENGINE=SEQUENCE; |
ALTER TABLE t1 DROP COLUMN c2; |
SELECT get_lock ('test', 1.5); |
|
10.5.14 fb40a2fabf8d8cf765c83a0b8e609dd893c75ec3 (Optimized) |
Core was generated by `/test/GAL_MD030222-mariadb-10.5.14-linux-x86_64-opt/bin/mysqld --defaults-file='.
|
Program terminated with signal SIGSEGV, Segmentation fault.
|
#0 __pthread_kill (threadid=<optimized out>, signo=signo@entry=11)
|
at ../sysdeps/unix/sysv/linux/pthread_kill.c:56
|
[Current thread is 1 (Thread 0x145bd0e49700 (LWP 4029129))]
|
(gdb) bt
|
#0 __pthread_kill (threadid=<optimized out>, signo=signo@entry=11) at ../sysdeps/unix/sysv/linux/pthread_kill.c:56
|
#1 0x000055f1f65ed96f in my_write_core (sig=sig@entry=11) at /test/10.5_opt/mysys/stacktrace.c:424
|
#2 0x000055f1f600ad10 in handle_fatal_signal (sig=11) at /test/10.5_opt/sql/signal_handler.cc:344
|
#3 <signal handler called>
|
#4 ull_get_key (ptr=<optimized out>, length=0x145bd0e470d8, not_used=<optimized out>) at /test/10.5_opt/sql/mdl.h:398
|
#5 0x000055f1f65ccf25 in my_hash_key (first=1 '\001', length=0x145bd0e470d8, record=<optimized out>, hash=0x145b54003068) at /test/10.5_opt/mysys/hash.c:196
|
#6 hashcmp (pos=0x145b54021cf8, length=7, key=0x145bd0e47198 "\btest", hash=0x145b54003068) at /test/10.5_opt/mysys/hash.c:371
|
#7 my_hash_first_from_hash_value (hash=hash@entry=0x145b54003068, hash_value=<optimized out>, key=key@entry=0x145bd0e47198 "\btest", length=7, current_record=current_record@entry=0x145bd0e4712c) at /test/10.5_opt/mysys/hash.c:288
|
#8 0x000055f1f65cd01d in my_hash_first (hash=hash@entry=0x145b54003068, key=key@entry=0x145bd0e47198 "\btest", length=<optimized out>, current_record=current_record@entry=0x145bd0e4712c) at /test/10.5_opt/mysys/hash.c:262
|
#9 0x000055f1f65cd035 in my_hash_search (hash=hash@entry=0x145b54003068, key=key@entry=0x145bd0e47198 "\btest", length=<optimized out>) at /test/10.5_opt/mysys/hash.c:235
|
#10 0x000055f1f607622a in Item_func_get_lock::val_int (this=0x145b54010b10) at /test/10.5_opt/sql/mdl.h:398
|
#11 0x000055f1f5f7122d in Type_handler::Item_send_long (this=<optimized out>, item=0x145b54010b10, protocol=0x145b540011a8, buf=<optimized out>) at /test/10.5_opt/sql/sql_type.cc:7487
|
#12 0x000055f1f5d2e810 in Protocol::send_result_set_row (this=this@entry=0x145b540011a8, row_items=row_items@entry=0x145b540105e8) at /test/10.5_opt/sql/protocol.cc:1083
|
#13 0x000055f1f5da2367 in select_send::send_data (this=0x145b54011518, items=@0x145b540105e8: {<base_list> = {<Sql_alloc> = {<No data fields>}, first = 0x145b54010bf8, last = 0x145b54010bf8, elements = 1}, <No data fields>}) at /test/10.5_opt/sql/sql_class.cc:3081
|
#14 0x000055f1f5e62518 in select_result_sink::send_data_with_check (u=<optimized out>, sent=0, items=<optimized out>, this=<optimized out>) at /test/10.5_opt/sql/sql_class.h:5342
|
#15 select_result_sink::send_data_with_check (sent=0, u=<optimized out>, items=<optimized out>, this=<optimized out>) at /test/10.5_opt/sql/sql_class.h:5332
|
#16 JOIN::exec_inner (this=0x145b54011540) at /test/10.5_opt/sql/sql_select.cc:4384
|
#17 0x000055f1f5e62919 in JOIN::exec (this=this@entry=0x145b54011540) at /test/10.5_opt/sql/sql_select.cc:4296
|
#18 0x000055f1f5e608da in mysql_select (thd=0x145b54000c58, tables=0x0, fields=@0x145b540105e8: {<base_list> = {<Sql_alloc> = {<No data fields>}, first = 0x145b54010bf8, last = 0x145b54010bf8, elements = 1}, <No data fields>}, conds=0x0, og_num=0, order=0x0, group=0x0, having=0x0, proc_param=0x0, select_options=<optimized out>, result=0x145b54011518, unit=0x145b54004c40, select_lex=0x145b54010498) at /test/10.5_opt/sql/sql_select.cc:4773
|
#19 0x000055f1f5e612c7 in handle_select (thd=thd@entry=0x145b54000c58, lex=lex@entry=0x145b54004b78, result=result@entry=0x145b54011518, setup_tables_done_option=setup_tables_done_option@entry=0) at /test/10.5_opt/sql/sql_select.cc:444
|
#20 0x000055f1f5deda31 in execute_sqlcom_select (thd=0x145b54000c58, all_tables=0x0) at /test/10.5_opt/sql/sql_parse.cc:6314
|
#21 0x000055f1f5dfc79b in mysql_execute_command (thd=0x145b54000c58) at /test/10.5_opt/sql/sql_parse.cc:4005
|
#22 0x000055f1f5de7fbf in mysql_parse (thd=thd@entry=0x145b54000c58, rawbuf=rawbuf@entry=0x145b54010400 "SELECT get_lock ('test', 1.5)", length=length@entry=29, parser_state=parser_state@entry=0x145bd0e48410, is_com_multi=is_com_multi@entry=false, is_next_command=is_next_command@entry=false) at /test/10.5_opt/sql/sql_parse.cc:8100
|
#23 0x000055f1f5de7729 in wsrep_mysql_parse (thd=0x145b54000c58, rawbuf=0x145b54010400 "SELECT get_lock ('test', 1.5)", length=29, parser_state=0x145bd0e48410, is_com_multi=<optimized out>, is_next_command=<optimized out>) at /test/10.5_opt/sql/sql_parse.cc:7903
|
#24 0x000055f1f5df684a in dispatch_command (command=COM_QUERY, thd=0x145b54000c58, packet=<optimized out>, packet_length=<optimized out>, is_com_multi=<optimized out>, is_next_command=<optimized out>) at /test/10.5_opt/sql/sql_class.h:1290
|
#25 0x000055f1f5df772c in do_command (thd=0x145b54000c58) at /test/10.5_opt/sql/sql_parse.cc:1370
|
#26 0x000055f1f5eff631 in do_handle_one_connection (connect=<optimized out>, connect@entry=0x55f1f824b448, put_in_cache=put_in_cache@entry=true) at /test/10.5_opt/sql/sql_connect.cc:1418
|
#27 0x000055f1f5effaad in handle_one_connection (arg=arg@entry=0x55f1f824b448) at /test/10.5_opt/sql/sql_connect.cc:1312
|
#28 0x000055f1f6291cef in pfs_spawn_thread (arg=0x55f1f8262d98) at /test/10.5_opt/storage/perfschema/pfs.cc:2201
|
#29 0x0000145be14dc609 in start_thread (arg=<optimized out>) at pthread_create.c:477
|
#30 0x0000145be10ca293 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
|
Bug confirmed present in:
MariaDB: 10.5.14 (dbg), 10.5.14 (opt), 10.6.6 (dbg), 10.6.6 (opt), 10.7.2 (dbg), 10.7.2 (opt), 10.8.1 (dbg)
Bug (or feature/syntax) confirmed not present in:
MariaDB: 10.2.42 (dbg), 10.2.42 (opt), 10.3.33 (dbg), 10.3.33 (opt), 10.4.23 (dbg), 10.4.23 (opt)
| Affects Version/s | 10.5 [ 23123 ] | |
| Affects Version/s | 10.6 [ 24028 ] | |
| Affects Version/s | 10.7 [ 24805 ] | |
| Affects Version/s | 10.8 [ 26121 ] |
| Fix Version/s | 10.5 [ 23123 ] | |
| Fix Version/s | 10.6 [ 24028 ] | |
| Fix Version/s | 10.7 [ 24805 ] | |
| Fix Version/s | 10.8 [ 26121 ] |
It looks that root cause of this segfault is related to https://jira.mariadb.org/browse/MDEV-27713 , current fix is to handle properly clean thread ull structures after BF abort is triggered.
Please retest with fix found on related ticket.
| Assignee | Seppo Jaakola [ seppo ] | Jan Lindström [ jplindst ] |
| issue.field.resolutiondate | 2022-03-18 07:59:41.0 | 2022-03-18 07:59:41.829 |
| Fix Version/s | 10.4.25 [ 27510 ] | |
| Fix Version/s | 10.5.16 [ 27508 ] | |
| Fix Version/s | 10.6.8 [ 27506 ] | |
| Fix Version/s | 10.7.4 [ 27504 ] | |
| Fix Version/s | 10.2 [ 14601 ] | |
| Fix Version/s | 10.3 [ 22126 ] | |
| Fix Version/s | 10.5 [ 23123 ] | |
| Fix Version/s | 10.6 [ 24028 ] | |
| Fix Version/s | 10.7 [ 24805 ] | |
| Fix Version/s | 10.8 [ 26121 ] | |
| Resolution | Fixed [ 1 ] | |
| Status | Open [ 1 ] | Closed [ 6 ] |
I encountered the same issue.
Here is my case:
MariaDB 10.5.13
Debian 11
Galera 4.10
Steps to reproduce:
On node 1:
START TRANSACTION; |
SELECT GET_LOCK('name', 5); |
Exit.
|
The database crashes with signal 11.
This issue has been fixed since this commit.
This is the last query that was ran on this box. I replaced one of the IP's from the queries for privacy reasons.
201105 11:13:35 7265 Query drop procedure if exists __insert_10_0S17265 Query create procedure __insert_10_0S1()BEGININSERT IGNORE INTO rtp_stat ( `id_sensor`,`time`,`saddr`,`mosf1_min`,`mosf1_avg`,`mosf2_min`,`mosf2_avg`,`mosAD_min`,`mosAD_avg`,`jitter_max`,`jitter_avg`,`loss_max_mult10`,`loss_avg_mult10`,`counter` ) VALUES ( '1','2020-11-05 11:12:50',inet6_aton('46.116.65.84'),'45','45','45','45','45','45','6','6','0','0','1' ),( '1','2020-11-05 11:12:50',inet6_aton('192.73.251.107'),'45','45','45','45','45','45','0','0','0','0','1' ),( '1','2020-11-05 11:12:50',inet6_aton('192.168.31.32'),'45','45','45','45','45','45','7','3','0','0','2' );INSERT IGNORE INTO rtp_stat ( `id_sensor`,`time`,`saddr`,`mosf1_min`,`mosf1_avg`,`mosf2_min`,`mosf2_avg`,`mosAD_min`,`mosAD_avg`,`jitter_max`,`jitter_avg`,`loss_max_mult10`,`loss_avg_mult10`,`counter` ) VALUES ( '1','2020-11-05 11:13:00',inet6_aton('46.116.65.84'),'45','45','45','45','45','45','7','7','0','0','1' ),( '1','2020-11-05 11:13:00',inet6_aton('192.73.251.107'),'45','45','45','45','45','45','0','0','0','0','1' ),( '1','2020-11-05 11:13:00',inet6_aton('192.168.31.32'),'45','45','45','45','45','45','7','3','0','0','2' );do get_lock('vm_cdr_callid_57351970fdffd984ef4fd91cc55980da', 60);set @exists_call_id = coalesce((select cdr.ID from cdrjoin cdr_next on (cdr_next.cdr_ID = cdr.ID and cdr_next.calldate = cdr.calldate)where cdr.calldate > ('2020-11-05 11:12:17' - interval 1 hour) andcdr.calldate < ('2020-11-05 11:12:17' + interval 1 hour) andfbasename = 'KtisOgVrJU3dGa1YukpCI3..' limit 1), 0);set @exists_rtp =if(@exists_call_id,exists (select * from cdr_rtp where cdr_id = @exists_call_id),0);if @exists_call_id and not @exists_rtp and 0 thendelete from cdr where id = @exists_call_id;delete from cdr_next where cdr_id = @exists_call_id;delete from cdr_next_1 where cdr_ID = @exists_call_id;delete from cdr_country_code where cdr_id = @exists_call_id;delete from cdr_rtp where cdr_id = @exists_call_id;delete from cdr_dtmf where cdr_id = @exists_call_id;delete from cdr_sipresp where cdr_id = @exists_call_id;delete from cdr_tar_part where cdr_id = @exists_call_id;set @exists_call_id = 0;end if;if not @exists_call_id thenINSERT INTO cdr ( `id_sensor`,`caller`,`caller_reverse`,`called`,`called_reverse`,`caller_domain`,`called_domain`,`callername`,`callername_reverse`,`sipcallerip`,`sipcalledip`,`sipcallerport`,`sipcalledport`,`duration`,`progress_time`,`first_rtp_time`,`connect_duration`,`vlan`,`calldate`,`callend`,`sighup`,`lastSIPresponseNum`,`response_time_xxx`,`bye`,`dscp`,`flags`,`lastSIPresponse_id`,`a_ua_id` ) VALUES ( '1','1316','6131','000390237920793','397029732093000','192.168.31.15','192.168.31.15','','',inet6_aton('103.145.13.60'),inet6_aton('192.168.31.66'),'17824','5060','0',NULL,NULL,NULL,'2','2020-11-05 11:12:17','2020-11-05 11:12:17','0','407','3.000000','0','262144','5376','2','46' );if row_count() > 0 thenset @MI_NEW_ID = last_insert_id();INSERT INTO cdr_next ( `fbasename`,`calldate`,`cdr_ID` ) VALUES ( 'KtisOgVrJU3dGa1YukpCI3..','2020-11-05 11:12:17',@MI_NEW_ID );do release_lock('vm_cdr_callid_57351970fdffd984ef4fd91cc55980da');INSERT INTO cdr_country_code ( `sipcallerip_country_code`,`sipcalledip_country_code`,`caller_number_country_code`,`called_number_country_code`,`calldate`,`cdr_ID` ) VALUES ( 'NL','US','','','2020-11-05 11:12:17',@MI_NEW_ID );INSERT INTO cdr_proxy ( `cdr_ID`,`calldate`,`dst` ) VALUES ( @MI_NEW_ID,'2020-11-05 11:12:17',inet6_aton('192.168.31.15') ),( @MI_NEW_ID,'2020-11-05 11:12:17',inet6_aton('192.168.31.29') );INSERT INTO cdr_sdp ( `cdr_ID`,`ip`,`port`,`is_caller`,`calldate` ) VALUES ( @MI_NEW_ID,inet6_aton('189.144.116.105'),'8000','0','2020-11-05 11:12:17' );INSERT INTO cdr_tar_part ( `cdr_ID`,`type`,`pos`,`calldate` ) VALUES ( @MI_NEW_ID,'1','0','2020-11-05 11:12:17' );end if;end if;do release_lock('vm_cdr_callid_57351970fdffd984ef4fd91cc55980da');END7265 Query call __insert_10_0S1()