[MDEV-17479] Assertion `mysql_socket.fd != -1' failed in inline_mysql_socket_send on server shutdown Created: 2018-10-16  Updated: 2019-01-31  Resolved: 2019-01-31

Status: Closed
Project: MariaDB Server
Component/s: Server
Affects Version/s: 10.2, 10.3, 10.4
Fix Version/s: 10.4.3, 10.2.22, 10.3.13

Type: Bug Priority: Major
Reporter: Elena Stepanova Assignee: Vladislav Vaintroub
Resolution: Fixed Votes: 0
Labels: None


 Description   

10.2 af6077b5358198384eb873ce26f88e7a7ecfe106

mysqld: /home/travis/src/include/mysql/psi/mysql_socket.h:738: inline_mysql_socket_send: Assertion `mysql_socket.fd != -1' failed.
181016  3:19:08 [ERROR] mysqld got signal 6 ;
 
#7  0x00007f8b93c78ca2 in __assert_fail () from /lib/x86_64-linux-gnu/libc.so.6
#8  0x000056163a63210a in inline_mysql_socket_send (src_file=0x56163aa74f18 "/home/travis/src/vio/viosocket.c", src_line=286, mysql_socket=..., buf=0x7f8b8c6aa480, n=40, flags=64) at /home/travis/src/include/mysql/psi/mysql_socket.h:738
#9  0x000056163a632d44 in vio_write (vio=0x56163d9739d0, buf=0x7f8b8c6aa480 "$", size=40) at /home/travis/src/vio/viosocket.c:285
#10 0x0000561639b27703 in net_real_write (net=0x7f8b8c33a6a8, packet=0x7f8b8c6aa480 "$", len=40) at /home/travis/src/sql/net_serv.cc:678
#11 0x0000561639b26dd5 in net_flush (net=0x7f8b8c33a6a8) at /home/travis/src/sql/net_serv.cc:380
#12 0x0000561639b27215 in net_write_command (net=0x7f8b8c33a6a8, command=255 '\377', header=0x56163a66dd40 "", head_len=0, packet=0x7f8b9124b9c0 "\035\004#08S01Server shutdown in progress", len=35) at /home/travis/src/sql/net_serv.cc:518
#13 0x0000561639b2e026 in net_send_error_packet (thd=0x7f8b8c33a3c0, sql_errno=1053, err=0x56163cd70019 "Server shutdown in progress", sqlstate=0x56163a6a268d "08S01") at /home/travis/src/sql/protocol.cc:465
#14 0x0000561639b2d5c5 in net_send_error (thd=0x7f8b8c33a3c0, sql_errno=1053, err=0x56163cd70019 "Server shutdown in progress", sqlstate=0x56163a6a268d "08S01") at /home/travis/src/sql/protocol.cc:173
#15 0x0000561639b15bcf in close_connection (thd=0x7f8b8c33a3c0, sql_errno=1053) at /home/travis/src/sql/mysqld.cc:2823
#16 0x0000561639b135fa in close_connections () at /home/travis/src/sql/mysqld.cc:1788
#17 0x0000561639b13dae in kill_server (sig_ptr=0x0) at /home/travis/src/sql/mysqld.cc:2012
#18 0x0000561639b13e29 in kill_server_thread (arg=0x7f8b9152fe10) at /home/travis/src/sql/mysqld.cc:2044
#19 0x00007f8b9483a184 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#20 0x00007f8b93d46ffd in clone () from /lib/x86_64-linux-gnu/libc.so.6

Debug injection to reproduce the problem, courtesy of svoj:

For 10.2

diff --git a/sql/mysqld.cc b/sql/mysqld.cc
index 7773660..64e87ea 100644
--- a/sql/mysqld.cc
+++ b/sql/mysqld.cc
@@ -1760,8 +1760,9 @@ static void close_connections(void)
   */
   DBUG_PRINT("info", ("thread_count: %d", thread_count));
 
-  for (int i= 0; *(volatile int32*) &thread_count && i < 1000; i++)
-    my_sleep(20000);
+//  for (int i= 0; *(volatile int32*) &thread_count && i < 1000; i++)
+//    my_sleep(20000);
+  sleep(3);
 
   /*
     Force remaining threads to die by closing the connection to the client
diff --git a/sql/sql_class.cc b/sql/sql_class.cc
index 7b51907..9948f6f 100644
--- a/sql/sql_class.cc
+++ b/sql/sql_class.cc
@@ -1793,6 +1793,7 @@ void THD::disconnect()
 {
   Vio *vio= NULL;
 
+  sleep(2);
   mysql_mutex_lock(&LOCK_thd_data);
 
   set_killed(KILL_CONNECTION);
diff --git a/vio/viosocket.c b/vio/viosocket.c
index 6d76d99..89a1689 100644
--- a/vio/viosocket.c
+++ b/vio/viosocket.c
@@ -584,6 +584,7 @@ int vio_close(Vio *vio)
   }
   vio->type= VIO_CLOSED;
   vio->mysql_socket= MYSQL_INVALID_SOCKET;
+  sleep(100);
   DBUG_RETURN(r);
 }

For 10.4, likely needs to be applied manually

diff --git a/sql/mysqld.cc b/sql/mysqld.cc
index d9c0ffa..faf10f8 100644
--- a/sql/mysqld.cc
+++ b/sql/mysqld.cc
@@ -1772,8 +1772,8 @@ static void close_connections(void)
   */
   DBUG_PRINT("info", ("thread_count: %u", uint32_t(thread_count)));
 
-  for (int i= 0; thread_count && i < 1000; i++)
-    my_sleep(20000);
+//  for (int i= 0; thread_count && i < 1000; i++)
+//    my_sleep(20000);
 
   /*
     Force remaining threads to die by closing the connection to the client
diff --git a/vio/viosocket.c b/vio/viosocket.c
index 8fc58ef..71d3bd4 100644
--- a/vio/viosocket.c
+++ b/vio/viosocket.c
@@ -627,6 +627,7 @@ int vio_close(Vio *vio)
   }
   vio->type= VIO_CLOSED;
   vio->mysql_socket= MYSQL_INVALID_SOCKET;
+  sleep(100);
   DBUG_RETURN(r);
 }

After applying it, any test on a debug build will probably suffice. Tried main.1st and

select 1;

worker[1] Using MTR_BUILD_THREAD 301, with reserved ports 16020..16039
main.1st                                 [ pass ]      1
***Warnings generated in error logs during shutdown after running tests: main.1st
 
2019-01-27  1:19:33 140291366881024 [Warning] /data/src/10.2-bug/sql/mysqld: Forcing close of thread 7  user: 'root'
mysqld: /data/src/10.2-bug/include/mysql/psi/mysql_socket.h:738: inline_mysql_socket_send: Assertion `mysql_socket.fd != -1' failed.
Attempting backtrace. You can use the following information to find out


Generated at Thu Feb 08 08:36:46 UTC 2024 using Jira 8.20.16#820016-sha1:9d11dbea5f4be3d4cc21f03a88dd11d8c8687422.