Uploaded image for project: 'MariaDB Server'
  1. MariaDB Server
  2. MDEV-17479

Assertion `mysql_socket.fd != -1' failed in inline_mysql_socket_send on server shutdown

    XMLWordPrintable

    Details

      Description

      10.2 af6077b5358198384eb873ce26f88e7a7ecfe106

      mysqld: /home/travis/src/include/mysql/psi/mysql_socket.h:738: inline_mysql_socket_send: Assertion `mysql_socket.fd != -1' failed.
      181016  3:19:08 [ERROR] mysqld got signal 6 ;
       
      #7  0x00007f8b93c78ca2 in __assert_fail () from /lib/x86_64-linux-gnu/libc.so.6
      #8  0x000056163a63210a in inline_mysql_socket_send (src_file=0x56163aa74f18 "/home/travis/src/vio/viosocket.c", src_line=286, mysql_socket=..., buf=0x7f8b8c6aa480, n=40, flags=64) at /home/travis/src/include/mysql/psi/mysql_socket.h:738
      #9  0x000056163a632d44 in vio_write (vio=0x56163d9739d0, buf=0x7f8b8c6aa480 "$", size=40) at /home/travis/src/vio/viosocket.c:285
      #10 0x0000561639b27703 in net_real_write (net=0x7f8b8c33a6a8, packet=0x7f8b8c6aa480 "$", len=40) at /home/travis/src/sql/net_serv.cc:678
      #11 0x0000561639b26dd5 in net_flush (net=0x7f8b8c33a6a8) at /home/travis/src/sql/net_serv.cc:380
      #12 0x0000561639b27215 in net_write_command (net=0x7f8b8c33a6a8, command=255 '\377', header=0x56163a66dd40 "", head_len=0, packet=0x7f8b9124b9c0 "\035\004#08S01Server shutdown in progress", len=35) at /home/travis/src/sql/net_serv.cc:518
      #13 0x0000561639b2e026 in net_send_error_packet (thd=0x7f8b8c33a3c0, sql_errno=1053, err=0x56163cd70019 "Server shutdown in progress", sqlstate=0x56163a6a268d "08S01") at /home/travis/src/sql/protocol.cc:465
      #14 0x0000561639b2d5c5 in net_send_error (thd=0x7f8b8c33a3c0, sql_errno=1053, err=0x56163cd70019 "Server shutdown in progress", sqlstate=0x56163a6a268d "08S01") at /home/travis/src/sql/protocol.cc:173
      #15 0x0000561639b15bcf in close_connection (thd=0x7f8b8c33a3c0, sql_errno=1053) at /home/travis/src/sql/mysqld.cc:2823
      #16 0x0000561639b135fa in close_connections () at /home/travis/src/sql/mysqld.cc:1788
      #17 0x0000561639b13dae in kill_server (sig_ptr=0x0) at /home/travis/src/sql/mysqld.cc:2012
      #18 0x0000561639b13e29 in kill_server_thread (arg=0x7f8b9152fe10) at /home/travis/src/sql/mysqld.cc:2044
      #19 0x00007f8b9483a184 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
      #20 0x00007f8b93d46ffd in clone () from /lib/x86_64-linux-gnu/libc.so.6
      

      Debug injection to reproduce the problem, courtesy of Sergey Vojtovich:

      For 10.2

      diff --git a/sql/mysqld.cc b/sql/mysqld.cc
      index 7773660..64e87ea 100644
      --- a/sql/mysqld.cc
      +++ b/sql/mysqld.cc
      @@ -1760,8 +1760,9 @@ static void close_connections(void)
         */
         DBUG_PRINT("info", ("thread_count: %d", thread_count));
       
      -  for (int i= 0; *(volatile int32*) &thread_count && i < 1000; i++)
      -    my_sleep(20000);
      +//  for (int i= 0; *(volatile int32*) &thread_count && i < 1000; i++)
      +//    my_sleep(20000);
      +  sleep(3);
       
         /*
           Force remaining threads to die by closing the connection to the client
      diff --git a/sql/sql_class.cc b/sql/sql_class.cc
      index 7b51907..9948f6f 100644
      --- a/sql/sql_class.cc
      +++ b/sql/sql_class.cc
      @@ -1793,6 +1793,7 @@ void THD::disconnect()
       {
         Vio *vio= NULL;
       
      +  sleep(2);
         mysql_mutex_lock(&LOCK_thd_data);
       
         set_killed(KILL_CONNECTION);
      diff --git a/vio/viosocket.c b/vio/viosocket.c
      index 6d76d99..89a1689 100644
      --- a/vio/viosocket.c
      +++ b/vio/viosocket.c
      @@ -584,6 +584,7 @@ int vio_close(Vio *vio)
         }
         vio->type= VIO_CLOSED;
         vio->mysql_socket= MYSQL_INVALID_SOCKET;
      +  sleep(100);
         DBUG_RETURN(r);
       }
      

      For 10.4, likely needs to be applied manually

      diff --git a/sql/mysqld.cc b/sql/mysqld.cc
      index d9c0ffa..faf10f8 100644
      --- a/sql/mysqld.cc
      +++ b/sql/mysqld.cc
      @@ -1772,8 +1772,8 @@ static void close_connections(void)
         */
         DBUG_PRINT("info", ("thread_count: %u", uint32_t(thread_count)));
       
      -  for (int i= 0; thread_count && i < 1000; i++)
      -    my_sleep(20000);
      +//  for (int i= 0; thread_count && i < 1000; i++)
      +//    my_sleep(20000);
       
         /*
           Force remaining threads to die by closing the connection to the client
      diff --git a/vio/viosocket.c b/vio/viosocket.c
      index 8fc58ef..71d3bd4 100644
      --- a/vio/viosocket.c
      +++ b/vio/viosocket.c
      @@ -627,6 +627,7 @@ int vio_close(Vio *vio)
         }
         vio->type= VIO_CLOSED;
         vio->mysql_socket= MYSQL_INVALID_SOCKET;
      +  sleep(100);
         DBUG_RETURN(r);
       }
      

      After applying it, any test on a debug build will probably suffice. Tried main.1st and

      select 1;
      

      worker[1] Using MTR_BUILD_THREAD 301, with reserved ports 16020..16039
      main.1st                                 [ pass ]      1
      ***Warnings generated in error logs during shutdown after running tests: main.1st
       
      2019-01-27  1:19:33 140291366881024 [Warning] /data/src/10.2-bug/sql/mysqld: Forcing close of thread 7  user: 'root'
      mysqld: /data/src/10.2-bug/include/mysql/psi/mysql_socket.h:738: inline_mysql_socket_send: Assertion `mysql_socket.fd != -1' failed.
      Attempting backtrace. You can use the following information to find out
      

        Attachments

          Activity

            People

            Assignee:
            wlad Vladislav Vaintroub
            Reporter:
            elenst Elena Stepanova
            Votes:
            0 Vote for this issue
            Watchers:
            1 Start watching this issue

              Dates

              Created:
              Updated:
              Resolved: