[MDEV-17898] FLUSH PRIVILEGES crashes server with segfault Created: 2018-12-04  Updated: 2020-08-25  Resolved: 2018-12-06

Status: Closed
Project: MariaDB Server
Component/s: Authentication and Privilege System
Affects Version/s: 10.0, 10.1, 10.1.26, 10.1.37, 10.2, 10.3, 10.4
Fix Version/s: 10.1.38, 10.0.38, 10.2.20, 10.3.12

Type: Bug Priority: Critical
Reporter: Claudio Nanni Assignee: Sergei Golubchik
Resolution: Fixed Votes: 2
Labels: None

Issue Links:
Blocks
is blocked by MDEV-18509 Crashes server with segfault during r... Closed
Problem/Incident
causes MDEV-18298 Crashes server with segfault during r... Closed

 Description   

With a set of data in the mysql system schema `FLUSH PRIVILEGES` crashes the server with segfault.
There seem to be an aleatory factor that determines the number of times it's needed to FLUSH PRIVILEGES before the crash, between 0 and 19 was tested.

sql_acl.cc:5306:

GRANT_NAME **first= NULL, *UNINIT_VAR(merged);
 
  ulong UNINIT_VAR(privs);
 
  for (GRANT_NAME **cur= grants.front(); cur <= grants.back(); cur++)
 
  {
 
   ACL_DB **first= NULL, *UNINIT_VAR(merged);
 
  ulong UNINIT_VAR(access), update_flags= 0;
 
  for (ACL_DB **cur= dbs.front(); cur <= dbs.back(); cur++)
 
  {
 
    if (!first || (!dbname && strcmp(cur[0]->db, cur[-1]->db)))
 
    { // new db name series 
      update_flags|= *update_role_db*(merged, first, access, grantee->user.str);
 
      merged= NULL;
 
      access= 0;
 
      first= cur;
 
    }
 
    if (strcmp(cur[0]->user, grantee->user.str) == 0)   *# SEGFAULT*
 
      access|= (merged= cur[0])->initial_access;
 
    else
 
      access|= cur[0]->access;
 
  }

What I noticed is that when the problem occurs `cur` is valued before the call to update_role_db(), but after it comes back `cur` is empty (at least if I'm not mistaken).
It definitely depends on data in the mysql schema tables.
I could not identify one specific data culprit, also because the occurrence seems to happen after a variable number of `FLUSH PRIVILEGES`.

Stack trace:

Thread pointer: 0x7f3ea83a8008
 
Attempting backtrace. You can use the following information to find out
 
where mysqld died. If you see no messages after this, something went
 
terribly wrong...
 
stack_bottom = 0x7f3e9d517110 thread_stack 0x48400
 
mysys/stacktrace.c:268(my_print_stacktrace)[0x555a1a1bdd4b]
 
sql/signal_handler.cc:168(handle_fatal_signal)[0x555a19d1a485]
 
/lib64/libpthread.so.0(+0x3639a0f7e0)[0x7f43230827e0]
 
/lib64/libc.so.6(+0x3638f28696)[0x7f43219e8696]
 
sql/sql_acl.cc:5317(merge_role_db_privileges)[0x555a19b42e9b]
 
sql/sql_acl.cc:5081(traverse_role_graph_impl)[0x555a19b35bab]
 
sql/sql_acl.cc:6779(propagate_role_grants_action)[0x555a19b35de4]
 
mysys/hash.c:769(my_hash_iterate)[0x555a1a19fd6c]
 
sql/sql_acl.cc:6845(grant_reload(THD*))[0x555a19b44dbc]
 
sql/sql_reload.cc:86(reload_acl_and_cache(THD*, unsigned long long, TABLE_LIST*, int*))[0x555a19c851ce]
 
sql/sql_parse.cc:4885(mysql_execute_command(THD*))[0x555a19b9e7e3]
 
sql/sql_parse.cc:7466(mysql_parse(THD*, char*, unsigned int, Parser_state*))[0x555a19ba6405]
 
sql/sql_parse.cc:1582(dispatch_command(enum_server_command, THD*, char*, unsigned int))[0x555a19ba9015]
 
sql/sql_parse.cc:1126(do_command(THD*))[0x555a19ba9687]
 
sql/sql_connect.cc:1330(do_handle_one_connection(THD*))[0x555a19c6b99f]
 
sql/sql_connect.cc:1244(handle_one_connection)[0x555a19c6bad7]
 
perfschema/pfs.cc:1864(pfs_spawn_thread)[0x555a19e8bb8d]
 
/lib64/libpthread.so.0(+0x3639a07aa1)[0x7f432307aaa1]
 
/lib64/libc.so.6(clone+0x6d)[0x7f43219a8bdd]
 
 
 
Trying to get some variables.
 
Some pointers may be invalid and cause the dump to abort.
 
Query (0x7f3eac4364e5): FLUSH PRIVILEGES
 
Connection ID (thread ID): 3
 
Status: NOT_KILLED



 Comments   
Comment by Elena Stepanova [ 2018-12-04 ]

10.1 328d7779bc

 
#3  <signal handler called>
 
#4  0x00007fa8cad9d34a in __strcmp_sse2_unaligned () from /lib/x86_64-linux-gnu/libc.so.6
 
#5  0x000055fe5b6dc942 in merge_role_db_privileges (grantee=0x7fa89a8d1ee0, dbname=0x0, rhash=0x7fa8ccb7a620) at /data/src/10.1/sql/sql_acl.cc:5317
 
#6  0x000055fe5b6dd942 in merge_role_privileges (role=0x7fa89da020e8, grantee=0x7fa89a8d1ee0, context=0x7fa8ccb7a800) at /data/src/10.1/sql/sql_acl.cc:5693
 
#7  0x000055fe5b6dc236 in traverse_role_graph_impl (user=0x7fa89da020e8, context=0x7fa8ccb7a800, offset=88, on_node=0x0, on_edge=0x55fe5b6dd78e <merge_role_privileges(ACL_ROLE*, ACL_ROLE*, void*)>) at /data/src/10.1/sql/sql_acl.cc:5080
 
#8  0x000055fe5b6dc4e5 in traverse_role_graph_up (role=0x7fa89da020e8, context=0x7fa8ccb7a800, on_node=0x0, on_edge=0x55fe5b6dd78e <merge_role_privileges(ACL_ROLE*, ACL_ROLE*, void*)>) at /data/src/10.1/sql/sql_acl.cc:5145
 
#9  0x000055fe5b6e0fe9 in propagate_role_grants_action (role_ptr=0x7fa89da020e8, ptr=0x0) at /data/src/10.1/sql/sql_acl.cc:6777
 
#10 0x000055fe5bfdf464 in my_hash_iterate (hash=0x55fe5ca5ebc0 <acl_roles>, action=0x55fe5b6e0f36 <propagate_role_grants_action(void*, void*)>, argument=0x0) at /data/src/10.1/mysys/hash.c:769
 
#11 0x000055fe5b6e16ab in grant_reload (thd=0x7fa8a7351070) at /data/src/10.1/sql/sql_acl.cc:6844
 
#12 0x000055fe5b8cbe0e in reload_acl_and_cache (thd=0x7fa8a7351070, options=1, tables=0x0, write_to_binlog=0x7fa8ccb7dda0) at /data/src/10.1/sql/sql_reload.cc:85
 
#13 0x000055fe5b769aea in mysql_execute_command (thd=0x7fa8a7351070) at /data/src/10.1/sql/sql_parse.cc:4886
 
#14 0x000055fe5b771075 in mysql_parse (thd=0x7fa8a7351070, rawbuf=0x7fa89d934088 "flush privileges", length=16, parser_state=0x7fa8ccb7e630) at /data/src/10.1/sql/sql_parse.cc:7467
 
#15 0x000055fe5b75f8c6 in dispatch_command (command=COM_QUERY, thd=0x7fa8a7351070, packet=0x7fa8a72b6071 "flush privileges", packet_length=16) at /data/src/10.1/sql/sql_parse.cc:1495
 
#16 0x000055fe5b75e64b in do_command (thd=0x7fa8a7351070) at /data/src/10.1/sql/sql_parse.cc:1124
 
#17 0x000055fe5b8991aa in do_handle_one_connection (thd_arg=0x7fa8a7351070) at /data/src/10.1/sql/sql_connect.cc:1330
 
#18 0x000055fe5b898f0e in handle_one_connection (arg=0x7fa8a7351070) at /data/src/10.1/sql/sql_connect.cc:1242
 
#19 0x00007fa8cc822494 in start_thread (arg=0x7fa8ccb7fb00) at pthread_create.c:333
 
#20 0x00007fa8cadf693f in clone () from /lib/x86_64-linux-gnu/libc.so.6

10.1 328d7779bc ASAN

 
==12063==ERROR: AddressSanitizer: heap-use-after-free on address 0x7efbdabf9358 at pc 0x55bed86d23d3 bp 0x7efbe5bd1010 sp 0x7efbe5bd1008
 
READ of size 8 at 0x7efbdabf9358 thread T29
 
    #0 0x55bed86d23d2 in merge_role_db_privileges /data/src/10.1/sql/sql_acl.cc:5317
 
    #1 0x55bed86d23d2 in merge_role_privileges /data/src/10.1/sql/sql_acl.cc:5693
 
    #2 0x55bed86b2602 in traverse_role_graph_impl /data/src/10.1/sql/sql_acl.cc:5080
 
    #3 0x55bed86bfceb in traverse_role_graph_up /data/src/10.1/sql/sql_acl.cc:5145
 
    #4 0x55bed86bfceb in propagate_role_grants_action /data/src/10.1/sql/sql_acl.cc:6777
 
    #5 0x55bed9afe41a in my_hash_iterate /data/src/10.1/mysys/hash.c:769
 
    #6 0x55bed86d6a8b in grant_reload(THD*) /data/src/10.1/sql/sql_acl.cc:6844
 
    #7 0x55bed8b296b2 in reload_acl_and_cache(THD*, unsigned long long, TABLE_LIST*, int*) /data/src/10.1/sql/sql_reload.cc:85
 
    #8 0x55bed8812801 in mysql_execute_command(THD*) /data/src/10.1/sql/sql_parse.cc:4886
 
    #9 0x55bed881b3ff in mysql_parse(THD*, char*, unsigned int, Parser_state*) /data/src/10.1/sql/sql_parse.cc:7467
 
    #10 0x55bed88221e3 in dispatch_command(enum_server_command, THD*, char*, unsigned int) /data/src/10.1/sql/sql_parse.cc:1495
 
    #11 0x55bed882896f in do_command(THD*) /data/src/10.1/sql/sql_parse.cc:1124
 
    #12 0x55bed8ad0023 in do_handle_one_connection(THD*) /data/src/10.1/sql/sql_connect.cc:1330
 
    #13 0x55bed8ad0534 in handle_one_connection /data/src/10.1/sql/sql_connect.cc:1242
 
    #14 0x7efc16aa2493 in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x7493)
 
    #15 0x7efc1507693e in __clone (/lib/x86_64-linux-gnu/libc.so.6+0xe893e)
 
 
 
0x7efbdabf9358 is located 605016 bytes inside of 653300-byte region [0x7efbdab65800,0x7efbdac04ff4)
 
freed by thread T29 here:
 
    #0 0x7efc16d0c527 in __interceptor_free (/usr/lib/x86_64-linux-gnu/libasan.so.1+0x54527)
 
    #1 0x55bed9b68b44 in free_memory /data/src/10.1/mysys/safemalloc.c:276
 
 
 
previously allocated by thread T29 here:
 
    #0 0x7efc16d0c73f in malloc (/usr/lib/x86_64-linux-gnu/libasan.so.1+0x5473f)
 
    #1 0x55bed9b68c15 in sf_malloc /data/src/10.1/mysys/safemalloc.c:115
 
 
 
Thread T29 created by T0 here:
 
    #0 0x7efc16cdbbba in pthread_create (/usr/lib/x86_64-linux-gnu/libasan.so.1+0x23bba)
 
    #1 0x55bed9b7f6bf in spawn_thread_noop /data/src/10.1/mysys/psi_noop.c:187
 
 
 
SUMMARY: AddressSanitizer: heap-use-after-free /data/src/10.1/sql/sql_acl.cc:5317 merge_role_db_privileges
 
Shadow bytes around the buggy address:
 
  0x0fdffb577210: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
 
  0x0fdffb577220: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
 
  0x0fdffb577230: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
 
  0x0fdffb577240: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
 
  0x0fdffb577250: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
 
=>0x0fdffb577260: fd fd fd fd fd fd fd fd fd fd fd[fd]fd fd fd fd
 
  0x0fdffb577270: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
 
  0x0fdffb577280: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
 
  0x0fdffb577290: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
 
  0x0fdffb5772a0: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
 
  0x0fdffb5772b0: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
 
Shadow byte legend (one shadow byte represents 8 application bytes):
 
  Addressable:           00
 
  Partially addressable: 01 02 03 04 05 06 07 
  Heap left redzone:       fa
 
  Heap right redzone:      fb
 
  Freed heap region:       fd
 
  Stack left redzone:      f1
 
  Stack mid redzone:       f2
 
  Stack right redzone:     f3
 
  Stack partial redzone:   f4
 
  Stack after return:      f5
 
  Stack use after scope:   f8
 
  Global redzone:          f9
 
  Global init order:       f6
 
  Poisoned by user:        f7
 
  Contiguous container OOB:fc
 
  ASan internal:           fe
 
==12063==ABORTING

Comment by Elena Stepanova [ 2018-12-04 ]

Note: the test case uses explicit lists of column names to make it applicable to all 10.x versions. The presence of the list is not essential for the issue.

Note: if it doesn't crash, try ASAN builds. Non-ASAN and ASAN variations of the failure are in the previous comment.

USE mysql;
 
 
 
INSERT INTO `db` (Host,Db,User,Select_priv,Insert_priv,Update_priv,Delete_priv,Create_priv,Drop_priv,Grant_priv,References_priv,Index_priv,Alter_priv,Create_tmp_table_priv,Lock_tables_priv,Create_view_priv,Show_view_priv,Create_routine_priv,Alter_routine_priv,Execute_priv,Event_priv,Trigger_priv)
 
VALUES 
  ('','foo','dwr_foo','Y','Y','Y','Y','N','N','N','N','N','N','N','N','Y','N','N','N','N','N','N'),
 
  ('','bar','dwr_bar','Y','Y','Y','Y','N','N','N','N','N','N','N','N','Y','N','N','N','N','N','N');
 
INSERT INTO `roles_mapping` (Host,User,Role,Admin_option) 
VALUES
 
  ('','dwr_qux_dev','dwr_foo','N'),('','dwr_qux_dev','dwr_bar','N');
 
INSERT INTO `user` (Host,User,Password,Select_priv,Insert_priv,Update_priv,Delete_priv,Create_priv,Drop_priv,Reload_priv,Shutdown_priv,Process_priv,File_priv,Grant_priv,References_priv,Index_priv,Alter_priv,Show_db_priv,Super_priv,Create_tmp_table_priv,Lock_tables_priv,Execute_priv,Repl_slave_priv,Repl_client_priv,Create_view_priv,Show_view_priv,Create_routine_priv,Alter_routine_priv,Create_user_priv,Event_priv,Trigger_priv,Create_tablespace_priv,ssl_type,ssl_cipher,x509_issuer,x509_subject,max_questions,max_updates,max_connections,max_user_connections,plugin,authentication_string,password_expired,is_role) 
VALUES 
  ('','dwr_foo','','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','','','','',0,0,0,0,'','','N','Y'),
 
  ('','dwr_bar','','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','N','','','','',0,0,0,0,'','','N','Y'),
 
  ('','dwr_qux_dev','','N','N','N','N','N','N','N','N','N','N','N','N','N','N','Y','N','N','N','N','N','N','N','N','N','N','N','N','N','N','','','','',0,0,0,0,'','','N','Y');
 
 
 
flush privileges;

Generated at Thu Feb 08 08:39:58 UTC 2024 using Jira 8.20.16#820016-sha1:9d11dbea5f4be3d4cc21f03a88dd11d8c8687422.