[MDEV-29680] ASAN heap-use-after-free in myrocks::Rdb_transaction::on_commit upon CREATE OR REPLACE .. SELECT Created: 2022-10-01  Updated: 2023-07-23

Status: Open
Project: MariaDB Server
Component/s: Data Definition - Create Table, Storage Engine - RocksDB
Affects Version/s: None
Fix Version/s: 11.2

Type: Bug Priority: Critical
Reporter: Elena Stepanova Assignee: Sergei Petrunia
Resolution: Unresolved Votes: 0
Labels: regression

Issue Links:
Problem/Incident
is caused by MDEV-25292 Atomic CREATE OR REPLACE TABLE Stalled
Relates

 Description    

INSTALL SONAME 'ha_rocksdb';
 
CREATE OR REPLACE TABLE t (a INT) ENGINE=RocksDB AS SELECT 1 AS a;

10.11 ba875e93

 
==1240302==ERROR: AddressSanitizer: heap-use-after-free on address 0x610000009be8 at pc 0x7f399cab1c3a bp 0x7f399e1a0a90 sp 0x7f399e1a0a88
 
WRITE of size 8 at 0x610000009be8 thread T5
 
    #0 0x7f399cab1c39 in std::__atomic_base<long>::store(long, std::memory_order) /usr/include/c++/10/bits/atomic_base.h:404
 
    #1 0x7f399cab1c39 in std::__atomic_base<long>::operator=(long) /usr/include/c++/10/bits/atomic_base.h:297
 
    #2 0x7f399ca9803e in myrocks::Rdb_transaction::on_commit() (/mnt8t/bld/10.11-asan-nightly/lib/plugin/ha_rocksdb.so+0xcf403e)
 
    #3 0x7f399ca9ad9f in myrocks::Rdb_transaction_impl::commit_no_binlog() (/mnt8t/bld/10.11-asan-nightly/lib/plugin/ha_rocksdb.so+0xcf6d9f)
 
    #4 0x7f399ca93e51 in myrocks::Rdb_transaction::commit() (/mnt8t/bld/10.11-asan-nightly/lib/plugin/ha_rocksdb.so+0xcefe51)
 
    #5 0x7f399ca19c47 in rocksdb_commit /data/src/10.11/storage/rocksdb/ha_rocksdb.cc:4239
 
    #6 0x5639015f3ce7 in commit_one_phase_2 /data/src/10.11/sql/handler.cc:2068
 
    #7 0x5639015f389d in ha_commit_one_phase(THD*, bool) /data/src/10.11/sql/handler.cc:2021
 
    #8 0x5639015f1bf7 in ha_commit_trans(THD*, bool) /data/src/10.11/sql/handler.cc:1815
 
    #9 0x56390120a82d in trans_commit_stmt(THD*) /data/src/10.11/sql/transaction.cc:472
 
    #10 0x563900c8fb23 in select_create::send_eof() /data/src/10.11/sql/sql_insert.cc:5321
 
    #11 0x563900e45936 in JOIN::exec_inner() /data/src/10.11/sql/sql_select.cc:4690
 
    #12 0x563900e4427c in JOIN::exec() /data/src/10.11/sql/sql_select.cc:4591
 
    #13 0x563900e4869d in mysql_select(THD*, TABLE_LIST*, List<Item>&, Item*, unsigned int, st_order*, st_order*, Item*, st_order*, unsigned long long, select_result*, st_select_lex_unit*, st_select_lex*) /data/src/10.11/sql/sql_select.cc:5071
 
    #14 0x563900e18c7c in handle_select(THD*, LEX*, select_result*, unsigned long) /data/src/10.11/sql/sql_select.cc:582
 
    #15 0x56390103384d in Sql_cmd_create_table_like::execute(THD*) /data/src/10.11/sql/sql_table.cc:12759
 
    #16 0x563900d3fec3 in mysql_execute_command(THD*, bool) /data/src/10.11/sql/sql_parse.cc:5997
 
    #17 0x563900d4d2bf in mysql_parse(THD*, char*, unsigned int, Parser_state*) /data/src/10.11/sql/sql_parse.cc:8037
 
    #18 0x563900d23921 in dispatch_command(enum_server_command, THD*, char*, unsigned int, bool) /data/src/10.11/sql/sql_parse.cc:1894
 
    #19 0x563900d206f9 in do_command(THD*, bool) /data/src/10.11/sql/sql_parse.cc:1407
 
    #20 0x5639011c7291 in do_handle_one_connection(CONNECT*, bool) /data/src/10.11/sql/sql_connect.cc:1416
 
    #21 0x5639011c6c27 in handle_one_connection /data/src/10.11/sql/sql_connect.cc:1318
 
    #22 0x563901da0f93 in pfs_spawn_thread /data/src/10.11/storage/perfschema/pfs.cc:2201
 
    #23 0x7f39a7985ea6 in start_thread nptl/pthread_create.c:477
 
    #24 0x7f39a7574aee in __clone (/lib/x86_64-linux-gnu/libc.so.6+0xfcaee)
 
 
 
0x610000009be8 is located 168 bytes inside of 192-byte region [0x610000009b40,0x610000009c00)
 
freed by thread T5 here:
 
    #0 0x7f39a7e8f017 in operator delete(void*) ../../../../src/libsanitizer/asan/asan_new_delete.cpp:160
 
    #1 0x7f399cba55ea in myrocks::Rdb_ddl_manager::remove(myrocks::Rdb_tbl_def*, rocksdb::WriteBatch*, bool) /data/src/10.11/storage/rocksdb/rdb_datadic.cc:4423
 
    #2 0x7f399cba5bfb in myrocks::Rdb_ddl_manager::rename(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::WriteBatch*) /data/src/10.11/storage/rocksdb/rdb_datadic.cc:4465
 
    #3 0x7f399ca597f0 in myrocks::ha_rocksdb::rename_table(char const*, char const*) /data/src/10.11/storage/rocksdb/ha_rocksdb.cc:11908
 
    #4 0x563901611d1a in handler::ha_rename_table(char const*, char const*) /data/src/10.11/sql/handler.cc:5347
 
    #5 0x563900ffdf93 in mysql_rename_table(handlerton*, st_mysql_const_lex_string const*, st_mysql_const_lex_string const*, st_mysql_const_lex_string const*, st_mysql_const_lex_string const*, st_mysql_const_unsigned_lex_string*, unsigned int) /data/src/10.11/sql/sql_table.cc:5536
 
    #6 0x563900dea5bf in rename_table_and_triggers(THD*, rename_param*, st_ddl_log_state*, Table_name*, st_mysql_const_lex_string const*, bool, bool*) /data/src/10.11/sql/sql_rename.cc:384
 
    #7 0x563900ff6fa5 in HA_CREATE_INFO::finalize_atomic_replace(THD*, TABLE_LIST*) /data/src/10.11/sql/sql_table.cc:4481
 
    #8 0x563900c8f94b in select_create::send_eof() /data/src/10.11/sql/sql_insert.cc:5307
 
    #9 0x563900e45936 in JOIN::exec_inner() /data/src/10.11/sql/sql_select.cc:4690
 
    #10 0x563900e4427c in JOIN::exec() /data/src/10.11/sql/sql_select.cc:4591
 
    #11 0x563900e4869d in mysql_select(THD*, TABLE_LIST*, List<Item>&, Item*, unsigned int, st_order*, st_order*, Item*, st_order*, unsigned long long, select_result*, st_select_lex_unit*, st_select_lex*) /data/src/10.11/sql/sql_select.cc:5071
 
    #12 0x563900e18c7c in handle_select(THD*, LEX*, select_result*, unsigned long) /data/src/10.11/sql/sql_select.cc:582
 
    #13 0x56390103384d in Sql_cmd_create_table_like::execute(THD*) /data/src/10.11/sql/sql_table.cc:12759
 
    #14 0x563900d3fec3 in mysql_execute_command(THD*, bool) /data/src/10.11/sql/sql_parse.cc:5997
 
    #15 0x563900d4d2bf in mysql_parse(THD*, char*, unsigned int, Parser_state*) /data/src/10.11/sql/sql_parse.cc:8037
 
    #16 0x563900d23921 in dispatch_command(enum_server_command, THD*, char*, unsigned int, bool) /data/src/10.11/sql/sql_parse.cc:1894
 
    #17 0x563900d206f9 in do_command(THD*, bool) /data/src/10.11/sql/sql_parse.cc:1407
 
    #18 0x5639011c7291 in do_handle_one_connection(CONNECT*, bool) /data/src/10.11/sql/sql_connect.cc:1416
 
    #19 0x5639011c6c27 in handle_one_connection /data/src/10.11/sql/sql_connect.cc:1318
 
    #20 0x563901da0f93 in pfs_spawn_thread /data/src/10.11/storage/perfschema/pfs.cc:2201
 
    #21 0x7f39a7985ea6 in start_thread nptl/pthread_create.c:477
 
 
 
previously allocated by thread T5 here:
 
    #0 0x7f39a7e8e647 in operator new(unsigned long) ../../../../src/libsanitizer/asan/asan_new_delete.cpp:99
 
    #1 0x7f399ca36dd6 in myrocks::ha_rocksdb::create_table(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, TABLE const*, unsigned long long) /data/src/10.11/storage/rocksdb/ha_rocksdb.cc:7748
 
    #2 0x7f399ca38172 in myrocks::ha_rocksdb::create(char const*, TABLE*, HA_CREATE_INFO*) /data/src/10.11/storage/rocksdb/ha_rocksdb.cc:7914
 
    #3 0x563901612851 in handler::ha_create(char const*, TABLE*, HA_CREATE_INFO*) /data/src/10.11/sql/handler.cc:5468
 
    #4 0x563901616e87 in ha_create_table(THD*, char const*, char const*, char const*, HA_CREATE_INFO*, st_mysql_const_unsigned_lex_string*, bool) /data/src/10.11/sql/handler.cc:5937
 
    #5 0x563900ff9d94 in create_table_impl /data/src/10.11/sql/sql_table.cc:4968
 
    #6 0x563900ffa984 in mysql_create_table_no_lock(THD*, st_mysql_const_lex_string const*, st_mysql_const_lex_string const*, st_mysql_const_lex_string const*, st_mysql_const_lex_string const*, Table_specification_st*, Alter_info*, bool*, int, TABLE_LIST*, st_mysql_const_unsigned_lex_string*) /data/src/10.11/sql/sql_table.cc:5095
 
    #7 0x563900c897e7 in select_create::create_table_from_items(THD*, List<Item>*, st_mysql_lock**) /data/src/10.11/sql/sql_insert.cc:4690
 
    #8 0x563900c8ba5b in select_create::prepare(List<Item>&, st_select_lex_unit*) /data/src/10.11/sql/sql_insert.cc:4945
 
    #9 0x563900e25568 in JOIN::prepare(TABLE_LIST*, Item*, unsigned int, st_order*, bool, st_order*, Item*, st_order*, st_select_lex*, st_select_lex_unit*) /data/src/10.11/sql/sql_select.cc:1738
 
    #10 0x563900e4841d in mysql_select(THD*, TABLE_LIST*, List<Item>&, Item*, unsigned int, st_order*, st_order*, Item*, st_order*, unsigned long long, select_result*, st_select_lex_unit*, st_select_lex*) /data/src/10.11/sql/sql_select.cc:5046
 
    #11 0x563900e18c7c in handle_select(THD*, LEX*, select_result*, unsigned long) /data/src/10.11/sql/sql_select.cc:582
 
    #12 0x56390103384d in Sql_cmd_create_table_like::execute(THD*) /data/src/10.11/sql/sql_table.cc:12759
 
    #13 0x563900d3fec3 in mysql_execute_command(THD*, bool) /data/src/10.11/sql/sql_parse.cc:5997
 
    #14 0x563900d4d2bf in mysql_parse(THD*, char*, unsigned int, Parser_state*) /data/src/10.11/sql/sql_parse.cc:8037
 
    #15 0x563900d23921 in dispatch_command(enum_server_command, THD*, char*, unsigned int, bool) /data/src/10.11/sql/sql_parse.cc:1894
 
    #16 0x563900d206f9 in do_command(THD*, bool) /data/src/10.11/sql/sql_parse.cc:1407
 
    #17 0x5639011c7291 in do_handle_one_connection(CONNECT*, bool) /data/src/10.11/sql/sql_connect.cc:1416
 
    #18 0x5639011c6c27 in handle_one_connection /data/src/10.11/sql/sql_connect.cc:1318
 
    #19 0x563901da0f93 in pfs_spawn_thread /data/src/10.11/storage/perfschema/pfs.cc:2201
 
    #20 0x7f39a7985ea6 in start_thread nptl/pthread_create.c:477
 
 
 
Thread T5 created by T0 here:
 
    #0 0x7f39a7e382a2 in __interceptor_pthread_create ../../../../src/libsanitizer/asan/asan_interceptors.cpp:214
 
    #1 0x563901d9ccfc in my_thread_create /data/src/10.11/storage/perfschema/my_thread.h:52
 
    #2 0x563901da1382 in pfs_spawn_thread_v1 /data/src/10.11/storage/perfschema/pfs.cc:2252
 
    #3 0x563900971599 in inline_mysql_thread_create /data/src/10.11/include/mysql/psi/mysql_thread.h:1139
 
    #4 0x563900988ac7 in create_thread_to_handle_connection(CONNECT*) /data/src/10.11/sql/mysqld.cc:6029
 
    #5 0x56390098910d in create_new_thread(CONNECT*) /data/src/10.11/sql/mysqld.cc:6088
 
    #6 0x56390098944f in handle_accepted_socket(st_mysql_socket, st_mysql_socket) /data/src/10.11/sql/mysqld.cc:6150
 
    #7 0x563900989dc3 in handle_connections_sockets() /data/src/10.11/sql/mysqld.cc:6274
 
    #8 0x563900988336 in mysqld_main(int, char**) /data/src/10.11/sql/mysqld.cc:5924
 
    #9 0x5639009708e4 in main /data/src/10.11/sql/main.cc:34
 
    #10 0x7f39a749bd09 in __libc_start_main ../csu/libc-start.c:308
 
 
 
SUMMARY: AddressSanitizer: heap-use-after-free /usr/include/c++/10/bits/atomic_base.h:404 in std::__atomic_base<long>::store(long, std::memory_order)
 
Shadow bytes around the buggy address:
 
  0x0c207fff9320: fa fa fa fa fa fa fa fa fd fd fd fd fd fd fd fd
 
  0x0c207fff9330: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
 
  0x0c207fff9340: fa fa fa fa fa fa fa fa 00 00 00 00 00 00 00 00
 
  0x0c207fff9350: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
 
  0x0c207fff9360: fa fa fa fa fa fa fa fa fd fd fd fd fd fd fd fd
 
=>0x0c207fff9370: fd fd fd fd fd fd fd fd fd fd fd fd fd[fd]fd fd
 
  0x0c207fff9380: fa fa fa fa fa fa fa fa fd fd fd fd fd fd fd fd
 
  0x0c207fff9390: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
 
  0x0c207fff93a0: fa fa fa fa fa fa fa fa 00 00 00 00 00 00 00 00
 
  0x0c207fff93b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
 
  0x0c207fff93c0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
 
Shadow byte legend (one shadow byte represents 8 application bytes):
 
  Addressable:           00
 
  Partially addressable: 01 02 03 04 05 06 07 
  Heap left redzone:       fa
 
  Freed heap region:       fd
 
  Stack left redzone:      f1
 
  Stack mid redzone:       f2
 
  Stack right redzone:     f3
 
  Stack after return:      f5
 
  Stack use after scope:   f8
 
  Global redzone:          f9
 
  Global init order:       f6
 
  Poisoned by user:        f7
 
  Container overflow:      fc
 
  Array cookie:            ac
 
  Intra object redzone:    bb
 
  ASan internal:           fe
 
  Left alloca redzone:     ca
 
  Right alloca redzone:    cb
 
  Shadow gap:              cc
 
==1240302==ABORTING

The failure appeared in 10.11 with

commit 93c8252f02faa8ad8dc5f005e52f1990c29d4a0d
 
Author: Aleksey Midenkov
 
Date:   Wed Aug 31 11:55:04 2022 +0300
 
 
 
    MDEV-25292 Atomic CREATE OR REPLACE TABLE



 Comments   
Comment by Aleksey Midenkov [ 2022-10-03 ]

Looks like internal RocksDB problem.

Comment by Elena Stepanova [ 2022-10-12 ]

Not sure whether it's the same problem or not, but anyway please fix this too:

Run with --mysqld=--plugin-load-add=ha_rocksdb, it is not reproducible if the plugin is installed at runtime within the test.

--source include/have_log_bin.inc
 
 
 
CREATE TABLE t (a INT) ENGINE=RocksDB;
 
LOCK TABLES t WRITE;
 
CREATE OR REPLACE TABLE t ENGINE=RocksDB AS SELECT 1 AS b;

bb-10.11-midenok 8e82c973e

 
#3  <signal handler called>
 
#4  0x00007f0d89906313 in myrocks::Rdb_transaction_impl::prepare (this=0x616000768380, name="\000\000\000\000\000\000\000\001\030\000MySQLXid\001\000\000\000\000\000\000\000\031\000\000\000\000\000\000") at /data/src/bb-10.11-midenok/storage/rocksdb/ha_rocksdb.cc:3208
 
#5  0x00007f0d8988384f in myrocks::rocksdb_prepare (hton=0x615000002618, thd=0x62c0000c0218, prepare_tx=false) at /data/src/bb-10.11-midenok/storage/rocksdb/ha_rocksdb.cc:3949
 
#6  0x000055e22899f3c6 in prepare_or_error (ht=0x615000002618, thd=0x62c0000c0218, all=false) at /data/src/bb-10.11-midenok/sql/handler.cc:1426
 
#7  0x000055e2289a12eb in ha_commit_trans (thd=0x62c0000c0218, all=false) at /data/src/bb-10.11-midenok/sql/handler.cc:1840
 
#8  0x000055e2285b9968 in trans_commit_stmt (thd=0x62c0000c0218) at /data/src/bb-10.11-midenok/sql/transaction.cc:472
 
#9  0x000055e22803cca1 in select_create::send_eof (this=0x62900004c8a8) at /data/src/bb-10.11-midenok/sql/sql_insert.cc:5329
 
#10 0x000055e2281f2dcb in JOIN::exec_inner (this=0x62900004c9f8) at /data/src/bb-10.11-midenok/sql/sql_select.cc:4690
 
#11 0x000055e2281f1711 in JOIN::exec (this=0x62900004c9f8) at /data/src/bb-10.11-midenok/sql/sql_select.cc:4591
 
#12 0x000055e2281f5b32 in mysql_select (thd=0x62c0000c0218, tables=0x0, fields=..., conds=0x0, og_num=0, order=0x0, group=0x0, having=0x0, proc_param=0x0, select_options=2201189878528, result=0x62900004c8a8, unit=0x62c0000c4658, select_lex=0x62900004bad8) at /data/src/bb-10.11-midenok/sql/sql_select.cc:5071
 
#13 0x000055e2281c6111 in handle_select (thd=0x62c0000c0218, lex=0x62c0000c4580, result=0x62900004c8a8, setup_tables_done_option=0) at /data/src/bb-10.11-midenok/sql/sql_select.cc:582
 
#14 0x000055e2283e1636 in Sql_cmd_create_table_like::execute (this=0x62900004b318, thd=0x62c0000c0218) at /data/src/bb-10.11-midenok/sql/sql_table.cc:12801
 
#15 0x000055e2280ed392 in mysql_execute_command (thd=0x62c0000c0218, is_called_from_prepared_stmt=false) at /data/src/bb-10.11-midenok/sql/sql_parse.cc:5997
 
#16 0x000055e2280fa692 in mysql_parse (thd=0x62c0000c0218, rawbuf=0x62900004b238 "CREATE OR REPLACE TABLE t ENGINE=RocksDB AS SELECT 1 AS b", length=57, parser_state=0x7f0d8384aa90) at /data/src/bb-10.11-midenok/sql/sql_parse.cc:8023
 
#17 0x000055e2280d0de4 in dispatch_command (command=COM_QUERY, thd=0x62c0000c0218, packet=0x62900026c219 "CREATE OR REPLACE TABLE t ENGINE=RocksDB AS SELECT 1 AS b", packet_length=57, blocking=true) at /data/src/bb-10.11-midenok/sql/sql_parse.cc:1894
 
#18 0x000055e2280cdbc8 in do_command (thd=0x62c0000c0218, blocking=true) at /data/src/bb-10.11-midenok/sql/sql_parse.cc:1407
 
#19 0x000055e2285763d8 in do_handle_one_connection (connect=0x60800000ac38, put_in_cache=true) at /data/src/bb-10.11-midenok/sql/sql_connect.cc:1416
 
#20 0x000055e228575d6e in handle_one_connection (arg=0x60800000abb8) at /data/src/bb-10.11-midenok/sql/sql_connect.cc:1318
 
#21 0x000055e2291506d8 in pfs_spawn_thread (arg=0x617000007418) at /data/src/bb-10.11-midenok/storage/perfschema/pfs.cc:2201
 
#22 0x00007f0d928e8ea7 in start_thread (arg=<optimized out>) at pthread_create.c:477
 
#23 0x00007f0d924d7aef in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

Comment by Aleksey Midenkov [ 2022-10-18 ]

I can disable atomic C-O-R for RocksDB if that is hard to fix and if more potential problems with atomic C-O-R in RocksDB are not payed off.

Generated at Thu Feb 08 10:10:28 UTC 2024 using Jira 8.20.16#820016-sha1:9d11dbea5f4be3d4cc21f03a88dd11d8c8687422.