rocksdb.rocksdb_cf_options failed in buildbot, server crashed

rocksdb.rocksdb_cf_options               w2 [ fail ]  Found warnings/errors in server log file!

        Test ended at 2018-06-16 07:16:01

line

Attempting backtrace. You can use the following information to find out

^ Found warnings in /dev/shm/var/2/log/mysqld.1.err

ok

 - found 'core' (0/0)

Trying 'dbx' to get a backtrace

worker[3] > Restart [mysqld.1 - pid: 9047, winpid: 9047] - using different config file

worker[3] > Restart [mysqld.2 - pid: 9049, winpid: 9049] - using different config file

Trying 'gdb' to get a backtrace from coredump /dev/shm/var/2/log/rocksdb.rocksdb_cf_options/mysqld.1/data/core

Core generated by '/usr/sbin/mysqld'

Output from gdb follows. The first stack trace is from the failing thread.

The following stack traces are from all threads (so the failing one is

duplicated).

--------------------------

[New LWP 7564]

[New LWP 7567]

[New LWP 7568]

[New LWP 7569]

[New LWP 7570]

[New LWP 7571]

[New LWP 7572]

[New LWP 7578]

[New LWP 7608]

[Thread debugging using libthread_db enabled]

Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".

Core was generated by `/usr/sbin/mysqld --defaults-group-suffix=.1 --defaults-file=/dev/shm/var/2/my.c'.

Program terminated with signal SIGABRT, Aborted.

#0  0x00007f8fd48fe0c1 in __pthread_kill (threadid=<optimized out>, signo=6) at ../nptl/sysdeps/unix/sysv/linux/pthread_kill.c:61

61	../nptl/sysdeps/unix/sysv/linux/pthread_kill.c: No such file or directory.

#0  0x00007f8fd48fe0c1 in __pthread_kill (threadid=<optimized out>, signo=6) at ../nptl/sysdeps/unix/sysv/linux/pthread_kill.c:61

#1  0x00007f8fd54b48f2 in handle_fatal_signal ()

#2  <signal handler called>

#3  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185

#4  0x00007f8fd521bb8d in mysqld_main(int, char**) ()

#5  0x00007f8fd2c79b45 in __libc_start_main (main=0x7f8fd51f7450 <main>, argc=22, argv=0x7ffd4d3a94e8, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7ffd4d3a94d8) at libc-start.c:287

#6  0x00007f8fd520d58d in _start ()

Thread 9 (Thread 0x7f8fc4e68700 (LWP 7608)):

#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135

#1  0x00007f8fd48fb479 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0

#2  0x00007f8fd48fb2a0 in __GI___pthread_mutex_lock (mutex=0x7f8fd6e268e0) at ../nptl/pthread_mutex_lock.c:79

#3  0x00007f8fcd045c11 in rocksdb::port::Mutex::Lock() () from /usr/lib/mysql/plugin/ha_rocksdb.so

#4  0x00007f8fcd0a5ed2 in rocksdb::ThreadLocalPtr::StaticMeta::Scrape(unsigned int, rocksdb::autovector<void*, 8ul>*, void*) () from /usr/lib/mysql/plugin/ha_rocksdb.so

#5  0x00007f8fcd060d24 in rocksdb::ColumnFamilyData::ResetThreadLocalSuperVersions() () from /usr/lib/mysql/plugin/ha_rocksdb.so

#6  0x00007f8fcd061172 in rocksdb::ColumnFamilyData::InstallSuperVersion(rocksdb::SuperVersionContext*, rocksdb::InstrumentedMutex*, rocksdb::MutableCFOptions const&) () from /usr/lib/mysql/plugin/ha_rocksdb.so

#7  0x00007f8fcceb92a2 in rocksdb::DBImpl::InstallSuperVersionAndScheduleWork(rocksdb::ColumnFamilyData*, rocksdb::SuperVersionContext*, rocksdb::MutableCFOptions const&) () from /usr/lib/mysql/plugin/ha_rocksdb.so

#8  0x00007f8fcceddfc3 in rocksdb::DBImpl::SwitchMemtable(rocksdb::ColumnFamilyData*, rocksdb::DBImpl::WriteContext*) () from /usr/lib/mysql/plugin/ha_rocksdb.so

#9  0x00007f8fcceb8de1 in rocksdb::DBImpl::FlushMemTable(rocksdb::ColumnFamilyData*, rocksdb::FlushOptions const&, bool) () from /usr/lib/mysql/plugin/ha_rocksdb.so

#10 0x00007f8fcceb9223 in rocksdb::DBImpl::Flush(rocksdb::FlushOptions const&, rocksdb::ColumnFamilyHandle*) () from /usr/lib/mysql/plugin/ha_rocksdb.so

#11 0x00007f8fcce39aee in ?? () from /usr/lib/mysql/plugin/ha_rocksdb.so

#12 0x00007f8fcce3e48d in ?? () from /usr/lib/mysql/plugin/ha_rocksdb.so

#13 0x00007f8fd54b6e66 in ha_finalize_handlerton(st_plugin_int*) ()

#14 0x00007f8fd52dc9f4 in ?? ()

#15 0x00007f8fd52e07ee in ?? ()

#16 0x00007f8fd52e1555 in plugin_shutdown() ()

#17 0x00007f8fd52110b6 in ?? ()

#18 0x00007f8fd521141b in unireg_end() ()

#19 0x00007f8fd52141f8 in ?? ()

#20 0x00007f8fd52145ae in kill_server_thread ()

#21 0x00007f8fd58d7421 in ?? ()

#22 0x00007f8fd48f9064 in start_thread (arg=0x7f8fc4e68700) at pthread_create.c:309

#23 0x00007f8fd2d4062d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 8 (Thread 0x7f8fcc0f3700 (LWP 7578)):

#0  do_sigwait (set=<optimized out>, sig=0x7f8fcc0f2d78) at ../nptl/sysdeps/unix/sysv/linux/../../../../../sysdeps/unix/sysv/linux/sigwait.c:63

#1  0x00007f8fd4900653 in __sigwait (set=0x7f8fcc0f2dd0, sig=0x0) at ../nptl/sysdeps/unix/sysv/linux/../../../../../sysdeps/unix/sysv/linux/sigwait.c:97

#2  0x00007f8fd521074b in signal_hand ()

#3  0x00007f8fd58d7421 in ?? ()

#4  0x00007f8fd48f9064 in start_thread (arg=0x7f8fcc0f3700) at pthread_create.c:309

#5  0x00007f8fd2d4062d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 7 (Thread 0x7f8fc67fc700 (LWP 7572)):

#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135

#1  0x00007f8fd48fb479 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0

#2  0x00007f8fd48fb2a0 in __GI___pthread_mutex_lock (mutex=0x7f8fd7086b00) at ../nptl/pthread_mutex_lock.c:79

#3  0x00007f8fcd045c11 in rocksdb::port::Mutex::Lock() () from /usr/lib/mysql/plugin/ha_rocksdb.so

#4  0x00007f8fccf75f18 in rocksdb::InstrumentedMutex::Lock() () from /usr/lib/mysql/plugin/ha_rocksdb.so

#5  0x00007f8fcd060afc in ?? () from /usr/lib/mysql/plugin/ha_rocksdb.so

#6  0x00007f8fcd0a5a75 in rocksdb::ThreadLocalPtr::StaticMeta::OnThreadExit(void*) () from /usr/lib/mysql/plugin/ha_rocksdb.so

#7  0x00007f8fd48f8e22 in __nptl_deallocate_tsd () at pthread_create.c:157

#8  0x00007f8fd48f9077 in start_thread (arg=0x7f8fc67fc700) at pthread_create.c:322

#9  0x00007f8fd2d4062d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 6 (Thread 0x7f8fc6ffd700 (LWP 7571)):

#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135

#1  0x00007f8fd48fb479 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0

#2  0x00007f8fd48fb2a0 in __GI___pthread_mutex_lock (mutex=0x7f8fd7086b00) at ../nptl/pthread_mutex_lock.c:79

#3  0x00007f8fcd045c11 in rocksdb::port::Mutex::Lock() () from /usr/lib/mysql/plugin/ha_rocksdb.so

#4  0x00007f8fccf75f18 in rocksdb::InstrumentedMutex::Lock() () from /usr/lib/mysql/plugin/ha_rocksdb.so

#5  0x00007f8fccea34b4 in rocksdb::DBImpl::SyncWAL() () from /usr/lib/mysql/plugin/ha_rocksdb.so

#6  0x00007f8fcce98148 in rocksdb::DBImpl::FlushWAL(bool) () from /usr/lib/mysql/plugin/ha_rocksdb.so

#7  0x00007f8fcce3aff2 in myrocks::Rdb_background_thread::run() () from /usr/lib/mysql/plugin/ha_rocksdb.so

#8  0x00007f8fcce85a1e in myrocks::Rdb_thread::thread_func(void*) () from /usr/lib/mysql/plugin/ha_rocksdb.so

#9  0x00007f8fd58d7421 in ?? ()

#10 0x00007f8fd48f9064 in start_thread (arg=0x7f8fc6ffd700) at pthread_create.c:309

#11 0x00007f8fd2d4062d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 5 (Thread 0x7f8fc77fe700 (LWP 7570)):

#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185

#1  0x00007f8fd33b6c7c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6

#2  0x00007f8fcd0a7679 in rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long) () from /usr/lib/mysql/plugin/ha_rocksdb.so

#3  0x00007f8fcd0a7a6f in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*) () from /usr/lib/mysql/plugin/ha_rocksdb.so

#4  0x00007f8fd33ba970 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6

#5  0x00007f8fd48f9064 in start_thread (arg=0x7f8fc77fe700) at pthread_create.c:309

#6  0x00007f8fd2d4062d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 4 (Thread 0x7f8fc7fff700 (LWP 7569)):

#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185

#1  0x00007f8fd33b6c7c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6

#2  0x00007f8fcd0a7679 in rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long) () from /usr/lib/mysql/plugin/ha_rocksdb.so

#3  0x00007f8fcd0a7a6f in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*) () from /usr/lib/mysql/plugin/ha_rocksdb.so

#4  0x00007f8fd33ba970 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6

#5  0x00007f8fd48f9064 in start_thread (arg=0x7f8fc7fff700) at pthread_create.c:309

#6  0x00007f8fd2d4062d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 3 (Thread 0x7f8fcc9cc700 (LWP 7568)):

#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185

#1  0x00007f8fcd045d1d in rocksdb::port::CondVar::Wait() () from /usr/lib/mysql/plugin/ha_rocksdb.so

#2  0x00007f8fccf761d8 in rocksdb::InstrumentedCondVar::Wait() () from /usr/lib/mysql/plugin/ha_rocksdb.so

#3  0x00007f8fccff4b38 in rocksdb::DeleteScheduler::BackgroundEmptyTrash() () from /usr/lib/mysql/plugin/ha_rocksdb.so

#4  0x00007f8fd33ba970 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6

#5  0x00007f8fd48f9064 in start_thread (arg=0x7f8fcc9cc700) at pthread_create.c:309

#6  0x00007f8fd2d4062d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 2 (Thread 0x7f8fcdcde700 (LWP 7567)):

#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238

#1  0x00007f8fd5a1dc2d in ?? ()

#2  0x00007f8fd58d7421 in ?? ()

#3  0x00007f8fd48f9064 in start_thread (arg=0x7f8fcdcde700) at pthread_create.c:309

#4  0x00007f8fd2d4062d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 1 (Thread 0x7f8fd4d1c780 (LWP 7564)):

#0  0x00007f8fd48fe0c1 in __pthread_kill (threadid=<optimized out>, signo=6) at ../nptl/sysdeps/unix/sysv/linux/pthread_kill.c:61

#1  0x00007f8fd54b48f2 in handle_fatal_signal ()

#2  <signal handler called>

#3  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185

#4  0x00007f8fd521bb8d in mysqld_main(int, char**) ()

#5  0x00007f8fd2c79b45 in __libc_start_main (main=0x7f8fd51f7450 <main>, argc=22, argv=0x7ffd4d3a94e8, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7ffd4d3a94d8) at libc-start.c:287

#6  0x00007f8fd520d58d in _start ()

Compressed file /dev/shm/var/2/log/rocksdb.rocksdb_cf_options/mysqld.1/data/core

 - saving '/dev/shm/var/2/log/rocksdb.rocksdb_cf_options/' to '/dev/shm/var/log/rocksdb.rocksdb_cf_options/'