Details
-
Bug
-
Status: Open (View Workflow)
-
Major
-
Resolution: Unresolved
-
10.6.16
-
None
Description
Crash happened during page flush with option FLUSH_KEEP_LAZY.
Simple bt
(gdb) bt
|
#0 0x0000147b5a9c8854 in pthread_kill () from /lib64/libpthread.so.0 |
#1 0x000055d65ec6bdf7 in my_write_core (sig=<optimized out>) at /local/p4clients/pkgbuild-wBgVO/workspace/src/RDSMariaDB/mysys/stacktrace.c:424 |
#2 0x000055d65e474b5e in handle_fatal_signal (sig=11) at /local/p4clients/pkgbuild-wBgVO/workspace/src/RDSMariaDB/sql/signal_handler.cc:357 |
#3 <signal handler called> |
#4 0x000055d65e6e3fdb in get_rdlock (pagecache=pagecache@entry=0x55d66039dce0 <maria_pagecache_var>, block=block@entry=0x147b4b3e7ee8) |
at /local/p4clients/pkgbuild-wBgVO/workspace/src/RDSMariaDB/storage/maria/ma_pagecache.c:2530 |
#5 0x000055d65e6e5058 in make_lock_and_pin (lock=PAGECACHE_LOCK_READ, pin=PAGECACHE_PIN, any=0 '\000', block=0x147b4b3e7ee8, pagecache=0x55d66039dce0 <maria_pagecache_var>) |
at /local/p4clients/pkgbuild-wBgVO/workspace/src/RDSMariaDB/storage/maria/ma_pagecache.c:2705 |
#6 flush_cached_blocks (pagecache=pagecache@entry=0x55d66039dce0 <maria_pagecache_var>, file=file@entry=0x147a095351f8, cache=<optimized out>, cache@entry=0x147b565f1620, end=end@entry=0x147b565f1668, |
type=type@entry=FLUSH_KEEP_LAZY, first_errno=first_errno@entry=0x147b565f15f4) at /local/p4clients/pkgbuild-wBgVO/workspace/src/RDSMariaDB/storage/maria/ma_pagecache.c:4803 |
#7 0x000055d65e6e58ee in flush_pagecache_blocks_int (pagecache=pagecache@entry=0x55d66039dce0 <maria_pagecache_var>, file=file@entry=0x147a095351f8, type=type@entry=FLUSH_KEEP_LAZY, |
filter=filter@entry=0x55d65e6e9810 <filter_flush_file_evenly>, filter_arg=0x147b565f5570) at /local/p4clients/pkgbuild-wBgVO/workspace/src/RDSMariaDB/storage/maria/ma_pagecache.c:5121 |
#8 0x000055d65e6e93ee in flush_pagecache_blocks_with_filter (pagecache=0x55d66039dce0 <maria_pagecache_var>, file=file@entry=0x147a095351f8, type=type@entry=FLUSH_KEEP_LAZY, |
filter=filter@entry=0x55d65e6e9810 <filter_flush_file_evenly>, filter_arg=filter_arg@entry=0x147b565f5570) at /local/p4clients/pkgbuild-wBgVO/workspace/src/RDSMariaDB/storage/maria/ma_pagecache.c:5236 |
#9 0x000055d65e6eab6f in ma_checkpoint_background (arg=0x1e) at /local/p4clients/pkgbuild-wBgVO/workspace/src/RDSMariaDB/storage/maria/ma_checkpoint.c:686 |
#10 0x0000147b5a9c144b in start_thread () from /lib64/libpthread.so.0 |
#11 0x0000147b5a6fc52f in clone () from /lib64/libc.so.6 |
bt full
#0 0x0000147b5a9c8854 in pthread_kill () from /lib64/libpthread.so.0 |
No symbol table info available.
|
#1 0x000055d65ec6bdf7 in my_write_core (sig=<optimized out>) at /local/p4clients/pkgbuild-wBgVO/workspace/src/RDSMariaDB/mysys/stacktrace.c:424 |
No locals.
|
#2 0x000055d65e474b5e in handle_fatal_signal (sig=11) at /local/p4clients/pkgbuild-wBgVO/workspace/src/RDSMariaDB/sql/signal_handler.cc:357 |
curr_time = 1709007369 |
tm = {tm_sec = 9, tm_min = 16, tm_hour = 4, tm_mday = 27, tm_mon = 1, tm_year = 124, tm_wday = 2, tm_yday = 57, tm_isdst = 0, tm_gmtoff = 0, tm_zone = 0x147b5a41b030 "UTC"} |
thd = 0x0 |
print_invalid_query_pointer = false |
#3 <signal handler called> |
No symbol table info available.
|
#4 0x000055d65e6e3fdb in get_rdlock (pagecache=pagecache@entry=0x55d66039dce0 <maria_pagecache_var>, block=block@entry=0x147b4b3e7ee8) at /local/p4clients/pkgbuild-wBgVO/workspace/src/RDSMariaDB/storage/maria/ma_pagecache.c:2530 |
file = <optimized out>
|
pageno = <optimized out>
|
locker = <optimized out>
|
#5 0x000055d65e6e5058 in make_lock_and_pin (lock=PAGECACHE_LOCK_READ, pin=PAGECACHE_PIN, any=0 '\000', block=0x147b4b3e7ee8, pagecache=0x55d66039dce0 <maria_pagecache_var>) at /local/p4clients/pkgbuild-wBgVO/workspace/src/RDSMariaDB/storage/maria/ma_pagecache.c:2705 |
No locals.
|
#6 flush_cached_blocks (pagecache=pagecache@entry=0x55d66039dce0 <maria_pagecache_var>, file=file@entry=0x147a095351f8, cache=<optimized out>, cache@entry=0x147b565f1620, end=end@entry=0x147b565f1668, type=type@entry=FLUSH_KEEP_LAZY, first_errno=first_errno@entry=0x147b565f15f4) |
at /local/p4clients/pkgbuild-wBgVO/workspace/src/RDSMariaDB/storage/maria/ma_pagecache.c:4803 |
block = 0x147b4b3e7ee8 |
rc = <optimized out>
|
count = 9 |
#7 0x000055d65e6e58ee in flush_pagecache_blocks_int (pagecache=pagecache@entry=0x55d66039dce0 <maria_pagecache_var>, file=file@entry=0x147a095351f8, type=type@entry=FLUSH_KEEP_LAZY, filter=filter@entry=0x55d65e6e9810 <filter_flush_file_evenly>, filter_arg=0x147b565f5570) at /local/p4clients/pkgbuild-wBgVO/workspace/src/RDSMariaDB/storage/maria/ma_pagecache.c:5121 |
count = <optimized out>
|
first_in_switch = 0x0 |
block = <optimized out>
|
us_flusher = {file = 641, flush_queue = {last_thread = 0x0}, first_in_switch = 0 '\000'} |
error = 0 |
pos = 0x147b565f1668 |
next = <optimized out>
|
other_flusher = <optimized out>
|
cache_buff = {0x147b4b4e7068, 0x147b4b492a68, 0x147b4b401508, 0x147b4b42b5c8, 0x147b4b3e7ee8, 0x147b4b300918, 0x147b4b4d7618, 0x147b4b30cfa8, 0x147b4b4c9758, 0x147b4b3c26b8, 0x147b4b466e18, 0x147b4b4e7338, 0x147b4b3e8758, 0x147b4b349b18, 0x147b4b307e18, 0x147b4b33bce8, 0x147b4b3cc0d8, 0x147b4b356f28, 0x147b4b4246f8, 0x147b4b495888, 0x147b4b44b518, |
0x147b4b376cc8, 0x147b4b323c28, 0x147b4b453e58, 0x147b4b4170a8, 0x147b4b473778, 0x147b4b3f2dd8, 0x147b4b2ec098, 0x147b4b49bca8, 0x147b4b3a4268, 0x147b4b49b558, 0x147b4b3ad808, 0x147b4b488088, 0x147b4b31ad48, 0x147b4b3e6e08, 0x147b4b464868, 0x147b4b3e8cf8, 0x147b4b38e758, 0x147b4b34cf68, 0x147b4b3df368, 0x147b4b353388, 0x147b4b391a88, 0x147b4b3b4f48, |
0x147b4b48b0e8, 0x147b4b2f8de8, 0x147b4b41a3d8, 0x147b4b47fb38, 0x147b4b47cb68, 0x147b4b4c4a48, 0x147b4b300eb8, 0x147b4b378a98, 0x147b4b476fb8, 0x147b4b38caa8, 0x147b4b322368, 0x147b4b3b9988, 0x147b4b35e818, 0x147b4b48b9e8, 0x147b4b37fcc8, 0x147b4b31dfe8, 0x147b4b4e5328, 0x147b4b491868, 0x147b4b4b13c8, 0x147b4b3e9208, 0x147b4b3a41d8, 0x147b4b4a4558, |
0x147b4b475a58, 0x147b4b435618, 0x147b4b3d8408, 0x147b4b39d4b8, 0x147b4b429f48, 0x147b4b43c968, 0x147b4b32c958, 0x147b4b4bde48, 0x147b4b3022f8, 0x147b4b2d2688, 0x147b4b3ed348, 0x147b4b343938, 0x147b4b427128, 0x147b4b3479e8, 0x147b4b33a668, 0x147b4b49df88, 0x147b4b4461d8, 0x147b4b30a728, 0x147b4b340e78, 0x147b4b4e3288, 0x147b4b4ccba8, 0x147b4b434ec8, |
0x147b4b4481e8, 0x147b4b4be238, 0x147b4b3fa368, 0x147b4b363d08, 0x147b4b4c4928, 0x147b4b3aec48, 0x147b4b3de048, 0x147b4b3f0b88, 0x147b4b2ef188, 0x147b4b3416e8, 0x147b4b460188, 0x147b4b32df48, 0x147b4b2cf598, 0x147b4b427368, 0x147b4b39b388, 0x147b4b4d4f48, 0x147b4b4e4b48, 0x147b4b3f4e78, 0x147b4b308d48, 0x147b4b3b9c58, 0x147b4b443a78, 0x147b4b2c39b8, |
0x147b4b4d6fe8, 0x147b4b3cde18, 0x147b4b4a7498, 0x147b4b2cf478, 0x147b4b47cb68, 0x147b4b2c4228, 0x147b4b2f6838, 0x147b4b3e9568, 0x147b4b4af7a8, 0x147b4b3c4098, 0x147b4b443b98, 0x147b4b385bd8, 0x147b4b447e88, 0x147b4b34ed38, 0x147b4b479688, 0x147b4b2f67a8, 0x147b4b3ac218, 0x147b4b38a588, 0x147b4b38e998, 0x147b4b2f4678, 0x147b4b30b268, 0x147b4b447468, |
0x147b4b451f68, 0x147b4b2f5f38, 0x147b4b32b5a8, 0x147b4b2de268, 0x147b4b33a158, 0x147b4b3b60b8, 0x147b4b435618, 0x147b4b41bd28, 0x147b4b46cb78, 0x147b4b3695e8, 0x147b4b360d38, 0x147b4b454d88, 0x147b4b4b6828, 0x147b4b3049c8, 0x147b4b2cc808, 0x147b4b4e6258, 0x147b4b2fd4c8, 0x147b4b45a5d8, 0x147b4b2e9ae8, 0x147b4b4ad9d8, 0x147b4b329508, 0x147b4b3c1788, |
0x147b4b36e418, 0x147b4b2f0fe8, 0x147b4b3454c8, 0x147b4b4624f8, 0x147b4b37e498, 0x147b4b4c6818, 0x147b4b4e2ce8, 0x147b4b3a6668, 0x147b4b2c8638, 0x147b4b333168, 0x147b4b2e5768, 0x147b4b3f4f08, 0x147b4b3cc558, 0x147b4b446898, 0x147b4b3fa908, 0x147b4b39f168, 0x147b4b2e3518, 0x147b4b353808, 0x147b4b495528, 0x147b4b3b1e58, 0x147b4b457458, 0x147b4b3d8b58, |
0x147b4b361ab8, 0x147b4b421218, 0x147b4b428c28, 0x147b4b2d2208, 0x147b4b4bade8, 0x147b4b39d5d8, 0x147b4b4af568, 0x147b4b4c3a88, 0x147b4b4ad1f8, 0x147b4b3e5788, 0x147b4b45ed48, 0x147b4b42f8b8, 0x147b4b43e978, 0x147b4b3d5a68, 0x147b4b3001c8, 0x147b4b44ede8, 0x147b4b384318, 0x147b4b350d48, 0x147b4b3fe028, 0x147b4b481518, 0x147b4b3790c8, 0x147b4b477048, |
0x147b4b453948, 0x147b4b3a58e8, 0x147b4b38b398...} |
cache = 0x147b565f1620 |
last_errno = 0 |
rc = 0 |
#8 0x000055d65e6e93ee in flush_pagecache_blocks_with_filter (pagecache=0x55d66039dce0 <maria_pagecache_var>, file=file@entry=0x147a095351f8, type=type@entry=FLUSH_KEEP_LAZY, filter=filter@entry=0x55d65e6e9810 <filter_flush_file_evenly>, filter_arg=filter_arg@entry=0x147b565f5570) |
at /local/p4clients/pkgbuild-wBgVO/workspace/src/RDSMariaDB/storage/maria/ma_pagecache.c:5236 |
res = <optimized out>
|
#9 0x000055d65e6eab6f in ma_checkpoint_background (arg=0x1e) at /local/p4clients/pkgbuild-wBgVO/workspace/src/RDSMariaDB/storage/maria/ma_checkpoint.c:686 |
res = <optimized out>
|
interval = 30 |
sleeps = 1057681 |
sleep_time = <optimized out>
|
log_horizon_at_last_checkpoint = 102680372452046 |
pagecache_flushes_at_last_checkpoint = 23434953 |
pages_bunch_size = 74 |
filter_param = {up_to_lsn = 102680372414771, max_pages = 49} |
dfile = 0x147a095351f8 |
kfile = 0x1479f68bf398 |
#10 0x0000147b5a9c144b in start_thread () from /lib64/libpthread.so.0 |
No symbol table info available.
|
#11 0x0000147b5a6fc52f in clone () from /lib64/libc.so.6 |
No symbol table info available.
|
Looking at the threads this server is using rocksdb
Thread 3 (Thread 0x147b571fd700 (LWP 474)): |
#0 0x0000147b5a9c7377 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 |
#1 0x000055d65ed10ffc in __gthread_cond_wait (__mutex=<optimized out>, __cond=__cond@entry=0x147b5a454418) at /local/p4clients/pkgbuild-cf8xL/workspace/build/LibGCC/LibGCC-gcc.237826.0/AL2_x86_64/DEV.STD.PTHREAD/build/private/gcc-src/build/private/src/gcc-7.5.0-build2/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:864 |
#2 std::condition_variable::wait (this=this@entry=0x147b5a454418, __lock=...) at /local/p4clients/pkgbuild-cf8xL/workspace/build/LibGCC/LibGCC-gcc.237826.0/AL2_x86_64/DEV.STD.PTHREAD/build/private/gcc-src/build/private/src/gcc-7.5.0/libstdc++-v3/src/c++11/condition_variable.cc:53 |
#3 0x0000147b58216eb7 in rocksdb::ThreadPoolImpl::Impl::BGThread (this=this@entry=0x147b5a454380, thread_id=thread_id@entry=0) at /local/p4clients/pkgbuild-wBgVO/workspace/src/RDSMariaDB/storage/rocksdb/rocksdb/util/threadpool_imp.cc:196 |
#4 0x0000147b58217246 in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper (arg=0x147b5a451170) at /local/p4clients/pkgbuild-wBgVO/workspace/src/RDSMariaDB/storage/rocksdb/rocksdb/util/threadpool_imp.cc:306 |
#5 0x000055d65ed785cf in std::execute_native_thread_routine (__p=0x147b5991f2a0) at /local/p4clients/pkgbuild-cf8xL/workspace/build/LibGCC/LibGCC-gcc.237826.0/AL2_x86_64/DEV.STD.PTHREAD/build/private/gcc-src/build/private/src/gcc-7.5.0/libstdc++-v3/src/c++11/thread.cc:83 |
#6 0x0000147b5a9c144b in start_thread () from /lib64/libpthread.so.0 |
#7 0x0000147b5a6fc52f in clone () from /lib64/libc.so.6 |
 |
Thread 2 (Thread 0x147b569f9700 (LWP 508)): |
#0 0x0000147b5a9c771e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 |
#1 0x0000147b57fbb700 in inline_mysql_cond_timedwait (src_file=0x147b583be4e0 "/local/p4clients/pkgbuild-wBgVO/workspace/src/RDSMariaDB/storage/rocksdb/ha_rocksdb.cc", src_line=11670, abstime=0x147b569f73f0, mutex=0x147b586bf8d8 <myrocks::rdb_drop_idx_thread+56>, that=0x147b586bf908 <myrocks::rdb_drop_idx_thread+104>) |
at /local/p4clients/pkgbuild-wBgVO/workspace/src/RDSMariaDB/include/mysql/psi/mysql_thread.h:1088 |
#2 myrocks::Rdb_drop_index_thread::run (this=0x147b586bf8a0 <myrocks::rdb_drop_idx_thread>) at /local/p4clients/pkgbuild-wBgVO/workspace/src/RDSMariaDB/storage/rocksdb/ha_rocksdb.cc:11670 |
#3 0x0000147b58000c31 in myrocks::Rdb_thread::thread_func (thread_ptr=0x147b586bf8a0 <myrocks::rdb_drop_idx_thread>) at /local/p4clients/pkgbuild-wBgVO/workspace/src/RDSMariaDB/storage/rocksdb/rdb_threads.cc:34 |
#4 0x0000147b5a9c144b in start_thread () from /lib64/libpthread.so.0 |
#5 0x0000147b5a6fc52f in clone () from /lib64/libc.so.6 |
The crash happens because the hash link is pointing to memory address 0x0 and this memory is out of bounds. See here:
(gdb) f 4 |
#4 0x000055d65e6e3fdb in get_rdlock (pagecache=pagecache@entry=0x55d66039dce0 <maria_pagecache_var>, block=block@entry=0x147b4b3e7ee8) at /local/p4clients/pkgbuild-wBgVO/workspace/src/RDSMariaDB/storage/maria/ma_pagecache.c:2530 |
2530 in /local/p4clients/pkgbuild-wBgVO/workspace/src/RDSMariaDB/storage/maria/ma_pagecache.c |
(gdb) p block
|
$22 = (PAGECACHE_BLOCK_LINK *) 0x147b4b3e7ee8 |
(gdb) p block->hash_link
|
$23 = (struct st_pagecache_hash_link *) 0x0 |
(gdb) p *block->hash_link
|
Cannot access memory at address 0x0 |
As to why the hash_link is pointing to memory address 0x0 at this point I can't exactly tell.Only thing I can think of is that the page that the link is pointing to was flushed/released and therefore the link is pointing to nothing (0x0).
Attachments
Issue Links
- duplicates
-
MDEV-33576 Crash on get_rdlock
- Closed
- relates to
-
MDEV-15256 Crash in flush_cached_blocks
- Confirmed