Details
-
Bug
-
Status: Closed (View Workflow)
-
Major
-
Resolution: Cannot Reproduce
-
10.1(EOL)
-
None
Description
Server crashes upon crash recovery after crash-upgrade from 10.1.22.
The datadir: ftp://ftp.askmonty.org/public/mdev18054-data.tar.gz
It was created by creating data and then running DML and crashing during concurrent DML on 10.1.22 release build. The general log from the start and up to the intentional crash on 10.1.22 is in the same datadir, mysql.log.
The server was running with encryption and page size 8K, I have no indication of whether any of it is important.
An attempt to start the current 10.1 server on the datadir causes an assertion failure on a debug build and a crash on a non-debug build.
To reproduce, unpack the datadir linked above and start the server with
--file-key-management --file-key-management-filename=`pwd`/mysql-test/std_data/keys.txt --plugin-load-add=file_key_management.so --innodb-encrypt-tables --innodb-encrypt-log --innodb-encryption-threads=4 --aria-encrypt-tables=1 --encrypt-tmp-disk-tables=1 --encrypt-binlog --innodb-page-size=8K --innodb-compression-algorithm=none --innodb-file-format=Barracuda --innodb_log_compressed_pages=on
|
(modify the path to keys.txt if necessary).
10.1 9f4a4cb401 debug build |
2018-12-21 15:29:26 7f11fb885780 InnoDB: Assertion failure in thread 139715211188096 in file rem0rec.ic line 437
|
InnoDB: Failing assertion: ret > 0
|
|
#5 0x00007f11f998e3fa in abort () from /lib/x86_64-linux-gnu/libc.so.6
|
#6 0x000056258e674d1b in rec_get_n_fields_old (rec=0x7f11e2178ff1 "") at /data/src/10.1/storage/xtradb/include/rem0rec.ic:437
|
#7 0x000056258e676a94 in rec_get_offsets_func (rec=0x7f11e2178ff1 "", index=0x7f11f902bf78, offsets=0x7ffdfd4dc130, n_fields=18446744073709551615, file=0x56258eaec520 "/data/src/10.1/storage/xtradb/page/page0cur.cc", line=870, heap=0x7ffdfd4dc0e0) at /data/src/10.1/storage/xtradb/rem/rem0rec.cc:585
|
#8 0x000056258e6472b6 in page_cur_parse_insert_rec (is_short=0, ptr=0x7f11e5ff6d47 "40,($\035\026\020\b", end_ptr=0x7f11e5ff6d8a "", block=0x7f11dd62b1e0, index=0x7f11f902bf78, mtr=0x7ffdfd4dc990) at /data/src/10.1/storage/xtradb/page/page0cur.cc:870
|
#9 0x000056258e622650 in recv_parse_or_apply_log_rec_body (type=9 '\t', ptr=0x7f11e5ff6d40 <incomplete sequence \361\200\207>, end_ptr=0x7f11e5ff6d8a "", block=0x7f11dd62b1e0, mtr=0x7ffdfd4dc990, space_id=0) at /data/src/10.1/storage/xtradb/log/log0recv.cc:1174
|
#10 0x000056258e623bcd in recv_recover_page_func (just_read_in=1, block=0x7f11dd62b1e0) at /data/src/10.1/storage/xtradb/log/log0recv.cc:1745
|
#11 0x000056258e79ff02 in buf_page_io_complete (bpage=0x7f11dd62b1e0) at /data/src/10.1/storage/xtradb/buf/buf0buf.cc:5034
|
#12 0x000056258e7be32a in buf_read_page_low (err=0x7ffdfd4dcfdc, sync=true, mode=132, space=0, zip_size=0, unzip=0, tablespace_version=1, offset=11, trx=0x0, should_buffer=false) at /data/src/10.1/storage/xtradb/buf/buf0rea.cc:279
|
#13 0x000056258e7be7fa in buf_read_page (space_id=0, zip_size=0, offset=11, trx=0x0) at /data/src/10.1/storage/xtradb/buf/buf0rea.cc:512
|
#14 0x000056258e79bd4c in buf_page_get_gen (space=0, zip_size=0, offset=11, rw_latch=3, guess=0x0, mode=10, file=0x56258eb44048 "/data/src/10.1/storage/xtradb/dict/dict0load.cc", line=1859, mtr=0x7ffdfd4dd770, err=0x7ffdfd4dd128) at /data/src/10.1/storage/xtradb/buf/buf0buf.cc:3211
|
#15 0x000056258e76e472 in btr_cur_search_to_nth_level (index=0x7f11f902b078, level=0, tuple=0x7f11f9029c78, mode=2, latch_mode=1, cursor=0x7ffdfd4dd690, has_search_latch=0, file=0x56258eb44048 "/data/src/10.1/storage/xtradb/dict/dict0load.cc", line=1859, mtr=0x7ffdfd4dd770) at /data/src/10.1/storage/xtradb/btr/btr0cur.cc:659
|
#16 0x000056258e77c1a1 in btr_pcur_open_low (index=0x7f11f902b078, level=0, tuple=0x7f11f9029c78, mode=2, latch_mode=1, cursor=0x7ffdfd4dd690, file=0x56258eb44048 "/data/src/10.1/storage/xtradb/dict/dict0load.cc", line=1859, mtr=0x7ffdfd4dd770) at /data/src/10.1/storage/xtradb/include/btr0pcur.ic:440
|
#17 0x000056258e77d575 in btr_pcur_open_on_user_rec_func (index=0x7f11f902b078, tuple=0x7f11f9029c78, mode=2, latch_mode=1, cursor=0x7ffdfd4dd690, file=0x56258eb44048 "/data/src/10.1/storage/xtradb/dict/dict0load.cc", line=1859, mtr=0x7ffdfd4dd770) at /data/src/10.1/storage/xtradb/btr/btr0pcur.cc:606
|
#18 0x000056258e7ed523 in dict_load_indexes (table=0x7f11e73cf278, heap=0x7f11f9029c00, ignore_err=DICT_ERR_IGNORE_NONE) at /data/src/10.1/storage/xtradb/dict/dict0load.cc:1859
|
#19 0x000056258e7ef920 in dict_load_sys_table (table=0x7f11e73cf278) at /data/src/10.1/storage/xtradb/dict/dict0load.cc:2791
|
#20 0x000056258e7c6aea in dict_boot () at /data/src/10.1/storage/xtradb/dict/dict0boot.cc:485
|
#21 0x000056258e70b25e in innobase_start_or_create_for_mysql () at /data/src/10.1/storage/xtradb/srv/srv0start.cc:2646
|
#22 0x000056258e598f71 in innobase_init (p=0x7f11f9036a70) at /data/src/10.1/storage/xtradb/handler/ha_innodb.cc:4525
|
#23 0x000056258e262b22 in ha_initialize_handlerton (plugin=0x7f11f9324880) at /data/src/10.1/sql/handler.cc:522
|
#24 0x000056258e039401 in plugin_initialize (tmp_root=0x7ffdfd4e1e90, plugin=0x7f11f9324880, argc=0x56258f31cba8 <remaining_argc>, argv=0x7f11f9033428, options_only=false) at /data/src/10.1/sql/sql_plugin.cc:1409
|
#25 0x000056258e039fef in plugin_init (argc=0x56258f31cba8 <remaining_argc>, argv=0x7f11f9033428, flags=2) at /data/src/10.1/sql/sql_plugin.cc:1687
|
#26 0x000056258df56eaf in init_server_components () at /data/src/10.1/sql/mysqld.cc:5169
|
#27 0x000056258df57f6b in mysqld_main (argc=24, argv=0x7f11f9033428) at /data/src/10.1/sql/mysqld.cc:5760
|
#28 0x000056258df4d4c0 in main (argc=24, argv=0x7ffdfd4e2c28) at /data/src/10.1/sql/main.cc:25
|
10.1 9f4a4cb401 |
#2 <signal handler called>
|
#3 mach_read_from_1 (b=0xfffffffffffffffa <error: Cannot access memory at address 0xfffffffffffffffa>) at /data/src/10.1/storage/xtradb/include/mach0
|
data.ic:56
|
#4 rec_get_bit_field_1 (shift=0, mask=15, offs=6, rec=0x0) at /data/src/10.1/storage/xtradb/include/rem0rec.ic:170
|
#5 rec_get_n_owned_old (rec=0x0) at /data/src/10.1/storage/xtradb/include/rem0rec.ic:523
|
#6 page_rec_find_owner_rec (rec=0x0) at /data/src/10.1/storage/xtradb/include/page0page.ic:938
|
#7 page_cur_insert_rec_low (current_rec=0x7f7502d98ff1 "", index=index@entry=0x7f7519f36068, rec=0x7ffc50fa7ecf "", offsets=<optimized out>, mtr=mtr@
|
entry=0x7ffc50fa8400) at /data/src/10.1/storage/xtradb/page/page0cur.cc:1152
|
#8 0x00005616282a327a in page_cur_rec_insert (mtr=0x7ffc50fa8400, offsets=<optimized out>, index=0x7f7519f36068, rec=<optimized out>, cursor=0x7ffc50fa7b90) at /data/src/10.1/storage/xtradb/include/page0cur.ic:326
|
#9 page_cur_parse_insert_rec (is_short=is_short@entry=0, ptr=0x7f7506bf6ec7 "40,($\035\026\020\b", ptr@entry=0x7f7506bf6ec0 <incomplete sequence \361\200\207>, end_ptr=end_ptr@entry=0x7f7506bf6f0a "", block=block@entry=0x7f74fe616a60, index=0x7f7519f36068, mtr=mtr@entry=0x7ffc50fa8400) at /data/src/10.1/storage/xtradb/page/page0cur.cc:926
|
#10 0x00005616282871e6 in recv_parse_or_apply_log_rec_body (type=<optimized out>, ptr=ptr@entry=0x7f7506bf6ec0 <incomplete sequence \361\200\207>, end_ptr=0x7f7506bf6f0a "", block=block@entry=0x7f74fe616a60, mtr=mtr@entry=0x7ffc50fa8400, space_id=<optimized out>) at /data/src/10.1/storage/xtradb/log/log0recv.cc:1174
|
#11 0x000056162828960d in recv_recover_page_func (just_read_in=just_read_in@entry=1, block=block@entry=0x7f74fe616a60) at /data/src/10.1/storage/xtradb/log/log0recv.cc:1745
|
#12 0x000056162836e1d5 in buf_page_io_complete (bpage=bpage@entry=0x7f74fe616a60) at /data/src/10.1/storage/xtradb/buf/buf0buf.cc:5034
|
#13 0x0000561628384d0c in buf_read_page_low (err=err@entry=0x7ffc50fa8a4c, sync=sync@entry=true, mode=mode@entry=132, space=space@entry=0, zip_size=zip_size@entry=0, unzip=unzip@entry=0, tablespace_version=1, offset=11, trx=0x0, should_buffer=false) at /data/src/10.1/storage/xtradb/buf/buf0rea.cc:279
|
#14 0x0000561628385ea4 in buf_read_page (space_id=space_id@entry=0, zip_size=zip_size@entry=0, offset=offset@entry=11, trx=trx@entry=0x0) at /data/src/10.1/storage/xtradb/buf/buf0rea.cc:512
|
#15 0x0000561628368385 in buf_page_get_gen (space=<optimized out>, zip_size=<optimized out>, offset=<optimized out>, rw_latch=<optimized out>, guess=<optimized out>, mode=<optimized out>, file=0x561628571438 "/data/src/10.1/storage/xtradb/dict/dict0load.cc", line=1859, mtr=0x7ffc50fa92b0, err=0x7ffc50fa8c84) at /data/src/10.1/storage/xtradb/buf/buf0buf.cc:3211
|
#16 0x0000561628348168 in btr_cur_search_to_nth_level (index=0x0, level=18446744073709543424, level@entry=0, tuple=0x7f7502d98236, mode=0, mode@entry=2, latch_mode=18446744073709551615, cursor=0x7ffc50fa90f0, has_search_latch=0, file=0x561628571438 "/data/src/10.1/storage/xtradb/dict/dict0load.cc", line=1859, mtr=0x7ffc50fa92b0) at /data/src/10.1/storage/xtradb/btr/btr0cur.cc:659
|
#17 0x00005616283549d8 in btr_pcur_open_low (level=0, mtr=0x7ffc50fa92b0, line=1859, file=<optimized out>, cursor=0x7ffc50fa90f0, latch_mode=<optimized out>, mode=2, tuple=<optimized out>, index=<optimized out>) at /data/src/10.1/storage/xtradb/include/btr0pcur.ic:440
|
#18 btr_pcur_open_on_user_rec_func (index=<optimized out>, tuple=<optimized out>, mode=2, latch_mode=<optimized out>, cursor=0x7ffc50fa90f0, file=<optimized out>, line=1859, mtr=0x7ffc50fa92b0) at /data/src/10.1/storage/xtradb/btr/btr0pcur.cc:606
|
#19 0x00005616283a63ba in dict_load_indexes (table=0x0, table@entry=0x7f7519c35a68, heap=0xffffffffffffe000, heap@entry=0x7f7519c2b500, ignore_err=47809078, ignore_err@entry=DICT_ERR_IGNORE_NONE) at /data/src/10.1/storage/xtradb/dict/dict0load.cc:1859
|
#20 0x00005616283a73dc in dict_load_sys_table (table=0x7f7519c35a68) at /data/src/10.1/storage/xtradb/dict/dict0load.cc:2791
|
#21 0x000056162838b080 in dict_boot () at /data/src/10.1/storage/xtradb/dict/dict0boot.cc:485
|
#22 0x000056162830bd88 in innobase_start_or_create_for_mysql () at /data/src/10.1/storage/xtradb/srv/srv0start.cc:2646
|
#23 0x0000561628241b1e in innobase_init (p=<optimized out>) at /data/src/10.1/storage/xtradb/handler/ha_innodb.cc:4525
|
#24 0x000056162800eb01 in ha_initialize_handlerton (plugin=0x7f7519ee6818) at /data/src/10.1/sql/handler.cc:522
|
#25 0x0000561627ea9435 in plugin_initialize (tmp_root=tmp_root@entry=0x7ffc50faddb0, plugin=plugin@entry=0x7f7519ee6818, argc=argc@entry=0x561628cbce58 <remaining_argc>, argv=argv@entry=0x7f7519c3d4c0, options_only=options_only@entry=false) at /data/src/10.1/sql/sql_plugin.cc:1409
|
#26 0x0000561627eaa3aa in plugin_init (argc=argc@entry=0x561628cbce58 <remaining_argc>, argv=0x7f7519c3d4c0, flags=2) at /data/src/10.1/sql/sql_plugin.cc:1687
|
#27 0x0000561627e13502 in init_server_components () at /data/src/10.1/sql/mysqld.cc:5169
|
#28 0x0000561627e17f5a in mysqld_main (argc=24, argv=0x7f7519c3d4c0) at /data/src/10.1/sql/mysqld.cc:5760
|
#29 0x00007f751a4bf2b1 in __libc_start_main () from /lib/x86_64-linux-gnu/libc.so.6
|
#30 0x0000561627e0c1fa in _start ()
|
The failure originally happen during a crash-upgrade test, upgrade from 10.1.22 release build to 10.1 8c43f963882 here: https://travis-ci.org/elenst/travis-tests/jobs/469081457.
Attachments
Issue Links
- relates to
-
MDEV-11799 InnoDB can abort if the doublewrite buffer contains a bad and a good copy
-
- Closed
-
Here is yet another one, same circumstances (crash-upgrade from 10.1.22 to 10.1)
10.1 27f1de5
2019-01-29 4:49:40 140272197707648 [Note] InnoDB: Starting crash recovery from checkpoint LSN=2962511
2019-01-29 4:49:40 140272197707648 [Note] InnoDB: Restoring possible half-written data pages from the doublewrite buffer...
2019-01-29 4:49:40 140272197707648 [Note] InnoDB: Trying to recover page 19:1 from the doublewrite buffer.
2019-01-29 4:49:40 140272197707648 [Warning] InnoDB: A doublewrite copy of page 19:1 is corrupted.
2019-01-29 4:49:40 140272197707648 [Note] InnoDB: Trying to recover page 19:2 from the doublewrite buffer.
...
2019-01-29 4:49:40 140272197707648 [Note] InnoDB: Trying to recover page 0:429 from the doublewrite buffer.
2019-01-29 4:49:40 140272197707648 [Note] InnoDB: Recovered page 0:429 from the doublewrite buffer.
Is short 0, info_and_status_bits 0, offset 163, o_offset 7
mismatch index 18446744073709551610, end_seg_len 13
parsed len 3
Dump of 300 bytes of log:
len 300; hex 00a31a2700000000000000a00000028d04000000000000000400000000000000204cf68b937f000046d32e0000000000d3d32e000000000000000000000000000000000000000000000000000000000083020000000000000100000000000000b84bf68b937f0000b84bf68b937f000000000000000000000000000000000000000c82840000000004000000000000000400000000000000684cf68b937f00004ed32e0000000000d3d32e00000000004848f68b937f0000784cf68b937f00000000000000000000000882830000000004000000000000000400000000000000b04cf68b937f000056d32e0000000000d3d32e0000000000304cf68b937f00005087be8b937f00000000000000000000000c828d0000000004000000000000000400000000000000f84cf68b; asc ' L F . . K K hL N . . HH xL L V . . 0L P L ;
2019-01-29 04:49:42 7f93aa842780 InnoDB: Page dump in ascii and hex (8192 bytes):
...
InnoDB: End of page dump
2019-01-29 04:49:42 7f93aa842780 InnoDB: uncompressed page, stored checksum in field1 2040680547, calculated checksums for field1: crc32 3419765740, innodb 2040680547, none 3735928559, stored checksum in field2 1499582561, calculated checksums for field2: crc32 3419765740, innodb 1499582561, none 3735928559, page LSN 0 900592, low 4 bytes of LSN at page end 900592, page number (if stored to page already) 11, space id (if created with >= MySQL-4.1.1 and stored already) 0
InnoDB: page type 17855 meaning INDEX
InnoDB: Page may be an update undo log page
InnoDB: Page may be an index page where index id is 3
InnoDB: (index "CLUST_IND" of table "SYS_INDEXES")
2019-01-29 04:49:42 7f93aa842780 InnoDB: Assertion failure in thread 140272197707648 in file page0cur.cc line 908
InnoDB: We intentionally generate a memory trap.
InnoDB: Submit a detailed bug report to https://jira.mariadb.org/
InnoDB: If you get repeated assertion failures or crashes, even
InnoDB: immediately after the mysqld startup, there may be
InnoDB: corruption in the InnoDB tablespace. Please refer to
InnoDB: http://dev.mysql.com/doc/refman/5.6/en/forcing-innodb-recovery.html
InnoDB: about forcing recovery.
190129 4:49:42 [ERROR] mysqld got signal 6 ;
#5 0x00007f93a87be028 in abort () from /lib/x86_64-linux-gnu/libc.so.6
#6 0x0000557c51d7a799 in page_cur_parse_insert_rec (is_short=0, ptr=0x7f938bf64bab "'", end_ptr=0x7f938bf64bb8 "\004", block=0x7f9383678c98, index=0x557c54c95aa8, mtr=0x7ffccee4aee0) at /home/travis/src/storage/xtradb/page/page0cur.cc:908
#7 0x0000557c51d55bc0 in recv_parse_or_apply_log_rec_body (type=9 '\t', ptr=0x7f938bf64ba8 "", end_ptr=0x7f938bf64bb8 "\004", block=0x7f9383678c98, mtr=0x7ffccee4aee0, space_id=0) at /home/travis/src/storage/xtradb/log/log0recv.cc:1174
#8 0x0000557c51d57127 in recv_recover_page_func (just_read_in=1, block=0x7f9383678c98) at /home/travis/src/storage/xtradb/log/log0recv.cc:1745
#9 0x0000557c51ed274b in buf_page_io_complete (bpage=0x7f9383678c98) at /home/travis/src/storage/xtradb/buf/buf0buf.cc:5034
#10 0x0000557c51ef0c10 in buf_read_page_low (err=0x7ffccee4b55c, sync=true, mode=132, space=0, zip_size=0, unzip=0, tablespace_version=1, offset=11, trx=0x0, should_buffer=false) at /home/travis/src/storage/xtradb/buf/buf0rea.cc:279
#11 0x0000557c51ef110c in buf_read_page (space_id=0, zip_size=0, offset=11, trx=0x0) at /home/travis/src/storage/xtradb/buf/buf0rea.cc:512
#12 0x0000557c51ece583 in buf_page_get_gen (space=0, zip_size=0, offset=11, rw_latch=3, guess=0x0, mode=10, file=0x557c52258e98 "/home/travis/src/storage/xtradb/dict/dict0load.cc", line=1859, mtr=0x7ffccee4bd00, err=0x7ffccee4b6b8) at /home/travis/src/storage/xtradb/buf/buf0buf.cc:3211
#13 0x0000557c51ea0d58 in btr_cur_search_to_nth_level (index=0x557c54c93628, level=0, tuple=0x557c54c97018, mode=2, latch_mode=1, cursor=0x7ffccee4bc20, has_search_latch=0, file=0x557c52258e98 "/home/travis/src/storage/xtradb/dict/dict0load.cc", line=1859, mtr=0x7ffccee4bd00) at /home/travis/src/storage/xtradb/btr/btr0cur.cc:659
#14 0x0000557c51eaea6d in btr_pcur_open_low (index=0x557c54c93628, level=0, tuple=0x557c54c97018, mode=2, latch_mode=1, cursor=0x7ffccee4bc20, file=0x557c52258e98 "/home/travis/src/storage/xtradb/dict/dict0load.cc", line=1859, mtr=0x7ffccee4bd00) at /home/travis/src/storage/xtradb/include/btr0pcur.ic:440
#15 0x0000557c51eafe35 in btr_pcur_open_on_user_rec_func (index=0x557c54c93628, tuple=0x557c54c97018, mode=2, latch_mode=1, cursor=0x7ffccee4bc20, file=0x557c52258e98 "/home/travis/src/storage/xtradb/dict/dict0load.cc", line=1859, mtr=0x7ffccee4bd00) at /home/travis/src/storage/xtradb/btr/btr0pcur.cc:606
#16 0x0000557c51f1fbb9 in dict_load_indexes (table=0x557c54c8d8c8, heap=0x557c54c96fa0, ignore_err=DICT_ERR_IGNORE_NONE) at /home/travis/src/storage/xtradb/dict/dict0load.cc:1859
#17 0x0000557c51f21fad in dict_load_sys_table (table=0x557c54c8d8c8) at /home/travis/src/storage/xtradb/dict/dict0load.cc:2791
#18 0x0000557c51ef9489 in dict_boot () at /home/travis/src/storage/xtradb/dict/dict0boot.cc:485
#19 0x0000557c51e3dcff in innobase_start_or_create_for_mysql () at /home/travis/src/storage/xtradb/srv/srv0start.cc:2649
#20 0x0000557c51ccc477 in innobase_init (p=0x557c54664670) at /home/travis/src/storage/xtradb/handler/ha_innodb.cc:4525
#21 0x0000557c5198d86d in ha_initialize_handlerton (plugin=0x557c546562c0) at /home/travis/src/sql/handler.cc:522
#22 0x0000557c5175d843 in plugin_initialize (tmp_root=0x7ffccee50450, plugin=0x557c546562c0, argc=0x557c52a20968 <remaining_argc>, argv=0x557c54611598, options_only=false) at /home/travis/src/sql/sql_plugin.cc:1409
#23 0x0000557c5175e471 in plugin_init (argc=0x557c52a20968 <remaining_argc>, argv=0x557c54611598, flags=2) at /home/travis/src/sql/sql_plugin.cc:1687
#24 0x0000557c5167aba7 in init_server_components () at /home/travis/src/sql/mysqld.cc:5169
#25 0x0000557c5167bc7b in mysqld_main (argc=32, argv=0x557c54611598) at /home/travis/src/sql/mysqld.cc:5760
#26 0x0000557c51671175 in main (argc=32, argv=0x7ffccee51208) at /home/travis/src/sql/main.cc:25
Origin: https://travis-ci.org/elenst/travis-tests/jobs/485462408 [2619 17 10]