[ERROR] [FATAL] InnoDB: Page ... name ... page_type ... key_version 1 lsn ... compressed_len ...

The problem looks roughly like the closed bug https://jira.mariadb.org/browse/MDEV-23199.

Work flow:

1. Start the DB server and generate some data.

2. Stress the DB server with DDL and DML

     Encryption+Compression are involved, but no explicite XA commands or replication.

3. SIGKILL the DB server during 2.

4. Attempt to restart the DB Server

[rr 2043283 3708]2020-10-19 16:19:14 0 [Note] InnoDB: Starting crash recovery from checkpoint LSN=154755265

...

2020-10-19 16:19:16 0 [Note] InnoDB: Ignoring data file './test/#sql-alter-1c8fbe-12.ibd' with space ID 145. Another data file called ./test/t3.ibd exists with the same space ID.

2020-10-19 16:19:17 0 [Note] InnoDB: Transaction 351694 was in the XA prepared state.

2020-10-19 16:19:17 0 [ERROR] [FATAL] InnoDB: Page 1115494424:3 name test/t5 page_type 32770 key_version 1 lsn 150031025 compressed_len 26538

201019 16:19:17 [ERROR] mysqld got signal 6 ;

...

(rr) bt

#0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50

#1  0x000014c09952a859 in __GI_abort () at abort.c:79

#2  0x0000563e8df53670 in ib::fatal::~fatal (this=0x7ffe6b4e1810, __in_chrg=<optimized out>) at /home/mleich/Server/bb-10.5-MDEV-23855/storage/innobase/ut/ut0ut.cc:580

#3  0x0000563e8e05a9ae in fil_page_type_validate (space=0x615000004198, page=page@entry=0x1bf109030000 "") at /home/mleich/Server/bb-10.5-MDEV-23855/storage/innobase/include/mach0data.ic:84

#4  0x0000563e8e061e78 in buf_page_decrypt_after_read (bpage=bpage@entry=0x1bf107b12718, node=...) at /home/mleich/Server/bb-10.5-MDEV-23855/storage/innobase/buf/buf0buf.cc:497

#5  0x0000563e8e0661a2 in buf_page_read_complete (bpage=bpage@entry=0x1bf107b12718, node=...) at /home/mleich/Server/bb-10.5-MDEV-23855/storage/innobase/buf/buf0buf.cc:4200

#6  0x0000563e8e0ec03d in buf_read_page_low (err=err@entry=0x7ffe6b4e3590, space=<optimized out>, sync=sync@entry=true, mode=mode@entry=132, page_id=..., zip_size=zip_size@entry=0, unzip=<optimized out>)

    at /home/mleich/Server/bb-10.5-MDEV-23855/storage/innobase/buf/buf0rea.cc:360

#7  0x0000563e8e0ec2c5 in buf_read_page (page_id=..., zip_size=zip_size@entry=0) at /home/mleich/Server/bb-10.5-MDEV-23855/storage/innobase/buf/buf0rea.cc:489

#8  0x0000563e8e072b53 in buf_page_get_low (page_id=..., zip_size=zip_size@entry=0, rw_latch=rw_latch@entry=1, guess=guess@entry=0x0, mode=mode@entry=10, 

    file=file@entry=0x563e8efe4e80 "/home/mleich/Server/bb-10.5-MDEV-23855/storage/innobase/dict/dict0load.cc", line=<optimized out>, mtr=<optimized out>, err=<optimized out>, allow_ibuf_merge=<optimized out>)

    at /home/mleich/Server/bb-10.5-MDEV-23855/storage/innobase/buf/buf0buf.cc:3119

#9  0x0000563e8e075541 in buf_page_get_gen (page_id=..., zip_size=0, rw_latch=rw_latch@entry=1, guess=guess@entry=0x0, mode=mode@entry=10, file=file@entry=0x563e8efe4e80 "/home/mleich/Server/bb-10.5-MDEV-23855/storage/innobase/dict/dict0load.cc", 

    line=2988, mtr=0x7ffe6b4e4960, err=0x0, allow_ibuf_merge=false) at /home/mleich/Server/bb-10.5-MDEV-23855/storage/innobase/buf/buf0buf.cc:3576

#10 0x0000563e8e17ffb3 in dict_load_table_one (name=..., ignore_err=ignore_err@entry=DICT_ERR_IGNORE_RECOVER_LOCK, fk_tables=std::deque with 0 elements) at /home/mleich/Server/bb-10.5-MDEV-23855/storage/innobase/include/fil0fil.h:674

#11 0x0000563e8e1816e0 in dict_load_table (name=0x614000005938 "test/t5", ignore_err=ignore_err@entry=DICT_ERR_IGNORE_RECOVER_LOCK) at /home/mleich/Server/bb-10.5-MDEV-23855/storage/innobase/include/dict0types.h:114

#12 0x0000563e8e1957c5 in dict_load_table_on_id (table_id=table_id@entry=137, ignore_err=ignore_err@entry=DICT_ERR_IGNORE_RECOVER_LOCK) at /home/mleich/Server/bb-10.5-MDEV-23855/storage/innobase/dict/dict0load.cc:3176

#13 0x0000563e8e1401ea in dict_table_open_on_id_low (table_id=table_id@entry=137, ignore_err=<optimized out>, cached_only=<optimized out>) at /home/mleich/Server/bb-10.5-MDEV-23855/storage/innobase/dict/dict0dict.cc:222

#14 0x0000563e8e15321b in dict_table_open_on_id (table_id=137, dict_locked=dict_locked@entry=false, table_op=table_op@entry=DICT_TABLE_OP_LOAD_TABLESPACE, thd=thd@entry=0x0, mdl=mdl@entry=0x0)

    at /home/mleich/Server/bb-10.5-MDEV-23855/storage/innobase/dict/dict0dict.cc:948

#15 0x0000563e8df1a459 in trx_resurrect_table_locks (trx=trx@entry=0x4a1e648a29d8, undo=undo@entry=0x6120000049d8) at /usr/include/c++/9/bits/stl_tree.h:356

#16 0x0000563e8df2001c in trx_resurrect (undo=undo@entry=0x6120000049d8, rseg=rseg@entry=0x613000004f18, start_time=start_time@entry=1603124357, start_time_micro=start_time_micro@entry=2644134905403, rows_to_undo=rows_to_undo@entry=0x7ffe6b4e6b40, 

    is_old_insert=is_old_insert@entry=false) at /home/mleich/Server/bb-10.5-MDEV-23855/storage/innobase/trx/trx0trx.cc:713

#17 0x0000563e8df2220a in trx_lists_init_at_db_start () at /home/mleich/Server/bb-10.5-MDEV-23855/storage/innobase/trx/trx0trx.cc:774

#18 0x0000563e8de23a44 in srv_start (create_new_db=<optimized out>) at /home/mleich/Server/bb-10.5-MDEV-23855/storage/innobase/srv/srv0start.cc:1510

#19 0x0000563e8d8c8fac in innodb_init (p=<optimized out>) at /home/mleich/Server/bb-10.5-MDEV-23855/storage/innobase/handler/ha_innodb.cc:3896

#20 0x0000563e8cc894c8 in ha_initialize_handlerton (plugin=0x62100002e010) at /home/mleich/Server/bb-10.5-MDEV-23855/sql/handler.cc:645

#21 0x0000563e8c3fba8e in plugin_initialize (tmp_root=tmp_root@entry=0x7ffe6b4ead10, plugin=plugin@entry=0x62100002e010, argc=argc@entry=0x563e8fdf2ec0 <remaining_argc>, argv=argv@entry=0x615000000d78, options_only=<optimized out>)

    at /home/mleich/Server/bb-10.5-MDEV-23855/sql/sql_plugin.cc:1459

#22 0x0000563e8c3ff34d in plugin_init (argc=argc@entry=0x563e8fdf2ec0 <remaining_argc>, argv=<optimized out>, flags=0) at /home/mleich/Server/bb-10.5-MDEV-23855/sql/sql_plugin.cc:1751

#23 0x0000563e8c064074 in init_server_components () at /home/mleich/Server/bb-10.5-MDEV-23855/sql/mysqld.cc:4913

#24 0x0000563e8c06eb96 in mysqld_main (argc=46, argv=<optimized out>) at /home/mleich/Server/bb-10.5-MDEV-23855/sql/mysqld.cc:5496

#25 0x0000563e8c04cf76 in main (argc=<optimized out>, argv=<optimized out>) at /home/mleich/Server/bb-10.5-MDEV-23855/sql/main.cc:25

(rr) quit

origin/bb-10.5-MDEV-23855 6155c7e55fafe2c0bd03c41e181e3d4531cfe0c5 2020-10-19T18:08:53+03:00 compiled with debug+ASAN+Og

Per Marko: The problem is not related to MDEV-23855.

Hitting this problem is extreme rare.

RQG

====

git clone https://github.com/mleich1/rqg --branch experimental RQG

perl rqg.pl \

--grammar=conf/mariadb/innodb_compression_encryption.yy \

--gendata=conf/mariadb/innodb_compression_encryption.zz \

--mysqld=--plugin-load-add=file_key_management.so \

--mysqld=--loose-file-key-management-filename=/data/mleich/RQG_mleich/conf/mariadb/encryption_keys.txt \

--mysqld=--innodb-encrypt-log \

--mysqld=--innodb-encrypt-tables \

--reporters=CrashRecovery1 \

--mysqld=--innodb_use_native_aio=1 \

--mysqld=--innodb_lock_schedule_algorithm=fcfs \

--mysqld=--loose-idle_write_transaction_timeout=0 \

--mysqld=--loose-idle_transaction_timeout=0 \

--mysqld=--loose-idle_readonly_transaction_timeout=0 \

--mysqld=--connect_timeout=60 \

--mysqld=--interactive_timeout=28800 \

--mysqld=--slave_net_timeout=60 \

--mysqld=--net_read_timeout=30 \

--mysqld=--net_write_timeout=60 \

--mysqld=--loose-table_lock_wait_timeout=50 \

--mysqld=--wait_timeout=28800 \

--mysqld=--lock-wait-timeout=86400 \

--mysqld=--innodb-lock-wait-timeout=50 \

--no-mask \

--queries=10000000 \

--seed=random \

--reporters=Backtrace \

--reporters=ErrorLog \

--reporters=Deadlock1 \

--validators=None \

--mysqld=--log_output=none \

--mysqld=--log-bin \

--mysqld=--log_bin_trust_function_creators=1 \

--mysqld=--loose-debug_assert_on_not_freed_memory=0 \

--engine=InnoDB \

--restart_timeout=120 \

--duration=400 \

--mysqld=--loose-innodb_fatal_semaphore_wait_threshold=400 \

--mysqld=--loose-innodb-sync-debug \

--mysqld=--innodb_stats_persistent=on \

--mysqld=--loose-max-statement-time=30 \

--threads=9 \

--mysqld=--innodb_page_size=64K \

--mysqld=--innodb-buffer-pool-size=24M \

--duration=400 \

--no_mask \

--workdir=<local settings> \

--vardir=<local settings> \

--mtr-build-thread=<local settings> \

--basedir1=<local settings> \

--script_debug=_nix_ \

--rr=Server \

--rr_options=--chaos