Assertion `rec_get_deleted_flag(rec, 0)' failed in dict0load.cc

    several times

    [ERROR] InnoDB: Could not find a valid tablespace file for test/FTS_0000000000003cfa_DELETED. Please refer to ...

    [ERROR] InnoDB: Refusing to load './test/FTS_0000000000003cfa_DELETED_CACHE.ibd' (id=15635, flags=0x13); dictionary contains id=15624, flags=0x13

mariadbd: /data/Server/10.11/storage/innobase/dict/dict0load.cc:2645: dict_table_t* dict_load_table_on_id(table_id_t, dict_err_ignore_t): Assertion `rec_get_deleted_flag(rec, 0)' failed.

(rr) bt

#0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=1744820480) at ./nptl/pthread_kill.c:44

#1  __pthread_kill_internal (signo=6, threadid=1744820480) at ./nptl/pthread_kill.c:78

#2  __GI___pthread_kill (threadid=1744820480, signo=signo@entry=6) at ./nptl/pthread_kill.c:89

#3  0x000056f2110ed476 in __GI_raise (sig=sig@entry=6) at ../sysdeps/posix/raise.c:26

#4  0x000056f2110d37f3 in __GI_abort () at ./stdlib/abort.c:79

#5  0x000056f2110d371b in __assert_fail_base (fmt=0x56f211288150 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=0x55c9e4c19195 "rec_get_deleted_flag(rec, 0)", 

    file=0x55c9e4c194a0 "/data/Server/10.11/storage/innobase/dict/dict0load.cc", line=2645, function=<optimized out>) at ./assert/assert.c:92

#6  0x000056f2110e4e96 in __GI___assert_fail (assertion=0x55c9e4c19195 "rec_get_deleted_flag(rec, 0)", file=0x55c9e4c194a0 "/data/Server/10.11/storage/innobase/dict/dict0load.cc", line=2645, 

    function=0x55c9e4c1ac80 "dict_table_t* dict_load_table_on_id(table_id_t, dict_err_ignore_t)") at ./assert/assert.c:101

#7  0x000055c9e45f1c7b in dict_load_table_on_id (table_id=table_id@entry=15630, ignore_err=DICT_ERR_IGNORE_RECOVER_LOCK) at /data/Server/10.11/storage/innobase/dict/dict0load.cc:2645

#8  0x000055c9e45d4655 in dict_table_open_on_id (table_id=15630, dict_locked=dict_locked@entry=false, table_op=table_op@entry=DICT_TABLE_OP_LOAD_TABLESPACE, thd=thd@entry=0x0, mdl=mdl@entry=0x0)

    at /data/Server/10.11/storage/innobase/dict/dict0dict.cc:869

#9  0x000055c9e44fe5f0 in trx_resurrect_table_locks (trx=trx@entry=0x22c673fa4c00, undo=...) at /data/Server/10.11/storage/innobase/trx/trx0trx.cc:619

#10 0x000055c9e44fff2e in trx_resurrect (undo=undo@entry=0x55c9e676e500, rseg=rseg@entry=0x55c9e5396c00 <trx_sys+31488>, start_time=start_time@entry=1716478320, start_time_micro=start_time_micro@entry=2440300438130, 

    rows_to_undo=rows_to_undo@entry=0x7ffcc639de10) at /data/Server/10.11/storage/innobase/trx/trx0trx.cc:699

#11 0x000055c9e45002d9 in trx_lists_init_at_db_start () at /data/Server/10.11/storage/innobase/trx/trx0trx.cc:756

#12 0x000055c9e44bac3d in srv_start (create_new_db=<optimized out>) at /data/Server/10.11/storage/innobase/srv/srv0start.cc:1487

#13 0x000055c9e425ab28 in innodb_init (p=<optimized out>) at /data/Server/10.11/storage/innobase/handler/ha_innodb.cc:4213

#14 0x000055c9e3f0b164 in ha_initialize_handlerton (plugin=0x55c9e66356c0) at /data/Server/10.11/sql/handler.cc:648

#15 0x000055c9e3c1531c in plugin_do_initialize (plugin=plugin@entry=0x55c9e66356c0, state=@0x7ffcc639e8fc: 4) at /data/Server/10.11/sql/sql_plugin.cc:1453

#16 0x000055c9e3c1a8ee in plugin_initialize (tmp_root=tmp_root@entry=0x7ffcc639eea0, plugin=plugin@entry=0x55c9e66356c0, argc=argc@entry=0x55c9e544ecc0 <remaining_argc>, argv=argv@entry=0x55c9e6581bd0, 

    options_only=<optimized out>) at /data/Server/10.11/sql/sql_plugin.cc:1506

#17 0x000055c9e3c1b922 in plugin_init (argc=argc@entry=0x55c9e544ecc0 <remaining_argc>, argv=<optimized out>, flags=0) at /data/Server/10.11/sql/sql_plugin.cc:1764

#18 0x000055c9e3aac259 in init_server_components () at /data/Server/10.11/sql/mysqld.cc:5247

#19 0x000055c9e3ab07ba in mysqld_main (argc=<optimized out>, argv=<optimized out>) at /data/Server/10.11/sql/mysqld.cc:5873

#20 0x000055c9e3aa3d76 in main (argc=<optimized out>, argv=<optimized out>) at /data/Server/10.11/sql/main.cc:34

(rr)

cd sdp:/data1/results/1716461064/MB-2133

_RR_TRACE_DIR=./1/rr rr replay --mark-stdio # Fate of source server

mariadb-backup --backup was running without rr tracing

_RR_TRACE_DIR=./1_clone/rr rr replay --mark-stdio mariadb-backup-0 # mariadb-backup --prepare

./1_clone/fbackup contains a file backup made after the prepare

_RR_TRACE_DIR=./1_clone/rr rr replay --mark-stdio # Failing start attempt

RQG

===

# git clone https://github.com/mleich1/rqg  RQG

#

# GIT_SHOW: HEAD -> master, origin/master, origin/HEAD a744f7f7753f7ecfe17ee4943673dd54723a2262 2024-05-23T14:53:07+02:00

# rqg.pl  : Version 4.5.0 (2023-12)

#

# $RQG_HOME/rqg.pl \

# --grammar=conf/runtime/alter_online.yy \

# --gendata=conf/runtime/alter_online.zz \

# --reporters=Mariabackup_linux \

# --mysqld=--loose-innodb-log-file-size=200M \

# --mysqld=--loose-innodb_lock_schedule_algorithm=fcfs \

# --mysqld=--loose-idle_write_transaction_timeout=0 \

# --mysqld=--loose-idle_transaction_timeout=0 \

# --mysqld=--loose-idle_readonly_transaction_timeout=0 \

# --mysqld=--connect_timeout=60 \

# --mysqld=--interactive_timeout=28800 \

# --mysqld=--slave_net_timeout=60 \

# --mysqld=--net_read_timeout=30 \

# --mysqld=--net_write_timeout=60 \

# --mysqld=--loose-table_lock_wait_timeout=50 \

# --mysqld=--wait_timeout=28800 \

# --no_mask \

# --queries=10000000 \

# --seed=random \

# --reporters=None \

# --reporters=ErrorLog \

# --reporters=Deadlock \

# --validators=None \

# --mysqld=--log_output=none \

# --mysqld=--log_bin_trust_function_creators=1 \

# --mysqld=--loose-debug_assert_on_not_freed_memory=0 \

# --engine=InnoDB \

# --restart_timeout=240 \

# --mysqld=--plugin-load-add=file_key_management.so \

# --mysqld=--loose-file-key-management-filename=$RQG_HOME/conf/mariadb/encryption_keys.txt \

# --mysqld=--plugin-load-add=provider_lzo.so \

# --mysqld=--plugin-load-add=provider_bzip2.so \

# --mysqld=--plugin-load-add=provider_lzma.so \

# --mysqld=--plugin-load-add=provider_snappy.so \

# --mysqld=--plugin-load-add=provider_lz4.so \

# --mysqld=--loose-innodb_compression_level=1 \

# --duration=300 \

# --mysqld=--loose-innodb_fatal_semaphore_wait_threshold=300 \

# --mysqld=--lock-wait-timeout=15 \

# --mysqld=--innodb-lock-wait-timeout=10 \

# --mysqld=--sql_mode=STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION \

# --mysqld=--innodb_file_per_table=1 \

# --mysqld=--loose-innodb_read_only_compressed=OFF \

# --mysqld=--innodb_stats_persistent=off \

# --mysqld=--innodb_adaptive_hash_index=on \

# --mysqld=--innodb_random_read_ahead=OFF \

# --mysqld=--loose-innodb_evict_tables_on_commit_debug=on \

# --threads=2 \

# --rr=Extended \

# --rr_options=--chaos --wait \

# --mysqld=--innodb_undo_log_truncate=ON \

# --mysqld=--loose_innodb_change_buffering=inserts \

# --mysqld=--innodb_undo_tablespaces=16 \

# --mysqld=--innodb_rollback_on_timeout=ON \

# --mysqld=--innodb_page_size=4K \

# --mysqld=--innodb-buffer-pool-size=6M \

# <local settings>