RDS MariaDB Instance Unable to Start: InnoDB Data Structure Corruption in Undo Log Page

2025-04-18 15:28:19 0 [Note] InnoDB: Starting crash recovery from checkpoint LSN=5840473063

2025-04-18 15:28:22 0 [Note] InnoDB: End of log at LSN=6646269219

2025-04-18 15:28:22 0 [Note] InnoDB: Retry with innodb_force_recovery=5

2025-04-18 15:28:22 0 [ERROR] InnoDB: Plugin initialization aborted with error Data structure corruption

2025-04-18 15:28:22 0 [Note] InnoDB: Starting shutdown...

2025-04-18 15:28:22 0 [ERROR] Plugin 'InnoDB' registration as a STORAGE ENGINE failed.

2025-04-18 15:28:22 0 [ERROR] Unknown/unsupported storage engine: InnoDB

2025-04-18 15:28:22 0 [ERROR] Aborting

#::301        |        Undo log page            |, Undo log of an active transaction, -

# storage/innobase/trx/trx0undo.cc

trx_undo_mem_create_at_db_start(trx_rseg_t *rseg, ulint id, uint32_t page_no)

{

    mtr_t        mtr;

    XID        xid;

    ut_ad(id < TRX_RSEG_N_SLOTS);

    mtr.start();

    const page_id_t page_id{rseg->space->id, page_no};

    const buf_block_t* block = buf_page_get(page_id, 0, RW_X_LATCH, &mtr);

    if (UNIV_UNLIKELY(!block)) {

corrupted:

        mtr.commit();

        return nullptr; # This line returning nullptr caused it

    }

#4  0x00005555569bbda0 in buf_page_get_gen (page_id=..., zip_size=0, rw_latch=RW_X_LATCH, guess=0x0, mode=10, mtr=0x7fffffffa230, err=0x0, allow_ibuf_merge=false)

    at /MariaDB/storage/innobase/buf/buf0buf.cc:3406

        block = 0x7fff94923fb0

        s = 3

        must_merge = false

        __PRETTY_FUNCTION__ = <optimized out>

#0  0x00007ffff6eb0ca0 in raise () from /lib64/libc.so.6

#1  0x00007ffff6eb2148 in abort () from /lib64/libc.so.6

#2  0x00007ffff6ea9a4a in __assert_fail_base () from /lib64/libc.so.6

#3  0x00007ffff6ea9ac2 in __assert_fail () from /lib64/libc.so.6

#4  0x00005555569bbda0 in buf_page_get_gen (page_id=..., zip_size=0, rw_latch=RW_X_LATCH, guess=0x0, mode=10, mtr=0x7fffffffa230, err=0x0, allow_ibuf_merge=false)

    at /MariaDB/storage/innobase/buf/buf0buf.cc:3406

#5  0x000055555693aeef in trx_undo_mem_create_at_db_start (rseg=0x555557be1740 <trx_sys+33792>, id=0, page_no=301)

    at /MariaDB/storage/innobase/trx/trx0undo.cc:979

#6  0x000055555691cef2 in trx_undo_lists_init (rseg=0x555557be1740 <trx_sys+33792>, rseg_header=0x7fff94923ee0)

    at /MariaDB/storage/innobase/trx/trx0rseg.cc:436

#7  0x000055555691d21d in trx_rseg_mem_restore (rseg=0x555557be1740 <trx_sys+33792>, mtr=0x7fffffffa710)

    at /MariaDB/storage/innobase/trx/trx0rseg.cc:532

#8  0x000055555691d7ee in trx_rseg_array_init () at /MariaDB/storage/innobase/trx/trx0rseg.cc:668

#9  0x0000555556925a04 in trx_lists_init_at_db_start () at /MariaDB/storage/innobase/trx/trx0trx.cc:725

#10 0x00005555568e4ce8 in srv_start (create_new_db=false) at /MariaDB/storage/innobase/srv/srv0start.cc:1501

#11 0x000055555664d7c2 in innodb_init (p=0x7ffff6a17d88) at /MariaDB/storage/innobase/handler/ha_innodb.cc:4240

#12 0x00005555562b314a in ha_initialize_handlerton (plugin=0x7ffff6b644a0) at /MariaDB/sql/handler.cc:657

#13 0x0000555555f2d691 in plugin_do_initialize (plugin=0x7ffff6b644a0, state=@0x7fffffffb818: 4) at /MariaDB/sql/sql_plugin.cc:1454

#14 0x0000555555f2da30 in plugin_initialize (tmp_root=0x7fffffffbef0, plugin=0x7ffff6b644a0, argc=0x555557e31e18 <remaining_argc>, argv=0x7ffff6a376e0, options_only=false)

    at /MariaDB/sql/sql_plugin.cc:1507

#15 0x0000555555f2e75c in plugin_init (argc=0x555557e31e18 <remaining_argc>, argv=0x7ffff6a376e0, flags=0)

    at /MariaDB/sql/sql_plugin.cc:1765

#16 0x0000555555d7b65b in init_server_components () at /MariaDB/sql/mysqld.cc:5280

#17 0x0000555555d7ca17 in mysqld_main (argc=87, argv=0x7ffff6a376e0) at /MariaDB/sql/mysqld.cc:5924

#18 0x0000555555d6bb42 in main (argc=29, argv=0x7fffffffe1f8) at /MariaDB/sql/main.cc:34

(gdb) f 5

#5  0x000055555693aeef in trx_undo_mem_create_at_db_start (rseg=0x555557be1740 <trx_sys+33792>, id=0, page_no=301)

    at /MariaDB/storage/innobase/trx/trx0undo.cc:979

979 /MariaDB/storage/innobase/trx/trx0undo.cc: No such file or directory.

(gdb) p rseg->space->id

$8 = 0

(gdb) p page_no

$9 = 301

(gdb) f 4

#4  0x00005555569bbda0 in buf_page_get_gen (page_id=..., zip_size=0, rw_latch=RW_X_LATCH, guess=0x0, mode=10, mtr=0x7fffffffa230, err=0x0, allow_ibuf_merge=false)

    at /MariaDB/storage/innobase/buf/buf0buf.cc:3406

3406    /MariaDB/storage/innobase/buf/buf0buf.cc: No such file or directory.

(gdb) p block

$10 = (buf_block_t *) 0x7fff94923fb0

(gdb) p mode

$11 = 10

(gdb) p block->page

$12 = {id_ = {m_id = 301}, hash = 0x0, oldest_modification_ = {m = {<std::__atomic_base<unsigned long>> = {static _S_alignment = 8, _M_i = 0}, <No data fields>}},

  static NOT_USED = 0, static MEMORY = 1, static REMOVE_HASH = 2, static FREED = 3, static UNFIXED = 536870912, static IBUF_EXIST = 1073741824, static REINIT = 1610612736,

  static READ_FIX = 2147483648, static WRITE_FIX = 2684354560, static WRITE_FIX_IBUF = 3221225472, static WRITE_FIX_REINIT = 3758096384, static LRU_MASK = 3758096384, lock = {lock = {

      writer = {lock = {<std::__atomic_base<unsigned int>> = {static _S_alignment = 4, _M_i = 0}, <No data fields>}, static HOLDER = 1, static WAITER = 2},

      readers = {<std::__atomic_base<unsigned int>> = {static _S_alignment = 4, _M_i = 0}, <No data fields>}, static WRITER = 2147483648}, recursive = 0,

    writer = {<std::__atomic_base<unsigned long>> = {static _S_alignment = 8, _M_i = 0}, <No data fields>}, static FOR_IO = 18446744073709551615, readers_lock = {

      lock = {<std::__atomic_base<unsigned int>> = {static _S_alignment = 4, _M_i = 0}, <No data fields>}, static HOLDER = 1, static WAITER = 2}, readers = {_M_b = {_M_p = 0x0}},

    static RECURSIVE_X = 1, static RECURSIVE_U = 65536, static RECURSIVE_MAX = 65535}, frame = 0x7fff955f8000 "", zip = {data = 0x0, m_end = 0, m_nonempty = 0, n_blobs = 0,

    ssize = 0, m_start = 0, m_external = false, fix = {m = {<std::__atomic_base<unsigned int>> = {static _S_alignment = 4, _M_i = 4}, <No data fields>}}}, in_zip_hash = false,

  in_LRU_list = true, in_page_hash = true, in_free_list = false, list = {prev = 0x0, next = 0x0}, LRU = {prev = 0x0, next = 0x7fff94923ee0}, old = 0, freed_page_clock = 0,

  access_time = {m_counter = {<std::__atomic_base<unsigned int>> = {static _S_alignment = 4, _M_i = 2949992558}, <No data fields>}}}