From a4338767fc07f69a651bb4d9150618edcffa4474 Mon Sep 17 00:00:00 2001 From: Alessandro Vetere Date: Tue, 3 Nov 2026 17:19:59 +0100 Subject: [PATCH 1/3] MDEV-42009: Hang on recovery after a corrupted page If redo log could not be applied to a page that buf_page_t::read_complete() had read during crash recovery, buf_pool_t::corrupted_evict() would wait forever for the buffer-fix that the reading thread itself was holding. Both buf_read_page_low() and IORequest::read_complete() hold such a buffer-fix, and both expect buf_pool_t::corrupted_evict() to release it, like it does when the page checksum fails. The hang was observed in dict_boot() during the test innodb.blob-crash. recv_recover_page(): Add the parameter caller_fixed, to tell whether buf_pool_t::corrupted_evict() must release a buffer-fix of the caller. Only the buf_page_t::read_complete() code path holds one; IORequest::fake_read_complete() and recv_sys_t::recover_low() do not. IORequest::fake_read_complete(): If redo log could not be applied to a page that was initialized by the log, flag the corruption with recv_sys_t::set_corrupt_fs(). Previously, the page was evicted and the startup completed, so that a later log checkpoint discarded the changes. recv_simulate_corruption(): Debug instrumentation that simulates a failure to apply log on each of the code paths. Assisted-by: Claude:claude-opus-5-5 --- .../innodb/r/recovery_corrupted_evict.result | 39 +++++++++++ .../innodb/t/recovery_corrupted_evict.test | 54 +++++++++++++++ storage/innobase/log/log0recv.cc | 66 +++++++++++++++++-- 3 files changed, 153 insertions(+), 6 deletions(-) create mode 100644 mysql-test/suite/innodb/r/recovery_corrupted_evict.result create mode 100644 mysql-test/suite/innodb/t/recovery_corrupted_evict.test diff --git a/mysql-test/suite/innodb/r/recovery_corrupted_evict.result b/mysql-test/suite/innodb/r/recovery_corrupted_evict.result new file mode 100644 index 00000000000..2e1398cccae --- /dev/null +++ b/mysql-test/suite/innodb/r/recovery_corrupted_evict.result @@ -0,0 +1,39 @@ +# +# MDEV-42009 Corruption and hang on recovery in innodb.blob-crash +# +call mtr.add_suppression("InnoDB: Simulating corruption on \\[page id: space="); +call mtr.add_suppression("InnoDB: Cannot apply log to \\[page id: space=[1-9][0-9]*, page number=0\\] of corrupted file '\\./test/t_new\\.ibd'"); +call mtr.add_suppression("InnoDB: Plugin initialization aborted"); +call mtr.add_suppression("Plugin 'InnoDB' (init function returned error|registration as a STORAGE ENGINE failed)"); +CREATE TABLE t_old (a INT PRIMARY KEY, b VARCHAR(255)) ENGINE=InnoDB; +INSERT INTO t_old SELECT seq, REPEAT('x', 200) FROM seq_1_to_100; +SET GLOBAL innodb_log_checkpoint_now=ON; +UPDATE t_old SET b=REPEAT('y', 200); +CREATE TABLE t_new (a INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t_new SELECT seq FROM seq_1_to_100; +# Kill the server +# Synchronous read of a system tablespace page by dict_boot() +# restart: --debug-dbug=+d,recv_corrupt_system_read +FOUND 1 /InnoDB: Plugin initialization aborted at srv0start\.cc\[\d+\] with error/ in mysqld.1.err +# Asynchronous read of a user tablespace page by recv_sys_t::apply() +# restart: --debug-dbug=+d,recv_corrupt_user_read +FOUND 2 /InnoDB: Plugin initialization aborted at srv0start\.cc\[\d+\] with error/ in mysqld.1.err +# Page initialized by the log (fake read in recv_sys_t::apply()) +# restart: --debug-dbug=+d,recv_corrupt_user_init +FOUND 3 /InnoDB: Plugin initialization aborted at srv0start\.cc\[\d+\] with error/ in mysqld.1.err +# First page of a tablespace in recv_sys_t::recover_deferred() +# restart: --debug-dbug=+d,recv_corrupt_user_page0_init +FOUND 4 /InnoDB: Plugin initialization aborted at srv0start\.cc\[\d+\] with error/ in mysqld.1.err +# No log was discarded by the failed attempts +# restart +CHECK TABLE t_old, t_new; +Table Op Msg_type Msg_text +test.t_old check status OK +test.t_new check status OK +SELECT COUNT(*) FROM t_old WHERE b=REPEAT('y', 200); +COUNT(*) +100 +SELECT COUNT(*) FROM t_new; +COUNT(*) +100 +DROP TABLE t_old, t_new; diff --git a/mysql-test/suite/innodb/t/recovery_corrupted_evict.test b/mysql-test/suite/innodb/t/recovery_corrupted_evict.test new file mode 100644 index 00000000000..8d5115055cf --- /dev/null +++ b/mysql-test/suite/innodb/t/recovery_corrupted_evict.test @@ -0,0 +1,54 @@ +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/have_sequence.inc +--source include/not_embedded.inc + +--echo # +--echo # MDEV-42009 Corruption and hang on recovery in innodb.blob-crash +--echo # + +call mtr.add_suppression("InnoDB: Simulating corruption on \\[page id: space="); +call mtr.add_suppression("InnoDB: Cannot apply log to \\[page id: space=[1-9][0-9]*, page number=0\\] of corrupted file '\\./test/t_new\\.ibd'"); +call mtr.add_suppression("InnoDB: Plugin initialization aborted"); +call mtr.add_suppression("Plugin 'InnoDB' (init function returned error|registration as a STORAGE ENGINE failed)"); + +CREATE TABLE t_old (a INT PRIMARY KEY, b VARCHAR(255)) ENGINE=InnoDB; +INSERT INTO t_old SELECT seq, REPEAT('x', 200) FROM seq_1_to_100; +SET GLOBAL innodb_log_checkpoint_now=ON; +# t_old pages will be read from the data file, t_new pages will be +# initialized by the log, and t_new.ibd will be recovered as deferred. +UPDATE t_old SET b=REPEAT('y', 200); +CREATE TABLE t_new (a INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t_new SELECT seq FROM seq_1_to_100; +--source include/kill_mysqld.inc + +--let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err +--let SEARCH_PATTERN= InnoDB: Plugin initialization aborted at srv0start\.cc\[\d+\] with error + +--echo # Synchronous read of a system tablespace page by dict_boot() +--let $restart_parameters= --debug-dbug=+d,recv_corrupt_system_read +--source include/start_mysqld.inc +--source include/search_pattern_in_file.inc + +--echo # Asynchronous read of a user tablespace page by recv_sys_t::apply() +--let $restart_parameters= --debug-dbug=+d,recv_corrupt_user_read +--source include/restart_mysqld.inc +--source include/search_pattern_in_file.inc + +--echo # Page initialized by the log (fake read in recv_sys_t::apply()) +--let $restart_parameters= --debug-dbug=+d,recv_corrupt_user_init +--source include/restart_mysqld.inc +--source include/search_pattern_in_file.inc + +--echo # First page of a tablespace in recv_sys_t::recover_deferred() +--let $restart_parameters= --debug-dbug=+d,recv_corrupt_user_page0_init +--source include/restart_mysqld.inc +--source include/search_pattern_in_file.inc + +--echo # No log was discarded by the failed attempts +--let $restart_parameters= +--source include/restart_mysqld.inc +CHECK TABLE t_old, t_new; +SELECT COUNT(*) FROM t_old WHERE b=REPEAT('y', 200); +SELECT COUNT(*) FROM t_new; +DROP TABLE t_old, t_new; diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index b892f9e4b41..4df56c8af22 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -3993,6 +3993,35 @@ recv_sys_t::parser recv_sys_t::get_backup_parser() noexcept ut_error; } +#ifndef DBUG_OFF +/** Check whether a debug keyword requests a simulated failure to apply +log to a page. The keywords select the code path that invoked +recv_recover_page(), as follows: +recv_corrupt_system_read: page in the system tablespace that is read from +the data file (buf_page_get_gen() during startup) +recv_corrupt_user_read: page in a user tablespace that is read from +the data file (recv_sys_t::apply() submitting an asynchronous read) +recv_corrupt_user_init: page in a user tablespace that was initialized +by the log (recv_sys_t::apply() submitting os_fake_read()) +recv_corrupt_user_page0_init: first page of a user tablespace that was +initialized by the log (recv_sys_t::recover_deferred()) +@param id page identifier +@param init_lsn page initialization LSN, or 0 if the page was read +@return whether the application of log should be reported as corrupted */ +static bool recv_simulate_corruption(page_id_t id, lsn_t init_lsn) noexcept +{ + if (!id.space()) + return !init_lsn && DBUG_IF("recv_corrupt_system_read"); + if (srv_is_undo_tablespace(id.space())) + return false; + if (!init_lsn) + return DBUG_IF("recv_corrupt_user_read"); + return id.page_no() + ? DBUG_IF("recv_corrupt_user_init") + : DBUG_IF("recv_corrupt_user_page0_init"); +} +#endif + /** Apply the hashed log records to the page, if the page lsn is less than the lsn of a log record. @param[in,out] block buffer pool page @@ -4000,11 +4029,15 @@ lsn of a log record. @param[in,out] recs log records to apply @param[in,out] space tablespace, or NULL if not looked up yet @param[in,out] init_lsn page initialization LSN, or 0 +@param[in] caller_fixed whether the caller holds a buffer-fix on + block, which buf_pool_t::corrupted_evict() + will release on failure @return the recovered page @retval nullptr on failure */ static buf_block_t *recv_recover_page(buf_block_t *block, mtr_t &mtr, page_recv_t &recs, - fil_space_t *space, lsn_t init_lsn = 0) + fil_space_t *space, lsn_t init_lsn, + bool caller_fixed) { mysql_mutex_assert_not_owner(&recv_sys.mutex); ut_ad(recv_sys.apply_log_recs); @@ -4092,6 +4125,17 @@ static buf_block_t *recv_recover_page(buf_block_t *block, mtr_t &mtr, block->page.id().page_no())); log_phys_t::apply_status a= l->apply(*block, recs.last_offset); +#ifndef DBUG_OFF + if (a != log_phys_t::APPLIED_NO + && recv_simulate_corruption(block->page.id(), init_lsn)) { + sql_print_error("InnoDB: Simulating corruption on" + " [page id: space=%" PRIu32 + ", page number=%" PRIu32 "]", + block->page.id().space(), + block->page.id().page_no()); + a = log_phys_t::APPLIED_CORRUPTED; + } +#endif switch (a) { case log_phys_t::APPLIED_NO: @@ -4160,8 +4204,9 @@ static buf_block_t *recv_recover_page(buf_block_t *block, mtr_t &mtr, mtr.commit(); buf_pool.corrupted_evict(&block->page, - block->page.state() & - buf_page_t::LRU_MASK); + (block->page.state() & + buf_page_t::LRU_MASK) + + uint32_t{caller_fixed}); return nullptr; } @@ -4281,7 +4326,8 @@ bool recv_recover_page(fil_space_t* space, buf_page_t* bpage) p->second.being_processed= 1; const lsn_t init_lsn{p->second.skip_read ? mlog_init.last(id) : 0}; mysql_mutex_unlock(&recv_sys.mutex); - success= recv_recover_page(success, mtr, p->second, space, init_lsn); + success= recv_recover_page(success, mtr, p->second, space, init_lsn, + true); p->second.being_processed= -1; goto func_exit; } @@ -4322,11 +4368,19 @@ void IORequest::fake_read_complete(os_offset_t offset) const noexcept ut_ad(init_lsn > 1); if (recv_recover_page(reinterpret_cast(bpage), - mtr, recs, node->space, init_lsn)) + mtr, recs, node->space, init_lsn, false)) { ut_ad(bpage->oldest_modification() || bpage->is_freed()); bpage->lock.x_unlock(true); } + else + { + /* The page was evicted. Prevent the completion of recovery, + so that the log will not be discarded by a checkpoint. */ + mysql_mutex_lock(&recv_sys.mutex); + recv_sys.set_corrupt_fs(); + mysql_mutex_unlock(&recv_sys.mutex); + } recs.being_processed= -1; ut_ad(mtr.has_committed()); @@ -4594,7 +4648,7 @@ inline buf_block_t *recv_sys_t::recover_low(const map::iterator &p, mtr_t &mtr, ut_d(mysql_mutex_lock(&mutex)); ut_ad(&recs == &pages.find(p->first)->second); ut_d(mysql_mutex_unlock(&mutex)); - block= recv_recover_page(block, mtr, recs, space, init_lsn); + block= recv_recover_page(block, mtr, recs, space, init_lsn, false); ut_ad(mtr.has_committed()); if (space) -- 2.56.0