How to replay

1. cd <Top directory with MariaDB binaries>/mysql-test

2. tar xvzf MDEV-23514.tgz

    This unpacks

    - shellscript mysqltest_background.sh

   - test               suite/innodb/t/ml_TBR-631.test

3. ./mtr --mem --parallel=4 -rr -rr-arg='--chaos' --mysqld=--innodb_page_size=16K ml_TBR-631{,,,}

     or similar

4. After observing the test failure you might need to wait lets say 10 seconds and than run

          killall -9 perl mysqld mariadbd mysqltest

     because otherwise certain processes will make useless load for several minutes.

Some notes:

The test and the shape of the MTR call are a brute force attempt to replay the problem.

I assume that neither "rr" nor ASAN build nor parallel > 1 nor the flood of SQL inside of ml_TBR-631 are strict required.

But its quite probable that their use made replays of the problem more likely.

The grammar of the corresponding RQG test is

__clone__3:

    COMMIT |

    ROLLBACK ;

thread1:

    ALTER TABLE t2 ADD PRIMARY KEY ( col_text(9) , col2 ) |

    ALTER TABLE t2 DROP PRIMARY KEY ;

thread1_connect:

    ;

thread1_init:

    CREATE TABLE t2 ( col1 INT, col2 INT, col_int INTEGER , col_string INTEGER, col_text TEXT ) ENGINE = InnoDB ROW_FORMAT = Compressed ;

thread2:

    UPDATE t2 SET col_int = { $my_int= $prng->int( 65, 512) } LIMIT 2 |

    INSERT INTO t2 (col2,col_int,col_string, col_text) VALUES ( { $my_int= $prng->int( 65, 512) } , $my_int, 1111111111, REPEAT(SUBSTR(CAST( $my_int AS CHAR),1,1), @fill_amount) ); __clone__3 ;

thread2_connect:

    SET AUTOCOMMIT = 0; SET @fill_amount = (@@innodb_page_size / 2 ) + 1 ;

thread2_init:

    ;

Thread 30 hit Hardware watchpoint 1: -location $1->online_log

Old value = (row_log_t *) 0x6e957008ac00

New value = (row_log_t *) 0x0

row_log_free (log=@0x6e95700215b8: 0x0)

    at /mariadb/10.2o/storage/innobase/row/row0log.cc:3050

3050	}

(rr) bt

#0  row_log_free (log=@0x6e95700215b8: 0)

    at /mariadb/10.2o/storage/innobase/row/row0log.cc:3049

#1  0x000055d70ca0a472 in innobase_online_rebuild_log_free (

    table=<optimized out>)

    at /mariadb/10.2o/storage/innobase/handler/handler0alter.cc:6391

#2  0x000055d70ca0d0e2 in rollback_inplace_alter_table (

    ha_alter_info=0x79110e27f730, table=0x6e95700394c8, 

    prebuilt=0x6e95700c7e60)

    at /mariadb/10.2o/storage/innobase/handler/handler0alter.cc:6524

#3  0x000055d70ca06c14 in ha_innobase::commit_inplace_alter_table (this=

    0x6e9570049730, altered_table=0x6e95700c8e78, 

    ha_alter_info=0x79110e27f730, commit=false)

…

(rr) thread 3

(rr) bt

…

#19 mtr_t::s_lock (this=0x3f9a30b397c0, lock=0x6e95700216a8, 

    file=0x55d70d07c3b0 "/mariadb/10.2o/storage/innobase/row/row0umod.cc", 

    line=284) at /mariadb/10.2o/storage/innobase/include/mtr0mtr.ic:237

#20 0x000055d70cd17765 in row_undo_mod_clust (node=0x6e956c03e850, 

    thr=0x6e956c039e18) at /mariadb/10.2o/storage/innobase/row/row0umod.cc:284

…

#33 0x000055d70c646e04 in mysql_parse (thd=0x6e956c000d28, rawbuf=0x6e956c0107e0 "ROLLBACK", length=<optimized out>, parser_state=0x3f9a30b3b6d0, is_com_multi=<optimized out>, is_next_command=false) at /mariadb/10.2o/sql/sql_parse.cc:7733

	ut_ad(rw_lock_own_flagged(

			&index->lock,

			RW_LOCK_FLAG_S | RW_LOCK_FLAG_X | RW_LOCK_FLAG_SX));

diff --git a/storage/innobase/row/row0umod.cc b/storage/innobase/row/row0umod.cc

index b1cc994cdd8..0de0760834e 100644

--- a/storage/innobase/row/row0umod.cc

+++ b/storage/innobase/row/row0umod.cc

@@ -319,17 +319,7 @@ row_undo_mod_clust(

 		ut_ad(err == DB_SUCCESS || err == DB_OUT_OF_FILE_SPACE);

 	}

-	/* Online rebuild cannot be initiated while we are holding

-	dict_operation_lock and index->lock. (It can be aborted.) */

-	ut_ad(online || !dict_index_is_online_ddl(index));

-

-	if (err == DB_SUCCESS && online) {

-

-		ut_ad(rw_lock_own_flagged(

-				&index->lock,

-				RW_LOCK_FLAG_S | RW_LOCK_FLAG_X

-				| RW_LOCK_FLAG_SX));

-

+	if (err == DB_SUCCESS && online && dict_index_is_online_ddl(index)) {

 		switch (node->rec_type) {

 		case TRX_UNDO_DEL_MARK_REC:

 			row_log_table_insert(

	if (ctx->online) {

		error = DB_SUCCESS;

	} else {

		error = row_merge_lock_table(

			ctx->prebuilt->trx, ctx->new_table, LOCK_S);

		if (error != DB_SUCCESS) {

			goto error_handling;

		}

	}

diff --git a/storage/innobase/row/row0undo.cc b/storage/innobase/row/row0undo.cc

index 34b55f25527..185a71e670b 100644

--- a/storage/innobase/row/row0undo.cc

+++ b/storage/innobase/row/row0undo.cc

@@ -279,6 +279,19 @@ row_undo(

 			? UNDO_NODE_INSERT : UNDO_NODE_MODIFY;

 	}

+	/* Prevent prepare_inplace_alter_table_dict() from adding

+	dict_table_t::indexes while we are processing the record.

+	Recovered transactions are not protected by MDL, and the

+	secondary index creation is not protected by table locks

+	for online operation. (A table lock would only be acquired

+	when committing the ALTER TABLE operation.) */

+	const bool locked_data_dict = UNIV_UNLIKELY(trx->is_recovered)

+		&& !trx->dict_operation_lock_mode;

+

+	if (UNIV_UNLIKELY(locked_data_dict)) {

+		row_mysql_freeze_data_dictionary(trx);

+	}

+

 	dberr_t err;

 	if (node->state == UNDO_NODE_INSERT) {

@@ -291,6 +304,10 @@ row_undo(

 		err = row_undo_mod(node, thr);

 	}

+	if (UNIV_UNLIKELY(locked_data_dict)) {

+		row_mysql_unfreeze_data_dictionary(trx);

+	}

+

 	/* Do some cleanup */

 	btr_pcur_close(&(node->pcur));