Please try the uploaded tests.

MDEV-20483-C.test  replays the assert.

MDEV-20483-B.test

              Is too some huge extent like MDEV-20483-C.test.

              It tries in addition first to insert illegal values via INSERT ... VALUES ...

              This gets denied (ERROR 22003: Out of range value for column 'pk' at row 1).

              Than it tries to insert the same ollegal values via INSERT ...  SELECT and harvests

              again ERROR 22003: Out of range value for column 'pk' at row 1

              whereas the same INSERT ... SELECT has success in MDEV-20483-C.test.

That MDEV-20483-C.test shows finally the assert is one thing.

But that MDEV-20483-B.test has just a denied statement in addition and does not

harvest finally the same assert is IMHO quite suspicious.

diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc

index f612be4d177..59a27ee112e 100644

--- a/storage/innobase/trx/trx0trx.cc

+++ b/storage/innobase/trx/trx0trx.cc

@@ -1363,6 +1363,13 @@ trx_commit_in_memory(

 		DBUG_LOG("trx", "Autocommit in memory: " << trx);

 		trx->state = TRX_STATE_NOT_STARTED;

 	} else {

+#ifdef UNIV_DEBUG

+		if (!UT_LIST_GET_LEN(trx->lock.trx_locks)) {

+			for (const lock_t* l : trx->lock.table_locks) {

+				ut_ad(!l);

+			}

+		}

+#endif

 		trx->commit_state();

 		if (trx->id) {

The patch direct above avoids to hit the assertion Assertion `(trx)->lock.table_locks.empty()' failed

but certain tests hit

    mysqld: storage/innobase/trx/trx0trx.cc:1369: void trx_commit_in_memory(trx_t*, const mtr_t*): Assertion `!l' failed.

MTR Based replay test:

--source include/have_innodb.inc

--connect (con15,localhost,root,,)

CREATE TABLE t1 ( col1 INT, pk INTEGER AUTO_INCREMENT, PRIMARY KEY (pk)) ENGINE = InnoDB  ;

INSERT INTO t1 VALUES (NULL, NULL)  ;

--connect (con16,localhost,root,,)

--connection con16

# Without AUTOCOMMIT = OFF no assert!

SET AUTOCOMMIT = OFF  ;

# With LOCK TABLES t1 WRITE (no CONCURRENT) no assert! Just the MDL timout kicks in.

# LOCK TABLES t1 WRITE ;

  LOCK TABLES t1 WRITE CONCURRENT ;

--connection con15

SET SESSION lock_wait_timeout = 2;

SET SESSION innodb_lock_wait_timeout = 1;

# Variant 1 which replays

# SELECT * FROM t1 INTO OUTFILE 'load_t1' ; LOAD DATA INFILE 'load_t1' REPLACE INTO TABLE t1 ;

# Variant 2 which replays

  REPLACE INTO t1 (col1) SELECT col1 FROM (SELECT col1 FROM t1) AS load_tab ;

DROP TABLE t1;

diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc

index b7e17097546..9fdc8feee90 100644

--- a/storage/innobase/lock/lock0lock.cc

+++ b/storage/innobase/lock/lock0lock.cc

@@ -6480,6 +6480,8 @@ lock_cancel_waiting_and_release(

                }

                lock_table_dequeue(lock);

+               /* Remove the lock from table lock vector too. */

+               lock_trx_table_locks_remove(lock);

        }

        /* Reset the wait flag and the back pointer to lock in trx. */

diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc

index 1dcca7c1f72..200335d605e 100644

--- a/storage/innobase/trx/trx0trx.cc

+++ b/storage/innobase/trx/trx0trx.cc

@@ -1695,6 +1695,13 @@ trx_commit_in_memory(

                DBUG_LOG("trx", "Autocommit in memory: " << trx);

                trx->state = TRX_STATE_NOT_STARTED;

        } else {

+#ifdef UNIV_DEBUG

+               if (!UT_LIST_GET_LEN(trx->lock.trx_locks)) {

+                       for (const lock_t* l : trx->lock.table_locks) {

+                               ut_ad(!l);

+                       }

+               }

+#endif

                trx_mutex_enter(trx);

                trx->commit_state();

                trx_mutex_exit(trx);