delayed BF abort with multi-master load

2026-05-19 22:11:43 0x7e1ed0969640  InnoDB: Assertion failure in file /home/seppo/work/wsrep/mariadb-foudation/server/storage/innobase/trx/trx0roll.cc line 68

InnoDB: Failing assertion: error_state == DB_INTERRUPTED

Thread 1 (Thread 0x7e1ed0969640 (LWP 129922)):

#0  0x00007e1ed64969fc in pthread_kill () from /lib/x86_64-linux-gnu/libc.so.6

#1  0x00005f63e19f096c in my_write_core (sig=6) at /home/seppo/work/wsrep/mariadb-foudation/server/mysys/stacktrace.c:424

#2  0x00005f63e101504b in handle_fatal_signal (sig=6) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/signal_handler.cc:298

#3  <signal handler called>

#4  0x00007e1ed64969fc in pthread_kill () from /lib/x86_64-linux-gnu/libc.so.6

#5  0x00007e1ed6442476 in raise () from /lib/x86_64-linux-gnu/libc.so.6

#6  0x00007e1ed64287f3 in abort () from /lib/x86_64-linux-gnu/libc.so.6

#7  0x00005f63e17819ee in ut_dbg_assertion_failed (expr=0x5f63e1faf641 "error_state == DB_INTERRUPTED", file=0x5f63e1faf5f0 "/home/seppo/work/wsrep/mariadb-foudation/server/storage/innobase/trx/trx0roll.cc", line=68) at /home/seppo/work/wsrep/mariadb-foudation/server/storage/innobase/ut/ut0dbg.cc:60

#8  0x00005f63e17561bd in trx_t::rollback_finish (this=0x7e1ec8203a00) at /home/seppo/work/wsrep/mariadb-foudation/server/storage/innobase/trx/trx0roll.cc:68

#9  0x00005f63e175688d in trx_t::rollback_low (this=0x7e1ec8203a00, savept=0x0) at /home/seppo/work/wsrep/mariadb-foudation/server/storage/innobase/trx/trx0roll.cc:126

#10 0x00005f63e1756f49 in trx_t::rollback (this=0x7e1ec8203a00, savept=0x0) at /home/seppo/work/wsrep/mariadb-foudation/server/storage/innobase/trx/trx0roll.cc:181

#11 0x00005f63e16a0956 in row_mysql_handle_errors (new_err=0x7e1ed09667b8, trx=0x7e1ec8203a00, thr=0x761dd001b268, savept=0x0) at /home/seppo/work/wsrep/mariadb-foudation/server/storage/innobase/row/row0mysql.cc:714

#12 0x00005f63e16eb2d7 in row_search_mvcc (buf=0x761dd0019380 "\377", mode=PAGE_CUR_GE, prebuilt=0x761dd001abd8, match_mode=1, direction=0) at /home/seppo/work/wsrep/mariadb-foudation/server/storage/innobase/row/row0sel.cc:5875

#13 0x00005f63e147292b in ha_innobase::index_read (this=0x761dd0018b68, buf=0x761dd0019380 "\377", key_ptr=0x761de8046898 "\210\026\001", key_len=4, find_flag=HA_READ_KEY_EXACT) at /home/seppo/work/wsrep/mariadb-foudation/server/storage/innobase/handler/ha_innodb.cc:8993

#14 0x00005f63e0f4391c in handler::index_read_map (this=0x761dd0018b68, buf=0x761dd0019380 "\377", key=0x761de8046898 "\210\026\001", keypart_map=1, find_flag=HA_READ_KEY_EXACT) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/handler.h:4201

#15 0x00005f63e1022c53 in handler::ha_index_read_map (this=0x761dd0018b68, buf=0x761dd0019380 "\377", key=0x761de8046898 "\210\026\001", keypart_map=1, find_flag=HA_READ_KEY_EXACT) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/handler.cc:3866

#16 0x00005f63e0c94bfe in join_read_always_key (tab=0x761de8045a28) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/sql_select.cc:25141

#17 0x00005f63e0c92cf9 in sub_select (join=0x761de803eb40, join_tab=0x761de8045a28, end_of_records=false) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/sql_select.cc:24349

#18 0x00005f63e0c935e2 in evaluate_join_record (join=0x761de803eb40, join_tab=0x761de80455b8, error=0) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/sql_select.cc:24585

#19 0x00005f63e0c92d6e in sub_select (join=0x761de803eb40, join_tab=0x761de80455b8, end_of_records=false) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/sql_select.cc:24352

#20 0x00005f63e0c91e0b in do_select (join=0x761de803eb40, procedure=0x0) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/sql_select.cc:23863

#21 0x00005f63e0c5c3c1 in JOIN::exec_inner (this=0x761de803eb40) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/sql_select.cc:5102

#22 0x00005f63e0c5b464 in JOIN::exec (this=0x761de803eb40) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/sql_select.cc:4893

#23 0x00005f63e0caef59 in Sql_cmd_dml::execute_inner (this=0x761de803cbe0, thd=0x761de8000dc8) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/sql_select.cc:34641

#24 0x00005f63e0d63e0b in Sql_cmd_update::execute_inner (this=0x761de803cbe0, thd=0x761de8000dc8) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/sql_update.cc:3181

#25 0x00005f63e0caec74 in Sql_cmd_dml::execute (this=0x761de803cbe0, thd=0x761de8000dc8) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/sql_select.cc:34573

#26 0x00005f63e0be3c4b in mysql_execute_command (thd=0x761de8000dc8, is_called_from_prepared_stmt=false) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/sql_parse.cc:4461

#27 0x00005f63e0bef89b in mysql_parse (thd=0x761de8000dc8, rawbuf=0x761de802dcc0 "UPDATE t3 SET count = count + 1 \n      WHERE t2_id IN (\n          SELECT id FROM t2 WHERE t1_id = 51\n      ) LIMIT 5", length=116, parser_state=0x7e1ed09682f0) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/sql_parse.cc:7945

#28 0x00005f63e0beef37 in wsrep_mysql_parse (thd=0x761de8000dc8, rawbuf=0x761de802dcc0 "UPDATE t3 SET count = count + 1 \n      WHERE t2_id IN (\n          SELECT id FROM t2 WHERE t1_id = 51\n      ) LIMIT 5", length=116, parser_state=0x7e1ed09682f0) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/sql_parse.cc:7755

#29 0x00005f63e0bdbc0b in dispatch_command (command=COM_QUERY, thd=0x761de8000dc8, packet=0x761de800bd99 "      UPDATE t3 SET count = count + 1 \n      WHERE t2_id IN (\n          SELECT id FROM t2 WHERE t1_id = 51\n      ) LIMIT 5\n   ", packet_length=126, blocking=true) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/sql_parse.cc:1912

#30 0x00005f63e0bda60f in do_command (thd=0x761de8000dc8, blocking=true) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/sql_parse.cc:1433

#31 0x00005f63e0de2c4e in do_handle_one_connection (connect=0x5f6404a22ff8, put_in_cache=true) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/sql_connect.cc:1497

#32 0x00005f63e0de29c3 in handle_one_connection (arg=0x5f6404a22ff8) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/sql_connect.cc:1409

#0  0x00007e1ed6491117 in ?? () from /lib/x86_64-linux-gnu/libc.so.6

#1  0x00007e1ed6493e9b in pthread_cond_timedwait () from /lib/x86_64-linux-gnu/libc.so.6

#2  0x00005f63e19f514f in safe_cond_timedwait (cond=0x7e1ec8200c90, mp=0x5f63e325e580 <lock_sys+192>, abstime=0x7e1ed2cb4c50, file=0x5f63e1eeca10 "/home/seppo/work/wsrep/mariadb-foudation/server/storage/innobase/lock/lock0lock.cc", 

    line=2382) at /home/seppo/work/wsrep/mariadb-foudation/server/mysys/thr_mutex.c:543

#3  0x00005f63e15278d1 in lock_wait (thr=0x7e1eb0ddb248) at /home/seppo/work/wsrep/mariadb-foudation/server/storage/innobase/lock/lock0lock.cc:2382

#4  0x00005f63e16a462c in row_update_cascade_for_mysql (thr=0x7e1eb0ddb248, node=0x7e1eb0dcd678, table=0x7e1eb04925a8) at /home/seppo/work/wsrep/mariadb-foudation/server/storage/innobase/row/row0mysql.cc:1997

#5  0x00005f63e16714d9 in row_ins_foreign_check_on_constraint (thr=0x7e1eb0ddb248, foreign=0x7e1eb048dfa8, pcur=0x7e1ed2cb5050, entry=0x7e1eb0ddef38, mtr=0x7e1ed2cb53b0)

    at /home/seppo/work/wsrep/mariadb-foudation/server/storage/innobase/row/row0ins.cc:1354

#6  0x00005f63e16725b3 in row_ins_check_foreign_constraint (check_ref=0, foreign=0x7e1eb048dfa8, table=0x7e1eb0488968, entry=0x7e1eb0ddef38, thr=0x7e1eb0ddb248)

    at /home/seppo/work/wsrep/mariadb-foudation/server/storage/innobase/row/row0ins.cc:1747

#7  0x00005f63e1700cb3 in row_upd_check_references_constraints (node=0x7e1eb049a080, pcur=0x7e1eb049a1a8, table=0x7e1eb0488968, index=0x7e1eb048f738, offsets=0x7e1ed2cb5960, thr=0x7e1eb0ddb248, mtr=0x7e1ed2cb5bc0)

    at /home/seppo/work/wsrep/mariadb-foudation/server/storage/innobase/row/row0upd.cc:252

#8  0x00005f63e17089e4 in row_upd_del_mark_clust_rec (node=0x7e1eb049a080, index=0x7e1eb048f738, offsets=0x7e1ed2cb5960, thr=0x7e1eb0ddb248, referenced=true, foreign=false, mtr=0x7e1ed2cb5bc0)

    at /home/seppo/work/wsrep/mariadb-foudation/server/storage/innobase/row/row0upd.cc:2494

#9  0x00005f63e17091f1 in row_upd_clust_step (node=0x7e1eb049a080, thr=0x7e1eb0ddb248) at /home/seppo/work/wsrep/mariadb-foudation/server/storage/innobase/row/row0upd.cc:2628

#10 0x00005f63e17097f5 in row_upd (node=0x7e1eb049a080, thr=0x7e1eb0ddb248) at /home/seppo/work/wsrep/mariadb-foudation/server/storage/innobase/row/row0upd.cc:2750

#11 0x00005f63e1709df1 in row_upd_step (thr=0x7e1eb0ddb248) at /home/seppo/work/wsrep/mariadb-foudation/server/storage/innobase/row/row0upd.cc:2892

#12 0x00005f63e16a36aa in row_update_for_mysql (prebuilt=0x7e1eb0499648) at /home/seppo/work/wsrep/mariadb-foudation/server/storage/innobase/row/row0mysql.cc:1678

#13 0x00005f63e1471cf9 in ha_innobase::delete_row (this=0x7e1eb04975d8, record=0x7e1eb0497df0 "\375\063") at /home/seppo/work/wsrep/mariadb-foudation/server/storage/innobase/handler/ha_innodb.cc:8674

#14 0x00005f63e10328f4 in handler::ha_delete_row (this=0x7e1eb04975d8, buf=0x7e1eb0497df0 "\375\063") at /home/seppo/work/wsrep/mariadb-foudation/server/sql/handler.cc:8205

#15 0x00005f63e11e66c5 in Delete_rows_log_event::do_exec_row (this=0x7e1eb0d95448, rgi=0x7e1eb00299d0) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/log_event_server.cc:7772

#16 0x00005f63e11ddf11 in Rows_log_event::do_apply_event (this=0x7e1eb0d95448, rgi=0x7e1eb00299d0) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/log_event_server.cc:5178

#17 0x00005f63e11c5d98 in Log_event::apply_event (this=0x7e1eb0d95448, rgi=0x7e1eb00299d0) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/log_event.cc:3951

#18 0x00005f63e142b5a9 in wsrep_apply_events (thd=0x7e1eb0001008, rli=0x7e1eb001b050, events_buf=0x7e1ed11993c8, buf_len=4580) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/wsrep_applier.cc:343

#19 0x00005f63e13ee016 in apply_events (thd=0x7e1eb0001008, rli=0x7e1eb001b050, data=..., err=..., include_msg=true) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/wsrep_high_priority_service.cc:127

#20 0x00005f63e13f06bd in Wsrep_applier_service::apply_write_set (this=0x7e1ed2cb7ba0, ws_meta=..., data=..., err=...) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/wsrep_high_priority_service.cc:612

#21 0x00005f63e1ad8189 in apply_write_set (server_state=..., high_priority_service=..., ws_handle=..., ws_meta=..., data=...) at /home/seppo/work/wsrep/mariadb-foudation/server/wsrep-lib/src/server_state.cpp:333

#22 0x00005f63e1adc1b3 in wsrep::server_state::on_apply (this=0x5f64049d15c0, high_priority_service=..., ws_handle=..., ws_meta=..., data=...) at /home/seppo/work/wsrep/mariadb-foudation/server/wsrep-lib/src/server_state.cpp:1132

#23 0x00005f63e1b04e05 in wsrep::high_priority_service::apply (this=0x7e1ed2cb7ba0, ws_handle=..., ws_meta=..., data=...) at /home/seppo/work/wsrep/mariadb-foudation/server/wsrep-lib/include/wsrep/high_priority_service.hpp:47

#24 0x00005f63e1b01651 in (anonymous namespace)::apply_cb (ctx=0x7e1ed2cb7ba0, wsh=0x7e1ed2cb6cd0, flags=73, buf=0x7e1ed2cb6cf0, meta=0x7e1ed2cb6fb0, exit_loop=0x7e1ed2cb6f3d)

    at /home/seppo/work/wsrep/mariadb-foudation/server/wsrep-lib/src/wsrep_provider_v26.cpp:510

#25 0x00007e1ecbc3ac5d in galera::TrxHandleSlave::apply (this=0x7e1eb0dab1d0, recv_ctx=0x7e1ed2cb7ba0, 

    apply_cb=0x5f63e1b01415 <(anonymous namespace)::apply_cb(void*, wsrep_ws_handle_t const*, uint32_t, wsrep_buf_t const*, wsrep_trx_meta_t const*, wsrep_bool_t*)>, meta=..., exit_loop=@0x7e1ed2cb6f3d: false)

    at /home/seppo/work/galera-bugs/galera/src/trx_handle.cpp:396

#26 0x00007e1ecbc5d5cc in galera::ReplicatorSMM::apply_trx (this=0x5f64049e7440, recv_ctx=0x7e1ed2cb7ba0, ts=...) at /home/seppo/work/galera-bugs/galera/src/replicator_smm.cpp:518

#27 0x00007e1ecbc68219 in galera::ReplicatorSMM::process_trx (this=0x5f64049e7440, recv_ctx=0x7e1ed2cb7ba0, ts_ptr=...) at /home/seppo/work/galera-bugs/galera/src/replicator_smm.cpp:2154

#28 0x00007e1ecbcb65bb in galera::GcsActionSource::process_writeset (this=0x5f64049dfdd0, recv_ctx=0x7e1ed2cb7ba0, act=..., exit_loop=@0x7e1ed2cb791a: false) at /home/seppo/work/galera-bugs/galera/src/gcs_action_source.cpp:62

#29 0x00007e1ecbcb6d24 in galera::GcsActionSource::dispatch (this=0x5f64049dfdd0, recv_ctx=0x7e1ed2cb7ba0, act=..., exit_loop=@0x7e1ed2cb791a: false) at /home/seppo/work/galera-bugs/galera/src/gcs_action_source.cpp:110

#30 0x00007e1ecbcb7565 in galera::GcsActionSource::process (this=0x5f64049dfdd0, recv_ctx=0x7e1ed2cb7ba0, exit_loop=@0x7e1ed2cb791a: false) at /home/seppo/work/galera-bugs/galera/src/gcs_action_source.cpp:186

#31 0x00007e1ecbc5c976 in galera::ReplicatorSMM::async_recv (this=0x5f64049e7440, recv_ctx=0x7e1ed2cb7ba0) at /home/seppo/work/galera-bugs/galera/src/replicator_smm.cpp:405

#32 0x00007e1ecbc19db7 in galera_recv (gh=0x5f64049d20a0, recv_ctx=0x7e1ed2cb7ba0) at /home/seppo/work/galera-bugs/galera/src/wsrep_provider.cpp:263

#33 0x00005f63e1b02d7a in wsrep::wsrep_provider_v26::run_applier (this=0x5f6404850fb0, applier_ctx=0x7e1ed2cb7ba0) at /home/seppo/work/wsrep/mariadb-foudation/server/wsrep-lib/src/wsrep_provider_v26.cpp:899

#34 0x00005f63e142cc47 in wsrep_replication_process (thd=0x7e1eb0001008, arg=0x5f6404a20450) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/wsrep_thd.cc:57

#35 0x00005f63e1410714 in start_wsrep_THD (arg=0x5f6404a20450) at /home/seppo/work/wsrep/mariadb-foudation/server/sql/wsrep_mysqld.cc:4109