Trying to take a DDL aware backup with mariabackup --lock-ddl-per-table causes backup to wait indefinitely in FLUSH NO_WRITE_TO_BINLOG TABLES if an ALTER TABLE is issued during the backup.
To reproduce, run backup with:
mariabackup --lock-ddl-per-table --backup --user user --password password --stream=xbstream --parallel=4 --tmpdir=/tmp | gzip > backup.xbs.gz
|
While backup is running, issue an alter table to add a column, eg:
alter table testpk add column MARIA_ID bigint unsigned not null auto_increment primary key
|
The backup will hang in FLUSH TABLES forever.
Show processlist shows that there is some kind of conflict between the MDL taken by the lock-ddl-per-table and the flush tables command:
MariaDB [(none)]> show processlist;
|
+----+-------------+-----------+------+---------+------+---------------------------------+--------------------------------------------------------------------------------------------+----------+
|
| Id | User | Host | db | Command | Time | State | Info | Progress |
|
+----+-------------+-----------+------+---------+------+---------------------------------+--------------------------------------------------------------------------------------------+----------+
|
| 1 | system user | | NULL | Daemon | NULL | InnoDB purge worker | NULL | 0.000 |
|
| 2 | system user | | NULL | Daemon | NULL | InnoDB purge worker | NULL | 0.000 |
|
| 3 | system user | | NULL | Daemon | NULL | InnoDB purge worker | NULL | 0.000 |
|
| 4 | system user | | NULL | Daemon | NULL | InnoDB purge coordinator | NULL | 0.000 |
|
| 5 | system user | | NULL | Daemon | NULL | InnoDB shutdown handler | NULL | 0.000 |
|
| 63 | root | localhost | rick | Query | 113 | Waiting for table metadata lock | alter table testpk add column MARIA_ID bigint unsigned not null auto_increment primary key | 0.000 |
|
| 80 | root | localhost | NULL | Query | 52 | Waiting for table flush | FLUSH NO_WRITE_TO_BINLOG TABLES | 0.000 |
|
| 81 | root | localhost | NULL | Sleep | 117 | | NULL | 0.000 |
|
| 82 | root | localhost | NULL | Query | 0 | init | show processlist | 0.000 |
|
+----+-------------+-----------+------+---------+------+---------------------------------+--------------------------------------------------------------------------------------------+----------+
|
9 rows in set (0.00 sec)
|
|
Thread dump show what looks like a metadata deadlock, both threads waiting in MDL_wait:
10 libaio::??(libaio.so.1),LinuxAIOHandler::collect(os0file.cc:1893),LinuxAIOHandler::poll(os0file.cc:2038),os_aio_linux_handler(os0file.cc:2092),os_aio_handler(os0file.cc:2092),fil_aio_wait(fil0fil.cc:5330),io_handler_thread(srv0start.cc:343),start_thread,clone
|
3 pthread_cond_wait,wait(os0event.cc:166),wait_low(os0event.cc:166),os_event_wait_low(os0event.cc:166),srv_resume_thread(srv0srv.cc:935),srv_worker_thread(srv0srv.cc:935),start_thread,clone
|
1 pthread_cond_wait,wait(os0event.cc:166),wait_low(os0event.cc:166),os_event_wait_low(os0event.cc:166),srv_resume_thread(srv0srv.cc:935),srv_purge_coordinator_suspend(srv0srv.cc:935),srv_purge_coordinator_thread(srv0srv.cc:935),start_thread,clone
|
1 pthread_cond_wait,wait(os0event.cc:166),wait_low(os0event.cc:166),os_event_wait_low(os0event.cc:166),buf_resize_thread(buf0buf.cc:3083),start_thread,clone
|
1 pthread_cond_wait,wait(os0event.cc:166),wait_low(os0event.cc:166),os_event_wait_low(os0event.cc:166),buf_dump_thread(buf0dump.cc:777),start_thread,clone
|
1 pthread_cond_wait,inline_mysql_cond_wait(mysql_thread.h:1149),inline_mysql_cond_wait(mysql_thread.h:1149),thd_destructor_proxy(mysql_thread.h:1149),start_thread,clone
|
1 pthread_cond_wait,inline_mysql_cond_wait(mysql_thread.h:1149),inline_mysql_cond_wait(mysql_thread.h:1149),handle_slave_background(mysql_thread.h:1149),start_thread,clone
|
1 pthread_cond_timedwait,os_event::timed_wait(os0event.cc:286),os_event::wait_time_low(os0event.cc:413),srv_monitor_thread(srv0srv.cc:1754),start_thread,clone
|
1 pthread_cond_timedwait,os_event::timed_wait(os0event.cc:286),os_event::wait_time_low(os0event.cc:413),srv_error_monitor_thread(srv0srv.cc:1923),start_thread,clone
|
1 pthread_cond_timedwait,os_event::timed_wait(os0event.cc:286),os_event::wait_time_low(os0event.cc:413),pc_sleep_if_needed(buf0flu.cc:2718),buf_flush_page_cleaner_coordinator(buf0flu.cc:2718),start_thread,clone
|
1 pthread_cond_timedwait,os_event::timed_wait(os0event.cc:286),os_event::wait_time_low(os0event.cc:413),lock_wait_timeout_thread(lock0wait.cc:547),start_thread,clone
|
1 pthread_cond_timedwait,os_event::timed_wait(os0event.cc:286),os_event::wait_time_low(os0event.cc:413),ib_wqueue_timedwait(ut0wqueue.cc:163),fts_optimize_thread(fts0opt.cc:3032),start_thread,clone
|
1 pthread_cond_timedwait,os_event::timed_wait(os0event.cc:286),os_event::wait_time_low(os0event.cc:413),dict_stats_thread(dict0stats_bg.cc:451),start_thread,clone
|
1 pthread_cond_timedwait,inline_mysql_cond_timedwait(mysql_thread.h:1186),timer_handler(mysql_thread.h:1186),start_thread,clone
|
1 pthread_cond_timedwait,inline_mysql_cond_timedwait(mysql_thread.h:1186),my_service_thread_sleep(mysql_thread.h:1186),ma_checkpoint_background(ma_checkpoint.c:709),start_thread,clone
|
1 pthread_cond_timedwait,inline_mysql_cond_timedwait(mysql_thread.h:1186),MDL_wait::timed_wait(mysql_thread.h:1186),TABLE_SHARE::wait_for_old_version(table.cc:4410),close_cached_tables(sql_base.cc:454),reload_acl_and_cache(sql_reload.cc:334),mysql_execute_command(sql_parse.cc:5384),mysql_parse(sql_parse.cc:7902),dispatch_command(sql_parse.cc:1806),do_command(sql_parse.cc:1360),do_handle_one_connection(sql_connect.cc:1335),handle_one_connection(sql_connect.cc:1241),start_thread,clone
|
1 pthread_cond_timedwait,inline_mysql_cond_timedwait(mysql_thread.h:1186),MDL_wait::timed_wait(mysql_thread.h:1186),MDL_context::acquire_lock(mdl.cc:2138),MDL_context::upgrade_shared_lock(mdl.cc:2333),mysql_inplace_alter_table(sql_table.cc:7245),mysql_alter_table(sql_table.cc:7245),Sql_cmd_alter_table::execute(sql_alter.cc:324),mysql_execute_command(sql_parse.cc:6208),mysql_parse(sql_parse.cc:7902),dispatch_command(sql_parse.cc:1806),do_command(sql_parse.cc:1360),do_handle_one_connection(sql_connect.cc:1335),handle_one_connection(sql_connect.cc:1241),start_thread,clone
|
1 pthread_cond_timedwait,inline_mysql_cond_timedwait(mysql_thread.h:1186),inline_mysql_cond_timedwait(mysql_thread.h:1186),cache_thread(mysql_thread.h:1186),one_thread_per_connection_end(mysql_thread.h:1186),do_handle_one_connection(sql_connect.cc:1354),handle_one_connection(sql_connect.cc:1241),start_thread,clone
|
1 __poll,vio_io_wait(viosocket.c:945),vio_socket_io_wait(viosocket.c:108),vio_read(viosocket.c:184),my_real_read(net_serv.cc:892),my_net_read_packet_reallen(net_serv.cc:1162),my_net_read_packet(net_serv.cc:1146),do_command(sql_parse.cc:1242),do_handle_one_connection(sql_connect.cc:1335),handle_one_connection(sql_connect.cc:1241),start_thread,clone
|
1 __poll,handle_connections_sockets(mysqld.cc:6584),mysqld_main(mysqld.cc:6050),__libc_start_main,_start
|
1 nanosleep,os_thread_sleep(os0thread.cc:230),srv_master_sleep(srv0srv.cc:2412),srv_master_thread(srv0srv.cc:2412),start_thread,clone
|
1 nanosleep,os_thread_sleep(os0thread.cc:230),btr_defragment_thread(btr0defragment.cc:772),start_thread,clone
|
1 do_sigwait,__sigwait,signal_hand(mysqld.cc:3505),start_thread,clone
|
{"report":{"fcp":1045.7999999970198,"ttfb":396.6000000089407,"pageVisibility":"visible","entityId":66457,"key":"jira.project.issue.view-issue","isInitial":true,"threshold":1000,"elementTimings":{},"userDeviceMemory":8,"userDeviceProcessors":64,"apdex":0.5,"journeyId":"18b809c6-6595-499a-bebc-5e3ac3f49c8b","navigationType":0,"readyForUser":1127.2000000029802,"redirectCount":0,"resourceLoadedEnd":1150.7000000029802,"resourceLoadedStart":402.1000000089407,"resourceTiming":[{"duration":93.5,"initiatorType":"link","name":"https://jira.mariadb.org/s/2c21342762a6a02add1c328bed317ffd-CDN/lu2bv2/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/css/_super/batch.css","startTime":402.1000000089407,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":402.1000000089407,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":495.6000000089407,"responseStart":0,"secureConnectionStart":0},{"duration":94.20000000298023,"initiatorType":"link","name":"https://jira.mariadb.org/s/7ebd35e77e471bc30ff0eba799ebc151-CDN/lu2bv2/820016/12ta74/2380add21a9a1006587582385952de73/_/download/contextbatch/css/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true","startTime":402.40000000596046,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":402.40000000596046,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":496.6000000089407,"responseStart":0,"secureConnectionStart":0},{"duration":158.5,"initiatorType":"script","name":"https://jira.mariadb.org/s/e9b27a47da5fb0f74a35acd57e9847fb-CDN/lu2bv2/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/js/_super/batch.js?locale=en","startTime":402.5,"connectEnd":402.5,"connectStart":402.5,"domainLookupEnd":402.5,"domainLookupStart":402.5,"fetchStart":402.5,"redirectEnd":0,"redirectStart":0,"requestStart":402.5,"responseEnd":561,"responseStart":561,"secureConnectionStart":402.5},{"duration":216.70000000298023,"initiatorType":"script","name":"https://jira.mariadb.org/s/c32eb0da7ad9831253f8397e6cc26afd-CDN/lu2bv2/820016/12ta74/2380add21a9a1006587582385952de73/_/download/contextbatch/js/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true","startTime":402.70000000298023,"connectEnd":402.70000000298023,"connectStart":402.70000000298023,"domainLookupEnd":402.70000000298023,"domainLookupStart":402.70000000298023,"fetchStart":402.70000000298023,"redirectEnd":0,"redirectStart":0,"requestStart":402.70000000298023,"responseEnd":619.4000000059605,"responseStart":619.4000000059605,"secureConnectionStart":402.70000000298023},{"duration":220,"initiatorType":"script","name":"https://jira.mariadb.org/s/bc0bcb146314416123c992714ee00ff7-CDN/lu2bv2/820016/12ta74/c92c0caa9a024ae85b0ebdbed7fb4bd7/_/download/contextbatch/js/atl.global,-_super/batch.js?locale=en","startTime":402.90000000596046,"connectEnd":402.90000000596046,"connectStart":402.90000000596046,"domainLookupEnd":402.90000000596046,"domainLookupStart":402.90000000596046,"fetchStart":402.90000000596046,"redirectEnd":0,"redirectStart":0,"requestStart":402.90000000596046,"responseEnd":622.9000000059605,"responseStart":622.9000000059605,"secureConnectionStart":402.90000000596046},{"duration":220.19999998807907,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bv2/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-en/jira.webresources:calendar-en.js","startTime":403.1000000089407,"connectEnd":403.1000000089407,"connectStart":403.1000000089407,"domainLookupEnd":403.1000000089407,"domainLookupStart":403.1000000089407,"fetchStart":403.1000000089407,"redirectEnd":0,"redirectStart":0,"requestStart":403.1000000089407,"responseEnd":623.2999999970198,"responseStart":623.2999999970198,"secureConnectionStart":403.1000000089407},{"duration":220.40000000596046,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bv2/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-localisation-moment/jira.webresources:calendar-localisation-moment.js","startTime":403.29999999701977,"connectEnd":403.29999999701977,"connectStart":403.29999999701977,"domainLookupEnd":403.29999999701977,"domainLookupStart":403.29999999701977,"fetchStart":403.29999999701977,"redirectEnd":0,"redirectStart":0,"requestStart":403.29999999701977,"responseEnd":623.7000000029802,"responseStart":623.7000000029802,"secureConnectionStart":403.29999999701977},{"duration":297.79999999701977,"initiatorType":"link","name":"https://jira.mariadb.org/s/b04b06a02d1959df322d9cded3aeecc1-CDN/lu2bv2/820016/12ta74/a2ff6aa845ffc9a1d22fe23d9ee791fc/_/download/contextbatch/css/jira.global.look-and-feel,-_super/batch.css","startTime":403.40000000596046,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":403.40000000596046,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":701.2000000029802,"responseStart":0,"secureConnectionStart":0},{"duration":220.5,"initiatorType":"script","name":"https://jira.mariadb.org/rest/api/1.0/shortcuts/820016/47140b6e0a9bc2e4913da06536125810/shortcuts.js?context=issuenavigation&context=issueaction","startTime":403.6000000089407,"connectEnd":403.6000000089407,"connectStart":403.6000000089407,"domainLookupEnd":403.6000000089407,"domainLookupStart":403.6000000089407,"fetchStart":403.6000000089407,"redirectEnd":0,"redirectStart":0,"requestStart":403.6000000089407,"responseEnd":624.1000000089407,"responseStart":624.1000000089407,"secureConnectionStart":403.6000000089407},{"duration":297.6000000089407,"initiatorType":"link","name":"https://jira.mariadb.org/s/3ac36323ba5e4eb0af2aa7ac7211b4bb-CDN/lu2bv2/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/css/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.css?jira.create.linked.issue=true","startTime":403.79999999701977,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":403.79999999701977,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":701.4000000059605,"responseStart":0,"secureConnectionStart":0},{"duration":220.79999999701977,"initiatorType":"script","name":"https://jira.mariadb.org/s/719848dd97ebe0663199f49a3936487a-CDN/lu2bv2/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/js/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.js?jira.create.linked.issue=true&locale=en","startTime":403.90000000596046,"connectEnd":403.90000000596046,"connectStart":403.90000000596046,"domainLookupEnd":403.90000000596046,"domainLookupStart":403.90000000596046,"fetchStart":403.90000000596046,"redirectEnd":0,"redirectStart":0,"requestStart":403.90000000596046,"responseEnd":624.7000000029802,"responseStart":624.7000000029802,"secureConnectionStart":403.90000000596046},{"duration":376.90000000596046,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bv2/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-js/jira.webresources:bigpipe-js.js","startTime":404.79999999701977,"connectEnd":404.79999999701977,"connectStart":404.79999999701977,"domainLookupEnd":404.79999999701977,"domainLookupStart":404.79999999701977,"fetchStart":404.79999999701977,"redirectEnd":0,"redirectStart":0,"requestStart":404.79999999701977,"responseEnd":781.7000000029802,"responseStart":781.7000000029802,"secureConnectionStart":404.79999999701977},{"duration":377.79999999701977,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bv2/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-init/jira.webresources:bigpipe-init.js","startTime":405,"connectEnd":405,"connectStart":405,"domainLookupEnd":405,"domainLookupStart":405,"fetchStart":405,"redirectEnd":0,"redirectStart":0,"requestStart":405,"responseEnd":782.7999999970198,"responseStart":782.7999999970198,"secureConnectionStart":405},{"duration":39.5,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":727.2999999970198,"connectEnd":727.2999999970198,"connectStart":727.2999999970198,"domainLookupEnd":727.2999999970198,"domainLookupStart":727.2999999970198,"fetchStart":727.2999999970198,"redirectEnd":0,"redirectStart":0,"requestStart":727.2999999970198,"responseEnd":766.7999999970198,"responseStart":766.7999999970198,"secureConnectionStart":727.2999999970198},{"duration":196.19999998807907,"initiatorType":"link","name":"https://jira.mariadb.org/s/d5715adaadd168a9002b108b2b039b50-CDN/lu2bv2/820016/12ta74/be4b45e9cec53099498fa61c8b7acba4/_/download/contextbatch/css/jira.project.sidebar,-_super,-project.issue.navigator,-jira.general,-jira.browse.project,-jira.view.issue,-jira.global,-atl.general,-com.atlassian.jira.projects.sidebar.init/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true","startTime":772.6000000089407,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":772.6000000089407,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":968.7999999970198,"responseStart":0,"secureConnectionStart":0},{"duration":374.8999999910593,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bv2/820016/12ta74/e65b778d185daf5aee24936755b43da6/_/download/contextbatch/js/browser-metrics-plugin.contrib,-_super,-project.issue.navigator,-jira.view.issue,-atl.general/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true","startTime":773.6000000089407,"connectEnd":773.6000000089407,"connectStart":773.6000000089407,"domainLookupEnd":773.6000000089407,"domainLookupStart":773.6000000089407,"fetchStart":773.6000000089407,"redirectEnd":0,"redirectStart":0,"requestStart":773.6000000089407,"responseEnd":1148.5,"responseStart":1148.5,"secureConnectionStart":773.6000000089407},{"duration":376.79999999701977,"initiatorType":"script","name":"https://jira.mariadb.org/s/53a43b6764f587426c7bb9a150184c00-CDN/lu2bv2/820016/12ta74/be4b45e9cec53099498fa61c8b7acba4/_/download/contextbatch/js/jira.project.sidebar,-_super,-project.issue.navigator,-jira.general,-jira.browse.project,-jira.view.issue,-jira.global,-atl.general,-com.atlassian.jira.projects.sidebar.init/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true","startTime":773.9000000059605,"connectEnd":773.9000000059605,"connectStart":773.9000000059605,"domainLookupEnd":773.9000000059605,"domainLookupStart":773.9000000059605,"fetchStart":773.9000000059605,"redirectEnd":0,"redirectStart":0,"requestStart":773.9000000059605,"responseEnd":1150.7000000029802,"responseStart":1150.7000000029802,"secureConnectionStart":773.9000000059605},{"duration":205.09999999403954,"initiatorType":"script","name":"https://www.google-analytics.com/analytics.js","startTime":1039.9000000059605,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":1039.9000000059605,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1245,"responseStart":0,"secureConnectionStart":0},{"duration":167.5,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":1047.4000000059605,"connectEnd":1047.4000000059605,"connectStart":1047.4000000059605,"domainLookupEnd":1047.4000000059605,"domainLookupStart":1047.4000000059605,"fetchStart":1047.4000000059605,"redirectEnd":0,"redirectStart":0,"requestStart":1047.4000000059605,"responseEnd":1214.9000000059605,"responseStart":1214.9000000059605,"secureConnectionStart":1047.4000000059605}],"fetchStart":0,"domainLookupStart":0,"domainLookupEnd":0,"connectStart":0,"connectEnd":0,"requestStart":165,"responseStart":397,"responseEnd":398,"domLoading":400,"domInteractive":1199,"domContentLoadedEventStart":1199,"domContentLoadedEventEnd":1239,"domComplete":1486,"loadEventStart":1486,"loadEventEnd":1488,"userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","marks":[{"name":"bigPipe.sidebar-id.start","time":1167.5},{"name":"bigPipe.sidebar-id.end","time":1168.2999999970198},{"name":"bigPipe.activity-panel-pipe-id.start","time":1168.5},{"name":"bigPipe.activity-panel-pipe-id.end","time":1170.7000000029802},{"name":"activityTabFullyLoaded","time":1250.7999999970198}],"measures":[],"correlationId":"22d792b61d4efe","effectiveType":"4g","downlink":10,"rtt":0,"serverDuration":163,"dbReadsTimeInMs":16,"dbConnsTimeInMs":25,"applicationHash":"9d11dbea5f4be3d4cc21f03a88dd11d8c8687422","experiments":[]}}