I encountered the following crash when running tests locally:
|
10.2 842616532a76cd078ba1d526ae4b85d3ad4be069
|
innodb.xa_recovery 'innodb' w34 [ fail ] Found warnings/errors in server log file!
|
Test ended at 2020-09-29 10:10:40
|
line
|
mysqld: /mariadb/10.2o/storage/innobase/include/sync0rw.ic:284: void rw_lock_s_lock_func(rw_lock_t*, ulint, const char*, unsigned int): Assertion `!rw_lock_own_flagged(lock, RW_LOCK_FLAG_X | RW_LOCK_FLAG_S)' failed.
|
…
|
#7 0x00007fe19f0835b2 in __GI___assert_fail (assertion=assertion@entry=0x561773fc7158 "!rw_lock_own_flagged(lock, RW_LOCK_FLAG_X | RW_LOCK_FLAG_S)", file=file@entry=0x561773fb3980 "/mariadb/10.2o/storage/innobase/include/sync0rw.ic", line=line@entry=284, function=function@entry=0x561773fc7110 "void rw_lock_s_lock_func(rw_lock_t*, ulint, const char*, unsigned int)") at assert.c:101
|
#8 0x0000561773bc1d2b in rw_lock_s_lock_func (line=1183, file_name=0x561774022b88 "/mariadb/10.2o/storage/innobase/btr/btr0cur.cc", pass=0, lock=0x561775d11918) at /mariadb/10.2o/storage/innobase/include/sync0rw.ic:286
|
#9 pfs_rw_lock_s_lock_func (lock=lock@entry=0x561775d11918, pass=pass@entry=0, file_name=file_name@entry=0x561774022b88 "/mariadb/10.2o/storage/innobase/btr/btr0cur.cc", line=line@entry=1183) at /mariadb/10.2o/storage/innobase/include/sync0rw.ic:631
|
#10 0x0000561773bd9aab in mtr_t::s_lock (line=1183, file=0x561774022b88 "/mariadb/10.2o/storage/innobase/btr/btr0cur.cc", lock=0x561775d11918, this=0x7fe1837fa8e0) at /mariadb/10.2o/storage/innobase/include/mtr0mtr.ic:237
|
#11 btr_cur_search_to_nth_level_func (index=index@entry=0x561775d117b0, level=level@entry=0, tuple=tuple@entry=0x7fe1880012d0, mode=mode@entry=PAGE_CUR_GE, latch_mode=<optimized out>, latch_mode@entry=2, cursor=cursor@entry=0x7fe1837fa7e8, has_search_latch=0, file=0x561773fc8df8 "/mariadb/10.2o/storage/innobase/ibuf/ibuf0ibuf.cc", line=4332, mtr=0x7fe1837fa8e0, autoinc=0) at /mariadb/10.2o/storage/innobase/btr/btr0cur.cc:1183
|
#12 0x00005617739ea387 in btr_pcur_open_low (index=0x561775d117b0, level=<optimized out>, level@entry=0, tuple=tuple@entry=0x7fe1880012d0, mode=mode@entry=PAGE_CUR_GE, latch_mode=latch_mode@entry=2, cursor=cursor@entry=0x7fe1837fa7e8, file=<optimized out>, line=<optimized out>, autoinc=0, mtr=0x7fe1837fa8e0) at /mariadb/10.2o/storage/innobase/include/btr0pcur.ic:457
|
#13 0x00005617739ee5fa in ibuf_delete_recs (page_id=<optimized out>) at /mariadb/10.2o/storage/innobase/ibuf/ibuf0ibuf.cc:4332
|
#14 0x00005617739eec38 in ibuf_merge_or_delete_for_page (block=block@entry=0x7fe1995cee70, page_id={m_space = 0, m_page_no = 835}, page_size=page_size@entry=0x7fe1995cee80, update_ibuf_bitmap=update_ibuf_bitmap@entry=1) at /mariadb/10.2o/storage/innobase/ibuf/ibuf0ibuf.cc:4480
|
#15 0x0000561773c10b3c in buf_page_io_complete (bpage=bpage@entry=0x7fe1995cee70, dblwr=dblwr@entry=false, evict=evict@entry=false) at /mariadb/10.2o/storage/innobase/buf/buf0buf.cc:6206
|
#16 0x0000561773c3845a in buf_read_page_low (err=err@entry=0x7fe1837fbaa4, sync=sync@entry=true, type=type@entry=0, mode=mode@entry=132, page_id={m_space = 0, m_page_no = 835}, page_size=@0x7fe1837fc130: {m_physical = 16384, m_logical = 16384, m_is_compressed = 0}, unzip=false, ignore_missing_space=false) at /mariadb/10.2o/storage/innobase/buf/buf0rea.cc:215
|
#17 0x0000561773c38d9e in buf_read_page (page_id={m_space = 0, m_page_no = 835}, page_size=@0x7fe1837fc130: {m_physical = 16384, m_logical = 16384, m_is_compressed = 0}) at /mariadb/10.2o/storage/innobase/buf/buf0rea.cc:412
|
#18 0x0000561773c0db4e in buf_page_get_low (page_id=page_id@entry={m_space = 0, m_page_no = 835}, page_size=@0x7fe1837fc130: {m_physical = 16384, m_logical = 16384, m_is_compressed = 0}, rw_latch=rw_latch@entry=8, guess=<optimized out>, guess@entry=0x0, mode=mode@entry=10, file=file@entry=0x561773fc8df8 "/mariadb/10.2o/storage/innobase/ibuf/ibuf0ibuf.cc", line=4786, mtr=0x7fe1837fd050, err=0x7fe1837fc104) at /mariadb/10.2o/storage/innobase/buf/buf0buf.cc:4387
|
#19 0x0000561773c0f22e in buf_page_get_gen (page_id={m_space = 0, m_page_no = 835}, page_size=@0x7fe1837fc130: {m_physical = 16384, m_logical = 16384, m_is_compressed = 0}, rw_latch=rw_latch@entry=8, guess=guess@entry=0x0, mode=mode@entry=10, file=file@entry=0x561773fc8df8 "/mariadb/10.2o/storage/innobase/ibuf/ibuf0ibuf.cc", line=4786, mtr=0x7fe1837fd050, err=0x7fe1837fc104) at /mariadb/10.2o/storage/innobase/buf/buf0buf.cc:4907
|
#20 0x0000561773bda515 in btr_cur_search_to_nth_level_func (index=index@entry=0x561775d117b0, level=level@entry=0, tuple=tuple@entry=0x7fe188001040, mode=mode@entry=PAGE_CUR_GE, latch_mode=<optimized out>, latch_mode@entry=2, cursor=cursor@entry=0x7fe1837fcf50, has_search_latch=<optimized out>, file=<optimized out>, line=<optimized out>, mtr=<optimized out>, autoinc=<optimized out>) at /mariadb/10.2o/storage/innobase/btr/btr0cur.cc:1281
|
#21 0x0000561773be5fcc in btr_pcur_open_low (index=0x561775d117b0, level=<optimized out>, level@entry=0, tuple=tuple@entry=0x7fe188001040, mode=mode@entry=PAGE_CUR_GE, latch_mode=latch_mode@entry=2, cursor=cursor@entry=0x7fe1837fcf50, file=<optimized out>, line=<optimized out>, autoinc=0, mtr=0x7fe1837fd050) at /mariadb/10.2o/storage/innobase/include/btr0pcur.ic:457
|
#22 0x0000561773be6192 in btr_pcur_open_on_user_rec_func (index=<optimized out>, tuple=tuple@entry=0x7fe188001040, mode=mode@entry=PAGE_CUR_GE, latch_mode=latch_mode@entry=2, cursor=cursor@entry=0x7fe1837fcf50, file=file@entry=0x561773fc8df8 "/mariadb/10.2o/storage/innobase/ibuf/ibuf0ibuf.cc", line=4786, mtr=0x7fe1837fd050) at /mariadb/10.2o/storage/innobase/btr/btr0pcur.cc:604
|
#23 0x00005617739ef9a4 in ibuf_delete_for_discarded_space (space=37) at /mariadb/10.2o/storage/innobase/ibuf/ibuf0ibuf.cc:4786
|
#24 0x0000561773c3a133 in buf_read_ibuf_merge_pages (sync=sync@entry=false, space_ids=space_ids@entry=0x7fe1837fd7a0, page_nos=page_nos@entry=0x7fe1837fd760, n_stored=1) at /mariadb/10.2o/storage/innobase/buf/buf0rea.cc:846
|
#25 0x00005617739ed1ac in ibuf_merge_pages (n_pages=0x7fe1837fddf8, sync=sync@entry=false) at /mariadb/10.2o/storage/innobase/ibuf/ibuf0ibuf.cc:2519
|
#26 0x00005617739ed1f5 in ibuf_merge (n_pages=n_pages@entry=0x7fe1837fddf8, sync=sync@entry=false) at /mariadb/10.2o/storage/innobase/ibuf/ibuf0ibuf.cc:2624
|
#27 0x00005617739f10e3 in ibuf_merge_in_background (full=full@entry=false) at /mariadb/10.2o/storage/innobase/ibuf/ibuf0ibuf.cc:2695
|
#28 0x0000561773b395c0 in srv_master_do_active_tasks () at /mariadb/10.2o/storage/innobase/srv/srv0srv.cc:2234
|
#29 0x0000561773b3c036 in srv_master_thread (arg=<optimized out>) at /mariadb/10.2o/storage/innobase/srv/srv0srv.cc:2468
|
#30 0x00007fe19f549ea7 in start_thread (arg=<optimized out>) at pthread_create.c:477
|
#31 0x00007fe19f14ceaf in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
|
The problem is that ibuf_merge_or_delete_for_page() is invoking ibuf_delete_recs() on the change buffer tree itself. By design, the changes to the change buffer are never buffered by the change buffer.
The assertion fails, because ibuf->index->lock was already S-latched at the time when we are trying to acquire an S-latch. In a non-debug build, such an execution should cause a server hang on shutdown.
I was able to repeat the crash by running the following (on 1 of 2 tries):
./mtr --no-reorder innodb.innodb-page_compression_default innodb.innodb-page_compression_snappy innodb.innodb-page_compression_zip innodb.innodb_wl6326_big innodb.xa_recovery
|
The following change fixes the bug:
diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc
|
index 7e01c864a50..7068dab77a4 100644
|
--- a/storage/innobase/ibuf/ibuf0ibuf.cc
|
+++ b/storage/innobase/ibuf/ibuf0ibuf.cc
|
@@ -4472,7 +4472,10 @@ ibuf_merge_or_delete_for_page(
|
|
fil_space_release(space);
|
if (UNIV_UNLIKELY(srv_shutdown_state)
|
- && !srv_fast_shutdown) {
|
+ && !srv_fast_shutdown
|
+ && (!block
|
+ || btr_page_get_index_id(block->frame)
|
+ != DICT_IBUF_ID_MIN + IBUF_SPACE_ID)) {
|
/* Prevent an infinite loop on slow
|
shutdown, in case the bitmap bits are
|
wrongly clear even though buffered
|
I successfully ran the above sequence of tests 5 times with the above fix.
{"report":{"fcp":797,"ttfb":172.2999997138977,"pageVisibility":"visible","entityId":92178,"key":"jira.project.issue.view-issue","isInitial":true,"threshold":1000,"elementTimings":{},"userDeviceMemory":8,"userDeviceProcessors":64,"apdex":1,"journeyId":"2a8a1f23-8fca-4d1c-9362-47f743f42ea0","navigationType":0,"readyForUser":882.5999999046326,"redirectCount":0,"resourceLoadedEnd":744.3999996185303,"resourceLoadedStart":177.59999990463257,"resourceTiming":[{"duration":49.69999980926514,"initiatorType":"link","name":"https://jira.mariadb.org/s/2c21342762a6a02add1c328bed317ffd-CDN/lu2cib/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/css/_super/batch.css","startTime":177.59999990463257,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":177.59999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":227.2999997138977,"responseStart":0,"secureConnectionStart":0},{"duration":51.80000019073486,"initiatorType":"link","name":"https://jira.mariadb.org/s/7ebd35e77e471bc30ff0eba799ebc151-CDN/lu2cib/820016/12ta74/494e4c556ecbb29f90a3d3b4f09cb99c/_/download/contextbatch/css/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true&whisper-enabled=true","startTime":177.7999997138977,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":177.7999997138977,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":229.59999990463257,"responseStart":0,"secureConnectionStart":0},{"duration":103.7999997138977,"initiatorType":"script","name":"https://jira.mariadb.org/s/0917945aaa57108d00c5076fea35e069-CDN/lu2cib/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/js/_super/batch.js?locale=en","startTime":178.09999990463257,"connectEnd":178.09999990463257,"connectStart":178.09999990463257,"domainLookupEnd":178.09999990463257,"domainLookupStart":178.09999990463257,"fetchStart":178.09999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":178.09999990463257,"responseEnd":281.8999996185303,"responseStart":281.8999996185303,"secureConnectionStart":178.09999990463257},{"duration":182.30000019073486,"initiatorType":"script","name":"https://jira.mariadb.org/s/2d8175ec2fa4c816e8023260bd8c1786-CDN/lu2cib/820016/12ta74/494e4c556ecbb29f90a3d3b4f09cb99c/_/download/contextbatch/js/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true&whisper-enabled=true","startTime":178.19999980926514,"connectEnd":178.19999980926514,"connectStart":178.19999980926514,"domainLookupEnd":178.19999980926514,"domainLookupStart":178.19999980926514,"fetchStart":178.19999980926514,"redirectEnd":0,"redirectStart":0,"requestStart":178.19999980926514,"responseEnd":360.5,"responseStart":360.5,"secureConnectionStart":178.19999980926514},{"duration":186.30000019073486,"initiatorType":"script","name":"https://jira.mariadb.org/s/a9324d6758d385eb45c462685ad88f1d-CDN/lu2cib/820016/12ta74/c92c0caa9a024ae85b0ebdbed7fb4bd7/_/download/contextbatch/js/atl.global,-_super/batch.js?locale=en","startTime":178.39999961853027,"connectEnd":178.39999961853027,"connectStart":178.39999961853027,"domainLookupEnd":178.39999961853027,"domainLookupStart":178.39999961853027,"fetchStart":178.39999961853027,"redirectEnd":0,"redirectStart":0,"requestStart":178.39999961853027,"responseEnd":364.69999980926514,"responseStart":364.69999980926514,"secureConnectionStart":178.39999961853027},{"duration":186.5,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-en/jira.webresources:calendar-en.js","startTime":178.69999980926514,"connectEnd":178.69999980926514,"connectStart":178.69999980926514,"domainLookupEnd":178.69999980926514,"domainLookupStart":178.69999980926514,"fetchStart":178.69999980926514,"redirectEnd":0,"redirectStart":0,"requestStart":178.69999980926514,"responseEnd":365.19999980926514,"responseStart":365.19999980926514,"secureConnectionStart":178.69999980926514},{"duration":186.60000038146973,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-localisation-moment/jira.webresources:calendar-localisation-moment.js","startTime":178.89999961853027,"connectEnd":178.89999961853027,"connectStart":178.89999961853027,"domainLookupEnd":178.89999961853027,"domainLookupStart":178.89999961853027,"fetchStart":178.89999961853027,"redirectEnd":0,"redirectStart":0,"requestStart":178.89999961853027,"responseEnd":365.5,"responseStart":365.5,"secureConnectionStart":178.89999961853027},{"duration":246.2999997138977,"initiatorType":"link","name":"https://jira.mariadb.org/s/b04b06a02d1959df322d9cded3aeecc1-CDN/lu2cib/820016/12ta74/a2ff6aa845ffc9a1d22fe23d9ee791fc/_/download/contextbatch/css/jira.global.look-and-feel,-_super/batch.css","startTime":179.09999990463257,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":179.09999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":425.3999996185303,"responseStart":0,"secureConnectionStart":0},{"duration":186.80000019073486,"initiatorType":"script","name":"https://jira.mariadb.org/rest/api/1.0/shortcuts/820016/47140b6e0a9bc2e4913da06536125810/shortcuts.js?context=issuenavigation&context=issueaction","startTime":179.19999980926514,"connectEnd":179.19999980926514,"connectStart":179.19999980926514,"domainLookupEnd":179.19999980926514,"domainLookupStart":179.19999980926514,"fetchStart":179.19999980926514,"redirectEnd":0,"redirectStart":0,"requestStart":179.19999980926514,"responseEnd":366,"responseStart":366,"secureConnectionStart":179.19999980926514},{"duration":246.10000038146973,"initiatorType":"link","name":"https://jira.mariadb.org/s/3ac36323ba5e4eb0af2aa7ac7211b4bb-CDN/lu2cib/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/css/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.css?jira.create.linked.issue=true","startTime":179.39999961853027,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":179.39999961853027,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":425.5,"responseStart":0,"secureConnectionStart":0},{"duration":187,"initiatorType":"script","name":"https://jira.mariadb.org/s/5d5e8fe91fbc506585e83ea3b62ccc4b-CDN/lu2cib/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/js/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.js?jira.create.linked.issue=true&locale=en","startTime":179.5,"connectEnd":179.5,"connectStart":179.5,"domainLookupEnd":179.5,"domainLookupStart":179.5,"fetchStart":179.5,"redirectEnd":0,"redirectStart":0,"requestStart":179.5,"responseEnd":366.5,"responseStart":366.5,"secureConnectionStart":179.5},{"duration":494,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-js/jira.webresources:bigpipe-js.js","startTime":184.69999980926514,"connectEnd":184.69999980926514,"connectStart":184.69999980926514,"domainLookupEnd":184.69999980926514,"domainLookupStart":184.69999980926514,"fetchStart":184.69999980926514,"redirectEnd":0,"redirectStart":0,"requestStart":184.69999980926514,"responseEnd":678.6999998092651,"responseStart":678.6999998092651,"secureConnectionStart":184.69999980926514},{"duration":559.5999999046326,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-init/jira.webresources:bigpipe-init.js","startTime":184.7999997138977,"connectEnd":184.7999997138977,"connectStart":184.7999997138977,"domainLookupEnd":184.7999997138977,"domainLookupStart":184.7999997138977,"fetchStart":184.7999997138977,"redirectEnd":0,"redirectStart":0,"requestStart":184.7999997138977,"responseEnd":744.3999996185303,"responseStart":744.3999996185303,"secureConnectionStart":184.7999997138977},{"duration":242.30000019073486,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":436.7999997138977,"connectEnd":436.7999997138977,"connectStart":436.7999997138977,"domainLookupEnd":436.7999997138977,"domainLookupStart":436.7999997138977,"fetchStart":436.7999997138977,"redirectEnd":0,"redirectStart":0,"requestStart":436.7999997138977,"responseEnd":679.0999999046326,"responseStart":679.0999999046326,"secureConnectionStart":436.7999997138977},{"duration":261.40000009536743,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/e65b778d185daf5aee24936755b43da6/_/download/contextbatch/js/browser-metrics-plugin.contrib,-_super,-project.issue.navigator,-jira.view.issue,-atl.general/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true&whisper-enabled=true","startTime":698.7999997138977,"connectEnd":698.7999997138977,"connectStart":698.7999997138977,"domainLookupEnd":698.7999997138977,"domainLookupStart":698.7999997138977,"fetchStart":698.7999997138977,"redirectEnd":0,"redirectStart":0,"requestStart":698.7999997138977,"responseEnd":960.1999998092651,"responseStart":960.1999998092651,"secureConnectionStart":698.7999997138977},{"duration":217.30000019073486,"initiatorType":"script","name":"https://www.google-analytics.com/analytics.js","startTime":790.6999998092651,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":790.6999998092651,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1008,"responseStart":0,"secureConnectionStart":0}],"fetchStart":0,"domainLookupStart":0,"domainLookupEnd":0,"connectStart":0,"connectEnd":0,"requestStart":4,"responseStart":172,"responseEnd":176,"domLoading":175,"domInteractive":948,"domContentLoadedEventStart":948,"domContentLoadedEventEnd":998,"domComplete":1562,"loadEventStart":1562,"loadEventEnd":1563,"userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","marks":[{"name":"bigPipe.sidebar-id.start","time":924},{"name":"bigPipe.sidebar-id.end","time":924.6999998092651},{"name":"bigPipe.activity-panel-pipe-id.start","time":924.8999996185303},{"name":"bigPipe.activity-panel-pipe-id.end","time":927.6999998092651},{"name":"activityTabFullyLoaded","time":1012.3999996185303}],"measures":[],"correlationId":"e41328fa4065a","effectiveType":"4g","downlink":10,"rtt":0,"serverDuration":99,"dbReadsTimeInMs":10,"dbConnsTimeInMs":19,"applicationHash":"9d11dbea5f4be3d4cc21f03a88dd11d8c8687422","experiments":[]}}
Bug
Major