Type:
Bug
Priority:
Major
Resolution:
Fixed
Affects Version/s:
10.5
Upon upgrade from 10.1, InnoDB sporadically fails to startup with
10.5 d23c3e03
mysqld: /data/src/10.5/storage/innobase/buf/buf0buf.cc:2723: buf_page_t* buf_pool_watch_set(page_id_t, rw_lock_t**): Assertion `bpage->in_page_hash' failed.
200325 0:36:27 [ERROR] mysqld got signal 6 ;
#3 0x00007fc65dbaff12 in __GI___assert_fail (assertion=0x5578057cbd16 "bpage->in_page_hash", file=0x5578057cc350 "/data/src/10.5/storage/innobase/buf/buf0buf.cc", line=2723,
function=0x5578057d0e20 <buf_pool_watch_set(page_id_t, rw_lock_t**)::__PRETTY_FUNCTION__> "buf_page_t* buf_pool_watch_set(page_id_t, rw_lock_t**)") at assert.c:101
#4 0x00005578050e2376 in buf_pool_watch_set (page_id=..., hash_lock=0x7fc6395f9ba0) at /data/src/10.5/storage/innobase/buf/buf0buf.cc:2723
#5 0x00005578050e491d in buf_page_get_gen (page_id=..., zip_size=0, rw_latch=2, guess=0x0, mode=15, file=0x557805765b48 "/data/src/10.5/storage/innobase/row/row0row.cc", line=1300, mtr=0x7fc6395fb0f0, err=0x7fc6395f9e30,
allow_ibuf_merge=true) at /data/src/10.5/storage/innobase/buf/buf0buf.cc:3349
#6 0x00005578050a2ecf in btr_cur_search_to_nth_level_func (index=0x7fc62c00e988, level=0, tuple=0x7fc62c00a6f8, mode=PAGE_CUR_LE, latch_mode=2, cursor=0x7fc6395fae50, ahi_latch=0x0,
file=0x557805765b48 "/data/src/10.5/storage/innobase/row/row0row.cc", line=1300, mtr=0x7fc6395fb0f0, autoinc=0) at /data/src/10.5/storage/innobase/btr/btr0cur.cc:1606
#7 0x0000557804fb4e4e in btr_pcur_open_low (index=0x7fc62c00e988, level=0, tuple=0x7fc62c00a6f8, mode=PAGE_CUR_LE, latch_mode=8194, cursor=0x7fc6395fae50, file=0x557805765b48 "/data/src/10.5/storage/innobase/row/row0row.cc",
line=1300, autoinc=0, mtr=0x7fc6395fb0f0) at /data/src/10.5/storage/innobase/include/btr0pcur.ic:441
#8 0x0000557804fb8fa4 in row_search_index_entry (index=0x7fc62c00e988, entry=0x7fc62c00a6f8, mode=8194, pcur=0x7fc6395fae50, mtr=0x7fc6395fb0f0) at /data/src/10.5/storage/innobase/row/row0row.cc:1300
#9 0x0000557804fac8c0 in row_purge_remove_sec_if_poss_leaf (node=0x557809e9e1e8, index=0x7fc62c00e988, entry=0x7fc62c00a6f8) at /data/src/10.5/storage/innobase/row/row0purge.cc:457
#10 0x0000557804face18 in row_purge_remove_sec_if_poss (node=0x557809e9e1e8, index=0x7fc62c00e988, entry=0x7fc62c00a6f8) at /data/src/10.5/storage/innobase/row/row0purge.cc:569
#11 0x0000557804fad029 in row_purge_del_mark (node=0x557809e9e1e8) at /data/src/10.5/storage/innobase/row/row0purge.cc:639
#12 0x0000557804faea4e in row_purge_record_func (node=0x557809e9e1e8, undo_rec=0x557809ea0170 "", thr=0x557809e9e130, updated_extern=false) at /data/src/10.5/storage/innobase/row/row0purge.cc:1048
#13 0x0000557804faed17 in row_purge (node=0x557809e9e1e8, undo_rec=0x557809ea0170 "", thr=0x557809e9e130) at /data/src/10.5/storage/innobase/row/row0purge.cc:1110
#14 0x0000557804faee95 in row_purge_step (thr=0x557809e9e130) at /data/src/10.5/storage/innobase/row/row0purge.cc:1159
#15 0x0000557804f188bd in que_thr_step (thr=0x557809e9e130) at /data/src/10.5/storage/innobase/que/que0que.cc:1038
#16 0x0000557804f18b54 in que_run_threads_low (thr=0x557809e9e130) at /data/src/10.5/storage/innobase/que/que0que.cc:1100
#17 0x0000557804f18da4 in que_run_threads (thr=0x557809e9e130) at /data/src/10.5/storage/innobase/que/que0que.cc:1140
#18 0x0000557804ff6519 in srv_task_execute () at /data/src/10.5/storage/innobase/srv/srv0srv.cc:2055
#19 0x0000557804ff6ca1 in purge_worker_callback () at /data/src/10.5/storage/innobase/srv/srv0srv.cc:2205
#20 0x000055780522bbca in tpool::task_group::execute (this=0x557806a503a0 <purge_task_group>, t=0x557806a50160 <purge_worker_task>) at /data/src/10.5/tpool/task_group.cc:55
#21 0x000055780522bf80 in tpool::task::execute (this=0x557806a50160 <purge_worker_task>) at /data/src/10.5/tpool/task.cc:47
#22 0x00005578052256a0 in tpool::thread_pool_generic::worker_main (this=0x557807c89b70, thread_var=0x557807c99700) at /data/src/10.5/tpool/tpool_generic.cc:518
#23 0x000055780522b9db in std::__invoke_impl<void, void (tpool::thread_pool_generic::* const&)(tpool::worker_data*), tpool::thread_pool_generic*, tpool::worker_data*>(std::__invoke_memfun_deref, void (tpool::thread_pool_generic::* const&)(tpool::worker_data*), tpool::thread_pool_generic*&&, tpool::worker_data*&&) (__f=
@0x7fc650001d68: (void (tpool::thread_pool_generic::*)(tpool::thread_pool_generic * const, tpool::worker_data *)) 0x557805225608 <tpool::thread_pool_generic::worker_main(tpool::worker_data*)>,
__t=<unknown type in /data/bld/10.5-daily/bin/mariadbd, CU 0x7fb5ae4, DIE 0x7fc725c>, __args#0=<unknown type in /data/bld/10.5-daily/bin/mariadbd, CU 0x7fb5ae4, DIE 0x7fc7268>) at /usr/include/c++/6/functional:227
#24 0x000055780522b94d in std::__invoke<void (tpool::thread_pool_generic::* const&)(tpool::worker_data*), tpool::thread_pool_generic*, tpool::worker_data*>(void (tpool::thread_pool_generic::* const&)(tpool::worker_data*), tpool::thread_pool_generic*&&, tpool::worker_data*&&) (__fn=
@0x7fc650001d68: (void (tpool::thread_pool_generic::*)(tpool::thread_pool_generic * const, tpool::worker_data *)) 0x557805225608 <tpool::thread_pool_generic::worker_main(tpool::worker_data*)>,
__args#0=<unknown type in /data/bld/10.5-daily/bin/mariadbd, CU 0x7fb5ae4, DIE 0x7fc725c>, __args#1=<unknown type in /data/bld/10.5-daily/bin/mariadbd, CU 0x7fb5ae4, DIE 0x7fc7268>) at /usr/include/c++/6/functional:251
#25 0x000055780522b8e1 in std::_Mem_fn_base<void (tpool::thread_pool_generic::*)(tpool::worker_data*), true>::operator()<tpool::thread_pool_generic*, tpool::worker_data*>(tpool::thread_pool_generic*&&, tpool::worker_data*&&) const (this=0x7fc650001d68, __args#0=<unknown type in /data/bld/10.5-daily/bin/mariadbd, CU 0x7fb5ae4, DIE 0x7fc725c>, __args#1=<unknown type in /data/bld/10.5-daily/bin/mariadbd, CU 0x7fb5ae4, DIE 0x7fc7268>)
at /usr/include/c++/6/functional:604
#26 0x000055780522b893 in std::_Bind_simple<std::_Mem_fn<void (tpool::thread_pool_generic::*)(tpool::worker_data*)> (tpool::thread_pool_generic*, tpool::worker_data*)>::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) (
this=0x7fc650001d58) at /usr/include/c++/6/functional:1391
#27 0x000055780522b791 in std::_Bind_simple<std::_Mem_fn<void (tpool::thread_pool_generic::*)(tpool::worker_data*)> (tpool::thread_pool_generic*, tpool::worker_data*)>::operator()() (this=0x7fc650001d58)
at /usr/include/c++/6/functional:1380
#28 0x000055780522b770 in std::thread::_State_impl<std::_Bind_simple<std::_Mem_fn<void (tpool::thread_pool_generic::*)(tpool::worker_data*)> (tpool::thread_pool_generic*, tpool::worker_data*)> >::_M_run() (this=0x7fc650001d50)
at /usr/include/c++/6/thread:197
#29 0x00007fc65e4f7e6f in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#30 0x00007fc65fb384a4 in start_thread (arg=0x7fc6395fc700) at pthread_create.c:456
#31 0x00007fc65dc6cd0f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:97
The failure can be reproduced by starting the server on the attached datadir numerous times in the loop. The data was created on 10.1.44 release server by running DDL / DML flow and shutting down the server normally at the end. The general and error log are also in the attached archive.
Startup options:
--innodb-page-size=4K --innodb-compression-algorithm=none --loose-innodb_log_compressed_pages=on
Couldn't reproduce on 10.4 in reasonable time.
relates to
MDEV-15053
Reduce buf_pool_t::mutex contention
Closed
{"report":{"fcp":829.7000000476837,"ttfb":175.30000007152557,"pageVisibility":"visible","entityId":84528,"key":"jira.project.issue.view-issue","isInitial":true,"threshold":1000,"elementTimings":{},"userDeviceMemory":8,"userDeviceProcessors":64,"apdex":1,"journeyId":"0ccf8345-75d0-4be7-8e51-4ef39177ba2d","navigationType":0,"readyForUser":918.3000000715256,"redirectCount":0,"resourceLoadedEnd":889.8999999761581,"resourceLoadedStart":180.70000004768372,"resourceTiming":[{"duration":178.69999992847443,"initiatorType":"link","name":"https://jira.mariadb.org/s/2c21342762a6a02add1c328bed317ffd-CDN/lu2bu7/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/css/_super/batch.css","startTime":180.70000004768372,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":180.70000004768372,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":359.39999997615814,"responseStart":0,"secureConnectionStart":0},{"duration":178.5,"initiatorType":"link","name":"https://jira.mariadb.org/s/7ebd35e77e471bc30ff0eba799ebc151-CDN/lu2bu7/820016/12ta74/8679b4946efa1a0bb029a3a22206fb5d/_/download/contextbatch/css/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true","startTime":181,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":181,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":359.5,"responseStart":0,"secureConnectionStart":0},{"duration":187.39999997615814,"initiatorType":"script","name":"https://jira.mariadb.org/s/fbf975c0cce4b1abf04784eeae9ba1f4-CDN/lu2bu7/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/js/_super/batch.js?locale=en","startTime":181.10000002384186,"connectEnd":181.10000002384186,"connectStart":181.10000002384186,"domainLookupEnd":181.10000002384186,"domainLookupStart":181.10000002384186,"fetchStart":181.10000002384186,"redirectEnd":0,"redirectStart":0,"requestStart":181.10000002384186,"responseEnd":368.5,"responseStart":368.5,"secureConnectionStart":181.10000002384186},{"duration":277.1999999284744,"initiatorType":"script","name":"https://jira.mariadb.org/s/099b33461394b8015fc36c0a4b96e19f-CDN/lu2bu7/820016/12ta74/8679b4946efa1a0bb029a3a22206fb5d/_/download/contextbatch/js/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true","startTime":181.20000004768372,"connectEnd":181.20000004768372,"connectStart":181.20000004768372,"domainLookupEnd":181.20000004768372,"domainLookupStart":181.20000004768372,"fetchStart":181.20000004768372,"redirectEnd":0,"redirectStart":0,"requestStart":181.20000004768372,"responseEnd":458.39999997615814,"responseStart":458.39999997615814,"secureConnectionStart":181.20000004768372},{"duration":280.6999999284744,"initiatorType":"script","name":"https://jira.mariadb.org/s/94c15bff32baef80f4096a08aceae8bc-CDN/lu2bu7/820016/12ta74/c92c0caa9a024ae85b0ebdbed7fb4bd7/_/download/contextbatch/js/atl.global,-_super/batch.js?locale=en","startTime":181.30000007152557,"connectEnd":181.30000007152557,"connectStart":181.30000007152557,"domainLookupEnd":181.30000007152557,"domainLookupStart":181.30000007152557,"fetchStart":181.30000007152557,"redirectEnd":0,"redirectStart":0,"requestStart":181.30000007152557,"responseEnd":462,"responseStart":462,"secureConnectionStart":181.30000007152557},{"duration":281.10000002384186,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bu7/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-en/jira.webresources:calendar-en.js","startTime":181.39999997615814,"connectEnd":181.39999997615814,"connectStart":181.39999997615814,"domainLookupEnd":181.39999997615814,"domainLookupStart":181.39999997615814,"fetchStart":181.39999997615814,"redirectEnd":0,"redirectStart":0,"requestStart":181.39999997615814,"responseEnd":462.5,"responseStart":462.5,"secureConnectionStart":181.39999997615814},{"duration":281.7999999523163,"initiatorType":"link","name":"https://jira.mariadb.org/s/b04b06a02d1959df322d9cded3aeecc1-CDN/lu2bu7/820016/12ta74/a2ff6aa845ffc9a1d22fe23d9ee791fc/_/download/contextbatch/css/jira.global.look-and-feel,-_super/batch.css","startTime":181.60000002384186,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":181.60000002384186,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":463.39999997615814,"responseStart":0,"secureConnectionStart":0},{"duration":281.2999999523163,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bu7/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-localisation-moment/jira.webresources:calendar-localisation-moment.js","startTime":181.60000002384186,"connectEnd":181.60000002384186,"connectStart":181.60000002384186,"domainLookupEnd":181.60000002384186,"domainLookupStart":181.60000002384186,"fetchStart":181.60000002384186,"redirectEnd":0,"redirectStart":0,"requestStart":181.60000002384186,"responseEnd":462.89999997615814,"responseStart":462.89999997615814,"secureConnectionStart":181.60000002384186},{"duration":281.5,"initiatorType":"script","name":"https://jira.mariadb.org/rest/api/1.0/shortcuts/820016/47140b6e0a9bc2e4913da06536125810/shortcuts.js?context=issuenavigation&context=issueaction","startTime":181.70000004768372,"connectEnd":181.70000004768372,"connectStart":181.70000004768372,"domainLookupEnd":181.70000004768372,"domainLookupStart":181.70000004768372,"fetchStart":181.70000004768372,"redirectEnd":0,"redirectStart":0,"requestStart":181.70000004768372,"responseEnd":463.2000000476837,"responseStart":463.2000000476837,"secureConnectionStart":181.70000004768372},{"duration":282,"initiatorType":"link","name":"https://jira.mariadb.org/s/3ac36323ba5e4eb0af2aa7ac7211b4bb-CDN/lu2bu7/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/css/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.css?jira.create.linked.issue=true","startTime":181.80000007152557,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":181.80000007152557,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":463.8000000715256,"responseStart":0,"secureConnectionStart":0},{"duration":281.8000000715256,"initiatorType":"script","name":"https://jira.mariadb.org/s/3339d87fa2538a859872f2df449bf8d0-CDN/lu2bu7/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/js/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.js?jira.create.linked.issue=true&locale=en","startTime":182,"connectEnd":182,"connectStart":182,"domainLookupEnd":182,"domainLookupStart":182,"fetchStart":182,"redirectEnd":0,"redirectStart":0,"requestStart":182,"responseEnd":463.8000000715256,"responseStart":463.8000000715256,"secureConnectionStart":182},{"duration":408.2999999523163,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bu7/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-js/jira.webresources:bigpipe-js.js","startTime":187.30000007152557,"connectEnd":187.30000007152557,"connectStart":187.30000007152557,"domainLookupEnd":187.30000007152557,"domainLookupStart":187.30000007152557,"fetchStart":187.30000007152557,"redirectEnd":0,"redirectStart":0,"requestStart":187.30000007152557,"responseEnd":595.6000000238419,"responseStart":595.6000000238419,"secureConnectionStart":187.30000007152557},{"duration":702.5,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bu7/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-init/jira.webresources:bigpipe-init.js","startTime":187.39999997615814,"connectEnd":187.39999997615814,"connectStart":187.39999997615814,"domainLookupEnd":187.39999997615814,"domainLookupStart":187.39999997615814,"fetchStart":187.39999997615814,"redirectEnd":0,"redirectStart":0,"requestStart":187.39999997615814,"responseEnd":889.8999999761581,"responseStart":889.8999999761581,"secureConnectionStart":187.39999997615814},{"duration":75.29999995231628,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":523.7000000476837,"connectEnd":523.7000000476837,"connectStart":523.7000000476837,"domainLookupEnd":523.7000000476837,"domainLookupStart":523.7000000476837,"fetchStart":523.7000000476837,"redirectEnd":0,"redirectStart":0,"requestStart":523.7000000476837,"responseEnd":599,"responseStart":599,"secureConnectionStart":523.7000000476837},{"duration":281.2000000476837,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2bu7/820016/12ta74/e65b778d185daf5aee24936755b43da6/_/download/contextbatch/js/browser-metrics-plugin.contrib,-_super,-atl.general/batch.js?agile_global_admin_condition=true&jag=true&slack-enabled=true","startTime":746.5,"connectEnd":746.5,"connectStart":746.5,"domainLookupEnd":746.5,"domainLookupStart":746.5,"fetchStart":746.5,"redirectEnd":0,"redirectStart":0,"requestStart":746.5,"responseEnd":1027.7000000476837,"responseStart":1027.6000000238419,"secureConnectionStart":746.5},{"duration":207.89999997615814,"initiatorType":"script","name":"https://www.google-analytics.com/analytics.js","startTime":821.5,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":821.5,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1029.3999999761581,"responseStart":0,"secureConnectionStart":0}],"fetchStart":0,"domainLookupStart":0,"domainLookupEnd":0,"connectStart":0,"connectEnd":0,"requestStart":45,"responseStart":175,"responseEnd":180,"domLoading":179,"domInteractive":982,"domContentLoadedEventStart":982,"domContentLoadedEventEnd":1034,"domComplete":1819,"loadEventStart":1819,"loadEventEnd":1821,"userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","marks":[{"name":"bigPipe.sidebar-id.start","time":962.3999999761581},{"name":"bigPipe.sidebar-id.end","time":963.2000000476837},{"name":"bigPipe.activity-panel-pipe-id.start","time":963.3000000715256},{"name":"bigPipe.activity-panel-pipe-id.end","time":965.8999999761581},{"name":"activityTabFullyLoaded","time":1042}],"measures":[],"correlationId":"3f434a2c1a710c","effectiveType":"4g","downlink":9.7,"rtt":0,"serverDuration":73,"dbReadsTimeInMs":11,"dbConnsTimeInMs":18,"applicationHash":"9d11dbea5f4be3d4cc21f03a88dd11d8c8687422","experiments":[]}}