We quite often see stale mysqld process on Power8. At least the following deadlock was identified during debugging:
#0 0x00003fff86742190 in __pthread_cond_wait (cond=0x3fff857f0640, mutex=0x3fff857f0600) at pthread_cond_wait.c:187
|
#1 0x0000000010a18dd8 in os_cond_wait (cond=0x3fff857f0640, fast_mutex=0x3fff857f0600)
|
at /home/u0013565/svoj/10.0/storage/xtradb/os/os0sync.cc:214
|
#2 0x0000000010a1961c in os_event_wait_low (event=0x3fff857f0600, reset_sig_count=23)
|
at /home/u0013565/svoj/10.0/storage/xtradb/os/os0sync.cc:610
|
#3 0x0000000010b22c2c in rw_lock_debug_mutex_enter () at /home/u0013565/svoj/10.0/storage/xtradb/sync/sync0rw.cc:936
|
#4 0x0000000010b22ebc in rw_lock_remove_debug_info (lock=0x3fff84b423d8, pass=0, lock_type=352)
|
at /home/u0013565/svoj/10.0/storage/xtradb/sync/sync0rw.cc:1012
|
#5 0x0000000010bd89d0 in rw_lock_s_unlock_func (pass=0, lock=0x3fff84b423d8) at /home/u0013565/svoj/10.0/storage/xtradb/include/sync0rw.ic:614
|
#6 0x0000000010bd9410 in pfs_rw_lock_s_unlock_func (pass=0, lock=0x3fff84b423d8)
|
at /home/u0013565/svoj/10.0/storage/xtradb/include/sync0rw.ic:1251
|
#7 0x0000000010bde57c in buf_flush_try_neighbors (space=0, offset=104, flush_type=BUF_FLUSH_LIST, n_flushed=75, n_to_flush=100)
|
at /home/u0013565/svoj/10.0/storage/xtradb/buf/buf0flu.cc:1291
|
#8 0x0000000010bde9f4 in buf_flush_page_and_try_neighbors (bpage=0x3fff73ff0e08, flush_type=BUF_FLUSH_LIST, n_to_flush=100, count=0x3fffd40fab18)
|
at /home/u0013565/svoj/10.0/storage/xtradb/buf/buf0flu.cc:1392
|
#9 0x0000000010bdf64c in buf_do_flush_list_batch (buf_pool=0x3fff84be0878, min_n=100, lsn_limit=18446744073709551615)
|
at /home/u0013565/svoj/10.0/storage/xtradb/buf/buf0flu.cc:1719
|
#10 0x0000000010bdfaa8 in buf_flush_batch (buf_pool=0x3fff84be0878, flush_type=BUF_FLUSH_LIST, min_n=100, lsn_limit=18446744073709551615,
|
limited_lru_scan=false, n=0x3fffd40fac58) at /home/u0013565/svoj/10.0/storage/xtradb/buf/buf0flu.cc:1796
|
#11 0x0000000010be02bc in buf_flush_list (min_n=18446744073709551614, lsn_limit=18446744073709551615, n_processed=0x3fffd40faf48)
|
at /home/u0013565/svoj/10.0/storage/xtradb/buf/buf0flu.cc:2051
|
#12 0x00000000109e77bc in log_preflush_pool_modified_pages (new_oldest=18446744073709551615)
|
at /home/u0013565/svoj/10.0/storage/xtradb/log/log0log.cc:1871
|
#13 0x00000000109e8d04 in log_make_checkpoint_at (lsn=18446744073709551615, write_always=1)
|
at /home/u0013565/svoj/10.0/storage/xtradb/log/log0log.cc:2333
|
#14 0x0000000010bd2c08 in buf_dblwr_create () at /home/u0013565/svoj/10.0/storage/xtradb/buf/buf0dblwr.cc:339
|
#15 0x0000000010b1c40c in innobase_start_or_create_for_mysql () at /home/u0013565/svoj/10.0/storage/xtradb/srv/srv0start.cc:2667
|
#16 0x000000001095f320 in innobase_init (p=0x3fff853e0e70) at /home/u0013565/svoj/10.0/storage/xtradb/handler/ha_innodb.cc:3760
|
#17 0x000000001050d800 in ha_initialize_handlerton (plugin=0x3fff856c4150) at /home/u0013565/svoj/10.0/sql/handler.cc:502
|
#18 0x000000001029e730 in plugin_initialize (tmp_root=0x3fffd40ff198, plugin=0x3fff856c4150, argc=0x114a35d0 <remaining_argc>,
|
argv=0x3fff85650940, options_only=false) at /home/u0013565/svoj/10.0/sql/sql_plugin.cc:1372
|
#19 0x000000001029f30c in plugin_init (argc=0x114a35d0 <remaining_argc>, argv=0x3fff85650940, flags=2)
|
at /home/u0013565/svoj/10.0/sql/sql_plugin.cc:1608
|
#20 0x000000001018e3c4 in init_server_components () at /home/u0013565/svoj/10.0/sql/mysqld.cc:4790
|
#21 0x000000001018fb48 in mysqld_main (argc=48, argv=0x3fff85650940) at /home/u0013565/svoj/10.0/sql/mysqld.cc:5385
|
#22 0x00000000101824c8 in main (argc=48, argv=0x3fffd40ffa88) at /home/u0013565/svoj/10.0/sql/main.cc:25
|
From the above trace we can see that it happens during server initialization. The server won't continue initialization and waits for signal forever.
The problem is race conditions in rw_lock_debug_mutex_enter and rw_lock_debug_mutex_exit.
- is part of
-
MDEV-6478
MariaDB on Power8
-
-
Closed
- links to
-
{"report":{"fcp":2390.4000000953674,"ttfb":745.3000001907349,"pageVisibility":"visible","entityId":41311,"key":"jira.project.issue.view-issue","isInitial":true,"threshold":1000,"elementTimings":{},"userDeviceMemory":8,"userDeviceProcessors":64,"apdex":0.5,"journeyId":"7c7e901c-7e44-4108-8efe-99c8945f9873","navigationType":0,"readyForUser":2483.2000002861023,"redirectCount":0,"resourceLoadedEnd":3051.5999999046326,"resourceLoadedStart":828,"resourceTiming":[{"duration":797.0999999046326,"initiatorType":"link","name":"https://jira.mariadb.org/s/2c21342762a6a02add1c328bed317ffd-CDN/lu2cib/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/css/_super/batch.css","startTime":828,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":828,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1625.0999999046326,"responseStart":0,"secureConnectionStart":0},{"duration":796.9000000953674,"initiatorType":"link","name":"https://jira.mariadb.org/s/7ebd35e77e471bc30ff0eba799ebc151-CDN/lu2cib/820016/12ta74/494e4c556ecbb29f90a3d3b4f09cb99c/_/download/contextbatch/css/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true&whisper-enabled=true","startTime":828.4000000953674,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":828.4000000953674,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1625.3000001907349,"responseStart":0,"secureConnectionStart":0},{"duration":877.7000002861023,"initiatorType":"script","name":"https://jira.mariadb.org/s/0917945aaa57108d00c5076fea35e069-CDN/lu2cib/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/js/_super/batch.js?locale=en","startTime":828.5999999046326,"connectEnd":828.5999999046326,"connectStart":828.5999999046326,"domainLookupEnd":828.5999999046326,"domainLookupStart":828.5999999046326,"fetchStart":828.5999999046326,"redirectEnd":0,"redirectStart":0,"requestStart":828.5999999046326,"responseEnd":1706.3000001907349,"responseStart":1706.3000001907349,"secureConnectionStart":828.5999999046326},{"duration":1004.2999997138977,"initiatorType":"script","name":"https://jira.mariadb.org/s/2d8175ec2fa4c816e8023260bd8c1786-CDN/lu2cib/820016/12ta74/494e4c556ecbb29f90a3d3b4f09cb99c/_/download/contextbatch/js/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true&whisper-enabled=true","startTime":828.8000001907349,"connectEnd":828.8000001907349,"connectStart":828.8000001907349,"domainLookupEnd":828.8000001907349,"domainLookupStart":828.8000001907349,"fetchStart":828.8000001907349,"redirectEnd":0,"redirectStart":0,"requestStart":828.8000001907349,"responseEnd":1833.0999999046326,"responseStart":1833.0999999046326,"secureConnectionStart":828.8000001907349},{"duration":1030.7000002861023,"initiatorType":"script","name":"https://jira.mariadb.org/s/a9324d6758d385eb45c462685ad88f1d-CDN/lu2cib/820016/12ta74/c92c0caa9a024ae85b0ebdbed7fb4bd7/_/download/contextbatch/js/atl.global,-_super/batch.js?locale=en","startTime":829,"connectEnd":829,"connectStart":829,"domainLookupEnd":829,"domainLookupStart":829,"fetchStart":829,"redirectEnd":0,"redirectStart":0,"requestStart":829,"responseEnd":1859.7000002861023,"responseStart":1859.7000002861023,"secureConnectionStart":829},{"duration":1031,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-en/jira.webresources:calendar-en.js","startTime":829.3000001907349,"connectEnd":829.3000001907349,"connectStart":829.3000001907349,"domainLookupEnd":829.3000001907349,"domainLookupStart":829.3000001907349,"fetchStart":829.3000001907349,"redirectEnd":0,"redirectStart":0,"requestStart":829.3000001907349,"responseEnd":1860.3000001907349,"responseStart":1860.3000001907349,"secureConnectionStart":829.3000001907349},{"duration":1031.5,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-localisation-moment/jira.webresources:calendar-localisation-moment.js","startTime":829.4000000953674,"connectEnd":829.4000000953674,"connectStart":829.4000000953674,"domainLookupEnd":829.4000000953674,"domainLookupStart":829.4000000953674,"fetchStart":829.4000000953674,"redirectEnd":0,"redirectStart":0,"requestStart":829.4000000953674,"responseEnd":1860.9000000953674,"responseStart":1860.9000000953674,"secureConnectionStart":829.4000000953674},{"duration":1101.5,"initiatorType":"link","name":"https://jira.mariadb.org/s/b04b06a02d1959df322d9cded3aeecc1-CDN/lu2cib/820016/12ta74/a2ff6aa845ffc9a1d22fe23d9ee791fc/_/download/contextbatch/css/jira.global.look-and-feel,-_super/batch.css","startTime":829.5999999046326,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":829.5999999046326,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1931.0999999046326,"responseStart":0,"secureConnectionStart":0},{"duration":1032,"initiatorType":"script","name":"https://jira.mariadb.org/rest/api/1.0/shortcuts/820016/47140b6e0a9bc2e4913da06536125810/shortcuts.js?context=issuenavigation&context=issueaction","startTime":829.8000001907349,"connectEnd":829.8000001907349,"connectStart":829.8000001907349,"domainLookupEnd":829.8000001907349,"domainLookupStart":829.8000001907349,"fetchStart":829.8000001907349,"redirectEnd":0,"redirectStart":0,"requestStart":829.8000001907349,"responseEnd":1861.8000001907349,"responseStart":1861.8000001907349,"secureConnectionStart":829.8000001907349},{"duration":1101.5999999046326,"initiatorType":"link","name":"https://jira.mariadb.org/s/3ac36323ba5e4eb0af2aa7ac7211b4bb-CDN/lu2cib/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/css/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.css?jira.create.linked.issue=true","startTime":829.9000000953674,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":829.9000000953674,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1931.5,"responseStart":0,"secureConnectionStart":0},{"duration":1032.3000001907349,"initiatorType":"script","name":"https://jira.mariadb.org/s/5d5e8fe91fbc506585e83ea3b62ccc4b-CDN/lu2cib/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/js/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.js?jira.create.linked.issue=true&locale=en","startTime":830.0999999046326,"connectEnd":830.0999999046326,"connectStart":830.0999999046326,"domainLookupEnd":830.0999999046326,"domainLookupStart":830.0999999046326,"fetchStart":830.0999999046326,"redirectEnd":0,"redirectStart":0,"requestStart":830.0999999046326,"responseEnd":1862.4000000953674,"responseStart":1862.4000000953674,"secureConnectionStart":830.0999999046326},{"duration":2110.6000003814697,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-js/jira.webresources:bigpipe-js.js","startTime":832.0999999046326,"connectEnd":832.0999999046326,"connectStart":832.0999999046326,"domainLookupEnd":832.0999999046326,"domainLookupStart":832.0999999046326,"fetchStart":832.0999999046326,"redirectEnd":0,"redirectStart":0,"requestStart":832.0999999046326,"responseEnd":2942.7000002861023,"responseStart":2942.7000002861023,"secureConnectionStart":832.0999999046326},{"duration":2219.2999997138977,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-init/jira.webresources:bigpipe-init.js","startTime":832.3000001907349,"connectEnd":832.3000001907349,"connectStart":832.3000001907349,"domainLookupEnd":832.3000001907349,"domainLookupStart":832.3000001907349,"fetchStart":832.3000001907349,"redirectEnd":0,"redirectStart":0,"requestStart":832.3000001907349,"responseEnd":3051.5999999046326,"responseStart":3051.5999999046326,"secureConnectionStart":832.3000001907349},{"duration":995.6999998092651,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":1947.4000000953674,"connectEnd":1947.4000000953674,"connectStart":1947.4000000953674,"domainLookupEnd":1947.4000000953674,"domainLookupStart":1947.4000000953674,"fetchStart":1947.4000000953674,"redirectEnd":0,"redirectStart":0,"requestStart":1947.4000000953674,"responseEnd":2943.0999999046326,"responseStart":2943.0999999046326,"secureConnectionStart":1947.4000000953674}],"fetchStart":0,"domainLookupStart":0,"domainLookupEnd":0,"connectStart":0,"connectEnd":0,"requestStart":584,"responseStart":745,"responseEnd":772,"domLoading":779,"domInteractive":3154,"domContentLoadedEventStart":3154,"domContentLoadedEventEnd":3229,"domComplete":3936,"loadEventStart":3936,"loadEventEnd":3937,"userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","marks":[{"name":"bigPipe.sidebar-id.start","time":3129.300000190735},{"name":"bigPipe.sidebar-id.end","time":3130.0999999046326},{"name":"bigPipe.activity-panel-pipe-id.start","time":3130.300000190735},{"name":"bigPipe.activity-panel-pipe-id.end","time":3131.4000000953674},{"name":"activityTabFullyLoaded","time":3245.7000002861023}],"measures":[],"correlationId":"89fbdcfa972c13","effectiveType":"4g","downlink":10,"rtt":0,"serverDuration":100,"dbReadsTimeInMs":19,"dbConnsTimeInMs":29,"applicationHash":"9d11dbea5f4be3d4cc21f03a88dd11d8c8687422","experiments":[]}}
Bug
Critical
