There is stuck query processing situation on a cluster with MinIO S3 implementation.
PP waits for data to be returned from SM indefinitely. Here is some information collected:
TID 3950697:
|
#0 0x00007fd55013345c pthread_cond_wait@@GLIBC_2.3.2
|
#1 0x0000556694d242eb primitiveprocessor::prefetchBlocks(unsigned long, int, unsigned int*)
|
#2 0x0000556694d24e50 primitiveprocessor::loadBlocks(long*, BRM::QueryContext, int, int, unsigned char**, unsigned int*, bool, unsigned int, unsigned int, bool*, bool, std::tr1::unordered_map<long, BRM::VSSData, std::tr1::hash<long>, std::equal_to<long>, std::allocator<std::pair<long const, BRM::VSSData> > >*)
|
#3 0x0000556694d08822 void primitiveprocessor::ColumnCommand::_loadData<8>()
|
#4 0x0000556694d0545d primitiveprocessor::ColumnCommand::issuePrimitive()
|
#5 0x0000556694d01ef4 primitiveprocessor::ColumnCommand::projectIntoRowGroup(rowgroup::RowGroup&, unsigned int)
|
#6 0x0000556694ceb50f primitiveprocessor::BatchPrimitiveProcessor::execute()
|
#7 0x0000556694cee1a1 primitiveprocessor::BatchPrimitiveProcessor::operator()()
|
#8 0x0000556694cfb48d primitiveprocessor::BPPSeeder::operator()()
|
#9 0x00007fd54fed8ac3 threadpool::FairThreadPool::threadFcn(threadpool::PriorityThreadPool::Priority)
|
#10 0x0000556694d42c97 thread_proxy
|
#11 0x00007fd55012d1ca start_thread
|
#12 0x00007fd54eb3de73 __clone
|
..
|
TID 3950869:
|
#0 0x00007fd550136ab4 read
|
#1 0x00007fd54def394b idbdatafile::SocketPool::send_recv(messageqcpp::ByteStream&, messageqcpp::ByteStream*)
|
#2 0x00007fd54dee912e idbdatafile::SMComm::pread(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, void*, unsigned long, long)
|
#3 0x0000556694d54fac (anonymous namespace)::thr_popper(dbbc::ioManager*)
|
#4 0x0000556694d59970 boost::detail::thread_data<dbbc::LambdaKludge>::run()
|
#5 0x0000556694d42c97 thread_proxy
|
#6 0x00007fd55012d1ca start_thread
|
#7 0x00007fd54eb3de73 __clone
|
As one sees PP processing threads wait for async data load. That in turn waits for SM to return a message.
SM threads backtrace doesn't have threads that answers that mentioned `idbdatafile::SMComm` communication.
There are no comments yet on this issue.
{"report":{"fcp":1818.7000002861023,"ttfb":238.60000038146973,"pageVisibility":"visible","entityId":127974,"key":"jira.project.issue.view-issue","isInitial":true,"threshold":1000,"elementTimings":{},"userDeviceMemory":8,"userDeviceProcessors":64,"apdex":0.5,"journeyId":"d3df21e9-fb5b-4f0c-97c1-816eefa5ce2b","navigationType":0,"readyForUser":1876.7000002861023,"redirectCount":0,"resourceLoadedEnd":2315,"resourceLoadedStart":253.10000038146973,"resourceTiming":[{"duration":719.2999997138977,"initiatorType":"link","name":"https://jira.mariadb.org/s/2c21342762a6a02add1c328bed317ffd-CDN/lu2cib/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/css/_super/batch.css","startTime":253.10000038146973,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":253.10000038146973,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":972.4000000953674,"responseStart":0,"secureConnectionStart":0},{"duration":719.4000000953674,"initiatorType":"link","name":"https://jira.mariadb.org/s/7ebd35e77e471bc30ff0eba799ebc151-CDN/lu2cib/820016/12ta74/494e4c556ecbb29f90a3d3b4f09cb99c/_/download/contextbatch/css/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true&whisper-enabled=true","startTime":253.40000009536743,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":253.40000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":972.8000001907349,"responseStart":0,"secureConnectionStart":0},{"duration":729.7999997138977,"initiatorType":"script","name":"https://jira.mariadb.org/s/0917945aaa57108d00c5076fea35e069-CDN/lu2cib/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/js/_super/batch.js?locale=en","startTime":253.60000038146973,"connectEnd":253.60000038146973,"connectStart":253.60000038146973,"domainLookupEnd":253.60000038146973,"domainLookupStart":253.60000038146973,"fetchStart":253.60000038146973,"redirectEnd":0,"redirectStart":0,"requestStart":253.60000038146973,"responseEnd":983.4000000953674,"responseStart":983.4000000953674,"secureConnectionStart":253.60000038146973},{"duration":830.9000000953674,"initiatorType":"script","name":"https://jira.mariadb.org/s/2d8175ec2fa4c816e8023260bd8c1786-CDN/lu2cib/820016/12ta74/494e4c556ecbb29f90a3d3b4f09cb99c/_/download/contextbatch/js/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true&whisper-enabled=true","startTime":253.80000019073486,"connectEnd":253.80000019073486,"connectStart":253.80000019073486,"domainLookupEnd":253.80000019073486,"domainLookupStart":253.80000019073486,"fetchStart":253.80000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":253.80000019073486,"responseEnd":1084.7000002861023,"responseStart":1084.7000002861023,"secureConnectionStart":253.80000019073486},{"duration":882.7000002861023,"initiatorType":"script","name":"https://jira.mariadb.org/s/a9324d6758d385eb45c462685ad88f1d-CDN/lu2cib/820016/12ta74/c92c0caa9a024ae85b0ebdbed7fb4bd7/_/download/contextbatch/js/atl.global,-_super/batch.js?locale=en","startTime":254,"connectEnd":254,"connectStart":254,"domainLookupEnd":254,"domainLookupStart":254,"fetchStart":254,"redirectEnd":0,"redirectStart":0,"requestStart":254,"responseEnd":1136.7000002861023,"responseStart":1136.6000003814697,"secureConnectionStart":254},{"duration":964.0999999046326,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-en/jira.webresources:calendar-en.js","startTime":254.30000019073486,"connectEnd":254.30000019073486,"connectStart":254.30000019073486,"domainLookupEnd":254.30000019073486,"domainLookupStart":254.30000019073486,"fetchStart":254.30000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":254.30000019073486,"responseEnd":1218.4000000953674,"responseStart":1218.4000000953674,"secureConnectionStart":254.30000019073486},{"duration":965.1999998092651,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-localisation-moment/jira.webresources:calendar-localisation-moment.js","startTime":254.30000019073486,"connectEnd":254.30000019073486,"connectStart":254.30000019073486,"domainLookupEnd":254.30000019073486,"domainLookupStart":254.30000019073486,"fetchStart":254.30000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":254.30000019073486,"responseEnd":1219.5,"responseStart":1219.5,"secureConnectionStart":254.30000019073486},{"duration":967.0999999046326,"initiatorType":"link","name":"https://jira.mariadb.org/s/b04b06a02d1959df322d9cded3aeecc1-CDN/lu2cib/820016/12ta74/a2ff6aa845ffc9a1d22fe23d9ee791fc/_/download/contextbatch/css/jira.global.look-and-feel,-_super/batch.css","startTime":254.60000038146973,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":254.60000038146973,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1221.7000002861023,"responseStart":0,"secureConnectionStart":0},{"duration":965.4000000953674,"initiatorType":"script","name":"https://jira.mariadb.org/rest/api/1.0/shortcuts/820016/47140b6e0a9bc2e4913da06536125810/shortcuts.js?context=issuenavigation&context=issueaction","startTime":254.7000002861023,"connectEnd":254.7000002861023,"connectStart":254.7000002861023,"domainLookupEnd":254.7000002861023,"domainLookupStart":254.7000002861023,"fetchStart":254.7000002861023,"redirectEnd":0,"redirectStart":0,"requestStart":254.7000002861023,"responseEnd":1220.1000003814697,"responseStart":1220,"secureConnectionStart":254.7000002861023},{"duration":967,"initiatorType":"link","name":"https://jira.mariadb.org/s/3ac36323ba5e4eb0af2aa7ac7211b4bb-CDN/lu2cib/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/css/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.css?jira.create.linked.issue=true","startTime":255,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":255,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":1222,"responseStart":0,"secureConnectionStart":0},{"duration":965.6999998092651,"initiatorType":"script","name":"https://jira.mariadb.org/s/5d5e8fe91fbc506585e83ea3b62ccc4b-CDN/lu2cib/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/js/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.js?jira.create.linked.issue=true&locale=en","startTime":255.10000038146973,"connectEnd":255.10000038146973,"connectStart":255.10000038146973,"domainLookupEnd":255.10000038146973,"domainLookupStart":255.10000038146973,"fetchStart":255.10000038146973,"redirectEnd":0,"redirectStart":0,"requestStart":255.10000038146973,"responseEnd":1220.8000001907349,"responseStart":1220.8000001907349,"secureConnectionStart":255.10000038146973},{"duration":1293.0999999046326,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-js/jira.webresources:bigpipe-js.js","startTime":255.90000009536743,"connectEnd":255.90000009536743,"connectStart":255.90000009536743,"domainLookupEnd":255.90000009536743,"domainLookupStart":255.90000009536743,"fetchStart":255.90000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":255.90000009536743,"responseEnd":1549,"responseStart":1549,"secureConnectionStart":255.90000009536743},{"duration":2037.5,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-init/jira.webresources:bigpipe-init.js","startTime":255.90000009536743,"connectEnd":255.90000009536743,"connectStart":255.90000009536743,"domainLookupEnd":255.90000009536743,"domainLookupStart":255.90000009536743,"fetchStart":255.90000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":255.90000009536743,"responseEnd":2293.4000000953674,"responseStart":2293.4000000953674,"secureConnectionStart":255.90000009536743},{"duration":321.5,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":1227,"connectEnd":1227,"connectStart":1227,"domainLookupEnd":1227,"domainLookupStart":1227,"fetchStart":1227,"redirectEnd":0,"redirectStart":0,"requestStart":1227,"responseEnd":1548.5,"responseStart":1548.5,"secureConnectionStart":1227},{"duration":652.2000002861023,"initiatorType":"link","name":"https://jira.mariadb.org/s/d5715adaadd168a9002b108b2b039b50-CDN/lu2cib/820016/12ta74/be4b45e9cec53099498fa61c8b7acba4/_/download/contextbatch/css/jira.project.sidebar,-_super,-project.issue.navigator,-jira.general,-jira.browse.project,-jira.view.issue,-jira.global,-atl.general,-com.atlassian.jira.projects.sidebar.init/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true&whisper-enabled=true","startTime":1662.5,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":1662.5,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":2314.7000002861023,"responseStart":0,"secureConnectionStart":0},{"duration":651.0999999046326,"initiatorType":"link","name":"https://jira.mariadb.org/s/50bc9be5bfead1a25e72c1a9338c94f6-CDN/lu2cib/820016/12ta74/e108c7645258ccb43280ed3404e3e949/_/download/contextbatch/css/com.atlassian.jira.plugins.jira-development-integration-plugin:0,-_super,-jira.view.issue,-jira.global,-jira.general,-jira.browse.project,-project.issue.navigator,-atl.general/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true&whisper-enabled=true","startTime":1663.9000000953674,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":1663.9000000953674,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":2315,"responseStart":0,"secureConnectionStart":0},{"duration":669.6999998092651,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/e65b778d185daf5aee24936755b43da6/_/download/contextbatch/js/browser-metrics-plugin.contrib,-_super,-project.issue.navigator,-jira.view.issue,-atl.general/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true&whisper-enabled=true","startTime":1665.3000001907349,"connectEnd":1665.3000001907349,"connectStart":1665.3000001907349,"domainLookupEnd":1665.3000001907349,"domainLookupStart":1665.3000001907349,"fetchStart":1665.3000001907349,"redirectEnd":0,"redirectStart":0,"requestStart":1665.3000001907349,"responseEnd":2335,"responseStart":2335,"secureConnectionStart":1665.3000001907349},{"duration":675.5999999046326,"initiatorType":"script","name":"https://jira.mariadb.org/s/097ae97cb8fbec7d6ea4bbb1f26955b9-CDN/lu2cib/820016/12ta74/be4b45e9cec53099498fa61c8b7acba4/_/download/contextbatch/js/jira.project.sidebar,-_super,-project.issue.navigator,-jira.general,-jira.browse.project,-jira.view.issue,-jira.global,-atl.general,-com.atlassian.jira.projects.sidebar.init/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true&whisper-enabled=true","startTime":1665.8000001907349,"connectEnd":1665.8000001907349,"connectStart":1665.8000001907349,"domainLookupEnd":1665.8000001907349,"domainLookupStart":1665.8000001907349,"fetchStart":1665.8000001907349,"redirectEnd":0,"redirectStart":0,"requestStart":1665.8000001907349,"responseEnd":2341.4000000953674,"responseStart":2341.4000000953674,"secureConnectionStart":1665.8000001907349},{"duration":676.5999999046326,"initiatorType":"script","name":"https://jira.mariadb.org/s/e0bf5781d46ea69fb123572974cf39de-CDN/lu2cib/820016/12ta74/e108c7645258ccb43280ed3404e3e949/_/download/contextbatch/js/com.atlassian.jira.plugins.jira-development-integration-plugin:0,-_super,-jira.view.issue,-jira.global,-jira.general,-jira.browse.project,-project.issue.navigator,-atl.general/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true&whisper-enabled=true","startTime":1666.2000002861023,"connectEnd":1666.2000002861023,"connectStart":1666.2000002861023,"domainLookupEnd":1666.2000002861023,"domainLookupStart":1666.2000002861023,"fetchStart":1666.2000002861023,"redirectEnd":0,"redirectStart":0,"requestStart":1666.2000002861023,"responseEnd":2342.800000190735,"responseStart":2342.800000190735,"secureConnectionStart":1666.2000002861023},{"duration":676.8000001907349,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:issue-table-dnd/jira.webresources:issue-table-dnd.js","startTime":1666.5,"connectEnd":1666.5,"connectStart":1666.5,"domainLookupEnd":1666.5,"domainLookupStart":1666.5,"fetchStart":1666.5,"redirectEnd":0,"redirectStart":0,"requestStart":1666.5,"responseEnd":2343.300000190735,"responseStart":2343.300000190735,"secureConnectionStart":1666.5}],"fetchStart":0,"domainLookupStart":0,"domainLookupEnd":0,"connectStart":0,"connectEnd":0,"requestStart":87,"responseStart":239,"responseEnd":246,"domLoading":251,"domInteractive":2322,"domContentLoadedEventStart":2322,"domContentLoadedEventEnd":2366,"domComplete":3279,"loadEventStart":3279,"loadEventEnd":3280,"userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","marks":[{"name":"bigPipe.sidebar-id.start","time":2295.7000002861023},{"name":"bigPipe.sidebar-id.end","time":2296.5},{"name":"bigPipe.activity-panel-pipe-id.start","time":2296.7000002861023},{"name":"bigPipe.activity-panel-pipe-id.end","time":2297.300000190735},{"name":"activityTabFullyLoaded","time":2376.9000000953674}],"measures":[],"correlationId":"882066d908d852","effectiveType":"4g","downlink":10,"rtt":0,"serverDuration":88,"dbReadsTimeInMs":10,"dbConnsTimeInMs":19,"applicationHash":"9d11dbea5f4be3d4cc21f03a88dd11d8c8687422","experiments":[]}}