The granularity for JSON doc that contains a list of S3 object files is MCS data file. This subtask describes the optimization to reduce the granularity of the S3 meta to partition. The suggested value size for FoundationDB is 10KB and this is enough to store meta for a partition. This would reduce the number of keys in FDB down to 8-10X from the original number of metadata files.
Roman
added a comment - The granularity for JSON doc that contains a list of S3 object files is MCS data file. This subtask describes the optimization to reduce the granularity of the S3 meta to partition. The suggested value size for FoundationDB is 10KB and this is enough to store meta for a partition. This would reduce the number of keys in FDB down to 8-10X from the original number of metadata files.
People
Unassigned
Julien Fritsch
Votes:
0Vote for this issue
Watchers:
2Start watching this issue
Dates
Created:
Updated:
Git Integration
Error rendering 'com.xiplink.jira.git.jira_git_plugin:git-issue-webpanel'. Please contact your Jira administrators.
{"report":{"fcp":726.3000001907349,"ttfb":184.40000009536743,"pageVisibility":"visible","entityId":128766,"key":"jira.project.issue.view-issue","isInitial":true,"threshold":1000,"elementTimings":{},"userDeviceMemory":8,"userDeviceProcessors":64,"apdex":1,"journeyId":"05719d04-e57b-4623-a153-ea462c1592c3","navigationType":0,"readyForUser":771.9000000953674,"redirectCount":0,"resourceLoadedEnd":526.5999999046326,"resourceLoadedStart":191.90000009536743,"resourceTiming":[{"duration":15.5,"initiatorType":"link","name":"https://jira.mariadb.org/s/2c21342762a6a02add1c328bed317ffd-CDN/lu2cib/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/css/_super/batch.css","startTime":191.90000009536743,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":191.90000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":207.40000009536743,"responseStart":0,"secureConnectionStart":0},{"duration":16.100000381469727,"initiatorType":"link","name":"https://jira.mariadb.org/s/7ebd35e77e471bc30ff0eba799ebc151-CDN/lu2cib/820016/12ta74/494e4c556ecbb29f90a3d3b4f09cb99c/_/download/contextbatch/css/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.css?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&slack-enabled=true&whisper-enabled=true","startTime":192.09999990463257,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":192.09999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":208.2000002861023,"responseStart":0,"secureConnectionStart":0},{"duration":85.19999980926514,"initiatorType":"script","name":"https://jira.mariadb.org/s/0917945aaa57108d00c5076fea35e069-CDN/lu2cib/820016/12ta74/0a8bac35585be7fc6c9cc5a0464cd4cf/_/download/contextbatch/js/_super/batch.js?locale=en","startTime":192.40000009536743,"connectEnd":192.40000009536743,"connectStart":192.40000009536743,"domainLookupEnd":192.40000009536743,"domainLookupStart":192.40000009536743,"fetchStart":192.40000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":211.80000019073486,"responseEnd":277.59999990463257,"responseStart":224.09999990463257,"secureConnectionStart":192.40000009536743},{"duration":124.30000019073486,"initiatorType":"script","name":"https://jira.mariadb.org/s/2d8175ec2fa4c816e8023260bd8c1786-CDN/lu2cib/820016/12ta74/494e4c556ecbb29f90a3d3b4f09cb99c/_/download/contextbatch/js/jira.browse.project,project.issue.navigator,jira.view.issue,jira.general,jira.global,atl.general,-_super/batch.js?agile_global_admin_condition=true&jag=true&jira.create.linked.issue=true&locale=en&slack-enabled=true&whisper-enabled=true","startTime":192.59999990463257,"connectEnd":192.59999990463257,"connectStart":192.59999990463257,"domainLookupEnd":192.59999990463257,"domainLookupStart":192.59999990463257,"fetchStart":192.59999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":213.09999990463257,"responseEnd":316.90000009536743,"responseStart":237.09999990463257,"secureConnectionStart":192.59999990463257},{"duration":38.69999980926514,"initiatorType":"script","name":"https://jira.mariadb.org/s/a9324d6758d385eb45c462685ad88f1d-CDN/lu2cib/820016/12ta74/c92c0caa9a024ae85b0ebdbed7fb4bd7/_/download/contextbatch/js/atl.global,-_super/batch.js?locale=en","startTime":192.80000019073486,"connectEnd":192.80000019073486,"connectStart":192.80000019073486,"domainLookupEnd":192.80000019073486,"domainLookupStart":192.80000019073486,"fetchStart":192.80000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":213.7000002861023,"responseEnd":231.5,"responseStart":230.80000019073486,"secureConnectionStart":192.80000019073486},{"duration":53.19999980926514,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-en/jira.webresources:calendar-en.js","startTime":192.90000009536743,"connectEnd":192.90000009536743,"connectStart":192.90000009536743,"domainLookupEnd":192.90000009536743,"domainLookupStart":192.90000009536743,"fetchStart":192.90000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":214.59999990463257,"responseEnd":246.09999990463257,"responseStart":245.59999990463257,"secureConnectionStart":192.90000009536743},{"duration":69.2999997138977,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:calendar-localisation-moment/jira.webresources:calendar-localisation-moment.js","startTime":193.2000002861023,"connectEnd":193.2000002861023,"connectStart":193.2000002861023,"domainLookupEnd":193.2000002861023,"domainLookupStart":193.2000002861023,"fetchStart":193.2000002861023,"redirectEnd":0,"redirectStart":0,"requestStart":216.30000019073486,"responseEnd":262.5,"responseStart":261.90000009536743,"secureConnectionStart":193.2000002861023},{"duration":21.699999809265137,"initiatorType":"link","name":"https://jira.mariadb.org/s/b04b06a02d1959df322d9cded3aeecc1-CDN/lu2cib/820016/12ta74/a2ff6aa845ffc9a1d22fe23d9ee791fc/_/download/contextbatch/css/jira.global.look-and-feel,-_super/batch.css","startTime":193.30000019073486,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":193.30000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":215,"responseStart":0,"secureConnectionStart":0},{"duration":75.5,"initiatorType":"script","name":"https://jira.mariadb.org/rest/api/1.0/shortcuts/820016/47140b6e0a9bc2e4913da06536125810/shortcuts.js?context=issuenavigation&context=issueaction","startTime":193.5,"connectEnd":193.5,"connectStart":193.5,"domainLookupEnd":193.5,"domainLookupStart":193.5,"fetchStart":193.5,"redirectEnd":0,"redirectStart":0,"requestStart":219.40000009536743,"responseEnd":269,"responseStart":268.5,"secureConnectionStart":193.5},{"duration":25.100000381469727,"initiatorType":"link","name":"https://jira.mariadb.org/s/3ac36323ba5e4eb0af2aa7ac7211b4bb-CDN/lu2cib/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/css/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.css?jira.create.linked.issue=true","startTime":193.59999990463257,"connectEnd":0,"connectStart":0,"domainLookupEnd":0,"domainLookupStart":0,"fetchStart":193.59999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":0,"responseEnd":218.7000002861023,"responseStart":0,"secureConnectionStart":0},{"duration":74.59999990463257,"initiatorType":"script","name":"https://jira.mariadb.org/s/5d5e8fe91fbc506585e83ea3b62ccc4b-CDN/lu2cib/820016/12ta74/d176f0986478cc64f24226b3d20c140d/_/download/contextbatch/js/com.atlassian.jira.projects.sidebar.init,-_super,-project.issue.navigator,-jira.view.issue/batch.js?jira.create.linked.issue=true&locale=en","startTime":193.80000019073486,"connectEnd":193.80000019073486,"connectStart":193.80000019073486,"domainLookupEnd":193.80000019073486,"domainLookupStart":193.80000019073486,"fetchStart":193.80000019073486,"redirectEnd":0,"redirectStart":0,"requestStart":219.5,"responseEnd":268.40000009536743,"responseStart":267.30000019073486,"secureConnectionStart":193.80000019073486},{"duration":324.7000002861023,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-js/jira.webresources:bigpipe-js.js","startTime":199.5,"connectEnd":199.5,"connectStart":199.5,"domainLookupEnd":199.5,"domainLookupStart":199.5,"fetchStart":199.5,"redirectEnd":0,"redirectStart":0,"requestStart":263.5,"responseEnd":524.2000002861023,"responseStart":521.3000001907349,"secureConnectionStart":199.5},{"duration":327,"initiatorType":"script","name":"https://jira.mariadb.org/s/d41d8cd98f00b204e9800998ecf8427e-CDN/lu2cib/820016/12ta74/1.0/_/download/batch/jira.webresources:bigpipe-init/jira.webresources:bigpipe-init.js","startTime":199.59999990463257,"connectEnd":199.59999990463257,"connectStart":199.59999990463257,"domainLookupEnd":199.59999990463257,"domainLookupStart":199.59999990463257,"fetchStart":199.59999990463257,"redirectEnd":0,"redirectStart":0,"requestStart":266.7000002861023,"responseEnd":526.5999999046326,"responseStart":521.9000000953674,"secureConnectionStart":199.59999990463257},{"duration":203,"initiatorType":"xmlhttprequest","name":"https://jira.mariadb.org/rest/webResources/1.0/resources","startTime":486.90000009536743,"connectEnd":486.90000009536743,"connectStart":486.90000009536743,"domainLookupEnd":486.90000009536743,"domainLookupStart":486.90000009536743,"fetchStart":486.90000009536743,"redirectEnd":0,"redirectStart":0,"requestStart":652.4000000953674,"responseEnd":689.9000000953674,"responseStart":688.0999999046326,"secureConnectionStart":486.90000009536743}],"fetchStart":0,"domainLookupStart":0,"domainLookupEnd":0,"connectStart":0,"connectEnd":0,"requestStart":44,"responseStart":184,"responseEnd":198,"domLoading":188,"domInteractive":823,"domContentLoadedEventStart":823,"domContentLoadedEventEnd":862,"domComplete":1451,"loadEventStart":1451,"loadEventEnd":1452,"userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","marks":[{"name":"bigPipe.sidebar-id.start","time":803.8000001907349},{"name":"bigPipe.sidebar-id.end","time":804.5999999046326},{"name":"bigPipe.activity-panel-pipe-id.start","time":804.8000001907349},{"name":"bigPipe.activity-panel-pipe-id.end","time":805.7000002861023},{"name":"activityTabFullyLoaded","time":877.8000001907349}],"measures":[],"correlationId":"a762b5b82464ad","effectiveType":"4g","downlink":10,"rtt":0,"serverDuration":81,"dbReadsTimeInMs":10,"dbConnsTimeInMs":18,"applicationHash":"9d11dbea5f4be3d4cc21f03a88dd11d8c8687422","experiments":[]}}
Sub-Task
Major
The granularity for JSON doc that contains a list of S3 object files is MCS data file. This subtask describes the optimization to reduce the granularity of the S3 meta to partition. The suggested value size for FoundationDB is 10KB and this is enough to store meta for a partition. This would reduce the number of keys in FDB down to 8-10X from the original number of metadata files.