Details
-
Bug
-
Status: Closed (View Workflow)
-
Major
-
Resolution: Incomplete
-
10.6.11, 10.6.12
-
None
-
None
-
Debian 11
Description
Hello,
On MariaDB 10.6.11 and 10.6.12 we are seeing a very high memory usage of our databases. The server can be stable for several hours and then suddenly use way more RAM than it should.
See the attached screenshot of how quickly it drops.
Our my.cnf:
[client]
|
default-character-set = utf8mb4
|
port = 3306
|
socket = /run/mysqld/mysqld.sock
|
|
|
[mysqld]
|
basedir = /usr
|
bind-address = 0.0.0.0
|
character-set-server = utf8mb4
|
collation-server = utf8mb4_general_ci
|
connect_timeout = 10
|
datadir = /var/lib/mysql
|
default-storage-engine = innodb
|
expire_logs_days = 10
|
init-connect = "SET NAMES utf8mb4"
|
innodb_buffer_pool_size = 6394M
|
innodb_file_per_table = 1
|
innodb_flush_method = O_DIRECT
|
innodb_log_file_size = 256M
|
interactive_timeout = 900
|
key_buffer_size = 256M
|
max_allowed_packet = 64M
|
max_connections = 200
|
max_heap_table_size = 64M
|
max_statement_time = 1200
|
myisam-recover-options = BACKUP,FORCE
|
myisam_sort_buffer_size = 32M
|
pid-file = /run/mysqld/mysqld.pid
|
port = 3306
|
query_cache_type = 0
|
skip-name-resolve
|
socket = /run/mysqld/mysqld.sock
|
sort_buffer_size = 2M
|
thread_cache_size = 20
|
tmp_table_size = 64M
|
tmpdir = /tmp
|
user = mysql
|
wait_timeout = 900
|
|
|
[mysqldump]
|
default-character-set = utf8mb4
|
max_allowed_packet = 64M
|
quick
|
quote-names
|
|
|
[myisamchk]
|
key_buffer = 64M
|
read_buffer = 1M
|
sort_buffer = 64M
|
write_buffer = 1M
|
The entire /var/lib/mysql directory is 2GB.
We see MariaDB use about 14GB RAM on a 16GB server. With only a 2GB dataset I don't understand how it will use this much memory.
I followed https://mariadb.com/kb/en/debugging-memory-usage/ and also added a 7.dot file that hopefully helps with debugging.
In case it helps, I also have an output of the pmap command:
3074645: /usr/sbin/mariadbd
|
Address Kbytes RSS Dirty Mode Mapping
|
000055da84294000 6280 5544 0 r---- mariadbd
|
000055da848b6000 9816 9212 0 r-x-- mariadbd
|
000055da8524c000 6092 2580 0 r---- mariadbd
|
000055da8583f000 1328 1328 1328 r---- mariadbd
|
000055da8598b000 788 788 452 rw--- mariadbd
|
000055da85a50000 9060 8728 8728 rw--- [ anon ]
|
000055da87819000 20533076 10925760 10925760 rw--- [ anon ]
|
00007faf3ca0f000 896 172 172 rw--- [ anon ]
|
00007faf3caef000 4 0 0 ----- [ anon ]
|
00007faf3caf0000 296 40 40 rw--- [ anon ]
|
00007faf3cb3a000 4 0 0 ----- [ anon ]
|
00007faf3cb3b000 296 40 40 rw--- [ anon ]
|
00007faf3cb85000 4 0 0 ----- [ anon ]
|
00007faf3cb86000 618152 335584 335584 rw--- [ anon ]
|
00007faf62730000 4 0 0 ----- [ anon ]
|
00007faf62731000 296 40 40 rw--- [ anon ]
|
00007faf6277b000 4 0 0 ----- [ anon ]
|
00007faf6277c000 680 424 424 rw--- [ anon ]
|
00007faf62826000 4 0 0 ----- [ anon ]
|
00007faf62827000 296 40 40 rw--- [ anon ]
|
00007faf62871000 4 0 0 ----- [ anon ]
|
00007faf62872000 680 424 424 rw--- [ anon ]
|
00007faf6291c000 4 0 0 ----- [ anon ]
|
00007faf6291d000 187496 103976 103976 rw--- [ anon ]
|
00007faf6e037000 4 0 0 ----- [ anon ]
|
00007faf6e038000 296 48 48 rw--- [ anon ]
|
00007faf6e082000 4 0 0 ----- [ anon ]
|
00007faf6e083000 296 48 48 rw--- [ anon ]
|
00007faf6e0cd000 4 0 0 ----- [ anon ]
|
00007faf6e0ce000 10152 5312 5312 rw--- [ anon ]
|
00007faf6eab8000 4 0 0 ----- [ anon ]
|
00007faf6eab9000 148968 81844 81844 rw--- [ anon ]
|
00007faf77c33000 4 0 0 ----- [ anon ]
|
00007faf77c34000 552 296 296 rw--- [ anon ]
|
00007faf77cbe000 4 0 0 ----- [ anon ]
|
00007faf77cbf000 8192 24 24 rw--- [ anon ]
|
00007faf784c0000 8192 4476 4476 rw--- [ anon ]
|
00007faf78cc0000 4 0 0 ----- [ anon ]
|
00007faf78cc1000 8192 24 24 rw--- [ anon ]
|
00007faf794c1000 4 0 0 ----- [ anon ]
|
00007faf794c2000 8192 24 24 rw--- [ anon ]
|
00007faf79cc3000 8192 4352 4352 rw--- [ anon ]
|
00007faf7a4c3000 4 0 0 ----- [ anon ]
|
00007faf7a4c4000 8192 24 24 rw--- [ anon ]
|
00007faf7acc4000 4 0 0 ----- [ anon ]
|
00007faf7acc5000 8192 24 24 rw--- [ anon ]
|
00007faf7b4d0000 6272 3396 3396 rw--- [ anon ]
|
00007faf7baf0000 4 0 0 ----- [ anon ]
|
00007faf7baf1000 296 48 48 rw--- [ anon ]
|
00007faf7bb3b000 4 0 0 ----- [ anon ]
|
00007faf7bb3c000 1576 792 792 rw--- [ anon ]
|
00007faf7bcc6000 4 0 0 ----- [ anon ]
|
00007faf7bcc7000 8192 24 24 rw--- [ anon ]
|
00007faf7c4dd000 4352 2312 2312 rw--- [ anon ]
|
00007faf7c91d000 4 0 0 ----- [ anon ]
|
00007faf7c91e000 4392 2592 2592 rw--- [ anon ]
|
00007faf7cd68000 4 0 0 ----- [ anon ]
|
00007faf7cd69000 8448 268 268 rw--- [ anon ]
|
00007faf7d5c8000 768 508 508 rw--- [ anon ]
|
00007faf7d688000 4 0 0 ----- [ anon ]
|
00007faf7d689000 424 116 116 rw--- [ anon ]
|
00007faf7d6f3000 4 0 0 ----- [ anon ]
|
00007faf7d6f4000 424 80 80 rw--- [ anon ]
|
00007faf7d75e000 4 0 0 ----- [ anon ]
|
00007faf7d75f000 296 32 32 rw--- [ anon ]
|
00007faf7d7a9000 32768 88 88 rw--- [ anon ]
|
00007faf7f7a9000 256 136 136 rw--- [ anon ]
|
00007faf7f7e9000 6553600 697688 697688 rw--- [ anon ]
|
00007fb10f7e9000 128 72 72 rw--- [ anon ]
|
00007fb10f809000 4 0 0 ----- [ anon ]
|
00007fb10f80a000 8192 12 12 rw--- [ anon ]
|
00007fb11000a000 128 0 0 rw-s- [ anon ]
|
00007fb11002a000 76 0 0 rw-s- [ anon ]
|
00007fb11003d000 128 72 72 rw--- [ anon ]
|
00007fb11005d000 4 0 0 ----- [ anon ]
|
00007fb11005e000 350288 2396 2396 rw--- [ anon ]
|
00007fb125672000 4 0 0 ----- [ anon ]
|
00007fb125673000 192 84 84 rw--- [ anon ]
|
00007fb1256a3000 12 12 0 r---- libnss_files-2.31.so
|
00007fb1256a6000 28 28 0 r-x-- libnss_files-2.31.so
|
00007fb1256ad000 8 0 0 r---- libnss_files-2.31.so
|
00007fb1256af000 4 4 4 r---- libnss_files-2.31.so
|
00007fb1256b0000 4 4 4 rw--- libnss_files-2.31.so
|
00007fb1256b1000 24 0 0 rw--- [ anon ]
|
00007fb1256c2000 2844 2672 2672 rw--- [ anon ]
|
00007fb125989000 16 16 0 r---- libgpg-error.so.0.29.0
|
00007fb12598d000 84 72 0 r-x-- libgpg-error.so.0.29.0
|
00007fb1259a2000 40 40 0 r---- libgpg-error.so.0.29.0
|
00007fb1259ac000 4 0 0 ----- libgpg-error.so.0.29.0
|
00007fb1259ad000 4 4 4 r---- libgpg-error.so.0.29.0
|
00007fb1259ae000 4 4 4 rw--- libgpg-error.so.0.29.0
|
00007fb1259af000 16 16 0 r---- libkmod.so.2.3.6
|
00007fb1259b3000 64 64 0 r-x-- libkmod.so.2.3.6
|
00007fb1259c3000 24 0 0 r---- libkmod.so.2.3.6
|
00007fb1259c9000 4 4 4 r---- libkmod.so.2.3.6
|
00007fb1259ca000 4 4 4 rw--- libkmod.so.2.3.6
|
00007fb1259cb000 8 8 0 r---- libuuid.so.1.3.0
|
00007fb1259cd000 16 16 0 r-x-- libuuid.so.1.3.0
|
00007fb1259d1000 4 0 0 r---- libuuid.so.1.3.0
|
00007fb1259d2000 4 4 4 r---- libuuid.so.1.3.0
|
00007fb1259d3000 4 4 4 rw--- libuuid.so.1.3.0
|
00007fb1259d4000 16 16 0 r---- libudev.so.1.7.0
|
00007fb1259d8000 100 64 0 r-x-- libudev.so.1.7.0
|
00007fb1259f1000 36 0 0 r---- libudev.so.1.7.0
|
00007fb1259fa000 4 4 4 r---- libudev.so.1.7.0
|
00007fb1259fb000 4 4 4 rw--- libudev.so.1.7.0
|
00007fb1259fc000 8 8 8 rw--- [ anon ]
|
00007fb1259fe000 12 12 0 r---- libgcc_s.so.1
|
00007fb125a01000 68 64 0 r-x-- libgcc_s.so.1
|
00007fb125a12000 16 8 0 r---- libgcc_s.so.1
|
00007fb125a16000 4 4 4 r---- libgcc_s.so.1
|
00007fb125a17000 4 4 4 rw--- libgcc_s.so.1
|
00007fb125a18000 48 48 0 r---- libgcrypt.so.20.2.8
|
00007fb125a24000 832 64 0 r-x-- libgcrypt.so.20.2.8
|
00007fb125af4000 244 0 0 r---- libgcrypt.so.20.2.8
|
00007fb125b31000 8 8 8 r---- libgcrypt.so.20.2.8
|
00007fb125b33000 20 20 20 rw--- libgcrypt.so.20.2.8
|
00007fb125b38000 44 44 0 r---- libzstd.so.1.4.8
|
00007fb125b43000 748 64 0 r-x-- libzstd.so.1.4.8
|
00007fb125bfe000 72 0 0 r---- libzstd.so.1.4.8
|
00007fb125c10000 4 0 0 ----- libzstd.so.1.4.8
|
00007fb125c11000 4 4 4 r---- libzstd.so.1.4.8
|
00007fb125c12000 4 4 4 rw--- libzstd.so.1.4.8
|
00007fb125c13000 12 12 0 r---- liblzma.so.5.2.5
|
00007fb125c16000 96 64 0 r-x-- liblzma.so.5.2.5
|
00007fb125c2e000 44 0 0 r---- liblzma.so.5.2.5
|
00007fb125c39000 4 4 4 r---- liblzma.so.5.2.5
|
00007fb125c3a000 4 4 4 rw--- liblzma.so.5.2.5
|
00007fb125c3b000 8 8 0 r---- librt-2.31.so
|
00007fb125c3d000 16 16 0 r-x-- librt-2.31.so
|
00007fb125c41000 8 0 0 r---- librt-2.31.so
|
00007fb125c43000 4 4 4 r---- librt-2.31.so
|
00007fb125c44000 4 4 4 rw--- librt-2.31.so
|
00007fb125c45000 8 8 8 rw--- [ anon ]
|
00007fb125c47000 12 12 0 r---- libdaxctl.so.1.5.0
|
00007fb125c4a000 20 20 0 r-x-- libdaxctl.so.1.5.0
|
00007fb125c4f000 8 0 0 r---- libdaxctl.so.1.5.0
|
00007fb125c51000 4 0 0 ----- libdaxctl.so.1.5.0
|
00007fb125c52000 4 4 4 r---- libdaxctl.so.1.5.0
|
00007fb125c53000 4 4 4 rw--- libdaxctl.so.1.5.0
|
00007fb125c54000 40 40 0 r---- libndctl.so.6.19.1
|
00007fb125c5e000 76 64 0 r-x-- libndctl.so.6.19.1
|
00007fb125c71000 36 0 0 r---- libndctl.so.6.19.1
|
00007fb125c7a000 4 4 4 r---- libndctl.so.6.19.1
|
00007fb125c7b000 4 4 4 rw--- libndctl.so.6.19.1
|
00007fb125c7c000 12 12 0 r---- libunwind.so.8.0.1
|
00007fb125c7f000 36 36 0 r-x-- libunwind.so.8.0.1
|
00007fb125c88000 12 4 0 r---- libunwind.so.8.0.1
|
00007fb125c8b000 4 4 4 r---- libunwind.so.8.0.1
|
00007fb125c8c000 4 4 4 rw--- libunwind.so.8.0.1
|
00007fb125c8d000 40 28 28 rw--- [ anon ]
|
00007fb125c97000 136 136 0 r---- libc-2.31.so
|
00007fb125cb9000 1384 1180 0 r-x-- libc-2.31.so
|
00007fb125e13000 316 308 0 r---- libc-2.31.so
|
00007fb125e62000 16 16 16 r---- libc-2.31.so
|
00007fb125e66000 8 8 8 rw--- libc-2.31.so
|
00007fb125e68000 16 16 16 rw--- [ anon ]
|
00007fb125e6c000 52 52 0 r---- libm-2.31.so
|
00007fb125e79000 616 320 0 r-x-- libm-2.31.so
|
00007fb125f13000 620 176 0 r---- libm-2.31.so
|
00007fb125fae000 4 4 4 r---- libm-2.31.so
|
00007fb125faf000 4 4 4 rw--- libm-2.31.so
|
00007fb125fb0000 600 600 0 r---- libstdc++.so.6.0.28
|
00007fb126046000 880 616 0 r-x-- libstdc++.so.6.0.28
|
00007fb126122000 296 184 0 r---- libstdc++.so.6.0.28
|
00007fb12616c000 44 44 44 r---- libstdc++.so.6.0.28
|
00007fb126177000 12 12 12 rw--- libstdc++.so.6.0.28
|
00007fb12617a000 20 20 20 rw--- [ anon ]
|
00007fb12617f000 4 4 0 r---- libdl-2.31.so
|
00007fb126180000 8 8 0 r-x-- libdl-2.31.so
|
00007fb126182000 4 4 0 r---- libdl-2.31.so
|
00007fb126183000 4 4 4 r---- libdl-2.31.so
|
00007fb126184000 4 4 4 rw--- libdl-2.31.so
|
00007fb126185000 24 24 0 r---- libpthread-2.31.so
|
00007fb12618b000 64 64 0 r-x-- libpthread-2.31.so
|
00007fb12619b000 24 24 0 r---- libpthread-2.31.so
|
00007fb1261a1000 4 4 4 r---- libpthread-2.31.so
|
00007fb1261a2000 4 4 4 rw--- libpthread-2.31.so
|
00007fb1261a3000 16 4 4 rw--- [ anon ]
|
00007fb1261a7000 536 536 0 r---- libcrypto.so.1.1
|
00007fb12622d000 1692 336 0 r-x-- libcrypto.so.1.1
|
00007fb1263d4000 580 64 0 r---- libcrypto.so.1.1
|
00007fb126465000 192 192 192 r---- libcrypto.so.1.1
|
00007fb126495000 8 8 8 rw--- libcrypto.so.1.1
|
00007fb126497000 16 4 4 rw--- [ anon ]
|
00007fb12649b000 116 116 0 r---- libssl.so.1.1
|
00007fb1264b8000 312 64 0 r-x-- libssl.so.1.1
|
00007fb126506000 104 0 0 r---- libssl.so.1.1
|
00007fb126520000 4 0 0 ----- libssl.so.1.1
|
00007fb126521000 36 36 36 r---- libssl.so.1.1
|
00007fb12652a000 16 16 16 rw--- libssl.so.1.1
|
00007fb12652e000 12 12 0 r---- libz.so.1.2.11
|
00007fb126531000 68 64 0 r-x-- libz.so.1.2.11
|
00007fb126542000 24 0 0 r---- libz.so.1.2.11
|
00007fb126548000 4 0 0 ----- libz.so.1.2.11
|
00007fb126549000 4 4 4 r---- libz.so.1.2.11
|
00007fb12654a000 4 4 4 rw--- libz.so.1.2.11
|
00007fb12654b000 72 72 0 r---- libsystemd.so.0.30.0
|
00007fb12655d000 480 276 0 r-x-- libsystemd.so.0.30.0
|
00007fb1265d5000 148 64 0 r---- libsystemd.so.0.30.0
|
00007fb1265fa000 16 16 16 r---- libsystemd.so.0.30.0
|
00007fb1265fe000 4 4 4 rw--- libsystemd.so.0.30.0
|
00007fb1265ff000 12 8 8 rw--- [ anon ]
|
00007fb126602000 12 12 0 r---- libpmem.so.1.0.0
|
00007fb126605000 292 64 0 r-x-- libpmem.so.1.0.0
|
00007fb12664e000 24 0 0 r---- libpmem.so.1.0.0
|
00007fb126654000 4 0 0 ----- libpmem.so.1.0.0
|
00007fb126655000 4 4 4 r---- libpmem.so.1.0.0
|
00007fb126656000 4 4 4 rw--- libpmem.so.1.0.0
|
00007fb126657000 12 12 0 r---- liblz4.so.1.9.3
|
00007fb12665a000 108 64 0 r-x-- liblz4.so.1.9.3
|
00007fb126675000 12 0 0 r---- liblz4.so.1.9.3
|
00007fb126678000 4 4 4 r---- liblz4.so.1.9.3
|
00007fb126679000 4 4 4 rw--- liblz4.so.1.9.3
|
00007fb12667a000 4 4 0 r---- liburing.so.1.0.7
|
00007fb12667b000 4 4 0 r-x-- liburing.so.1.0.7
|
00007fb12667c000 4 0 0 r---- liburing.so.1.0.7
|
00007fb12667d000 4 4 4 r---- liburing.so.1.0.7
|
00007fb12667e000 4 4 4 rw--- liburing.so.1.0.7
|
00007fb12667f000 8 8 0 r---- libcrypt.so.1.1.0
|
00007fb126681000 84 64 0 r-x-- libcrypt.so.1.1.0
|
00007fb126696000 104 0 0 r---- libcrypt.so.1.1.0
|
00007fb1266b0000 4 4 4 r---- libcrypt.so.1.1.0
|
00007fb1266b1000 4 4 4 rw--- libcrypt.so.1.1.0
|
00007fb1266b2000 32 0 0 rw--- [ anon ]
|
00007fb1266ba000 8 8 0 r---- libpcre2-8.so.0.11.0
|
00007fb1266bc000 436 64 0 r-x-- libpcre2-8.so.0.11.0
|
00007fb126729000 180 0 0 r---- libpcre2-8.so.0.11.0
|
00007fb126756000 4 4 4 r---- libpcre2-8.so.0.11.0
|
00007fb126757000 4 4 4 rw--- libpcre2-8.so.0.11.0
|
00007fb126762000 4 4 4 rw--- [ anon ]
|
00007fb126763000 84 84 0 r---- libtcmalloc.so.4.5.6
|
00007fb126778000 152 152 0 r-x-- libtcmalloc.so.4.5.6
|
00007fb12679e000 60 60 0 r---- libtcmalloc.so.4.5.6
|
00007fb1267ad000 4 0 0 ----- libtcmalloc.so.4.5.6
|
00007fb1267ae000 4 4 4 r---- libtcmalloc.so.4.5.6
|
00007fb1267af000 4 4 4 rw--- libtcmalloc.so.4.5.6
|
00007fb1267b0000 1764 1684 1684 rw--- [ anon ]
|
00007fb126969000 4 4 0 r---- ld-2.31.so
|
00007fb12696a000 128 128 4 r-x-- ld-2.31.so
|
00007fb12698a000 32 32 0 r---- ld-2.31.so
|
00007fb126992000 4 4 4 rw--- [ anon ]
|
00007fb126993000 4 4 4 r---- ld-2.31.so
|
00007fb126994000 4 4 4 rw--- ld-2.31.so
|
00007fb126995000 4 4 4 rw--- [ anon ]
|
00007ffc1bb7d000 132 60 60 rw--- [ stack ]
|
00007ffc1bbb3000 16 0 0 r---- [ anon ]
|
00007ffc1bbb7000 8 8 0 r-x-- [ anon ]
|
---------------- ------- ------- -------
|
total kB 28593924 12214000 12189312
|
I also have a gcore core-dump, but since it is 14GB and probably contains sensitive information regarding the database I have not uploaded it at this point.
Is there anything we can do to help narrow down the cause of this issue? Unfortunately I am not knowledgable enough to interprete these debugging results.
