[MDEV-30782] Very high memory usage in MariaDB 10.6.12 Created: 2023-03-03  Updated: 2024-02-07  Resolved: 2024-02-07

Status: Closed
Project: MariaDB Server
Component/s: None
Affects Version/s: 10.6.11, 10.6.12
Fix Version/s: N/A

Type: Bug Priority: Major
Reporter: Niels Hendriks Assignee: Unassigned
Resolution: Incomplete Votes: 0
Labels: None
Environment:

Debian 11

Attachments: Microsoft Word 7.dot     PNG File Screenshot 2023-03-03 at 23.13.52.png    

 Description   

Hello,

On MariaDB 10.6.11 and 10.6.12 we are seeing a very high memory usage of our databases. The server can be stable for several hours and then suddenly use way more RAM than it should.

See the attached screenshot of how quickly it drops.
Our my.cnf:

[client]
 
default-character-set = utf8mb4
 
port = 3306
 
socket = /run/mysqld/mysqld.sock
 
 
 
[mysqld]
 
basedir = /usr
 
bind-address = 0.0.0.0
 
character-set-server = utf8mb4
 
collation-server = utf8mb4_general_ci
 
connect_timeout = 10
 
datadir = /var/lib/mysql
 
default-storage-engine = innodb
 
expire_logs_days = 10
 
init-connect = "SET NAMES utf8mb4"
 
innodb_buffer_pool_size = 6394M
 
innodb_file_per_table = 1
 
innodb_flush_method = O_DIRECT
 
innodb_log_file_size = 256M
 
interactive_timeout = 900
 
key_buffer_size = 256M
 
max_allowed_packet = 64M
 
max_connections = 200
 
max_heap_table_size = 64M
 
max_statement_time = 1200
 
myisam-recover-options = BACKUP,FORCE
 
myisam_sort_buffer_size = 32M
 
pid-file = /run/mysqld/mysqld.pid
 
port = 3306
 
query_cache_type = 0
 
skip-name-resolve
 
socket = /run/mysqld/mysqld.sock
 
sort_buffer_size = 2M
 
thread_cache_size = 20
 
tmp_table_size = 64M
 
tmpdir = /tmp
 
user = mysql
 
wait_timeout = 900
 
 
 
[mysqldump]
 
default-character-set = utf8mb4
 
max_allowed_packet = 64M
 
quick
 
quote-names
 
 
 
[myisamchk]
 
key_buffer = 64M
 
read_buffer = 1M
 
sort_buffer = 64M
 
write_buffer = 1M

The entire /var/lib/mysql directory is 2GB.

We see MariaDB use about 14GB RAM on a 16GB server. With only a 2GB dataset I don't understand how it will use this much memory.

I followed https://mariadb.com/kb/en/debugging-memory-usage/ and also added a 7.dot file that hopefully helps with debugging.

In case it helps, I also have an output of the pmap command:

3074645:   /usr/sbin/mariadbd
 
Address           Kbytes     RSS   Dirty Mode  Mapping
 
000055da84294000    6280    5544       0 r---- mariadbd
 
000055da848b6000    9816    9212       0 r-x-- mariadbd
 
000055da8524c000    6092    2580       0 r---- mariadbd
 
000055da8583f000    1328    1328    1328 r---- mariadbd
 
000055da8598b000     788     788     452 rw--- mariadbd
 
000055da85a50000    9060    8728    8728 rw---   [ anon ]
 
000055da87819000 20533076 10925760 10925760 rw---   [ anon ]
 
00007faf3ca0f000     896     172     172 rw---   [ anon ]
 
00007faf3caef000       4       0       0 -----   [ anon ]
 
00007faf3caf0000     296      40      40 rw---   [ anon ]
 
00007faf3cb3a000       4       0       0 -----   [ anon ]
 
00007faf3cb3b000     296      40      40 rw---   [ anon ]
 
00007faf3cb85000       4       0       0 -----   [ anon ]
 
00007faf3cb86000  618152  335584  335584 rw---   [ anon ]
 
00007faf62730000       4       0       0 -----   [ anon ]
 
00007faf62731000     296      40      40 rw---   [ anon ]
 
00007faf6277b000       4       0       0 -----   [ anon ]
 
00007faf6277c000     680     424     424 rw---   [ anon ]
 
00007faf62826000       4       0       0 -----   [ anon ]
 
00007faf62827000     296      40      40 rw---   [ anon ]
 
00007faf62871000       4       0       0 -----   [ anon ]
 
00007faf62872000     680     424     424 rw---   [ anon ]
 
00007faf6291c000       4       0       0 -----   [ anon ]
 
00007faf6291d000  187496  103976  103976 rw---   [ anon ]
 
00007faf6e037000       4       0       0 -----   [ anon ]
 
00007faf6e038000     296      48      48 rw---   [ anon ]
 
00007faf6e082000       4       0       0 -----   [ anon ]
 
00007faf6e083000     296      48      48 rw---   [ anon ]
 
00007faf6e0cd000       4       0       0 -----   [ anon ]
 
00007faf6e0ce000   10152    5312    5312 rw---   [ anon ]
 
00007faf6eab8000       4       0       0 -----   [ anon ]
 
00007faf6eab9000  148968   81844   81844 rw---   [ anon ]
 
00007faf77c33000       4       0       0 -----   [ anon ]
 
00007faf77c34000     552     296     296 rw---   [ anon ]
 
00007faf77cbe000       4       0       0 -----   [ anon ]
 
00007faf77cbf000    8192      24      24 rw---   [ anon ]
 
00007faf784c0000    8192    4476    4476 rw---   [ anon ]
 
00007faf78cc0000       4       0       0 -----   [ anon ]
 
00007faf78cc1000    8192      24      24 rw---   [ anon ]
 
00007faf794c1000       4       0       0 -----   [ anon ]
 
00007faf794c2000    8192      24      24 rw---   [ anon ]
 
00007faf79cc3000    8192    4352    4352 rw---   [ anon ]
 
00007faf7a4c3000       4       0       0 -----   [ anon ]
 
00007faf7a4c4000    8192      24      24 rw---   [ anon ]
 
00007faf7acc4000       4       0       0 -----   [ anon ]
 
00007faf7acc5000    8192      24      24 rw---   [ anon ]
 
00007faf7b4d0000    6272    3396    3396 rw---   [ anon ]
 
00007faf7baf0000       4       0       0 -----   [ anon ]
 
00007faf7baf1000     296      48      48 rw---   [ anon ]
 
00007faf7bb3b000       4       0       0 -----   [ anon ]
 
00007faf7bb3c000    1576     792     792 rw---   [ anon ]
 
00007faf7bcc6000       4       0       0 -----   [ anon ]
 
00007faf7bcc7000    8192      24      24 rw---   [ anon ]
 
00007faf7c4dd000    4352    2312    2312 rw---   [ anon ]
 
00007faf7c91d000       4       0       0 -----   [ anon ]
 
00007faf7c91e000    4392    2592    2592 rw---   [ anon ]
 
00007faf7cd68000       4       0       0 -----   [ anon ]
 
00007faf7cd69000    8448     268     268 rw---   [ anon ]
 
00007faf7d5c8000     768     508     508 rw---   [ anon ]
 
00007faf7d688000       4       0       0 -----   [ anon ]
 
00007faf7d689000     424     116     116 rw---   [ anon ]
 
00007faf7d6f3000       4       0       0 -----   [ anon ]
 
00007faf7d6f4000     424      80      80 rw---   [ anon ]
 
00007faf7d75e000       4       0       0 -----   [ anon ]
 
00007faf7d75f000     296      32      32 rw---   [ anon ]
 
00007faf7d7a9000   32768      88      88 rw---   [ anon ]
 
00007faf7f7a9000     256     136     136 rw---   [ anon ]
 
00007faf7f7e9000 6553600  697688  697688 rw---   [ anon ]
 
00007fb10f7e9000     128      72      72 rw---   [ anon ]
 
00007fb10f809000       4       0       0 -----   [ anon ]
 
00007fb10f80a000    8192      12      12 rw---   [ anon ]
 
00007fb11000a000     128       0       0 rw-s-   [ anon ]
 
00007fb11002a000      76       0       0 rw-s-   [ anon ]
 
00007fb11003d000     128      72      72 rw---   [ anon ]
 
00007fb11005d000       4       0       0 -----   [ anon ]
 
00007fb11005e000  350288    2396    2396 rw---   [ anon ]
 
00007fb125672000       4       0       0 -----   [ anon ]
 
00007fb125673000     192      84      84 rw---   [ anon ]
 
00007fb1256a3000      12      12       0 r---- libnss_files-2.31.so
 
00007fb1256a6000      28      28       0 r-x-- libnss_files-2.31.so
 
00007fb1256ad000       8       0       0 r---- libnss_files-2.31.so
 
00007fb1256af000       4       4       4 r---- libnss_files-2.31.so
 
00007fb1256b0000       4       4       4 rw--- libnss_files-2.31.so
 
00007fb1256b1000      24       0       0 rw---   [ anon ]
 
00007fb1256c2000    2844    2672    2672 rw---   [ anon ]
 
00007fb125989000      16      16       0 r---- libgpg-error.so.0.29.0
 
00007fb12598d000      84      72       0 r-x-- libgpg-error.so.0.29.0
 
00007fb1259a2000      40      40       0 r---- libgpg-error.so.0.29.0
 
00007fb1259ac000       4       0       0 ----- libgpg-error.so.0.29.0
 
00007fb1259ad000       4       4       4 r---- libgpg-error.so.0.29.0
 
00007fb1259ae000       4       4       4 rw--- libgpg-error.so.0.29.0
 
00007fb1259af000      16      16       0 r---- libkmod.so.2.3.6
 
00007fb1259b3000      64      64       0 r-x-- libkmod.so.2.3.6
 
00007fb1259c3000      24       0       0 r---- libkmod.so.2.3.6
 
00007fb1259c9000       4       4       4 r---- libkmod.so.2.3.6
 
00007fb1259ca000       4       4       4 rw--- libkmod.so.2.3.6
 
00007fb1259cb000       8       8       0 r---- libuuid.so.1.3.0
 
00007fb1259cd000      16      16       0 r-x-- libuuid.so.1.3.0
 
00007fb1259d1000       4       0       0 r---- libuuid.so.1.3.0
 
00007fb1259d2000       4       4       4 r---- libuuid.so.1.3.0
 
00007fb1259d3000       4       4       4 rw--- libuuid.so.1.3.0
 
00007fb1259d4000      16      16       0 r---- libudev.so.1.7.0
 
00007fb1259d8000     100      64       0 r-x-- libudev.so.1.7.0
 
00007fb1259f1000      36       0       0 r---- libudev.so.1.7.0
 
00007fb1259fa000       4       4       4 r---- libudev.so.1.7.0
 
00007fb1259fb000       4       4       4 rw--- libudev.so.1.7.0
 
00007fb1259fc000       8       8       8 rw---   [ anon ]
 
00007fb1259fe000      12      12       0 r---- libgcc_s.so.1
 
00007fb125a01000      68      64       0 r-x-- libgcc_s.so.1
 
00007fb125a12000      16       8       0 r---- libgcc_s.so.1
 
00007fb125a16000       4       4       4 r---- libgcc_s.so.1
 
00007fb125a17000       4       4       4 rw--- libgcc_s.so.1
 
00007fb125a18000      48      48       0 r---- libgcrypt.so.20.2.8
 
00007fb125a24000     832      64       0 r-x-- libgcrypt.so.20.2.8
 
00007fb125af4000     244       0       0 r---- libgcrypt.so.20.2.8
 
00007fb125b31000       8       8       8 r---- libgcrypt.so.20.2.8
 
00007fb125b33000      20      20      20 rw--- libgcrypt.so.20.2.8
 
00007fb125b38000      44      44       0 r---- libzstd.so.1.4.8
 
00007fb125b43000     748      64       0 r-x-- libzstd.so.1.4.8
 
00007fb125bfe000      72       0       0 r---- libzstd.so.1.4.8
 
00007fb125c10000       4       0       0 ----- libzstd.so.1.4.8
 
00007fb125c11000       4       4       4 r---- libzstd.so.1.4.8
 
00007fb125c12000       4       4       4 rw--- libzstd.so.1.4.8
 
00007fb125c13000      12      12       0 r---- liblzma.so.5.2.5
 
00007fb125c16000      96      64       0 r-x-- liblzma.so.5.2.5
 
00007fb125c2e000      44       0       0 r---- liblzma.so.5.2.5
 
00007fb125c39000       4       4       4 r---- liblzma.so.5.2.5
 
00007fb125c3a000       4       4       4 rw--- liblzma.so.5.2.5
 
00007fb125c3b000       8       8       0 r---- librt-2.31.so
 
00007fb125c3d000      16      16       0 r-x-- librt-2.31.so
 
00007fb125c41000       8       0       0 r---- librt-2.31.so
 
00007fb125c43000       4       4       4 r---- librt-2.31.so
 
00007fb125c44000       4       4       4 rw--- librt-2.31.so
 
00007fb125c45000       8       8       8 rw---   [ anon ]
 
00007fb125c47000      12      12       0 r---- libdaxctl.so.1.5.0
 
00007fb125c4a000      20      20       0 r-x-- libdaxctl.so.1.5.0
 
00007fb125c4f000       8       0       0 r---- libdaxctl.so.1.5.0
 
00007fb125c51000       4       0       0 ----- libdaxctl.so.1.5.0
 
00007fb125c52000       4       4       4 r---- libdaxctl.so.1.5.0
 
00007fb125c53000       4       4       4 rw--- libdaxctl.so.1.5.0
 
00007fb125c54000      40      40       0 r---- libndctl.so.6.19.1
 
00007fb125c5e000      76      64       0 r-x-- libndctl.so.6.19.1
 
00007fb125c71000      36       0       0 r---- libndctl.so.6.19.1
 
00007fb125c7a000       4       4       4 r---- libndctl.so.6.19.1
 
00007fb125c7b000       4       4       4 rw--- libndctl.so.6.19.1
 
00007fb125c7c000      12      12       0 r---- libunwind.so.8.0.1
 
00007fb125c7f000      36      36       0 r-x-- libunwind.so.8.0.1
 
00007fb125c88000      12       4       0 r---- libunwind.so.8.0.1
 
00007fb125c8b000       4       4       4 r---- libunwind.so.8.0.1
 
00007fb125c8c000       4       4       4 rw--- libunwind.so.8.0.1
 
00007fb125c8d000      40      28      28 rw---   [ anon ]
 
00007fb125c97000     136     136       0 r---- libc-2.31.so
 
00007fb125cb9000    1384    1180       0 r-x-- libc-2.31.so
 
00007fb125e13000     316     308       0 r---- libc-2.31.so
 
00007fb125e62000      16      16      16 r---- libc-2.31.so
 
00007fb125e66000       8       8       8 rw--- libc-2.31.so
 
00007fb125e68000      16      16      16 rw---   [ anon ]
 
00007fb125e6c000      52      52       0 r---- libm-2.31.so
 
00007fb125e79000     616     320       0 r-x-- libm-2.31.so
 
00007fb125f13000     620     176       0 r---- libm-2.31.so
 
00007fb125fae000       4       4       4 r---- libm-2.31.so
 
00007fb125faf000       4       4       4 rw--- libm-2.31.so
 
00007fb125fb0000     600     600       0 r---- libstdc++.so.6.0.28
 
00007fb126046000     880     616       0 r-x-- libstdc++.so.6.0.28
 
00007fb126122000     296     184       0 r---- libstdc++.so.6.0.28
 
00007fb12616c000      44      44      44 r---- libstdc++.so.6.0.28
 
00007fb126177000      12      12      12 rw--- libstdc++.so.6.0.28
 
00007fb12617a000      20      20      20 rw---   [ anon ]
 
00007fb12617f000       4       4       0 r---- libdl-2.31.so
 
00007fb126180000       8       8       0 r-x-- libdl-2.31.so
 
00007fb126182000       4       4       0 r---- libdl-2.31.so
 
00007fb126183000       4       4       4 r---- libdl-2.31.so
 
00007fb126184000       4       4       4 rw--- libdl-2.31.so
 
00007fb126185000      24      24       0 r---- libpthread-2.31.so
 
00007fb12618b000      64      64       0 r-x-- libpthread-2.31.so
 
00007fb12619b000      24      24       0 r---- libpthread-2.31.so
 
00007fb1261a1000       4       4       4 r---- libpthread-2.31.so
 
00007fb1261a2000       4       4       4 rw--- libpthread-2.31.so
 
00007fb1261a3000      16       4       4 rw---   [ anon ]
 
00007fb1261a7000     536     536       0 r---- libcrypto.so.1.1
 
00007fb12622d000    1692     336       0 r-x-- libcrypto.so.1.1
 
00007fb1263d4000     580      64       0 r---- libcrypto.so.1.1
 
00007fb126465000     192     192     192 r---- libcrypto.so.1.1
 
00007fb126495000       8       8       8 rw--- libcrypto.so.1.1
 
00007fb126497000      16       4       4 rw---   [ anon ]
 
00007fb12649b000     116     116       0 r---- libssl.so.1.1
 
00007fb1264b8000     312      64       0 r-x-- libssl.so.1.1
 
00007fb126506000     104       0       0 r---- libssl.so.1.1
 
00007fb126520000       4       0       0 ----- libssl.so.1.1
 
00007fb126521000      36      36      36 r---- libssl.so.1.1
 
00007fb12652a000      16      16      16 rw--- libssl.so.1.1
 
00007fb12652e000      12      12       0 r---- libz.so.1.2.11
 
00007fb126531000      68      64       0 r-x-- libz.so.1.2.11
 
00007fb126542000      24       0       0 r---- libz.so.1.2.11
 
00007fb126548000       4       0       0 ----- libz.so.1.2.11
 
00007fb126549000       4       4       4 r---- libz.so.1.2.11
 
00007fb12654a000       4       4       4 rw--- libz.so.1.2.11
 
00007fb12654b000      72      72       0 r---- libsystemd.so.0.30.0
 
00007fb12655d000     480     276       0 r-x-- libsystemd.so.0.30.0
 
00007fb1265d5000     148      64       0 r---- libsystemd.so.0.30.0
 
00007fb1265fa000      16      16      16 r---- libsystemd.so.0.30.0
 
00007fb1265fe000       4       4       4 rw--- libsystemd.so.0.30.0
 
00007fb1265ff000      12       8       8 rw---   [ anon ]
 
00007fb126602000      12      12       0 r---- libpmem.so.1.0.0
 
00007fb126605000     292      64       0 r-x-- libpmem.so.1.0.0
 
00007fb12664e000      24       0       0 r---- libpmem.so.1.0.0
 
00007fb126654000       4       0       0 ----- libpmem.so.1.0.0
 
00007fb126655000       4       4       4 r---- libpmem.so.1.0.0
 
00007fb126656000       4       4       4 rw--- libpmem.so.1.0.0
 
00007fb126657000      12      12       0 r---- liblz4.so.1.9.3
 
00007fb12665a000     108      64       0 r-x-- liblz4.so.1.9.3
 
00007fb126675000      12       0       0 r---- liblz4.so.1.9.3
 
00007fb126678000       4       4       4 r---- liblz4.so.1.9.3
 
00007fb126679000       4       4       4 rw--- liblz4.so.1.9.3
 
00007fb12667a000       4       4       0 r---- liburing.so.1.0.7
 
00007fb12667b000       4       4       0 r-x-- liburing.so.1.0.7
 
00007fb12667c000       4       0       0 r---- liburing.so.1.0.7
 
00007fb12667d000       4       4       4 r---- liburing.so.1.0.7
 
00007fb12667e000       4       4       4 rw--- liburing.so.1.0.7
 
00007fb12667f000       8       8       0 r---- libcrypt.so.1.1.0
 
00007fb126681000      84      64       0 r-x-- libcrypt.so.1.1.0
 
00007fb126696000     104       0       0 r---- libcrypt.so.1.1.0
 
00007fb1266b0000       4       4       4 r---- libcrypt.so.1.1.0
 
00007fb1266b1000       4       4       4 rw--- libcrypt.so.1.1.0
 
00007fb1266b2000      32       0       0 rw---   [ anon ]
 
00007fb1266ba000       8       8       0 r---- libpcre2-8.so.0.11.0
 
00007fb1266bc000     436      64       0 r-x-- libpcre2-8.so.0.11.0
 
00007fb126729000     180       0       0 r---- libpcre2-8.so.0.11.0
 
00007fb126756000       4       4       4 r---- libpcre2-8.so.0.11.0
 
00007fb126757000       4       4       4 rw--- libpcre2-8.so.0.11.0
 
00007fb126762000       4       4       4 rw---   [ anon ]
 
00007fb126763000      84      84       0 r---- libtcmalloc.so.4.5.6
 
00007fb126778000     152     152       0 r-x-- libtcmalloc.so.4.5.6
 
00007fb12679e000      60      60       0 r---- libtcmalloc.so.4.5.6
 
00007fb1267ad000       4       0       0 ----- libtcmalloc.so.4.5.6
 
00007fb1267ae000       4       4       4 r---- libtcmalloc.so.4.5.6
 
00007fb1267af000       4       4       4 rw--- libtcmalloc.so.4.5.6
 
00007fb1267b0000    1764    1684    1684 rw---   [ anon ]
 
00007fb126969000       4       4       0 r---- ld-2.31.so
 
00007fb12696a000     128     128       4 r-x-- ld-2.31.so
 
00007fb12698a000      32      32       0 r---- ld-2.31.so
 
00007fb126992000       4       4       4 rw---   [ anon ]
 
00007fb126993000       4       4       4 r---- ld-2.31.so
 
00007fb126994000       4       4       4 rw--- ld-2.31.so
 
00007fb126995000       4       4       4 rw---   [ anon ]
 
00007ffc1bb7d000     132      60      60 rw---   [ stack ]
 
00007ffc1bbb3000      16       0       0 r----   [ anon ]
 
00007ffc1bbb7000       8       8       0 r-x--   [ anon ]
 
---------------- ------- ------- -------
 
total kB         28593924 12214000 12189312

I also have a gcore core-dump, but since it is 14GB and probably contains sensitive information regarding the database I have not uploaded it at this point.

Is there anything we can do to help narrow down the cause of this issue? Unfortunately I am not knowledgable enough to interprete these debugging results.

 Comments   
Comment by Daniel Black [ 2023-03-04 ]

https://github.com/iovisor/bcc/blob/master/tools/memleak_example.txt maybe. Also samples of show global status at times might be useful.

Comment by Niels Hendriks [ 2023-03-06 ]

Hi Daniel,

Perhaps I am using meamleak incorrectly. But I don't see any significant output:

memleak-bpfcc  1 10 -p $(pidof mariadbd)
 
In file included from <built-in>:2:
 
In file included from /virtual/include/bcc/bpf.h:12:
 
In file included from include/linux/types.h:6:
 
In file included from include/uapi/linux/types.h:14:
 
In file included from include/uapi/linux/posix_types.h:5:
 
In file included from include/linux/stddef.h:5:
 
In file included from include/uapi/linux/stddef.h:2:
 
In file included from include/linux/compiler_types.h:69:
 
include/linux/compiler-clang.h:51:9: warning: '__HAVE_BUILTIN_BSWAP32__' macro redefined [-Wmacro-redefined]
 
#define __HAVE_BUILTIN_BSWAP32__
 
        ^
 
<command line>:4:9: note: previous definition is here
 
#define __HAVE_BUILTIN_BSWAP32__ 1
 
        ^
 
In file included from <built-in>:2:
 
In file included from /virtual/include/bcc/bpf.h:12:
 
In file included from include/linux/types.h:6:
 
In file included from include/uapi/linux/types.h:14:
 
In file included from include/uapi/linux/posix_types.h:5:
 
In file included from include/linux/stddef.h:5:
 
In file included from include/uapi/linux/stddef.h:2:
 
In file included from include/linux/compiler_types.h:69:
 
include/linux/compiler-clang.h:52:9: warning: '__HAVE_BUILTIN_BSWAP64__' macro redefined [-Wmacro-redefined]
 
#define __HAVE_BUILTIN_BSWAP64__
 
        ^
 
<command line>:5:9: note: previous definition is here
 
#define __HAVE_BUILTIN_BSWAP64__ 1
 
        ^
 
In file included from <built-in>:2:
 
In file included from /virtual/include/bcc/bpf.h:12:
 
In file included from include/linux/types.h:6:
 
In file included from include/uapi/linux/types.h:14:
 
In file included from include/uapi/linux/posix_types.h:5:
 
In file included from include/linux/stddef.h:5:
 
In file included from include/uapi/linux/stddef.h:2:
 
In file included from include/linux/compiler_types.h:69:
 
include/linux/compiler-clang.h:53:9: warning: '__HAVE_BUILTIN_BSWAP16__' macro redefined [-Wmacro-redefined]
 
#define __HAVE_BUILTIN_BSWAP16__
 
        ^
 
<command line>:3:9: note: previous definition is here
 
#define __HAVE_BUILTIN_BSWAP16__ 1
 
        ^
 
3 warnings generated.
 
Attaching to pid 163020, Ctrl+C to quit.
 
[10:27:20] Top 10 stacks with outstanding allocations:
 
[10:27:21] Top 10 stacks with outstanding allocations:
 
[10:27:22] Top 10 stacks with outstanding allocations:
 
[10:27:23] Top 10 stacks with outstanding allocations:
 
[10:27:24] Top 10 stacks with outstanding allocations:
 
[10:27:25] Top 10 stacks with outstanding allocations:
 
[10:27:26] Top 10 stacks with outstanding allocations:
 
[10:27:27] Top 10 stacks with outstanding allocations:
 
[10:27:28] Top 10 stacks with outstanding allocations:
 
[10:27:29] Top 10 stacks with outstanding allocations:

Here's the `show global status` output with +-10GB memory in-use:

Variable_name   Value
 
Aborted_clients 0
 
Aborted_connects        0
 
Aborted_connects_preauth        0
 
Access_denied_errors    0
 
Acl_column_grants       0
 
Acl_database_grants     15
 
Acl_function_grants     0
 
Acl_procedure_grants    0
 
Acl_package_spec_grants 0
 
Acl_package_body_grants 0
 
Acl_proxy_users 0
 
Acl_role_grants 0
 
Acl_roles       0
 
Acl_table_grants        1
 
Acl_users       23
 
Aria_pagecache_blocks_not_flushed       2
 
Aria_pagecache_blocks_unused    15643
 
Aria_pagecache_blocks_used      323
 
Aria_pagecache_read_requests    9612889
 
Aria_pagecache_reads    946359
 
Aria_pagecache_write_requests   2033179
 
Aria_pagecache_writes   2033105
 
Aria_transaction_log_syncs      924
 
Binlog_commits  0
 
Binlog_group_commits    0
 
Binlog_group_commit_trigger_count       0
 
Binlog_group_commit_trigger_lock_wait   0
 
Binlog_group_commit_trigger_timeout     0
 
Binlog_snapshot_file
 
Binlog_snapshot_position        0
 
Binlog_bytes_written    0
 
Binlog_cache_disk_use   0
 
Binlog_cache_use        0
 
Binlog_stmt_cache_disk_use      0
 
Binlog_stmt_cache_use   0
 
Busy_time       0.000000
 
Bytes_received  7354658204
 
Bytes_sent      16949638091
 
Column_compressions     0
 
Column_decompressions   0
 
Com_admin_commands      0
 
Com_alter_db    0
 
Com_alter_db_upgrade    0
 
Com_alter_event 0
 
Com_alter_function      0
 
Com_alter_procedure     0
 
Com_alter_server        0
 
Com_alter_sequence      0
 
Com_alter_table 0
 
Com_alter_tablespace    0
 
Com_alter_user  0
 
Com_analyze     0
 
Com_assign_to_keycache  0
 
Com_backup      0
 
Com_backup_lock 0
 
Com_begin       36
 
Com_binlog      0
 
Com_call_procedure      0
 
Com_change_db   1104
 
Com_change_master       0
 
Com_check       0
 
Com_checksum    0
 
Com_commit      20
 
Com_compound_sql        0
 
Com_create_db   0
 
Com_create_event        0
 
Com_create_function     0
 
Com_create_index        0
 
Com_create_package      0
 
Com_create_package_body 0
 
Com_create_procedure    0
 
Com_create_role 0
 
Com_create_sequence     0
 
Com_create_server       0
 
Com_create_table        0
 
Com_create_temporary_table      0
 
Com_create_trigger      0
 
Com_create_udf  0
 
Com_create_user 0
 
Com_create_view 0
 
Com_dealloc_sql 0
 
Com_delete      469
 
Com_delete_multi        2
 
Com_do  0
 
Com_drop_db     0
 
Com_drop_event  0
 
Com_drop_function       0
 
Com_drop_index  0
 
Com_drop_procedure      0
 
Com_drop_package        0
 
Com_drop_package_body   0
 
Com_drop_role   0
 
Com_drop_server 0
 
Com_drop_sequence       0
 
Com_drop_table  0
 
Com_drop_temporary_table        0
 
Com_drop_trigger        0
 
Com_drop_user   0
 
Com_drop_view   0
 
Com_empty_query 0
 
Com_execute_immediate   0
 
Com_execute_sql 0
 
Com_flush       0
 
Com_get_diagnostics     0
 
Com_grant       0
 
Com_grant_role  0
 
Com_ha_close    0
 
Com_ha_open     0
 
Com_ha_read     0
 
Com_help        0
 
Com_insert      82216
 
Com_insert_select       0
 
Com_install_plugin      0
 
Com_kill        0
 
Com_load        0
 
Com_lock_tables 0
 
Com_optimize    0
 
Com_preload_keys        0
 
Com_prepare_sql 0
 
Com_purge       0
 
Com_purge_before_date   0
 
Com_release_savepoint   14
 
Com_rename_table        0
 
Com_rename_user 0
 
Com_repair      0
 
Com_replace     0
 
Com_replace_select      0
 
Com_reset       0
 
Com_resignal    0
 
Com_revoke      0
 
Com_revoke_all  0
 
Com_revoke_role 0
 
Com_rollback    2
 
Com_rollback_to_savepoint       960
 
Com_savepoint   14
 
Com_select      3353853
 
Com_set_option  45317
 
Com_show_authors        0
 
Com_show_binlog_events  0
 
Com_show_binlogs        0
 
Com_show_charsets       0
 
Com_show_collations     0
 
Com_show_contributors   0
 
Com_show_create_db      0
 
Com_show_create_event   0
 
Com_show_create_func    44
 
Com_show_create_package 0
 
Com_show_create_package_body    0
 
Com_show_create_proc    56
 
Com_show_create_table   1224
 
Com_show_create_trigger 0
 
Com_show_create_user    0
 
Com_show_databases      2
 
Com_show_engine_logs    0
 
Com_show_engine_mutex   0
 
Com_show_engine_status  0
 
Com_show_errors 0
 
Com_show_events 0
 
Com_show_explain        0
 
Com_show_fields 2447
 
Com_show_function_status        14
 
Com_show_generic        0
 
Com_show_grants 529
 
Com_show_keys   0
 
Com_show_binlog_status  0
 
Com_show_open_tables    0
 
Com_show_package_status 14
 
Com_show_package_body_status    14
 
Com_show_plugins        0
 
Com_show_privileges     0
 
Com_show_procedure_status       14
 
Com_show_processlist    4
 
Com_show_profile        0
 
Com_show_profiles       0
 
Com_show_relaylog_events        0
 
Com_show_slave_hosts    0
 
Com_show_slave_status   0
 
Com_show_status 606
 
Com_show_storage_engines        3
 
Com_show_table_status   0
 
Com_show_tables 18
 
Com_show_triggers       0
 
Com_show_variables      3
 
Com_show_warnings       0
 
Com_shutdown    0
 
Com_signal      0
 
Com_start_all_slaves    0
 
Com_start_slave 0
 
Com_stmt_close  0
 
Com_stmt_execute        0
 
Com_stmt_fetch  0
 
Com_stmt_prepare        0
 
Com_stmt_reprepare      0
 
Com_stmt_reset  0
 
Com_stmt_send_long_data 0
 
Com_stop_all_slaves     0
 
Com_stop_slave  0
 
Com_truncate    0
 
Com_uninstall_plugin    0
 
Com_unlock_tables       14
 
Com_update      8625
 
Com_update_multi        0
 
Com_xa_commit   0
 
Com_xa_end      0
 
Com_xa_prepare  0
 
Com_xa_recover  0
 
Com_xa_rollback 0
 
Com_xa_start    0
 
Compression     OFF
 
Connection_errors_accept        0
 
Connection_errors_internal      0
 
Connection_errors_max_connections       0
 
Connection_errors_peer_address  0
 
Connection_errors_select        0
 
Connection_errors_tcpwrap       0
 
Connections     20390
 
Cpu_time        0.000000
 
Created_tmp_disk_tables 945784
 
Created_tmp_files       4
 
Created_tmp_tables      7226713
 
Delayed_errors  0
 
Delayed_insert_threads  0
 
Delayed_writes  0
 
Delete_scan     2
 
Empty_queries   103541
 
Executed_events 0
 
Executed_triggers       0
 
Feature_application_time_periods        0
 
Feature_check_constraint        1
 
Feature_custom_aggregate_functions      0
 
Feature_delay_key_write 0
 
Feature_dynamic_columns 0
 
Feature_fulltext        0
 
Feature_gis     0
 
Feature_insert_returning        0
 
Feature_invisible_columns       0
 
Feature_json    2468
 
Feature_locale  0
 
Feature_subquery        6035724
 
Feature_system_versioning       0
 
Feature_timezone        14
 
Feature_trigger 0
 
Feature_window_functions        0
 
Feature_xml     0
 
Handler_commit  3441021
 
Handler_delete  990
 
Handler_discover        54
 
Handler_external_lock   0
 
Handler_icp_attempts    1769055029
 
Handler_icp_match       43399103
 
Handler_mrr_init        0
 
Handler_mrr_key_refills 0
 
Handler_mrr_rowid_refills       0
 
Handler_prepare 0
 
Handler_read_first      464983
 
Handler_read_key        679447325
 
Handler_read_last       1
 
Handler_read_next       207154592
 
Handler_read_prev       2484306
 
Handler_read_retry      0
 
Handler_read_rnd        9077472
 
Handler_read_rnd_deleted        3
 
Handler_read_rnd_next   95878572
 
Handler_rollback        52
 
Handler_savepoint       14
 
Handler_savepoint_rollback      960
 
Handler_tmp_delete      0
 
Handler_tmp_update      0
 
Handler_tmp_write       279468327
 
Handler_update  79787
 
Handler_write   92003
 
Innodb_adaptive_hash_hash_searches      0
 
Innodb_adaptive_hash_non_hash_searches  0
 
Innodb_background_log_sync      72408
 
Innodb_buffer_pool_dump_status
 
Innodb_buffer_pool_load_status  Buffer pool(s) load completed at 230305 14:13:36
 
Innodb_buffer_pool_resize_status
 
Innodb_buffer_pool_load_incomplete      OFF
 
Innodb_buffer_pool_pages_data   40392
 
Innodb_buffer_pool_bytes_data   661782528
 
Innodb_buffer_pool_pages_dirty  1478
 
Innodb_buffer_pool_bytes_dirty  24215552
 
Innodb_buffer_pool_pages_flushed        0
 
Innodb_buffer_pool_pages_free   365208
 
Innodb_buffer_pool_pages_made_not_young 0
 
Innodb_buffer_pool_pages_made_young     2
 
Innodb_buffer_pool_pages_misc   0
 
Innodb_buffer_pool_pages_old    14890
 
Innodb_buffer_pool_pages_total  405600
 
Innodb_buffer_pool_pages_lru_flushed    0
 
Innodb_buffer_pool_pages_lru_freed      0
 
Innodb_buffer_pool_read_ahead_rnd       0
 
Innodb_buffer_pool_read_ahead   0
 
Innodb_buffer_pool_read_ahead_evicted   0
 
Innodb_buffer_pool_read_requests        811337470
 
Innodb_buffer_pool_reads        39245
 
Innodb_buffer_pool_wait_free    0
 
Innodb_buffer_pool_write_requests       904161
 
Innodb_checkpoint_age   32739858
 
Innodb_checkpoint_max_age       216723273
 
Innodb_data_fsyncs      91581
 
Innodb_data_pending_fsyncs      0
 
Innodb_data_pending_reads       0
 
Innodb_data_pending_writes      0
 
Innodb_data_read        642990080
 
Innodb_data_reads       39607
 
Innodb_data_writes      91581
 
Innodb_data_written     0
 
Innodb_dblwr_pages_written      0
 
Innodb_dblwr_writes     0
 
Innodb_deadlocks        0
 
Innodb_history_list_length      122
 
Innodb_ibuf_discarded_delete_marks      0
 
Innodb_ibuf_discarded_deletes   0
 
Innodb_ibuf_discarded_inserts   0
 
Innodb_ibuf_free_list   145
 
Innodb_ibuf_merged_delete_marks 0
 
Innodb_ibuf_merged_deletes      0
 
Innodb_ibuf_merged_inserts      0
 
Innodb_ibuf_merges      0
 
Innodb_ibuf_segment_size        147
 
Innodb_ibuf_size        1
 
Innodb_log_waits        0
 
Innodb_log_write_requests       1448
 
Innodb_log_writes       91581
 
Innodb_lsn_current      49687657232
 
Innodb_lsn_flushed      49687657232
 
Innodb_lsn_last_checkpoint      49654917374
 
Innodb_master_thread_active_loops       0
 
Innodb_master_thread_idle_loops 72415
 
Innodb_max_trx_id       161374037
 
Innodb_mem_adaptive_hash        0
 
Innodb_mem_dictionary   40670760
 
Innodb_os_log_fsyncs    91581
 
Innodb_os_log_pending_fsyncs    0
 
Innodb_os_log_pending_writes    0
 
Innodb_os_log_written   107915776
 
Innodb_page_size        16384
 
Innodb_pages_created    1147
 
Innodb_pages_read       39245
 
Innodb_pages_written    0
 
Innodb_row_lock_current_waits   0
 
Innodb_row_lock_time    0
 
Innodb_row_lock_time_avg        0
 
Innodb_row_lock_time_max        0
 
Innodb_row_lock_waits   31
 
Innodb_rows_deleted     1067
 
Innodb_rows_inserted    12923
 
Innodb_rows_read        404273030
 
Innodb_rows_updated     79786
 
Innodb_system_rows_deleted      0
 
Innodb_system_rows_inserted     0
 
Innodb_system_rows_read 0
 
Innodb_system_rows_updated      0
 
Innodb_num_open_files   353
 
Innodb_truncated_status_writes  0
 
Innodb_available_undo_logs      128
 
Innodb_undo_truncations 0
 
Innodb_page_compression_saved   0
 
Innodb_num_pages_page_compressed        0
 
Innodb_num_page_compressed_trim_op      0
 
Innodb_num_pages_page_decompressed      0
 
Innodb_num_pages_page_compression_error 0
 
Innodb_num_pages_encrypted      0
 
Innodb_num_pages_decrypted      0
 
Innodb_have_lz4 ON
 
Innodb_have_lzo OFF
 
Innodb_have_lzma        OFF
 
Innodb_have_bzip2       OFF
 
Innodb_have_snappy      OFF
 
Innodb_have_punch_hole  ON
 
Innodb_defragment_compression_failures  0
 
Innodb_defragment_failures      0
 
Innodb_defragment_count 0
 
Innodb_instant_alter_column     0
 
Innodb_onlineddl_rowlog_rows    0
 
Innodb_onlineddl_rowlog_pct_used        0
 
Innodb_onlineddl_pct_progress   0
 
Innodb_secondary_index_triggered_cluster_reads  128234385
 
Innodb_secondary_index_triggered_cluster_reads_avoided  0
 
Innodb_encryption_rotation_pages_read_from_cache        0
 
Innodb_encryption_rotation_pages_read_from_disk 0
 
Innodb_encryption_rotation_pages_modified       0
 
Innodb_encryption_rotation_pages_flushed        0
 
Innodb_encryption_rotation_estimated_iops       0
 
Innodb_encryption_n_merge_blocks_encrypted      0
 
Innodb_encryption_n_merge_blocks_decrypted      0
 
Innodb_encryption_n_rowlog_blocks_encrypted     0
 
Innodb_encryption_n_rowlog_blocks_decrypted     0
 
Innodb_encryption_n_temp_blocks_encrypted       0
 
Innodb_encryption_n_temp_blocks_decrypted       0
 
Innodb_encryption_num_key_requests      0
 
Key_blocks_not_flushed  0
 
Key_blocks_unused       214334
 
Key_blocks_used 541
 
Key_blocks_warm 0
 
Key_read_requests       105908
 
Key_reads       5904
 
Key_write_requests      0
 
Key_writes      0
 
Last_query_cost 0.000000
 
Master_gtid_wait_count  0
 
Master_gtid_wait_time   0
 
Master_gtid_wait_timeouts       0
 
Max_statement_time_exceeded     0
 
Max_used_connections    13
 
Memory_used     4681624104
 
Memory_used_initial     58913720
 
Not_flushed_delayed_rows        0
 
Open_files      9
 
Open_streams    4
 
Open_table_definitions  400
 
Open_tables     5818
 
Opened_files    3785435
 
Opened_plugin_libraries 0
 
Opened_table_definitions        998
 
Opened_tables   2669153
 
Opened_views    857
 
Performance_schema_accounts_lost        0
 
Performance_schema_cond_classes_lost    0
 
Performance_schema_cond_instances_lost  0
 
Performance_schema_digest_lost  0
 
Performance_schema_file_classes_lost    0
 
Performance_schema_file_handles_lost    0
 
Performance_schema_file_instances_lost  0
 
Performance_schema_hosts_lost   0
 
Performance_schema_index_stat_lost      0
 
Performance_schema_locker_lost  0
 
Performance_schema_memory_classes_lost  0
 
Performance_schema_metadata_lock_lost   0
 
Performance_schema_mutex_classes_lost   0
 
Performance_schema_mutex_instances_lost 0
 
Performance_schema_nested_statement_lost        0
 
Performance_schema_prepared_statements_lost     0
 
Performance_schema_program_lost 0
 
Performance_schema_rwlock_classes_lost  0
 
Performance_schema_rwlock_instances_lost        0
 
Performance_schema_session_connect_attrs_lost   0
 
Performance_schema_socket_classes_lost  0
 
Performance_schema_socket_instances_lost        0
 
Performance_schema_stage_classes_lost   0
 
Performance_schema_statement_classes_lost       0
 
Performance_schema_table_handles_lost   0
 
Performance_schema_table_instances_lost 0
 
Performance_schema_table_lock_stat_lost 0
 
Performance_schema_thread_classes_lost  0
 
Performance_schema_thread_instances_lost        0
 
Performance_schema_users_lost   0
 
Prepared_stmt_count     0
 
Qcache_free_blocks      1
 
Qcache_free_memory      1031304
 
Qcache_hits     0
 
Qcache_inserts  0
 
Qcache_lowmem_prunes    0
 
Qcache_not_cached       0
 
Qcache_queries_in_cache 0
 
Qcache_total_blocks     1
 
Queries 3518014
 
Questions       3518014
 
Resultset_metadata_skipped      0
 
Rows_read       346144038
 
Rows_sent       16386013
 
Rows_tmp_read   17069832
 
Rpl_semi_sync_master_clients    0
 
Rpl_semi_sync_master_get_ack    0
 
Rpl_semi_sync_master_net_avg_wait_time  0
 
Rpl_semi_sync_master_net_wait_time      0
 
Rpl_semi_sync_master_net_waits  0
 
Rpl_semi_sync_master_no_times   0
 
Rpl_semi_sync_master_no_tx      0
 
Rpl_semi_sync_master_request_ack        0
 
Rpl_semi_sync_master_status     OFF
 
Rpl_semi_sync_master_timefunc_failures  0
 
Rpl_semi_sync_master_tx_avg_wait_time   0
 
Rpl_semi_sync_master_tx_wait_time       0
 
Rpl_semi_sync_master_tx_waits   0
 
Rpl_semi_sync_master_wait_pos_backtraverse      0
 
Rpl_semi_sync_master_wait_sessions      0
 
Rpl_semi_sync_master_yes_tx     0
 
Rpl_semi_sync_slave_send_ack    0
 
Rpl_semi_sync_slave_status      OFF
 
Rpl_status      AUTH_MASTER
 
Rpl_transactions_multi_engine   0
 
Select_full_join        1588
 
Select_full_range_join  48977
 
Select_range    80824
 
Select_range_check      0
 
Select_scan     1155434
 
Slave_connections       0
 
Slave_heartbeat_period  0.000
 
Slave_open_temp_tables  0
 
Slave_received_heartbeats       0
 
Slave_retried_transactions      0
 
Slave_running   OFF
 
Slave_skipped_errors    0
 
Slaves_connected        0
 
Slaves_running  0
 
Slow_launch_threads     0
 
Slow_queries    175
 
Sort_merge_passes       0
 
Sort_priority_queue_sorts       343324
 
Sort_range      812852
 
Sort_rows       12771629
 
Sort_scan       1484138
 
Ssl_accept_renegotiates 0
 
Ssl_accepts     0
 
Ssl_callback_cache_hits 0
 
Ssl_cipher
 
Ssl_cipher_list
 
Ssl_client_connects     0
 
Ssl_connect_renegotiates        0
 
Ssl_ctx_verify_depth    0
 
Ssl_ctx_verify_mode     0
 
Ssl_default_timeout     0
 
Ssl_finished_accepts    0
 
Ssl_finished_connects   0
 
Ssl_server_not_after
 
Ssl_server_not_before
 
Ssl_session_cache_hits  0
 
Ssl_session_cache_misses        0
 
Ssl_session_cache_mode  NONE
 
Ssl_session_cache_overflows     0
 
Ssl_session_cache_size  0
 
Ssl_session_cache_timeouts      0
 
Ssl_sessions_reused     0
 
Ssl_used_session_cache_entries  0
 
Ssl_verify_depth        0
 
Ssl_verify_mode 0
 
Ssl_version
 
Subquery_cache_hit      1492253
 
Subquery_cache_miss     259952940
 
Syncs   2060
 
Table_locks_immediate   2515
 
Table_locks_waited      0
 
Table_open_cache_active_instances       1
 
Table_open_cache_hits   22930346
 
Table_open_cache_misses 2670018
 
Table_open_cache_overflows      52309
 
Tc_log_max_pages_used   0
 
Tc_log_page_size        0
 
Tc_log_page_waits       0
 
Threadpool_idle_threads 0
 
Threadpool_threads      0
 
Threads_cached  3
 
Threads_connected       11
 
Threads_created 14
 
Threads_running 11
 
Transactions_gtid_foreign_engine        0
 
Transactions_multi_engine       0
 
Update_scan     0
 
Uptime  72437
 
Uptime_since_flush_status       72437
 
wsrep   0
 
wsrep_applier_thread_count      0
 
wsrep_cluster_capabilities
 
wsrep_cluster_conf_id   18446744073709551615
 
wsrep_cluster_size      0
 
wsrep_cluster_state_uuid
 
wsrep_cluster_status    Disconnected
 
wsrep_connected OFF
 
wsrep_local_bf_aborts   0
 
wsrep_local_index       18446744073709551615
 
wsrep_provider_capabilities
 
wsrep_provider_name
 
wsrep_provider_vendor
 
wsrep_provider_version
 
wsrep_ready     OFF
 
wsrep_rollbacker_thread_count   0
 
wsrep_thread_count      0

Comment by Sergei Golubchik [ 2023-03-30 ]

you can try to check memory allocations in performance_schema, perhaps it'll provide a clue

Generated at Thu Feb 08 10:18:51 UTC 2024 using Jira 8.20.16#820016-sha1:9d11dbea5f4be3d4cc21f03a88dd11d8c8687422.