large pages - out of memory handling results in SIGBUS sql/sql_show.cc:6822(get_schema_views_record

#  cat /sys/devices/system/node/node0/hugepages

/hugepages-1048576kB/free_hugepages

63

[mariadb]

skip-networking

thread_handling=pool-of-threads

thread_pool_stall_limit=1ms

max-connections=4096

large_pages=1

#memlock=1

query_cache_type=0

innodb_page_size=64k

innodb-buffer-pool-size=64G

# 1G - 32M - based off buf_chunk_init to keep the

# allocation in 1G chunks

# 16k page size

# (gdb) p (1 << 30) / (16 * 1024) * sizeof(buf_block_t) + (16384 - 1)

#$2 = 25182207

# 64k page size

#(gdb) p (1 << 30) / (65536) * sizeof(buf_block_t) + (65536 - 1)

#$3 = 6356991

#(gdb) p 1 << 30 - (32 * 1<< 20)

#$4 = 1073741824

innodb_buffer_pool_chunk_size=1000M

innodb_file_per_table=1

innodb_flush_method=O_DIRECT

innodb_sync_spin_loops=5

innodb_lru_scan_depth=20000

innodb_io_capacity=10k

innodb_io_capacity_max=20k

innodb_flush_neighbors=0

innodb_log_buffer_size=64M

innodb_thread_concurrency=0

innodb_max_dirty_pages_pct=75

innodb_adaptive_flushing=1

innodb_read_io_threads=16

innodb_write_io_threads=4

innodb_purge_threads=1

innodb_log_file_size=1G

innodb_log_files_in_group=4

performance_schema=OFF

max_prepared_stmt_count=16M

Mar 26 23:05:51 opsky7.aus.stglabs.ibm.com numactl[83379]: 2018-03-26 23:05:51 0 [Note] /usr/local/mysql/bin/mysqld (mysqld 10.3.6-MariaDB) starting as process 83379 ...

Mar 26 23:05:51 opsky7.aus.stglabs.ibm.com numactl[83379]: 2018-03-26 23:05:51 0 [Note] InnoDB: innodb_page_size=65536

Mar 26 23:05:51 opsky7.aus.stglabs.ibm.com numactl[83379]: 2018-03-26 23:05:51 0 [Note] InnoDB: Mutexes and rw_locks use GCC atomic builtins

Mar 26 23:05:51 opsky7.aus.stglabs.ibm.com numactl[83379]: 2018-03-26 23:05:51 0 [Note] InnoDB: Uses event mutexes

Mar 26 23:05:51 opsky7.aus.stglabs.ibm.com numactl[83379]: 2018-03-26 23:05:51 0 [Note] InnoDB: Compressed tables use zlib 1.2.7

Mar 26 23:05:51 opsky7.aus.stglabs.ibm.com numactl[83379]: 2018-03-26 23:05:51 0 [Note] InnoDB: Using Linux native AIO

Mar 26 23:05:51 opsky7.aus.stglabs.ibm.com numactl[83379]: 2018-03-26 23:05:51 0 [Note] InnoDB: Number of pools: 1

Mar 26 23:05:51 opsky7.aus.stglabs.ibm.com numactl[83379]: 2018-03-26 23:05:51 0 [Note] InnoDB: Using SSE2 crc32 instructions

Mar 26 23:05:51 opsky7.aus.stglabs.ibm.com numactl[83379]: 2018-03-26 23:05:51 0 [Note] InnoDB: Initializing buffer pool, total size = 70.3125G, instances = 8, chunk size = 1000M

Mar 26 23:06:04 opsky7.aus.stglabs.ibm.com numactl[83379]: 180326 23:06:04 [ERROR] mysqld got signal 7 ;

Mar 26 23:06:04 opsky7.aus.stglabs.ibm.com numactl[83379]: This could be because you hit a bug. It is also possible that this binary

Mar 26 23:06:04 opsky7.aus.stglabs.ibm.com numactl[83379]: or one of the libraries it was linked against is corrupt, improperly built,

Mar 26 23:06:04 opsky7.aus.stglabs.ibm.com numactl[83379]: or misconfigured. This error can also be caused by malfunctioning hardware.

Mar 26 23:06:04 opsky7.aus.stglabs.ibm.com numactl[83379]: To report this bug, see https://mariadb.com/kb/en/reporting-bugs

Mar 26 23:06:04 opsky7.aus.stglabs.ibm.com numactl[83379]: We will try our best to scrape up some info that will hopefully help

Mar 26 23:06:04 opsky7.aus.stglabs.ibm.com numactl[83379]: diagnose the problem, but since we have already crashed,

Mar 26 23:06:04 opsky7.aus.stglabs.ibm.com numactl[83379]: something is definitely wrong and this may fail.

Mar 26 23:06:04 opsky7.aus.stglabs.ibm.com numactl[83379]: Server version: 10.3.6-MariaDB

Mar 26 23:06:04 opsky7.aus.stglabs.ibm.com numactl[83379]: key_buffer_size=134217728

Mar 26 23:06:04 opsky7.aus.stglabs.ibm.com numactl[83379]: read_buffer_size=131072

Mar 26 23:06:04 opsky7.aus.stglabs.ibm.com numactl[83379]: max_used_connections=0

Mar 26 23:06:04 opsky7.aus.stglabs.ibm.com numactl[83379]: max_threads=65537

Mar 26 23:06:04 opsky7.aus.stglabs.ibm.com numactl[83379]: thread_count=0

Mar 26 23:06:04 opsky7.aus.stglabs.ibm.com numactl[83379]: It is possible that mysqld could use up to

Mar 26 23:06:04 opsky7.aus.stglabs.ibm.com numactl[83379]: key_buffer_size + (read_buffer_size + sort_buffer_size)*max_threads = 142830806 K  bytes of memory

Mar 26 23:06:04 opsky7.aus.stglabs.ibm.com numactl[83379]: Hope that's ok; if not, decrease some variables in the equation.

Mar 26 23:06:04 opsky7.aus.stglabs.ibm.com numactl[83379]: Thread pointer: 0x0

Mar 26 23:06:04 opsky7.aus.stglabs.ibm.com numactl[83379]: Attempting backtrace. You can use the following information to find out

Mar 26 23:06:04 opsky7.aus.stglabs.ibm.com numactl[83379]: where mysqld died. If you see no messages after this, something went

Mar 26 23:06:04 opsky7.aus.stglabs.ibm.com numactl[83379]: terribly wrong...

Mar 26 23:06:04 opsky7.aus.stglabs.ibm.com numactl[83379]: stack_bottom = 0x0 thread_stack 0x49000

Mar 26 23:06:04 opsky7.aus.stglabs.ibm.com numactl[83379]: /usr/local/mysql/bin/mysqld(my_print_stacktrace+0x29)[0xc5fd99]

Mar 26 23:06:05 opsky7.aus.stglabs.ibm.com numactl[83379]: /usr/local/mysql/bin/mysqld(handle_fatal_signal+0x25f)[0x7ef92f]

Mar 26 23:06:05 opsky7.aus.stglabs.ibm.com numactl[83379]: sigaction.c:0(__restore_rt)[0x7f46c1eff5e0]

Mar 26 23:06:05 opsky7.aus.stglabs.ibm.com numactl[83379]: /usr/local/mysql/bin/mysqld[0xaa9a80]

Mar 26 23:06:05 opsky7.aus.stglabs.ibm.com numactl[83379]: sql/sql_string.h:211(String::length(unsigned long))[0xa99b20]

Mar 26 23:06:05 opsky7.aus.stglabs.ibm.com numactl[83379]: sql/sql_show.cc:6822(get_schema_views_record(THD*, TABLE_LIST*, TABLE*, bool, st_mysql_const_lex_string const*, st_mysql_const_lex_string const*))[0xa3d5b9]

Mar 26 23:06:05 opsky7.aus.stglabs.ibm.com numactl[83379]: psi/mysql_thread.h:1097(inline_mysql_prlock_unlock(st_mysql_prlock*))[0x968ad3]

Mar 26 23:06:05 opsky7.aus.stglabs.ibm.com numactl[83379]: sql/sys_vars.ic:154(global constructors keyed to a)[0x7f069d]

Mar 26 23:06:05 opsky7.aus.stglabs.ibm.com numactl[83379]: /usr/local/mysql/bin/mysqld[0x637f45]

Mar 26 23:06:05 opsky7.aus.stglabs.ibm.com numactl[83379]: /usr/local/mysql/bin/mysqld(_Z11plugin_initPiPPci+0xa8d)[0x637c2d]

Mar 26 23:06:05 opsky7.aus.stglabs.ibm.com numactl[83379]: /usr/local/mysql/bin/mysqld[0x584a7b]

Mar 26 23:06:05 opsky7.aus.stglabs.ibm.com numactl[83379]: /usr/local/mysql/bin/mysqld(_Z11mysqld_mainiPPc+0x561)[0x5807d1]

Mar 26 23:06:05 opsky7.aus.stglabs.ibm.com numactl[83379]: /lib64/libc.so.6(__libc_start_main+0xf5)[0x7f46bffd6c05]

Mar 26 23:06:05 opsky7.aus.stglabs.ibm.com numactl[83379]: /usr/local/mysql/bin/mysqld[0x57c759]

Mar 26 23:06:05 opsky7.aus.stglabs.ibm.com numactl[83379]: The manual page at http://dev.mysql.com/doc/mysql/en/crashing.html contains

Mar 26 23:06:05 opsky7.aus.stglabs.ibm.com numactl[83379]: information that should help you find out what is causing the crash.

Mar 26 23:06:05 opsky7.aus.stglabs.ibm.com systemd[1]: mariadb0@1.service: main process exited, code=killed, status=7/BUS

Mar 26 23:06:32 opsky7.aus.stglabs.ibm.com numactl[83463]: 2018-03-26 23:06:32 0 [Note] /usr/local/mysql/bin/mysqld (mysqld 10.3.6-MariaDB) starting as process 83463 ...

Mar 26 23:06:33 opsky7.aus.stglabs.ibm.com numactl[83463]: 2018-03-26 23:06:33 0 [Note] InnoDB: innodb_page_size=65536

Mar 26 23:06:33 opsky7.aus.stglabs.ibm.com numactl[83463]: 2018-03-26 23:06:33 0 [Note] InnoDB: Mutexes and rw_locks use GCC atomic builtins

Mar 26 23:06:33 opsky7.aus.stglabs.ibm.com numactl[83463]: 2018-03-26 23:06:33 0 [Note] InnoDB: Uses event mutexes

Mar 26 23:06:33 opsky7.aus.stglabs.ibm.com numactl[83463]: 2018-03-26 23:06:33 0 [Note] InnoDB: Compressed tables use zlib 1.2.7

Mar 26 23:06:33 opsky7.aus.stglabs.ibm.com numactl[83463]: 2018-03-26 23:06:33 0 [Note] InnoDB: Using Linux native AIO

Mar 26 23:06:33 opsky7.aus.stglabs.ibm.com numactl[83463]: 2018-03-26 23:06:33 0 [Note] InnoDB: Number of pools: 1

Mar 26 23:06:33 opsky7.aus.stglabs.ibm.com numactl[83463]: 2018-03-26 23:06:33 0 [Note] InnoDB: Using SSE2 crc32 instructions

Mar 26 23:06:33 opsky7.aus.stglabs.ibm.com numactl[83463]: 2018-03-26 23:06:33 0 [Note] InnoDB: Initializing buffer pool, total size = 70.3125G, instances = 8, chunk size = 1000M

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: 180326 23:06:46 [ERROR] mysqld got signal 7 ;

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: This could be because you hit a bug. It is also possible that this binary

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: or one of the libraries it was linked against is corrupt, improperly built,

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: or misconfigured. This error can also be caused by malfunctioning hardware.

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: To report this bug, see https://mariadb.com/kb/en/reporting-bugs

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: We will try our best to scrape up some info that will hopefully help

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: diagnose the problem, but since we have already crashed,

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: something is definitely wrong and this may fail.

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: Server version: 10.3.6-MariaDB

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: key_buffer_size=134217728

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: read_buffer_size=131072

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: max_used_connections=0

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: max_threads=65537

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: thread_count=0

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: It is possible that mysqld could use up to

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: key_buffer_size + (read_buffer_size + sort_buffer_size)*max_threads = 142830806 K  bytes of memory

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: Hope that's ok; if not, decrease some variables in the equation.

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: Thread pointer: 0x0

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: Attempting backtrace. You can use the following information to find out

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: where mysqld died. If you see no messages after this, something went

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: terribly wrong...

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: stack_bottom = 0x0 thread_stack 0x49000

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: /usr/local/mysql/bin/mysqld(my_print_stacktrace+0x29)[0xc5fd99]

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: /usr/local/mysql/bin/mysqld(handle_fatal_signal+0x25f)[0x7ef92f]

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: sigaction.c:0(__restore_rt)[0x7fc51f84a5e0]

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: /usr/local/mysql/bin/mysqld[0xaa9a80]

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: sql/sql_string.h:211(String::length(unsigned long))[0xa99b20]

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: sql/sql_show.cc:6822(get_schema_views_record(THD*, TABLE_LIST*, TABLE*, bool, st_mysql_const_lex_string const*, st_mysql_const_lex_string const*))[0xa3d5b9]

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: psi/mysql_thread.h:1097(inline_mysql_prlock_unlock(st_mysql_prlock*))[0x968ad3]

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: sql/sys_vars.ic:154(global constructors keyed to a)[0x7f069d]

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: /usr/local/mysql/bin/mysqld[0x637f45]

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: /usr/local/mysql/bin/mysqld(_Z11plugin_initPiPPci+0xa8d)[0x637c2d]

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: /usr/local/mysql/bin/mysqld[0x584a7b]

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: /usr/local/mysql/bin/mysqld(_Z11mysqld_mainiPPc+0x561)[0x5807d1]

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: /lib64/libc.so.6(__libc_start_main+0xf5)[0x7fc51d921c05]

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: /usr/local/mysql/bin/mysqld[0x57c759]

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: The manual page at http://dev.mysql.com/doc/mysql/en/crashing.html contains

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com numactl[83463]: information that should help you find out what is causing the crash.

Mar 26 23:06:46 opsky7.aus.stglabs.ibm.com systemd[1]: mariadb0@1.service: main process exited, code=killed, status=7/BUS

 systemctl show mariadb0@1.service

Type=notify

Restart=on-abort

NotifyAccess=main

RestartUSec=5s

TimeoutStartUSec=1min 30s

TimeoutStopUSec=1min 30s

WatchdogUSec=0

WatchdogTimestampMonotonic=0

StartLimitInterval=10000000

StartLimitBurst=5

StartLimitAction=none

FailureAction=none

PermissionsStartOnly=no

RootDirectoryStartOnly=no

RemainAfterExit=no

GuessMainPID=yes

MainPID=84555

ControlPID=0

FileDescriptorStoreMax=0

StatusErrno=0

Result=success

ExecMainStartTimestamp=Mon 2018-03-26 23:15:28 CDT

ExecMainStartTimestampMonotonic=75746396191

ExecMainExitTimestampMonotonic=0

ExecMainPID=84555

ExecMainCode=0

ExecMainStatus=0

ExecStartPre={ path=/usr/local/mysql/scripts/mysql_install_db ; argv[]=/usr/local/mysql/scripts/mysql_install_db --basedir=/usr/local/mysql $MYSQLD_MULTI_INSTANCE ; ignore_errors=no ; start_time=[Mon 2018-03-26 

ExecStart={ path=/usr/bin/numactl ; argv[]=/usr/bin/numactl --cpunodebind=0 --membind=0 /usr/local/mysql/bin/mysqld $MYSQLD_MULTI_INSTANCE ; ignore_errors=no ; start_time=[Mon 2018-03-26 23:15:28 CDT] ; stop_tim

Slice=system-mariadb0.slice

ControlGroup=/system.slice/system-mariadb0.slice/mariadb0@1.service

MemoryCurrent=18446744073709551615

TasksCurrent=18446744073709551615

Delegate=no

CPUAccounting=no

CPUShares=18446744073709551615

StartupCPUShares=18446744073709551615

CPUQuotaPerSecUSec=infinity

BlockIOAccounting=no

BlockIOWeight=18446744073709551615

StartupBlockIOWeight=18446744073709551615

MemoryAccounting=no

MemoryLimit=18446744073709551615

DevicePolicy=closed

TasksAccounting=no

TasksMax=18446744073709551615

Environment=MYSQLD_MULTI_INSTANCE=--datadir=/var/lib/mysql/node0/1\x20--socket=/tmp/mysqld-n0-1.sock PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/usr/local/mysql/scripts:/usr/local/mysql/bin

UMask=0007

LimitCPU=18446744073709551615

LimitFSIZE=18446744073709551615

LimitDATA=18446744073709551615

LimitSTACK=18446744073709551615

LimitCORE=18446744073709551615

LimitRSS=18446744073709551615

LimitNOFILE=16364

LimitAS=18446744073709551615

LimitNPROC=1541525

LimitMEMLOCK=18446744073709551615

LimitLOCKS=18446744073709551615

LimitSIGPENDING=1541525

LimitMSGQUEUE=819200

LimitNICE=0

LimitRTPRIO=0

LimitRTTIME=18446744073709551615

OOMScoreAdjust=0

Nice=0

IOScheduling=0

CPUSchedulingPolicy=0

CPUSchedulingPriority=0

TimerSlackNSec=50000

CPUSchedulingResetOnFork=no

NonBlocking=no

StandardInput=null

StandardOutput=journal

StandardError=inherit

TTYReset=no

TTYVHangup=no

TTYVTDisallocate=no

SyslogPriority=30

SyslogLevelPrefix=yes

SecureBits=0

CapabilityBoundingSet=16384

AmbientCapabilities=0

User=mysql

Group=mysql

MountFlags=0

PrivateTmp=no

PrivateNetwork=no

PrivateDevices=yes

ProtectHome=yes

ProtectSystem=full

SameProcessGroup=no

IgnoreSIGPIPE=yes

NoNewPrivileges=no

SystemCallErrorNumber=0

RuntimeDirectoryMode=0755

KillMode=control-group

KillSignal=15

SendSIGKILL=no

SendSIGHUP=no

Id=mariadb0@1.service

Names=mariadb0@1.service

Requires=basic.target

Wants=system-mariadb0.slice

Conflicts=shutdown.target

Before=shutdown.target

After=network.target basic.target system-mariadb0.slice syslog.target systemd-journald.socket

Documentation=man:mysqld(8) https://mariadb.com/kb/en/library/systemd/

Description=MariaDB @VERSION@ database server (multi-instance)

LoadState=loaded

ActiveState=activating

SubState=start

FragmentPath=/etc/systemd/system/mariadb0@.service

DropInPaths=/etc/systemd/system/mariadb0@.service.d/multi.conf

UnitFileState=disabled

UnitFilePreset=disabled

InactiveExitTimestamp=Mon 2018-03-26 23:15:07 CDT

InactiveExitTimestampMonotonic=75724660933

ActiveEnterTimestampMonotonic=0

ActiveExitTimestampMonotonic=0

InactiveEnterTimestamp=Mon 2018-03-26 23:15:07 CDT

InactiveEnterTimestampMonotonic=75724659785

CanStart=yes

CanStop=yes

CanReload=no

CanIsolate=no

Job=11143

StopWhenUnneeded=no

RefuseManualStart=no

RefuseManualStop=no

AllowIsolate=no

DefaultDependencies=yes

OnFailureJobMode=replace

IgnoreOnIsolate=no

IgnoreOnSnapshot=no

NeedDaemonReload=no

JobTimeoutUSec=0

JobTimeoutAction=none

ConditionResult=yes

AssertResult=yes

ConditionTimestamp=Mon 2018-03-26 23:15:07 CDT

ConditionTimestampMonotonic=75724659834

AssertTimestamp=Mon 2018-03-26 23:15:07 CDT

AssertTimestampMonotonic=75724659834

Transient=no