AIX crash on mariadbd --bootstrap - invalid mariadb_stats pointer

(gdb) r

Starting program: /home/mdborg/build-mariadb-server-debug/sql/mariadbd --no-defaults --bootstrap --datadir=/home/mdborg/tmp/tmp.W88Kk3BlwK

[New Thread 1]

2023-09-13  8:46:08 0 [Note] Starting MariaDB 10.6.16-MariaDB-debug source revision 736901b4435f6f6785060d3de4d17e4c09035e06 as process 61014222

..

(gdb) bt full

#0  0x090000003dfbbcf0 in mariadb_increment_pages_accessed () at /home/mdborg/mariadb-server/storage/innobase/buf/buf0buf.cc:51

No locals.

#1  0x090000003d7c2ab0 in buf_page_get_low (page_id=<incomplete type>, zip_size=<incomplete type>, rw_latch=<incomplete type>, guess=<incomplete type>, mode=<incomplete type>, mtr=<incomplete type>, 

    err=0x0, allow_ibuf_merge=false) at /home/mdborg/mariadb-server/storage/innobase/buf/buf0buf.cc:2451

        access_time = 0

        retries = <incomplete type>

        chain = <incomplete type>

        hash_lock = <incomplete type>

        block = <incomplete type>

        state = <incomplete type>

#2  0x090000003d7c5014 in buf_page_get_gen (page_id=<incomplete type>, zip_size=<incomplete type>, rw_latch=<incomplete type>, guess=<incomplete type>, mode=<incomplete type>, mtr=<incomplete type>, 

    err=0x0, allow_ibuf_merge=false) at /home/mdborg/mariadb-server/storage/innobase/buf/buf0buf.cc:2930

        block = <incomplete type>

        s = 16777217

        must_merge = <incomplete type>

#3  0x090000003d996df8 in ibuf_bitmap_get_map_page (page_id=<incomplete type>, zip_size=0, mtr=<incomplete type>) at /home/mdborg/mariadb-server/storage/innobase/ibuf/ibuf0ibuf.cc:667

No locals.

#4  0x090000003d9a54e8 in ibuf_reset_bitmap (block=<incomplete type>, page_id=<incomplete type>, zip_size=0) at /home/mdborg/mariadb-server/storage/innobase/ibuf/ibuf0ibuf.cc:4093

        bitmap_bits = 0

        bitmap_page = <incomplete type>

        physical_size = 16384

        pcur = <incomplete type>

        err = <incomplete type>

        mtr = <incomplete type>

        space = <incomplete type>

        heap = 0x112224a78

        search_tuple = <incomplete type>

        mops = {720575940618949812, 12403668970434532176, 1152921504606823920}

        dops = {13527542225392, 4534637392, 720575940618949660}

#5  0x090000003d7c7fd0 in buf_page_create_low (page_id=<incomplete type>, zip_size=<incomplete type>, mtr=<incomplete type>, free_block=<incomplete type>)

    at /home/mdborg/mariadb-server/storage/innobase/buf/buf0buf.cc:3304

        chain = <incomplete type>

        bpage = <incomplete type>

#6  0x090000003d7c81f0 in buf_page_create_low (space=<incomplete type>, offset=<incomplete type>, zip_size=<incomplete type>, mtr=<incomplete type>, free_block=<incomplete type>)

    at /home/mdborg/mariadb-server/storage/innobase/buf/buf0buf.cc:3342

No locals.

#7  0x090000003d92fa10 in fsp_page_create (space=<incomplete type>, offset=<incomplete type>, mtr=<incomplete type>) at /home/mdborg/mariadb-server/storage/innobase/fsp/fsp0fsp.cc:1062

        block = <incomplete type>

        free_block = <incomplete type>

#8  0x090000003d9300a0 in fsp_alloc_free_page (space=<incomplete type>, hint=<incomplete type>, mtr=<incomplete type>, init_mtr=<incomplete type>, err=0xfffffffffffa948)

    at /home/mdborg/mariadb-server/storage/innobase/fsp/fsp0fsp.cc:1175

        block = <incomplete type>

--Type <RET> for more, q to quit, c to continue without paging--

        xdes = <incomplete type>

        descr = <incomplete type>

        free = <incomplete type>

        page_no = <incomplete type>

        space_size = <incomplete type>

#9  0x090000003d931120 in fsp_alloc_seg_inode_page (space=<incomplete type>, header=<incomplete type>, mtr=<incomplete type>) at /home/mdborg/mariadb-server/storage/innobase/fsp/fsp0fsp.cc:1373

        err = <incomplete type>

        block = <incomplete type>

        inode = 0x0

        i = 1152921504606824768

#10 0x090000003d93148c in fsp_alloc_seg_inode (space=<incomplete type>, header=<incomplete type>, iblock=0xfffffffffffaad8, mtr=<incomplete type>, err=0xfffffffffffca50)

    at /home/mdborg/mariadb-server/storage/innobase/fsp/fsp0fsp.cc:1410

        page_id = <incomplete type>

        block = <incomplete type>

        physical_size = 1152921504606824928

        n = 1152921504606825184

        inode = 0xfffffffffffa9f0 "\n"

#11 0x090000003d932970 in fseg_get_n_frag_pages (space=<incomplete type>, byte_offset=94, mtr=<incomplete type>, err=0xfffffffffffca50, has_done_reservation=false, block=<incomplete type>)

    at /home/mdborg/mariadb-server/storage/innobase/fsp/fsp0fsp.cc:1694

        inode = 0x0

        seg_id = 1051067616

        n_reserved = <incomplete type>

        reserved_extent = false

        _db_stack_frame_ = <incomplete type>

        header = <incomplete type>

        iblock = <incomplete type>

#12 0x090000003d75352c in btr_root_page_init (type=<incomplete type>, space=<incomplete type>, index_id=<incomplete type>, index=<incomplete type>, mtr=<incomplete type>, err=0xfffffffffffca50)

    at /home/mdborg/mariadb-server/storage/innobase/btr/btr0btr.cc:1039

        ibuf_hdr_block = <incomplete type>

        block = <incomplete type>

#13 0x090000003d94d6f0 in srv_start(bool)::{lambda()#1}::operator()() const (create_new_db=9) at /home/mdborg/mariadb-server/storage/innobase/srv/srv0start.cc:1393

        ibuf_root = <incomplete type>

        flushed_lsn = <incomplete type>

        err = <incomplete type>

        srv_log_file_found = true

        mtr = <incomplete type>

        sum_of_new_sizes = <incomplete type>

        logfile0 = <incomplete type>

        create_new_log = true

#14 0x090000003d6ffbbc in innodb_init (p=0x11205ec18) at /home/mdborg/mariadb-server/storage/innobase/handler/ha_innodb.cc:4291

        _db_stack_frame_ = <incomplete type>

        innobase_hton = <incomplete type>

--Type <RET> for more, q to quit, c to continue without paging--

        test_filename = "\000\000"

        test_tablename = "#mysql50#-@"

        count = 3

        create_new_db = true

        err = <incomplete type>

        error = 0

#15 0x090000003c77beb4 in hton_drop_table (plugin=0x111967380) at /home/mdborg/mariadb-server/sql/handler.cc:665

        hton = 0x11205ec18

        ret = 0

        _db_stack_frame_ = <incomplete type>

        tmp = 0

        fslot = <incomplete type>

#16 0x090000003caec8a0 in plugin_do_initialize (plugin=0x111967380, state=@0xfffffffffffd198: 4) at /home/mdborg/mariadb-server/sql/sql_plugin.cc:1454

        ret = 0

        _db_stack_frame_ = <incomplete type>

        init = @0x8001000a04d3e40: 0x90000003c77bd1c <hton_drop_table(handlerton*, char const*)>

#17 0x090000003caecd90 in plugin_initialize (tmp_root=<incomplete type>, plugin=0x111967380, argc=0x8001000a0e59cf0 <_mysqld.bss_+120>, argv=0x1100389f8, options_only=false)

    at /home/mdborg/mariadb-server/sql/sql_plugin.cc:1507

        ret = 1

        _db_stack_frame_ = <incomplete type>

        state = 4

#18 0x090000003caedf54 in init_plugin_psi_keys (argc=0x90000003c89e060 <init_server_components()+4316>, argv=0x8001000a021de80 <type_handler_varchar>, flags=3)

    at /home/mdborg/mariadb-server/sql/sql_plugin.cc:1766

        plugin_table_engine = false

        opts_only = false

        idx = 8

        hash = 0x8001000a0627760 <_sqlplugin.bss_3_+224>

        error = 0

        i = 4

        builtins = 0x8001000a0374318 <mysql_optional_plugins+56>

        plugin = 0x8001000a028d610 <builtin_maria_partition_plugin+104>

        tmp = <incomplete type>

        plugin_ptr = 0x111967380

        reap = 0xfffffffffffd438

        retry_end = 0xfffffffffffd240

        retry_start = 0xfffffffffffd240

        tmp_root = <incomplete type>

        reaped_mandatory_plugin = false

        mandatory = false

        opt_plugin_load_list_iter = <incomplete type>

        plugin_table_engine_name_buf = "\000\000\000\000\000\000\000\000\b\000\020\000\240!,X\000\000\000\000\000\000\000\060\272\334\017\376\340\335\360\r\017\377\377\377\377\377\327\240\017\377\377\377\377\377\330p\272\334\017\376\340\335\360\r\t\000\000\000<\211\223\340\272"

--Type <RET> for more, q to quit, c to continue without paging--

        plugin_table_engine_name = <incomplete type>

        MyISAM = <incomplete type>

        _db_stack_frame_ = <incomplete type>

        item = 0x0

#19 0x090000003c89e060 in init_server_components () at /home/mdborg/mariadb-server/sql/mysqld.cc:5110

        _db_stack_frame_ = <incomplete type>

#20 0x090000003c89fbfc in test_lc_time_sz (argc=5, argv=0x1100389f8) at /home/mdborg/mariadb-server/sql/mysqld.cc:5727

        please_close_stdin = true

        ho_error = 0

        new_thread_stack_size = <incomplete type>

        user = 0xbadc0ffee0ddf00d <error: Cannot access memory at address 0xbadc0ffee0ddf00d>

#21 0x0000000100000558 in main ()

No symbol table info available.

(gdb) 

(gdb) p mariadb_stats

Cannot access memory at address 0x80010008003e938

(gdb)  p &mariadb_dummy_stats 

$1 = (ha_handler_stats *) 0x80010008003e900

(gdb) b mariadb_set_stats::mariadb_set_stats

Function "mariadb_set_stats::mariadb_set_stats" not defined.

Make breakpoint pending on future shared library load? (y or [n]) y

Breakpoint 1 (mariadb_set_stats::mariadb_set_stats) pending.

(gdb) b mariadb_set_stats::~mariadb_set_stats

Function "mariadb_set_stats::~mariadb_set_stats" not defined.

Make breakpoint pending on future shared library load? (y or [n]) y

Breakpoint 2 (mariadb_set_stats::~mariadb_set_stats) pending.

diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc

index e0af4f728e3..b8690d79ce8 100644

--- a/storage/innobase/handler/ha_innodb.cc

+++ b/storage/innobase/handler/ha_innodb.cc

@@ -4221,6 +4221,8 @@ static int innodb_init(void* p)

        innobase_hton->prepare_commit_versioned

                = innodb_prepare_commit_versioned;

+mariadb_stats= &mariadb_dummy_stats;

+

        innodb_remember_check_sysvar_funcs();

        compile_time_assert(DATA_MYSQL_TRUE_VARCHAR == MYSQL_TYPE_VARCHAR);

2023-09-13  9:19:26 0 [Note] InnoDB: File './ibdata1' size is now 12 MB.

2023-09-13  9:19:26 0 [Note] InnoDB: Setting log file ./ib_logfile101 size to 100663296 bytes

2023-09-13  9:19:26 0 [Note] InnoDB: Renaming log file ./ib_logfile101 to ./ib_logfile0

2023-09-13  9:19:26 0 [Note] InnoDB: New log file created, LSN=10449

2023-09-13  9:19:26 0 [Note] InnoDB: Doublewrite buffer not found: creating new

terminate called after throwing an instance of 'std::system_error'

  what():  Invalid argument

[New Thread 258]

[New Thread 515]

[New Thread 772]

[New Thread 1029]

[New Thread 1286]

[New Thread 1543]

[New Thread 1800]

[New Thread 2057]

[New Thread 2314]

[New Thread 2571]

[New Thread 2828]

[New Thread 3085]

[New Thread 3342]

[New Thread 3599]

[New Thread 3856]

[New Thread 4113]

[New Thread 4370]

[New Thread 4627]

[New Thread 4884]

[New Thread 5141]

[New Thread 5398]

[New Thread 5655]

[New Thread 5912]

[New Thread 6169]

[New Thread 6426]

[New Thread 6683]

[New Thread 6940]

[New Thread 7197]

[New Thread 7454]

[New Thread 7711]

[New Thread 7968]

[New Thread 8225]

[New Thread 8482]

[New Thread 8739]

[New Thread 8996]

Thread 1 received signal SIGABRT, Aborted.

0x0900000000534f94 in pthread_kill () from /usr/lib/libpthreads.a(shr_xpg5_64.o)

(gdb) bt full

#0  0x0900000000534f94 in pthread_kill () from /usr/lib/libpthreads.a(shr_xpg5_64.o)

No symbol table info available.

#1  0x09000000005347e8 in _p_raise () from /usr/lib/libpthreads.a(shr_xpg5_64.o)

No symbol table info available.

#2  0x090000000003946c in raise () from /usr/lib/libc.a(shr_64.o)

No symbol table info available.

#3  0x0900000000055e68 in abort () from /usr/lib/libc.a(shr_64.o)

No symbol table info available.

#4  0x090000000d01fbcc in __gnu_cxx::__verbose_terminate_handler() () at ../../../../../gcc-10.3.0/libstdc++-v3/libsupc++/vterminate.cc:42

        w = 0x12e99208 <error: Cannot access memory at address 0x12e99208>

        terminating = 9

        t = 0x0

#5  0x090000000d02d9c4 in __cxa_end_catch () from /opt/freeware/lib/gcc/powerpc-ibm-aix7.1.0.0/10/pthread/libstdc++.a(libstdc++.so.6)

No symbol table info available.

#6  0x090000000d01fa08 in __gnu_cxx::__verbose_terminate_handler() () at ../../../../../gcc-10.3.0/libstdc++-v3/libsupc++/vterminate.cc:7

        terminating = 9

        t = 0x0

#7  0x090000000d02d5b4 in __gnu_cxx::__concurrence_lock_error::~__concurrence_lock_error (this=<incomplete type>) from /opt/freeware/lib/gcc/powerpc-ibm-aix7.1.0.0/10/pthread/libstdc++.a(libstdc++.so.6)

No locals.

#8  0x090000000d03e074 in std::out_of_range::out_of_range(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) ()

   from /opt/freeware/lib/gcc/powerpc-ibm-aix7.1.0.0/10/pthread/libstdc++.a(libstdc++.so.6)

No symbol table info available.

#9  0x0900000040c6da58 in std::mutex::lock (this=0x90000000d016880 <__thenan_df+32>) at /home/mdborg/mariadb-server/tpool/tpool_generic.cc:104

        __e = <incomplete type>

#10 0x0900000040c66f64 in std::unique_lock<std::mutex>::lock (this=<incomplete type>) at /home/mdborg/mariadb-server/tpool/tpool_generic.cc:138

No locals.

#11 0x0900000040c6a820 in std::unique_lock<std::mutex>::unique_lock (this=0x112216160, __m=@0x11000ade8: ???) at /home/mdborg/mariadb-server/tpool/tpool_generic.cc:68

No locals.

#12 0x090000004133ab94 in binary_semaphore::wake (this=0x11000ade8) at /home/mdborg/mariadb-server/storage/innobase/log/log0sync.cc:153

        lk = <incomplete type>

#13 0x090000004133ba90 in group_commit_lock::release(unsigned long)::{lambda(std::pair<unsigned long, completion_callback> const&)#1}::operator()(std::pair<unsigned long, completion_callback> const&) const (this=0x8001000a0e4eef0 <_log0log.bss_3_+128>, num=<incomplete type>) at /home/mdborg/mariadb-server/storage/innobase/log/log0sync.cc:393

        callbacks = 0x112216288

        callback_count = <incomplete type>

        ret = <incomplete type>

        lk = <incomplete type>

        cur = <incomplete type>

        prev = <incomplete type>

        next = <incomplete type>

        wakeup_list = <incomplete type>

        it = <incomplete type>

#14 0x0900000040dcb79c in log_write_up_to (lsn=<incomplete type>, flush_to_disk=true, rotate_key=true, callback=0x0) at /home/mdborg/mariadb-server/storage/innobase/log/log0log.cc:848

--Type <RET> for more, q to quit, c to continue without paging--c

        flush_lsn = <incomplete type>

        ret_lsn1 = <incomplete type>

        ret_lsn2 = <incomplete type>

#15 0x0900000040ddb3dc in log_checkpoint_low (oldest_lsn=<incomplete type>, end_lsn=<incomplete type>) at /home/mdborg/mariadb-server/storage/innobase/buf/buf0flu.cc:1806

        age = <incomplete type>

        flush_lsn = <incomplete type>

#16 0x0900000040ddcd24 in buf_flush_sync_for_checkpoint (lsn=<incomplete type>) at /home/mdborg/mariadb-server/storage/innobase/buf/buf0flu.cc:2030

        newest_lsn = <incomplete type>

        measure = <incomplete type>

        checkpoint_lsn = <incomplete type>

        target = <incomplete type>

#17 0x0900000040dde820 in buf_flush_page_cleaner () at /home/mdborg/mariadb-server/storage/innobase/buf/buf0flu.cc:2264

        soft_lsn_limit = <incomplete type>

        oldest_lsn = <incomplete type>

        pct_lwm = -9.9261575707946013e-234

        n_flushed = <incomplete type>

        n = <incomplete type>

        dirty_blocks = <incomplete type>

        dirty_pct = -9.9261575707946013e-234

        last_pages = <incomplete type>

        abstime = {tv_sec = 0, tv_nsec = 1694596767}

        lsn_limit = <incomplete type>

        last_activity_count = <incomplete type>

        psi = 0x0