Fatal Python error: PyEval_SaveThread: the function must be called with the GIL held, but the GIL is released (the current Python thread state is NULL)

Python runtime state: initialized

Thread 0x00007fffe3fff640 (most recent call first):

  File sqldb execute error: No connection available SELECT * FROM table1 ORDER BY id DESC LIMIT 10

"/home/ec2-user/venv/lib64/python3.11/site-packages/mariadb/cursors.py"got exception No connection available

, line 310 in execute

  File "/home/ec2-user/test_sqldb_segv-bug.py", line 74 in execute

  File "/home/ec2-user/test_sqldb_segv-bug.py", line 117 in test_sqldb1

  File "/usr/lib64/python3.11/threading.py", line 982 in run

  File "/usr/lib64/python3.11/threading.py", line 1045 in _bootstrap_inner

  File "/usr/lib64/python3.11/threading.py", line 1002 in _bootstrap

Thread 0x00007fffe8ccb640 (most recent call first):

  File "/home/ec2-user/test_sqldb_segv-bug.py", line 127 in test_sqldb1

  File "/usr/lib64/python3.11/threading.py", line 982 in run

  File "/usr/lib64/python3.11/threading.py", line 1045 in _bootstrap_inner

  File "/usr/lib64/python3.11/threading.py", line 1002 in _bootstrap

Current thread 0x00007fffe96cc640 (most recent call first):

  File "/home/ec2-user/test_sqldb_segv-bug.py", line 107 in test_sqldb_insert

  File "/usr/lib64/python3.11/threading.py", line 982 in run

  File "/usr/lib64/python3.11/threading.py", line 1045 in _bootstrap_inner

  File "/usr/lib64/python3.11/threading.py", line 1002 in _bootstrap

Thread 0x00007ffff7fb7740 (most recent call first):

  File "/usr/lib64/python3.11/threading.py", line 1590 in _shutdown

Extension modules: mariadb._mariadb (total: 1)

Thread 2 "python3" received signal SIGABRT, Aborted.

[Switching to Thread 0x7fffe96cc640 (LWP 42106)]

0x00007ffff76a157c in __pthread_kill_implementation () from /lib64/libc.so.6

Missing separate debuginfos, use: dnf debuginfo-install glibc-2.34-117.amzn2023.0.1.x86_64 libuuid-2.37.4-1.amzn2023.0.4.x86_64 mpdecimal-2.5.1-3.amzn2023.0.3.x86_64 openssl-libs-3.0.8-1.amzn2023.0.18.x86_64 zlib-1.2.11-33.amzn2023.0.5.x86_64

(gdb) 

hread 3 "python3" received signal SIGSEGV, Segmentation fault.

[Switching to Thread 0x7fffe8ccb640 (LWP 42274)]

method_vectorcall_NOARGS (func=<method_descriptor at remote 0x7fffe9d3bbf0>, args=0x7ffff7538358, nargsf=<optimized out>, kwnames=0x0) at /usr/src/debug/python3.11-3.11.6-1.amzn2023.0.5.x86_64/Objects/descrobject.c:449

449	    PyCFunction meth = (PyCFunction)method_enter_call(tstate, func);

(gdb) info locals

tstate = 0x0

nargs = 1

meth = <optimized out>

result = <optimized out>

(gdb) 

import os

import sys

import time

import uuid

from threading import Thread

import mariadb

###

#

# The SEGV seems to occur when we hit limit of connections from pool a number of times

#

# Run this script with 2 threads and 100ms sleep :

#

#  python3 test_sqldb_segv-bug.py 2 100

#

# Also tried using python 3.12.4, this also segvs

#

###

DEBUG=0

POOL_NUM=1

def debug(s, *args, **kwargs):

    if DEBUG:

        print(s, *args, **kwargs)

pool = mariadb.ConnectionPool(pool_name="sqldb", pool_size=5)

pool.set_config(

            user="test",

            password="password",

            host="127.0.0.1",

            database="TEST",

            ssl=False,

            autocommit=True,

)

for num in range(POOL_NUM):

    try:

        conn = mariadb.connect(

            user="test",

            password="password",

            host="127.0.0.1",

            database="TEST",

            ssl=False,

        )

        conn.autocommit = True

        conn.auto_reconnect = True

        pool.add_connection(conn)

        print(f"startup - added pool {conn}")

    except mariadb.PoolError:

        print("got Pool error")

def get_cursor(_id):

    try:

        conn = pool.get_connection()

    except mariadb.PoolError:

        raise

    except Exception as exc:

        print(f"get_cursor: exception {exc}")

    cur = conn.cursor(dictionary=False, buffered=True)

    return cur

def execute(_id, sql, *args, **kwargs):

    rows = []

    try:

        cur = get_cursor(_id)

    except Exception as exc:

        print(f"sqldb execute error: {exc} {sql}")

        raise exc

    try:

        res = cur.execute(sql, *args, **kwargs)

        if cur.description:

            try:

                rows = cur.fetchall()

            except mariadb.Error as exc:

                # probably "Cursor doesn't have a result set" which is OK for tasks like INSERT

                if str(exc) != "Cursor doesn't have a result set":

                    print(f"sqldb cursor fetchall error: {exc}")

    except Exception as exc:

        print(f"sqldb execute error: {exc} {sql}")

        cur.connection.close()

        raise exc

    cur.connection.close()

    return rows

def test_sqldb_insert(_id, sleep_time):

    print("start insert")

    num = 0

    while True:

        debug("I",end='',flush=True)

        try:

            res = execute(_id, "INSERT INTO table1 (name, uid, call_from, clientid) VALUES (?,?,?,?)", (num, str(uuid.uuid1()), num, str(uuid.uuid1())))

        except Exception as exc:

            print(f"got exception {exc}")

        num += 1

        time.sleep(0.2)

def test_sqldb1(_id, sleep_time):

    print(f"start {_id}")

    while True:

        debug(f".",end='',flush=True)

        try:

            res = execute(_id, "SELECT * FROM table1 ORDER BY id DESC LIMIT 10")

            debug(f"{_id}",end='',flush=True)

            if res:

                for x in res:

                    pass

        except Exception as exc:

            print(f"got exception {exc}")

        time.sleep(sleep_time)

if __name__ == "__main__":

    if len(sys.argv) < 3:

        print(f"Usage: {sys.argv[0]} <num threads> <sleep ms>")

        sys.exit(1)

    num_threads = int(sys.argv[1])

    sleep_time = int(sys.argv[2]) / 1000

    try:

        res = execute('99', "CREATE TABLE IF NOT EXISTS table1 (id INT auto_increment primary key, name VARCHAR(255), uid VARCHAR(36), call_from VARCHAR(36), clientid VARCHAR(36)) CHARACTER SET 'utf8'")

    except Exception as exc:

        print(f"got exception {exc}")

    tid = Thread(

        target=test_sqldb_insert,

        args=(99, sleep_time),

    )

    print(f"starting {99}")        

    tid.start()

    for thread_id in range(num_threads):

        tid = Thread(

            target=test_sqldb1,

            args=(thread_id, sleep_time),

        )

        print(f"starting {thread_id}")

        tid.start()

#!/bin/bash

# This is for an AMI 2023 on AWS

sudo yum install -y pip

sudo yum install -y python3-devel python3-debug

sudo yum install -y python3.11-devel python3.11-debug python3.11-pip

sudo yum install -y python3-virtualenv

sudo yum install -y gdb

python3.11 -m venv ${HOME}/venv

source ${HOME}/venv/bin/activate

pip3.11 install --upgrade pip

sudo dnf install -y mariadb105 mariadb105-server mariadb-connector-c mariadb-connector-c-devel

sudo systemctl enable mariadb

sudo systemctl start mariadb

wget https://dlm.mariadb.com/3677127/Connectors/c/connector-c-3.3.8/mariadb-connector-c-3.3.8-src.tar.gz

tar zxf mariadb-connector-c-3.3.8-src.tar.gz

cd mariadb-connector-c-3.3.8-src

mkdir build

cd build

sudo yum install -y cmake

cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local

make

sudo make install

cd ../..

pip3.11 install mariadb

# Ok messy, the installation of mariadb requires the old mariadb-connector-c to be there, but the new mariadb python package will need to access new connector, so going to hijak softlink

pushd /lib64

sudo rm libmariadb.so.3

sudo ln -s /usr/local/lib/mariadb/libmariadb.so.3 libmariadb.so.3

popd

sudo mysqladmin password "password"

sudo mysql --user=root <<_EOF_

DELETE FROM mysql.user WHERE User='';

DELETE FROM mysql.user WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1');

DROP DATABASE IF EXISTS test;

DELETE FROM mysql.db WHERE Db='test' OR Db='test\\_%';

CREATE USER IF NOT EXISTS 'test'@'localhost' IDENTIFIED BY 'password';

GRANT ALL PRIVILEGES ON *.* TO 'test'@'localhost' IDENTIFIED BY 'password';

FLUSH PRIVILEGES;

CREATE DATABASE TEST;

_EOF_

# For debuginfo in gdb

sudo dnf debuginfo-install -y python3.11-3.11.6-1.amzn2023.0.5.x86_64

sudo dnf debuginfo-install -y glibc-2.34-117.amzn2023.0.1.x86_64 libuuid-2.37.4-1.amzn2023.0.4.x86_64 mpdecimal-2.5.1-3.amzn2023.0.3.x86_64 openssl-libs-3.0.8-1.amzn2023.0.18.x86_64 zlib-1.2.11-33.amzn2023.0.5.x86_64