#!/bin/bash
#
# MDEV-38190 -- manual repro for the confirmed SHUTDOWN hang:
#
#   Under BiE (--binlog-storage-engine=innodb) with semi-sync master
#   enabled, if SHUTDOWN WAIT FOR ALL SLAVES is issued while several
#   transactions across DIFFERENT gtid_domain_ids are genuinely pending a
#   semi-sync ACK (slave IO thread connected the whole time; ack withheld
#   via a DBUG hook, NOT via STOP SLAVE IO_THREAD), the shutdown never
#   completes.  A client thread that was killed by close_connections()
#   while parked inside the semi-sync commit wait spins at ~100% CPU
#   forever instead of unwinding, and the rest of shutdown
#   (close_connections() -> InnoDB shutdown) never runs.
#
# This script builds a fresh 2-node master/slave pair from a debug build,
# arms the ack-withholding DBUG hook (sql/slave.cc:
# "simulate_delay_semisync_slave_reply"), fires 3 blocking transactions in
# 3 GTID domains, issues SHUTDOWN WAIT FOR ALL SLAVES, releases the held
# ack, and then proves the hang two ways:
#   1. the error log stalls forever after "Stopping ack receiver thread"
#      (no "InnoDB: Starting shutdown", no "Shutdown completed"), and
#   2. a live one_connection thread is sampled twice and shown to be
#      consuming ~100% CPU continuously (state R, wchan=0), not just slow.
#
# Usage: ./repro_shutdown_hang.sh <path-to-debug-build-dir>
#
#   <path-to-debug-build-dir> is the built debug tree containing sql/mariadbd,
#   e.g. ~/mariadb/bie/debug (the source checkout is assumed to be its parent
#   directory).
#
# On exit, if the hang reproduced, the master process is LEFT RUNNING
# (still hung) for further inspection -- kill it with:
#   kill -9 <master_pid>
# The script prints that PID at the end.

set -uo pipefail

# ---- configuration ------------------------------------------------------
REPO="${1:?Usage: $0 <path-to-debug-build-dir>}"
BIN="$REPO/sql/mariadbd"
CLI="$REPO/client/mysql"
INSTALL_DB="$REPO/scripts/mariadb-install-db"
SRCDIR="$(dirname "$REPO")"

BASE="$(dirname "$0")/sd04-manual"
MASTER_PORT=15801
SLAVE_PORT=15802
# ------------------------------------------------------------------------

[ -x "$BIN" ] || { echo "FATAL: no mariadbd at $BIN (is $REPO a built debug tree?)" >&2; exit 1; }

M="$CLI --socket=$BASE/master/mysqld.sock -uroot"
S="$CLI --socket=$BASE/slave/mysqld.sock -uroot"

log() { echo "[$(date +%T)] $*"; }
die() { echo "FATAL: $*" >&2; exit 1; }

# ---- 0. clean slate -----------------------------------------------------
log "Killing any leftover servers from a previous run..."
pkill -9 -f "$BASE/master/mysqld.sock" 2>/dev/null
pkill -f "$BASE/slave/mysqld.sock" 2>/dev/null
sleep 1

rm -rf "$BASE"
mkdir -p "$BASE"/{master,slave}/{data,binlogdir} "$BASE/logs"

# ---- 1. install fresh datadirs ------------------------------------------
log "Running mariadb-install-db for master and slave..."
"$INSTALL_DB" --no-defaults --datadir="$BASE/master/data" \
  --auth-root-authentication-method=normal --srcdir="$SRCDIR" \
  > "$BASE/logs/install_master.log" 2>&1 \
  || die "master install-db failed, see $BASE/logs/install_master.log"
"$INSTALL_DB" --no-defaults --datadir="$BASE/slave/data" \
  --auth-root-authentication-method=normal --srcdir="$SRCDIR" \
  > "$BASE/logs/install_slave.log" 2>&1 \
  || die "slave install-db failed, see $BASE/logs/install_slave.log"

# ---- 2. start both servers ----------------------------------------------
log "Starting master (port $MASTER_PORT) and slave (port $SLAVE_PORT)..."

"$BIN" --no-defaults \
  --datadir="$BASE/master/data" --socket="$BASE/master/mysqld.sock" \
  --port=$MASTER_PORT --bind-address=127.0.0.1 \
  --pid-file="$BASE/master/mysqld.pid" \
  --log-error="$BASE/logs/master.err" \
  --server-id=1 --log-bin --binlog-directory="$BASE/master/binlogdir" \
  --binlog-storage-engine=innodb --gtid-domain-id=0 \
  --innodb-buffer-pool-size=64M --innodb-log-file-size=32M \
  --skip-name-resolve --rpl-semi-sync-master-enabled=1 \
  --debug-sync-timeout=300 --core-file \
  > "$BASE/logs/master.out" 2>&1 &
disown

"$BIN" --no-defaults \
  --datadir="$BASE/slave/data" --socket="$BASE/slave/mysqld.sock" \
  --port=$SLAVE_PORT --bind-address=127.0.0.1 \
  --pid-file="$BASE/slave/mysqld.pid" \
  --log-error="$BASE/logs/slave.err" \
  --server-id=2 --log-bin --binlog-directory="$BASE/slave/binlogdir" \
  --binlog-storage-engine=innodb --gtid-domain-id=0 \
  --innodb-buffer-pool-size=64M --innodb-log-file-size=32M \
  --skip-name-resolve --rpl-semi-sync-slave-enabled=1 --log-slave-updates=1 \
  --debug-sync-timeout=300 --core-file \
  > "$BASE/logs/slave.out" 2>&1 &
disown

# wait for both sockets to come up
for sock in "$BASE/master/mysqld.sock" "$BASE/slave/mysqld.sock"; do
  for i in $(seq 1 30); do
    [ -S "$sock" ] && break
    sleep 0.5
  done
  [ -S "$sock" ] || die "server socket $sock never appeared, check logs/*.err"
done
sleep 1

# ---- 3. wire up replication ----------------------------------------------
log "Setting up replication (master -> slave)..."
$M -e "CREATE TABLE test.t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB;" \
  || die "CREATE TABLE failed"
$S -e "CHANGE MASTER TO MASTER_HOST='127.0.0.1', MASTER_PORT=$MASTER_PORT,
       MASTER_USER='root', MASTER_USE_GTID=slave_pos; START SLAVE;" \
  || die "CHANGE MASTER/START SLAVE failed"

sleep 2
io_running=$($S -e "SHOW SLAVE STATUS\G" 2>/dev/null | awk -F': ' '/Slave_IO_Running/{print $2}')
[ "$io_running" = "Yes" ] || die "slave IO thread did not start"

status=$($M -e "SHOW STATUS LIKE 'Rpl_semi_sync_master_status';" 2>/dev/null | awk 'NR==2{print $2}')
[ "$status" = "ON" ] || die "semi-sync master status is not ON ($status)"
log "Replication up, semi-sync master status=ON."

# ---- 4. arm the setup: long timeout + ack-withholding hook --------------
log "Setting a long semi-sync master timeout so the 3 waiters below don't"
log "auto-resolve via the ordinary 10s timeout before we can observe them."
$M -e "SET GLOBAL rpl_semi_sync_master_timeout= 3600000;" \
  || die "failed to set rpl_semi_sync_master_timeout"

log "Arming ack-withholding DBUG hook on the slave IO thread" \
    "(simulate_delay_semisync_slave_reply)..."
$S -e "SET GLOBAL debug_dbug= '+d,simulate_delay_semisync_slave_reply';" \
  || die "failed to set debug_dbug on slave"

# ---- 5. fire 3 blocking transactions across 3 GTID domains --------------
log "Firing 3 transactions in 3 different GTID domains (each will block" \
    "waiting for a semi-sync ACK that is being withheld)..."
for d in 1 2 3; do
  nohup $CLI --socket="$BASE/master/mysqld.sock" -uroot \
    -e "SET SESSION gtid_domain_id=$d; INSERT INTO test.t1 VALUES ($d, $d);" \
    > "$BASE/logs/txn_domain${d}.out" 2> "$BASE/logs/txn_domain${d}.err" &
  disown
done

# poll for all 3 to show up as pending waiters
ok=0
for i in $(seq 1 30); do
  n=$($M -e "SHOW STATUS LIKE 'Rpl_semi_sync_master_wait_sessions';" 2>/dev/null | awk 'NR==2{print $2}')
  if [ "$n" = "3" ]; then ok=1; break; fi
  sleep 0.5
done
[ "$ok" = "1" ] || die "never saw Rpl_semi_sync_master_wait_sessions=3 (got '$n')"
log "Confirmed: Rpl_semi_sync_master_wait_sessions=3, all 3 transactions genuinely pending."

# ---- 6. issue the shutdown that should never come back ------------------
log "Issuing SHUTDOWN WAIT FOR ALL SLAVES from a 4th connection..."
nohup $CLI --socket="$BASE/master/mysqld.sock" -uroot \
  -e "SHUTDOWN WAIT FOR ALL SLAVES;" \
  > "$BASE/logs/shutdown_cmd.out" 2> "$BASE/logs/shutdown_cmd.err" &
disown

sleep 2
grep -q "Normal shutdown" "$BASE/logs/master.err" \
  || log "WARNING: 'Normal shutdown' not seen yet in master.err (may still be in flight)"

# ---- 7. release the withheld ack ----------------------------------------
log "Releasing the withheld ack (all 3 real ACKs should now land)..."
$S -e "SET GLOBAL debug_dbug= ''; SET DEBUG_SYNC= 'now SIGNAL io_thd_do_reply';" \
  2>/dev/null
# (the slave connection above is unaffected by the master's shutdown)

ok=0
for i in $(seq 1 30); do
  grep -q "Stopping ack receiver thread" "$BASE/logs/master.err" 2>/dev/null && { ok=1; break; }
  sleep 0.5
done
[ "$ok" = "1" ] && log "Confirmed: 'Stopping ack receiver thread' logged -- all 3 ACKs landed." \
                 || log "WARNING: never saw 'Stopping ack receiver thread' within 15s."

# ---- 8. now watch: does shutdown actually complete? ---------------------
MASTER_PID=$(pgrep -f "$BASE/master/mysqld.sock" | head -1)
[ -n "$MASTER_PID" ] || die "master process already gone -- can't observe the hang"

WATCH_SECS=30
log "Watching for up to ${WATCH_SECS}s for 'Shutdown completed' in master.err..."
completed=0
for i in $(seq 1 $WATCH_SECS); do
  grep -q "Shutdown completed" "$BASE/logs/master.err" 2>/dev/null && { completed=1; break; }
  kill -0 "$MASTER_PID" 2>/dev/null || { completed=1; break; }  # process exited some other way
  sleep 1
done

if [ "$completed" = "1" ]; then
  echo
  echo "=== RESULT: shutdown completed normally -- hang did NOT reproduce this run ==="
  tail -5 "$BASE/logs/master.err"
  exit 0
fi

# ---- 9. hang confirmed: prove it's a genuine spin, not just slow --------
log "No 'Shutdown completed' after ${WATCH_SECS}s and process is still alive."
log "Sampling thread CPU usage twice, 3s apart, to confirm a genuine spin..."

spin_tid=""
declare -A u1
for t in /proc/$MASTER_PID/task/*; do
  tid=$(basename "$t")
  u1[$tid]=$(awk '{print $14}' "$t/stat" 2>/dev/null)
done
sleep 3
for t in /proc/$MASTER_PID/task/*; do
  tid=$(basename "$t")
  u2=$(awk '{print $14}' "$t/stat" 2>/dev/null)
  comm=$(cat "$t/comm" 2>/dev/null)
  state=$(awk '{print $3}' "$t/stat" 2>/dev/null)
  before=${u1[$tid]:-0}
  delta=$(( ${u2:-0} - before ))
  if [ "$delta" -gt 200 ]; then   # >~2s of CPU time burned in 3s wall-clock
    spin_tid=$tid
    echo "  -> tid=$tid comm=$comm state=$state utime_delta=${delta} ticks / 3s wall  <-- SPINNING"
  fi
done

echo
if [ -n "$spin_tid" ]; then
  echo "=== RESULT: HANG REPRODUCED ==="
  echo "  master pid=$MASTER_PID is alive but never finished shutdown."
  echo "  thread $spin_tid is pinned at near-100% CPU with no progress in the error log."
else
  echo "=== RESULT: shutdown stalled (no 'Shutdown completed') but no spinning thread found."
  echo "  master pid=$MASTER_PID is still alive; inspect $BASE/logs/master.err manually."
fi
echo
echo "master.err tail:"
tail -10 "$BASE/logs/master.err"
echo
echo "Master (pid $MASTER_PID) has been left running for inspection."
echo "Slave is still up too. To clean up:"
echo "  kill -9 $MASTER_PID"
echo "  pkill -f '$BASE/slave/mysqld.sock'"