#!/bin/bash
#
# MDEV-38190 -- manual repro for the confirmed SHUTDOWN hang, VARIANT 2:
# same bug, but the semi-sync ACK is withheld over a real TCP proxy
# instead of a server-side DBUG hook. No debug build feature is required
# for the withholding mechanism itself; this variant would work the same
# way against a release build of the server.
#
#   slave  <-->  proxy (semisync_ack_proxy.py)  <-->  real master
#
# master->slave traffic (the binlog dump) always passes through. slave->
# master traffic (which carries the semi-sync ACK, interleaved on the same
# connection) is gated by a plain file: while it says CLOSED, the proxy
# buffers those bytes in memory instead of forwarding them -- the ACK
# genuinely never reaches the master, over the wire, for real. Flipping
# the gate file to OPEN flushes it.
#
# Usage: ./repro_shutdown_hang_proxy.sh <path-to-build-dir>
#
#   <path-to-build-dir> is the built tree containing sql/mariadbd -- debug
#   or release, since this variant needs no debug-build feature -- e.g.
#   ~/mariadb/bie/debug (the source checkout is assumed to be its parent
#   directory).
#
# On exit, if the hang reproduced, the master process is LEFT RUNNING
# (still hung) for inspection -- kill it with the PID printed at the end.

set -uo pipefail

# ---- configuration ------------------------------------------------------
REPO="${1:?Usage: $0 <path-to-build-dir>}"
BIN="$REPO/sql/mariadbd"
CLI="$REPO/client/mysql"
INSTALL_DB="$REPO/scripts/mariadb-install-db"
SRCDIR="$(dirname "$REPO")"
PROXY_SCRIPT="$(dirname "$0")/semisync_ack_proxy.py"

BASE="$(dirname "$0")/sd04-manual-proxy"
MASTER_PORT=15811
SLAVE_PORT=15812
PROXY_PORT=15813
GATE_FILE="$BASE/proxy_gate"
# ------------------------------------------------------------------------

M="$CLI --socket=$BASE/master/mysqld.sock -uroot"
S="$CLI --socket=$BASE/slave/mysqld.sock -uroot"

log() { echo "[$(date +%T)] $*"; }
die() { echo "FATAL: $*" >&2; exit 1; }
gate() { echo -n "$1" > "$GATE_FILE"; }

[ -x "$BIN" ] || die "no mariadbd at $BIN (is $REPO a built tree?)"

# ---- 0. clean slate -----------------------------------------------------
log "Killing any leftover servers/proxy from a previous run..."
pkill -9 -f "$BASE/master/mysqld.sock" 2>/dev/null
pkill -f "$BASE/slave/mysqld.sock" 2>/dev/null
pkill -f "semisync_ack_proxy.py.*$PROXY_PORT" 2>/dev/null
sleep 1

rm -rf "$BASE"
mkdir -p "$BASE"/{master,slave}/{data,binlogdir} "$BASE/logs"

# ---- 1. install fresh datadirs ------------------------------------------
log "Running mariadb-install-db for master and slave..."
"$INSTALL_DB" --no-defaults --datadir="$BASE/master/data" \
  --auth-root-authentication-method=normal --srcdir="$SRCDIR" \
  > "$BASE/logs/install_master.log" 2>&1 \
  || die "master install-db failed, see $BASE/logs/install_master.log"
"$INSTALL_DB" --no-defaults --datadir="$BASE/slave/data" \
  --auth-root-authentication-method=normal --srcdir="$SRCDIR" \
  > "$BASE/logs/install_slave.log" 2>&1 \
  || die "slave install-db failed, see $BASE/logs/install_slave.log"

# ---- 2. start master, slave, and the proxy between them ----------------
log "Starting master (port $MASTER_PORT) and slave (port $SLAVE_PORT)..."

"$BIN" --no-defaults \
  --datadir="$BASE/master/data" --socket="$BASE/master/mysqld.sock" \
  --port=$MASTER_PORT --bind-address=127.0.0.1 \
  --pid-file="$BASE/master/mysqld.pid" \
  --log-error="$BASE/logs/master.err" \
  --server-id=1 --log-bin --binlog-directory="$BASE/master/binlogdir" \
  --binlog-storage-engine=innodb --gtid-domain-id=0 \
  --innodb-buffer-pool-size=64M --innodb-log-file-size=32M \
  --skip-name-resolve --rpl-semi-sync-master-enabled=1 --core-file \
  > "$BASE/logs/master.out" 2>&1 &
disown

"$BIN" --no-defaults \
  --datadir="$BASE/slave/data" --socket="$BASE/slave/mysqld.sock" \
  --port=$SLAVE_PORT --bind-address=127.0.0.1 \
  --pid-file="$BASE/slave/mysqld.pid" \
  --log-error="$BASE/logs/slave.err" \
  --server-id=2 --log-bin --binlog-directory="$BASE/slave/binlogdir" \
  --binlog-storage-engine=innodb --gtid-domain-id=0 \
  --innodb-buffer-pool-size=64M --innodb-log-file-size=32M \
  --skip-name-resolve --rpl-semi-sync-slave-enabled=1 --log-slave-updates=1 \
  --core-file \
  > "$BASE/logs/slave.out" 2>&1 &
disown

for sock in "$BASE/master/mysqld.sock" "$BASE/slave/mysqld.sock"; do
  for i in $(seq 1 30); do
    [ -S "$sock" ] && break
    sleep 0.5
  done
  [ -S "$sock" ] || die "server socket $sock never appeared, check logs/*.err"
done
sleep 1

log "Starting the ACK-gating proxy: 127.0.0.1:$PROXY_PORT -> 127.0.0.1:$MASTER_PORT"
python3 "$PROXY_SCRIPT" "$PROXY_PORT" 127.0.0.1 "$MASTER_PORT" "$GATE_FILE" \
  > "$BASE/logs/proxy.log" 2>&1 &
disown
sleep 1
pgrep -f "semisync_ack_proxy.py $PROXY_PORT" >/dev/null || die "proxy did not start, see $BASE/logs/proxy.log"
gate OPEN
log "Gate is OPEN (normal passthrough) for initial setup."

# ---- 3. wire up replication THROUGH THE PROXY ---------------------------
log "Setting up replication (slave -> proxy -> master)..."
$M -e "CREATE TABLE test.t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB;" \
  || die "CREATE TABLE failed"
$S -e "CHANGE MASTER TO MASTER_HOST='127.0.0.1', MASTER_PORT=$PROXY_PORT,
       MASTER_USER='root', MASTER_USE_GTID=slave_pos; START SLAVE;" \
  || die "CHANGE MASTER/START SLAVE failed"

sleep 2
io_running=$($S -e "SHOW SLAVE STATUS\G" 2>/dev/null | awk -F': ' '/Slave_IO_Running/{print $2}')
[ "$io_running" = "Yes" ] || die "slave IO thread did not start (check it connected via the proxy)"

status=$($M -e "SHOW STATUS LIKE 'Rpl_semi_sync_master_status';" 2>/dev/null | awk 'NR==2{print $2}')
[ "$status" = "ON" ] || die "semi-sync master status is not ON ($status)"
log "Replication up THROUGH THE PROXY, semi-sync master status=ON."

# ---- 4. long timeout, then CLOSE the gate -------------------------------
log "Setting a long semi-sync master timeout so the 3 waiters below don't"
log "auto-resolve via the ordinary 10s timeout before we can observe them."
$M -e "SET GLOBAL rpl_semi_sync_master_timeout= 3600000;" \
  || die "failed to set rpl_semi_sync_master_timeout"

log "Closing the gate: slave->master ACK bytes will now be withheld for real."
gate CLOSED

# ---- 5. fire 3 blocking transactions across 3 GTID domains --------------
log "Firing 3 transactions in 3 different GTID domains (each will block" \
    "waiting for a semi-sync ACK that the proxy is withholding)..."
for d in 1 2 3; do
  nohup $CLI --socket="$BASE/master/mysqld.sock" -uroot \
    -e "SET SESSION gtid_domain_id=$d; INSERT INTO test.t1 VALUES ($d, $d);" \
    > "$BASE/logs/txn_domain${d}.out" 2> "$BASE/logs/txn_domain${d}.err" &
  disown
done

ok=0
for i in $(seq 1 30); do
  n=$($M -e "SHOW STATUS LIKE 'Rpl_semi_sync_master_wait_sessions';" 2>/dev/null | awk 'NR==2{print $2}')
  if [ "$n" = "3" ]; then ok=1; break; fi
  sleep 0.5
done
[ "$ok" = "1" ] || die "never saw Rpl_semi_sync_master_wait_sessions=3 (got '$n')"
log "Confirmed: Rpl_semi_sync_master_wait_sessions=3, all 3 transactions genuinely pending" \
    "(their ACKs are sitting in the proxy's buffer, never touched master's socket)."

# ---- 6. issue the shutdown that should never come back ------------------
log "Issuing SHUTDOWN WAIT FOR ALL SLAVES from a 4th connection..."
nohup $CLI --socket="$BASE/master/mysqld.sock" -uroot \
  -e "SHUTDOWN WAIT FOR ALL SLAVES;" \
  > "$BASE/logs/shutdown_cmd.out" 2> "$BASE/logs/shutdown_cmd.err" &
disown

sleep 2
grep -q "Normal shutdown" "$BASE/logs/master.err" \
  || log "WARNING: 'Normal shutdown' not seen yet in master.err (may still be in flight)"

# ---- 7. open the gate: proxy flushes the real, withheld ACK bytes -------
log "Opening the gate -- proxy flushes the buffered ACK bytes to the master..."
gate OPEN

ok=0
for i in $(seq 1 30); do
  grep -q "Stopping ack receiver thread" "$BASE/logs/master.err" 2>/dev/null && { ok=1; break; }
  sleep 0.5
done
[ "$ok" = "1" ] && log "Confirmed: 'Stopping ack receiver thread' logged -- all 3 ACKs landed." \
                 || log "WARNING: never saw 'Stopping ack receiver thread' within 15s."

# ---- 8. now watch: does shutdown actually complete? ---------------------
MASTER_PID=$(pgrep -f "$BASE/master/mysqld.sock" | head -1)
[ -n "$MASTER_PID" ] || die "master process already gone -- can't observe the hang"

WATCH_SECS=30
log "Watching for up to ${WATCH_SECS}s for 'Shutdown completed' in master.err..."
completed=0
for i in $(seq 1 $WATCH_SECS); do
  grep -q "Shutdown completed" "$BASE/logs/master.err" 2>/dev/null && { completed=1; break; }
  kill -0 "$MASTER_PID" 2>/dev/null || { completed=1; break; }
  sleep 1
done

if [ "$completed" = "1" ]; then
  echo
  echo "=== RESULT: shutdown completed normally -- hang did NOT reproduce this run ==="
  tail -5 "$BASE/logs/master.err"
  exit 0
fi

# ---- 9. hang confirmed: prove it's a genuine spin, not just slow --------
log "No 'Shutdown completed' after ${WATCH_SECS}s and process is still alive."
log "Sampling thread CPU usage twice, 3s apart, to confirm a genuine spin..."

spin_tid=""
declare -A u1
for t in /proc/$MASTER_PID/task/*; do
  tid=$(basename "$t")
  u1[$tid]=$(awk '{print $14}' "$t/stat" 2>/dev/null)
done
sleep 3
for t in /proc/$MASTER_PID/task/*; do
  tid=$(basename "$t")
  u2=$(awk '{print $14}' "$t/stat" 2>/dev/null)
  comm=$(cat "$t/comm" 2>/dev/null)
  state=$(awk '{print $3}' "$t/stat" 2>/dev/null)
  before=${u1[$tid]:-0}
  delta=$(( ${u2:-0} - before ))
  if [ "$delta" -gt 200 ]; then
    spin_tid=$tid
    echo "  -> tid=$tid comm=$comm state=$state utime_delta=${delta} ticks / 3s wall  <-- SPINNING"
  fi
done

echo
if [ -n "$spin_tid" ]; then
  echo "=== RESULT: HANG REPRODUCED (via network proxy, no DBUG hook) ==="
  echo "  master pid=$MASTER_PID is alive but never finished shutdown."
  echo "  thread $spin_tid is pinned at near-100% CPU with no progress in the error log."
else
  echo "=== RESULT: shutdown stalled (no 'Shutdown completed') but no spinning thread found."
  echo "  master pid=$MASTER_PID is still alive; inspect $BASE/logs/master.err manually."
fi
echo
echo "master.err tail:"
tail -10 "$BASE/logs/master.err"
echo
echo "Master (pid $MASTER_PID) has been left running for inspection."
echo "Slave and proxy are still up too. To clean up:"
echo "  kill -9 $MASTER_PID"
echo "  pkill -f '$BASE/slave/mysqld.sock'"
echo "  pkill -f 'semisync_ack_proxy.py $PROXY_PORT'"