#!/usr/bin/env bash
#
# MDEV-41709: Aria: a write error during OPTIMIZE/REPAIR TABLE (maria_repair_by_sort)
# makes maria_close() of the new data handle fail; the error path then uses and frees
# the already-freed MARIA_HA (heap-use-after-free, double free, server abort).
#
# !!! WARNING: this CRASHES the mariadbd process (SIGABRT via glibc's double-free
# !!! detector). Every connection is dropped. Use a throwaway server and set
# !!! RESTART_CMD to a real command that restarts the SAME server (same datadir).
#
# Usage:
#   DB_PORT=<port> RESTART_CMD='<restart command>' ./mdev41709_repro.sh
#   SCAN=1 DB_PORT=<port> RESTART_CMD='...' ./mdev41709_repro.sh   # try several N until one crashes
#
# Environment:
#   N            which pwrite64() to t2.TMD to fail (default 73, as in the report;
#                also reported to work: 200)
#   SCAN=1       try N in 73 200 50 100 150 300 500 30 until one crashes the server
#   DB_HOST, DB_USER, DB_PASS, MARIADB_BIN   client connection (defaults: 127.0.0.1, root, mariadb)
#   SERVER_PID   mariadbd PID (default: from @@pid_file, then pgrep on --port)
#   RESTART_CMD  REQUIRED for the post-crash checks: a command that cleanly starts the
#                same server again on the same datadir/port/socket after it has aborted
#   YES=1        skip the confirmation prompt
#   KEEP=1       keep the test database afterwards
#
# Needs: sudo, strace >= 4.22 (for -P and inject=), ptrace allowed on mariadbd,
#        read access to the datadir, aria_recover_options with auto-repair (the default).

set -u
DB_HOST="${DB_HOST:-127.0.0.1}"
DB_USER="${DB_USER:-root}"
MARIADB_BIN="${MARIADB_BIN:-mariadb}"
DB="mdev41709"
OUT="${OUT_DIR:-/tmp/mdev41709_$(date +%Y%m%d_%H%M%S)}"
[ -n "${DB_PORT:-}" ] || { echo "Set DB_PORT (e.g. DB_PORT=11551 $0)" >&2; exit 2; }
mkdir -p "$OUT"

pass=0; fail=0
say()   { printf '\n=== %s ===\n' "$*"; }
check() { if [ "$2" -eq 0 ]; then echo "  [ok]   $1"; pass=$((pass+1)); else echo "  [MISS] $1"; fail=$((fail+1)); fi; }
has()   { grep -q -- "$2" <<<"$1"; }
cargs() { printf '%s\n' "-u$DB_USER" ${DB_PASS:+-p"$DB_PASS"} "-h$DB_HOST" "-P$DB_PORT"; }
q() { "$MARIADB_BIN" $(cargs) -N -B -e "$1" 2>&1 | grep -v 'ssl-verify-server-cert'; }

case "${RESTART_CMD:-}" in *"<"*|*"..."*)
  echo "RESTART_CMD looks like a placeholder: $RESTART_CMD" >&2
  echo "Give the real command that restarts THIS server (same datadir/port/socket)." >&2
  exit 2 ;;
esac
[ -n "${RESTART_CMD:-}" ] || { echo "RESTART_CMD is required (the server will crash)." >&2; exit 2; }

say "Server"
q "SELECT @@version, @@port, @@datadir" | tee "$OUT/server.txt"
DATADIR=$(q "SELECT @@datadir" | tail -1); DATADIR="${DATADIR%/}"
LOG=$(q "SELECT @@log_error" | tail -1)
pf=$(q "SELECT @@pid_file" | tail -1)
[ -r "$pf" ] && pid=$(cat "$pf")
[ -z "${SERVER_PID:-${pid:-}}" ] && pid=$(pgrep -f -- "mariadbd.*--port=${DB_PORT}( |\$)" | head -1)
PID="${SERVER_PID:-${pid:-}}"
[ -n "$PID" ] && [ -d "/proc/$PID" ] || { echo "Cannot find mariadbd PID; set SERVER_PID" >&2; exit 2; }
echo "mariadbd PID: $PID"
sudo -v || { echo "sudo required"; exit 2; }
strace -V 2>&1 | head -1
if [ "${YES:-0}" != "1" ]; then
  echo
  echo "This will CRASH (abort) the mariadbd process on port $DB_PORT. RESTART_CMD will be used"
  echo "to bring it back for the post-crash checks."
  read -r -p "Continue? [y/N] " a; [ "$a" = "y" ] || exit 1
fi

restart_server() {
  eval "$RESTART_CMD"
  local i
  for i in $(seq 1 120); do q "SELECT 1" | grep -q '^1$' && break; sleep 1; done
  if [ -d "/proc/$PID" ]; then
    echo "  WARNING: the old mariadbd (PID $PID) is still running - RESTART_CMD did not restart THIS server." >&2
    return 2
  fi
  pid=$(cat "$pf" 2>/dev/null); [ -z "$pid" ] && pid=$(pgrep -f -- "mariadbd.*--port=${DB_PORT}( |\$)" | head -1)
  [ -n "$pid" ] && [ -d "/proc/$pid" ] || return 1
  PID="$pid"
  return 0
}

create_table() {
  q "USE $DB;
DROP TABLE IF EXISTS t2;
CREATE TABLE t2 (id BIGINT PRIMARY KEY, k BIGINT, n BIGINT, v TEXT) ENGINE=Aria TRANSACTIONAL=1 ROW_FORMAT=PAGE;
INSERT INTO t2 SELECT seq, seq, seq, RPAD(MD5(seq), 900, MD5(seq+1)) FROM seq_1_to_2016;
DELETE FROM t2 WHERE id IN (954, 987, 1523, 1780, 1849);" | grep -i error && return 1
  return 0
}

# try_one <N> -> sets CRASHED (0/1), QOUT, TR
try_one() {
  local n="$1" spid_file="$OUT/strace_${n}.pid" i=0
  TR="$OUT/trace_${n}.txt"
  local f; f=$(readlink -f "$DATADIR/$DB/t2.TMD" 2>/dev/null || echo "$DATADIR/$DB/t2.TMD")
  nohup sudo -n sh -c "echo \$\$ > '$spid_file'; exec strace -f -qq -y -p $PID \
      -e trace=pwrite64 -e inject=pwrite64:error=EIO:when=$n -P '$f' -o '$TR'" >/dev/null 2>&1 &
  while [ "$(awk '/^TracerPid:/{print $2}' /proc/$PID/status 2>/dev/null)" = "0" ]; do
    i=$((i+1)); [ $i -gt 100 ] && { echo "strace did not attach" >&2; exit 1; }
    sleep 0.1
  done
  QOUT=$(timeout 30 "$MARIADB_BIN" $(cargs) -e "OPTIMIZE TABLE $DB.t2" 2>&1)
  sudo -n kill -INT "$(cat "$spid_file")" 2>/dev/null
  i=0
  while [ -d "/proc/$PID" ] && [ "$(awk '/^TracerPid:/{print $2}' /proc/$PID/status 2>/dev/null)" != "0" ]; do
    i=$((i+1)); [ $i -gt 50 ] && break; sleep 0.1
  done
  if [ -d "/proc/$PID" ]; then CRASHED=0; else CRASHED=1; fi
}

# ---------------------------------------------------------------- run
say "Setup"
q "DROP DATABASE IF EXISTS $DB; CREATE DATABASE $DB;" | grep -i error && exit 2
create_table || { echo "table setup failed" >&2; exit 2; }
echo "created $DB.t2 (2016 rows, 5 deleted, Aria TRANSACTIONAL=1 ROW_FORMAT=PAGE)"

CANDIDATES="${N:-73}"
[ "${SCAN:-0}" = "1" ] && CANDIDATES="73 200 50 100 150 300 500 30"

CRASHED=0; USED_N=""
for n in $CANDIDATES; do
  say "Injecting EIO on the ${n}th pwrite64() to t2.TMD, then OPTIMIZE TABLE t2"
  try_one "$n"
  echo "  OPTIMIZE TABLE output: ${QOUT:-<empty>}"
  if [ "$CRASHED" -eq 1 ]; then
    echo "  mariadbd PID $PID is GONE - looks like a crash"
    USED_N="$n"
    break
  else
    echo "  mariadbd PID $PID still running - no crash at N=$n, resetting the table and trying again"
    restart_server >/dev/null 2>&1 || true   # harmless if not needed; PID may be unchanged
    create_table || { echo "table re-setup failed" >&2; exit 2; }
  fi
done

say "Immediate evidence (waiting for the crash handler to finish writing the log)"
sleep 5   # core-dump writing + backtrace can take a few seconds after the process disappears
if [ -r "$LOG" ]; then
  CRASHLINE=$(grep -n -E "got signal|double free|corruption \(out\)" "$LOG" | tail -1 | cut -d: -f1)
  if [ -n "$CRASHLINE" ]; then
    START=$(awk -v end="$CRASHLINE" '
      NR<=end && /double free|corruption \(out\)|Thread pointer:|Printing a BT/{l=NR}
      END{print l+0}' "$LOG")
    [ "${START:-0}" -gt 0 ] || START=$((CRASHLINE>8 ? CRASHLINE-8 : 1))
    ENDLINE=$(awk -v start="$CRASHLINE" '
      NR>=start && /Kernel version:/{print NR+1; exit}' "$LOG")
    [ -n "$ENDLINE" ] || ENDLINE=$(wc -l < "$LOG")
    echo "  FULL crash report (log lines $START-$ENDLINE):"
    sed -n "${START},${ENDLINE}p" "$LOG" | tee "$OUT/crash_backtrace.txt" | sed 's/^/    /'
    echo "  (full backtrace saved to $OUT/crash_backtrace.txt)"
  else
    echo "  no crash signature found yet in the log - showing the last 40 lines:"
    tail -40 "$LOG" | sed 's/^/    /'
  fi
fi

say "Restart"
RESTOK=0
if restart_server; then RESTOK=1; echo "  server is back up"; else echo "  restart did not complete cleanly"; fi

CT=""; SC=""; LOGAFTER=""
if [ "$RESTOK" -eq 1 ]; then
  CT=$(q "CHECK TABLE $DB.t2")
  SC=$(q "SELECT COUNT(*) FROM $DB.t2")
  echo "  CHECK TABLE t2  -> $(echo "$CT" | tr '\t\n' '  ' | cut -c1-250)"
  echo "  SELECT COUNT(*) -> $SC"
  [ -r "$LOG" ] && LOGAFTER=$(grep -E "crashed, skipping|Please repair|Last repair was aborted" "$LOG" | tail -5)
  echo "$LOGAFTER" | sed 's/^/  log: /'
fi

# ---------------------------------------------------------------- checks
say "Checks"
grep -q INJECTED "$TR" 2>/dev/null;                 check "EIO injected on a pwrite64() to t2.TMD (N=${USED_N:-none})" $?
[ "$CRASHED" -eq 1 ];                                check "the mariadbd process is gone after OPTIMIZE TABLE (crash)" $?
has "$QOUT" "2013" || has "$QOUT" "2006" || has "$QOUT" "2026" || has "$QOUT" "Lost connection" || has "$QOUT" "unexpected eof"; check "client reports a lost/broken connection during the query" $?
grep -q "Got error" "$LOG" 2>/dev/null || echo "  (needs --log-warnings=3 on the server to show \"Got error N on close\"; not required for the other checks)"
grep -q "Got error" "$LOG" 2>/dev/null;               check "error log: \"Got error <n> on close\" for the table (needs --log-warnings=3)" $?
grep -qE "double free|corruption \(out\)|got signal" "$LOG" 2>/dev/null; check "error log shows the double-free abort (signal 6 / double free)" $?
[ "$RESTOK" -eq 1 ];                                 check "the server restarts after the crash" $?
if [ "$RESTOK" -eq 1 ]; then
  has "$LOGAFTER" "crashed, skipping" || has "$LOGAFTER" "Last repair was aborted"; check "the table is marked crashed at the next open" $?
  has "$CT" "Corrupt" || has "$CT" "Found";          check "CHECK TABLE reports a problem before auto-repair runs" $?
  has "$SC" "2011";                                  check "SELECT COUNT(*) is 2011 after auto-repair (no data lost)" $?
fi

# ---------------------------------------------------------------- cleanup
if [ "${KEEP:-0}" != "1" ] && [ "$RESTOK" -eq 1 ]; then
  say "Cleanup"; q "DROP DATABASE IF EXISTS $DB" >/dev/null; echo "  dropped $DB"
elif [ "$RESTOK" -ne 1 ]; then
  echo
  echo "  NOTE: the server did not restart cleanly; check it by hand, then: DROP DATABASE IF EXISTS $DB;"
fi
say "Summary"
echo "  matched: $pass   not matched: $fail"
if [ "$fail" -eq 0 ]; then echo "  RESULT: BUG REPRODUCED (all ticket symptoms observed)"
else echo "  RESULT: NOT (fully) reproduced - see [MISS] lines and $OUT"; fi
echo "  Artifacts: $OUT"
[ "$fail" -eq 0 ]