#!/usr/bin/env bash
#
# Reproducer: stale OID 1043 (syscolumn.charsetnum) extents left behind by a
# failed MCOL-5162 automatic syscat upgrade in dbbuilder, and the resulting
# ~50 s TupleBPS::goodExtentCount() retry stalls.
#
# Mechanism under test (FileOp::createFile(fid,...), we_fileop.cpp):
#   allocateColExtentExactFile()  -> extent is in the EM (unavailable, HWM 0)
#   isDiskSpaceAvail() / create   -> fails, nothing rolls the extent back
#   next dbbuilder run            -> file still absent, upgrade again, new extent
#                                    at the next FBO (4096, 8192, ...)
#   first successful run          -> writes a 1 MiB abbreviated extent at file
#                                    offset 0, tagged with the LAST extent's LBID
#
# Fault injection:
#   f1  WriteEngine.MaxFileSystemDiskUsagePct=1, so isDiskSpaceAvail() fails
#       right after the allocation (ERR_FILE_DISK_SPACE). No code change.
#   f2  run dbbuilder under gdb, stop when allocateColExtentExactFile() returns,
#       SIGKILL it (models a crash, kill or restart in the middle of the upgrade).
#
# DESTRUCTIVE: deletes OID 1043 from the Extent Map and its segment file.
# Run it only on a disposable single-node install (local storage or StorageManager).
#
# Usage: repro_syscat_upgrade_extents.sh --mode f1|f2 [--attempts N]
#                                        [--workdir DIR] [--timeout SEC] --yes

set -uo pipefail

MODE=""
ATTEMPTS=8
WORKDIR="./repro-1043-$(date +%Y%m%d-%H%M%S)"
QTIMEOUT=900
CONFIRMED=0

OID=1043              # OID_SYSCOLUMN_CHARSETNUM
REF_OID=1024          # OID_SYSCOLUMN_OBJECTID, the reference column used by the upgrade
COLWIDTH=4
DBUSER=mysql
TESTDB=mcol_repro

usage() { sed -n '2,27p' "$0" | sed 's/^# \{0,1\}//'; exit 1; }

while [[ $# -gt 0 ]]; do
  case "$1" in
    --mode) MODE="$2"; shift 2 ;;
    --attempts) ATTEMPTS="$2"; shift 2 ;;
    --workdir) WORKDIR="$2"; shift 2 ;;
    --timeout) QTIMEOUT="$2"; shift 2 ;;
    --yes) CONFIRMED=1; shift ;;
    -h|--help) usage ;;
    *) echo "unknown argument: $1"; usage ;;
  esac
done

[[ "$MODE" == f1 || "$MODE" == f2 ]] || usage
[[ $CONFIRMED -eq 1 ]] || { echo "Refusing to run without --yes (destructive, see header)."; exit 1; }
[[ $EUID -eq 0 ]] || { echo "Must run as root."; exit 1; }

mkdir -p "$WORKDIR" || exit 1
WORKDIR=$(cd "$WORKDIR" && pwd)
LOG="$WORKDIR/repro.log"
RESULTS=()

log()  { echo "[$(date +%T)] $*" | tee -a "$LOG"; }
die()  { log "FATAL: $*"; exit 1; }
check() {  # check <name> <expected> <actual>
  local st=FAIL; [[ "$2" == "$3" ]] && st=PASS
  RESULTS+=("$st  $1 (expected: $2, got: $3)")
  log "$st  $1 (expected: $2, got: $3)"
}

need() { command -v "$1" >/dev/null 2>&1 || die "missing tool: $1"; }
for t in editem save_brm dbbuilder mcsGetConfig mcsSetConfig mariadb systemctl bc; do need "$t"; done
[[ "$MODE" == f2 ]] && need gdb

SM_ENABLED=$(mcsGetConfig StorageManager Enabled 2>/dev/null)
[[ "$SM_ENABLED" == Y ]] && { need smls; need smrm; need smcat; }
DBROOT1=$(mcsGetConfig SystemConfig DBRoot1)
EXTENT_ROWS=$(mcsGetConfig ExtentMap ExtentRows 2>/dev/null); EXTENT_ROWS=${EXTENT_ROWS:-8388608}
EXTENT_BLOCKS=$(( EXTENT_ROWS * COLWIDTH / 8192 ))
ABBREV_BYTES=$(( 256 * 1024 * COLWIDTH ))    # INITIAL_EXTENT_ROWS_TO_DISK * width
SEGDIR="$DBROOT1/000.dir/000.dir/004.dir/019.dir/000.dir"   # oid2FileName(1043, p0, s0)
SEGFILE="$SEGDIR/FILE000.cdf"
CS_LOGDIR=/var/log/mariadb/columnstore

log "mode=$MODE attempts=$ATTEMPTS storagemanager=$SM_ENABLED dbroot1=$DBROOT1"
log "extent blocks for width $COLWIDTH: $EXTENT_BLOCKS; abbreviated extent bytes: $ABBREV_BYTES"
log "workdir=$WORKDIR"

# ---------------------------------------------------------------- helpers ---
sql() { timeout "$QTIMEOUT" mariadb -N -B -e "$1" 2>&1; }

timed_sql() {  # timed_sql <label> <query>; prints "<rc> <seconds>" and saves the output
  local s e rc
  s=$(date +%s.%N)
  sql "$2" >"$WORKDIR/sql-$1.out"; rc=$?
  e=$(date +%s.%N)
  printf '%d %.2f\n' "$rc" "$(echo "$e - $s" | bc)"
}

em_dump() { editem -o "$1" 2>&1; }
em_extents() { em_dump "$1" | grep 'fbo:'; }
em_count() { em_extents "$1" | wc -l; }
field() { sed -n "s/.*$1: \([^,;]*\).*/\1/p"; }   # pull "<name>: value" from an editem line

file_size() {  # size of the 1043 segment file in bytes, or "absent"
  if [[ "$SM_ENABLED" == Y ]]; then
    local sz; sz=$(smls "$SEGDIR" 2>/dev/null | awk '$NF=="FILE000.cdf"{print $1}')
    echo "${sz:-absent}"
  else
    [[ -e "$SEGFILE" ]] && stat -c %s "$SEGFILE" || echo absent
  fi
}

remove_segfile() {
  if [[ "$SM_ENABLED" == Y ]]; then smrm "$SEGFILE"; else rm -f "$SEGFILE"; fi
}

backup_segfile() {
  if [[ "$SM_ENABLED" == Y ]]; then smcat "$SEGFILE" >"$WORKDIR/backup/FILE000.cdf.1043"
  else cp -p "$SEGFILE" "$WORKDIR/backup/FILE000.cdf.1043"; fi
}

svc() { systemctl "$1" "$2" >>"$LOG" 2>&1; }

stop_query_side() {
  # Stop what CMAPI could use to re-run dbbuilder on its own (mcs-ddlproc start),
  # and anything that caches blocks. controllernode/workernode (and the
  # storagemanager) stay up: editem and dbbuilder need them.
  for s in mariadb-columnstore-cmapi mcs-ddlproc mcs-dmlproc mcs-writeengineserver mcs-primproc; do
    systemctl is-active --quiet "$s" && { log "stopping $s"; svc stop "$s"; }
  done
}

start_query_side() {
  # Start through systemctl, not CMAPI: CMAPI would run dbbuilder again after DDLProc.
  for s in mcs-primproc mcs-writeengineserver mcs-dmlproc mcs-ddlproc; do
    log "starting $s"; svc start "$s"
  done
  for _ in $(seq 60); do sql "select 1" >/dev/null && return 0; sleep 2; done
  die "server did not come back"
}

run_dbbuilder_clean() { su -s /bin/sh -c "dbbuilder 7" "$DBUSER" >"$1" 2>&1; }

run_dbbuilder_fault() {  # run_dbbuilder_fault <logfile>
  case "$MODE" in
    f1)
      run_dbbuilder_clean "$1" ;;
    f2)
      # Stop when allocateColExtentExactFile() returns, i.e. the extent is in
      # the EM and the file is not created yet, then SIGKILL the process.
      su -s /bin/sh -c "gdb -q -batch -nx \
          -ex 'set pagination off' -ex 'set confirm off' \
          -ex 'set breakpoint pending on' \
          -ex 'break WriteEngine::BRMWrapper::allocateColExtentExactFile' \
          -ex run -ex bt -ex finish -ex 'echo ===ALLOCATED, KILLING===\n' \
          -ex 'signal SIGKILL' \
          --args \$(command -v dbbuilder) 7" "$DBUSER" >"$1" 2>&1 ;;
  esac
}

log_mark() { for f in "$CS_LOGDIR"/{crit,err,warning,info,debug}.log; do [[ -f $f ]] && wc -l <"$f" || echo 0; done | paste -sd' '; }

logs_since() {  # logs_since "<mark>" -> new lines of all Columnstore logs
  local i=0 m=($1)
  for f in "$CS_LOGDIR"/{crit,err,warning,info,debug}.log; do
    [[ -f $f ]] && tail -n +"$(( ${m[$i]} + 1 ))" "$f"
    i=$((i + 1))
  done
}

# ---------------------------------------------------------------- 1. baseline
log "== 1. baseline"
mkdir -p "$WORKDIR/backup"
cp -p /etc/columnstore/Columnstore.xml "$WORKDIR/backup/"
save_brm >>"$LOG" 2>&1 || die "save_brm failed"
DBRMROOT=$(mcsGetConfig SystemConfig DBRMRoot)
cp -p "$DBRMROOT"* "$WORKDIR/backup/" || die "cannot back up BRM files"
em_dump "$OID" >"$WORKDIR/em-$OID-baseline.txt"
em_dump "$REF_OID" >"$WORKDIR/em-$REF_OID-baseline.txt"
[[ "$(file_size)" != absent ]] && backup_segfile

sql "create database if not exists $TESTDB" >/dev/null
sql "drop table if exists $TESTDB.t1; create table $TESTDB.t1 (a int) engine=columnstore; insert into $TESTDB.t1 values (1)" \
  >>"$LOG" || die "cannot create test table"

REF_HWM=$(em_extents "$REF_OID" | tail -1 | field HWM)
log "ref OID $REF_OID extents=$(em_count $REF_OID) HWM=$REF_HWM; OID $OID extents=$(em_count $OID) size=$(file_size)"
read -r rc t <<<"$(timed_sql base-select "select * from $TESTDB.t1")";  log "baseline select t1: rc=$rc ${t}s"
read -r rc t <<<"$(timed_sql base-charset "select count(charsetnum) from calpontsys.syscolumn")"; log "baseline count(charsetnum): rc=$rc ${t}s"

# ------------------------------------------------- 2. pre-upgrade catalog state
log "== 2. remove OID $OID from EM and storage (pre-MCOL-5162 state)"
stop_query_side
editem -f -e "$OID" >>"$LOG" 2>&1 || die "editem -e $OID failed"
remove_segfile >>"$LOG" 2>&1
save_brm >>"$LOG" 2>&1
check "OID $OID extents after removal" 0 "$(em_count $OID)"
check "OID $OID file after removal" absent "$(file_size)"

# --------------------------------------------------- 3. failed upgrade attempts
log "== 3. $ATTEMPTS failed upgrade attempts (mode $MODE)"
if [[ "$MODE" == f1 ]]; then
  ORIG_PCT=$(mcsGetConfig WriteEngine MaxFileSystemDiskUsagePct); ORIG_PCT=${ORIG_PCT:-98}
  mcsSetConfig WriteEngine MaxFileSystemDiskUsagePct 1 || die "mcsSetConfig failed"
  trap 'mcsSetConfig WriteEngine MaxFileSystemDiskUsagePct "$ORIG_PCT"' EXIT
fi

for i in $(seq 1 "$ATTEMPTS"); do
  run_dbbuilder_fault "$WORKDIR/dbbuilder-attempt-$i.log"
  em_dump "$OID" >"$WORKDIR/em-$OID-attempt-$i.txt"
  last=$(em_extents "$OID" | tail -1)
  log "attempt $i: extents=$(em_count $OID) last=[$last] file=$(file_size)"
  check "attempt $i: extent count" "$i" "$(em_count $OID)"
  check "attempt $i: new extent fbo" "$(( (i - 1) * EXTENT_BLOCKS ))" "$(echo "$last" | field fbo)"
  check "attempt $i: new extent status/HWM" "unavail/0" "$(echo "$last" | field status)/$(echo "$last" | field HWM)"
  check "attempt $i: file" absent "$(file_size)"
  if [[ "$(em_count $OID)" != "$i" ]]; then
    log "extent count did not grow as predicted; see $WORKDIR/dbbuilder-attempt-$i.log"
    break
  fi
done

# ------------------------------------------------------ 4. successful attempt
log "== 4. fault removed, one clean dbbuilder run"
if [[ "$MODE" == f1 ]]; then mcsSetConfig WriteEngine MaxFileSystemDiskUsagePct "$ORIG_PCT"; trap - EXIT; fi
run_dbbuilder_clean "$WORKDIR/dbbuilder-final.log"
save_brm >>"$LOG" 2>&1
em_dump "$OID" >"$WORKDIR/em-$OID-final.txt"
N=$(em_count $OID); last=$(em_extents "$OID" | tail -1)
log "final: extents=$N last=[$last] file=$(file_size)"
check "final: extent count" "$(( ATTEMPTS + 1 ))" "$N"
check "final: last extent fbo" "$(( ATTEMPTS * EXTENT_BLOCKS ))" "$(echo "$last" | field fbo)"
check "final: last extent status/HWM" "avail/$REF_HWM" "$(echo "$last" | field status)/$(echo "$last" | field HWM)"
check "final: stale extents (unavail, HWM 0)" "$ATTEMPTS" "$(em_extents $OID | grep -c 'HWM: 0; status: unavail')"
check "final: file size (1 MiB abbreviated extent only)" "$ABBREV_BYTES" "$(file_size)"

# ---------------------------------------------------------------- 5. symptoms
log "== 5. symptoms"
start_query_side
MARK=$(log_mark)

read -r rc t <<<"$(timed_sql control-systable "select count(*) from calpontsys.systable")"
log "control count(*) systable: rc=$rc ${t}s"

read -r rc t <<<"$(timed_sql charset "select count(charsetnum) from calpontsys.syscolumn")"
log "count(charsetnum): rc=$rc ${t}s: $(head -c 300 "$WORKDIR/sql-charset.out")"
check "count(charsetnum) fails" "fails" "$([[ $rc -ne 0 ]] || grep -qi error "$WORKDIR/sql-charset.out" && echo fails || echo succeeds)"

# SELECT with a stack sample in the middle of the stall, to show where the time goes.
( sleep 15
  gdb -q -batch -nx -p "$(pidof PrimProc)" -ex 'set pagination off' -ex 'thread apply all bt' \
    >"$WORKDIR/primproc-stacks-during-select.txt" 2>&1 ) &
SAMPLER=$!
read -r rc t <<<"$(timed_sql select-t1 "select * from $TESTDB.t1")"
wait "$SAMPLER" 2>/dev/null
log "select t1: rc=$rc ${t}s (baseline was fast)"
check "select t1 stalls >= 50s" yes "$(echo "$t >= 50" | bc | sed 's/1/yes/;s/0/no/')"
check "select t1 still succeeds (dead ERR_TUPLE_BPS branch)" 0 "$rc"
STALL_FRAMES=$(grep -c 'TupleBPS::run' "$WORKDIR/primproc-stacks-during-select.txt" 2>/dev/null)
check "PrimProc thread in TupleBPS::run during stall" yes "$([[ ${STALL_FRAMES:-0} -gt 0 ]] && echo yes || echo no)"

read -r rc t <<<"$(timed_sql insert-t1 "insert into $TESTDB.t1 values (2); commit")"
log "insert+commit t1: rc=$rc ${t}s"

logs_since "$MARK" >"$WORKDIR/columnstore-logs-symptoms.txt"
log "log lines about OID $OID / EOF / HWM:"
grep -Ei "1043|early eof|hwm|consistent extent count" "$WORKDIR/columnstore-logs-symptoms.txt" | head -20 | tee -a "$LOG"

# ---------------------------------------------------------------- summary
log "== summary"
printf '%s\n' "${RESULTS[@]}" | tee "$WORKDIR/summary.txt"
FAILS=$(grep -c '^FAIL' "$WORKDIR/summary.txt")
log "$FAILS check(s) failed. Artifacts: $WORKDIR"
cat <<EOF | tee -a "$LOG"

The cluster is left in the broken state on purpose (it is the test fixture):
  BRM backup of the broken EM: run save_brm and copy $DBRMROOT* now
  Pre-test BRM + 1043 file:    $WORKDIR/backup/
To go back to the pre-test state: stop all mcs-* services, restore the BRM_saves
files from $WORKDIR/backup into $(dirname "$DBRMROOT"), restore FILE000.cdf.1043
to $SEGFILE (smput with StorageManager), then start the cluster.
CMAPI (mariadb-columnstore-cmapi) was stopped and is left stopped.
EOF
exit $(( FAILS > 0 ))