#!/bin/bash
# mdev-semisync-repro.sh [IMAGE]
#
# Minimal reproducer: after a network partition on the replication port heals, a semi-sync
# primary stops completing the handshake of NEW connections while already-established
# sessions keep working. Builds everything from stock images and cleans up after itself.
#
# Requires: docker, nft, and root (or an equivalent) for both.
# Default image: mariadb:11.8.6 . Tested also with mariadb:11.8 (11.8.8) and mariadb:12 (12.3.2).
#
# Two things this harness is careful about, because four earlier designs failed silently:
#   * it refuses to continue unless the scenario is really in place (replica running, semi-sync
#     ON with one client, schema replicated) — otherwise "did not reproduce" would be meaningless;
#   * the firewall rule carries a counter and is checked, because a rule can exist, return rc=0
#     and match nothing at all.
#
# Why the topology looks convoluted: the two servers must be on DIFFERENT docker networks, with
# the primary reached through a published (DNAT'd) port. Containers sharing one bridge exchange
# frames at L2 and, with br_netfilter unloaded, that traffic never reaches netfilter, so no
# forward rule can drop it. And the cut must be a silent drop: closing the connection cleanly
# does not reproduce the bug.
set -uo pipefail

IMG="${1:-mariadb:11.8.6}"
NETA=mdevneta; NETB=mdevnetb; GW=172.29.0.1; PORT=33061
M=mdev-master; S=mdev-slave
D=/tmp/mdev-repro; mkdir -p "$D"
ts(){ date -u '+%FT%T.%3NZ'; }
say(){ echo "[$(ts)] $*"; }

cleanup(){
  nft delete table inet mdevfence 2>/dev/null
  [ -f "$D/writer.pid" ] && kill "$(cat "$D/writer.pid")" 2>/dev/null; rm -f "$D/writer.pid"
  [ -f "$D/witness.pid" ] && kill "$(cat "$D/witness.pid")" 2>/dev/null; rm -f "$D/witness.pid"
  docker rm -f -v "$M" "$S" >/dev/null 2>&1
  docker network rm "$NETA" "$NETB" >/dev/null 2>&1
}
cleanup
docker network create --subnet 172.28.0.0/24 "$NETA" >/dev/null
docker network create --subnet 172.29.0.0/24 "$NETB" >/dev/null

say "starting primary and replica ($IMG)"
docker run -d --name "$M" --network "$NETA" -p "$GW:$PORT:3306" \
  -e MARIADB_ALLOW_EMPTY_ROOT_PASSWORD=yes "$IMG" \
  --server-id=1 --log-bin=mb --binlog-format=ROW --log-slave-updates=ON >/dev/null
docker run -d --name "$S" --network "$NETB" \
  -e MARIADB_ALLOW_EMPTY_ROOT_PASSWORD=yes "$IMG" \
  --server-id=2 --log-bin=sb --binlog-format=ROW --log-slave-updates=ON >/dev/null

# Readiness over TCP on purpose: the image runs a temporary server with --skip-networking
# during initialisation, so a socket check passes far too early.
wait_ready(){ local c=$1 i; for i in $(seq 1 90); do
  docker exec "$c" mariadb -h 127.0.0.1 -P 3306 -N -B -e "SELECT 1" >/dev/null 2>&1 && return 0
  sleep 2; done; return 1; }
wait_ready "$M" || { say "ABORT: primary never served over TCP"; cleanup; exit 2; }
wait_ready "$S" || { say "ABORT: replica never served over TCP"; cleanup; exit 2; }
say "version: $(docker exec "$M" mariadb -N -B -e 'SELECT VERSION()')"

docker exec "$M" mariadb -e "
  SET GLOBAL rpl_semi_sync_master_enabled=ON;
  SET GLOBAL rpl_semi_sync_master_timeout=1000;
  SET GLOBAL rpl_semi_sync_master_wait_point=AFTER_SYNC;
  SET GLOBAL rpl_semi_sync_master_wait_no_slave=OFF;
  CREATE USER IF NOT EXISTS repl@'%' IDENTIFIED BY 'repl';
  GRANT REPLICATION SLAVE ON *.* TO repl@'%';"
docker exec "$S" mariadb -e "
  SET GLOBAL rpl_semi_sync_slave_enabled=ON;
  CHANGE MASTER TO MASTER_HOST='$GW', MASTER_PORT=$PORT, MASTER_USER='repl',
                   MASTER_PASSWORD='repl', MASTER_USE_GTID=slave_pos;
  START SLAVE;"
sleep 8
# schema created AFTER replication starts, so it travels through the binlog
docker exec "$M" mariadb -e "CREATE DATABASE IF NOT EXISTS demo;
  CREATE TABLE IF NOT EXISTS demo.t (id BIGINT AUTO_INCREMENT PRIMARY KEY, ts DATETIME(3), seq BIGINT) ENGINE=InnoDB;"
sleep 4

IO=$(docker exec "$S" mariadb -e "SHOW SLAVE STATUS\G" | awk -F': ' '/Slave_IO_Running:/{print $2}' | tr -d ' ')
ST=$(docker exec "$M" mariadb -N -B -e "SHOW GLOBAL STATUS LIKE 'Rpl_semi_sync_master_status'" | awk '{print $2}')
CL=$(docker exec "$M" mariadb -N -B -e "SHOW GLOBAL STATUS LIKE 'Rpl_semi_sync_master_clients'" | awk '{print $2}')
TB=$(docker exec "$S" mariadb -N -B -e "SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA='demo'")
say "scenario: replica IO=$IO | semi-sync=$ST clients=$CL | replicated tables=$TB"
[ "$IO" = Yes ] && [ "$ST" = ON ] && [ "$CL" = 1 ] && [ "$TB" = 1 ] || { say "ABORT: scenario not in place"; cleanup; exit 3; }

# witness: one connection opened BEFORE the partition, kept open throughout
setsid nohup bash -c "while :; do
    echo \"SELECT CONCAT('=== ', UTC_TIMESTAMP(3));\"
    echo \"SHOW GLOBAL STATUS WHERE Variable_name IN ('Threads_connected','Threads_running','Aborted_connects','Rpl_semi_sync_master_status','Rpl_semi_sync_master_clients');\"
    sleep 5
  done | docker exec -i $M mariadb -B" > "$D/witness.log" 2>&1 &
echo $! > "$D/witness.pid"

# 1 transaction per second, so commits are waiting for acks
setsid nohup bash -c "i=0; while :; do i=\$((i+1));
  docker exec $M mariadb -N -e \"INSERT INTO demo.t (ts,seq) VALUES (UTC_TIMESTAMP(3),\$i);\" >/dev/null 2>&1
  sleep 1; done" >/dev/null 2>&1 &
echo $! > "$D/writer.pid"

# probe: a NEW connection every 2 s
( while :; do
    if timeout 6 docker exec "$M" mariadb --connect-timeout=3 -N -B -e "SELECT 1" >/dev/null 2>&1
    then echo "[$(ts)] NEW=OK"; else echo "[$(ts)] NEW=FAIL"; fi; sleep 2
  done ) > "$D/probe.log" 2>&1 &
PROBE=$!
sleep 15

SIP=$(docker inspect "$S" --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}')
say "PARTITION: silent drop, replica -> $GW:$PORT (matched on the pre-DNAT destination)"
nft add table inet mdevfence
nft 'add chain inet mdevfence f { type filter hook forward priority -150 ; policy accept ; }'
nft add rule inet mdevfence f ip saddr "$SIP" ct original proto-dst "$PORT" counter drop
sleep 10
PK=$(nft -a list table inet mdevfence | grep -oE 'packets [0-9]+' | head -1 | awk '{print $2}')
say "rule counter after 10 s: ${PK:-0} packets"
[ "${PK:-0}" -ge 1 ] || { say "ABORT: the rule matches nothing; the partition is not happening"; kill $PROBE 2>/dev/null; cleanup; exit 4; }
sleep 80
say "semi-sync during the partition: $(docker exec "$M" mariadb -N -B -e "SHOW GLOBAL STATUS LIKE 'Rpl_semi_sync_master_status'" | awk '{print $2}') (expected OFF)"
T_UNDO=$(date -u +%s)
say "HEAL: removing the rule"
nft delete table inet mdevfence

say "observing for 5 minutes"
sleep 300
kill $PROBE 2>/dev/null

TOT=$(wc -l < "$D/probe.log"); FAIL=$(grep -c FAIL "$D/probe.log")
say ""
say "########## RESULT ##########"
say "  probe samples: $TOT   failures: $FAIL"
say "  first failure: $(grep -m1 FAIL "$D/probe.log" || echo none)"
say "  last sample:   $(tail -1 "$D/probe.log")"
say "  max Threads_connected seen by the witness: $(grep -oE '^Threads_connected[[:space:]]+[0-9]+' "$D/witness.log" | awk '{print $2}' | sort -n | tail -1)"
TEAR=$(docker logs "$M" 2>&1 | grep -i 'Stop semi-sync binlog_dump' | tail -1)
if [ -n "$TEAR" ]; then
  TS=$(date -u -d "$(echo "$TEAR" | awk '{print $1" "$2}')" +%s 2>/dev/null)
  [ -n "$TS" ] && say "  teardown happened $((TS - T_UNDO)) s after the heal"
fi
say "  primary log, semi-sync lines:"
docker logs "$M" 2>&1 | grep -i 'semi-sync' | tail -4 | sed 's/^/      /'
[ "$FAIL" -gt 0 ] && say "  >>> REPRODUCED: the primary stopped completing new handshakes" \
                  || say "  >>> not reproduced in this run"
say "  logs kept in $D (probe.log, witness.log)"
cleanup