#!/usr/bin/env bash # # MDEV-41708: Aria: one failed fdatasync() of aria_log silently makes the log read-only; # later writes to Aria tables (incl. mysql.global_priv) fail with misleading errors yet # persist after a restart, tables show corrupt, a new table loops in auto-repair, and a # failed ALTER leaves a #sql-alter table behind. # # !!! WARNING: this POISONS the server it runs against (Aria log read-only until a restart, # !!! mysql.global_priv may be corrupted and a CREATE USER of 'u1' may persist). Use a # !!! throwaway server. Set RESTART_CMD so the script can restart it for you. # # Usage: # DB_PORT= RESTART_CMD='' ./mdev41708_repro.sh # # Environment (all optional except DB_PORT): # DB_HOST default 127.0.0.1, DB_USER default root, DB_PASS, MARIADB_BIN default mariadb # SERVER_PID mariadbd PID (default: from @@pid_file, then pgrep on --port) # RESTART_CMD command that cleanly restarts the SAME server (same datadir). Without it # only the first part runs (no restart checks, no ALTER case). # YES=1 skip the confirmation prompt # KEEP=1 keep the test database # # Needs: sudo, strace >= 4.22 (for -P and inject=), ptrace allowed on mariadbd, read access # to the datadir, and a server that runs with real fsync. set -u DB_HOST="${DB_HOST:-127.0.0.1}" DB_USER="${DB_USER:-root}" MARIADB_BIN="${MARIADB_BIN:-mariadb}" DB="mdev41708" OUT="${OUT_DIR:-/tmp/mdev41708_$(date +%Y%m%d_%H%M%S)}" [ -n "${DB_PORT:-}" ] || { echo "Set DB_PORT (e.g. DB_PORT=11551 $0)" >&2; exit 2; } mkdir -p "$OUT" pass=0; fail=0 say() { printf '\n=== %s ===\n' "$*"; } check() { if [ "$2" -eq 0 ]; then echo " [ok] $1"; pass=$((pass+1)); else echo " [MISS] $1"; fail=$((fail+1)); fi; } has() { grep -q -- "$2" <<<"$1"; } cargs() { printf '%s\n' "-u$DB_USER" ${DB_PASS:+-p"$DB_PASS"} "-h$DB_HOST" "-P$DB_PORT"; } q() { "$MARIADB_BIN" $(cargs) -N -B -e "$1" 2>&1 | grep -v 'ssl-verify-server-cert'; } # ---------------------------------------------------------------- server say "Server" q "SELECT @@version, @@port, @@datadir" | tee "$OUT/server.txt" DATADIR=$(q "SELECT @@datadir" | tail -1); DATADIR="${DATADIR%/}" LOG=$(q "SELECT @@log_error" | tail -1) pf=$(q "SELECT @@pid_file" | tail -1) [ -r "$pf" ] && pid=$(cat "$pf") [ -z "${SERVER_PID:-${pid:-}}" ] && pid=$(pgrep -f -- "mariadbd.*--port=${DB_PORT}( |\$)" | head -1) PID="${SERVER_PID:-${pid:-}}" [ -n "$PID" ] && [ -d "/proc/$PID" ] || { echo "Cannot find mariadbd PID; set SERVER_PID" >&2; exit 2; } echo "mariadbd PID: $PID" q "SELECT CONCAT('aria_recover_options=', @@aria_recover_options)" 2>/dev/null alog_file() { ls "$DATADIR"/aria_log.[0-9]* 2>/dev/null | sort | tail -1; } ALOG=$(alog_file) [ -n "$ALOG" ] || { echo "No aria_log.* file visible in $DATADIR (no read access?)" >&2; exit 2; } echo "aria log: $ALOG" sudo -v || { echo "sudo required"; exit 2; } strace -V 2>&1 | head -1 case "${RESTART_CMD:-}" in *"<"*|*"..."*) echo "RESTART_CMD looks like a placeholder: $RESTART_CMD" >&2 echo "Give the REAL commands, e.g.:" >&2 echo " B=; RESTART_CMD=\"\$B/bin/mariadb-admin --no-defaults -uroot --socket=\$B/socket.sock shutdown; sleep 3; nohup \$B/bin/mariadbd >/dev/null 2>&1 &\"" >&2 exit 2 ;; esac if [ "${YES:-0}" != "1" ]; then echo echo "This will make the Aria log of THIS server read-only and may corrupt mysql.global_priv" echo "until the server is restarted. RESTART_CMD is ${RESTART_CMD:+set}${RESTART_CMD:-NOT set}." read -r -p "Continue? [y/N] " a; [ "$a" = "y" ] || exit 1 fi restart_server() { [ -n "${RESTART_CMD:-}" ] || return 1 eval "$RESTART_CMD" local i for i in $(seq 1 120); do q "SELECT 1" | grep -q '^1$' && break; sleep 1; done if [ -d "/proc/$PID" ]; then echo " WARNING: the old mariadbd (PID $PID) is still running - RESTART_CMD did not restart THIS server." return 2 fi PID=$(cat "$pf" 2>/dev/null || echo "") [ -n "$PID" ] || PID=$(pgrep -f -- "mariadbd.*--port=${DB_PORT}( |\$)" | head -1) ALOG=$(alog_file) return 0 } # inject_aria : EIO on the first fdatasync of the Aria log while runs -> INJ_OUT, NINJ inject_aria() { local sql="$1" spid_file="$OUT/strace.pid" i=0 TR="$OUT/trace_$RANDOM.txt" f f=$(readlink -f "$ALOG" 2>/dev/null || echo "$ALOG") nohup sudo -n sh -c "echo \$\$ > '$spid_file'; exec strace -f -qq -y -p $PID \ -e trace=fdatasync -e inject=fdatasync:error=EIO:when=1 -P '$f' -o '$TR'" >/dev/null 2>&1 & while [ "$(awk '/^TracerPid:/{print $2}' /proc/$PID/status)" = "0" ]; do i=$((i+1)); [ $i -gt 100 ] && { echo "strace did not attach" >&2; exit 1; } sleep 0.1 done INJ_OUT=$(q "$sql") sudo -n kill -INT "$(cat "$spid_file")" 2>/dev/null i=0 while [ "$(awk '/^TracerPid:/{print $2}' /proc/$PID/status)" != "0" ]; do i=$((i+1)); [ $i -gt 100 ] && break; sleep 0.1 done NINJ=$(grep -c INJECTED "$TR" 2>/dev/null || true); NINJ=${NINJ:-0} echo " injected fdatasync on the Aria log: $NINJ" grep INJECTED "$TR" | head -2 | sed -E 's/^[0-9]+ / /' } # ---------------------------------------------------------------- Case 1 say "Setup" q "DROP DATABASE IF EXISTS $DB; CREATE DATABASE $DB; USE $DB; CREATE TABLE t1 (id INT PRIMARY KEY, n INT) ENGINE=Aria TRANSACTIONAL=1; CREATE TABLE t2 LIKE t1; INSERT INTO t1 SELECT seq, seq FROM seq_1_to_1000; INSERT INTO t2 SELECT seq, seq FROM seq_1_to_1000; DROP USER IF EXISTS u1@localhost;" | grep -i error && exit 2 echo "created $DB.t1 and $DB.t2 (1000 rows each, Aria TRANSACTIONAL=1)" say "Case 1: EIO on the next fdatasync of the Aria log, during INSERT INTO t1" inject_aria "INSERT INTO $DB.t1 VALUES (1001, 1001)" R1="$INJ_OUT"; N1=$NINJ; echo " INSERT INTO t1 -> $R1" R2=$(q "INSERT INTO $DB.t2 VALUES (1001, 1001)"); echo " INSERT INTO t2 -> $R2" C=$(q "SELECT COUNT(*) FROM $DB.t2 WHERE id=1001"); echo " SELECT count (id=1001) -> $C" CT=$(q "CHECK TABLE $DB.t2"); echo " CHECK TABLE t2 -> $(echo "$CT" | tr '\t\n' ' ' | cut -c1-200)" R3=$(q "CREATE USER u1@localhost"); echo " CREATE USER u1 -> $R3" GP=$(q "CHECK TABLE mysql.global_priv"); echo " CHECK TABLE global_priv -> $(echo "$GP" | tr '\t\n' ' ' | cut -c1-200)" q "CREATE TABLE $DB.t3 (a INT) ENGINE=Aria TRANSACTIONAL=1" >/dev/null timeout 15 "$MARIADB_BIN" $(cargs) -e "INSERT INTO $DB.t3 VALUES (1)" >/dev/null 2>&1; HANG=$? BAK1=$(ls "$DATADIR/$DB" 2>/dev/null | grep -c BAK); sleep 4; BAK2=$(ls "$DATADIR/$DB" 2>/dev/null | grep -c BAK) echo " INSERT INTO new table t3: exit code $HANG (124 = did not return within 15 s)" echo " .BAK files in the database directory: $BAK1, then $BAK2 a few seconds later" echo " error log (last lines):"; [ -r "$LOG" ] && tail -5 "$LOG" | cut -c1-170 | sed 's/^/ /' READONLY_MSG=0; [ -r "$LOG" ] && grep -qiE "read-?only" "$LOG" && READONLY_MSG=1 RROW="n/a"; RUSER="n/a"; RCHK="n/a" if [ -n "${RESTART_CMD:-}" ]; then say "Restart (RESTART_CMD)" if restart_server; then RROW=$(q "SELECT COUNT(*) FROM $DB.t2 WHERE id=1001" | tail -1) RUSER=$(q "SELECT COUNT(*) FROM mysql.global_priv WHERE user='u1'" | tail -1) q "SELECT COUNT(*) FROM $DB.t1; SELECT COUNT(*) FROM $DB.t2; SELECT COUNT(*) FROM mysql.global_priv" >/dev/null # opening a crashed Aria table triggers the auto-repair RCHK=$(q "CHECK TABLE $DB.t1, $DB.t2, mysql.global_priv" | tr '\t\n' ' ') echo " rows with id=1001 in t2 after the restart: $RROW" echo " users u1 in mysql.global_priv after restart: $RUSER" echo " CHECK TABLE after the restart: $(echo "$RCHK" | cut -c1-200)" [ -r "$LOG" ] && grep -E "Found .* rows when repairing" "$LOG" | tail -2 | sed 's/^/ log: /' else RROW="invalid"; fi fi # ---------------------------------------------------------------- Case 2 A1=""; A2=""; A3=""; LEFT1="n/a"; LEFT2="n/a"; N2=0 if [ -n "${RESTART_CMD:-}" ] && [ "$RROW" != "invalid" ]; then say "Case 2: EIO on the next fdatasync of the Aria log, during ALTER TABLE ... ADD INDEX" # fresh tables: after case 1, t1 may still be flagged corrupt (it was written by a failed statement) q "USE $DB; DROP TABLE IF EXISTS t3, t4; CREATE TABLE c2a (id INT PRIMARY KEY, n INT) ENGINE=Aria TRANSACTIONAL=1; CREATE TABLE c2b LIKE c2a; INSERT INTO c2a SELECT seq, seq FROM seq_1_to_1000; INSERT INTO c2b SELECT seq, seq FROM seq_1_to_1000;" | grep -i error && exit 2 inject_aria "ALTER TABLE $DB.c2a ADD INDEX ix1 (n)" A1="$INJ_OUT"; N2=$NINJ; echo " ALTER TABLE c2a ADD INDEX -> $A1" A2=$(q "ALTER TABLE $DB.c2b ADD INDEX ix3 (n)"); echo " next ALTER TABLE c2b -> $A2" A3=$(q "RENAME TABLE $DB.c2a TO $DB.c2c"); echo " RENAME TABLE c2a TO c2c -> $A3" LEFT1=$(ls "$DATADIR/$DB" 2>/dev/null | grep -c '^#sql-alter') echo " #sql-alter-* files in the database directory: $LEFT1" say "Restart again" if restart_server; then LEFT2=$(ls "$DATADIR/$DB" 2>/dev/null | grep -c '^#sql-alter') echo " #sql-alter-* files after the restart: $LEFT2" else LEFT2="invalid"; fi fi # ---------------------------------------------------------------- checks say "Checks" [ "$N1" -ge 1 ]; check "case 1: EIO injected on a fdatasync of the Aria log" $? has "$R1" "Can't sync file"; check "case 1: INSERT INTO t1 -> error 27 \"Can't sync file ... aria_log\"" $? has "$R2" "error 122"; check "case 1: later INSERT INTO t2 -> 1030 \"Got error 122 Internal (unspecified) error\"" $? [ "$C" = "0" ]; check "case 1: the failed INSERT is not visible while the server runs" $? has "$CT$GP" "Corrupt"; check "case 1: CHECK TABLE reports Corrupt (t2 and/or mysql.global_priv)" $? has "$R3" "1032"; check "case 1: CREATE USER -> error 1032 \"Can't find record in 'global_priv'\"" $? [ "$HANG" -eq 124 ]; check "case 1: INSERT into a new Aria table does not return (auto-repair loop)" $? [ "$BAK2" -gt "$BAK1" ] 2>/dev/null; check "case 1: .BAK files keep being created ($BAK1 -> $BAK2)" $? [ "$READONLY_MSG" -eq 0 ]; check "case 1: the error log never says the Aria log became read-only" $? if [ "$RROW" != "n/a" ] && [ "$RROW" != "invalid" ]; then [ "$RROW" = "1" ]; check "case 1: the row of the FAILED INSERT exists after the restart" $? [ "$RUSER" = "1" ]; check "case 1: the user of the FAILED CREATE USER exists after the restart" $? ! has "$RCHK" "Corrupt"; check "case 1: CHECK TABLE of t1, t2, mysql.global_priv reports no Corrupt after the restart (auto-repaired)" $? else echo " (restart checks and case 2 skipped: set RESTART_CMD to run them)" fi if [ -n "$A1" ]; then [ "$N2" -ge 1 ]; check "case 2: EIO injected on a fdatasync of the Aria log" $? has "$A1" "Can't sync file"; check "case 2: ALTER TABLE ADD INDEX -> error 27 \"Can't sync file\"" $? has "$A2" "145"; check "case 2: next ALTER fails with 145 on the #sql-alter table" $? has "$A3" "errno: 1"; check "case 2: RENAME TABLE fails with 1025 (errno 1 Operation not permitted)" $? [ "$LEFT1" -gt 0 ] 2>/dev/null; check "case 2: #sql-alter-* files left behind ($LEFT1)" $? [ "$LEFT2" != "invalid" ] && [ "$LEFT2" -gt 0 ] 2>/dev/null; check "case 2: #sql-alter-* files still there after a restart ($LEFT2)" $? fi # ---------------------------------------------------------------- cleanup RESTARTED_OK=0 if [ -n "${RESTART_CMD:-}" ] && [ "$RROW" != "invalid" ] && [ "$RROW" != "n/a" ]; then RESTARTED_OK=1; fi if [ "${KEEP:-0}" != "1" ] && [ "$RESTARTED_OK" -eq 1 ]; then say "Cleanup" q "DROP USER IF EXISTS u1@localhost; DROP DATABASE IF EXISTS $DB" >/dev/null rm -f "$DATADIR/$DB"/'#sql-alter'* "$DATADIR/$DB"/*BAK 2>/dev/null echo " dropped user u1 and database $DB (server was restarted, so the log is writable again)" else echo echo " NOTE: this server was NOT restarted successfully, so it is still poisoned (Aria log read-only," echo " mysql.global_priv may be corrupt, .BAK files in $DATADIR/$DB). Cleanup was skipped." echo " Restart it, then: DROP USER IF EXISTS u1@localhost; DROP DATABASE IF EXISTS $DB;" fi say "Summary" echo " matched: $pass not matched: $fail" if [ "$fail" -eq 0 ]; then echo " RESULT: BUG REPRODUCED (all ticket symptoms observed)" else echo " RESULT: NOT (fully) reproduced - see [MISS] lines and $OUT"; fi echo " Artifacts: $OUT" [ "$fail" -eq 0 ]