Uploaded image for project: 'MariaDB Server'
  1. MariaDB Server
  2. MDEV-41229

mariadb-backup returns success after --galera-info loses the current binary log to EIO

    XMLWordPrintable

Details

    • Bug
    • Status: Open (View Workflow)
    • Major
    • Resolution: Unresolved
    • 12.3.3
    • 11.4, 11.8, 12.3
    • mariabackup
    • None

    Description

      On a synced single-node Galera server with GTID and binary logging enabled,
      `mariadb-backup --galera-info` flushes and copies the current binary log into
      the backup. If the exact source-file read returns `EIO`, `copy_file()` reports
      failure and removes the incomplete destination. The error is not propagated:
      the tool then prints `Writing Galera info succeeded`, prints `completed OK!`,
      and exits 0 while the current binary log named in the backup metadata is absent.

      Steps to reproduce

      The following uses a small `LD_PRELOAD` fault shim. It affects only the first
      `pread()` of a file whose basename starts with `mariadb-bin.` and returns one
      `EIO`; all other I/O is passed through unchanged.

      #!/usr/bin/env bash
      set -euo pipefail
      IMAGE='mariadb@sha256:dd9b303aed4f4890ed09f766d8ca9ddfd176c0c6f6267feff53b3192ec65a979'
      C="mdev-mcf203-$$"
      OWNER="mdev-mcf203-$$"
      TMP=$(mktemp -d)
      cleanup() {
        if docker inspect --format '{{ index .Config.Labels "io.encryptiondbfuzz.owner" }}' \
             "$C" 2>/dev/null | grep -Fqx "$OWNER"; then
          docker rm -f "$C" >/dev/null 2>&1 || true
        fi
        rm -rf -- "$TMP"
      }
      trap cleanup EXIT
       
      cat >"$TMP/fail.c" <<'EOF_C'
      #define _GNU_SOURCE
      #include <errno.h>
      #include <fcntl.h>
      #include <limits.h>
      #include <stdio.h>
      #include <string.h>
      #include <sys/syscall.h>
      #include <sys/types.h>
      #include <unistd.h>
      static int delivered;
      static int target(int fd) {
        char p[64], v[PATH_MAX+1];
        snprintf(p,sizeof(p),"/proc/self/fd/%d",fd);
        ssize_t n=readlink(p,v,sizeof(v)-1); if(n<=0)return 0; v[n]=0;
        char *b=strrchr(v,'/'); b=b?b+1:v;
        return !strncmp(b,"mariadb-bin.",12) && strcmp(b,"mariadb-bin.index");
      }
      static void note(void) {
        const char s[]="injected EIO into current-binlog pread\n";
        int fd=open("/tmp/edbf-binlog-eio.log",O_WRONLY|O_CREAT|O_APPEND,0600);
        if(fd>=0){ssize_t n=write(fd,s,sizeof(s)-1);int rc=close(fd);(void)n;(void)rc;}
      }
      static ssize_t call(int fd,void *buf,size_t len,off_t off) {
        if(target(fd) && __sync_bool_compare_and_swap(&delivered,0,1)) {
          note(); errno=EIO; return -1;
        }
        return syscall(SYS_pread64,fd,buf,len,off);
      }
      ssize_t pread(int fd,void *buf,size_t len,off_t off){return call(fd,buf,len,off);}
      ssize_t p64(int fd,void *buf,size_t len,off64_t off) __asm__("pread64");
      ssize_t p64(int fd,void *buf,size_t len,off64_t off){return call(fd,buf,len,(off_t)off);}
      EOF_C
      gcc -shared -fPIC -O2 -Wall -Wextra -Werror \
        -o "$TMP/fail.so" "$TMP/fail.c"
      chmod 755 "$TMP" && chmod 444 "$TMP/fail.so"
       
      docker run --detach --name "$C" \
        --label "io.encryptiondbfuzz.owner=$OWNER" \
        --network none --read-only --cap-drop ALL \
        --security-opt no-new-privileges=true --pids-limit 224 \
        --memory 1700m --cpus 1 --user 999:999 \
        --tmpfs /var/lib/mysql:rw,nosuid,nodev,size=700m,uid=999,gid=999 \
        --tmpfs /run/mysqld:rw,nosuid,nodev,size=16m,uid=999,gid=999 \
        --tmpfs /tmp:rw,nosuid,nodev,size=700m,uid=999,gid=999 \
        --mount "type=bind,src=$TMP,dst=/edbf,readonly" \
        --env MARIADB_ALLOW_EMPTY_ROOT_PASSWORD=1 "$IMAGE" \
        --log-bin=mariadb-bin --binlog-format=ROW --server-id=1 \
        --gtid-strict-mode=ON --wsrep-on=ON \
        --wsrep-provider=/usr/lib/galera/libgalera_smm.so \
        --wsrep-cluster-address=gcomm:// --wsrep-new-cluster \
        --wsrep-node-address=127.0.0.1 --wsrep-node-name=edbf-node \
        --wsrep-provider-options=gmcast.listen_addr=tcp://127.0.0.1:4567 \
        --bind-address=127.0.0.1 >/dev/null
       
      db() {
        docker exec --user 999:999 "$C" mariadb --protocol=socket \
          --socket=/run/mysqld/mysqld.sock -uroot --batch --skip-column-names "$@"
      }
      READY=0
      for I in $(seq 1 120); do
        STATE=$(db -NBe "SELECT CONCAT(@@wsrep_on,'|',
          COALESCE((SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS
            WHERE VARIABLE_NAME='WSREP_READY'),''),'|',
          COALESCE((SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS
            WHERE VARIABLE_NAME='WSREP_LOCAL_STATE_COMMENT'),''))" 2>/dev/null || true)
        if [ "$STATE" = 'ON|ON|Synced' ]; then READY=1; break; fi
        sleep 1
      done
      test "$READY" -eq 1
       
      db -e "CREATE DATABASE edbf; USE edbf;
        CREATE TABLE t(id INT PRIMARY KEY,pad VARCHAR(255)) ENGINE=InnoDB;
        INSERT INTO t VALUES(1,REPEAT('A',220)),(2,REPEAT('B',220)),(3,REPEAT('C',220));
        SELECT CONCAT('wsrep=',@@wsrep_on,', log_bin=',@@log_bin,
                      ', gtid=',@@gtid_binlog_state,', rows=',COUNT(*)) FROM t;"
       
      set +e
      docker exec --user 999:999 \
        --env LD_PRELOAD=/edbf/fail.so "$C" timeout 170 mariadb-backup \
        --backup --galera-info --protocol=socket \
        --socket=/run/mysqld/mysqld.sock --user=root \
        --target-dir=/tmp/backup 2>&1 | tee "$TMP/backup.log"
      RC=${PIPESTATUS[0]}
      set -e
       
      BINLOG=$(db -NBe 'SHOW MASTER STATUS' | cut -f1)
      echo "backup_rc=$RC current_binlog=$BINLOG"
      grep -E 'copy_file\(\) failed|Writing Galera info succeeded|completed OK' \
        "$TMP/backup.log"
      docker exec --user 999:999 "$C" cat /tmp/edbf-binlog-eio.log
      if docker exec --user 999:999 "$C" test -f "/tmp/backup/$BINLOG"; then
        echo 'backup_binlog=present'
      else
        echo 'backup_binlog=MISSING'
      fi
      db -NBe "SELECT CONCAT('server_health_rows=',COUNT(*)) FROM edbf.t"
      

      Actual result

      The exact timestamps and UUID vary. The significant output is:

      wsrep=ON, log_bin=ON, gtid=0-1-3, rows=3
      Error: copy_file() failed.
      Writing Galera info succeeded with <uuid>:4 0
      completed OK!
      backup_rc=0 current_binlog=mariadb-bin.000003
      injected EIO into current-binlog pread
      backup_binlog=MISSING
      server_health_rows=3
      

      The generated `mariadb_backup_galera_info` and `donor_galera_info` files remain
      present, but their associated current binary log is missing. The server remains
      running; no crash was observed.

      Expected result

      Failure to copy the current binary log selected by `--galera-info` should make
      `write_galera_info()` fail. `mariadb-backup` should exit nonzero and must not
      print either `Writing Galera info succeeded` or `completed OK!` for this backup.

      Attachments

        Activity

          People

            seppo Seppo Jaakola
            csfuzz csfuzz
            Votes:
            0 Vote for this issue
            Watchers:
            2 Start watching this issue

            Dates

              Created:
              Updated:

              Git Integration

                Error rendering 'com.xiplink.jira.git.jira_git_plugin:git-issue-webpanel'. Please contact your Jira administrators.