Uploaded image for project: 'MariaDB Server'
  1. MariaDB Server
  2. MDEV-41235

mariadb-backup --log-copy-interval overflows its timeout conversion and enters an immediate timed-wait loop

    XMLWordPrintable

Details

    • Bug
    • Status: Open (View Workflow)
    • Major
    • Resolution: Unresolved
    • 12.3.3
    • 11.4, 11.8, 12.3
    • mariabackup
    • None

    Description

      `--log-copy-interval` accepts values up to `LONG_MAX` milliseconds. The log
      copying thread multiplies the parsed value by `1000000ULL` before building an
      absolute `timespec`. For `2^58` milliseconds, the multiplication wraps modulo
      `2^64` and becomes zero. The thread consequently submits deadlines at or just
      before the current time and loops immediately instead of sleeping.

      A read-only `LD_PRELOAD` observer records the absolute deadlines without
      changing `pthread_cond_timedwait()` behavior. A 1000 ms control produced no
      nonpositive deadlines; the trigger produced 374 during the same small backup.

      Steps to reproduce

      #!/usr/bin/env bash
      set -euo pipefail
      IMAGE='mariadb@sha256:dd9b303aed4f4890ed09f766d8ca9ddfd176c0c6f6267feff53b3192ec65a979'
      C="mdev-mcf2s10-$$"; OWNER="mdev-mcf2s10-$$"; TMP=$(mktemp -d)
      cleanup() {
        if docker inspect --format '{{ index .Config.Labels "io.encryptiondbfuzz.owner" }}' \
             "$C" 2>/dev/null | grep -Fqx "$OWNER"; then
          docker rm -f "$C" >/dev/null 2>&1 || true
        fi
        rm -rf -- "$TMP"
      }
      trap cleanup EXIT
       
      cat >"$TMP/observe.c" <<'EOF_C'
      #define _GNU_SOURCE
      #include <dlfcn.h>
      #include <pthread.h>
      #include <stdatomic.h>
      #include <stdint.h>
      #include <stdio.h>
      #include <stdlib.h>
      #include <time.h>
      static pthread_mutex_t lock=PTHREAD_MUTEX_INITIALIZER;
      static _Atomic unsigned int count;
      int pthread_cond_timedwait(pthread_cond_t *c,pthread_mutex_t *m,const struct timespec *a){
        static int (*real_fn)(pthread_cond_t*,pthread_mutex_t*,const struct timespec*);
        struct timespec now; const char *path=getenv("WAIT_LOG");
        if(!real_fn)real_fn=dlsym(RTLD_NEXT,"pthread_cond_timedwait");
        unsigned int n=atomic_fetch_add(&count,1);
        if(n<10000 && path && clock_gettime(CLOCK_REALTIME,&now)==0){
          int64_t d=((int64_t)a->tv_sec-(int64_t)now.tv_sec)*INT64_C(1000000000)+a->tv_nsec-now.tv_nsec;
          pthread_mutex_lock(&lock); FILE *f=fopen(path,"a");
          if(f){fprintf(f,"ordinal=%u delta_ns=%lld\n",n,(long long)d);fclose(f);}
          pthread_mutex_unlock(&lock);
        }
        return real_fn(c,m,a);
      }
      EOF_C
      gcc -shared -fPIC -O2 -Wall -Wextra -Werror -o "$TMP/observe.so" "$TMP/observe.c" -ldl -pthread
      chmod 755 "$TMP"; chmod 444 "$TMP/observe.so"
       
      docker run -d --name "$C" --label "io.encryptiondbfuzz.owner=$OWNER" \
        --network none --read-only --cap-drop ALL \
        --security-opt no-new-privileges=true --pids-limit 180 --memory 1500m \
        --cpus 1 --user 999:999 \
        --tmpfs /var/lib/mysql:rw,nosuid,nodev,size=512m,uid=999,gid=999 \
        --tmpfs /run/mysqld:rw,nosuid,nodev,size=16m,uid=999,gid=999 \
        --tmpfs /tmp:rw,nosuid,nodev,size=384m,uid=999,gid=999 \
        --mount "type=bind,src=$TMP,dst=/edbf,readonly" \
        --env MARIADB_ALLOW_EMPTY_ROOT_PASSWORD=1 "$IMAGE" >/dev/null
      READY=0
      for I in $(seq 1 90); do
        if docker exec --user 999:999 "$C" mariadb --protocol=socket \
             --socket=/run/mysqld/mysqld.sock -uroot -NBe 'SELECT VERSION()' 2>/dev/null; then READY=1; break; fi
        sleep 1
      done
      test "$READY" -eq 1
      docker exec --user 999:999 "$C" mariadb --protocol=socket \
        --socket=/run/mysqld/mysqld.sock -uroot -e \
        "CREATE DATABASE edbf; CREATE TABLE edbf.t(id INT PRIMARY KEY,payload MEDIUMBLOB); INSERT INTO edbf.t SELECT seq,REPEAT(CHAR(65+(seq MOD 20)),8192) FROM edbf.seq_1_to_500"
       
      run_case() {
        CASE=$1; INTERVAL=$2
        docker exec --user 999:999 --env LD_PRELOAD=/edbf/observe.so \
          --env WAIT_LOG=/tmp/wait-$CASE.log "$C" timeout 150 mariadb-backup \
          --backup --log-copy-interval="$INTERVAL" --protocol=socket \
          --socket=/run/mysqld/mysqld.sock --user=root \
          --target-dir="/tmp/backup-$CASE" 2>&1 | grep 'completed OK!'
        docker exec --user 999:999 "$C" awk -v name="$CASE" '
          {split($2,a,"="); n++; if(a[2]<=0) nonpos++; if(a[2]>=500000000) future++}
          END{printf "%s observations=%d nonpositive=%d future_ge_500ms=%d\n",name,n,nonpos,future}' \
          "/tmp/wait-$CASE.log"
        docker exec --user 999:999 "$C" rm -rf "/tmp/backup-$CASE"
      }
      run_case control 1000
      run_case trigger 288230376151711744
      docker exec --user 999:999 "$C" mariadb --protocol=socket \
        --socket=/run/mysqld/mysqld.sock -uroot -NBe "SELECT CONCAT('server_health_rows=',COUNT(*)) FROM edbf.t"
      

      Actual result

      The authoritative run produced:

      [00] ... completed OK!
      control observations=8 nonpositive=0 future_ge_500ms=4
      [00] ... completed OK!
      trigger observations=377 nonpositive=374 future_ge_500ms=1
      server_health_rows=500
      

      Representative trigger deltas were `-1238`, `-428`, and `-678` nanoseconds,
      showing deadlines already expired when passed to `pthread_cond_timedwait()`.
      Both backups completed and the server remained healthy; no crash was observed.

      Expected result

      The option parser should reject intervals whose nanosecond conversion cannot be
      represented, or the conversion should use checked arithmetic and a normalized,
      representable deadline. A large accepted interval must not become an immediate
      retry loop.

      Attachments

        Activity

          People

            tafzeel Mohammad Tafzeel Shams
            csfuzz csfuzz
            Votes:
            0 Vote for this issue
            Watchers:
            2 Start watching this issue

            Dates

              Created:
              Updated:

              Git Integration

                Error rendering 'com.xiplink.jira.git.jira_git_plugin:git-issue-webpanel'. Please contact your Jira administrators.